diff --git a/pdfmajor/lexer/__init__.py b/pdfmajor/lexer/__init__.py index 19e4b1c..f3808cb 100644 --- a/pdfmajor/lexer/__init__.py +++ b/pdfmajor/lexer/__init__.py @@ -19,8 +19,19 @@ from pdfmajor.streambuffer import BufferStream class PDFLexer: - def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None: - self.buffer = BufferStream(fp, buffer_size=buffer_size) + def __init__(self, buffer: BufferStream) -> None: + self.buffer = buffer + + @classmethod + def from_iobuffer(cls, fp: io.BufferedIOBase, buffer_size: int = 4096): + return cls(BufferStream(fp, buffer_size=buffer_size)) + + @classmethod + def from_bytes(cls, data: bytes, buffer_size: int = 4096): + return cls(BufferStream(io.BytesIO(data), buffer_size=buffer_size)) + + def tell(self) -> int: + return self.buffer.tell() def seek(self, pos: int) -> int: """moves the offset of the reader to a certain spot, and returns the new offset diff --git a/pdfmajor/parser/PDFParser.py b/pdfmajor/parser/PDFParser.py index a333359..feee107 100644 --- a/pdfmajor/parser/PDFParser.py +++ b/pdfmajor/parser/PDFParser.py @@ -1,7 +1,12 @@ import io -from pdfmajor.parser.objects.stream import PDFStream -from typing import Iterator, List, Optional, Union +from typing import Dict, Iterator, Optional, Tuple + +from pdfmajor.streambuffer import BufferStream +from pdfmajor.parser.objects.stream import PDFStream +from pdfmajor.parser.exceptions import InvalidXref +from pdfmajor.parser.objects.base import PDFObject +from pdfmajor.lexer.token import TokenKeyword from pdfmajor.lexer import PDFLexer from pdfmajor.lexer.token import ( TArrayValue, @@ -11,15 +16,54 @@ from pdfmajor.lexer.token import ( TokenDictionary, TokenKeyword, ) - -from .state import ParsingState -from .parsers import attempt_parse_prim, deal_with_collection_object -from .exceptions import InvalidKeywordPos, ParserError - +from .objects.indirect import IndirectObject from .objects.base import PDFContextualObject, PDFObject from .objects.collections import PDFArray, PDFDictionary from .objects.comment import PDFComment -from .objects.indirect import IndirectObject +from .state import ParsingState +from .parsers import attempt_parse_prim, deal_with_collection_object +from .exceptions import InvalidKeywordPos, BrokenFile, InvalidKeywordPos, ParserError + + +class XRefDB: + """A class for collection references to objects""" + + def __init__(self, parser: "PDFParser") -> None: + self.objs: Dict[Tuple[int, int], IndirectObject] = {} + self.parser = parser + + def save_indobject(self, indobj: IndirectObject): + if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None: + self.objs[(indobj.obj_num, indobj.gen_num)] = indobj + + def get_object_from_indobj(self, indobj: IndirectObject) -> PDFObject: + return self.get_object(indobj.obj_num, indobj.gen_num) + + def get_object(self, obj_num: int, gen_num: int) -> PDFObject: + """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10 + + Args: + obj_num (int): object number + gen_num (int): generation number + + Returns: + PDFObject + """ + indobj = self.objs.get((obj_num, gen_num), None) + if indobj is None: + raise InvalidXref( + f"xref not found for object retrieval {obj_num}, {gen_num}" + ) + else: + obj = indobj.get_object() + if obj is not None: + return obj + cur_pos = self.parser.tell() + self.parser.seek(indobj.offset) + obj = next(self.parser.iter_objects()) + indobj.save_object(obj) + self.parser.seek(cur_pos) + return obj class PDFParser: @@ -27,11 +71,9 @@ class PDFParser: def __init__( self, - fp: io.BufferedIOBase, - buffer_size: int = 4096, - state: Optional[ParsingState] = None, + buffer: BufferStream, + db: Optional[XRefDB] = None, strict: bool = True, - retain_obj_on_seek: bool = False, ) -> None: """ @@ -40,48 +82,62 @@ class PDFParser: buffer_size (int, optional): the size of the reads. Defaults to 4096. state (Optional[ParsingState], optional): the parsing state. Defaults to None. strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True. - retain_obj_on_seek (bool, optional): whether we want to ensure we keep the object collection when changing the offset using the self.seek methodm this does not apply if you specify the state when using the seek method!. Defaults to False. """ - self.lexer = PDFLexer(fp, buffer_size) + self.lexer = PDFLexer(buffer) + self.db = XRefDB(self) self.strict = strict - self.state = ParsingState([], []) if state is None else state - self.retain_obj_on_seek = retain_obj_on_seek - def seek(self, offset: int, state: Optional[ParsingState] = None) -> int: + @classmethod + def from_iobuffer( + cls, + fp: io.BufferedIOBase, + buffer_size: int = 4096, + strict: bool = True, + ): + return cls( + BufferStream(fp, buffer_size=buffer_size), + strict=strict, + ) + + @classmethod + def from_bytes( + cls, + data: bytes, + buffer_size: int = 4096, + strict: bool = True, + ): + return cls( + BufferStream(io.BytesIO(data), buffer_size=buffer_size), + strict=strict, + ) + + def seek(self, offset: int) -> int: """moves the offset of the reader to a certain spot, and returns the new offset Args: offset (int) - state (Optional[ParsingState]): this will override our current state Returns: int: new offset """ - self.state = ( - ParsingState( - [], - [], - indirect_object_collection=self.state.indirect_object_collection - if not self.retain_obj_on_seek - else None, - ) - if state is None - else state - ) return self.lexer.seek(offset) + def tell(self) -> int: + return self.lexer.tell() + def iter_objects(self) -> Iterator[PDFObject]: """Iterates over the objects in the current byte stream Yields: PDFObject """ + state = ParsingState([], [], strict=self.strict) for token in self.lexer.iter_tokens(): - attempt_prim = attempt_parse_prim(token, self.state) + attempt_prim = attempt_parse_prim(token, state) if attempt_prim.success: - yield from self._feed_or_yield_objs(attempt_prim.next_objs) + yield from state.feed_or_yield_objs(attempt_prim.next_objs) elif isinstance(token, TokenComment): - yield self.state.set_last_obj( + yield state.set_last_obj( PDFComment( token.value, (token.start_loc, token.end_loc), @@ -89,67 +145,99 @@ class PDFParser: ) elif isinstance(token, TokenKeyword): if token.value == b"R": - yield from self._on_indirect_ref_close() + yield from self._on_indirect_ref_close(state) elif token.value == b"obj": - self.state.initialize_indirect_obj() + state.initialize_indirect_obj(token.start_loc) elif token.value == b"endobj": - yield from self._on_endobj(token) + yield self._on_endobj(state, token) elif token.value == b"stream": - self._on_stream(token) + self._on_stream( + state, + token, + ) elif token.value == b"endstream": - self._on_endstream(token) + self._on_endstream(state, token) else: raise InvalidKeywordPos(token) elif isinstance(token, TokenArray): - yield from self.state.flush_int_collections() + yield from state.flush_int_collections() yield from deal_with_collection_object( - self.state, token, PDFArray, TArrayValue.OPEN, strict=self.strict + state, token, PDFArray, TArrayValue.OPEN, strict=self.strict ) elif isinstance(token, TokenDictionary): - yield from self.state.flush_int_collections() + yield from state.flush_int_collections() yield from deal_with_collection_object( - self.state, + state, token=token, cls_const=PDFDictionary, open_t=TDictValue.OPEN, strict=self.strict, ) - def _feed_or_yield_objs(self, next_objs: List[PDFObject]): - cur_ctx = self.state.current_context - for obj in next_objs: - if cur_ctx is not None: - cur_ctx.pass_item(obj) - else: - yield obj + def _on_endobj(self, state: ParsingState, token: TokenKeyword) -> IndirectObject: + """finish off the current context, assuming we are handling an indirect-object - def _on_indirect_ref_close(self): - obj = self.state.get_indobjects() - cur_ctx = self.state.current_context + Args: + token (TokenKeyword) + + Raises: + BrokenFile: if the object fif not get initilized with two integers + InvalidKeywordPos: If the context is not an indirect object + + Yields: + PDFObject + """ + last_ctx = state.context_stack.pop() + if isinstance(last_ctx, IndirectObject): + if len(state.int_collection) > 1: + raise BrokenFile(f"Invalid number of integers in obj defition {token}") + for num in state.flush_int_collections(): + last_ctx.pass_item(num) + obj = last_ctx.into_readonly_copy() + self.db.save_indobject(obj) + return obj + else: + raise InvalidKeywordPos(token) + + def _on_stream(self, state: ParsingState, token: TokenKeyword): + """Initilize a PDFStream + + Args: + token (TokenKeyword) + + Raises: + ParserError: if the current context is not an indirect object + """ + last_ctx = state.current_context + if last_ctx is not None and isinstance(last_ctx, IndirectObject): + self.db.save_indobject(last_ctx) + stream = PDFStream(token.end_loc, 0) + obj = self.db.get_object_from_indobj(last_ctx) + stream.pass_item(obj) + last_ctx.save_object(PDFStream(token.end_loc, 0)) + state.context_stack.append(stream) + self.seek(stream.offset + stream.length) + else: + raise ParserError(f"PDFStream is missing initilization dictionary") + + def _on_endstream(self, state: ParsingState, token: TokenKeyword): + """Finish a PDFStream + + Args: + token (TokenKeyword) + + Raises: + InvalidKeywordPos: if the current context is not an indirect stream + """ + stream: PDFContextualObject = state.context_stack.pop() + if not isinstance(stream, PDFStream): + raise InvalidKeywordPos(token) + + def _on_indirect_ref_close(self, state: ParsingState): + obj_num, gen_num = state.get_indobj_values() + obj = self.db.get_object(obj_num, gen_num) + cur_ctx = state.current_context if cur_ctx is not None: cur_ctx.pass_item(obj) else: yield obj - - def _on_endobj(self, token: TokenKeyword): - last_ctx = self.state.context_stack.pop() - if isinstance(last_ctx, IndirectObject): - yield last_ctx.clone() - elif self.strict: - raise InvalidKeywordPos(token) - - def _on_stream(self, token: TokenKeyword): - last_ctx = self.state.current_context - if last_ctx is not None and isinstance(last_ctx, IndirectObject): - stream = PDFStream(token.end_loc, 0) - stream.pass_item(last_ctx.get_object()) - last_ctx.save_object(PDFStream(token.end_loc, 0)) - self.state.context_stack.append(stream) - self.lexer.seek(stream.offset + stream.length) - else: - raise ParserError(f"PDFStream is missing initilization dictionary") - - def _on_endstream(self, token: TokenKeyword): - stream: PDFContextualObject = self.state.context_stack.pop() - if not isinstance(stream, PDFStream): - raise InvalidKeywordPos(token) diff --git a/pdfmajor/parser/PDFStreamParser.py b/pdfmajor/parser/PDFStreamParser.py index a15f350..df76ee3 100644 --- a/pdfmajor/parser/PDFStreamParser.py +++ b/pdfmajor/parser/PDFStreamParser.py @@ -1,7 +1,6 @@ import io - +from pdfmajor.streambuffer import BufferStream from .objects.stream import PDFStream -from .state import ParsingState from .PDFParser import PDFParser from .fliters import process_filters_on_data @@ -10,20 +9,17 @@ class PDFStreamPasrser(PDFParser): def __init__( self, stream: PDFStream, - fp: io.BufferedIOBase, - state: ParsingState, - buffer_size: int = 4096, + buffer: BufferStream, strict: bool = True, ) -> None: - fp.seek(stream.offset) - data = fp.read(stream.length) + cur_pos = buffer.tell() + buffer.seek(stream.offset) + data = buffer.read(stream.length).data + buffer.seek(cur_pos) data = process_filters_on_data(data, stream.filter, stream.decode_parms) data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms) self.stream = stream super().__init__( - fp=io.BytesIO(data), - buffer_size=buffer_size, - state=state, + buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length), strict=strict, - retain_obj_on_seek=True, ) diff --git a/pdfmajor/parser/exceptions.py b/pdfmajor/parser/exceptions.py index 6632a64..57b49a6 100644 --- a/pdfmajor/parser/exceptions.py +++ b/pdfmajor/parser/exceptions.py @@ -6,7 +6,15 @@ class ParserError(PDFMajorException): pass -class InvalidKeywordPos(ParserError): +class BrokenFile(ParserError): + pass + + +class UnexpectedEOF(BrokenFile, EOFError): + pass + + +class InvalidKeywordPos(BrokenFile): def __init__(self, token: TokenKeyword) -> None: self.token = token super().__init__( @@ -18,9 +26,17 @@ class InvalidIndirectObjAccess(ParserError): pass -class DecodeFailed(ParserError): +class DecodeFailed(BrokenFile): pass class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError): pass + + +class PDFNoValidXRef(BrokenFile, EOFError): + pass + + +class InvalidXref(ParserError, KeyError): + pass diff --git a/pdfmajor/parser/objects/collections.py b/pdfmajor/parser/objects/collections.py index 8ee35d4..ebb37d8 100644 --- a/pdfmajor/parser/objects/collections.py +++ b/pdfmajor/parser/objects/collections.py @@ -23,6 +23,13 @@ class PDFDictionary(Dict[str, PDFObject], PDFContextualObject): self.last_name: Optional[PDFName] = None self.strict = strict + @classmethod + def from_dict(cls, d: Dict[str, PDFObject]): + new_obj = cls() + for key, val in d.items(): + new_obj[key] = val + return new_obj + @property def last_item_value(self): return self[self.last_name] diff --git a/pdfmajor/parser/objects/indirect.py b/pdfmajor/parser/objects/indirect.py index 54314e9..53623eb 100644 --- a/pdfmajor/parser/objects/indirect.py +++ b/pdfmajor/parser/objects/indirect.py @@ -1,88 +1,55 @@ -from typing import Dict, Tuple -from ..exceptions import InvalidIndirectObjAccess, ParserError +from typing import Optional +from ..exceptions import InvalidIndirectObjAccess from .base import PDFContextualObject, PDFObject from .primitives import PDFNull -class IndirectObjectCollection: - """A class for collection references to objects""" - - def __init__(self) -> None: - self.objs: Dict[Tuple[int, int], PDFObject] = {} - - def create_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject": - """Create an indirect object and store its value in the collection - return the reference to where this object lies at - - Returns: - IndirectObject: - """ - return IndirectObject(self, obj_num, gen_num, read_only=False) - - def get_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject": - """creates a reference to an object - - Returns: - IndirectObject - """ - return IndirectObject(self, obj_num, gen_num, read_only=True) - - def get_object(self, obj_num: int, gen_num: int) -> PDFObject: - """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10 - - Args: - obj_num (int): object number - gen_num (int): generation number - - Returns: - PDFObject - """ - return self.objs.get((obj_num, gen_num), PDFNull()) - - class IndirectObject(PDFContextualObject): """A class representing a reference for other objects""" def __init__( self, - collection: IndirectObjectCollection, obj_num: int, gen_num: int, - read_only: bool = True, + offset: int, + data: Optional[PDFObject] = None, ) -> None: + self.offset = offset self.obj_num = obj_num self.gen_num = gen_num - self.__read_only = read_only - self.__col = collection + self.__data = data def __repr__(self) -> str: - return f"IndirectObject({self.obj_num}, {self.gen_num})" + return f"IndirectObject(obj_num={self.obj_num}, gen_num={self.gen_num}, offset={self.offset}, data={self.get_object()})" - def get_object(self) -> PDFObject: - return self.__col.get_object(self.obj_num, self.gen_num) + def get_object(self) -> Optional[PDFObject]: + return self.__data def save_object(self, obj: PDFObject): - if self.__read_only: - raise InvalidIndirectObjAccess( - f"attempted to override a read-only reference of an object {obj}" - ) - self.__col.objs[(self.obj_num, self.gen_num)] = obj + # if self.__data is not None: + # raise InvalidIndirectObjAccess( + # f"attempted to override an initilized reference of an object {obj}" + # ) + self.__data = obj - def clone(self): - return IndirectObject(self.__col, self.obj_num, self.gen_num, True) + def into_readonly_copy(self): + return IndirectObject( + self.obj_num, self.gen_num, self.offset, self.get_object() + ) def to_python(self): - return self.get_object().to_python() + obj = self.get_object() + if obj is None: + return None + return obj.to_python() def pass_item(self, item: PDFObject): - if self.__read_only: - raise InvalidIndirectObjAccess( - f"attempted to add to an read-only reference of an object {item}" - ) + # if self.__read_only: + # raise InvalidIndirectObjAccess( + # f"attempted to add to an read-only reference of an object {item}" + # ) cur_obj = self.get_object() if isinstance(cur_obj, PDFContextualObject): cur_obj.pass_item(item) - elif isinstance(cur_obj, PDFNull): - self.save_object(item) else: - raise InvalidIndirectObjAccess(f"Invalid redefinition of an object {item}") + self.save_object(item) diff --git a/pdfmajor/parser/state.py b/pdfmajor/parser/state.py index 99903f9..c594a11 100644 --- a/pdfmajor/parser/state.py +++ b/pdfmajor/parser/state.py @@ -4,33 +4,32 @@ from .exceptions import ParserError from .objects.base import PDFContextualObject, PDFObject from .objects.primitives import PDFInteger from .objects.comment import PDFComment -from .objects.indirect import IndirectObject, IndirectObjectCollection +from .objects.indirect import IndirectObject class ParsingState: - __slots__ = [ - "context_stack", - "int_collection", - "last_obj", - "indirect_object_collection", - ] + __slots__ = ["context_stack", "int_collection", "last_obj", "strict"] def __init__( self, context_stack: List[PDFContextualObject], int_collection: List[PDFInteger], last_obj: Optional[PDFObject] = None, - indirect_object_collection: Optional[IndirectObjectCollection] = None, + strict: bool = False, ) -> None: self.context_stack = context_stack self.int_collection = int_collection - self.int_collection = int_collection self.last_obj = last_obj - self.indirect_object_collection = ( - indirect_object_collection - if indirect_object_collection is not None - else IndirectObjectCollection() + self.strict = strict + + def __repr__(self) -> str: + return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj}, strict={strict})".format( + stackc=len(self.context_stack), + cur_stack=self.current_context, + ints=self.int_collection, + lobj=self.last_obj, + strict=self.strict, ) @property @@ -76,22 +75,15 @@ class ParsingState: return (obj_num.to_python(), gen_num.to_python()) raise ParserError("missing leading tokens for object reference") - def get_indobjects(self) -> IndirectObject: - """Gets an indirect object, based on the values in 'self.int_collection' + def initialize_indirect_obj(self, offset: int): + """Initilizes an indirect object based off the values in self.int_collection - Raises: - ParserError: if the int_collection does not have two elements - - Returns: - IndirectObject + Args: + offset (int): position for the 'obj' keyword """ - obj_num, gen_num = self.get_indobj_values() - return self.indirect_object_collection.get_indobject(obj_num, gen_num) - - def initialize_indirect_obj(self): obj_num, gen_num = self.get_indobj_values() self.context_stack.append( - self.indirect_object_collection.create_indobject(obj_num, gen_num) + IndirectObject(obj_num=obj_num, gen_num=gen_num, offset=offset) ) def flush_int_collections(self) -> Iterator[PDFObject]: @@ -108,3 +100,19 @@ class ParsingState: for obj in self.int_collection: yield obj self.int_collection = [] + + def feed_or_yield_objs(self, next_objs: List[PDFObject]) -> Iterator[PDFObject]: + """If there context, then feed the objects into it, otherwise yield them + + Args: + next_objs (List[PDFObject]) + + Yields: + PDFObject + """ + cur_ctx = self.current_context + for obj in next_objs: + if cur_ctx is not None: + cur_ctx.pass_item(obj) + else: + yield obj diff --git a/pdfmajor/parser/xref.py b/pdfmajor/parser/xref.py index b28b04f..e1179c4 100644 --- a/pdfmajor/parser/xref.py +++ b/pdfmajor/parser/xref.py @@ -1,3 +1,131 @@ +# from pdfmajor.parser.objects.indirect import IndirectObjectCollection +from pdfmajor.lexer.token import TokenInteger, TokenKeyword +from pdfmajor.lexer import PDFLexer +from typing import Iterator, Literal, NamedTuple, Union +from pdfmajor.streambuffer import BufferStream +from .exceptions import PDFNoValidXRef, BrokenFile, UnexpectedEOF +class XRefRow(NamedTuple): + """a parsed version of a row in the xref table + as defined in PDF spec 1.7 section 7.5.4 + """ + offset: int + obj_num: int + gen_num: int + use: Union[Literal["n"], Literal["f"]] + + +def find_start_of_xref(buffer: BufferStream, strict: bool = False) -> int: + """Internal function used to locate the start of the XRef table""" + prev = None + found_eof = False + for _, line in buffer.into_reverse_reader_iter(): + line = line.strip() + found_eof |= line.strip() == b"%%EOF" + if line == b"startxref": + break + if line: + prev = line + if prev is None: + raise PDFNoValidXRef() + if strict and not found_eof: + raise BrokenFile( + "Failed to find %%EOF at end of file, this can indicate the file is incomplete or broken." + ) + return int(prev) + + +def iter_over_xref(buffer: BufferStream, strict: bool = False) -> Iterator[XRefRow]: + start_offset = find_start_of_xref(buffer, strict) + buffer.seek(start_offset) + lexer = PDFLexer(buffer) + first_token = next(lexer.iter_tokens()) + if isinstance(first_token, TokenInteger): + raise NotImplementedError("XRefStream: PDF-1.5") + elif isinstance(first_token, TokenKeyword): + if first_token.value == b"xref": + return iter_over_standard_xref(buffer, strict) + raise BrokenFile("Missing a valid XRef table") + + +# # read xref table +# def read_xref_from(self, parser, start, xrefs): +# """Reads XRefs from the given location.""" +# parser.seek(start) +# parser.reset() +# try: +# (pos, token) = parser.nexttoken() +# except PSEOF: +# raise PDFNoValidXRef("Unexpected EOF") +# log.info("read_xref_from: start=%d, token=%r", start, token) +# if isinstance(token, int): +# # XRefStream: PDF-1.5 +# parser.seek(pos) +# parser.reset() +# xref = PDFXRefStream() +# xref.load(parser) +# else: +# if token is parser.KEYWORD_XREF: +# parser.nextline() +# xref = PDFXRef() +# xref.load(parser) +# xrefs.append(xref) +# trailer = xref.get_trailer() +# log.info("trailer: %r", trailer) +# if "XRefStm" in trailer: +# pos = int_value(trailer["XRefStm"]) +# self.read_xref_from(parser, pos, xrefs) +# if "Prev" in trailer: +# # find previous xref +# pos = int_value(trailer["Prev"]) +# self.read_xref_from(parser, pos, xrefs) +# return + + +def iter_over_standard_xref( + buffer: BufferStream, strict: bool = False +) -> Iterator[XRefRow]: + while True: + bbline = buffer.get_firstnonempty_line() + if bbline is None: + raise UnexpectedEOF + pos, line = bbline + if line.startswith(b"trailer"): + buffer.seek(pos) + break + descriptors = line.strip().split(b" ") + if len(descriptors) != 2: + raise PDFNoValidXRef(f"Trailer not found: pos={pos} line={line!r}") + try: + (start, nobjs) = map(int, descriptors) + except ValueError: + raise PDFNoValidXRef( + f"Failed to decipher xref row: pos={pos} line={line!r}" + ) + for obj_num in range(start, start + nobjs): + _, line = buffer.readline() + f = line.strip().split(b" ") + if len(f) != 3: + raise PDFNoValidXRef(f"Invalid XRef format: pos={pos} line={line!r}") + try: + pos = int(f[0]) + gen_num = int(f[1]) + use = f[2] + except ValueError: + raise PDFNoValidXRef( + f"Failed to decipher xref row: pos={pos} line={line!r}" + ) + if use not in [b"n", b"f"] and not strict: + raise PDFNoValidXRef( + f"Invalid 'use' value for xref {use!r}: pos={pos} line={line!r}" + ) + yield XRefRow(offset=pos, obj_num=obj_num, gen_num=gen_num, use=use) # type: ignore + + +# def construct_indobj_collection( +# buffer: BufferStream, it_xref: Iterator[XRefRow] +# ) -> IndirectObjectCollection: +# collection = IndirectObjectCollection() +# parser = diff --git a/pdfmajor/streambuffer.py b/pdfmajor/streambuffer.py index 7e91671..23fabdf 100644 --- a/pdfmajor/streambuffer.py +++ b/pdfmajor/streambuffer.py @@ -1,5 +1,5 @@ import io -from typing import Iterator, NamedTuple +from typing import Iterator, NamedTuple, Optional from pdfmajor.exceptions import PDFMajorException @@ -19,9 +19,35 @@ class BufferStream: __slots__ = ["fp", "buffer_size"] def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None: + """[summary] + + Args: + fp (io.BufferedIOBase) + buffer_size (int, optional): maximum length of the bytes this will read at each iteration. Defaults to 4096. + """ self.fp = fp self.buffer_size = buffer_size + def get_slice( + self, offset: int, length: int, buffer_size: Optional[int] = None + ) -> "BufferStream": + """Creates a slice of the current byte-stream and returns a BufferStream based on that slice + Args: + offset (int): where the slice starts + length (int): length of the slice + buffer_size (int, optional) buffer size for new slice. Defaults to self.buffer_size + Returns: + BufferStream + """ + cur_pos = self.tell() + self.seek(offset) + bslice = self.fp.read(length) + self.seek(cur_pos) + return BufferStream( + io.BytesIO(bslice), + buffer_size=buffer_size if buffer_size is not None else self.buffer_size, + ) + def seekd(self, step: int) -> int: """similar to self.fp.seek but you move the offset relative to where you are now @@ -63,6 +89,27 @@ class BufferStream: buf = self.fp.read(size) return BufferedBytes(pos, buf) + def readline(self) -> BufferedBytes: + """Reads upto a line-end + + Returns: + BufferedBytes + """ + pos = self.tell() + buf = self.fp.readline() + return BufferedBytes(pos, buf) + + def get_firstnonempty_line(self) -> Optional[BufferedBytes]: + """reads line-by-line until we find a none-empty line + + Returns: + Optional[BufferedBytes] + """ + for bbline in self.into_line_iter(): + if bbline.data.strip(): + return bbline + return None + def __iter__(self): if self.fp.read(1) == b"": raise StreamEOF @@ -84,6 +131,23 @@ class BufferStream: def __exit__(self): self.close() + def into_line_iter(self) -> Iterator[BufferedBytes]: + """reads the stream line by line + + Raises: + StreamEOF + + Yields: + BufferedBytes + """ + last_pos = self.tell() + while True: + bbyte = self.readline() + if self.tell() == last_pos: + raise StreamEOF + yield bbyte + last_pos = bbyte.pos + def into_reverse_reader_iter(self) -> Iterator[BufferedBytes]: """reads the file in reverse, outputting complete lines diff --git a/tests/test_lexer.py b/tests/test_lexer.py index ce5b361..44ee271 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -222,7 +222,7 @@ class Basics(TestCase): class Lexer(TestCase): def run_test(self, raw: bytes, expected: List[Token]): - tokenizer = PDFLexer(io.BytesIO(raw)) + tokenizer = PDFLexer.from_bytes(raw) for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1): self.assertEqual(token, etoken, f"Failed at example #{i}") try: diff --git a/tests/test_parser.py b/tests/test_parser.py index 3f06450..016b5f4 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,12 +1,13 @@ -import io from pathlib import Path +from pdfmajor.parser.objects.collections import PDFDictionary +from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString -from typing import Callable, List +from typing import List from decimal import Decimal from unittest import TestCase from pdfmajor.streambuffer import StreamEOF -from pdfmajor.parser.objects.indirect import IndirectObject, IndirectObjectCollection +from pdfmajor.parser.objects.indirect import IndirectObject from pdfmajor.parser.objects.base import PDFObject from pdfmajor.parser.objects.comment import PDFComment from pdfmajor.parser.objects.stream import PDFStream @@ -17,9 +18,8 @@ CURRENT_FOLDER = Path(__file__).parent class Collections(TestCase): def test_parse_dict(self): - parser = PDFParser( - io.BytesIO( - br"""<< /Type /Example + parser = PDFParser.from_bytes( + br"""<< /Type /Example /Subtype /DictionaryExample /Version 0.01 /IntegerItem 12 @@ -29,8 +29,7 @@ class Collections(TestCase): /Item2 true /LastItem ( not! ) /VeryLastItem ( OK ) >> - >>""" - ), + >>""", strict=True, ) obj = next(parser.iter_objects()) @@ -52,8 +51,8 @@ class Collections(TestCase): ) def test_parse_array(self): - parser = PDFParser( - io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]"""), + parser = PDFParser.from_bytes( + br"""[ 549 3.14 false ( Ralph ) /SomeName ]""", strict=True, ) obj = next(parser.iter_objects()) @@ -63,13 +62,11 @@ class Collections(TestCase): ) def test_parse_nested_array(self): - parser = PDFParser( - io.BytesIO( - br"""[ [10 30] [22 33] [ + parser = PDFParser.from_bytes( + br"""[ [10 30] [22 33] [ << /x 0 /y 5 >> << /x 10 /y 5 >> - ] ]""" - ), + ] ]""", strict=True, ) obj = next(parser.iter_objects()) @@ -88,9 +85,8 @@ class Collections(TestCase): class IndirectObjects(TestCase): def test_parse_simple_indobj(self): - parser = PDFParser( - io.BytesIO( - br""" + parser = PDFParser.from_bytes( + br""" 12 1 obj (Bring) endobj 100 0 obj <33> @@ -98,47 +94,45 @@ class IndirectObjects(TestCase): 13 8 obj << /x 1 /y 3>> endobj - """ - ), + """, strict=True, ) expected = [ - (12, 1, "Bring"), - (100, 0, b"3"), - (13, 8, {"x": 1, "y": 3}), + IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)), + IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)), + IndirectObject( + 13, + 8, + 144, + PDFDictionary.from_dict( + {"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)} + ), + ), ] for obj, eobj in zip(parser.iter_objects(), expected): self.assertIsInstance(obj, IndirectObject) - self.assertEqual( - obj, - parser.state.indirect_object_collection.get_indobject(eobj[0], eobj[1]), - ) - self.assertEqual(obj.to_python(), eobj[2]) + self.assertEqual(obj, eobj) with self.assertRaises(StreamEOF): next(parser.iter_objects()) def test_parse_obj_with_references(self): - parser = PDFParser( - io.BytesIO( - br""" + parser = PDFParser.from_bytes( + br""" 12 1 obj (Bring) endobj 10 2 obj << /lastone 12 1 R >> endobj - """ - ), + """, strict=True, ) expected = [ - parser.state.indirect_object_collection.get_indobject(12, 1), - parser.state.indirect_object_collection.get_indobject(10, 2), + "Bring", + {"lastone": "Bring"}, ] for obj, exp in zip(parser.iter_objects(), expected): - self.assertEqual(obj, exp) + self.assertEqual(obj.to_python(), exp) with self.assertRaises(StreamEOF): next(parser.iter_objects()) - sec_obj = parser.state.indirect_object_collection.get_indobject( - 10, 2 - ).get_object() + sec_obj = parser.db.get_object(10, 2) self.assertEqual( sec_obj.to_python(), {"lastone": "Bring"}, @@ -147,13 +141,11 @@ class IndirectObjects(TestCase): def run_test( self, raw: bytes, - expected: Callable[[IndirectObjectCollection], List[PDFObject]], + expected: List[PDFObject], ): - parser = PDFParser(io.BytesIO(raw), strict=True) + parser = PDFParser.from_bytes(raw, strict=True) for i, (eobj, obj) in enumerate( - zip( - expected(parser.state.indirect_object_collection), parser.iter_objects() - ), + zip(expected, parser.iter_objects()), 1, ): self.assertEqual(obj, eobj, f"Failed at example #{i}") @@ -172,20 +164,19 @@ class IndirectObjects(TestCase): endobj %closing the object 11 1 obj 2 endobj """, - expected=lambda indobjc: [ + expected=[ PDFComment(b" object references", (22, 41)), PDFComment(b" start of object", (62, 79)), PDFComment(b" object content", (110, 126)), - indobjc.get_indobject(10, 0), + IndirectObject(10, 0, 58, PDFString("testing", 0, 0)), PDFComment(b"closing the object", (150, 169)), - indobjc.get_indobject(11, 1), + IndirectObject(11, 1, 191, PDFInteger(2, 0, 0)), ], ) def test_parse_stream(self): - parser = PDFParser( - io.BytesIO( - br""" + parser = PDFParser.from_bytes( + br""" 5 0 obj << /Type /XRef /Length 124 /Filter /FlateDecode >> stream @@ -193,7 +184,6 @@ stream <80>endstream endobj """ - ) ) for obj in parser.iter_objects(): self.assertIsInstance(obj, IndirectObject) diff --git a/tests/test_xref.py b/tests/test_xref.py new file mode 100644 index 0000000..e775381 --- /dev/null +++ b/tests/test_xref.py @@ -0,0 +1,15 @@ +from pathlib import Path +from unittest import TestCase + +from pdfmajor.streambuffer import BufferStream +from pdfmajor.parser.xref import iter_over_xref + +CURRENT_FOLDER = Path(__file__).parent + + +class Standard(TestCase): + def test_char_doc(self): + with (CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf").open("rb") as fp: + buffer = BufferStream(fp) + for xref in iter_over_xref(buffer): + print(xref)