From 3d39bd575fe0ab5564393811c13af96b13e3cbb6 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Tue, 27 Oct 2020 09:46:51 -0400 Subject: [PATCH] converted parser to functional fixed a bufsize issue on the lexer --- pdfmajor/lexer/__init__.py | 12 +- pdfmajor/parser/PDFParser.py | 243 ---------------------------- pdfmajor/parser/PDFStreamParser.py | 37 +++-- pdfmajor/parser/__init__.py | 197 ++++++++++++++++++++++ pdfmajor/parser/exceptions.py | 5 +- pdfmajor/parser/objects/indirect.py | 19 ++- pdfmajor/parser/parsers.py | 7 +- pdfmajor/parser/state.py | 7 +- pdfmajor/parser/xref.py | 5 +- pdfmajor/parser/xrefdb.py | 27 ++++ pdfmajor/streambuffer.py | 11 ++ tests/test_parser.py | 162 ++++++++++--------- tests/test_xref.py | 8 +- 13 files changed, 370 insertions(+), 370 deletions(-) delete mode 100644 pdfmajor/parser/PDFParser.py create mode 100644 pdfmajor/parser/xrefdb.py diff --git a/pdfmajor/lexer/__init__.py b/pdfmajor/lexer/__init__.py index 3b28586..1a1c6b8 100644 --- a/pdfmajor/lexer/__init__.py +++ b/pdfmajor/lexer/__init__.py @@ -30,9 +30,7 @@ def iter_tokens(buffer: BufferStream) -> Iterator[Token]: else: j = m.start(0) next_char = buf[j : j + 1] - if next_char == b"~": - continue - elif next_char == b"%": + if next_char == b"%": buffer.seek(bufpos + j + 1) yield parse_comment(buffer) elif next_char == b"/": @@ -54,10 +52,12 @@ def iter_tokens(buffer: BufferStream) -> Iterator[Token]: buffer.seek(bufpos + j + 1) yield parse_double_angled_bracket(buffer) elif next_char == b">": - buffer.seek(bufpos + j + 1) - if buf[j : j + 2] == b">>": - buffer.seekd(1) + buffer.seek(bufpos + j) + bufpos, buf = buffer.read(2) + if buf == b">>": yield TokenDictionary(bufpos + j, bufpos + j + 2, TDictValue.CLOSE) + else: + buffer.seekd(-1) elif next_char == b"[": buffer.seek(bufpos + j + 1) yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN) diff --git a/pdfmajor/parser/PDFParser.py b/pdfmajor/parser/PDFParser.py deleted file mode 100644 index feee107..0000000 --- a/pdfmajor/parser/PDFParser.py +++ /dev/null @@ -1,243 +0,0 @@ -import io - -from typing import Dict, Iterator, Optional, Tuple - -from pdfmajor.streambuffer import BufferStream -from pdfmajor.parser.objects.stream import PDFStream -from pdfmajor.parser.exceptions import InvalidXref -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.lexer.token import TokenKeyword -from pdfmajor.lexer import PDFLexer -from pdfmajor.lexer.token import ( - TArrayValue, - TDictValue, - TokenArray, - TokenComment, - TokenDictionary, - TokenKeyword, -) -from .objects.indirect import IndirectObject -from .objects.base import PDFContextualObject, PDFObject -from .objects.collections import PDFArray, PDFDictionary -from .objects.comment import PDFComment -from .state import ParsingState -from .parsers import attempt_parse_prim, deal_with_collection_object -from .exceptions import InvalidKeywordPos, BrokenFile, InvalidKeywordPos, ParserError - - -class XRefDB: - """A class for collection references to objects""" - - def __init__(self, parser: "PDFParser") -> None: - self.objs: Dict[Tuple[int, int], IndirectObject] = {} - self.parser = parser - - def save_indobject(self, indobj: IndirectObject): - if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None: - self.objs[(indobj.obj_num, indobj.gen_num)] = indobj - - def get_object_from_indobj(self, indobj: IndirectObject) -> PDFObject: - return self.get_object(indobj.obj_num, indobj.gen_num) - - def get_object(self, obj_num: int, gen_num: int) -> PDFObject: - """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10 - - Args: - obj_num (int): object number - gen_num (int): generation number - - Returns: - PDFObject - """ - indobj = self.objs.get((obj_num, gen_num), None) - if indobj is None: - raise InvalidXref( - f"xref not found for object retrieval {obj_num}, {gen_num}" - ) - else: - obj = indobj.get_object() - if obj is not None: - return obj - cur_pos = self.parser.tell() - self.parser.seek(indobj.offset) - obj = next(self.parser.iter_objects()) - indobj.save_object(obj) - self.parser.seek(cur_pos) - return obj - - -class PDFParser: - """PDF Parser""" - - def __init__( - self, - buffer: BufferStream, - db: Optional[XRefDB] = None, - strict: bool = True, - ) -> None: - """ - - Args: - fp (io.BufferedIOBase) - buffer_size (int, optional): the size of the reads. Defaults to 4096. - state (Optional[ParsingState], optional): the parsing state. Defaults to None. - strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True. - """ - self.lexer = PDFLexer(buffer) - self.db = XRefDB(self) - self.strict = strict - - @classmethod - def from_iobuffer( - cls, - fp: io.BufferedIOBase, - buffer_size: int = 4096, - strict: bool = True, - ): - return cls( - BufferStream(fp, buffer_size=buffer_size), - strict=strict, - ) - - @classmethod - def from_bytes( - cls, - data: bytes, - buffer_size: int = 4096, - strict: bool = True, - ): - return cls( - BufferStream(io.BytesIO(data), buffer_size=buffer_size), - strict=strict, - ) - - def seek(self, offset: int) -> int: - """moves the offset of the reader to a certain spot, and returns the new offset - - Args: - offset (int) - - Returns: - int: new offset - """ - return self.lexer.seek(offset) - - def tell(self) -> int: - return self.lexer.tell() - - def iter_objects(self) -> Iterator[PDFObject]: - """Iterates over the objects in the current byte stream - - Yields: - PDFObject - """ - state = ParsingState([], [], strict=self.strict) - for token in self.lexer.iter_tokens(): - attempt_prim = attempt_parse_prim(token, state) - if attempt_prim.success: - yield from state.feed_or_yield_objs(attempt_prim.next_objs) - elif isinstance(token, TokenComment): - yield state.set_last_obj( - PDFComment( - token.value, - (token.start_loc, token.end_loc), - ) - ) - elif isinstance(token, TokenKeyword): - if token.value == b"R": - yield from self._on_indirect_ref_close(state) - elif token.value == b"obj": - state.initialize_indirect_obj(token.start_loc) - elif token.value == b"endobj": - yield self._on_endobj(state, token) - elif token.value == b"stream": - self._on_stream( - state, - token, - ) - elif token.value == b"endstream": - self._on_endstream(state, token) - else: - raise InvalidKeywordPos(token) - elif isinstance(token, TokenArray): - yield from state.flush_int_collections() - yield from deal_with_collection_object( - state, token, PDFArray, TArrayValue.OPEN, strict=self.strict - ) - elif isinstance(token, TokenDictionary): - yield from state.flush_int_collections() - yield from deal_with_collection_object( - state, - token=token, - cls_const=PDFDictionary, - open_t=TDictValue.OPEN, - strict=self.strict, - ) - - def _on_endobj(self, state: ParsingState, token: TokenKeyword) -> IndirectObject: - """finish off the current context, assuming we are handling an indirect-object - - Args: - token (TokenKeyword) - - Raises: - BrokenFile: if the object fif not get initilized with two integers - InvalidKeywordPos: If the context is not an indirect object - - Yields: - PDFObject - """ - last_ctx = state.context_stack.pop() - if isinstance(last_ctx, IndirectObject): - if len(state.int_collection) > 1: - raise BrokenFile(f"Invalid number of integers in obj defition {token}") - for num in state.flush_int_collections(): - last_ctx.pass_item(num) - obj = last_ctx.into_readonly_copy() - self.db.save_indobject(obj) - return obj - else: - raise InvalidKeywordPos(token) - - def _on_stream(self, state: ParsingState, token: TokenKeyword): - """Initilize a PDFStream - - Args: - token (TokenKeyword) - - Raises: - ParserError: if the current context is not an indirect object - """ - last_ctx = state.current_context - if last_ctx is not None and isinstance(last_ctx, IndirectObject): - self.db.save_indobject(last_ctx) - stream = PDFStream(token.end_loc, 0) - obj = self.db.get_object_from_indobj(last_ctx) - stream.pass_item(obj) - last_ctx.save_object(PDFStream(token.end_loc, 0)) - state.context_stack.append(stream) - self.seek(stream.offset + stream.length) - else: - raise ParserError(f"PDFStream is missing initilization dictionary") - - def _on_endstream(self, state: ParsingState, token: TokenKeyword): - """Finish a PDFStream - - Args: - token (TokenKeyword) - - Raises: - InvalidKeywordPos: if the current context is not an indirect stream - """ - stream: PDFContextualObject = state.context_stack.pop() - if not isinstance(stream, PDFStream): - raise InvalidKeywordPos(token) - - def _on_indirect_ref_close(self, state: ParsingState): - obj_num, gen_num = state.get_indobj_values() - obj = self.db.get_object(obj_num, gen_num) - cur_ctx = state.current_context - if cur_ctx is not None: - cur_ctx.pass_item(obj) - else: - yield obj diff --git a/pdfmajor/parser/PDFStreamParser.py b/pdfmajor/parser/PDFStreamParser.py index df76ee3..676e6b1 100644 --- a/pdfmajor/parser/PDFStreamParser.py +++ b/pdfmajor/parser/PDFStreamParser.py @@ -1,25 +1,24 @@ import io from pdfmajor.streambuffer import BufferStream from .objects.stream import PDFStream -from .PDFParser import PDFParser from .fliters import process_filters_on_data -class PDFStreamPasrser(PDFParser): - def __init__( - self, - stream: PDFStream, - buffer: BufferStream, - strict: bool = True, - ) -> None: - cur_pos = buffer.tell() - buffer.seek(stream.offset) - data = buffer.read(stream.length).data - buffer.seek(cur_pos) - data = process_filters_on_data(data, stream.filter, stream.decode_parms) - data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms) - self.stream = stream - super().__init__( - buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length), - strict=strict, - ) +# class PDFStreamPasrser(PDFParser): +# def __init__( +# self, +# stream: PDFStream, +# buffer: BufferStream, +# strict: bool = True, +# ) -> None: +# cur_pos = buffer.tell() +# buffer.seek(stream.offset) +# data = buffer.read(stream.length).data +# buffer.seek(cur_pos) +# data = process_filters_on_data(data, stream.filter, stream.decode_parms) +# data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms) +# self.stream = stream +# super().__init__( +# buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length), +# strict=strict, +# ) diff --git a/pdfmajor/parser/__init__.py b/pdfmajor/parser/__init__.py index e69de29..71b1a04 100644 --- a/pdfmajor/parser/__init__.py +++ b/pdfmajor/parser/__init__.py @@ -0,0 +1,197 @@ +from pdfmajor.parser.xref import XRefRow +from pdfmajor.parser.xrefdb import XRefDB +from typing import Iterator, Optional +from pdfmajor.streambuffer import BufferStream +from pdfmajor.lexer.token import ( + TArrayValue, + TDictValue, + TokenArray, + TokenComment, + TokenDictionary, + TokenKeyword, +) +from pdfmajor.lexer import iter_tokens +from .objects.stream import PDFStream +from .objects.indirect import IndirectObject +from .objects.base import PDFContextualObject, PDFObject +from .objects.collections import PDFArray, PDFDictionary +from .objects.comment import PDFComment +from .state import ParsingState +from .parsers import attempt_parse_prim, deal_with_collection_object +from .exceptions import ( + InvalidKeywordPos, + BrokenFile, + InvalidKeywordPos, + InvalidXref, + ParserError, +) + + +def iter_objects( + buffer: BufferStream, + xrefdb: XRefDB, + state: Optional[ParsingState] = None, +) -> Iterator[PDFObject]: + """Iterates over the objects in the current byte stream + + Yields: + PDFObject + """ + state = ParsingState([], []) if state is None else state + for token in iter_tokens(buffer): + attempt_prim = attempt_parse_prim(token, state) + if attempt_prim.success: + yield from state.feed_or_yield_objs(attempt_prim.next_objs) + elif isinstance(token, TokenComment): + yield state.set_last_obj( + PDFComment( + token.value, + (token.start_loc, token.end_loc), + ) + ) + elif isinstance(token, TokenKeyword): + if token.value == b"R": + yield from _on_indirect_ref_close(buffer, state, xrefdb) + elif token.value == b"obj": + state.initialize_indirect_obj(token.start_loc) + elif token.value == b"endobj": + obj = _on_endobj(state, token) + xrefdb.save_indobject(obj) + yield obj + elif token.value == b"stream": + _on_stream( + buffer, + state, + xrefdb, + token, + ) + else: + raise InvalidKeywordPos(token, [b"R", b"obj", b"endobj", b"stream"]) + elif isinstance(token, TokenArray): + yield from state.flush_int_collections() + yield from deal_with_collection_object( + state, token, PDFArray, TArrayValue.OPEN + ) + elif isinstance(token, TokenDictionary): + yield from state.flush_int_collections() + yield from deal_with_collection_object( + state, + token=token, + cls_const=PDFDictionary, + open_t=TDictValue.OPEN, + ) + + +def _on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject: + """finish off the current context, assuming we are handling an indirect-object + + Args: + token (TokenKeyword) + + Raises: + BrokenFile: if the object fif not get initilized with two integers + InvalidKeywordPos: If the context is not an indirect object + + Yields: + PDFObject + """ + last_ctx = state.context_stack.pop() + if isinstance(last_ctx, IndirectObject): + if len(state.int_collection) > 1: + raise BrokenFile(f"Invalid number of integers in obj defition {token}") + for num in state.flush_int_collections(): + last_ctx.pass_item(num) + return last_ctx.into_readonly_copy() + else: + raise ParserError(f"Recived an endobj while not procesing an indirect object") + + +def _on_stream( + buffer: BufferStream, state: ParsingState, xrefdb: XRefDB, token: TokenKeyword +): + """Initilize a PDFStream + + Args: + token (TokenKeyword) + + Raises: + ParserError: if the current context is not an indirect object + """ + last_ctx = state.current_context + if last_ctx is not None and isinstance(last_ctx, IndirectObject): + xrefdb.save_indobject(last_ctx) + stream = PDFStream(token.end_loc + 1, 0) + obj = _save_or_get_object(buffer, xrefdb, last_ctx.obj_num, last_ctx.gen_num) + with buffer.get_window(): + next_token = next(iter_tokens(buffer)) + if not ( + isinstance(next_token, TokenKeyword) and next_token.value == b"endstream" + ): + raise BrokenFile( + f"stream did not contain 'endstream', instead the token {next_token} was found" + ) + stream.pass_item(obj) + buffer.seek(stream.offset + stream.length) + last_ctx.save_object(stream) + state.context_stack.append(stream) + else: + raise ParserError(f"PDFStream is missing initilization dictionary") + + +def _on_indirect_ref_close(buffer: BufferStream, state: ParsingState, xrefdb: XRefDB): + obj_num, gen_num = state.get_indobj_values() + obj = _save_or_get_object(buffer, xrefdb, obj_num, gen_num) + cur_ctx = state.current_context + if cur_ctx is not None: + cur_ctx.pass_item(obj) + else: + yield obj + + +def _save_or_get_object( + buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int +) -> PDFObject: + """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10 + + Args: + obj_num (int): object number + gen_num (int): generation number + + Returns: + PDFObject + """ + indobj = xrefdb.objs.get((obj_num, gen_num), None) + if indobj is None: + xref = xrefdb.xrefs.get((obj_num, gen_num), None) + if xref is None: + raise InvalidXref( + f"xref not found for object retrieval {obj_num}, {gen_num}" + ) + return _read_object_from_xref(xref, xrefdb, buffer) + else: + return _read_object_from_db_or_buffer(indobj, xrefdb, buffer) + + +def _read_object_from_xref( + xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream +) -> PDFObject: + indobj = IndirectObject(xref.obj_num, xref.gen_num, xref.offset) + with buffer.get_window(): + buffer.seek(indobj.offset) + obj = next(iter_objects(buffer, xrefdb)) + indobj.save_object(obj) + xrefdb.save_indobject(indobj) + return obj + + +def _read_object_from_db_or_buffer( + indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream +) -> PDFObject: + obj = indobj.get_object() + if obj is not None: + return obj + with buffer.get_window(): + buffer.seek(indobj.offset) + obj = next(iter_objects(buffer, xrefdb)) + indobj.save_object(obj) + return obj diff --git a/pdfmajor/parser/exceptions.py b/pdfmajor/parser/exceptions.py index 57b49a6..af47e3b 100644 --- a/pdfmajor/parser/exceptions.py +++ b/pdfmajor/parser/exceptions.py @@ -1,3 +1,4 @@ +from typing import List from pdfmajor.lexer.token import TokenKeyword from pdfmajor.exceptions import PDFMajorException @@ -15,10 +16,10 @@ class UnexpectedEOF(BrokenFile, EOFError): class InvalidKeywordPos(BrokenFile): - def __init__(self, token: TokenKeyword) -> None: + def __init__(self, token: TokenKeyword, expected_token: List[bytes]) -> None: self.token = token super().__init__( - f"keyword '{token.value!r}' was found at {token.start_loc}:{token.end_loc}" + f"keyword '{token.value!r}' was found at {token.start_loc}:{token.end_loc} when expecting {expected_token}" ) diff --git a/pdfmajor/parser/objects/indirect.py b/pdfmajor/parser/objects/indirect.py index 53623eb..dd1269b 100644 --- a/pdfmajor/parser/objects/indirect.py +++ b/pdfmajor/parser/objects/indirect.py @@ -26,10 +26,12 @@ class IndirectObject(PDFContextualObject): return self.__data def save_object(self, obj: PDFObject): - # if self.__data is not None: - # raise InvalidIndirectObjAccess( - # f"attempted to override an initilized reference of an object {obj}" - # ) + """This method will override the current stored object + + Args: + obj (PDFObject) + + """ self.__data = obj def into_readonly_copy(self): @@ -44,10 +46,11 @@ class IndirectObject(PDFContextualObject): return obj.to_python() def pass_item(self, item: PDFObject): - # if self.__read_only: - # raise InvalidIndirectObjAccess( - # f"attempted to add to an read-only reference of an object {item}" - # ) + """This method will either propogate the item to it's child (in case the child is a contexual object) or override it's own data with the incoming + + Args: + item (PDFObject): [description] + """ cur_obj = self.get_object() if isinstance(cur_obj, PDFContextualObject): cur_obj.pass_item(item) diff --git a/pdfmajor/parser/parsers.py b/pdfmajor/parser/parsers.py index b965d01..fccffad 100644 --- a/pdfmajor/parser/parsers.py +++ b/pdfmajor/parser/parsers.py @@ -1,4 +1,4 @@ -from typing import Iterator, List, Optional, NamedTuple, Type +from typing import Iterator, List, NamedTuple, Type from pdfmajor.lexer.token import Token, TokenComplexType, TokenComplexTypeVal @@ -49,7 +49,6 @@ def deal_with_collection_object( token: TokenComplexType, cls_const: Type[PDFContextualObject], open_t: TokenComplexTypeVal, - strict: bool = False, ) -> Iterator[PDFObject]: """generic method for processing things that have a start and end token @@ -72,9 +71,9 @@ def deal_with_collection_object( return else: cur_ctx.pass_item(last_ctx) - elif strict: + else: raise ParserError(f"Invalid end of {cls_const} specified at {token}") - elif strict: + else: raise ParserError( f"Invalid end of {cls_const} specified at {token} with no start" ) diff --git a/pdfmajor/parser/state.py b/pdfmajor/parser/state.py index c594a11..5e8204f 100644 --- a/pdfmajor/parser/state.py +++ b/pdfmajor/parser/state.py @@ -9,27 +9,24 @@ from .objects.indirect import IndirectObject class ParsingState: - __slots__ = ["context_stack", "int_collection", "last_obj", "strict"] + __slots__ = ["context_stack", "int_collection", "last_obj"] def __init__( self, context_stack: List[PDFContextualObject], int_collection: List[PDFInteger], last_obj: Optional[PDFObject] = None, - strict: bool = False, ) -> None: self.context_stack = context_stack self.int_collection = int_collection self.last_obj = last_obj - self.strict = strict def __repr__(self) -> str: - return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj}, strict={strict})".format( + return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj})".format( stackc=len(self.context_stack), cur_stack=self.current_context, ints=self.int_collection, lobj=self.last_obj, - strict=self.strict, ) @property diff --git a/pdfmajor/parser/xref.py b/pdfmajor/parser/xref.py index e1179c4..fb86db5 100644 --- a/pdfmajor/parser/xref.py +++ b/pdfmajor/parser/xref.py @@ -1,6 +1,6 @@ # from pdfmajor.parser.objects.indirect import IndirectObjectCollection from pdfmajor.lexer.token import TokenInteger, TokenKeyword -from pdfmajor.lexer import PDFLexer +from pdfmajor.lexer import iter_tokens from typing import Iterator, Literal, NamedTuple, Union from pdfmajor.streambuffer import BufferStream from .exceptions import PDFNoValidXRef, BrokenFile, UnexpectedEOF @@ -40,8 +40,7 @@ def find_start_of_xref(buffer: BufferStream, strict: bool = False) -> int: def iter_over_xref(buffer: BufferStream, strict: bool = False) -> Iterator[XRefRow]: start_offset = find_start_of_xref(buffer, strict) buffer.seek(start_offset) - lexer = PDFLexer(buffer) - first_token = next(lexer.iter_tokens()) + first_token = next(iter_tokens(buffer)) if isinstance(first_token, TokenInteger): raise NotImplementedError("XRefStream: PDF-1.5") elif isinstance(first_token, TokenKeyword): diff --git a/pdfmajor/parser/xrefdb.py b/pdfmajor/parser/xrefdb.py new file mode 100644 index 0000000..caa9507 --- /dev/null +++ b/pdfmajor/parser/xrefdb.py @@ -0,0 +1,27 @@ +from pdfmajor.parser.xref import XRefRow + +from typing import Dict, Iterator, Tuple + +from pdfmajor.parser.exceptions import InvalidXref +from pdfmajor.parser.objects.base import PDFObject +from .objects.indirect import IndirectObject +from .objects.base import PDFObject + + +class XRefDB: + """A class for collection references to objects""" + + def __init__(self) -> None: + self.objs: Dict[Tuple[int, int], IndirectObject] = {} + self.xrefs: Dict[Tuple[int, int], XRefRow] = {} + + def update_from_xrefiter(self, it_xref: Iterator[XRefRow]): + for xref in it_xref: + if xref.use == b"n": + self.xrefs[(xref.obj_num, xref.gen_num)] = xref + + def save_indobject(self, indobj: IndirectObject): + if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None: + self.objs[(indobj.obj_num, indobj.gen_num)] = indobj + else: + print("WARN: saving twice", indobj) diff --git a/pdfmajor/streambuffer.py b/pdfmajor/streambuffer.py index 23fabdf..2dd9de2 100644 --- a/pdfmajor/streambuffer.py +++ b/pdfmajor/streambuffer.py @@ -1,5 +1,6 @@ import io from typing import Iterator, NamedTuple, Optional +from contextlib import contextmanager from pdfmajor.exceptions import PDFMajorException @@ -28,6 +29,10 @@ class BufferStream: self.fp = fp self.buffer_size = buffer_size + @classmethod + def from_bytes(cls, raw: bytes, buffer_size: int = 4096): + return cls(io.BytesIO(raw), buffer_size) + def get_slice( self, offset: int, length: int, buffer_size: Optional[int] = None ) -> "BufferStream": @@ -175,3 +180,9 @@ class BufferStream: buf = b"" if len(buf) > 0 and pos == 0 and n == -1: yield BufferedBytes(pos=n + 1, data=buf) + + @contextmanager + def get_window(self): + cur_pos = self.tell() + yield self + self.seek(cur_pos) diff --git a/tests/test_parser.py b/tests/test_parser.py index 016b5f4..9893037 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,25 +1,46 @@ from pathlib import Path +from pdfmajor.parser.xrefdb import XRefDB +from pdfmajor.parser import iter_objects from pdfmajor.parser.objects.collections import PDFDictionary from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString -from typing import List +from typing import Any, List from decimal import Decimal from unittest import TestCase -from pdfmajor.streambuffer import StreamEOF +from pdfmajor.streambuffer import BufferStream from pdfmajor.parser.objects.indirect import IndirectObject from pdfmajor.parser.objects.base import PDFObject from pdfmajor.parser.objects.comment import PDFComment from pdfmajor.parser.objects.stream import PDFStream -from pdfmajor.parser.PDFParser import PDFParser CURRENT_FOLDER = Path(__file__).parent class Collections(TestCase): + def run_test(self, raw: bytes, expected: Any): + size = len(raw) + for buf_size in [ + 2, + 3, + size // 4, + size // 3, + size // 2, + 3 * size // 4, + size, + 2 * size, + 4 * size, + ]: + buffer = BufferStream.from_bytes(raw, buffer_size=buf_size) + it = iter_objects(buffer, XRefDB()) + obj = next(it) + self.assertEqual(obj.to_python(), expected) + with self.assertRaises(StopIteration): + next(it) + def test_parse_dict(self): - parser = PDFParser.from_bytes( - br"""<< /Type /Example + self.run_test( + raw=br"""<< /Type /Example /Subtype /DictionaryExample /Version 0.01 /IntegerItem 12 @@ -30,12 +51,7 @@ class Collections(TestCase): /LastItem ( not! ) /VeryLastItem ( OK ) >> >>""", - strict=True, - ) - obj = next(parser.iter_objects()) - self.assertDictEqual( - obj.to_python(), - { + expected={ "Type": "/Example", "Subtype": "/DictionaryExample", "Version": Decimal("0.01"), @@ -51,28 +67,18 @@ class Collections(TestCase): ) def test_parse_array(self): - parser = PDFParser.from_bytes( - br"""[ 549 3.14 false ( Ralph ) /SomeName ]""", - strict=True, - ) - obj = next(parser.iter_objects()) - self.assertListEqual( - obj.to_python(), - [549, Decimal("3.14"), False, " Ralph ", "/SomeName"], + self.run_test( + raw=br"""[ 549 3.14 false ( Ralph ) /SomeName ]""", + expected=[549, Decimal("3.14"), False, " Ralph ", "/SomeName"], ) def test_parse_nested_array(self): - parser = PDFParser.from_bytes( - br"""[ [10 30] [22 33] [ + self.run_test( + raw=br"""[ [10 30] [22 33] [ << /x 0 /y 5 >> << /x 10 /y 5 >> ] ]""", - strict=True, - ) - obj = next(parser.iter_objects()) - self.assertListEqual( - obj.to_python(), - [ + expected=[ [10, 30], [22, 33], [ @@ -84,9 +90,32 @@ class Collections(TestCase): class IndirectObjects(TestCase): + def run_test(self, raw: bytes, expected: List[PDFObject]) -> XRefDB: + db: XRefDB = XRefDB() + size = len(raw) + for buf_size in [ + 2, + 3, + size // 4, + size // 3, + size // 2, + 3 * size // 4, + size, + 2 * size, + 4 * size, + ]: + db = XRefDB() + buffer = BufferStream.from_bytes(raw, buffer_size=buf_size) + it = iter_objects(buffer, db) + for obj, eobj in zip(it, expected): + self.assertEqual(obj, eobj) + with self.assertRaises((StopIteration, EOFError)): + next(it) + return db + def test_parse_simple_indobj(self): - parser = PDFParser.from_bytes( - br""" + self.run_test( + raw=br""" 12 1 obj (Bring) endobj 100 0 obj <33> @@ -95,66 +124,43 @@ class IndirectObjects(TestCase): obj << /x 1 /y 3>> endobj """, - strict=True, - ) - expected = [ - IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)), - IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)), - IndirectObject( - 13, - 8, - 144, - PDFDictionary.from_dict( - {"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)} + expected=[ + IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)), + IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)), + IndirectObject( + 13, + 8, + 144, + PDFDictionary.from_dict( + {"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)} + ), ), - ), - ] - for obj, eobj in zip(parser.iter_objects(), expected): - self.assertIsInstance(obj, IndirectObject) - self.assertEqual(obj, eobj) - with self.assertRaises(StreamEOF): - next(parser.iter_objects()) + ], + ) def test_parse_obj_with_references(self): - parser = PDFParser.from_bytes( - br""" + db = self.run_test( + raw=br""" 12 1 obj (Bring) endobj 10 2 obj << /lastone 12 1 R >> endobj """, - strict=True, + expected=[ + IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)), + IndirectObject( + 0, + 2, + 54, + data=PDFDictionary.from_dict({"lastone": PDFString("Bring", 0, 0)}), + ), + ], ) - expected = [ - "Bring", - {"lastone": "Bring"}, - ] - for obj, exp in zip(parser.iter_objects(), expected): - self.assertEqual(obj.to_python(), exp) - with self.assertRaises(StreamEOF): - next(parser.iter_objects()) - sec_obj = parser.db.get_object(10, 2) + sec_obj = db.objs[(10, 2)] self.assertEqual( sec_obj.to_python(), {"lastone": "Bring"}, ) - def run_test( - self, - raw: bytes, - expected: List[PDFObject], - ): - parser = PDFParser.from_bytes(raw, strict=True) - for i, (eobj, obj) in enumerate( - zip(expected, parser.iter_objects()), - 1, - ): - self.assertEqual(obj, eobj, f"Failed at example #{i}") - try: - next_obj = next(parser.iter_objects()) - self.assertIsNone(next_obj) - except (EOFError, StopIteration) as e: - self.assertIsNotNone(e) - def test_parse_comments(self): self.run_test( br""" @@ -175,7 +181,7 @@ class IndirectObjects(TestCase): ) def test_parse_stream(self): - parser = PDFParser.from_bytes( + buffer = BufferStream.from_bytes( br""" 5 0 obj << /Type /XRef /Length 124 /Filter /FlateDecode >> @@ -185,7 +191,7 @@ stream endobj """ ) - for obj in parser.iter_objects(): + for obj in iter_objects(buffer, XRefDB()): self.assertIsInstance(obj, IndirectObject) self.assertIsInstance(obj.get_object(), PDFStream) self.assertDictEqual( diff --git a/tests/test_xref.py b/tests/test_xref.py index e775381..a070efb 100644 --- a/tests/test_xref.py +++ b/tests/test_xref.py @@ -1,4 +1,6 @@ from pathlib import Path +from pdfmajor.parser.xrefdb import XRefDB +from pdfmajor.parser import _save_or_get_object from unittest import TestCase from pdfmajor.streambuffer import BufferStream @@ -11,5 +13,7 @@ class Standard(TestCase): def test_char_doc(self): with (CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf").open("rb") as fp: buffer = BufferStream(fp) - for xref in iter_over_xref(buffer): - print(xref) + db = XRefDB() + db.update_from_xrefiter(iter_over_xref(buffer)) + for obj, gen in db.xrefs.keys(): + print(_save_or_get_object(buffer, db, obj, gen))