diff --git a/pdfmajor/parser/__init__.py b/pdfmajor/parser/__init__.py index 8009628..9a11df8 100644 --- a/pdfmajor/parser/__init__.py +++ b/pdfmajor/parser/__init__.py @@ -1,7 +1,7 @@ -from pdfmajor.parser.objects.primitives import PDFInteger -from pdfmajor.parser.stream.PDFStream import PDFStream +from pdfmajor.parser.objects.primitives import get_obj_from_token_primitive from typing import Iterator, Optional -from pdfmajor.streambuffer import BufferStream + +from pdfmajor.lexer import iter_tokens from pdfmajor.lexer.token import ( TArrayValue, TDictValue, @@ -10,20 +10,20 @@ from pdfmajor.lexer.token import ( TokenDictionary, TokenKeyword, ) -from pdfmajor.lexer import iter_tokens -from .objects.indirect import IndirectObject, ObjectRef +from pdfmajor.streambuffer import BufferStream + +from .exceptions import InvalidKeywordPos from .objects.base import PDFObject from .objects.collections import PDFArray, PDFDictionary from .objects.comment import PDFComment -from .state import ParsingState -from .parsers import attempt_parse_prim, deal_with_collection_object -from .exceptions import ( - EarlyStop, - InvalidKeywordPos, - BrokenFile, - InvalidKeywordPos, - ParserError, +from .parsers import ( + deal_with_collection_object, + on_endobj, + on_indirect_ref_close, + on_primitive, + on_stream, ) +from .state import ParsingState def get_first_object( @@ -44,9 +44,9 @@ def iter_objects( """ state = ParsingState([], []) if state is None else state for token in iter_tokens(buffer): - attempt_prim = attempt_parse_prim(token, state) - if attempt_prim.success: - yield from state.feed_or_yield_objs(attempt_prim.next_objs) + prim_obj = get_obj_from_token_primitive(token) + if prim_obj is not None: + yield from on_primitive(prim_obj, state) elif isinstance(token, TokenComment): yield state.set_last_obj( PDFComment( @@ -56,14 +56,14 @@ def iter_objects( ) elif isinstance(token, TokenKeyword): if token.value == b"R": - yield from _on_indirect_ref_close(state) + yield from on_indirect_ref_close(state) elif token.value == b"obj": state.initialize_indirect_obj(token.start_loc) elif token.value == b"endobj": - obj = _on_endobj(state, token) + obj = on_endobj(state, token) yield obj elif token.value == b"stream": - yield from _on_stream( + yield from on_stream( buffer, state, token, @@ -83,147 +83,3 @@ def iter_objects( cls_const=PDFDictionary, open_t=TDictValue.OPEN, ) - - -def _on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject: - """finish off the current context, assuming we are handling an indirect-object - - Args: - token (TokenKeyword) - - Raises: - BrokenFile: if the object fif not get initilized with two integers - InvalidKeywordPos: If the context is not an indirect object - - Yields: - PDFObject - """ - last_ctx = state.context_stack.pop() - if isinstance(last_ctx, IndirectObject): - if len(state.int_collection) > 1: - raise BrokenFile(f"Invalid number of integers in obj defition {token}") - for num in state.flush_int_collections(): - last_ctx.pass_item(num) - return last_ctx - else: - raise ParserError(f"Recived an endobj while not procesing an indirect object") - - -def _on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword): - """Initilize a PDFStream - - Args: - token (TokenKeyword) - - Raises: - ParserError: if the current context is not an indirect object - """ - last_ctx = state.current_context - if last_ctx is not None and isinstance(last_ctx, IndirectObject): - obj = last_ctx.get_object() - if not isinstance(obj, PDFDictionary): - raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}") - last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj) - if isinstance(stream.length, PDFInteger): - buffer.seek(stream.offset + stream.length.to_python()) - next_token = None - for i, next_token in enumerate(iter_tokens(buffer)): - if ( - isinstance(next_token, TokenKeyword) - and next_token.value == b"endstream" - ): - if i > 0: - stream.length.value = next_token.start_loc - stream.offset - break - if not ( - isinstance(next_token, TokenKeyword) - and next_token.value == b"endstream" - ): - raise BrokenFile( - f"stream did not contain 'endstream', instead the token {next_token} was found" - ) - elif isinstance(stream.length, ObjectRef): - yield last_ctx - raise EarlyStop( - "cannot proceed parsing as the stream.length object is an indirect object" - ) - else: - raise ParserError(f"PDFStream is missing initilization dictionary") - - -def _on_indirect_ref_close(state: ParsingState): - obj_num, gen_num = state.get_indobj_values() - obj = ObjectRef(obj_num, gen_num) - cur_ctx = state.current_context - if cur_ctx is not None: - cur_ctx.pass_item(obj) - else: - yield obj - - -# def _save_or_get_object( -# buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int -# ) -> PDFObject: -# """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10 - -# Args: -# obj_num (int): object number -# gen_num (int): generation number - -# Returns: -# PDFObject -# """ -# indobj = xrefdb.objs.get((obj_num, gen_num), None) -# print("trying to read", obj_num, gen_num) -# if indobj is None: -# xref = xrefdb.xrefs.get((obj_num, gen_num), None) -# if xref is None: -# raise InvalidXref( -# f"xref not found for object retrieval {obj_num}, {gen_num}" -# ) -# return _read_object_from_xref(xref, xrefdb, buffer) -# else: -# return _read_object_from_db_or_buffer(indobj, xrefdb, buffer) - - -# def _read_object_from_xref( -# xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream -# ) -> PDFObject: -# with buffer.get_window(): -# buffer.seek(xref.offset) -# obj = next(iter_objects(buffer, xrefdb)) -# if ( -# not isinstance(obj, IndirectObject) -# or obj.gen_num != xref.gen_num -# or obj.obj_num != xref.obj_num -# ): -# raise ParserError( -# f"Invalid offset for indirect object, expected {xref} but got {obj}" -# ) -# return obj - - -# def _read_object_from_db_or_buffer( -# indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream -# ) -> PDFObject: -# if indobj.read: -# return indobj -# with buffer.get_window(): -# buffer.seek(indobj.offset) -# next_token = next(iter_tokens(buffer)) -# if next_token.value != b"obj": -# if isinstance(next_token, TokenKeyword): -# raise InvalidKeywordPos(next_token, [b"obj"]) -# else: -# raise ParserError(f"expected a keyword instead got {next_token}") -# obj = next( -# iter_objects( -# buffer, -# xrefdb, -# ParsingState([indobj], []), -# ) -# ) -# if obj != indobj: -# raise ParserError(f"got a different indobj {obj} != {indobj}") -# indobj.read = True -# return obj diff --git a/pdfmajor/parser/parsers.py b/pdfmajor/parser/parsers.py index 520010d..e9acd9c 100644 --- a/pdfmajor/parser/parsers.py +++ b/pdfmajor/parser/parsers.py @@ -1,45 +1,50 @@ from typing import Iterator, List, NamedTuple, Type -from pdfmajor.lexer.token import Token, TokenComplexType, TokenComplexTypeVal +from pdfmajor.lexer import iter_tokens +from pdfmajor.lexer.token import ( + Token, + TokenComplexType, + TokenComplexTypeVal, + TokenKeyword, +) +from pdfmajor.streambuffer import BufferStream -from .state import ParsingState -from .exceptions import ParserError -from .objects.base import PDFObject +from .exceptions import BrokenFile, EarlyStop, ParserError +from .stream.PDFStream import PDFStream from .objects.base import PDFContextualObject, PDFObject -from .objects.primitives import PDFInteger, get_obj_from_token_primitive +from .objects.collections import PDFDictionary +from .objects.indirect import IndirectObject, ObjectRef +from .objects.primitives import ( + PDFInteger, + PDFPrimitiveObject, + get_obj_from_token_primitive, +) +from .state import ParsingState -class ParsingAttempt(NamedTuple): - success: bool - next_objs: List[PDFObject] - - -def attempt_parse_prim(token: Token, state: ParsingState) -> ParsingAttempt: - """Attempt to parse out a primitive value from the token +def on_primitive(pobj: PDFPrimitiveObject, state: ParsingState) -> Iterator[PDFObject]: + """Parses out a primitive value from the token Args: - token (Token): [description] - state (ParsingState): [description] + pobj (PDFPrimitiveObject) + state (ParsingState) - Returns: - ParsingAttempt + Yields: + Iterator[PDFObject] """ - prim_obj = get_obj_from_token_primitive(token) - if prim_obj is not None: - out: List[PDFObject] = [] - state.set_last_obj(prim_obj) - if isinstance(prim_obj, PDFInteger): - if len(state.int_collection) <= 1: - state.int_collection.append(prim_obj) - else: - for obj in state.flush_int_collections(): - out.append(obj) + state.set_last_obj(pobj) + if isinstance(pobj, PDFInteger): + for obj in state.flush_int_collections(1): + yield obj + state.int_collection.append(pobj) + else: + for obj in state.flush_int_collections(): + yield obj + cur_ctx = state.current_context + if cur_ctx is not None: + cur_ctx.pass_item(pobj) else: - for obj in state.flush_int_collections(): - out.append(obj) - out.append(prim_obj) - return ParsingAttempt(True, out) - return ParsingAttempt(False, []) + yield pobj def deal_with_collection_object( @@ -75,3 +80,87 @@ def deal_with_collection_object( raise ParserError( f"Invalid end of {cls_const} specified at {token} with no start" ) + + +def on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject: + """finish off the current context, assuming we are handling an indirect-object + + Args: + token (TokenKeyword) + + Raises: + BrokenFile: if the object fif not get initilized with two integers + InvalidKeywordPos: If the context is not an indirect object + + Yields: + PDFObject + """ + last_ctx = state.context_stack.pop() + if isinstance(last_ctx, IndirectObject): + if len(state.int_collection) > 1: + raise BrokenFile(f"Invalid number of integers in obj defition {token}") + for num in state.flush_int_collections(): + last_ctx.pass_item(num) + return last_ctx + else: + raise ParserError(f"Recived an endobj while not procesing an indirect object") + + +def on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword): + """Initilize a PDFStream + + Args: + token (TokenKeyword) + + Raises: + ParserError: if the current context is not an indirect object + """ + last_ctx = state.current_context + if last_ctx is not None and isinstance(last_ctx, IndirectObject): + obj = last_ctx.get_object() + if not isinstance(obj, PDFDictionary): + raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}") + last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj) + if isinstance(stream.length, PDFInteger): + buffer.seek(stream.offset + stream.length.to_python()) + next_token = None + for i, next_token in enumerate(iter_tokens(buffer)): + if ( + isinstance(next_token, TokenKeyword) + and next_token.value == b"endstream" + ): + if i > 0: + stream.length.value = next_token.start_loc - stream.offset + state.health_report.write( + issue_name="InvalidPDFStreamLength", + additional_info=f""" + start_token={token} + end_token={next_token} + object_dictionary={obj} + """, + ) + break + if not ( + isinstance(next_token, TokenKeyword) + and next_token.value == b"endstream" + ): + raise BrokenFile( + f"stream did not contain 'endstream', instead the token {next_token} was found" + ) + elif isinstance(stream.length, ObjectRef): + yield last_ctx + raise EarlyStop( + "cannot proceed parsing as the stream.length object is an indirect object" + ) + else: + raise ParserError(f"PDFStream is missing initilization dictionary") + + +def on_indirect_ref_close(state: ParsingState): + obj_num, gen_num = state.get_indobj_values() + obj = ObjectRef(obj_num, gen_num) + cur_ctx = state.current_context + if cur_ctx is not None: + cur_ctx.pass_item(obj) + else: + yield obj diff --git a/pdfmajor/parser/state.py b/pdfmajor/parser/state.py index 7f9a50f..d16aa19 100644 --- a/pdfmajor/parser/state.py +++ b/pdfmajor/parser/state.py @@ -1,25 +1,31 @@ from typing import Iterator, List, Optional, Tuple, Union +from pdfmajor.healthlog import PDFHealthReport + from .exceptions import ParserError from .objects.base import PDFContextualObject, PDFObject -from .objects.primitives import PDFInteger from .objects.comment import PDFComment from .objects.indirect import IndirectObject +from .objects.primitives import PDFInteger class ParsingState: - __slots__ = ["context_stack", "int_collection", "last_obj"] + __slots__ = ["context_stack", "int_collection", "last_obj", "health_report"] def __init__( self, context_stack: List[PDFContextualObject], int_collection: List[Union[PDFInteger, int]], last_obj: Optional[PDFObject] = None, + health_report: Optional[PDFHealthReport] = None, ) -> None: self.context_stack = context_stack self.int_collection = int_collection self.last_obj = last_obj + self.health_report = ( + health_report if health_report is not None else PDFHealthReport() + ) def __repr__(self) -> str: return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj})".format( @@ -87,7 +93,7 @@ class ParsingState: IndirectObject(obj_num=obj_num, gen_num=gen_num, offset=offset) ) - def flush_int_collections(self) -> Iterator[PDFObject]: + def flush_int_collections(self, up_to: int = 0) -> Iterator[PDFObject]: """clear out the int-collection Yields: @@ -95,20 +101,21 @@ class ParsingState: """ cur_ctx = self.current_context if cur_ctx is not None: - for obj in self.int_collection: + while len(self.int_collection) > up_to: + obj = self.int_collection.pop(0) if not isinstance(obj, PDFInteger): raise ParserError( f"Invalid int-collection contains none PDFInteger types {obj}" ) cur_ctx.pass_item(obj) else: - for obj in self.int_collection: + while len(self.int_collection) > up_to: + obj = self.int_collection.pop(0) if not isinstance(obj, PDFInteger): raise ParserError( f"Invalid int-collection contains none PDFInteger types {obj}" ) yield obj - self.int_collection = [] def feed_or_yield_objs(self, next_objs: List[PDFObject]) -> Iterator[PDFObject]: """If there context, then feed the objects into it, otherwise yield them diff --git a/tests/test_parser.py b/tests/test_parser.py index be5e03b..89e3aa0 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,6 +1,6 @@ from pathlib import Path from pdfmajor.parser import iter_objects -from pdfmajor.parser.objects.collections import PDFDictionary +from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString from typing import Any, Callable, List, Optional @@ -117,6 +117,23 @@ class IndirectObjects(TestCase): with self.assertRaises((StopIteration, EOFError)): next(it) + def test_array_ofindirects(self): + self.run_test( + raw=br"""[0 1 3 R 2 3 23 4 R 31 2 R]""", + expected=[ + PDFArray.from_list( + [ + PDFInteger(0, 0, 0), + ObjectRef(1, 3), + PDFInteger(2, 0, 0), + PDFInteger(3, 0, 0), + ObjectRef(23, 4), + ObjectRef(31, 2), + ] + ) + ], + ) + def test_parse_simple_indobj(self): self.run_test( raw=br"""