From 43ed8b4019653153864309b78753e927afff53f4 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sun, 18 Oct 2020 11:48:19 -0400 Subject: [PATCH] working prototype for l2 --- pdfmajor/lexer/token.py | 6 -- pdfmajor/parser_v2/indirect_objects.py | 88 +++++++++++++++++ pdfmajor/parser_v2/l1.py | 129 +++++++++++++++---------- pdfmajor/parser_v2/l2.py | 102 ++++++++++--------- pdfmajor/parser_v2/objects.py | 59 +++++++++-- tests/test_parser.py | 31 ++++++ 6 files changed, 304 insertions(+), 111 deletions(-) create mode 100644 pdfmajor/parser_v2/indirect_objects.py diff --git a/pdfmajor/lexer/token.py b/pdfmajor/lexer/token.py index 17106ae..b8db0bb 100644 --- a/pdfmajor/lexer/token.py +++ b/pdfmajor/lexer/token.py @@ -116,11 +116,8 @@ class TokenReal(Token[Decimal]): TokenNumber = Union[TokenInteger, TokenReal] TokenPrimitive = Union[ - TokenComment, - TokenKeyword, TokenName, TokenBoolean, - TokenNull, TokenString, TokenHexString, TokenInteger, @@ -138,11 +135,8 @@ def is_primitive(token: Token) -> bool: bool: True if primitive """ for opt in [ - TokenComment, - TokenKeyword, TokenName, TokenBoolean, - TokenNull, TokenString, TokenHexString, TokenInteger, diff --git a/pdfmajor/parser_v2/indirect_objects.py b/pdfmajor/parser_v2/indirect_objects.py new file mode 100644 index 0000000..87f2892 --- /dev/null +++ b/pdfmajor/parser_v2/indirect_objects.py @@ -0,0 +1,88 @@ +from pdfmajor.parser_v2.exceptions import ParserError +from pdfmajor.lexer.token import TokenNull +from typing import Dict, Tuple +from .objects import PDFContextualObject, PDFNull, PDFObject, PDFPrimitive + + +class IndirectObjectCollection: + """A class for collection references to objects""" + + def __init__(self) -> None: + self.objs: Dict[Tuple[int, int], PDFObject] = {} + + def create_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject": + """Create an indirect object and store its value in the collection + return the reference to where this object lies at + + Returns: + IndirectObject: + """ + return IndirectObject(self, obj_num, gen_num, read_only=False) + + def get_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject": + """creates a reference to an object + + Returns: + IndirectObject + """ + return IndirectObject(self, obj_num, gen_num, read_only=True) + + def get_object(self, obj_num: int, gen_num: int) -> PDFObject: + """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10 + + Args: + obj_num (int): object number + gen_num (int): generation number + + Returns: + PDFObject + """ + return self.objs.get((obj_num, gen_num), PDFNull()) + + +class IndirectObject(PDFContextualObject): + """A class representing a reference for other objects""" + + def __init__( + self, + collection: IndirectObjectCollection, + obj_num: int, + gen_num: int, + read_only: bool = True, + ) -> None: + self.obj_num = obj_num + self.gen_num = gen_num + self.__read_only = read_only + self.__col = collection + + def __repr__(self) -> str: + return f"IndirectObject({self.obj_num}, {self.gen_num})" + + def get_object(self) -> PDFObject: + return self.__col.get_object(self.obj_num, self.gen_num) + + def save_object(self, obj: PDFObject): + if self.__read_only: + raise ParserError( + f"attempted to override a read-only reference of an object {obj}" + ) + self.__col.objs[(self.obj_num, self.gen_num)] = obj + + def get_value(self): + return IndirectObject(self.__col, self.obj_num, self.gen_num, True) + + def to_python(self): + return self.get_object().to_python() + + def pass_item(self, item: PDFObject): + if self.__read_only: + raise ParserError( + f"attempted to add to an read-only reference of an object {item}" + ) + cur_obj = self.get_object() + if isinstance(cur_obj, PDFContextualObject): + cur_obj.pass_item(item) + elif isinstance(cur_obj, PDFNull): + self.save_object(item) + else: + raise ParserError(f"Invalid redefinition of an object {item}") diff --git a/pdfmajor/parser_v2/l1.py b/pdfmajor/parser_v2/l1.py index 8da1191..f74ef8f 100644 --- a/pdfmajor/parser_v2/l1.py +++ b/pdfmajor/parser_v2/l1.py @@ -2,24 +2,80 @@ import io from pdfmajor.parser_v2.exceptions import ParserError from pdfmajor.parser_v2.objects import ( PDFArray, + PDFComment, PDFContextualObject, PDFDictionary, - PDFPrimivite, + PDFKeyword, + PDFNull, + PDFPrimitive, PDFObject, ) -from typing import Any, Iterator, List, Optional +from typing import Any, Iterator, List, Optional, Type from pdfmajor.lexer.token import ( TArrayValue, TDictValue, TokenArray, + TokenComment, TokenDictionary, TokenComplexType, TokenComplexTypeVal, + TokenKeyword, + TokenNull, is_primitive, ) from pdfmajor.lexer import PDFLexer +def _deal_with_obj( + context_stack: List[PDFContextualObject], + token: TokenComplexType, + cls_const: Type[PDFContextualObject], + open_t: TokenComplexTypeVal, + strict: bool = False, +) -> Optional[PDFObject]: + """generic method for processing things that have a start and end token + + Args: + token (TokenComplexType) + cls_const (Any): the class constructor for the object + open_t (TokenComplexTypeVal): the value represnting 'open' for the object + + Returns: + Optional[PDFObject] + """ + if token.value == open_t: + context_stack.append(cls_const()) + elif len(context_stack) > 0: + last_ctx = context_stack.pop() + if isinstance(last_ctx, cls_const): + cur_ctx = _get_curr_context(context_stack) + if cur_ctx is None: + return last_ctx + else: + cur_ctx.pass_item(last_ctx) + elif strict: + raise ParserError(f"Invalid end of {cls_const} specified at {token}") + elif strict: + raise ParserError( + f"Invalid end of {cls_const} specified at {token} with no start" + ) + return None + + +def _get_curr_context( + context_stack: List[PDFContextualObject], +) -> Optional[PDFContextualObject]: + """Returns the current context of the parsing state + + Returns: + Optional[PDFContextualObject] + """ + if len(context_stack) > 0: + return context_stack[-1] + else: + return None + + class PDFL1Parser: """Level 1 PDF Parser""" @@ -28,19 +84,6 @@ class PDFL1Parser: ) -> None: self.lexer = PDFLexer(fp, buffer_size) self.strict = strict - self.context_stack: List[PDFContextualObject] = [] - - @property - def current_context(self) -> Optional[PDFContextualObject]: - """Returns the current context of the parsing state - - Returns: - Optional[PDFContextualObject] - """ - if len(self.context_stack) > 0: - return self.context_stack[-1] - else: - return None def seek(self, offset: int) -> int: """moves the offset of the reader to a certain spot, and returns the new offset @@ -59,54 +102,36 @@ class PDFL1Parser: Yields: PDFObject """ + context_stack: List[PDFContextualObject] = [] for token in self.lexer.iter_tokens(): if is_primitive(token): - cur_ctx = self.current_context - obj = PDFPrimivite(token) # type: ignore + cur_ctx = _get_curr_context(context_stack) + obj = PDFPrimitive(token) # type: ignore if cur_ctx is None: yield obj else: cur_ctx.pass_item(obj) + elif isinstance(token, TokenComment): + yield PDFComment(token) + elif isinstance(token, TokenKeyword): + yield PDFKeyword(token) + elif isinstance(token, TokenNull): + yield PDFNull() elif isinstance(token, TokenArray): - obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN) + obj = _deal_with_obj( + context_stack, token, PDFArray, TArrayValue.OPEN, strict=self.strict + ) if obj is not None: yield obj elif isinstance(token, TokenDictionary): - obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN) + obj = _deal_with_obj( + context_stack=context_stack, + token=token, + cls_const=PDFDictionary, + open_t=TDictValue.OPEN, + strict=self.strict, + ) if obj is not None: yield obj else: raise ParserError(f"Invalid token provided {token}") - - def __deal_with_obj( - self, - token: TokenComplexType, - cls_const: Any, - open_t: TokenComplexTypeVal, - ) -> Optional[PDFObject]: - """generic method for processing things that have a start and end token - - Args: - token (TokenComplexType) - cls_const (Any): the class constructor for the object - open_t (TokenComplexTypeVal): the value represnting 'open' for the object - - Returns: - Optional[PDFObject] - """ - if token.value == open_t: - self.context_stack.append(cls_const()) - elif len(self.context_stack) > 0: - last_ctx = self.context_stack.pop() - if isinstance(last_ctx, cls_const): - cur_ctx = self.current_context - if cur_ctx is None: - return last_ctx - else: - cur_ctx.pass_item(last_ctx) - elif self.strict: - raise ParserError(f"Invalid end of {cls_const} specified at {token}") - elif self.strict: - raise ParserError( - f"Invalid end of {cls_const} specified at {token} with no start" - ) diff --git a/pdfmajor/parser_v2/l2.py b/pdfmajor/parser_v2/l2.py index 9318213..9f5c8ab 100644 --- a/pdfmajor/parser_v2/l2.py +++ b/pdfmajor/parser_v2/l2.py @@ -1,46 +1,28 @@ import io +from pdfmajor.parser_v2.l1 import PDFL1Parser from pdfmajor.parser_v2.exceptions import ParserError from pdfmajor.parser_v2.objects import ( - PDFArray, - PDFDictionary, - PDFPrimivite, + PDFContextualObject, + PDFKeyword, + PDFPrimitive, PDFObject, ) -from typing import Any, Iterator, List, Optional +from typing import Iterator, List, Optional, Tuple from pdfmajor.lexer.token import ( - TArrayValue, - TDictValue, - TokenArray, - TokenDictionary, - TokenComplexType, - TokenComplexTypeVal, - is_primitive, + TokenInteger, ) -from pdfmajor.lexer import PDFLexer -from pdfmajor.parser_v2.context import PDFParseContext +from pdfmajor.parser_v2.indirect_objects import IndirectObjectCollection -class PDFL1Parser: - """Level 1 PDF Parser""" +class PDFL2Parser: + """Level 2 PDF Parser""" def __init__( self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True ) -> None: - self.lexer = PDFLexer(fp, buffer_size) + self.l1 = PDFL1Parser(fp, buffer_size, strict) self.strict = strict - self.context_stack: List[PDFParseContext] = [] - - @property - def current_context(self) -> Optional[PDFParseContext]: - """Returns the current context of the parsing state - - Returns: - Optional[PDFParseContext] - """ - if len(self.context_stack) > 0: - return self.context_stack[-1] - else: - return None + self.inobjects = IndirectObjectCollection() def seek(self, offset: int) -> int: """moves the offset of the reader to a certain spot, and returns the new offset @@ -51,7 +33,7 @@ class PDFL1Parser: Returns: int: new offset """ - return self.lexer.seek(offset) + return self.l1.seek(offset) def iter_objects(self) -> Iterator[PDFObject]: """Iterates over the objects in the current byte stream using level 2 parsing @@ -59,21 +41,47 @@ class PDFL1Parser: Yields: PDFObject """ - for token in self.lexer.iter_tokens(): - if is_primitive(token): - cur_ctx = self.current_context - obj = PDFPrimivite(token) # type: ignore - if cur_ctx is None: - yield obj - else: - cur_ctx.push(obj) - elif isinstance(token, TokenArray): - obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN) - if obj is not None: - yield obj - elif isinstance(token, TokenDictionary): - obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN) - if obj is not None: - yield obj + object_list: List[PDFObject] = [] + ctx_list: List[PDFContextualObject] = [] + for obj in self.l1.iter_objects(): + if isinstance(obj, PDFKeyword): + if obj.get_value() == b"obj": + if len(object_list) < 2: + raise ParserError("recieved 'obj' but missing leading tokens") + ctx_list.append( + self.inobjects.create_indobject( + *_get_indobj_values(object_list) + ) + ) + elif obj.get_value() == b"endobj": + if len(ctx_list) > 0: + last_ctx_obj = ctx_list.pop() + if len(ctx_list) > 0: + ctx_list[-1].pass_item(last_ctx_obj) + else: + yield last_ctx_obj.get_value() + elif obj.get_value() == b"R": + cur_indobj = self.inobjects.get_indobject( + *_get_indobj_values(object_list) + ) + if len(ctx_list) > 0: + ctx_list[-1].pass_item(cur_indobj) + else: + yield cur_indobj.get_value() + elif len(ctx_list) == 0: + object_list.append(obj) + elif len(ctx_list) > 0: + ctx_list[-1].pass_item(obj) else: - raise ParserError(f"Invalid token provided {token}") + raise ParserError(f"Invalid obj provided {obj}") + + +def _get_indobj_values(object_list: List[PDFObject]) -> Tuple[int, int]: + gen_num = object_list.pop() + obj_num = object_list.pop() + if isinstance(gen_num, PDFPrimitive) and isinstance(obj_num, PDFPrimitive): + if isinstance(gen_num.token, TokenInteger) and isinstance( + obj_num.token, TokenInteger + ): + return (obj_num.to_python(), gen_num.to_python()) + raise ParserError("missing leading tokens for object reference") diff --git a/pdfmajor/parser_v2/objects.py b/pdfmajor/parser_v2/objects.py index 7d4bbbe..3a81f20 100644 --- a/pdfmajor/parser_v2/objects.py +++ b/pdfmajor/parser_v2/objects.py @@ -3,8 +3,14 @@ from abc import ABCMeta, abstractmethod from pdfmajor.execptions import ParserError -from pdfmajor.lexer.token import PDFName, TokenName, TokenPrimitive -from typing import Any, Dict, Generic, List, NamedTuple, Optional, TypeVar, Union +from pdfmajor.lexer.token import ( + PDFName, + TokenComment, + TokenKeyword, + TokenName, + TokenPrimitive, +) +from typing import Any, Dict, Generic, List, Optional, TypeVar TToken = TypeVar("TToken", bound=TokenPrimitive) @@ -22,8 +28,13 @@ class PDFObject(metaclass=ABCMeta): """returns the python variation of this object""" raise NotImplementedError + def __eq__(self, o: object) -> bool: + if not isinstance(o, self.__class__): + return False + return o.to_python() == self.to_python() -class PDFContextualObject(PDFObject, metaclass=ABCMeta): + +class PDFContextualObject(PDFObject): """An abstract class representing objects that require context for construction""" @abstractmethod @@ -31,7 +42,7 @@ class PDFContextualObject(PDFObject, metaclass=ABCMeta): raise NotImplementedError -class PDFPrimivite(Generic[TToken], PDFObject): +class PDFPrimitive(Generic[TToken], PDFObject): """A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3""" def __init__(self, token: TToken) -> None: @@ -41,7 +52,43 @@ class PDFPrimivite(Generic[TToken], PDFObject): return self.token.value def to_python(self): - return self.value + return self.get_value() + + +class PDFKeyword(PDFObject): + """A class representing a PDF keyword""" + + def __init__(self, token: TokenKeyword) -> None: + self.token = token + + def get_value(self): + return self.token.value + + def to_python(self): + return self.get_value() + + +class PDFComment(PDFObject): + """A class representing a PDF comment""" + + def __init__(self, token: TokenComment) -> None: + self.token = token + + def get_value(self): + return self.token.value + + def to_python(self): + return self.get_value() + + +class PDFNull(PDFObject): + """A class representing a PDF null object""" + + def get_value(self): + return None + + def to_python(self): + return None class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject): @@ -54,7 +101,7 @@ class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject): def pass_item(self, item: PDFObject): if self.name is None: - if isinstance(item, PDFPrimivite) and isinstance(item.token, TokenName): + if isinstance(item, PDFPrimitive) and isinstance(item.token, TokenName): self.name = item.get_value() if self.strict: raise ParserError("Invalid token provided to dictionary") diff --git a/tests/test_parser.py b/tests/test_parser.py index 7b5fe69..e5375b7 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,7 +1,10 @@ from decimal import Decimal import io +from pdfmajor.parser_v2.indirect_objects import IndirectObject +from pdfmajor.parser_v2.l2 import PDFL2Parser from pdfmajor.parser_v2.objects import PDFName from pdfmajor.parser_v2.l1 import PDFL1Parser +from pdfmajor.streambuffer import StreamEOF from unittest import TestCase @@ -70,3 +73,31 @@ class L1(TestCase): ], ], ) + + +class L2(TestCase): + def test_parse_simple_indobj(self): + parser = PDFL2Parser( + io.BytesIO( + br""" + 12 1 obj (Bring) endobj + 100 0 obj + <33> + endobj + 13 8 + obj + << /x 1 /y 3>> endobj + """ + ) + ) + expected = [ + (12, 1, "Bring"), + (100, 0, b"3"), + (13, 8, {PDFName("x"): 1, PDFName("y"): 3}), + ] + for obj, eobj in zip(parser.iter_objects(), expected): + self.assertIsInstance(obj, IndirectObject) + self.assertEqual(obj, parser.inobjects.get_indobject(eobj[0], eobj[1])) + self.assertEqual(obj.to_python(), eobj[2]) + with self.assertRaises(StreamEOF): + next(parser.iter_objects())