diff --git a/pdfmajor/lexer/__init__.py b/pdfmajor/lexer/__init__.py index 45b0827..19e4b1c 100644 --- a/pdfmajor/lexer/__init__.py +++ b/pdfmajor/lexer/__init__.py @@ -70,7 +70,7 @@ class PDFLexer: yield parse_double_angled_bracket(self.buffer) elif next_char == b">": self.buffer.seek(bufpos + j + 1) - if buf[j : j + 2] == ">>": + if buf[j : j + 2] == b">>": self.buffer.seekd(1) yield TokenDictionary( bufpos + j, bufpos + j + 2, TDictValue.CLOSE diff --git a/pdfmajor/lexer/keyword.py b/pdfmajor/lexer/keyword.py index 5c1ce3a..edd8a69 100644 --- a/pdfmajor/lexer/keyword.py +++ b/pdfmajor/lexer/keyword.py @@ -1,13 +1,11 @@ from pdfmajor.streambuffer import BufferStream from pdfmajor.lexer.exceptions import LexerEOF -from pdfmajor.lexer.token import TokenBoolean, TokenKeyword, TokenNull +from pdfmajor.lexer.token import Token, TokenBoolean, TokenKeyword, TokenNull from pdfmajor.lexer.regex import END_KEYWORD from typing import Optional, Union -def parse_keyword( - buffer: BufferStream, initialpos: Optional[int] = None -) -> Union[TokenKeyword, TokenBoolean]: +def parse_keyword(buffer: BufferStream, initialpos: Optional[int] = None) -> Token: """Parses input stream into a keyword or bool Args: @@ -15,7 +13,7 @@ def parse_keyword( initialpos (Optional[int]) Returns: - Union[TokenKeyword, TokenBoolean] + Token """ curtoken = b"" initialpos = buffer.tell() - 1 if initialpos is None else initialpos diff --git a/pdfmajor/lexer/token.py b/pdfmajor/lexer/token.py index 78a0b2f..a01fe48 100644 --- a/pdfmajor/lexer/token.py +++ b/pdfmajor/lexer/token.py @@ -36,6 +36,7 @@ class Token(Generic[T]): return False +# Primitive Tokens class TokenComment(Token[bytes]): """Token representing PDF comments Detection of this type of token can be found in PDF 1.7 spec section 7.2.3 @@ -109,8 +110,45 @@ class TokenReal(Token[Decimal]): TokenNumber = Union[TokenInteger, TokenReal] +TokenPrimitive = Union[ + TokenComment, + TokenKeyword, + TokenName, + TokenBoolean, + TokenNull, + TokenString, + TokenHexString, + TokenInteger, + TokenReal, +] +def is_primitive(token: Token) -> bool: + """determins is a token represents a primitive pdf object such as boolean, null, integer etc + + Args: + token (Token) + + Returns: + bool: True if primitive + """ + for opt in [ + TokenComment, + TokenKeyword, + TokenName, + TokenBoolean, + TokenNull, + TokenString, + TokenHexString, + TokenInteger, + TokenReal, + ]: + if isinstance(token, opt): + return True + return False + + +# Tokens for complex objects class TDictValue(Enum): OPEN = "<<" CLOSE = ">>" diff --git a/pdfmajor/parser_v2/__init__.py b/pdfmajor/parser_v2/__init__.py new file mode 100644 index 0000000..3e808dd --- /dev/null +++ b/pdfmajor/parser_v2/__init__.py @@ -0,0 +1,99 @@ +import io +from pdfmajor.parser_v2.exceptions import ParserError +from pdfmajor.parser_v2.objects import PDFArray, PDFDictionary, PDFPrimivite, PDFObject +from typing import Iterator, List, Optional +from pdfmajor.lexer.token import ( + TArrayValue, + TDictValue, + TokenArray, + TokenDictionary, + TokenPrimitive, + is_primitive, +) +from pdfmajor.lexer import PDFLexer +from pdfmajor.parser_v2.context import PDFParseContext + + +class PDFParser: + def __init__( + self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True + ) -> None: + self.lexer = PDFLexer(fp, buffer_size) + self.strict = strict + self.context_stack: List[PDFParseContext] = [] + + @property + def current_context(self) -> Optional[PDFParseContext]: + """Returns the current context of the parsing state + + Returns: + Optional[PDFParseContext] + """ + if len(self.context_stack) > 0: + return self.context_stack[-1] + else: + return None + + def seek(self, offset: int) -> int: + """moves the offset of the reader to a certain spot, and returns the new offset + + Args: + offset (int) + + Returns: + int: new offset + """ + return self.lexer.seek(offset) + + def iter_objects(self) -> Iterator[PDFObject]: + """Iterates over the objects in the current stream + + Yields: + [type]: [description] + """ + for token in self.lexer.iter_tokens(): + if is_primitive(token): + cur_ctx = self.current_context + obj = PDFPrimivite(token) # type: ignore + if cur_ctx is None: + yield obj + else: + cur_ctx.push(obj) + elif isinstance(token, TokenArray): + if token.value == TArrayValue.OPEN: + self.context_stack.append(PDFParseContext(PDFArray())) + elif len(self.context_stack) > 0: + last_ctx = self.context_stack.pop() + if isinstance(last_ctx.pdfobject, PDFArray): + cur_ctx = self.current_context + if cur_ctx is None: + yield last_ctx.pdfobject + else: + cur_ctx.push(last_ctx.pdfobject) + elif self.strict: + raise ParserError(f"Invalid end of array specified at {token}") + elif self.strict: + raise ParserError( + f"Invalid end of array specified at {token} with no start" + ) + elif isinstance(token, TokenDictionary): + if token.value == TDictValue.OPEN: + self.context_stack.append(PDFParseContext(PDFDictionary())) + elif len(self.context_stack) > 0: + last_ctx = self.context_stack.pop() + if isinstance(last_ctx.pdfobject, PDFDictionary): + cur_ctx = self.current_context + if cur_ctx is None: + yield last_ctx.pdfobject + else: + cur_ctx.push(last_ctx.pdfobject) + elif self.strict: + raise ParserError( + f"Invalid end of dictionary specified at {token}" + ) + elif self.strict: + raise ParserError( + f"Invalid end of dictionary specified at {token} with no start" + ) + else: + raise ParserError(f"Invalid token provided {token}") diff --git a/pdfmajor/parser_v2/context.py b/pdfmajor/parser_v2/context.py new file mode 100644 index 0000000..b346984 --- /dev/null +++ b/pdfmajor/parser_v2/context.py @@ -0,0 +1,41 @@ +from pdfmajor.parser_v2.exceptions import ParserError +from pdfmajor.parser_v2.objects import ( + PDFArray, + PDFDictionary, + PDFObject, + PDFComplexType, + PDFStream, +) + + +class PDFParseContext: + """A parsing context + This class helps constructs complex PDF types + """ + + __slots__ = ["pdfobject"] + + def __init__(self, pdfobject: PDFComplexType) -> None: + """ + Args: + pdfobject (PDFComplexType): the complex pdf object we are trying to build + """ + self.pdfobject = pdfobject + + def push(self, item: PDFObject): + """push any pdf object to the context, process according to the type of complex object we are trying to build + + Args: + item (PDFObject) + + Raises: + ParserError: raises if an invalid complex object is used + """ + if isinstance(self.pdfobject, PDFArray): + self.pdfobject.append(item) + elif isinstance(self.pdfobject, PDFDictionary): + self.pdfobject.pass_item(item) + elif isinstance(self.pdfobject, PDFStream): + raise NotImplementedError + else: + raise ParserError(f"Invalid Context {type(self.pdfobject)}") diff --git a/pdfmajor/parser_v2/exceptions.py b/pdfmajor/parser_v2/exceptions.py new file mode 100644 index 0000000..2456cde --- /dev/null +++ b/pdfmajor/parser_v2/exceptions.py @@ -0,0 +1,5 @@ +from pdfmajor.execptions import PDFMajorException + + +class ParserError(PDFMajorException): + pass diff --git a/pdfmajor/parser_v2/objects.py b/pdfmajor/parser_v2/objects.py new file mode 100644 index 0000000..14235c9 --- /dev/null +++ b/pdfmajor/parser_v2/objects.py @@ -0,0 +1,58 @@ +# (Boolean values, Integer and Real numbers, Strings, Names), Arrays, Dictionaries, Streams, and the null object + + +from pdfmajor.execptions import ParserError +from pdfmajor.lexer.token import TokenName, TokenPrimitive +from typing import Dict, Generic, List, NamedTuple, Optional, TypeVar, Union + +TToken = TypeVar("TToken", bound=TokenPrimitive) + + +class PDFName(NamedTuple): + value: str + + +class PDFPrimivite(Generic[TToken]): + def __init__(self, token: TToken) -> None: + self.token: TToken = token + + @property + def value(self): + if isinstance(self.token, TokenName): + return PDFName(self.token.value) + return self.token.value + + +class PDFDictionary(Dict[PDFPrimivite[TokenName], "PDFObject"]): + def __init__(self, strict: bool = False) -> None: + super().__init__() + self.name: Optional[str] = None + self.strict = strict + + def pass_item(self, item: "PDFObject"): + if self.name is None: + if isinstance(item, PDFPrimivite) and isinstance(item.token, TokenName): + self.name = item.value + if self.strict: + raise ParserError("Invalid token provided to dictionary") + else: + self[self.name] = item.value + self.name = None + + @property + def value(self): + return dict(self) + + +class PDFArray(List["PDFObject"]): + @property + def value(self): + return self + + +class PDFStream(object): + pass + + +PDFComplexType = Union[PDFDictionary, PDFArray, PDFStream] +PDFObject = Union[PDFComplexType, PDFPrimivite] diff --git a/pdfmajor/streambuffer.py b/pdfmajor/streambuffer.py index 74fc634..1c5127d 100644 --- a/pdfmajor/streambuffer.py +++ b/pdfmajor/streambuffer.py @@ -72,7 +72,7 @@ class BufferStream: def __next__(self): bbyte = self.read(self.buffer_size) if len(bbyte.data) == 0: - raise StopIteration + raise StopIteration(f"no data {bbyte}") return bbyte def close(self): diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..15500a1 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,41 @@ +from decimal import Decimal +import io +from pdfmajor.parser_v2.objects import PDFName +from pdfmajor.parser_v2 import PDFParser +from unittest import TestCase + + +class Basic(TestCase): + def test_parse_dict(self): + parser = PDFParser( + io.BytesIO( + br"""<< /Type /Example + /Subtype /DictionaryExample + /Version 0.01 + /IntegerItem 12 + /StringItem ( a string ) + /Subdictionary << + /Item1 0.4 + /Item2 true + /LastItem ( not! ) + /VeryLastItem ( OK ) >> + >>""" + ) + ) + obj = next(parser.iter_objects()) + self.assertDictEqual( + obj, + { + PDFName("Type"): PDFName("Example"), + PDFName("Subtype"): PDFName("DictionaryExample"), + PDFName("Version"): Decimal("0.01"), + PDFName("IntegerItem"): 12, + PDFName("StringItem"): " a string ", + PDFName("Subdictionary"): { + PDFName("Item1"): Decimal("0.4"), + PDFName("Item2"): True, + PDFName("LastItem"): " not! ", + PDFName("VeryLastItem"): " OK ", + }, + }, + )