From 3fec7a1747e073c2be86804373db71660bca8835 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sat, 17 Oct 2020 23:45:31 -0400 Subject: [PATCH] undoing lexing of obj and stream --- pdfmajor/lexer/keyword.py | 12 ------------ pdfmajor/lexer/token.py | 26 ++------------------------ pdfmajor/parser_v2/__init__.py | 12 +++++++++--- pdfmajor/parser_v2/objects.py | 13 ++++++++++++- tests/test_lexer.py | 12 ++++-------- 5 files changed, 27 insertions(+), 48 deletions(-) diff --git a/pdfmajor/lexer/keyword.py b/pdfmajor/lexer/keyword.py index 5ebc0b6..25f0f58 100644 --- a/pdfmajor/lexer/keyword.py +++ b/pdfmajor/lexer/keyword.py @@ -1,14 +1,10 @@ from pdfmajor.streambuffer import BufferStream from pdfmajor.lexer.exceptions import LexerEOF from pdfmajor.lexer.token import ( - TObjValue, - TStreamValue, Token, TokenBoolean, TokenKeyword, TokenNull, - TokenObj, - TokenStream, ) from pdfmajor.lexer.regex import END_KEYWORD from typing import Optional, Union @@ -40,14 +36,6 @@ def parse_keyword(buffer: BufferStream, initialpos: Optional[int] = None) -> Tok return TokenBoolean(initialpos, buffer.tell(), False) elif curtoken == b"null": return TokenNull(initialpos, buffer.tell(), None) - elif curtoken == b"stream": - return TokenStream(initialpos, buffer.tell(), TStreamValue.START) - elif curtoken == b"endstream": - return TokenStream(initialpos, buffer.tell(), TStreamValue.END) - elif curtoken == b"obj": - return TokenObj(initialpos, buffer.tell(), TObjValue.START) - elif curtoken == b"endobj": - return TokenObj(initialpos, buffer.tell(), TObjValue.END) else: return TokenKeyword(initialpos, buffer.tell(), curtoken) raise LexerEOF diff --git a/pdfmajor/lexer/token.py b/pdfmajor/lexer/token.py index 4cb29ab..17106ae 100644 --- a/pdfmajor/lexer/token.py +++ b/pdfmajor/lexer/token.py @@ -180,27 +180,5 @@ class TokenArray(Token[TArrayValue]): pass -class TStreamValue(Enum): - START = "stream" - END = "endstream" - - -class TokenStream(Token[TStreamValue]): - """Tokens representing the starting and ending of a PDF stream as specified in PDF spec 1.7 section 7.3.8.1""" - - pass - - -class TObjValue(Enum): - START = "obj" - END = "endobj" - - -class TokenObj(Token[TObjValue]): - """Tokens representing the starting and ending of an indirect PDF object as specified in PDF spec 1.7 section 7.3.10""" - - pass - - -TokenComplexType = Union[TokenDictionary, TokenArray, TokenStream, TokenObj] -TokenComplexTypeVal = Union[TDictValue, TArrayValue, TStreamValue, TObjValue] +TokenComplexType = Union[TokenDictionary, TokenArray] +TokenComplexTypeVal = Union[TDictValue, TArrayValue] diff --git a/pdfmajor/parser_v2/__init__.py b/pdfmajor/parser_v2/__init__.py index 09f08d6..00ec683 100644 --- a/pdfmajor/parser_v2/__init__.py +++ b/pdfmajor/parser_v2/__init__.py @@ -5,9 +5,8 @@ from pdfmajor.parser_v2.objects import ( PDFDictionary, PDFPrimivite, PDFObject, - PDFComplexType, ) -from typing import Any, Callable, Iterator, List, Optional +from typing import Any, Iterator, List, Optional from pdfmajor.lexer.token import ( TArrayValue, TDictValue, @@ -22,6 +21,13 @@ from pdfmajor.parser_v2.context import PDFParseContext class PDFParser: + """A PDF token parser + this parser parses the bytes in three steps + 1. Generate tokens from the byte stream using the PDFLexer + 2. Level 1 Parsing: Match Low level objects such as primitive types, arrays, dictionaries + 3. Level 2 Parsing: Match high level objects such as Streams and Indirect objects + """ + def __init__( self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True ) -> None: @@ -53,7 +59,7 @@ class PDFParser: return self.lexer.seek(offset) def iter_objects(self) -> Iterator[PDFObject]: - """Iterates over the objects in the current byte stream + """Iterates over the objects in the current byte stream using level 1 parsing Yields: PDFObject diff --git a/pdfmajor/parser_v2/objects.py b/pdfmajor/parser_v2/objects.py index e10db8b..9b89cbb 100644 --- a/pdfmajor/parser_v2/objects.py +++ b/pdfmajor/parser_v2/objects.py @@ -32,7 +32,7 @@ class PDFPrimivite(Generic[TToken], PDFObject): def get_value(self): return self.token.value - def to_python(self) -> Union[TToken.value]: + def to_python(self): return self.value @@ -82,6 +82,17 @@ class PDFStream(PDFObject): fdecode_parms: Optional[List[PDFDictionary]] = None, dl: Optional[int] = None, ) -> None: + """ + Args: + offset (int): the offset where the stream itself starts + length (int): the length of the stream + filter (Optional[List[PDFName]], optional): The names of a filter that shall be applied in processing the stream data. Defaults to None. + decode_parms (Optional[List[PDFDictionary]], optional): Parameter dictionaries used by the filters specified by filter. Defaults to None. + f (Optional[Any], optional): The file containing the stream data. If this entry is present, the bytes between stream and endstream shall be ignored. However, the Length entry should still specify the number of those bytes (usually, there are no bytes and Length is 0).. Defaults to None. + ffilter (Optional[List[PDFName]], optional): The names of a filter that shall be applied in processing of the external file. Defaults to None. + fdecode_parms (Optional[List[PDFDictionary]], optional): Parameter dictionaries used by the filters specified by ffilter. Defaults to None. + dl (Optional[int], optional): A non-negative integer representing the number of bytes in the decoded (defiltered) stream. It can be used to determine, for example, whether enough disk space is available to write a stream to a file.This value shall be considered a hint only; for some stream filters, it may not be possible to determine this value precisely. Defaults to None. + """ self.offset = offset self.length = length self.filter = filter diff --git a/tests/test_lexer.py b/tests/test_lexer.py index cef0c06..7c228d6 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -16,21 +16,17 @@ from pdfmajor.lexer.token import ( PDFName, TArrayValue, TDictValue, - TObjValue, - TStreamValue, Token, TokenArray, TokenBoolean, TokenComment, TokenNull, - TokenObj, TokenReal, TokenDictionary, TokenHexString, TokenInteger, TokenKeyword, TokenName, - TokenStream, TokenString, ) from pdfmajor.lexer.comment import parse_comment @@ -266,13 +262,13 @@ endobj endobj """, expected=[ - TokenStream(0, 9, TStreamValue.END), - TokenObj(10, 16, TObjValue.END), + TokenKeyword(0, 9, b"endstream"), + TokenKeyword(10, 16, b"endobj"), TokenInteger(17, 18, 4), TokenInteger(19, 20, 0), - TokenObj(21, 24, TObjValue.START), + TokenKeyword(21, 24, b"obj"), TokenInteger(28, 32, 2618), - TokenObj(33, 39, TObjValue.END), + TokenKeyword(33, 39, b"endobj"), ], )