got rid of the parse context in favour of a contextual trait

This commit is contained in:
ari committed 2020-10-17 23:53:43 -04:00
1 parent 3fec7a1747
commit c7d618284b
6 files changed
+208 -164

No files matched your search

+2 -113
View File
@@ -1,117 +1,6 @@
import io """A PDF token parser
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFDictionary,
PDFPrimivite,
PDFObject,
)
from typing import Any, Iterator, List, Optional
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
from pdfmajor.parser_v2.context import PDFParseContext
class PDFParser:
"""A PDF token parser
this parser parses the bytes in three steps this parser parses the bytes in three steps
1. Generate tokens from the byte stream using the PDFLexer 1. Generate tokens from the byte stream using the PDFLexer
2. Level 1 Parsing: Match Low level objects such as primitive types, arrays, dictionaries 2. Level 1 Parsing: Match Low level objects such as primitive types, arrays, dictionaries
3. Level 2 Parsing: Match high level objects such as Streams and Indirect objects 3. Level 2 Parsing: Match high level objects such as Streams and Indirect objects
""" """
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.context_stack: List[PDFParseContext] = []
@property
def current_context(self) -> Optional[PDFParseContext]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFParseContext]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 1 parsing
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.push(obj)
elif isinstance(token, TokenArray):
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
if obj is not None:
yield obj
else:
raise ParserError(f"Invalid token provided {token}")
def __deal_with_obj(
self,
token: TokenComplexType,
cls_const: Any,
open_t: TokenComplexTypeVal,
) -> Optional[PDFObject]:
"""generic method for processing things that have a start and end token
Args:
token (TokenComplexType)
cls_const (Any): the class constructor for the object
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
Returns:
Optional[PDFObject]
"""
if token.value == open_t:
self.context_stack.append(PDFParseContext(cls_const()))
elif len(self.context_stack) > 0:
last_ctx = self.context_stack.pop()
if isinstance(last_ctx.pdfobject, cls_const):
cur_ctx = self.current_context
if cur_ctx is None:
return last_ctx.pdfobject
else:
cur_ctx.push(last_ctx.pdfobject)
elif self.strict:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif self.strict:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
-41
View File
@@ -1,41 +0,0 @@
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFDictionary,
PDFObject,
PDFComplexType,
PDFStream,
)
class PDFParseContext:
"""A parsing context
This class helps constructs complex PDF types
"""
__slots__ = ["pdfobject"]
def __init__(self, pdfobject: PDFComplexType) -> None:
"""
Args:
pdfobject (PDFComplexType): the complex pdf object we are trying to build
"""
self.pdfobject = pdfobject
def push(self, item: PDFObject):
"""push any pdf object to the context, process according to the type of complex object we are trying to build
Args:
item (PDFObject)
Raises:
ParserError: raises if an invalid complex object is used
"""
if isinstance(self.pdfobject, PDFArray):
self.pdfobject.pass_item(item)
elif isinstance(self.pdfobject, PDFDictionary):
self.pdfobject.pass_item(item)
elif isinstance(self.pdfobject, PDFStream):
raise NotImplementedError
else:
raise ParserError(f"Invalid Context {type(self.pdfobject)}")
+112
View File
@@ -0,0 +1,112 @@
import io
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFContextualObject,
PDFDictionary,
PDFPrimivite,
PDFObject,
)
from typing import Any, Iterator, List, Optional
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
class PDFL1Parser:
"""Level 1 PDF Parser"""
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.context_stack: List[PDFContextualObject] = []
@property
def current_context(self) -> Optional[PDFContextualObject]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFContextualObject]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 1 parsing
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.pass_item(obj)
elif isinstance(token, TokenArray):
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
if obj is not None:
yield obj
else:
raise ParserError(f"Invalid token provided {token}")
def __deal_with_obj(
self,
token: TokenComplexType,
cls_const: Any,
open_t: TokenComplexTypeVal,
) -> Optional[PDFObject]:
"""generic method for processing things that have a start and end token
Args:
token (TokenComplexType)
cls_const (Any): the class constructor for the object
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
Returns:
Optional[PDFObject]
"""
if token.value == open_t:
self.context_stack.append(cls_const())
elif len(self.context_stack) > 0:
last_ctx = self.context_stack.pop()
if isinstance(last_ctx, cls_const):
cur_ctx = self.current_context
if cur_ctx is None:
return last_ctx
else:
cur_ctx.pass_item(last_ctx)
elif self.strict:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif self.strict:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
+79
View File
@@ -0,0 +1,79 @@
import io
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFDictionary,
PDFPrimivite,
PDFObject,
)
from typing import Any, Iterator, List, Optional
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
from pdfmajor.parser_v2.context import PDFParseContext
class PDFL1Parser:
"""Level 1 PDF Parser"""
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.context_stack: List[PDFParseContext] = []
@property
def current_context(self) -> Optional[PDFParseContext]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFParseContext]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 2 parsing
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.push(obj)
elif isinstance(token, TokenArray):
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
if obj is not None:
yield obj
else:
raise ParserError(f"Invalid token provided {token}")
+10 -5
View File
@@ -23,6 +23,14 @@ class PDFObject(metaclass=ABCMeta):
raise NotImplementedError raise NotImplementedError
class PDFContextualObject(PDFObject, metaclass=ABCMeta):
"""An abstract class representing objects that require context for construction"""
@abstractmethod
def pass_item(self, item: PDFObject):
raise NotImplementedError
class PDFPrimivite(Generic[TToken], PDFObject): class PDFPrimivite(Generic[TToken], PDFObject):
"""A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3""" """A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3"""
@@ -36,7 +44,7 @@ class PDFPrimivite(Generic[TToken], PDFObject):
return self.value return self.value
class PDFDictionary(Dict[PDFName, PDFObject], PDFObject): class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject):
"""A class representing a PDF Dictionary as specified in PDF spec 1.7 section 7.3.7""" """A class representing a PDF Dictionary as specified in PDF spec 1.7 section 7.3.7"""
def __init__(self, strict: bool = False) -> None: def __init__(self, strict: bool = False) -> None:
@@ -58,7 +66,7 @@ class PDFDictionary(Dict[PDFName, PDFObject], PDFObject):
return dict(self) return dict(self)
class PDFArray(List[PDFObject], PDFObject): class PDFArray(List[PDFObject], PDFContextualObject):
"""A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6""" """A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6"""
def pass_item(self, item: PDFObject): def pass_item(self, item: PDFObject):
@@ -117,6 +125,3 @@ class PDFStream(PDFObject):
else [x.to_python() for x in self.fdecode_parms], else [x.to_python() for x in self.fdecode_parms],
"dl": self.dl, "dl": self.dl,
} }
PDFComplexType = Union[PDFDictionary, PDFArray, PDFStream]
+5 -5
View File
@@ -1,13 +1,13 @@
from decimal import Decimal from decimal import Decimal
import io import io
from pdfmajor.parser_v2.objects import PDFName from pdfmajor.parser_v2.objects import PDFName
from pdfmajor.parser_v2 import PDFParser from pdfmajor.parser_v2.l1 import PDFL1Parser
from unittest import TestCase from unittest import TestCase
class Basic(TestCase): class L1(TestCase):
def test_parse_dict(self): def test_parse_dict(self):
parser = PDFParser( parser = PDFL1Parser(
io.BytesIO( io.BytesIO(
br"""<< /Type /Example br"""<< /Type /Example
/Subtype /DictionaryExample /Subtype /DictionaryExample
@@ -41,7 +41,7 @@ class Basic(TestCase):
) )
def test_parse_array(self): def test_parse_array(self):
parser = PDFParser( parser = PDFL1Parser(
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]""") io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]""")
) )
obj = next(parser.iter_objects()) obj = next(parser.iter_objects())
@@ -50,7 +50,7 @@ class Basic(TestCase):
) )
def test_parse_nested_array(self): def test_parse_nested_array(self):
parser = PDFParser( parser = PDFL1Parser(
io.BytesIO( io.BytesIO(
br"""[ [10 30] [22 33] [ br"""[ [10 30] [22 33] [
<< /x 0 /y 5 >> << /x 0 /y 5 >>