got rid of the parse context in favour of a contextual trait

This commit is contained in:
ari committed 2020-10-17 23:53:43 -04:00
1 parent 3fec7a1747
commit c7d618284b
6 files changed
+208 -164

No files matched your search

+2 -113
View File
@@ -1,117 +1,6 @@
import io
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFDictionary,
PDFPrimivite,
PDFObject,
)
from typing import Any, Iterator, List, Optional
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
from pdfmajor.parser_v2.context import PDFParseContext
class PDFParser:
"""A PDF token parser
"""A PDF token parser
this parser parses the bytes in three steps
1. Generate tokens from the byte stream using the PDFLexer
2. Level 1 Parsing: Match Low level objects such as primitive types, arrays, dictionaries
3. Level 2 Parsing: Match high level objects such as Streams and Indirect objects
"""
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.context_stack: List[PDFParseContext] = []
@property
def current_context(self) -> Optional[PDFParseContext]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFParseContext]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 1 parsing
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.push(obj)
elif isinstance(token, TokenArray):
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
if obj is not None:
yield obj
else:
raise ParserError(f"Invalid token provided {token}")
def __deal_with_obj(
self,
token: TokenComplexType,
cls_const: Any,
open_t: TokenComplexTypeVal,
) -> Optional[PDFObject]:
"""generic method for processing things that have a start and end token
Args:
token (TokenComplexType)
cls_const (Any): the class constructor for the object
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
Returns:
Optional[PDFObject]
"""
if token.value == open_t:
self.context_stack.append(PDFParseContext(cls_const()))
elif len(self.context_stack) > 0:
last_ctx = self.context_stack.pop()
if isinstance(last_ctx.pdfobject, cls_const):
cur_ctx = self.current_context
if cur_ctx is None:
return last_ctx.pdfobject
else:
cur_ctx.push(last_ctx.pdfobject)
elif self.strict:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif self.strict:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
"""
-41
View File
@@ -1,41 +0,0 @@
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFDictionary,
PDFObject,
PDFComplexType,
PDFStream,
)
class PDFParseContext:
"""A parsing context
This class helps constructs complex PDF types
"""
__slots__ = ["pdfobject"]
def __init__(self, pdfobject: PDFComplexType) -> None:
"""
Args:
pdfobject (PDFComplexType): the complex pdf object we are trying to build
"""
self.pdfobject = pdfobject
def push(self, item: PDFObject):
"""push any pdf object to the context, process according to the type of complex object we are trying to build
Args:
item (PDFObject)
Raises:
ParserError: raises if an invalid complex object is used
"""
if isinstance(self.pdfobject, PDFArray):
self.pdfobject.pass_item(item)
elif isinstance(self.pdfobject, PDFDictionary):
self.pdfobject.pass_item(item)
elif isinstance(self.pdfobject, PDFStream):
raise NotImplementedError
else:
raise ParserError(f"Invalid Context {type(self.pdfobject)}")
+112
View File
@@ -0,0 +1,112 @@
import io
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFContextualObject,
PDFDictionary,
PDFPrimivite,
PDFObject,
)
from typing import Any, Iterator, List, Optional
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
class PDFL1Parser:
"""Level 1 PDF Parser"""
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.context_stack: List[PDFContextualObject] = []
@property
def current_context(self) -> Optional[PDFContextualObject]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFContextualObject]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 1 parsing
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.pass_item(obj)
elif isinstance(token, TokenArray):
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
if obj is not None:
yield obj
else:
raise ParserError(f"Invalid token provided {token}")
def __deal_with_obj(
self,
token: TokenComplexType,
cls_const: Any,
open_t: TokenComplexTypeVal,
) -> Optional[PDFObject]:
"""generic method for processing things that have a start and end token
Args:
token (TokenComplexType)
cls_const (Any): the class constructor for the object
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
Returns:
Optional[PDFObject]
"""
if token.value == open_t:
self.context_stack.append(cls_const())
elif len(self.context_stack) > 0:
last_ctx = self.context_stack.pop()
if isinstance(last_ctx, cls_const):
cur_ctx = self.current_context
if cur_ctx is None:
return last_ctx
else:
cur_ctx.pass_item(last_ctx)
elif self.strict:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif self.strict:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
+79
View File
@@ -0,0 +1,79 @@
import io
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFDictionary,
PDFPrimivite,
PDFObject,
)
from typing import Any, Iterator, List, Optional
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
from pdfmajor.parser_v2.context import PDFParseContext
class PDFL1Parser:
"""Level 1 PDF Parser"""
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.context_stack: List[PDFParseContext] = []
@property
def current_context(self) -> Optional[PDFParseContext]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFParseContext]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 2 parsing
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.push(obj)
elif isinstance(token, TokenArray):
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
if obj is not None:
yield obj
else:
raise ParserError(f"Invalid token provided {token}")
+10 -5
View File
@@ -23,6 +23,14 @@ class PDFObject(metaclass=ABCMeta):
raise NotImplementedError
class PDFContextualObject(PDFObject, metaclass=ABCMeta):
"""An abstract class representing objects that require context for construction"""
@abstractmethod
def pass_item(self, item: PDFObject):
raise NotImplementedError
class PDFPrimivite(Generic[TToken], PDFObject):
"""A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3"""
@@ -36,7 +44,7 @@ class PDFPrimivite(Generic[TToken], PDFObject):
return self.value
class PDFDictionary(Dict[PDFName, PDFObject], PDFObject):
class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject):
"""A class representing a PDF Dictionary as specified in PDF spec 1.7 section 7.3.7"""
def __init__(self, strict: bool = False) -> None:
@@ -58,7 +66,7 @@ class PDFDictionary(Dict[PDFName, PDFObject], PDFObject):
return dict(self)
class PDFArray(List[PDFObject], PDFObject):
class PDFArray(List[PDFObject], PDFContextualObject):
"""A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6"""
def pass_item(self, item: PDFObject):
@@ -117,6 +125,3 @@ class PDFStream(PDFObject):
else [x.to_python() for x in self.fdecode_parms],
"dl": self.dl,
}
PDFComplexType = Union[PDFDictionary, PDFArray, PDFStream]
+5 -5
View File
@@ -1,13 +1,13 @@
from decimal import Decimal
import io
from pdfmajor.parser_v2.objects import PDFName
from pdfmajor.parser_v2 import PDFParser
from pdfmajor.parser_v2.l1 import PDFL1Parser
from unittest import TestCase
class Basic(TestCase):
class L1(TestCase):
def test_parse_dict(self):
parser = PDFParser(
parser = PDFL1Parser(
io.BytesIO(
br"""<< /Type /Example
/Subtype /DictionaryExample
@@ -41,7 +41,7 @@ class Basic(TestCase):
)
def test_parse_array(self):
parser = PDFParser(
parser = PDFL1Parser(
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]""")
)
obj = next(parser.iter_objects())
@@ -50,7 +50,7 @@ class Basic(TestCase):
)
def test_parse_nested_array(self):
parser = PDFParser(
parser = PDFL1Parser(
io.BytesIO(
br"""[ [10 30] [22 33] [
<< /x 0 /y 5 >>