got rid of the parse context in favour of a contextual trait
This commit is contained in:
1 parent
3fec7a1747
commit
c7d618284b
6 files changed
+208
-164
No files matched your search
@@ -1,117 +1,6 @@
|
|||||||
import io
|
"""A PDF token parser
|
||||||
from pdfmajor.parser_v2.exceptions import ParserError
|
|
||||||
from pdfmajor.parser_v2.objects import (
|
|
||||||
PDFArray,
|
|
||||||
PDFDictionary,
|
|
||||||
PDFPrimivite,
|
|
||||||
PDFObject,
|
|
||||||
)
|
|
||||||
from typing import Any, Iterator, List, Optional
|
|
||||||
from pdfmajor.lexer.token import (
|
|
||||||
TArrayValue,
|
|
||||||
TDictValue,
|
|
||||||
TokenArray,
|
|
||||||
TokenDictionary,
|
|
||||||
TokenComplexType,
|
|
||||||
TokenComplexTypeVal,
|
|
||||||
is_primitive,
|
|
||||||
)
|
|
||||||
from pdfmajor.lexer import PDFLexer
|
|
||||||
from pdfmajor.parser_v2.context import PDFParseContext
|
|
||||||
|
|
||||||
|
|
||||||
class PDFParser:
|
|
||||||
"""A PDF token parser
|
|
||||||
this parser parses the bytes in three steps
|
this parser parses the bytes in three steps
|
||||||
1. Generate tokens from the byte stream using the PDFLexer
|
1. Generate tokens from the byte stream using the PDFLexer
|
||||||
2. Level 1 Parsing: Match Low level objects such as primitive types, arrays, dictionaries
|
2. Level 1 Parsing: Match Low level objects such as primitive types, arrays, dictionaries
|
||||||
3. Level 2 Parsing: Match high level objects such as Streams and Indirect objects
|
3. Level 2 Parsing: Match high level objects such as Streams and Indirect objects
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
|
|
||||||
) -> None:
|
|
||||||
self.lexer = PDFLexer(fp, buffer_size)
|
|
||||||
self.strict = strict
|
|
||||||
self.context_stack: List[PDFParseContext] = []
|
|
||||||
|
|
||||||
@property
|
|
||||||
def current_context(self) -> Optional[PDFParseContext]:
|
|
||||||
"""Returns the current context of the parsing state
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Optional[PDFParseContext]
|
|
||||||
"""
|
|
||||||
if len(self.context_stack) > 0:
|
|
||||||
return self.context_stack[-1]
|
|
||||||
else:
|
|
||||||
return None
|
|
||||||
|
|
||||||
def seek(self, offset: int) -> int:
|
|
||||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
|
||||||
|
|
||||||
Args:
|
|
||||||
offset (int)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
int: new offset
|
|
||||||
"""
|
|
||||||
return self.lexer.seek(offset)
|
|
||||||
|
|
||||||
def iter_objects(self) -> Iterator[PDFObject]:
|
|
||||||
"""Iterates over the objects in the current byte stream using level 1 parsing
|
|
||||||
|
|
||||||
Yields:
|
|
||||||
PDFObject
|
|
||||||
"""
|
|
||||||
for token in self.lexer.iter_tokens():
|
|
||||||
if is_primitive(token):
|
|
||||||
cur_ctx = self.current_context
|
|
||||||
obj = PDFPrimivite(token) # type: ignore
|
|
||||||
if cur_ctx is None:
|
|
||||||
yield obj
|
|
||||||
else:
|
|
||||||
cur_ctx.push(obj)
|
|
||||||
elif isinstance(token, TokenArray):
|
|
||||||
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
|
|
||||||
if obj is not None:
|
|
||||||
yield obj
|
|
||||||
elif isinstance(token, TokenDictionary):
|
|
||||||
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
|
|
||||||
if obj is not None:
|
|
||||||
yield obj
|
|
||||||
else:
|
|
||||||
raise ParserError(f"Invalid token provided {token}")
|
|
||||||
|
|
||||||
def __deal_with_obj(
|
|
||||||
self,
|
|
||||||
token: TokenComplexType,
|
|
||||||
cls_const: Any,
|
|
||||||
open_t: TokenComplexTypeVal,
|
|
||||||
) -> Optional[PDFObject]:
|
|
||||||
"""generic method for processing things that have a start and end token
|
|
||||||
|
|
||||||
Args:
|
|
||||||
token (TokenComplexType)
|
|
||||||
cls_const (Any): the class constructor for the object
|
|
||||||
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Optional[PDFObject]
|
|
||||||
"""
|
|
||||||
if token.value == open_t:
|
|
||||||
self.context_stack.append(PDFParseContext(cls_const()))
|
|
||||||
elif len(self.context_stack) > 0:
|
|
||||||
last_ctx = self.context_stack.pop()
|
|
||||||
if isinstance(last_ctx.pdfobject, cls_const):
|
|
||||||
cur_ctx = self.current_context
|
|
||||||
if cur_ctx is None:
|
|
||||||
return last_ctx.pdfobject
|
|
||||||
else:
|
|
||||||
cur_ctx.push(last_ctx.pdfobject)
|
|
||||||
elif self.strict:
|
|
||||||
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
|
|
||||||
elif self.strict:
|
|
||||||
raise ParserError(
|
|
||||||
f"Invalid end of {cls_const} specified at {token} with no start"
|
|
||||||
)
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
from pdfmajor.parser_v2.exceptions import ParserError
|
|
||||||
from pdfmajor.parser_v2.objects import (
|
|
||||||
PDFArray,
|
|
||||||
PDFDictionary,
|
|
||||||
PDFObject,
|
|
||||||
PDFComplexType,
|
|
||||||
PDFStream,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class PDFParseContext:
|
|
||||||
"""A parsing context
|
|
||||||
This class helps constructs complex PDF types
|
|
||||||
"""
|
|
||||||
|
|
||||||
__slots__ = ["pdfobject"]
|
|
||||||
|
|
||||||
def __init__(self, pdfobject: PDFComplexType) -> None:
|
|
||||||
"""
|
|
||||||
Args:
|
|
||||||
pdfobject (PDFComplexType): the complex pdf object we are trying to build
|
|
||||||
"""
|
|
||||||
self.pdfobject = pdfobject
|
|
||||||
|
|
||||||
def push(self, item: PDFObject):
|
|
||||||
"""push any pdf object to the context, process according to the type of complex object we are trying to build
|
|
||||||
|
|
||||||
Args:
|
|
||||||
item (PDFObject)
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
ParserError: raises if an invalid complex object is used
|
|
||||||
"""
|
|
||||||
if isinstance(self.pdfobject, PDFArray):
|
|
||||||
self.pdfobject.pass_item(item)
|
|
||||||
elif isinstance(self.pdfobject, PDFDictionary):
|
|
||||||
self.pdfobject.pass_item(item)
|
|
||||||
elif isinstance(self.pdfobject, PDFStream):
|
|
||||||
raise NotImplementedError
|
|
||||||
else:
|
|
||||||
raise ParserError(f"Invalid Context {type(self.pdfobject)}")
|
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
import io
|
||||||
|
from pdfmajor.parser_v2.exceptions import ParserError
|
||||||
|
from pdfmajor.parser_v2.objects import (
|
||||||
|
PDFArray,
|
||||||
|
PDFContextualObject,
|
||||||
|
PDFDictionary,
|
||||||
|
PDFPrimivite,
|
||||||
|
PDFObject,
|
||||||
|
)
|
||||||
|
from typing import Any, Iterator, List, Optional
|
||||||
|
from pdfmajor.lexer.token import (
|
||||||
|
TArrayValue,
|
||||||
|
TDictValue,
|
||||||
|
TokenArray,
|
||||||
|
TokenDictionary,
|
||||||
|
TokenComplexType,
|
||||||
|
TokenComplexTypeVal,
|
||||||
|
is_primitive,
|
||||||
|
)
|
||||||
|
from pdfmajor.lexer import PDFLexer
|
||||||
|
|
||||||
|
|
||||||
|
class PDFL1Parser:
|
||||||
|
"""Level 1 PDF Parser"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
|
||||||
|
) -> None:
|
||||||
|
self.lexer = PDFLexer(fp, buffer_size)
|
||||||
|
self.strict = strict
|
||||||
|
self.context_stack: List[PDFContextualObject] = []
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current_context(self) -> Optional[PDFContextualObject]:
|
||||||
|
"""Returns the current context of the parsing state
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[PDFContextualObject]
|
||||||
|
"""
|
||||||
|
if len(self.context_stack) > 0:
|
||||||
|
return self.context_stack[-1]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def seek(self, offset: int) -> int:
|
||||||
|
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||||
|
|
||||||
|
Args:
|
||||||
|
offset (int)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int: new offset
|
||||||
|
"""
|
||||||
|
return self.lexer.seek(offset)
|
||||||
|
|
||||||
|
def iter_objects(self) -> Iterator[PDFObject]:
|
||||||
|
"""Iterates over the objects in the current byte stream using level 1 parsing
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
PDFObject
|
||||||
|
"""
|
||||||
|
for token in self.lexer.iter_tokens():
|
||||||
|
if is_primitive(token):
|
||||||
|
cur_ctx = self.current_context
|
||||||
|
obj = PDFPrimivite(token) # type: ignore
|
||||||
|
if cur_ctx is None:
|
||||||
|
yield obj
|
||||||
|
else:
|
||||||
|
cur_ctx.pass_item(obj)
|
||||||
|
elif isinstance(token, TokenArray):
|
||||||
|
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
|
||||||
|
if obj is not None:
|
||||||
|
yield obj
|
||||||
|
elif isinstance(token, TokenDictionary):
|
||||||
|
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
|
||||||
|
if obj is not None:
|
||||||
|
yield obj
|
||||||
|
else:
|
||||||
|
raise ParserError(f"Invalid token provided {token}")
|
||||||
|
|
||||||
|
def __deal_with_obj(
|
||||||
|
self,
|
||||||
|
token: TokenComplexType,
|
||||||
|
cls_const: Any,
|
||||||
|
open_t: TokenComplexTypeVal,
|
||||||
|
) -> Optional[PDFObject]:
|
||||||
|
"""generic method for processing things that have a start and end token
|
||||||
|
|
||||||
|
Args:
|
||||||
|
token (TokenComplexType)
|
||||||
|
cls_const (Any): the class constructor for the object
|
||||||
|
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[PDFObject]
|
||||||
|
"""
|
||||||
|
if token.value == open_t:
|
||||||
|
self.context_stack.append(cls_const())
|
||||||
|
elif len(self.context_stack) > 0:
|
||||||
|
last_ctx = self.context_stack.pop()
|
||||||
|
if isinstance(last_ctx, cls_const):
|
||||||
|
cur_ctx = self.current_context
|
||||||
|
if cur_ctx is None:
|
||||||
|
return last_ctx
|
||||||
|
else:
|
||||||
|
cur_ctx.pass_item(last_ctx)
|
||||||
|
elif self.strict:
|
||||||
|
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
|
||||||
|
elif self.strict:
|
||||||
|
raise ParserError(
|
||||||
|
f"Invalid end of {cls_const} specified at {token} with no start"
|
||||||
|
)
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import io
|
||||||
|
from pdfmajor.parser_v2.exceptions import ParserError
|
||||||
|
from pdfmajor.parser_v2.objects import (
|
||||||
|
PDFArray,
|
||||||
|
PDFDictionary,
|
||||||
|
PDFPrimivite,
|
||||||
|
PDFObject,
|
||||||
|
)
|
||||||
|
from typing import Any, Iterator, List, Optional
|
||||||
|
from pdfmajor.lexer.token import (
|
||||||
|
TArrayValue,
|
||||||
|
TDictValue,
|
||||||
|
TokenArray,
|
||||||
|
TokenDictionary,
|
||||||
|
TokenComplexType,
|
||||||
|
TokenComplexTypeVal,
|
||||||
|
is_primitive,
|
||||||
|
)
|
||||||
|
from pdfmajor.lexer import PDFLexer
|
||||||
|
from pdfmajor.parser_v2.context import PDFParseContext
|
||||||
|
|
||||||
|
|
||||||
|
class PDFL1Parser:
|
||||||
|
"""Level 1 PDF Parser"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
|
||||||
|
) -> None:
|
||||||
|
self.lexer = PDFLexer(fp, buffer_size)
|
||||||
|
self.strict = strict
|
||||||
|
self.context_stack: List[PDFParseContext] = []
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current_context(self) -> Optional[PDFParseContext]:
|
||||||
|
"""Returns the current context of the parsing state
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[PDFParseContext]
|
||||||
|
"""
|
||||||
|
if len(self.context_stack) > 0:
|
||||||
|
return self.context_stack[-1]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def seek(self, offset: int) -> int:
|
||||||
|
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||||
|
|
||||||
|
Args:
|
||||||
|
offset (int)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int: new offset
|
||||||
|
"""
|
||||||
|
return self.lexer.seek(offset)
|
||||||
|
|
||||||
|
def iter_objects(self) -> Iterator[PDFObject]:
|
||||||
|
"""Iterates over the objects in the current byte stream using level 2 parsing
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
PDFObject
|
||||||
|
"""
|
||||||
|
for token in self.lexer.iter_tokens():
|
||||||
|
if is_primitive(token):
|
||||||
|
cur_ctx = self.current_context
|
||||||
|
obj = PDFPrimivite(token) # type: ignore
|
||||||
|
if cur_ctx is None:
|
||||||
|
yield obj
|
||||||
|
else:
|
||||||
|
cur_ctx.push(obj)
|
||||||
|
elif isinstance(token, TokenArray):
|
||||||
|
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
|
||||||
|
if obj is not None:
|
||||||
|
yield obj
|
||||||
|
elif isinstance(token, TokenDictionary):
|
||||||
|
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
|
||||||
|
if obj is not None:
|
||||||
|
yield obj
|
||||||
|
else:
|
||||||
|
raise ParserError(f"Invalid token provided {token}")
|
||||||
@@ -23,6 +23,14 @@ class PDFObject(metaclass=ABCMeta):
|
|||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class PDFContextualObject(PDFObject, metaclass=ABCMeta):
|
||||||
|
"""An abstract class representing objects that require context for construction"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def pass_item(self, item: PDFObject):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
class PDFPrimivite(Generic[TToken], PDFObject):
|
class PDFPrimivite(Generic[TToken], PDFObject):
|
||||||
"""A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3"""
|
"""A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3"""
|
||||||
|
|
||||||
@@ -36,7 +44,7 @@ class PDFPrimivite(Generic[TToken], PDFObject):
|
|||||||
return self.value
|
return self.value
|
||||||
|
|
||||||
|
|
||||||
class PDFDictionary(Dict[PDFName, PDFObject], PDFObject):
|
class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject):
|
||||||
"""A class representing a PDF Dictionary as specified in PDF spec 1.7 section 7.3.7"""
|
"""A class representing a PDF Dictionary as specified in PDF spec 1.7 section 7.3.7"""
|
||||||
|
|
||||||
def __init__(self, strict: bool = False) -> None:
|
def __init__(self, strict: bool = False) -> None:
|
||||||
@@ -58,7 +66,7 @@ class PDFDictionary(Dict[PDFName, PDFObject], PDFObject):
|
|||||||
return dict(self)
|
return dict(self)
|
||||||
|
|
||||||
|
|
||||||
class PDFArray(List[PDFObject], PDFObject):
|
class PDFArray(List[PDFObject], PDFContextualObject):
|
||||||
"""A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6"""
|
"""A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6"""
|
||||||
|
|
||||||
def pass_item(self, item: PDFObject):
|
def pass_item(self, item: PDFObject):
|
||||||
@@ -117,6 +125,3 @@ class PDFStream(PDFObject):
|
|||||||
else [x.to_python() for x in self.fdecode_parms],
|
else [x.to_python() for x in self.fdecode_parms],
|
||||||
"dl": self.dl,
|
"dl": self.dl,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
PDFComplexType = Union[PDFDictionary, PDFArray, PDFStream]
|
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
import io
|
import io
|
||||||
from pdfmajor.parser_v2.objects import PDFName
|
from pdfmajor.parser_v2.objects import PDFName
|
||||||
from pdfmajor.parser_v2 import PDFParser
|
from pdfmajor.parser_v2.l1 import PDFL1Parser
|
||||||
from unittest import TestCase
|
from unittest import TestCase
|
||||||
|
|
||||||
|
|
||||||
class Basic(TestCase):
|
class L1(TestCase):
|
||||||
def test_parse_dict(self):
|
def test_parse_dict(self):
|
||||||
parser = PDFParser(
|
parser = PDFL1Parser(
|
||||||
io.BytesIO(
|
io.BytesIO(
|
||||||
br"""<< /Type /Example
|
br"""<< /Type /Example
|
||||||
/Subtype /DictionaryExample
|
/Subtype /DictionaryExample
|
||||||
@@ -41,7 +41,7 @@ class Basic(TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def test_parse_array(self):
|
def test_parse_array(self):
|
||||||
parser = PDFParser(
|
parser = PDFL1Parser(
|
||||||
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]""")
|
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]""")
|
||||||
)
|
)
|
||||||
obj = next(parser.iter_objects())
|
obj = next(parser.iter_objects())
|
||||||
@@ -50,7 +50,7 @@ class Basic(TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def test_parse_nested_array(self):
|
def test_parse_nested_array(self):
|
||||||
parser = PDFParser(
|
parser = PDFL1Parser(
|
||||||
io.BytesIO(
|
io.BytesIO(
|
||||||
br"""[ [10 30] [22 33] [
|
br"""[ [10 30] [22 33] [
|
||||||
<< /x 0 /y 5 >>
|
<< /x 0 /y 5 >>
|
||||||
|
|||||||
Reference in new issue
Block a user