working prototype for l2

This commit is contained in:
ari committed 2020-10-18 11:48:19 -04:00
1 parent c7d618284b
commit 43ed8b4019
6 files changed
+304 -111

No files matched your search

-6
View File
@@ -116,11 +116,8 @@ class TokenReal(Token[Decimal]):
TokenNumber = Union[TokenInteger, TokenReal]
TokenPrimitive = Union[
TokenComment,
TokenKeyword,
TokenName,
TokenBoolean,
TokenNull,
TokenString,
TokenHexString,
TokenInteger,
@@ -138,11 +135,8 @@ def is_primitive(token: Token) -> bool:
bool: True if primitive
"""
for opt in [
TokenComment,
TokenKeyword,
TokenName,
TokenBoolean,
TokenNull,
TokenString,
TokenHexString,
TokenInteger,
+88
View File
@@ -0,0 +1,88 @@
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.lexer.token import TokenNull
from typing import Dict, Tuple
from .objects import PDFContextualObject, PDFNull, PDFObject, PDFPrimitive
class IndirectObjectCollection:
"""A class for collection references to objects"""
def __init__(self) -> None:
self.objs: Dict[Tuple[int, int], PDFObject] = {}
def create_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject":
"""Create an indirect object and store its value in the collection
return the reference to where this object lies at
Returns:
IndirectObject:
"""
return IndirectObject(self, obj_num, gen_num, read_only=False)
def get_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject":
"""creates a reference to an object
Returns:
IndirectObject
"""
return IndirectObject(self, obj_num, gen_num, read_only=True)
def get_object(self, obj_num: int, gen_num: int) -> PDFObject:
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
Args:
obj_num (int): object number
gen_num (int): generation number
Returns:
PDFObject
"""
return self.objs.get((obj_num, gen_num), PDFNull())
class IndirectObject(PDFContextualObject):
"""A class representing a reference for other objects"""
def __init__(
self,
collection: IndirectObjectCollection,
obj_num: int,
gen_num: int,
read_only: bool = True,
) -> None:
self.obj_num = obj_num
self.gen_num = gen_num
self.__read_only = read_only
self.__col = collection
def __repr__(self) -> str:
return f"IndirectObject({self.obj_num}, {self.gen_num})"
def get_object(self) -> PDFObject:
return self.__col.get_object(self.obj_num, self.gen_num)
def save_object(self, obj: PDFObject):
if self.__read_only:
raise ParserError(
f"attempted to override a read-only reference of an object {obj}"
)
self.__col.objs[(self.obj_num, self.gen_num)] = obj
def get_value(self):
return IndirectObject(self.__col, self.obj_num, self.gen_num, True)
def to_python(self):
return self.get_object().to_python()
def pass_item(self, item: PDFObject):
if self.__read_only:
raise ParserError(
f"attempted to add to an read-only reference of an object {item}"
)
cur_obj = self.get_object()
if isinstance(cur_obj, PDFContextualObject):
cur_obj.pass_item(item)
elif isinstance(cur_obj, PDFNull):
self.save_object(item)
else:
raise ParserError(f"Invalid redefinition of an object {item}")
+77 -52
View File
@@ -2,24 +2,80 @@ import io
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFComment,
PDFContextualObject,
PDFDictionary,
PDFPrimivite,
PDFKeyword,
PDFNull,
PDFPrimitive,
PDFObject,
)
from typing import Any, Iterator, List, Optional
from typing import Any, Iterator, List, Optional, Type
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenComment,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
TokenKeyword,
TokenNull,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
def _deal_with_obj(
context_stack: List[PDFContextualObject],
token: TokenComplexType,
cls_const: Type[PDFContextualObject],
open_t: TokenComplexTypeVal,
strict: bool = False,
) -> Optional[PDFObject]:
"""generic method for processing things that have a start and end token
Args:
token (TokenComplexType)
cls_const (Any): the class constructor for the object
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
Returns:
Optional[PDFObject]
"""
if token.value == open_t:
context_stack.append(cls_const())
elif len(context_stack) > 0:
last_ctx = context_stack.pop()
if isinstance(last_ctx, cls_const):
cur_ctx = _get_curr_context(context_stack)
if cur_ctx is None:
return last_ctx
else:
cur_ctx.pass_item(last_ctx)
elif strict:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif strict:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
return None
def _get_curr_context(
context_stack: List[PDFContextualObject],
) -> Optional[PDFContextualObject]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFContextualObject]
"""
if len(context_stack) > 0:
return context_stack[-1]
else:
return None
class PDFL1Parser:
"""Level 1 PDF Parser"""
@@ -28,19 +84,6 @@ class PDFL1Parser:
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.context_stack: List[PDFContextualObject] = []
@property
def current_context(self) -> Optional[PDFContextualObject]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFContextualObject]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
@@ -59,54 +102,36 @@ class PDFL1Parser:
Yields:
PDFObject
"""
context_stack: List[PDFContextualObject] = []
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
cur_ctx = _get_curr_context(context_stack)
obj = PDFPrimitive(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.pass_item(obj)
elif isinstance(token, TokenComment):
yield PDFComment(token)
elif isinstance(token, TokenKeyword):
yield PDFKeyword(token)
elif isinstance(token, TokenNull):
yield PDFNull()
elif isinstance(token, TokenArray):
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
obj = _deal_with_obj(
context_stack, token, PDFArray, TArrayValue.OPEN, strict=self.strict
)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
obj = _deal_with_obj(
context_stack=context_stack,
token=token,
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
strict=self.strict,
)
if obj is not None:
yield obj
else:
raise ParserError(f"Invalid token provided {token}")
def __deal_with_obj(
self,
token: TokenComplexType,
cls_const: Any,
open_t: TokenComplexTypeVal,
) -> Optional[PDFObject]:
"""generic method for processing things that have a start and end token
Args:
token (TokenComplexType)
cls_const (Any): the class constructor for the object
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
Returns:
Optional[PDFObject]
"""
if token.value == open_t:
self.context_stack.append(cls_const())
elif len(self.context_stack) > 0:
last_ctx = self.context_stack.pop()
if isinstance(last_ctx, cls_const):
cur_ctx = self.current_context
if cur_ctx is None:
return last_ctx
else:
cur_ctx.pass_item(last_ctx)
elif self.strict:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif self.strict:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
+55 -47
View File
@@ -1,46 +1,28 @@
import io
from pdfmajor.parser_v2.l1 import PDFL1Parser
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFDictionary,
PDFPrimivite,
PDFContextualObject,
PDFKeyword,
PDFPrimitive,
PDFObject,
)
from typing import Any, Iterator, List, Optional
from typing import Iterator, List, Optional, Tuple
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
is_primitive,
TokenInteger,
)
from pdfmajor.lexer import PDFLexer
from pdfmajor.parser_v2.context import PDFParseContext
from pdfmajor.parser_v2.indirect_objects import IndirectObjectCollection
class PDFL1Parser:
"""Level 1 PDF Parser"""
class PDFL2Parser:
"""Level 2 PDF Parser"""
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.l1 = PDFL1Parser(fp, buffer_size, strict)
self.strict = strict
self.context_stack: List[PDFParseContext] = []
@property
def current_context(self) -> Optional[PDFParseContext]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFParseContext]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
self.inobjects = IndirectObjectCollection()
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
@@ -51,7 +33,7 @@ class PDFL1Parser:
Returns:
int: new offset
"""
return self.lexer.seek(offset)
return self.l1.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 2 parsing
@@ -59,21 +41,47 @@ class PDFL1Parser:
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.push(obj)
elif isinstance(token, TokenArray):
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
if obj is not None:
yield obj
object_list: List[PDFObject] = []
ctx_list: List[PDFContextualObject] = []
for obj in self.l1.iter_objects():
if isinstance(obj, PDFKeyword):
if obj.get_value() == b"obj":
if len(object_list) < 2:
raise ParserError("recieved 'obj' but missing leading tokens")
ctx_list.append(
self.inobjects.create_indobject(
*_get_indobj_values(object_list)
)
)
elif obj.get_value() == b"endobj":
if len(ctx_list) > 0:
last_ctx_obj = ctx_list.pop()
if len(ctx_list) > 0:
ctx_list[-1].pass_item(last_ctx_obj)
else:
yield last_ctx_obj.get_value()
elif obj.get_value() == b"R":
cur_indobj = self.inobjects.get_indobject(
*_get_indobj_values(object_list)
)
if len(ctx_list) > 0:
ctx_list[-1].pass_item(cur_indobj)
else:
yield cur_indobj.get_value()
elif len(ctx_list) == 0:
object_list.append(obj)
elif len(ctx_list) > 0:
ctx_list[-1].pass_item(obj)
else:
raise ParserError(f"Invalid token provided {token}")
raise ParserError(f"Invalid obj provided {obj}")
def _get_indobj_values(object_list: List[PDFObject]) -> Tuple[int, int]:
gen_num = object_list.pop()
obj_num = object_list.pop()
if isinstance(gen_num, PDFPrimitive) and isinstance(obj_num, PDFPrimitive):
if isinstance(gen_num.token, TokenInteger) and isinstance(
obj_num.token, TokenInteger
):
return (obj_num.to_python(), gen_num.to_python())
raise ParserError("missing leading tokens for object reference")
+53 -6
View File
@@ -3,8 +3,14 @@
from abc import ABCMeta, abstractmethod
from pdfmajor.execptions import ParserError
from pdfmajor.lexer.token import PDFName, TokenName, TokenPrimitive
from typing import Any, Dict, Generic, List, NamedTuple, Optional, TypeVar, Union
from pdfmajor.lexer.token import (
PDFName,
TokenComment,
TokenKeyword,
TokenName,
TokenPrimitive,
)
from typing import Any, Dict, Generic, List, Optional, TypeVar
TToken = TypeVar("TToken", bound=TokenPrimitive)
@@ -22,8 +28,13 @@ class PDFObject(metaclass=ABCMeta):
"""returns the python variation of this object"""
raise NotImplementedError
def __eq__(self, o: object) -> bool:
if not isinstance(o, self.__class__):
return False
return o.to_python() == self.to_python()
class PDFContextualObject(PDFObject, metaclass=ABCMeta):
class PDFContextualObject(PDFObject):
"""An abstract class representing objects that require context for construction"""
@abstractmethod
@@ -31,7 +42,7 @@ class PDFContextualObject(PDFObject, metaclass=ABCMeta):
raise NotImplementedError
class PDFPrimivite(Generic[TToken], PDFObject):
class PDFPrimitive(Generic[TToken], PDFObject):
"""A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3"""
def __init__(self, token: TToken) -> None:
@@ -41,7 +52,43 @@ class PDFPrimivite(Generic[TToken], PDFObject):
return self.token.value
def to_python(self):
return self.value
return self.get_value()
class PDFKeyword(PDFObject):
"""A class representing a PDF keyword"""
def __init__(self, token: TokenKeyword) -> None:
self.token = token
def get_value(self):
return self.token.value
def to_python(self):
return self.get_value()
class PDFComment(PDFObject):
"""A class representing a PDF comment"""
def __init__(self, token: TokenComment) -> None:
self.token = token
def get_value(self):
return self.token.value
def to_python(self):
return self.get_value()
class PDFNull(PDFObject):
"""A class representing a PDF null object"""
def get_value(self):
return None
def to_python(self):
return None
class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject):
@@ -54,7 +101,7 @@ class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject):
def pass_item(self, item: PDFObject):
if self.name is None:
if isinstance(item, PDFPrimivite) and isinstance(item.token, TokenName):
if isinstance(item, PDFPrimitive) and isinstance(item.token, TokenName):
self.name = item.get_value()
if self.strict:
raise ParserError("Invalid token provided to dictionary")
+31
View File
@@ -1,7 +1,10 @@
from decimal import Decimal
import io
from pdfmajor.parser_v2.indirect_objects import IndirectObject
from pdfmajor.parser_v2.l2 import PDFL2Parser
from pdfmajor.parser_v2.objects import PDFName
from pdfmajor.parser_v2.l1 import PDFL1Parser
from pdfmajor.streambuffer import StreamEOF
from unittest import TestCase
@@ -70,3 +73,31 @@ class L1(TestCase):
],
],
)
class L2(TestCase):
def test_parse_simple_indobj(self):
parser = PDFL2Parser(
io.BytesIO(
br"""
12 1 obj (Bring) endobj
100 0 obj
<33>
endobj
13 8
obj
<< /x 1 /y 3>> endobj
"""
)
)
expected = [
(12, 1, "Bring"),
(100, 0, b"3"),
(13, 8, {PDFName("x"): 1, PDFName("y"): 3}),
]
for obj, eobj in zip(parser.iter_objects(), expected):
self.assertIsInstance(obj, IndirectObject)
self.assertEqual(obj, parser.inobjects.get_indobject(eobj[0], eobj[1]))
self.assertEqual(obj.to_python(), eobj[2])
with self.assertRaises(StreamEOF):
next(parser.iter_objects())