working prototype for l2
This commit is contained in:
1 parent
c7d618284b
commit
43ed8b4019
6 files changed
+304
-111
No files matched your search
@@ -116,11 +116,8 @@ class TokenReal(Token[Decimal]):
|
||||
|
||||
TokenNumber = Union[TokenInteger, TokenReal]
|
||||
TokenPrimitive = Union[
|
||||
TokenComment,
|
||||
TokenKeyword,
|
||||
TokenName,
|
||||
TokenBoolean,
|
||||
TokenNull,
|
||||
TokenString,
|
||||
TokenHexString,
|
||||
TokenInteger,
|
||||
@@ -138,11 +135,8 @@ def is_primitive(token: Token) -> bool:
|
||||
bool: True if primitive
|
||||
"""
|
||||
for opt in [
|
||||
TokenComment,
|
||||
TokenKeyword,
|
||||
TokenName,
|
||||
TokenBoolean,
|
||||
TokenNull,
|
||||
TokenString,
|
||||
TokenHexString,
|
||||
TokenInteger,
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
from pdfmajor.parser_v2.exceptions import ParserError
|
||||
from pdfmajor.lexer.token import TokenNull
|
||||
from typing import Dict, Tuple
|
||||
from .objects import PDFContextualObject, PDFNull, PDFObject, PDFPrimitive
|
||||
|
||||
|
||||
class IndirectObjectCollection:
|
||||
"""A class for collection references to objects"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.objs: Dict[Tuple[int, int], PDFObject] = {}
|
||||
|
||||
def create_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject":
|
||||
"""Create an indirect object and store its value in the collection
|
||||
return the reference to where this object lies at
|
||||
|
||||
Returns:
|
||||
IndirectObject:
|
||||
"""
|
||||
return IndirectObject(self, obj_num, gen_num, read_only=False)
|
||||
|
||||
def get_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject":
|
||||
"""creates a reference to an object
|
||||
|
||||
Returns:
|
||||
IndirectObject
|
||||
"""
|
||||
return IndirectObject(self, obj_num, gen_num, read_only=True)
|
||||
|
||||
def get_object(self, obj_num: int, gen_num: int) -> PDFObject:
|
||||
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
|
||||
|
||||
Args:
|
||||
obj_num (int): object number
|
||||
gen_num (int): generation number
|
||||
|
||||
Returns:
|
||||
PDFObject
|
||||
"""
|
||||
return self.objs.get((obj_num, gen_num), PDFNull())
|
||||
|
||||
|
||||
class IndirectObject(PDFContextualObject):
|
||||
"""A class representing a reference for other objects"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
collection: IndirectObjectCollection,
|
||||
obj_num: int,
|
||||
gen_num: int,
|
||||
read_only: bool = True,
|
||||
) -> None:
|
||||
self.obj_num = obj_num
|
||||
self.gen_num = gen_num
|
||||
self.__read_only = read_only
|
||||
self.__col = collection
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"IndirectObject({self.obj_num}, {self.gen_num})"
|
||||
|
||||
def get_object(self) -> PDFObject:
|
||||
return self.__col.get_object(self.obj_num, self.gen_num)
|
||||
|
||||
def save_object(self, obj: PDFObject):
|
||||
if self.__read_only:
|
||||
raise ParserError(
|
||||
f"attempted to override a read-only reference of an object {obj}"
|
||||
)
|
||||
self.__col.objs[(self.obj_num, self.gen_num)] = obj
|
||||
|
||||
def get_value(self):
|
||||
return IndirectObject(self.__col, self.obj_num, self.gen_num, True)
|
||||
|
||||
def to_python(self):
|
||||
return self.get_object().to_python()
|
||||
|
||||
def pass_item(self, item: PDFObject):
|
||||
if self.__read_only:
|
||||
raise ParserError(
|
||||
f"attempted to add to an read-only reference of an object {item}"
|
||||
)
|
||||
cur_obj = self.get_object()
|
||||
if isinstance(cur_obj, PDFContextualObject):
|
||||
cur_obj.pass_item(item)
|
||||
elif isinstance(cur_obj, PDFNull):
|
||||
self.save_object(item)
|
||||
else:
|
||||
raise ParserError(f"Invalid redefinition of an object {item}")
|
||||
+77
-52
@@ -2,24 +2,80 @@ import io
|
||||
from pdfmajor.parser_v2.exceptions import ParserError
|
||||
from pdfmajor.parser_v2.objects import (
|
||||
PDFArray,
|
||||
PDFComment,
|
||||
PDFContextualObject,
|
||||
PDFDictionary,
|
||||
PDFPrimivite,
|
||||
PDFKeyword,
|
||||
PDFNull,
|
||||
PDFPrimitive,
|
||||
PDFObject,
|
||||
)
|
||||
from typing import Any, Iterator, List, Optional
|
||||
from typing import Any, Iterator, List, Optional, Type
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
TokenArray,
|
||||
TokenComment,
|
||||
TokenDictionary,
|
||||
TokenComplexType,
|
||||
TokenComplexTypeVal,
|
||||
TokenKeyword,
|
||||
TokenNull,
|
||||
is_primitive,
|
||||
)
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
|
||||
|
||||
def _deal_with_obj(
|
||||
context_stack: List[PDFContextualObject],
|
||||
token: TokenComplexType,
|
||||
cls_const: Type[PDFContextualObject],
|
||||
open_t: TokenComplexTypeVal,
|
||||
strict: bool = False,
|
||||
) -> Optional[PDFObject]:
|
||||
"""generic method for processing things that have a start and end token
|
||||
|
||||
Args:
|
||||
token (TokenComplexType)
|
||||
cls_const (Any): the class constructor for the object
|
||||
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
|
||||
|
||||
Returns:
|
||||
Optional[PDFObject]
|
||||
"""
|
||||
if token.value == open_t:
|
||||
context_stack.append(cls_const())
|
||||
elif len(context_stack) > 0:
|
||||
last_ctx = context_stack.pop()
|
||||
if isinstance(last_ctx, cls_const):
|
||||
cur_ctx = _get_curr_context(context_stack)
|
||||
if cur_ctx is None:
|
||||
return last_ctx
|
||||
else:
|
||||
cur_ctx.pass_item(last_ctx)
|
||||
elif strict:
|
||||
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
|
||||
elif strict:
|
||||
raise ParserError(
|
||||
f"Invalid end of {cls_const} specified at {token} with no start"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _get_curr_context(
|
||||
context_stack: List[PDFContextualObject],
|
||||
) -> Optional[PDFContextualObject]:
|
||||
"""Returns the current context of the parsing state
|
||||
|
||||
Returns:
|
||||
Optional[PDFContextualObject]
|
||||
"""
|
||||
if len(context_stack) > 0:
|
||||
return context_stack[-1]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class PDFL1Parser:
|
||||
"""Level 1 PDF Parser"""
|
||||
|
||||
@@ -28,19 +84,6 @@ class PDFL1Parser:
|
||||
) -> None:
|
||||
self.lexer = PDFLexer(fp, buffer_size)
|
||||
self.strict = strict
|
||||
self.context_stack: List[PDFContextualObject] = []
|
||||
|
||||
@property
|
||||
def current_context(self) -> Optional[PDFContextualObject]:
|
||||
"""Returns the current context of the parsing state
|
||||
|
||||
Returns:
|
||||
Optional[PDFContextualObject]
|
||||
"""
|
||||
if len(self.context_stack) > 0:
|
||||
return self.context_stack[-1]
|
||||
else:
|
||||
return None
|
||||
|
||||
def seek(self, offset: int) -> int:
|
||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||
@@ -59,54 +102,36 @@ class PDFL1Parser:
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
context_stack: List[PDFContextualObject] = []
|
||||
for token in self.lexer.iter_tokens():
|
||||
if is_primitive(token):
|
||||
cur_ctx = self.current_context
|
||||
obj = PDFPrimivite(token) # type: ignore
|
||||
cur_ctx = _get_curr_context(context_stack)
|
||||
obj = PDFPrimitive(token) # type: ignore
|
||||
if cur_ctx is None:
|
||||
yield obj
|
||||
else:
|
||||
cur_ctx.pass_item(obj)
|
||||
elif isinstance(token, TokenComment):
|
||||
yield PDFComment(token)
|
||||
elif isinstance(token, TokenKeyword):
|
||||
yield PDFKeyword(token)
|
||||
elif isinstance(token, TokenNull):
|
||||
yield PDFNull()
|
||||
elif isinstance(token, TokenArray):
|
||||
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
|
||||
obj = _deal_with_obj(
|
||||
context_stack, token, PDFArray, TArrayValue.OPEN, strict=self.strict
|
||||
)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
elif isinstance(token, TokenDictionary):
|
||||
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
|
||||
obj = _deal_with_obj(
|
||||
context_stack=context_stack,
|
||||
token=token,
|
||||
cls_const=PDFDictionary,
|
||||
open_t=TDictValue.OPEN,
|
||||
strict=self.strict,
|
||||
)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
else:
|
||||
raise ParserError(f"Invalid token provided {token}")
|
||||
|
||||
def __deal_with_obj(
|
||||
self,
|
||||
token: TokenComplexType,
|
||||
cls_const: Any,
|
||||
open_t: TokenComplexTypeVal,
|
||||
) -> Optional[PDFObject]:
|
||||
"""generic method for processing things that have a start and end token
|
||||
|
||||
Args:
|
||||
token (TokenComplexType)
|
||||
cls_const (Any): the class constructor for the object
|
||||
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
|
||||
|
||||
Returns:
|
||||
Optional[PDFObject]
|
||||
"""
|
||||
if token.value == open_t:
|
||||
self.context_stack.append(cls_const())
|
||||
elif len(self.context_stack) > 0:
|
||||
last_ctx = self.context_stack.pop()
|
||||
if isinstance(last_ctx, cls_const):
|
||||
cur_ctx = self.current_context
|
||||
if cur_ctx is None:
|
||||
return last_ctx
|
||||
else:
|
||||
cur_ctx.pass_item(last_ctx)
|
||||
elif self.strict:
|
||||
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
|
||||
elif self.strict:
|
||||
raise ParserError(
|
||||
f"Invalid end of {cls_const} specified at {token} with no start"
|
||||
)
|
||||
+55
-47
@@ -1,46 +1,28 @@
|
||||
import io
|
||||
from pdfmajor.parser_v2.l1 import PDFL1Parser
|
||||
from pdfmajor.parser_v2.exceptions import ParserError
|
||||
from pdfmajor.parser_v2.objects import (
|
||||
PDFArray,
|
||||
PDFDictionary,
|
||||
PDFPrimivite,
|
||||
PDFContextualObject,
|
||||
PDFKeyword,
|
||||
PDFPrimitive,
|
||||
PDFObject,
|
||||
)
|
||||
from typing import Any, Iterator, List, Optional
|
||||
from typing import Iterator, List, Optional, Tuple
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
TokenArray,
|
||||
TokenDictionary,
|
||||
TokenComplexType,
|
||||
TokenComplexTypeVal,
|
||||
is_primitive,
|
||||
TokenInteger,
|
||||
)
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.parser_v2.context import PDFParseContext
|
||||
from pdfmajor.parser_v2.indirect_objects import IndirectObjectCollection
|
||||
|
||||
|
||||
class PDFL1Parser:
|
||||
"""Level 1 PDF Parser"""
|
||||
class PDFL2Parser:
|
||||
"""Level 2 PDF Parser"""
|
||||
|
||||
def __init__(
|
||||
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
|
||||
) -> None:
|
||||
self.lexer = PDFLexer(fp, buffer_size)
|
||||
self.l1 = PDFL1Parser(fp, buffer_size, strict)
|
||||
self.strict = strict
|
||||
self.context_stack: List[PDFParseContext] = []
|
||||
|
||||
@property
|
||||
def current_context(self) -> Optional[PDFParseContext]:
|
||||
"""Returns the current context of the parsing state
|
||||
|
||||
Returns:
|
||||
Optional[PDFParseContext]
|
||||
"""
|
||||
if len(self.context_stack) > 0:
|
||||
return self.context_stack[-1]
|
||||
else:
|
||||
return None
|
||||
self.inobjects = IndirectObjectCollection()
|
||||
|
||||
def seek(self, offset: int) -> int:
|
||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||
@@ -51,7 +33,7 @@ class PDFL1Parser:
|
||||
Returns:
|
||||
int: new offset
|
||||
"""
|
||||
return self.lexer.seek(offset)
|
||||
return self.l1.seek(offset)
|
||||
|
||||
def iter_objects(self) -> Iterator[PDFObject]:
|
||||
"""Iterates over the objects in the current byte stream using level 2 parsing
|
||||
@@ -59,21 +41,47 @@ class PDFL1Parser:
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
for token in self.lexer.iter_tokens():
|
||||
if is_primitive(token):
|
||||
cur_ctx = self.current_context
|
||||
obj = PDFPrimivite(token) # type: ignore
|
||||
if cur_ctx is None:
|
||||
yield obj
|
||||
else:
|
||||
cur_ctx.push(obj)
|
||||
elif isinstance(token, TokenArray):
|
||||
obj = self.__deal_with_obj(token, PDFArray, TArrayValue.OPEN)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
elif isinstance(token, TokenDictionary):
|
||||
obj = self.__deal_with_obj(token, PDFDictionary, TDictValue.OPEN)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
object_list: List[PDFObject] = []
|
||||
ctx_list: List[PDFContextualObject] = []
|
||||
for obj in self.l1.iter_objects():
|
||||
if isinstance(obj, PDFKeyword):
|
||||
if obj.get_value() == b"obj":
|
||||
if len(object_list) < 2:
|
||||
raise ParserError("recieved 'obj' but missing leading tokens")
|
||||
ctx_list.append(
|
||||
self.inobjects.create_indobject(
|
||||
*_get_indobj_values(object_list)
|
||||
)
|
||||
)
|
||||
elif obj.get_value() == b"endobj":
|
||||
if len(ctx_list) > 0:
|
||||
last_ctx_obj = ctx_list.pop()
|
||||
if len(ctx_list) > 0:
|
||||
ctx_list[-1].pass_item(last_ctx_obj)
|
||||
else:
|
||||
yield last_ctx_obj.get_value()
|
||||
elif obj.get_value() == b"R":
|
||||
cur_indobj = self.inobjects.get_indobject(
|
||||
*_get_indobj_values(object_list)
|
||||
)
|
||||
if len(ctx_list) > 0:
|
||||
ctx_list[-1].pass_item(cur_indobj)
|
||||
else:
|
||||
yield cur_indobj.get_value()
|
||||
elif len(ctx_list) == 0:
|
||||
object_list.append(obj)
|
||||
elif len(ctx_list) > 0:
|
||||
ctx_list[-1].pass_item(obj)
|
||||
else:
|
||||
raise ParserError(f"Invalid token provided {token}")
|
||||
raise ParserError(f"Invalid obj provided {obj}")
|
||||
|
||||
|
||||
def _get_indobj_values(object_list: List[PDFObject]) -> Tuple[int, int]:
|
||||
gen_num = object_list.pop()
|
||||
obj_num = object_list.pop()
|
||||
if isinstance(gen_num, PDFPrimitive) and isinstance(obj_num, PDFPrimitive):
|
||||
if isinstance(gen_num.token, TokenInteger) and isinstance(
|
||||
obj_num.token, TokenInteger
|
||||
):
|
||||
return (obj_num.to_python(), gen_num.to_python())
|
||||
raise ParserError("missing leading tokens for object reference")
|
||||
@@ -3,8 +3,14 @@
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from pdfmajor.execptions import ParserError
|
||||
from pdfmajor.lexer.token import PDFName, TokenName, TokenPrimitive
|
||||
from typing import Any, Dict, Generic, List, NamedTuple, Optional, TypeVar, Union
|
||||
from pdfmajor.lexer.token import (
|
||||
PDFName,
|
||||
TokenComment,
|
||||
TokenKeyword,
|
||||
TokenName,
|
||||
TokenPrimitive,
|
||||
)
|
||||
from typing import Any, Dict, Generic, List, Optional, TypeVar
|
||||
|
||||
TToken = TypeVar("TToken", bound=TokenPrimitive)
|
||||
|
||||
@@ -22,8 +28,13 @@ class PDFObject(metaclass=ABCMeta):
|
||||
"""returns the python variation of this object"""
|
||||
raise NotImplementedError
|
||||
|
||||
def __eq__(self, o: object) -> bool:
|
||||
if not isinstance(o, self.__class__):
|
||||
return False
|
||||
return o.to_python() == self.to_python()
|
||||
|
||||
class PDFContextualObject(PDFObject, metaclass=ABCMeta):
|
||||
|
||||
class PDFContextualObject(PDFObject):
|
||||
"""An abstract class representing objects that require context for construction"""
|
||||
|
||||
@abstractmethod
|
||||
@@ -31,7 +42,7 @@ class PDFContextualObject(PDFObject, metaclass=ABCMeta):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class PDFPrimivite(Generic[TToken], PDFObject):
|
||||
class PDFPrimitive(Generic[TToken], PDFObject):
|
||||
"""A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3"""
|
||||
|
||||
def __init__(self, token: TToken) -> None:
|
||||
@@ -41,7 +52,43 @@ class PDFPrimivite(Generic[TToken], PDFObject):
|
||||
return self.token.value
|
||||
|
||||
def to_python(self):
|
||||
return self.value
|
||||
return self.get_value()
|
||||
|
||||
|
||||
class PDFKeyword(PDFObject):
|
||||
"""A class representing a PDF keyword"""
|
||||
|
||||
def __init__(self, token: TokenKeyword) -> None:
|
||||
self.token = token
|
||||
|
||||
def get_value(self):
|
||||
return self.token.value
|
||||
|
||||
def to_python(self):
|
||||
return self.get_value()
|
||||
|
||||
|
||||
class PDFComment(PDFObject):
|
||||
"""A class representing a PDF comment"""
|
||||
|
||||
def __init__(self, token: TokenComment) -> None:
|
||||
self.token = token
|
||||
|
||||
def get_value(self):
|
||||
return self.token.value
|
||||
|
||||
def to_python(self):
|
||||
return self.get_value()
|
||||
|
||||
|
||||
class PDFNull(PDFObject):
|
||||
"""A class representing a PDF null object"""
|
||||
|
||||
def get_value(self):
|
||||
return None
|
||||
|
||||
def to_python(self):
|
||||
return None
|
||||
|
||||
|
||||
class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject):
|
||||
@@ -54,7 +101,7 @@ class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject):
|
||||
|
||||
def pass_item(self, item: PDFObject):
|
||||
if self.name is None:
|
||||
if isinstance(item, PDFPrimivite) and isinstance(item.token, TokenName):
|
||||
if isinstance(item, PDFPrimitive) and isinstance(item.token, TokenName):
|
||||
self.name = item.get_value()
|
||||
if self.strict:
|
||||
raise ParserError("Invalid token provided to dictionary")
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
from pdfmajor.parser_v2.indirect_objects import IndirectObject
|
||||
from pdfmajor.parser_v2.l2 import PDFL2Parser
|
||||
from pdfmajor.parser_v2.objects import PDFName
|
||||
from pdfmajor.parser_v2.l1 import PDFL1Parser
|
||||
from pdfmajor.streambuffer import StreamEOF
|
||||
from unittest import TestCase
|
||||
|
||||
|
||||
@@ -70,3 +73,31 @@ class L1(TestCase):
|
||||
],
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class L2(TestCase):
|
||||
def test_parse_simple_indobj(self):
|
||||
parser = PDFL2Parser(
|
||||
io.BytesIO(
|
||||
br"""
|
||||
12 1 obj (Bring) endobj
|
||||
100 0 obj
|
||||
<33>
|
||||
endobj
|
||||
13 8
|
||||
obj
|
||||
<< /x 1 /y 3>> endobj
|
||||
"""
|
||||
)
|
||||
)
|
||||
expected = [
|
||||
(12, 1, "Bring"),
|
||||
(100, 0, b"3"),
|
||||
(13, 8, {PDFName("x"): 1, PDFName("y"): 3}),
|
||||
]
|
||||
for obj, eobj in zip(parser.iter_objects(), expected):
|
||||
self.assertIsInstance(obj, IndirectObject)
|
||||
self.assertEqual(obj, parser.inobjects.get_indobject(eobj[0], eobj[1]))
|
||||
self.assertEqual(obj.to_python(), eobj[2])
|
||||
with self.assertRaises(StreamEOF):
|
||||
next(parser.iter_objects())
|
||||
Reference in new issue
Block a user