new attempt for parser

- now support clear indirect objects
This commit is contained in:
ari committed 2020-10-19 15:00:31 -04:00
1 parent b802f782fa
commit 07244aaf11
18 files changed
+606 -514

No files matched your search

+2 -2
View File
@@ -6,7 +6,7 @@ from pdfmajor.lexer.exceptions import (
)
from pdfmajor.streambuffer import BufferStream
from pdfmajor.lexer.token import PDFName, TokenName
from pdfmajor.lexer.token import TokenName
from pdfmajor.lexer.regex import HEX, END_LITERAL
@@ -62,6 +62,6 @@ def parse_name(buffer: BufferStream) -> TokenName:
return TokenName(
initialpos,
buffer.tell(),
PDFName(state.curtoken.decode()),
state.curtoken.decode(),
)
raise LexerEOF
+1 -6
View File
@@ -53,12 +53,7 @@ class TokenKeyword(Token[bytes]):
pass
class PDFName(str):
def __repr__(self) -> str:
return f"/{self}"
class TokenName(Token[PDFName]):
class TokenName(Token[str]):
"""Token representing literal name tokens
Detection of this type of token can be found in PDF 1.7 spec section 7.3.5
"""
-6
View File
@@ -1,6 +0,0 @@
"""A PDF token parser
this parser parses the bytes in three steps
1. Generate tokens from the byte stream using the PDFLexer
2. Level 1 Parsing: Match Low level objects such as primitive types, arrays, dictionaries
3. Level 2 Parsing: Match high level objects such as Streams and Indirect objects
"""
-136
View File
@@ -1,136 +0,0 @@
import io
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFComment,
PDFContextualObject,
PDFDictionary,
PDFNull,
PDFPrimitive,
PDFObject,
)
from typing import Iterator, List, Optional, Type, Union
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenComment,
TokenDictionary,
TokenComplexType,
TokenComplexTypeVal,
TokenKeyword,
TokenNull,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
def _deal_with_obj(
context_stack: List[PDFContextualObject],
token: TokenComplexType,
cls_const: Type[PDFContextualObject],
open_t: TokenComplexTypeVal,
strict: bool = False,
) -> Optional[PDFObject]:
"""generic method for processing things that have a start and end token
Args:
token (TokenComplexType)
cls_const (Any): the class constructor for the object
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
Returns:
Optional[PDFObject]
"""
if token.value == open_t:
context_stack.append(cls_const())
elif len(context_stack) > 0:
last_ctx = context_stack.pop()
if isinstance(last_ctx, cls_const):
cur_ctx = _get_curr_context(context_stack)
if cur_ctx is None:
return last_ctx
else:
cur_ctx.pass_item(last_ctx)
elif strict:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif strict:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
return None
def _get_curr_context(
context_stack: List[PDFContextualObject],
) -> Optional[PDFContextualObject]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFContextualObject]
"""
if len(context_stack) > 0:
return context_stack[-1]
else:
return None
class PDFL1Parser:
"""Level 1 PDF Parser"""
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[Union[PDFObject, TokenKeyword]]:
"""Iterates over the objects in the current byte stream using level 1 parsing
Yields:
PDFObject
"""
context_stack: List[PDFContextualObject] = []
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = _get_curr_context(context_stack)
obj = PDFPrimitive(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.pass_item(obj)
elif isinstance(token, TokenComment):
yield PDFComment(token.value, (token.start_loc, token.end_loc))
elif isinstance(token, TokenKeyword):
yield token
elif isinstance(token, TokenNull):
yield PDFNull()
elif isinstance(token, TokenArray):
obj = _deal_with_obj(
context_stack, token, PDFArray, TArrayValue.OPEN, strict=self.strict
)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
obj = _deal_with_obj(
context_stack=context_stack,
token=token,
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
strict=self.strict,
)
if obj is not None:
yield obj
else:
raise ParserError(f"Invalid token provided {token}")
-90
View File
@@ -1,90 +0,0 @@
import io
from pdfmajor.parser_v2.l1 import PDFL1Parser
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFComment,
PDFContextualObject,
PDFPrimitive,
PDFObject,
)
from typing import Iterator, List, Tuple
from pdfmajor.lexer.token import (
TokenInteger,
TokenKeyword,
)
from pdfmajor.parser_v2.indirect_objects import IndirectObjectCollection
class PDFL2Parser:
"""Level 2 PDF Parser"""
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.l1 = PDFL1Parser(fp, buffer_size, strict)
self.strict = strict
self.inobjects = IndirectObjectCollection()
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.l1.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 2 parsing
Yields:
PDFObject
"""
object_list: List[PDFObject] = []
ctx_list: List[PDFContextualObject] = []
for obj in self.l1.iter_objects():
if isinstance(obj, TokenKeyword):
if obj.value == b"obj":
if len(object_list) < 2:
raise ParserError("recieved 'obj' but missing leading tokens")
ctx_list.append(
self.inobjects.create_indobject(
*_get_indobj_values(object_list)
)
)
elif obj.value == b"endobj":
if len(ctx_list) > 0:
last_ctx_obj = ctx_list.pop()
if len(ctx_list) > 0:
ctx_list[-1].pass_item(last_ctx_obj)
else:
yield last_ctx_obj.get_value()
elif obj.value == b"R":
cur_indobj = self.inobjects.get_indobject(
*_get_indobj_values(object_list)
)
if len(ctx_list) > 0:
ctx_list[-1].pass_item(cur_indobj)
else:
yield cur_indobj.get_value()
elif isinstance(obj, PDFComment):
yield obj
elif len(ctx_list) == 0:
object_list.append(obj)
elif len(ctx_list) > 0:
ctx_list[-1].pass_item(obj)
else:
raise ParserError(f"Invalid obj provided {obj}")
def _get_indobj_values(object_list: List[PDFObject]) -> Tuple[int, int]:
gen_num = object_list.pop()
obj_num = object_list.pop()
if isinstance(gen_num, PDFPrimitive) and isinstance(obj_num, PDFPrimitive):
if isinstance(gen_num.token, TokenInteger) and isinstance(
obj_num.token, TokenInteger
):
return (obj_num.to_python(), gen_num.to_python())
raise ParserError("missing leading tokens for object reference")
-177
View File
@@ -1,177 +0,0 @@
# (Boolean values, Integer and Real numbers, Strings, Names), Arrays, Dictionaries, Streams, and the null object
from abc import ABCMeta, abstractmethod
from pdfmajor.execptions import ParserError
from pdfmajor.lexer.token import (
PDFName,
TokenComment,
TokenKeyword,
TokenName,
TokenPrimitive,
)
from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar
TToken = TypeVar("TToken", bound=TokenPrimitive)
class PDFObject(metaclass=ABCMeta):
"""A common abstract class that other PDF Objects will extend"""
@abstractmethod
def get_value(self):
"""return the base value of this object"""
return self
@abstractmethod
def to_python(self):
"""returns the python variation of this object"""
raise NotImplementedError
def __eq__(self, o: object) -> bool:
if not isinstance(o, self.__class__):
return False
return o.to_python() == self.to_python()
def __repr__(self) -> str:
return "{cls}({data})".format(
cls=self.__class__.__name__, data=self.to_python()
)
class PDFContextualObject(PDFObject):
"""An abstract class representing objects that require context for construction"""
@abstractmethod
def pass_item(self, item: PDFObject):
raise NotImplementedError
class PDFPrimitive(Generic[TToken], PDFObject):
"""A class representing any simple primitive data type as specified in PDF spec 1.7 section 7.3"""
def __init__(self, token: TToken) -> None:
self.token: TToken = token
def get_value(self):
return self.token.value
def to_python(self):
return self.get_value()
class PDFComment(PDFObject):
"""A class representing a PDF comment"""
def __init__(self, comment: bytes, loc: Tuple[int, int] = (0, 0)) -> None:
"""
Args:
comment (bytes): the comment data
loc (Tuple[int, int], optional): location of the comment in the byte-stream. Defaults to (0, 0).
"""
self.comment = comment
self.location = loc
def get_value(self):
return self.comment
def to_python(self) -> Tuple[bytes, Tuple[int, int]]:
"""returns the value of the comment and where it is in the byte-strea,
Returns:
Tuple[bytes, Tuple[int, int]]: the first value in the tuple is the comment value and the second is the start and end location in the byte stream
"""
return (self.get_value(), self.location)
class PDFNull(PDFObject):
"""A class representing a PDF null object"""
def get_value(self):
return None
def to_python(self):
return None
class PDFDictionary(Dict[PDFName, PDFObject], PDFContextualObject):
"""A class representing a PDF Dictionary as specified in PDF spec 1.7 section 7.3.7"""
def __init__(self, strict: bool = False) -> None:
super().__init__()
self.name: Optional[PDFName] = None
self.strict = strict
def pass_item(self, item: PDFObject):
if self.name is None:
if isinstance(item, PDFPrimitive) and isinstance(item.token, TokenName):
self.name = item.get_value()
if self.strict:
raise ParserError("Invalid token provided to dictionary")
else:
self[self.name] = item.get_value()
self.name = None
def to_python(self) -> Dict[PDFName, Any]:
return dict(self)
class PDFArray(List[PDFObject], PDFContextualObject):
"""A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6"""
def pass_item(self, item: PDFObject):
self.append(item.get_value())
def to_python(self) -> List[Any]:
return list(self)
class PDFStream(PDFObject):
"""A class representing a PDF Stream as specified in PDF spec 1.7 section 7.3.8"""
def __init__(
self,
offset: int,
length: int,
filter: Optional[List[PDFName]] = None,
decode_parms: Optional[List[PDFDictionary]] = None,
f: Optional[Any] = None, # file-specifications
ffilter: Optional[List[PDFName]] = None,
fdecode_parms: Optional[List[PDFDictionary]] = None,
dl: Optional[int] = None,
) -> None:
"""
Args:
offset (int): the offset where the stream itself starts
length (int): the length of the stream
filter (Optional[List[PDFName]], optional): The names of a filter that shall be applied in processing the stream data. Defaults to None.
decode_parms (Optional[List[PDFDictionary]], optional): Parameter dictionaries used by the filters specified by filter. Defaults to None.
f (Optional[Any], optional): The file containing the stream data. If this entry is present, the bytes between stream and endstream shall be ignored. However, the Length entry should still specify the number of those bytes (usually, there are no bytes and Length is 0).. Defaults to None.
ffilter (Optional[List[PDFName]], optional): The names of a filter that shall be applied in processing of the external file. Defaults to None.
fdecode_parms (Optional[List[PDFDictionary]], optional): Parameter dictionaries used by the filters specified by ffilter. Defaults to None.
dl (Optional[int], optional): A non-negative integer representing the number of bytes in the decoded (defiltered) stream. It can be used to determine, for example, whether enough disk space is available to write a stream to a file.This value shall be considered a hint only; for some stream filters, it may not be possible to determine this value precisely. Defaults to None.
"""
self.offset = offset
self.length = length
self.filter = filter
self.decode_parms = decode_parms
self.f = f
self.ffilter = ffilter
self.fdecode_parms = fdecode_parms
self.dl = dl
def to_python(self):
return {
"offset": self.offset,
"length": self.length,
"filter": self.filter,
"decode_parms": None
if self.decode_parms is None
else [x.to_python() for x in self.decode_parms],
"f": self.f,
"ffilter": self.ffilter,
"fdecode_parms": None
if self.fdecode_parms is None
else [x.to_python() for x in self.fdecode_parms],
"dl": self.dl,
}
+107
View File
@@ -0,0 +1,107 @@
import io
from typing import Iterator, Optional
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenComment,
TokenDictionary,
TokenKeyword,
)
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import ParserError
from .objects.base import PDFObject
from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment
from .objects.indirect import IndirectObject
class PDFParser:
"""PDF Parser"""
def __init__(
self,
fp: io.BufferedIOBase,
buffer_size: int = 4096,
state: Optional[ParsingState] = None,
strict: bool = True,
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.state = ParsingState([], []) if state is None else state
def seek(self, offset: int, state: Optional[ParsingState] = None) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
self.state = ParsingState([], []) if state is None else state
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 1 parsing
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
attempt_prim = attempt_parse_prim(token, self.state)
if attempt_prim.success:
cur_ctx = self.state.current_context
for obj in attempt_prim.next_objs:
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
elif isinstance(token, TokenComment):
yield self.state.set_last_obj(
PDFComment(
token.value,
(token.start_loc, token.end_loc),
)
)
elif isinstance(token, TokenKeyword):
if token.value == b"R":
obj = self.state.get_indobjects()
cur_ctx = self.state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
elif token.value == b"obj":
self.state.initialize_indirect_obj()
elif token.value == b"endobj":
last_ctx = self.state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
yield last_ctx.clone()
elif self.strict:
raise ParserError(f"Invalid position for 'endobj' {token}")
elif isinstance(token, TokenArray):
for obj in self.state.flush_int_collections():
yield obj
obj = deal_with_collection_object(
self.state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
for obj in self.state.flush_int_collections():
yield obj
obj = deal_with_collection_object(
self.state,
token=token,
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
strict=self.strict,
)
if obj is not None:
yield obj
File renamed without changes.
File renamed without changes.
+32
View File
@@ -0,0 +1,32 @@
# (Boolean values, Integer and Real numbers, Strings, Names), Arrays, Dictionaries, Streams, and the null object
from abc import ABCMeta, abstractmethod
from typing import Any
class PDFObject(metaclass=ABCMeta):
"""A common abstract class that other PDF Objects will extend"""
@abstractmethod
def to_python(self) -> Any:
"""returns the python variation of this object"""
raise NotImplementedError
def __eq__(self, o: object) -> bool:
if not isinstance(o, self.__class__):
return False
return o.to_python() == self.to_python()
def __repr__(self) -> str:
return "{cls}({data})".format(
cls=self.__class__.__name__, data=self.to_python()
)
class PDFContextualObject(PDFObject):
"""An abstract class representing objects that require context for construction"""
@abstractmethod
def pass_item(self, item: PDFObject) -> None:
raise NotImplementedError
+44
View File
@@ -0,0 +1,44 @@
from typing import Any, Dict, List, Optional
from ..exceptions import ParserError
from .base import PDFContextualObject, PDFObject
from .primitives import PDFName
class PDFArray(List[PDFObject], PDFContextualObject):
"""A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6"""
def pass_item(self, item: PDFObject):
self.append(item)
def to_python(self) -> List[Any]:
return [x.to_python() for x in self]
class PDFDictionary(Dict[str, PDFObject], PDFContextualObject):
"""A class representing a PDF Dictionary as specified in PDF spec 1.7 section 7.3.7"""
def __init__(self, strict: bool = False) -> None:
super().__init__()
self.name: Optional[PDFName] = None
self.last_name: Optional[PDFName] = None
self.strict = strict
@property
def last_item_value(self):
return self[self.last_name]
def pass_item(self, item: PDFObject):
if self.name is None:
if isinstance(item, PDFName):
self.name = item
elif self.strict:
raise ParserError(
f"Invalid token provided to dictionary as a key {item}"
)
else:
self[self.name.value] = item
self.last_name = self.name
self.name = None
def to_python(self) -> Dict[PDFName, Any]:
return {k: v.to_python() for k, v in self.items()}
+31
View File
@@ -0,0 +1,31 @@
from typing import Optional, Tuple
from .base import PDFObject
class PDFComment(PDFObject):
"""A class representing a PDF comment"""
def __init__(
self,
comment: bytes,
loc: Tuple[int, int] = (0, 0),
last_obj: Optional[PDFObject] = None,
next_obj: Optional[PDFObject] = None,
) -> None:
"""
Args:
comment (bytes): the comment data
loc (Tuple[int, int], optional): location of the comment in the byte-stream. Defaults to (0, 0).
"""
self.comment = comment
self.location = loc
self.last_obj = last_obj
self.next_obj = next_obj
def to_python(self) -> Tuple[bytes, Tuple[int, int]]:
"""returns the value of the comment and where it is in the byte-strea,
Returns:
Tuple[bytes, Tuple[int, int]]: the first value in the tuple is the comment value and the second is the start and end location in the byte stream
"""
return (self.comment, self.location)
@@ -1,7 +1,7 @@
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.lexer.token import TokenNull
from typing import Dict, Tuple
from .objects import PDFContextualObject, PDFNull, PDFObject, PDFPrimitive
from ..exceptions import ParserError
from .base import PDFContextualObject, PDFObject
from .primitives import PDFNull
class IndirectObjectCollection:
@@ -68,7 +68,7 @@ class IndirectObject(PDFContextualObject):
)
self.__col.objs[(self.obj_num, self.gen_num)] = obj
def get_value(self):
def clone(self):
return IndirectObject(self.__col, self.obj_num, self.gen_num, True)
def to_python(self):
+85
View File
@@ -0,0 +1,85 @@
from decimal import Decimal
from typing import Dict, Generic, Optional, Type, TypeVar
from pdfmajor.lexer.token import (
Token,
TokenBoolean,
TokenHexString,
TokenInteger,
TokenName,
TokenNull,
TokenReal,
TokenString,
)
from .base import PDFObject
T = TypeVar("T", str, bytes, int, bool, Decimal)
class PDFPrimitiveObject(PDFObject, Generic[T]):
def __init__(self, value: T) -> None:
self.value: T = value
def to_python(self) -> T:
return self.value
class PDFString(PDFPrimitiveObject[str]):
pass
class PDFName(PDFPrimitiveObject[str]):
def to_python(self):
return "/" + self.value
class PDFBoolean(PDFPrimitiveObject[bool]):
pass
class PDFHexString(PDFPrimitiveObject[bytes]):
pass
class PDFInteger(PDFPrimitiveObject[int]):
pass
class PDFReal(PDFPrimitiveObject[Decimal]):
pass
class PDFNull(PDFObject):
"""A class representing a PDF null object"""
def to_python(self):
return None
_token_to_obj_map: Dict[Type[Token], Type[PDFPrimitiveObject]] = {
TokenName: PDFName,
TokenString: PDFString,
TokenBoolean: PDFBoolean,
TokenHexString: PDFHexString,
TokenInteger: PDFInteger,
TokenReal: PDFReal,
}
def get_obj_from_token_primitive(token: Token) -> Optional[PDFObject]:
"""converts tokens to their primitive object counterparts
Args:
token (Token)
Returns:
Optional[PDFObject]: None if could not match them
"""
obj_const = _token_to_obj_map.get(type(token), None)
if obj_const:
return obj_const(token.value)
elif isinstance(token, TokenNull):
return PDFNull()
else:
return None
+80
View File
@@ -0,0 +1,80 @@
from typing import List, Optional, NamedTuple, Type
from pdfmajor.lexer.token import Token, TokenComplexType, TokenComplexTypeVal
from .state import ParsingState
from .exceptions import ParserError
from .objects.base import PDFObject
from .objects.base import PDFContextualObject, PDFObject
from .objects.primitives import PDFInteger, get_obj_from_token_primitive
class ParsingAttempt(NamedTuple):
success: bool
next_objs: List[PDFObject]
def attempt_parse_prim(token: Token, state: ParsingState) -> ParsingAttempt:
"""Attempt to parse out a primitive value from the token
Args:
token (Token): [description]
state (ParsingState): [description]
Returns:
ParsingAttempt
"""
prim_obj = get_obj_from_token_primitive(token)
if prim_obj is not None:
out = []
state.set_last_obj(prim_obj)
if isinstance(prim_obj, PDFInteger):
if len(state.int_collection) <= 1:
state.int_collection.append(prim_obj)
else:
for last_num in state.int_collection:
out.append(last_num)
state.int_collection = []
out.append(prim_obj)
else:
for obj in state.flush_int_collections():
out.append(obj)
out.append(prim_obj)
return ParsingAttempt(True, out)
return ParsingAttempt(False, [])
def deal_with_collection_object(
state: ParsingState,
token: TokenComplexType,
cls_const: Type[PDFContextualObject],
open_t: TokenComplexTypeVal,
strict: bool = False,
) -> Optional[PDFObject]:
"""generic method for processing things that have a start and end token
Args:
token (TokenComplexType)
cls_const (Any): the class constructor for the object
open_t (TokenComplexTypeVal): the value represnting 'open' for the object
Returns:
Optional[PDFObject]
"""
if token.value == open_t:
state.context_stack.append(cls_const())
elif len(state.context_stack) > 0:
last_ctx = state.context_stack.pop()
if isinstance(last_ctx, cls_const):
cur_ctx = state.current_context
if cur_ctx is None:
return last_ctx
else:
cur_ctx.pass_item(last_ctx)
elif strict:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif strict:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
return None
+110
View File
@@ -0,0 +1,110 @@
from typing import Iterator, List, Optional, Tuple
from .exceptions import ParserError
from .objects.base import PDFContextualObject, PDFObject
from .objects.primitives import PDFInteger
from .objects.comment import PDFComment
from .objects.indirect import IndirectObject, IndirectObjectCollection
class ParsingState:
__slots__ = [
"context_stack",
"int_collection",
"last_obj",
"indirect_object_collection",
]
def __init__(
self,
context_stack: List[PDFContextualObject],
int_collection: List[PDFInteger],
last_obj: Optional[PDFObject] = None,
indirect_object_collection: Optional[IndirectObjectCollection] = None,
) -> None:
self.context_stack = context_stack
self.int_collection = int_collection
self.int_collection = int_collection
self.last_obj = last_obj
self.indirect_object_collection = (
indirect_object_collection
if indirect_object_collection is not None
else IndirectObjectCollection()
)
@property
def current_context(self) -> Optional[PDFContextualObject]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFContextualObject]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def set_last_obj(self, new_obj: PDFObject) -> PDFObject:
"""Set the value for the 'last object', special consideration for comments
Args:
new_obj (PDFObject)
Returns:
PDFObject
"""
if isinstance(self.last_obj, PDFComment):
self.last_obj.next_obj = new_obj
elif isinstance(new_obj, PDFComment):
new_obj.last_obj = self.last_obj
self.last_obj = new_obj
return new_obj
def get_indobj_values(self) -> Tuple[int, int]:
"""Attempts to convert the values in self.int_collection to a tuple representing the generation and object number of an indirect object
Raises:
ParserError: if the int_collection does not have two elements
Returns:
Tuple[int, int]: (obj_num, gen_num)
"""
if len(self.int_collection) == 2:
gen_num = self.int_collection.pop()
obj_num = self.int_collection.pop()
return (obj_num.to_python(), gen_num.to_python())
raise ParserError("missing leading tokens for object reference")
def get_indobjects(self) -> IndirectObject:
"""Gets an indirect object, based on the values in 'self.int_collection'
Raises:
ParserError: if the int_collection does not have two elements
Returns:
IndirectObject
"""
obj_num, gen_num = self.get_indobj_values()
return self.indirect_object_collection.get_indobject(obj_num, gen_num)
def initialize_indirect_obj(self):
obj_num, gen_num = self.get_indobj_values()
self.context_stack.append(
self.indirect_object_collection.create_indobject(obj_num, gen_num)
)
def flush_int_collections(self) -> Iterator[PDFObject]:
"""clear out the int-collection
Yields:
PDFObject
"""
cur_ctx = self.current_context
if cur_ctx is not None:
for obj in self.int_collection:
cur_ctx.pass_item(obj)
else:
for obj in self.int_collection:
yield obj
self.int_collection = []
+21 -23
View File
@@ -5,7 +5,6 @@ from typing import List
from unittest import TestCase
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.exceptions import LexerError
from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
from pdfmajor.lexer.string import parse_string
@@ -13,11 +12,9 @@ from pdfmajor.lexer.name import parse_name
from pdfmajor.lexer.keyword import parse_keyword
from pdfmajor.streambuffer import BufferStream
from pdfmajor.lexer.token import (
PDFName,
TArrayValue,
TDictValue,
TArrayValue,
Token,
TokenArray,
TokenBoolean,
TokenComment,
TokenNull,
@@ -27,6 +24,7 @@ from pdfmajor.lexer.token import (
TokenInteger,
TokenKeyword,
TokenName,
TokenArray,
TokenString,
)
from pdfmajor.lexer.comment import parse_comment
@@ -114,7 +112,7 @@ class Basics(TestCase):
token = parse_name(buffer)
self.assertIsInstance(token, TokenName)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, PDFName("Some_Name"))
self.assertEqual(token.value, "Some_Name")
self.assertEqual(token.end_loc, 9)
def test_parse_literal_names_hex(self):
@@ -124,7 +122,7 @@ class Basics(TestCase):
token = parse_name(buffer)
self.assertIsInstance(token, TokenName)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, PDFName("foo_aa"))
self.assertEqual(token.value, "foo_aa")
self.assertEqual(token.end_loc, 8)
def test_parse_string(self):
@@ -246,7 +244,7 @@ class Lexer(TestCase):
self.run_test(
b"/Length 4 0 R\n",
expected=[
TokenName(0, 7, PDFName("Length")),
TokenName(0, 7, "Length"),
TokenInteger(8, 9, 4),
TokenInteger(10, 11, 0),
TokenKeyword(12, 13, b"R"),
@@ -299,10 +297,10 @@ func/a/b{(c)do*}def
TokenKeyword(start_loc=16, end_loc=17, value=b'"'),
TokenKeyword(start_loc=19, end_loc=20, value=b"@"),
TokenKeyword(start_loc=21, end_loc=22, value=b"#"),
TokenName(start_loc=23, end_loc=25, value=PDFName("a")),
TokenName(start_loc=25, end_loc=29, value=PDFName("BCD")),
TokenName(start_loc=30, end_loc=40, value=PDFName("Some_Name")),
TokenName(start_loc=41, end_loc=53, value=PDFName("foo_naa")),
TokenName(start_loc=23, end_loc=25, value="a"),
TokenName(start_loc=25, end_loc=29, value="BCD"),
TokenName(start_loc=30, end_loc=40, value="Some_Name"),
TokenName(start_loc=41, end_loc=53, value="foo_naa"),
TokenInteger(start_loc=54, end_loc=55, value=0),
TokenInteger(start_loc=56, end_loc=58, value=1),
TokenInteger(start_loc=59, end_loc=61, value=-2),
@@ -326,8 +324,8 @@ func/a/b{(c)do*}def
start_loc=211, end_loc=225, value=b"\xab\xcd\x00\x124\x05"
),
TokenKeyword(start_loc=226, end_loc=230, value=b"func"),
TokenName(start_loc=230, end_loc=232, value=PDFName("a")),
TokenName(start_loc=232, end_loc=234, value=PDFName("b")),
TokenName(start_loc=230, end_loc=232, value="a"),
TokenName(start_loc=232, end_loc=234, value="b"),
TokenKeyword(start_loc=234, end_loc=235, value=b"{"),
TokenString(start_loc=235, end_loc=238, value="c"),
TokenKeyword(start_loc=238, end_loc=241, value=b"do*"),
@@ -339,7 +337,7 @@ func/a/b{(c)do*}def
TokenKeyword(start_loc=254, end_loc=255, value=b"!"),
TokenArray(start_loc=256, end_loc=257, value=TArrayValue.CLOSE),
TokenDictionary(start_loc=258, end_loc=260, value=TDictValue.OPEN),
TokenName(start_loc=261, end_loc=265, value=PDFName("foo")),
TokenName(start_loc=261, end_loc=265, value="foo"),
TokenString(start_loc=266, end_loc=271, value="bar"),
TokenDictionary(start_loc=272, end_loc=274, value=TDictValue.CLOSE),
],
@@ -382,15 +380,15 @@ func/a/b
TokenKeyword(start_loc=44, end_loc=45, value=b'"'),
TokenKeyword(start_loc=47, end_loc=48, value=b"@"),
TokenKeyword(start_loc=49, end_loc=50, value=b"#"),
TokenName(start_loc=51, end_loc=53, value=PDFName("a")),
TokenName(start_loc=53, end_loc=57, value=PDFName("BCD")),
TokenName(start_loc=58, end_loc=68, value=PDFName("Some_Name")),
TokenName(start_loc=69, end_loc=81, value=PDFName("foo_naa")),
TokenName(start_loc=82, end_loc=89, value=PDFName("barj")),
TokenName(start_loc=90, end_loc=95, value=PDFName("need")),
TokenName(start_loc=96, end_loc=102, value=PDFName("oh1oh")),
TokenName(start_loc=51, end_loc=53, value="a"),
TokenName(start_loc=53, end_loc=57, value="BCD"),
TokenName(start_loc=58, end_loc=68, value="Some_Name"),
TokenName(start_loc=69, end_loc=81, value="foo_naa"),
TokenName(start_loc=82, end_loc=89, value="barj"),
TokenName(start_loc=90, end_loc=95, value="need"),
TokenName(start_loc=96, end_loc=102, value="oh1oh"),
TokenKeyword(start_loc=103, end_loc=107, value=b"func"),
TokenName(start_loc=107, end_loc=109, value=PDFName("a")),
TokenName(start_loc=109, end_loc=111, value=PDFName("b")),
TokenName(start_loc=107, end_loc=109, value="a"),
TokenName(start_loc=109, end_loc=111, value="b"),
],
)
+89 -70
View File
@@ -1,18 +1,19 @@
from decimal import Decimal
import io
from typing import Callable, List
from pdfmajor.lexer.token import TokenInteger, TokenKeyword, TokenString
from pdfmajor.parser_v2.indirect_objects import IndirectObject, IndirectObjectCollection
from pdfmajor.parser_v2.l2 import PDFL2Parser
from pdfmajor.parser_v2.objects import PDFComment, PDFName, PDFObject, PDFPrimitive
from pdfmajor.parser_v2.l1 import PDFL1Parser
from pdfmajor.streambuffer import StreamEOF
from decimal import Decimal
from unittest import TestCase
from pdfmajor.streambuffer import StreamEOF
from pdfmajor.parser_v3.objects.indirect import IndirectObject, IndirectObjectCollection
from pdfmajor.parser_v3.objects.base import PDFObject
from pdfmajor.parser_v3 import PDFParser
from pdfmajor.parser_v3.objects.comment import PDFComment
class L1(TestCase):
class Collections(TestCase):
def test_parse_dict(self):
parser = PDFL1Parser(
parser = PDFParser(
io.BytesIO(
br"""<< /Type /Example
/Subtype /DictionaryExample
@@ -25,97 +26,65 @@ class L1(TestCase):
/LastItem ( not! )
/VeryLastItem ( OK ) >>
>>"""
)
),
strict=True,
)
obj = next(parser.iter_objects())
self.assertDictEqual(
obj,
obj.to_python(),
{
PDFName("Type"): PDFName("Example"),
PDFName("Subtype"): PDFName("DictionaryExample"),
PDFName("Version"): Decimal("0.01"),
PDFName("IntegerItem"): 12,
PDFName("StringItem"): " a string ",
PDFName("Subdictionary"): {
PDFName("Item1"): Decimal("0.4"),
PDFName("Item2"): True,
PDFName("LastItem"): " not! ",
PDFName("VeryLastItem"): " OK ",
"Type": "/Example",
"Subtype": "/DictionaryExample",
"Version": Decimal("0.01"),
"IntegerItem": 12,
"StringItem": " a string ",
"Subdictionary": {
"Item1": Decimal("0.4"),
"Item2": True,
"LastItem": " not! ",
"VeryLastItem": " OK ",
},
},
)
def test_parse_array(self):
parser = PDFL1Parser(
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]""")
parser = PDFParser(
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]"""),
strict=True,
)
obj = next(parser.iter_objects())
self.assertListEqual(
obj, [549, Decimal("3.14"), False, " Ralph ", PDFName("SomeName")]
obj.to_python(),
[549, Decimal("3.14"), False, " Ralph ", "/SomeName"],
)
def test_parse_nested_array(self):
parser = PDFL1Parser(
parser = PDFParser(
io.BytesIO(
br"""[ [10 30] [22 33] [
<< /x 0 /y 5 >>
<< /x 10 /y 5 >>
] ]"""
)
),
strict=True,
)
obj = next(parser.iter_objects())
self.assertListEqual(
obj,
obj.to_python(),
[
[10, 30],
[22, 33],
[
{PDFName("x"): 0, PDFName("y"): 5},
{PDFName("x"): 10, PDFName("y"): 5},
{"x": 0, "y": 5},
{"x": 10, "y": 5},
],
],
)
def test_parse_keeps_keywords(self):
parser = PDFL1Parser(
io.BytesIO(
br"""
12 1 obj (Bring) endobj
"""
)
)
expected = [
PDFPrimitive(TokenInteger(13, 15, 12)),
PDFPrimitive(TokenInteger(16, 17, 1)),
TokenKeyword(18, 21, b"obj"),
PDFPrimitive(TokenString(22, 29, "Bring")),
TokenKeyword(30, 36, b"endobj"),
]
for obj, eobj in zip(parser.iter_objects(), expected):
self.assertEqual(obj, eobj)
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
class L2(TestCase):
def run_test(
self,
raw: bytes,
expected: Callable[[IndirectObjectCollection], List[PDFObject]],
):
parser = PDFL2Parser(io.BytesIO(raw))
for i, (eobj, obj) in enumerate(
zip(expected(parser.inobjects), parser.iter_objects()), 1
):
self.assertEqual(obj, eobj, f"Failed at example #{i}")
try:
next_obj = next(parser.iter_objects())
self.assertIsNone(next_obj)
except (EOFError, StopIteration) as e:
self.assertIsNotNone(e)
class IndirectObjects(TestCase):
def test_parse_simple_indobj(self):
parser = PDFL2Parser(
parser = PDFParser(
io.BytesIO(
br"""
12 1 obj (Bring) endobj
@@ -126,20 +95,70 @@ class L2(TestCase):
obj
<< /x 1 /y 3>> endobj
"""
)
),
strict=True,
)
expected = [
(12, 1, "Bring"),
(100, 0, b"3"),
(13, 8, {PDFName("x"): 1, PDFName("y"): 3}),
(13, 8, {"x": 1, "y": 3}),
]
for obj, eobj in zip(parser.iter_objects(), expected):
self.assertIsInstance(obj, IndirectObject)
self.assertEqual(obj, parser.inobjects.get_indobject(eobj[0], eobj[1]))
self.assertEqual(
obj,
parser.state.indirect_object_collection.get_indobject(eobj[0], eobj[1]),
)
self.assertEqual(obj.to_python(), eobj[2])
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
def test_parse_obj_with_references(self):
parser = PDFParser(
io.BytesIO(
br"""
12 1 obj (Bring) endobj
10 2 obj << /lastone 12 1 R >> endobj
"""
),
strict=True,
)
expected = [
parser.state.indirect_object_collection.get_indobject(12, 1),
parser.state.indirect_object_collection.get_indobject(10, 2),
]
for obj, exp in zip(parser.iter_objects(), expected):
self.assertEqual(obj, exp)
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
sec_obj = parser.state.indirect_object_collection.get_indobject(
10, 2
).get_object()
self.assertEqual(
sec_obj.to_python(),
{"lastone": "Bring"},
)
def run_test(
self,
raw: bytes,
expected: Callable[[IndirectObjectCollection], List[PDFObject]],
):
parser = PDFParser(io.BytesIO(raw), strict=True)
for i, (eobj, obj) in enumerate(
zip(
expected(parser.state.indirect_object_collection), parser.iter_objects()
),
1,
):
self.assertEqual(obj, eobj, f"Failed at example #{i}")
try:
next_obj = next(parser.iter_objects())
self.assertIsNone(next_obj)
except (EOFError, StopIteration) as e:
self.assertIsNotNone(e)
def test_parse_comments(self):
self.run_test(
br"""