succesful dictionary parsing

This commit is contained in:
ari committed 2020-10-17 21:41:33 -04:00
1 parent 7ee8254edb
commit 01ace1c57d
9 files changed
+287 -7

No files matched your search

+1 -1
View File
@@ -70,7 +70,7 @@ class PDFLexer:
yield parse_double_angled_bracket(self.buffer)
elif next_char == b">":
self.buffer.seek(bufpos + j + 1)
if buf[j : j + 2] == ">>":
if buf[j : j + 2] == b">>":
self.buffer.seekd(1)
yield TokenDictionary(
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
+3 -5
View File
@@ -1,13 +1,11 @@
from pdfmajor.streambuffer import BufferStream
from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword, TokenNull
from pdfmajor.lexer.token import Token, TokenBoolean, TokenKeyword, TokenNull
from pdfmajor.lexer.regex import END_KEYWORD
from typing import Optional, Union
def parse_keyword(
buffer: BufferStream, initialpos: Optional[int] = None
) -> Union[TokenKeyword, TokenBoolean]:
def parse_keyword(buffer: BufferStream, initialpos: Optional[int] = None) -> Token:
"""Parses input stream into a keyword or bool
Args:
@@ -15,7 +13,7 @@ def parse_keyword(
initialpos (Optional[int])
Returns:
Union[TokenKeyword, TokenBoolean]
Token
"""
curtoken = b""
initialpos = buffer.tell() - 1 if initialpos is None else initialpos
+38
View File
@@ -36,6 +36,7 @@ class Token(Generic[T]):
return False
# Primitive Tokens
class TokenComment(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
@@ -109,8 +110,45 @@ class TokenReal(Token[Decimal]):
TokenNumber = Union[TokenInteger, TokenReal]
TokenPrimitive = Union[
TokenComment,
TokenKeyword,
TokenName,
TokenBoolean,
TokenNull,
TokenString,
TokenHexString,
TokenInteger,
TokenReal,
]
def is_primitive(token: Token) -> bool:
"""determins is a token represents a primitive pdf object such as boolean, null, integer etc
Args:
token (Token)
Returns:
bool: True if primitive
"""
for opt in [
TokenComment,
TokenKeyword,
TokenName,
TokenBoolean,
TokenNull,
TokenString,
TokenHexString,
TokenInteger,
TokenReal,
]:
if isinstance(token, opt):
return True
return False
# Tokens for complex objects
class TDictValue(Enum):
OPEN = "<<"
CLOSE = ">>"
+99
View File
@@ -0,0 +1,99 @@
import io
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import PDFArray, PDFDictionary, PDFPrimivite, PDFObject
from typing import Iterator, List, Optional
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenDictionary,
TokenPrimitive,
is_primitive,
)
from pdfmajor.lexer import PDFLexer
from pdfmajor.parser_v2.context import PDFParseContext
class PDFParser:
def __init__(
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.context_stack: List[PDFParseContext] = []
@property
def current_context(self) -> Optional[PDFParseContext]:
"""Returns the current context of the parsing state
Returns:
Optional[PDFParseContext]
"""
if len(self.context_stack) > 0:
return self.context_stack[-1]
else:
return None
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current stream
Yields:
[type]: [description]
"""
for token in self.lexer.iter_tokens():
if is_primitive(token):
cur_ctx = self.current_context
obj = PDFPrimivite(token) # type: ignore
if cur_ctx is None:
yield obj
else:
cur_ctx.push(obj)
elif isinstance(token, TokenArray):
if token.value == TArrayValue.OPEN:
self.context_stack.append(PDFParseContext(PDFArray()))
elif len(self.context_stack) > 0:
last_ctx = self.context_stack.pop()
if isinstance(last_ctx.pdfobject, PDFArray):
cur_ctx = self.current_context
if cur_ctx is None:
yield last_ctx.pdfobject
else:
cur_ctx.push(last_ctx.pdfobject)
elif self.strict:
raise ParserError(f"Invalid end of array specified at {token}")
elif self.strict:
raise ParserError(
f"Invalid end of array specified at {token} with no start"
)
elif isinstance(token, TokenDictionary):
if token.value == TDictValue.OPEN:
self.context_stack.append(PDFParseContext(PDFDictionary()))
elif len(self.context_stack) > 0:
last_ctx = self.context_stack.pop()
if isinstance(last_ctx.pdfobject, PDFDictionary):
cur_ctx = self.current_context
if cur_ctx is None:
yield last_ctx.pdfobject
else:
cur_ctx.push(last_ctx.pdfobject)
elif self.strict:
raise ParserError(
f"Invalid end of dictionary specified at {token}"
)
elif self.strict:
raise ParserError(
f"Invalid end of dictionary specified at {token} with no start"
)
else:
raise ParserError(f"Invalid token provided {token}")
+41
View File
@@ -0,0 +1,41 @@
from pdfmajor.parser_v2.exceptions import ParserError
from pdfmajor.parser_v2.objects import (
PDFArray,
PDFDictionary,
PDFObject,
PDFComplexType,
PDFStream,
)
class PDFParseContext:
"""A parsing context
This class helps constructs complex PDF types
"""
__slots__ = ["pdfobject"]
def __init__(self, pdfobject: PDFComplexType) -> None:
"""
Args:
pdfobject (PDFComplexType): the complex pdf object we are trying to build
"""
self.pdfobject = pdfobject
def push(self, item: PDFObject):
"""push any pdf object to the context, process according to the type of complex object we are trying to build
Args:
item (PDFObject)
Raises:
ParserError: raises if an invalid complex object is used
"""
if isinstance(self.pdfobject, PDFArray):
self.pdfobject.append(item)
elif isinstance(self.pdfobject, PDFDictionary):
self.pdfobject.pass_item(item)
elif isinstance(self.pdfobject, PDFStream):
raise NotImplementedError
else:
raise ParserError(f"Invalid Context {type(self.pdfobject)}")
+5
View File
@@ -0,0 +1,5 @@
from pdfmajor.execptions import PDFMajorException
class ParserError(PDFMajorException):
pass
+58
View File
@@ -0,0 +1,58 @@
# (Boolean values, Integer and Real numbers, Strings, Names), Arrays, Dictionaries, Streams, and the null object
from pdfmajor.execptions import ParserError
from pdfmajor.lexer.token import TokenName, TokenPrimitive
from typing import Dict, Generic, List, NamedTuple, Optional, TypeVar, Union
TToken = TypeVar("TToken", bound=TokenPrimitive)
class PDFName(NamedTuple):
value: str
class PDFPrimivite(Generic[TToken]):
def __init__(self, token: TToken) -> None:
self.token: TToken = token
@property
def value(self):
if isinstance(self.token, TokenName):
return PDFName(self.token.value)
return self.token.value
class PDFDictionary(Dict[PDFPrimivite[TokenName], "PDFObject"]):
def __init__(self, strict: bool = False) -> None:
super().__init__()
self.name: Optional[str] = None
self.strict = strict
def pass_item(self, item: "PDFObject"):
if self.name is None:
if isinstance(item, PDFPrimivite) and isinstance(item.token, TokenName):
self.name = item.value
if self.strict:
raise ParserError("Invalid token provided to dictionary")
else:
self[self.name] = item.value
self.name = None
@property
def value(self):
return dict(self)
class PDFArray(List["PDFObject"]):
@property
def value(self):
return self
class PDFStream(object):
pass
PDFComplexType = Union[PDFDictionary, PDFArray, PDFStream]
PDFObject = Union[PDFComplexType, PDFPrimivite]
+1 -1
View File
@@ -72,7 +72,7 @@ class BufferStream:
def __next__(self):
bbyte = self.read(self.buffer_size)
if len(bbyte.data) == 0:
raise StopIteration
raise StopIteration(f"no data {bbyte}")
return bbyte
def close(self):
+41
View File
@@ -0,0 +1,41 @@
from decimal import Decimal
import io
from pdfmajor.parser_v2.objects import PDFName
from pdfmajor.parser_v2 import PDFParser
from unittest import TestCase
class Basic(TestCase):
def test_parse_dict(self):
parser = PDFParser(
io.BytesIO(
br"""<< /Type /Example
/Subtype /DictionaryExample
/Version 0.01
/IntegerItem 12
/StringItem ( a string )
/Subdictionary <<
/Item1 0.4
/Item2 true
/LastItem ( not! )
/VeryLastItem ( OK ) >>
>>"""
)
)
obj = next(parser.iter_objects())
self.assertDictEqual(
obj,
{
PDFName("Type"): PDFName("Example"),
PDFName("Subtype"): PDFName("DictionaryExample"),
PDFName("Version"): Decimal("0.01"),
PDFName("IntegerItem"): 12,
PDFName("StringItem"): " a string ",
PDFName("Subdictionary"): {
PDFName("Item1"): Decimal("0.4"),
PDFName("Item2"): True,
PDFName("LastItem"): " not! ",
PDFName("VeryLastItem"): " OK ",
},
},
)