succesful dictionary parsing
This commit is contained in:
1 parent
7ee8254edb
commit
01ace1c57d
9 files changed
+287
-7
No files matched your search
@@ -70,7 +70,7 @@ class PDFLexer:
|
||||
yield parse_double_angled_bracket(self.buffer)
|
||||
elif next_char == b">":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
if buf[j : j + 2] == ">>":
|
||||
if buf[j : j + 2] == b">>":
|
||||
self.buffer.seekd(1)
|
||||
yield TokenDictionary(
|
||||
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.lexer.exceptions import LexerEOF
|
||||
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword, TokenNull
|
||||
from pdfmajor.lexer.token import Token, TokenBoolean, TokenKeyword, TokenNull
|
||||
from pdfmajor.lexer.regex import END_KEYWORD
|
||||
from typing import Optional, Union
|
||||
|
||||
|
||||
def parse_keyword(
|
||||
buffer: BufferStream, initialpos: Optional[int] = None
|
||||
) -> Union[TokenKeyword, TokenBoolean]:
|
||||
def parse_keyword(buffer: BufferStream, initialpos: Optional[int] = None) -> Token:
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
@@ -15,7 +13,7 @@ def parse_keyword(
|
||||
initialpos (Optional[int])
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
Token
|
||||
"""
|
||||
curtoken = b""
|
||||
initialpos = buffer.tell() - 1 if initialpos is None else initialpos
|
||||
|
||||
@@ -36,6 +36,7 @@ class Token(Generic[T]):
|
||||
return False
|
||||
|
||||
|
||||
# Primitive Tokens
|
||||
class TokenComment(Token[bytes]):
|
||||
"""Token representing PDF comments
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
|
||||
@@ -109,8 +110,45 @@ class TokenReal(Token[Decimal]):
|
||||
|
||||
|
||||
TokenNumber = Union[TokenInteger, TokenReal]
|
||||
TokenPrimitive = Union[
|
||||
TokenComment,
|
||||
TokenKeyword,
|
||||
TokenName,
|
||||
TokenBoolean,
|
||||
TokenNull,
|
||||
TokenString,
|
||||
TokenHexString,
|
||||
TokenInteger,
|
||||
TokenReal,
|
||||
]
|
||||
|
||||
|
||||
def is_primitive(token: Token) -> bool:
|
||||
"""determins is a token represents a primitive pdf object such as boolean, null, integer etc
|
||||
|
||||
Args:
|
||||
token (Token)
|
||||
|
||||
Returns:
|
||||
bool: True if primitive
|
||||
"""
|
||||
for opt in [
|
||||
TokenComment,
|
||||
TokenKeyword,
|
||||
TokenName,
|
||||
TokenBoolean,
|
||||
TokenNull,
|
||||
TokenString,
|
||||
TokenHexString,
|
||||
TokenInteger,
|
||||
TokenReal,
|
||||
]:
|
||||
if isinstance(token, opt):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# Tokens for complex objects
|
||||
class TDictValue(Enum):
|
||||
OPEN = "<<"
|
||||
CLOSE = ">>"
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
import io
|
||||
from pdfmajor.parser_v2.exceptions import ParserError
|
||||
from pdfmajor.parser_v2.objects import PDFArray, PDFDictionary, PDFPrimivite, PDFObject
|
||||
from typing import Iterator, List, Optional
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
TokenArray,
|
||||
TokenDictionary,
|
||||
TokenPrimitive,
|
||||
is_primitive,
|
||||
)
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.parser_v2.context import PDFParseContext
|
||||
|
||||
|
||||
class PDFParser:
|
||||
def __init__(
|
||||
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
|
||||
) -> None:
|
||||
self.lexer = PDFLexer(fp, buffer_size)
|
||||
self.strict = strict
|
||||
self.context_stack: List[PDFParseContext] = []
|
||||
|
||||
@property
|
||||
def current_context(self) -> Optional[PDFParseContext]:
|
||||
"""Returns the current context of the parsing state
|
||||
|
||||
Returns:
|
||||
Optional[PDFParseContext]
|
||||
"""
|
||||
if len(self.context_stack) > 0:
|
||||
return self.context_stack[-1]
|
||||
else:
|
||||
return None
|
||||
|
||||
def seek(self, offset: int) -> int:
|
||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||
|
||||
Args:
|
||||
offset (int)
|
||||
|
||||
Returns:
|
||||
int: new offset
|
||||
"""
|
||||
return self.lexer.seek(offset)
|
||||
|
||||
def iter_objects(self) -> Iterator[PDFObject]:
|
||||
"""Iterates over the objects in the current stream
|
||||
|
||||
Yields:
|
||||
[type]: [description]
|
||||
"""
|
||||
for token in self.lexer.iter_tokens():
|
||||
if is_primitive(token):
|
||||
cur_ctx = self.current_context
|
||||
obj = PDFPrimivite(token) # type: ignore
|
||||
if cur_ctx is None:
|
||||
yield obj
|
||||
else:
|
||||
cur_ctx.push(obj)
|
||||
elif isinstance(token, TokenArray):
|
||||
if token.value == TArrayValue.OPEN:
|
||||
self.context_stack.append(PDFParseContext(PDFArray()))
|
||||
elif len(self.context_stack) > 0:
|
||||
last_ctx = self.context_stack.pop()
|
||||
if isinstance(last_ctx.pdfobject, PDFArray):
|
||||
cur_ctx = self.current_context
|
||||
if cur_ctx is None:
|
||||
yield last_ctx.pdfobject
|
||||
else:
|
||||
cur_ctx.push(last_ctx.pdfobject)
|
||||
elif self.strict:
|
||||
raise ParserError(f"Invalid end of array specified at {token}")
|
||||
elif self.strict:
|
||||
raise ParserError(
|
||||
f"Invalid end of array specified at {token} with no start"
|
||||
)
|
||||
elif isinstance(token, TokenDictionary):
|
||||
if token.value == TDictValue.OPEN:
|
||||
self.context_stack.append(PDFParseContext(PDFDictionary()))
|
||||
elif len(self.context_stack) > 0:
|
||||
last_ctx = self.context_stack.pop()
|
||||
if isinstance(last_ctx.pdfobject, PDFDictionary):
|
||||
cur_ctx = self.current_context
|
||||
if cur_ctx is None:
|
||||
yield last_ctx.pdfobject
|
||||
else:
|
||||
cur_ctx.push(last_ctx.pdfobject)
|
||||
elif self.strict:
|
||||
raise ParserError(
|
||||
f"Invalid end of dictionary specified at {token}"
|
||||
)
|
||||
elif self.strict:
|
||||
raise ParserError(
|
||||
f"Invalid end of dictionary specified at {token} with no start"
|
||||
)
|
||||
else:
|
||||
raise ParserError(f"Invalid token provided {token}")
|
||||
@@ -0,0 +1,41 @@
|
||||
from pdfmajor.parser_v2.exceptions import ParserError
|
||||
from pdfmajor.parser_v2.objects import (
|
||||
PDFArray,
|
||||
PDFDictionary,
|
||||
PDFObject,
|
||||
PDFComplexType,
|
||||
PDFStream,
|
||||
)
|
||||
|
||||
|
||||
class PDFParseContext:
|
||||
"""A parsing context
|
||||
This class helps constructs complex PDF types
|
||||
"""
|
||||
|
||||
__slots__ = ["pdfobject"]
|
||||
|
||||
def __init__(self, pdfobject: PDFComplexType) -> None:
|
||||
"""
|
||||
Args:
|
||||
pdfobject (PDFComplexType): the complex pdf object we are trying to build
|
||||
"""
|
||||
self.pdfobject = pdfobject
|
||||
|
||||
def push(self, item: PDFObject):
|
||||
"""push any pdf object to the context, process according to the type of complex object we are trying to build
|
||||
|
||||
Args:
|
||||
item (PDFObject)
|
||||
|
||||
Raises:
|
||||
ParserError: raises if an invalid complex object is used
|
||||
"""
|
||||
if isinstance(self.pdfobject, PDFArray):
|
||||
self.pdfobject.append(item)
|
||||
elif isinstance(self.pdfobject, PDFDictionary):
|
||||
self.pdfobject.pass_item(item)
|
||||
elif isinstance(self.pdfobject, PDFStream):
|
||||
raise NotImplementedError
|
||||
else:
|
||||
raise ParserError(f"Invalid Context {type(self.pdfobject)}")
|
||||
@@ -0,0 +1,5 @@
|
||||
from pdfmajor.execptions import PDFMajorException
|
||||
|
||||
|
||||
class ParserError(PDFMajorException):
|
||||
pass
|
||||
@@ -0,0 +1,58 @@
|
||||
# (Boolean values, Integer and Real numbers, Strings, Names), Arrays, Dictionaries, Streams, and the null object
|
||||
|
||||
|
||||
from pdfmajor.execptions import ParserError
|
||||
from pdfmajor.lexer.token import TokenName, TokenPrimitive
|
||||
from typing import Dict, Generic, List, NamedTuple, Optional, TypeVar, Union
|
||||
|
||||
TToken = TypeVar("TToken", bound=TokenPrimitive)
|
||||
|
||||
|
||||
class PDFName(NamedTuple):
|
||||
value: str
|
||||
|
||||
|
||||
class PDFPrimivite(Generic[TToken]):
|
||||
def __init__(self, token: TToken) -> None:
|
||||
self.token: TToken = token
|
||||
|
||||
@property
|
||||
def value(self):
|
||||
if isinstance(self.token, TokenName):
|
||||
return PDFName(self.token.value)
|
||||
return self.token.value
|
||||
|
||||
|
||||
class PDFDictionary(Dict[PDFPrimivite[TokenName], "PDFObject"]):
|
||||
def __init__(self, strict: bool = False) -> None:
|
||||
super().__init__()
|
||||
self.name: Optional[str] = None
|
||||
self.strict = strict
|
||||
|
||||
def pass_item(self, item: "PDFObject"):
|
||||
if self.name is None:
|
||||
if isinstance(item, PDFPrimivite) and isinstance(item.token, TokenName):
|
||||
self.name = item.value
|
||||
if self.strict:
|
||||
raise ParserError("Invalid token provided to dictionary")
|
||||
else:
|
||||
self[self.name] = item.value
|
||||
self.name = None
|
||||
|
||||
@property
|
||||
def value(self):
|
||||
return dict(self)
|
||||
|
||||
|
||||
class PDFArray(List["PDFObject"]):
|
||||
@property
|
||||
def value(self):
|
||||
return self
|
||||
|
||||
|
||||
class PDFStream(object):
|
||||
pass
|
||||
|
||||
|
||||
PDFComplexType = Union[PDFDictionary, PDFArray, PDFStream]
|
||||
PDFObject = Union[PDFComplexType, PDFPrimivite]
|
||||
@@ -72,7 +72,7 @@ class BufferStream:
|
||||
def __next__(self):
|
||||
bbyte = self.read(self.buffer_size)
|
||||
if len(bbyte.data) == 0:
|
||||
raise StopIteration
|
||||
raise StopIteration(f"no data {bbyte}")
|
||||
return bbyte
|
||||
|
||||
def close(self):
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
from pdfmajor.parser_v2.objects import PDFName
|
||||
from pdfmajor.parser_v2 import PDFParser
|
||||
from unittest import TestCase
|
||||
|
||||
|
||||
class Basic(TestCase):
|
||||
def test_parse_dict(self):
|
||||
parser = PDFParser(
|
||||
io.BytesIO(
|
||||
br"""<< /Type /Example
|
||||
/Subtype /DictionaryExample
|
||||
/Version 0.01
|
||||
/IntegerItem 12
|
||||
/StringItem ( a string )
|
||||
/Subdictionary <<
|
||||
/Item1 0.4
|
||||
/Item2 true
|
||||
/LastItem ( not! )
|
||||
/VeryLastItem ( OK ) >>
|
||||
>>"""
|
||||
)
|
||||
)
|
||||
obj = next(parser.iter_objects())
|
||||
self.assertDictEqual(
|
||||
obj,
|
||||
{
|
||||
PDFName("Type"): PDFName("Example"),
|
||||
PDFName("Subtype"): PDFName("DictionaryExample"),
|
||||
PDFName("Version"): Decimal("0.01"),
|
||||
PDFName("IntegerItem"): 12,
|
||||
PDFName("StringItem"): " a string ",
|
||||
PDFName("Subdictionary"): {
|
||||
PDFName("Item1"): Decimal("0.4"),
|
||||
PDFName("Item2"): True,
|
||||
PDFName("LastItem"): " not! ",
|
||||
PDFName("VeryLastItem"): " OK ",
|
||||
},
|
||||
},
|
||||
)
|
||||
Reference in new issue
Block a user