succesful dictionary parsing
This commit is contained in:
1 parent
7ee8254edb
commit
01ace1c57d
9 files changed
+287
-7
No files matched your search
@@ -70,7 +70,7 @@ class PDFLexer:
|
|||||||
yield parse_double_angled_bracket(self.buffer)
|
yield parse_double_angled_bracket(self.buffer)
|
||||||
elif next_char == b">":
|
elif next_char == b">":
|
||||||
self.buffer.seek(bufpos + j + 1)
|
self.buffer.seek(bufpos + j + 1)
|
||||||
if buf[j : j + 2] == ">>":
|
if buf[j : j + 2] == b">>":
|
||||||
self.buffer.seekd(1)
|
self.buffer.seekd(1)
|
||||||
yield TokenDictionary(
|
yield TokenDictionary(
|
||||||
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
|
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
|
||||||
|
|||||||
@@ -1,13 +1,11 @@
|
|||||||
from pdfmajor.streambuffer import BufferStream
|
from pdfmajor.streambuffer import BufferStream
|
||||||
from pdfmajor.lexer.exceptions import LexerEOF
|
from pdfmajor.lexer.exceptions import LexerEOF
|
||||||
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword, TokenNull
|
from pdfmajor.lexer.token import Token, TokenBoolean, TokenKeyword, TokenNull
|
||||||
from pdfmajor.lexer.regex import END_KEYWORD
|
from pdfmajor.lexer.regex import END_KEYWORD
|
||||||
from typing import Optional, Union
|
from typing import Optional, Union
|
||||||
|
|
||||||
|
|
||||||
def parse_keyword(
|
def parse_keyword(buffer: BufferStream, initialpos: Optional[int] = None) -> Token:
|
||||||
buffer: BufferStream, initialpos: Optional[int] = None
|
|
||||||
) -> Union[TokenKeyword, TokenBoolean]:
|
|
||||||
"""Parses input stream into a keyword or bool
|
"""Parses input stream into a keyword or bool
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -15,7 +13,7 @@ def parse_keyword(
|
|||||||
initialpos (Optional[int])
|
initialpos (Optional[int])
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Union[TokenKeyword, TokenBoolean]
|
Token
|
||||||
"""
|
"""
|
||||||
curtoken = b""
|
curtoken = b""
|
||||||
initialpos = buffer.tell() - 1 if initialpos is None else initialpos
|
initialpos = buffer.tell() - 1 if initialpos is None else initialpos
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ class Token(Generic[T]):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# Primitive Tokens
|
||||||
class TokenComment(Token[bytes]):
|
class TokenComment(Token[bytes]):
|
||||||
"""Token representing PDF comments
|
"""Token representing PDF comments
|
||||||
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
|
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
|
||||||
@@ -109,8 +110,45 @@ class TokenReal(Token[Decimal]):
|
|||||||
|
|
||||||
|
|
||||||
TokenNumber = Union[TokenInteger, TokenReal]
|
TokenNumber = Union[TokenInteger, TokenReal]
|
||||||
|
TokenPrimitive = Union[
|
||||||
|
TokenComment,
|
||||||
|
TokenKeyword,
|
||||||
|
TokenName,
|
||||||
|
TokenBoolean,
|
||||||
|
TokenNull,
|
||||||
|
TokenString,
|
||||||
|
TokenHexString,
|
||||||
|
TokenInteger,
|
||||||
|
TokenReal,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def is_primitive(token: Token) -> bool:
|
||||||
|
"""determins is a token represents a primitive pdf object such as boolean, null, integer etc
|
||||||
|
|
||||||
|
Args:
|
||||||
|
token (Token)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bool: True if primitive
|
||||||
|
"""
|
||||||
|
for opt in [
|
||||||
|
TokenComment,
|
||||||
|
TokenKeyword,
|
||||||
|
TokenName,
|
||||||
|
TokenBoolean,
|
||||||
|
TokenNull,
|
||||||
|
TokenString,
|
||||||
|
TokenHexString,
|
||||||
|
TokenInteger,
|
||||||
|
TokenReal,
|
||||||
|
]:
|
||||||
|
if isinstance(token, opt):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# Tokens for complex objects
|
||||||
class TDictValue(Enum):
|
class TDictValue(Enum):
|
||||||
OPEN = "<<"
|
OPEN = "<<"
|
||||||
CLOSE = ">>"
|
CLOSE = ">>"
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import io
|
||||||
|
from pdfmajor.parser_v2.exceptions import ParserError
|
||||||
|
from pdfmajor.parser_v2.objects import PDFArray, PDFDictionary, PDFPrimivite, PDFObject
|
||||||
|
from typing import Iterator, List, Optional
|
||||||
|
from pdfmajor.lexer.token import (
|
||||||
|
TArrayValue,
|
||||||
|
TDictValue,
|
||||||
|
TokenArray,
|
||||||
|
TokenDictionary,
|
||||||
|
TokenPrimitive,
|
||||||
|
is_primitive,
|
||||||
|
)
|
||||||
|
from pdfmajor.lexer import PDFLexer
|
||||||
|
from pdfmajor.parser_v2.context import PDFParseContext
|
||||||
|
|
||||||
|
|
||||||
|
class PDFParser:
|
||||||
|
def __init__(
|
||||||
|
self, fp: io.BufferedIOBase, buffer_size: int = 4096, strict: bool = True
|
||||||
|
) -> None:
|
||||||
|
self.lexer = PDFLexer(fp, buffer_size)
|
||||||
|
self.strict = strict
|
||||||
|
self.context_stack: List[PDFParseContext] = []
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current_context(self) -> Optional[PDFParseContext]:
|
||||||
|
"""Returns the current context of the parsing state
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[PDFParseContext]
|
||||||
|
"""
|
||||||
|
if len(self.context_stack) > 0:
|
||||||
|
return self.context_stack[-1]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def seek(self, offset: int) -> int:
|
||||||
|
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||||
|
|
||||||
|
Args:
|
||||||
|
offset (int)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int: new offset
|
||||||
|
"""
|
||||||
|
return self.lexer.seek(offset)
|
||||||
|
|
||||||
|
def iter_objects(self) -> Iterator[PDFObject]:
|
||||||
|
"""Iterates over the objects in the current stream
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
[type]: [description]
|
||||||
|
"""
|
||||||
|
for token in self.lexer.iter_tokens():
|
||||||
|
if is_primitive(token):
|
||||||
|
cur_ctx = self.current_context
|
||||||
|
obj = PDFPrimivite(token) # type: ignore
|
||||||
|
if cur_ctx is None:
|
||||||
|
yield obj
|
||||||
|
else:
|
||||||
|
cur_ctx.push(obj)
|
||||||
|
elif isinstance(token, TokenArray):
|
||||||
|
if token.value == TArrayValue.OPEN:
|
||||||
|
self.context_stack.append(PDFParseContext(PDFArray()))
|
||||||
|
elif len(self.context_stack) > 0:
|
||||||
|
last_ctx = self.context_stack.pop()
|
||||||
|
if isinstance(last_ctx.pdfobject, PDFArray):
|
||||||
|
cur_ctx = self.current_context
|
||||||
|
if cur_ctx is None:
|
||||||
|
yield last_ctx.pdfobject
|
||||||
|
else:
|
||||||
|
cur_ctx.push(last_ctx.pdfobject)
|
||||||
|
elif self.strict:
|
||||||
|
raise ParserError(f"Invalid end of array specified at {token}")
|
||||||
|
elif self.strict:
|
||||||
|
raise ParserError(
|
||||||
|
f"Invalid end of array specified at {token} with no start"
|
||||||
|
)
|
||||||
|
elif isinstance(token, TokenDictionary):
|
||||||
|
if token.value == TDictValue.OPEN:
|
||||||
|
self.context_stack.append(PDFParseContext(PDFDictionary()))
|
||||||
|
elif len(self.context_stack) > 0:
|
||||||
|
last_ctx = self.context_stack.pop()
|
||||||
|
if isinstance(last_ctx.pdfobject, PDFDictionary):
|
||||||
|
cur_ctx = self.current_context
|
||||||
|
if cur_ctx is None:
|
||||||
|
yield last_ctx.pdfobject
|
||||||
|
else:
|
||||||
|
cur_ctx.push(last_ctx.pdfobject)
|
||||||
|
elif self.strict:
|
||||||
|
raise ParserError(
|
||||||
|
f"Invalid end of dictionary specified at {token}"
|
||||||
|
)
|
||||||
|
elif self.strict:
|
||||||
|
raise ParserError(
|
||||||
|
f"Invalid end of dictionary specified at {token} with no start"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ParserError(f"Invalid token provided {token}")
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
from pdfmajor.parser_v2.exceptions import ParserError
|
||||||
|
from pdfmajor.parser_v2.objects import (
|
||||||
|
PDFArray,
|
||||||
|
PDFDictionary,
|
||||||
|
PDFObject,
|
||||||
|
PDFComplexType,
|
||||||
|
PDFStream,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PDFParseContext:
|
||||||
|
"""A parsing context
|
||||||
|
This class helps constructs complex PDF types
|
||||||
|
"""
|
||||||
|
|
||||||
|
__slots__ = ["pdfobject"]
|
||||||
|
|
||||||
|
def __init__(self, pdfobject: PDFComplexType) -> None:
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
pdfobject (PDFComplexType): the complex pdf object we are trying to build
|
||||||
|
"""
|
||||||
|
self.pdfobject = pdfobject
|
||||||
|
|
||||||
|
def push(self, item: PDFObject):
|
||||||
|
"""push any pdf object to the context, process according to the type of complex object we are trying to build
|
||||||
|
|
||||||
|
Args:
|
||||||
|
item (PDFObject)
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ParserError: raises if an invalid complex object is used
|
||||||
|
"""
|
||||||
|
if isinstance(self.pdfobject, PDFArray):
|
||||||
|
self.pdfobject.append(item)
|
||||||
|
elif isinstance(self.pdfobject, PDFDictionary):
|
||||||
|
self.pdfobject.pass_item(item)
|
||||||
|
elif isinstance(self.pdfobject, PDFStream):
|
||||||
|
raise NotImplementedError
|
||||||
|
else:
|
||||||
|
raise ParserError(f"Invalid Context {type(self.pdfobject)}")
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
from pdfmajor.execptions import PDFMajorException
|
||||||
|
|
||||||
|
|
||||||
|
class ParserError(PDFMajorException):
|
||||||
|
pass
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
# (Boolean values, Integer and Real numbers, Strings, Names), Arrays, Dictionaries, Streams, and the null object
|
||||||
|
|
||||||
|
|
||||||
|
from pdfmajor.execptions import ParserError
|
||||||
|
from pdfmajor.lexer.token import TokenName, TokenPrimitive
|
||||||
|
from typing import Dict, Generic, List, NamedTuple, Optional, TypeVar, Union
|
||||||
|
|
||||||
|
TToken = TypeVar("TToken", bound=TokenPrimitive)
|
||||||
|
|
||||||
|
|
||||||
|
class PDFName(NamedTuple):
|
||||||
|
value: str
|
||||||
|
|
||||||
|
|
||||||
|
class PDFPrimivite(Generic[TToken]):
|
||||||
|
def __init__(self, token: TToken) -> None:
|
||||||
|
self.token: TToken = token
|
||||||
|
|
||||||
|
@property
|
||||||
|
def value(self):
|
||||||
|
if isinstance(self.token, TokenName):
|
||||||
|
return PDFName(self.token.value)
|
||||||
|
return self.token.value
|
||||||
|
|
||||||
|
|
||||||
|
class PDFDictionary(Dict[PDFPrimivite[TokenName], "PDFObject"]):
|
||||||
|
def __init__(self, strict: bool = False) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.name: Optional[str] = None
|
||||||
|
self.strict = strict
|
||||||
|
|
||||||
|
def pass_item(self, item: "PDFObject"):
|
||||||
|
if self.name is None:
|
||||||
|
if isinstance(item, PDFPrimivite) and isinstance(item.token, TokenName):
|
||||||
|
self.name = item.value
|
||||||
|
if self.strict:
|
||||||
|
raise ParserError("Invalid token provided to dictionary")
|
||||||
|
else:
|
||||||
|
self[self.name] = item.value
|
||||||
|
self.name = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def value(self):
|
||||||
|
return dict(self)
|
||||||
|
|
||||||
|
|
||||||
|
class PDFArray(List["PDFObject"]):
|
||||||
|
@property
|
||||||
|
def value(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class PDFStream(object):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
PDFComplexType = Union[PDFDictionary, PDFArray, PDFStream]
|
||||||
|
PDFObject = Union[PDFComplexType, PDFPrimivite]
|
||||||
@@ -72,7 +72,7 @@ class BufferStream:
|
|||||||
def __next__(self):
|
def __next__(self):
|
||||||
bbyte = self.read(self.buffer_size)
|
bbyte = self.read(self.buffer_size)
|
||||||
if len(bbyte.data) == 0:
|
if len(bbyte.data) == 0:
|
||||||
raise StopIteration
|
raise StopIteration(f"no data {bbyte}")
|
||||||
return bbyte
|
return bbyte
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
from decimal import Decimal
|
||||||
|
import io
|
||||||
|
from pdfmajor.parser_v2.objects import PDFName
|
||||||
|
from pdfmajor.parser_v2 import PDFParser
|
||||||
|
from unittest import TestCase
|
||||||
|
|
||||||
|
|
||||||
|
class Basic(TestCase):
|
||||||
|
def test_parse_dict(self):
|
||||||
|
parser = PDFParser(
|
||||||
|
io.BytesIO(
|
||||||
|
br"""<< /Type /Example
|
||||||
|
/Subtype /DictionaryExample
|
||||||
|
/Version 0.01
|
||||||
|
/IntegerItem 12
|
||||||
|
/StringItem ( a string )
|
||||||
|
/Subdictionary <<
|
||||||
|
/Item1 0.4
|
||||||
|
/Item2 true
|
||||||
|
/LastItem ( not! )
|
||||||
|
/VeryLastItem ( OK ) >>
|
||||||
|
>>"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
obj = next(parser.iter_objects())
|
||||||
|
self.assertDictEqual(
|
||||||
|
obj,
|
||||||
|
{
|
||||||
|
PDFName("Type"): PDFName("Example"),
|
||||||
|
PDFName("Subtype"): PDFName("DictionaryExample"),
|
||||||
|
PDFName("Version"): Decimal("0.01"),
|
||||||
|
PDFName("IntegerItem"): 12,
|
||||||
|
PDFName("StringItem"): " a string ",
|
||||||
|
PDFName("Subdictionary"): {
|
||||||
|
PDFName("Item1"): Decimal("0.4"),
|
||||||
|
PDFName("Item2"): True,
|
||||||
|
PDFName("LastItem"): " not! ",
|
||||||
|
PDFName("VeryLastItem"): " OK ",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
Reference in new issue
Block a user