converted parser to functional
fixed a bufsize issue on the lexer
This commit is contained in:
1 parent
087c1c2fe9
commit
3d39bd575f
13 files changed
+370
-370
No files matched your search
@@ -30,9 +30,7 @@ def iter_tokens(buffer: BufferStream) -> Iterator[Token]:
|
||||
else:
|
||||
j = m.start(0)
|
||||
next_char = buf[j : j + 1]
|
||||
if next_char == b"~":
|
||||
continue
|
||||
elif next_char == b"%":
|
||||
if next_char == b"%":
|
||||
buffer.seek(bufpos + j + 1)
|
||||
yield parse_comment(buffer)
|
||||
elif next_char == b"/":
|
||||
@@ -54,10 +52,12 @@ def iter_tokens(buffer: BufferStream) -> Iterator[Token]:
|
||||
buffer.seek(bufpos + j + 1)
|
||||
yield parse_double_angled_bracket(buffer)
|
||||
elif next_char == b">":
|
||||
buffer.seek(bufpos + j + 1)
|
||||
if buf[j : j + 2] == b">>":
|
||||
buffer.seekd(1)
|
||||
buffer.seek(bufpos + j)
|
||||
bufpos, buf = buffer.read(2)
|
||||
if buf == b">>":
|
||||
yield TokenDictionary(bufpos + j, bufpos + j + 2, TDictValue.CLOSE)
|
||||
else:
|
||||
buffer.seekd(-1)
|
||||
elif next_char == b"[":
|
||||
buffer.seek(bufpos + j + 1)
|
||||
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
|
||||
|
||||
@@ -1,243 +0,0 @@
|
||||
import io
|
||||
|
||||
from typing import Dict, Iterator, Optional, Tuple
|
||||
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.parser.objects.stream import PDFStream
|
||||
from pdfmajor.parser.exceptions import InvalidXref
|
||||
from pdfmajor.parser.objects.base import PDFObject
|
||||
from pdfmajor.lexer.token import TokenKeyword
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
TokenArray,
|
||||
TokenComment,
|
||||
TokenDictionary,
|
||||
TokenKeyword,
|
||||
)
|
||||
from .objects.indirect import IndirectObject
|
||||
from .objects.base import PDFContextualObject, PDFObject
|
||||
from .objects.collections import PDFArray, PDFDictionary
|
||||
from .objects.comment import PDFComment
|
||||
from .state import ParsingState
|
||||
from .parsers import attempt_parse_prim, deal_with_collection_object
|
||||
from .exceptions import InvalidKeywordPos, BrokenFile, InvalidKeywordPos, ParserError
|
||||
|
||||
|
||||
class XRefDB:
|
||||
"""A class for collection references to objects"""
|
||||
|
||||
def __init__(self, parser: "PDFParser") -> None:
|
||||
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
|
||||
self.parser = parser
|
||||
|
||||
def save_indobject(self, indobj: IndirectObject):
|
||||
if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None:
|
||||
self.objs[(indobj.obj_num, indobj.gen_num)] = indobj
|
||||
|
||||
def get_object_from_indobj(self, indobj: IndirectObject) -> PDFObject:
|
||||
return self.get_object(indobj.obj_num, indobj.gen_num)
|
||||
|
||||
def get_object(self, obj_num: int, gen_num: int) -> PDFObject:
|
||||
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
|
||||
|
||||
Args:
|
||||
obj_num (int): object number
|
||||
gen_num (int): generation number
|
||||
|
||||
Returns:
|
||||
PDFObject
|
||||
"""
|
||||
indobj = self.objs.get((obj_num, gen_num), None)
|
||||
if indobj is None:
|
||||
raise InvalidXref(
|
||||
f"xref not found for object retrieval {obj_num}, {gen_num}"
|
||||
)
|
||||
else:
|
||||
obj = indobj.get_object()
|
||||
if obj is not None:
|
||||
return obj
|
||||
cur_pos = self.parser.tell()
|
||||
self.parser.seek(indobj.offset)
|
||||
obj = next(self.parser.iter_objects())
|
||||
indobj.save_object(obj)
|
||||
self.parser.seek(cur_pos)
|
||||
return obj
|
||||
|
||||
|
||||
class PDFParser:
|
||||
"""PDF Parser"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
buffer: BufferStream,
|
||||
db: Optional[XRefDB] = None,
|
||||
strict: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
|
||||
Args:
|
||||
fp (io.BufferedIOBase)
|
||||
buffer_size (int, optional): the size of the reads. Defaults to 4096.
|
||||
state (Optional[ParsingState], optional): the parsing state. Defaults to None.
|
||||
strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True.
|
||||
"""
|
||||
self.lexer = PDFLexer(buffer)
|
||||
self.db = XRefDB(self)
|
||||
self.strict = strict
|
||||
|
||||
@classmethod
|
||||
def from_iobuffer(
|
||||
cls,
|
||||
fp: io.BufferedIOBase,
|
||||
buffer_size: int = 4096,
|
||||
strict: bool = True,
|
||||
):
|
||||
return cls(
|
||||
BufferStream(fp, buffer_size=buffer_size),
|
||||
strict=strict,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(
|
||||
cls,
|
||||
data: bytes,
|
||||
buffer_size: int = 4096,
|
||||
strict: bool = True,
|
||||
):
|
||||
return cls(
|
||||
BufferStream(io.BytesIO(data), buffer_size=buffer_size),
|
||||
strict=strict,
|
||||
)
|
||||
|
||||
def seek(self, offset: int) -> int:
|
||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||
|
||||
Args:
|
||||
offset (int)
|
||||
|
||||
Returns:
|
||||
int: new offset
|
||||
"""
|
||||
return self.lexer.seek(offset)
|
||||
|
||||
def tell(self) -> int:
|
||||
return self.lexer.tell()
|
||||
|
||||
def iter_objects(self) -> Iterator[PDFObject]:
|
||||
"""Iterates over the objects in the current byte stream
|
||||
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
state = ParsingState([], [], strict=self.strict)
|
||||
for token in self.lexer.iter_tokens():
|
||||
attempt_prim = attempt_parse_prim(token, state)
|
||||
if attempt_prim.success:
|
||||
yield from state.feed_or_yield_objs(attempt_prim.next_objs)
|
||||
elif isinstance(token, TokenComment):
|
||||
yield state.set_last_obj(
|
||||
PDFComment(
|
||||
token.value,
|
||||
(token.start_loc, token.end_loc),
|
||||
)
|
||||
)
|
||||
elif isinstance(token, TokenKeyword):
|
||||
if token.value == b"R":
|
||||
yield from self._on_indirect_ref_close(state)
|
||||
elif token.value == b"obj":
|
||||
state.initialize_indirect_obj(token.start_loc)
|
||||
elif token.value == b"endobj":
|
||||
yield self._on_endobj(state, token)
|
||||
elif token.value == b"stream":
|
||||
self._on_stream(
|
||||
state,
|
||||
token,
|
||||
)
|
||||
elif token.value == b"endstream":
|
||||
self._on_endstream(state, token)
|
||||
else:
|
||||
raise InvalidKeywordPos(token)
|
||||
elif isinstance(token, TokenArray):
|
||||
yield from state.flush_int_collections()
|
||||
yield from deal_with_collection_object(
|
||||
state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
|
||||
)
|
||||
elif isinstance(token, TokenDictionary):
|
||||
yield from state.flush_int_collections()
|
||||
yield from deal_with_collection_object(
|
||||
state,
|
||||
token=token,
|
||||
cls_const=PDFDictionary,
|
||||
open_t=TDictValue.OPEN,
|
||||
strict=self.strict,
|
||||
)
|
||||
|
||||
def _on_endobj(self, state: ParsingState, token: TokenKeyword) -> IndirectObject:
|
||||
"""finish off the current context, assuming we are handling an indirect-object
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
BrokenFile: if the object fif not get initilized with two integers
|
||||
InvalidKeywordPos: If the context is not an indirect object
|
||||
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
last_ctx = state.context_stack.pop()
|
||||
if isinstance(last_ctx, IndirectObject):
|
||||
if len(state.int_collection) > 1:
|
||||
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
|
||||
for num in state.flush_int_collections():
|
||||
last_ctx.pass_item(num)
|
||||
obj = last_ctx.into_readonly_copy()
|
||||
self.db.save_indobject(obj)
|
||||
return obj
|
||||
else:
|
||||
raise InvalidKeywordPos(token)
|
||||
|
||||
def _on_stream(self, state: ParsingState, token: TokenKeyword):
|
||||
"""Initilize a PDFStream
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
ParserError: if the current context is not an indirect object
|
||||
"""
|
||||
last_ctx = state.current_context
|
||||
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
|
||||
self.db.save_indobject(last_ctx)
|
||||
stream = PDFStream(token.end_loc, 0)
|
||||
obj = self.db.get_object_from_indobj(last_ctx)
|
||||
stream.pass_item(obj)
|
||||
last_ctx.save_object(PDFStream(token.end_loc, 0))
|
||||
state.context_stack.append(stream)
|
||||
self.seek(stream.offset + stream.length)
|
||||
else:
|
||||
raise ParserError(f"PDFStream is missing initilization dictionary")
|
||||
|
||||
def _on_endstream(self, state: ParsingState, token: TokenKeyword):
|
||||
"""Finish a PDFStream
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
InvalidKeywordPos: if the current context is not an indirect stream
|
||||
"""
|
||||
stream: PDFContextualObject = state.context_stack.pop()
|
||||
if not isinstance(stream, PDFStream):
|
||||
raise InvalidKeywordPos(token)
|
||||
|
||||
def _on_indirect_ref_close(self, state: ParsingState):
|
||||
obj_num, gen_num = state.get_indobj_values()
|
||||
obj = self.db.get_object(obj_num, gen_num)
|
||||
cur_ctx = state.current_context
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
yield obj
|
||||
@@ -1,25 +1,24 @@
|
||||
import io
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from .objects.stream import PDFStream
|
||||
from .PDFParser import PDFParser
|
||||
from .fliters import process_filters_on_data
|
||||
|
||||
|
||||
class PDFStreamPasrser(PDFParser):
|
||||
def __init__(
|
||||
self,
|
||||
stream: PDFStream,
|
||||
buffer: BufferStream,
|
||||
strict: bool = True,
|
||||
) -> None:
|
||||
cur_pos = buffer.tell()
|
||||
buffer.seek(stream.offset)
|
||||
data = buffer.read(stream.length).data
|
||||
buffer.seek(cur_pos)
|
||||
data = process_filters_on_data(data, stream.filter, stream.decode_parms)
|
||||
data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms)
|
||||
self.stream = stream
|
||||
super().__init__(
|
||||
buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length),
|
||||
strict=strict,
|
||||
)
|
||||
# class PDFStreamPasrser(PDFParser):
|
||||
# def __init__(
|
||||
# self,
|
||||
# stream: PDFStream,
|
||||
# buffer: BufferStream,
|
||||
# strict: bool = True,
|
||||
# ) -> None:
|
||||
# cur_pos = buffer.tell()
|
||||
# buffer.seek(stream.offset)
|
||||
# data = buffer.read(stream.length).data
|
||||
# buffer.seek(cur_pos)
|
||||
# data = process_filters_on_data(data, stream.filter, stream.decode_parms)
|
||||
# data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms)
|
||||
# self.stream = stream
|
||||
# super().__init__(
|
||||
# buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length),
|
||||
# strict=strict,
|
||||
# )
|
||||
@@ -0,0 +1,197 @@
|
||||
from pdfmajor.parser.xref import XRefRow
|
||||
from pdfmajor.parser.xrefdb import XRefDB
|
||||
from typing import Iterator, Optional
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
TokenArray,
|
||||
TokenComment,
|
||||
TokenDictionary,
|
||||
TokenKeyword,
|
||||
)
|
||||
from pdfmajor.lexer import iter_tokens
|
||||
from .objects.stream import PDFStream
|
||||
from .objects.indirect import IndirectObject
|
||||
from .objects.base import PDFContextualObject, PDFObject
|
||||
from .objects.collections import PDFArray, PDFDictionary
|
||||
from .objects.comment import PDFComment
|
||||
from .state import ParsingState
|
||||
from .parsers import attempt_parse_prim, deal_with_collection_object
|
||||
from .exceptions import (
|
||||
InvalidKeywordPos,
|
||||
BrokenFile,
|
||||
InvalidKeywordPos,
|
||||
InvalidXref,
|
||||
ParserError,
|
||||
)
|
||||
|
||||
|
||||
def iter_objects(
|
||||
buffer: BufferStream,
|
||||
xrefdb: XRefDB,
|
||||
state: Optional[ParsingState] = None,
|
||||
) -> Iterator[PDFObject]:
|
||||
"""Iterates over the objects in the current byte stream
|
||||
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
state = ParsingState([], []) if state is None else state
|
||||
for token in iter_tokens(buffer):
|
||||
attempt_prim = attempt_parse_prim(token, state)
|
||||
if attempt_prim.success:
|
||||
yield from state.feed_or_yield_objs(attempt_prim.next_objs)
|
||||
elif isinstance(token, TokenComment):
|
||||
yield state.set_last_obj(
|
||||
PDFComment(
|
||||
token.value,
|
||||
(token.start_loc, token.end_loc),
|
||||
)
|
||||
)
|
||||
elif isinstance(token, TokenKeyword):
|
||||
if token.value == b"R":
|
||||
yield from _on_indirect_ref_close(buffer, state, xrefdb)
|
||||
elif token.value == b"obj":
|
||||
state.initialize_indirect_obj(token.start_loc)
|
||||
elif token.value == b"endobj":
|
||||
obj = _on_endobj(state, token)
|
||||
xrefdb.save_indobject(obj)
|
||||
yield obj
|
||||
elif token.value == b"stream":
|
||||
_on_stream(
|
||||
buffer,
|
||||
state,
|
||||
xrefdb,
|
||||
token,
|
||||
)
|
||||
else:
|
||||
raise InvalidKeywordPos(token, [b"R", b"obj", b"endobj", b"stream"])
|
||||
elif isinstance(token, TokenArray):
|
||||
yield from state.flush_int_collections()
|
||||
yield from deal_with_collection_object(
|
||||
state, token, PDFArray, TArrayValue.OPEN
|
||||
)
|
||||
elif isinstance(token, TokenDictionary):
|
||||
yield from state.flush_int_collections()
|
||||
yield from deal_with_collection_object(
|
||||
state,
|
||||
token=token,
|
||||
cls_const=PDFDictionary,
|
||||
open_t=TDictValue.OPEN,
|
||||
)
|
||||
|
||||
|
||||
def _on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject:
|
||||
"""finish off the current context, assuming we are handling an indirect-object
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
BrokenFile: if the object fif not get initilized with two integers
|
||||
InvalidKeywordPos: If the context is not an indirect object
|
||||
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
last_ctx = state.context_stack.pop()
|
||||
if isinstance(last_ctx, IndirectObject):
|
||||
if len(state.int_collection) > 1:
|
||||
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
|
||||
for num in state.flush_int_collections():
|
||||
last_ctx.pass_item(num)
|
||||
return last_ctx.into_readonly_copy()
|
||||
else:
|
||||
raise ParserError(f"Recived an endobj while not procesing an indirect object")
|
||||
|
||||
|
||||
def _on_stream(
|
||||
buffer: BufferStream, state: ParsingState, xrefdb: XRefDB, token: TokenKeyword
|
||||
):
|
||||
"""Initilize a PDFStream
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
ParserError: if the current context is not an indirect object
|
||||
"""
|
||||
last_ctx = state.current_context
|
||||
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
|
||||
xrefdb.save_indobject(last_ctx)
|
||||
stream = PDFStream(token.end_loc + 1, 0)
|
||||
obj = _save_or_get_object(buffer, xrefdb, last_ctx.obj_num, last_ctx.gen_num)
|
||||
with buffer.get_window():
|
||||
next_token = next(iter_tokens(buffer))
|
||||
if not (
|
||||
isinstance(next_token, TokenKeyword) and next_token.value == b"endstream"
|
||||
):
|
||||
raise BrokenFile(
|
||||
f"stream did not contain 'endstream', instead the token {next_token} was found"
|
||||
)
|
||||
stream.pass_item(obj)
|
||||
buffer.seek(stream.offset + stream.length)
|
||||
last_ctx.save_object(stream)
|
||||
state.context_stack.append(stream)
|
||||
else:
|
||||
raise ParserError(f"PDFStream is missing initilization dictionary")
|
||||
|
||||
|
||||
def _on_indirect_ref_close(buffer: BufferStream, state: ParsingState, xrefdb: XRefDB):
|
||||
obj_num, gen_num = state.get_indobj_values()
|
||||
obj = _save_or_get_object(buffer, xrefdb, obj_num, gen_num)
|
||||
cur_ctx = state.current_context
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
yield obj
|
||||
|
||||
|
||||
def _save_or_get_object(
|
||||
buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int
|
||||
) -> PDFObject:
|
||||
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
|
||||
|
||||
Args:
|
||||
obj_num (int): object number
|
||||
gen_num (int): generation number
|
||||
|
||||
Returns:
|
||||
PDFObject
|
||||
"""
|
||||
indobj = xrefdb.objs.get((obj_num, gen_num), None)
|
||||
if indobj is None:
|
||||
xref = xrefdb.xrefs.get((obj_num, gen_num), None)
|
||||
if xref is None:
|
||||
raise InvalidXref(
|
||||
f"xref not found for object retrieval {obj_num}, {gen_num}"
|
||||
)
|
||||
return _read_object_from_xref(xref, xrefdb, buffer)
|
||||
else:
|
||||
return _read_object_from_db_or_buffer(indobj, xrefdb, buffer)
|
||||
|
||||
|
||||
def _read_object_from_xref(
|
||||
xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream
|
||||
) -> PDFObject:
|
||||
indobj = IndirectObject(xref.obj_num, xref.gen_num, xref.offset)
|
||||
with buffer.get_window():
|
||||
buffer.seek(indobj.offset)
|
||||
obj = next(iter_objects(buffer, xrefdb))
|
||||
indobj.save_object(obj)
|
||||
xrefdb.save_indobject(indobj)
|
||||
return obj
|
||||
|
||||
|
||||
def _read_object_from_db_or_buffer(
|
||||
indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream
|
||||
) -> PDFObject:
|
||||
obj = indobj.get_object()
|
||||
if obj is not None:
|
||||
return obj
|
||||
with buffer.get_window():
|
||||
buffer.seek(indobj.offset)
|
||||
obj = next(iter_objects(buffer, xrefdb))
|
||||
indobj.save_object(obj)
|
||||
return obj
|
||||
@@ -1,3 +1,4 @@
|
||||
from typing import List
|
||||
from pdfmajor.lexer.token import TokenKeyword
|
||||
from pdfmajor.exceptions import PDFMajorException
|
||||
|
||||
@@ -15,10 +16,10 @@ class UnexpectedEOF(BrokenFile, EOFError):
|
||||
|
||||
|
||||
class InvalidKeywordPos(BrokenFile):
|
||||
def __init__(self, token: TokenKeyword) -> None:
|
||||
def __init__(self, token: TokenKeyword, expected_token: List[bytes]) -> None:
|
||||
self.token = token
|
||||
super().__init__(
|
||||
f"keyword '{token.value!r}' was found at {token.start_loc}:{token.end_loc}"
|
||||
f"keyword '{token.value!r}' was found at {token.start_loc}:{token.end_loc} when expecting {expected_token}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -26,10 +26,12 @@ class IndirectObject(PDFContextualObject):
|
||||
return self.__data
|
||||
|
||||
def save_object(self, obj: PDFObject):
|
||||
# if self.__data is not None:
|
||||
# raise InvalidIndirectObjAccess(
|
||||
# f"attempted to override an initilized reference of an object {obj}"
|
||||
# )
|
||||
"""This method will override the current stored object
|
||||
|
||||
Args:
|
||||
obj (PDFObject)
|
||||
|
||||
"""
|
||||
self.__data = obj
|
||||
|
||||
def into_readonly_copy(self):
|
||||
@@ -44,10 +46,11 @@ class IndirectObject(PDFContextualObject):
|
||||
return obj.to_python()
|
||||
|
||||
def pass_item(self, item: PDFObject):
|
||||
# if self.__read_only:
|
||||
# raise InvalidIndirectObjAccess(
|
||||
# f"attempted to add to an read-only reference of an object {item}"
|
||||
# )
|
||||
"""This method will either propogate the item to it's child (in case the child is a contexual object) or override it's own data with the incoming
|
||||
|
||||
Args:
|
||||
item (PDFObject): [description]
|
||||
"""
|
||||
cur_obj = self.get_object()
|
||||
if isinstance(cur_obj, PDFContextualObject):
|
||||
cur_obj.pass_item(item)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Iterator, List, Optional, NamedTuple, Type
|
||||
from typing import Iterator, List, NamedTuple, Type
|
||||
|
||||
from pdfmajor.lexer.token import Token, TokenComplexType, TokenComplexTypeVal
|
||||
|
||||
@@ -49,7 +49,6 @@ def deal_with_collection_object(
|
||||
token: TokenComplexType,
|
||||
cls_const: Type[PDFContextualObject],
|
||||
open_t: TokenComplexTypeVal,
|
||||
strict: bool = False,
|
||||
) -> Iterator[PDFObject]:
|
||||
"""generic method for processing things that have a start and end token
|
||||
|
||||
@@ -72,9 +71,9 @@ def deal_with_collection_object(
|
||||
return
|
||||
else:
|
||||
cur_ctx.pass_item(last_ctx)
|
||||
elif strict:
|
||||
else:
|
||||
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
|
||||
elif strict:
|
||||
else:
|
||||
raise ParserError(
|
||||
f"Invalid end of {cls_const} specified at {token} with no start"
|
||||
)
|
||||
@@ -9,27 +9,24 @@ from .objects.indirect import IndirectObject
|
||||
|
||||
class ParsingState:
|
||||
|
||||
__slots__ = ["context_stack", "int_collection", "last_obj", "strict"]
|
||||
__slots__ = ["context_stack", "int_collection", "last_obj"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
context_stack: List[PDFContextualObject],
|
||||
int_collection: List[PDFInteger],
|
||||
last_obj: Optional[PDFObject] = None,
|
||||
strict: bool = False,
|
||||
) -> None:
|
||||
self.context_stack = context_stack
|
||||
self.int_collection = int_collection
|
||||
self.last_obj = last_obj
|
||||
self.strict = strict
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj}, strict={strict})".format(
|
||||
return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj})".format(
|
||||
stackc=len(self.context_stack),
|
||||
cur_stack=self.current_context,
|
||||
ints=self.int_collection,
|
||||
lobj=self.last_obj,
|
||||
strict=self.strict,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# from pdfmajor.parser.objects.indirect import IndirectObjectCollection
|
||||
from pdfmajor.lexer.token import TokenInteger, TokenKeyword
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.lexer import iter_tokens
|
||||
from typing import Iterator, Literal, NamedTuple, Union
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from .exceptions import PDFNoValidXRef, BrokenFile, UnexpectedEOF
|
||||
@@ -40,8 +40,7 @@ def find_start_of_xref(buffer: BufferStream, strict: bool = False) -> int:
|
||||
def iter_over_xref(buffer: BufferStream, strict: bool = False) -> Iterator[XRefRow]:
|
||||
start_offset = find_start_of_xref(buffer, strict)
|
||||
buffer.seek(start_offset)
|
||||
lexer = PDFLexer(buffer)
|
||||
first_token = next(lexer.iter_tokens())
|
||||
first_token = next(iter_tokens(buffer))
|
||||
if isinstance(first_token, TokenInteger):
|
||||
raise NotImplementedError("XRefStream: PDF-1.5")
|
||||
elif isinstance(first_token, TokenKeyword):
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
from pdfmajor.parser.xref import XRefRow
|
||||
|
||||
from typing import Dict, Iterator, Tuple
|
||||
|
||||
from pdfmajor.parser.exceptions import InvalidXref
|
||||
from pdfmajor.parser.objects.base import PDFObject
|
||||
from .objects.indirect import IndirectObject
|
||||
from .objects.base import PDFObject
|
||||
|
||||
|
||||
class XRefDB:
|
||||
"""A class for collection references to objects"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
|
||||
self.xrefs: Dict[Tuple[int, int], XRefRow] = {}
|
||||
|
||||
def update_from_xrefiter(self, it_xref: Iterator[XRefRow]):
|
||||
for xref in it_xref:
|
||||
if xref.use == b"n":
|
||||
self.xrefs[(xref.obj_num, xref.gen_num)] = xref
|
||||
|
||||
def save_indobject(self, indobj: IndirectObject):
|
||||
if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None:
|
||||
self.objs[(indobj.obj_num, indobj.gen_num)] = indobj
|
||||
else:
|
||||
print("WARN: saving twice", indobj)
|
||||
@@ -1,5 +1,6 @@
|
||||
import io
|
||||
from typing import Iterator, NamedTuple, Optional
|
||||
from contextlib import contextmanager
|
||||
from pdfmajor.exceptions import PDFMajorException
|
||||
|
||||
|
||||
@@ -28,6 +29,10 @@ class BufferStream:
|
||||
self.fp = fp
|
||||
self.buffer_size = buffer_size
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, raw: bytes, buffer_size: int = 4096):
|
||||
return cls(io.BytesIO(raw), buffer_size)
|
||||
|
||||
def get_slice(
|
||||
self, offset: int, length: int, buffer_size: Optional[int] = None
|
||||
) -> "BufferStream":
|
||||
@@ -175,3 +180,9 @@ class BufferStream:
|
||||
buf = b""
|
||||
if len(buf) > 0 and pos == 0 and n == -1:
|
||||
yield BufferedBytes(pos=n + 1, data=buf)
|
||||
|
||||
@contextmanager
|
||||
def get_window(self):
|
||||
cur_pos = self.tell()
|
||||
yield self
|
||||
self.seek(cur_pos)
|
||||
+84
-78
@@ -1,25 +1,46 @@
|
||||
from pathlib import Path
|
||||
from pdfmajor.parser.xrefdb import XRefDB
|
||||
from pdfmajor.parser import iter_objects
|
||||
from pdfmajor.parser.objects.collections import PDFDictionary
|
||||
from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString
|
||||
|
||||
from typing import List
|
||||
from typing import Any, List
|
||||
from decimal import Decimal
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.streambuffer import StreamEOF
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.parser.objects.indirect import IndirectObject
|
||||
from pdfmajor.parser.objects.base import PDFObject
|
||||
from pdfmajor.parser.objects.comment import PDFComment
|
||||
from pdfmajor.parser.objects.stream import PDFStream
|
||||
from pdfmajor.parser.PDFParser import PDFParser
|
||||
|
||||
CURRENT_FOLDER = Path(__file__).parent
|
||||
|
||||
|
||||
class Collections(TestCase):
|
||||
def run_test(self, raw: bytes, expected: Any):
|
||||
size = len(raw)
|
||||
for buf_size in [
|
||||
2,
|
||||
3,
|
||||
size // 4,
|
||||
size // 3,
|
||||
size // 2,
|
||||
3 * size // 4,
|
||||
size,
|
||||
2 * size,
|
||||
4 * size,
|
||||
]:
|
||||
buffer = BufferStream.from_bytes(raw, buffer_size=buf_size)
|
||||
it = iter_objects(buffer, XRefDB())
|
||||
obj = next(it)
|
||||
self.assertEqual(obj.to_python(), expected)
|
||||
with self.assertRaises(StopIteration):
|
||||
next(it)
|
||||
|
||||
def test_parse_dict(self):
|
||||
parser = PDFParser.from_bytes(
|
||||
br"""<< /Type /Example
|
||||
self.run_test(
|
||||
raw=br"""<< /Type /Example
|
||||
/Subtype /DictionaryExample
|
||||
/Version 0.01
|
||||
/IntegerItem 12
|
||||
@@ -30,12 +51,7 @@ class Collections(TestCase):
|
||||
/LastItem ( not! )
|
||||
/VeryLastItem ( OK ) >>
|
||||
>>""",
|
||||
strict=True,
|
||||
)
|
||||
obj = next(parser.iter_objects())
|
||||
self.assertDictEqual(
|
||||
obj.to_python(),
|
||||
{
|
||||
expected={
|
||||
"Type": "/Example",
|
||||
"Subtype": "/DictionaryExample",
|
||||
"Version": Decimal("0.01"),
|
||||
@@ -51,28 +67,18 @@ class Collections(TestCase):
|
||||
)
|
||||
|
||||
def test_parse_array(self):
|
||||
parser = PDFParser.from_bytes(
|
||||
br"""[ 549 3.14 false ( Ralph ) /SomeName ]""",
|
||||
strict=True,
|
||||
)
|
||||
obj = next(parser.iter_objects())
|
||||
self.assertListEqual(
|
||||
obj.to_python(),
|
||||
[549, Decimal("3.14"), False, " Ralph ", "/SomeName"],
|
||||
self.run_test(
|
||||
raw=br"""[ 549 3.14 false ( Ralph ) /SomeName ]""",
|
||||
expected=[549, Decimal("3.14"), False, " Ralph ", "/SomeName"],
|
||||
)
|
||||
|
||||
def test_parse_nested_array(self):
|
||||
parser = PDFParser.from_bytes(
|
||||
br"""[ [10 30] [22 33] [
|
||||
self.run_test(
|
||||
raw=br"""[ [10 30] [22 33] [
|
||||
<< /x 0 /y 5 >>
|
||||
<< /x 10 /y 5 >>
|
||||
] ]""",
|
||||
strict=True,
|
||||
)
|
||||
obj = next(parser.iter_objects())
|
||||
self.assertListEqual(
|
||||
obj.to_python(),
|
||||
[
|
||||
expected=[
|
||||
[10, 30],
|
||||
[22, 33],
|
||||
[
|
||||
@@ -84,9 +90,32 @@ class Collections(TestCase):
|
||||
|
||||
|
||||
class IndirectObjects(TestCase):
|
||||
def run_test(self, raw: bytes, expected: List[PDFObject]) -> XRefDB:
|
||||
db: XRefDB = XRefDB()
|
||||
size = len(raw)
|
||||
for buf_size in [
|
||||
2,
|
||||
3,
|
||||
size // 4,
|
||||
size // 3,
|
||||
size // 2,
|
||||
3 * size // 4,
|
||||
size,
|
||||
2 * size,
|
||||
4 * size,
|
||||
]:
|
||||
db = XRefDB()
|
||||
buffer = BufferStream.from_bytes(raw, buffer_size=buf_size)
|
||||
it = iter_objects(buffer, db)
|
||||
for obj, eobj in zip(it, expected):
|
||||
self.assertEqual(obj, eobj)
|
||||
with self.assertRaises((StopIteration, EOFError)):
|
||||
next(it)
|
||||
return db
|
||||
|
||||
def test_parse_simple_indobj(self):
|
||||
parser = PDFParser.from_bytes(
|
||||
br"""
|
||||
self.run_test(
|
||||
raw=br"""
|
||||
12 1 obj (Bring) endobj
|
||||
100 0 obj
|
||||
<33>
|
||||
@@ -95,66 +124,43 @@ class IndirectObjects(TestCase):
|
||||
obj
|
||||
<< /x 1 /y 3>> endobj
|
||||
""",
|
||||
strict=True,
|
||||
)
|
||||
expected = [
|
||||
IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)),
|
||||
IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)),
|
||||
IndirectObject(
|
||||
13,
|
||||
8,
|
||||
144,
|
||||
PDFDictionary.from_dict(
|
||||
{"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)}
|
||||
expected=[
|
||||
IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)),
|
||||
IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)),
|
||||
IndirectObject(
|
||||
13,
|
||||
8,
|
||||
144,
|
||||
PDFDictionary.from_dict(
|
||||
{"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)}
|
||||
),
|
||||
),
|
||||
),
|
||||
]
|
||||
for obj, eobj in zip(parser.iter_objects(), expected):
|
||||
self.assertIsInstance(obj, IndirectObject)
|
||||
self.assertEqual(obj, eobj)
|
||||
with self.assertRaises(StreamEOF):
|
||||
next(parser.iter_objects())
|
||||
],
|
||||
)
|
||||
|
||||
def test_parse_obj_with_references(self):
|
||||
parser = PDFParser.from_bytes(
|
||||
br"""
|
||||
db = self.run_test(
|
||||
raw=br"""
|
||||
12 1 obj (Bring) endobj
|
||||
10 2 obj << /lastone 12 1 R >> endobj
|
||||
|
||||
""",
|
||||
strict=True,
|
||||
expected=[
|
||||
IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)),
|
||||
IndirectObject(
|
||||
0,
|
||||
2,
|
||||
54,
|
||||
data=PDFDictionary.from_dict({"lastone": PDFString("Bring", 0, 0)}),
|
||||
),
|
||||
],
|
||||
)
|
||||
expected = [
|
||||
"Bring",
|
||||
{"lastone": "Bring"},
|
||||
]
|
||||
for obj, exp in zip(parser.iter_objects(), expected):
|
||||
self.assertEqual(obj.to_python(), exp)
|
||||
with self.assertRaises(StreamEOF):
|
||||
next(parser.iter_objects())
|
||||
sec_obj = parser.db.get_object(10, 2)
|
||||
sec_obj = db.objs[(10, 2)]
|
||||
self.assertEqual(
|
||||
sec_obj.to_python(),
|
||||
{"lastone": "Bring"},
|
||||
)
|
||||
|
||||
def run_test(
|
||||
self,
|
||||
raw: bytes,
|
||||
expected: List[PDFObject],
|
||||
):
|
||||
parser = PDFParser.from_bytes(raw, strict=True)
|
||||
for i, (eobj, obj) in enumerate(
|
||||
zip(expected, parser.iter_objects()),
|
||||
1,
|
||||
):
|
||||
self.assertEqual(obj, eobj, f"Failed at example #{i}")
|
||||
try:
|
||||
next_obj = next(parser.iter_objects())
|
||||
self.assertIsNone(next_obj)
|
||||
except (EOFError, StopIteration) as e:
|
||||
self.assertIsNotNone(e)
|
||||
|
||||
def test_parse_comments(self):
|
||||
self.run_test(
|
||||
br"""
|
||||
@@ -175,7 +181,7 @@ class IndirectObjects(TestCase):
|
||||
)
|
||||
|
||||
def test_parse_stream(self):
|
||||
parser = PDFParser.from_bytes(
|
||||
buffer = BufferStream.from_bytes(
|
||||
br"""
|
||||
5 0 obj
|
||||
<< /Type /XRef /Length 124 /Filter /FlateDecode >>
|
||||
@@ -185,7 +191,7 @@ stream
|
||||
endobj
|
||||
"""
|
||||
)
|
||||
for obj in parser.iter_objects():
|
||||
for obj in iter_objects(buffer, XRefDB()):
|
||||
self.assertIsInstance(obj, IndirectObject)
|
||||
self.assertIsInstance(obj.get_object(), PDFStream)
|
||||
self.assertDictEqual(
|
||||
|
||||
+6
-2
@@ -1,4 +1,6 @@
|
||||
from pathlib import Path
|
||||
from pdfmajor.parser.xrefdb import XRefDB
|
||||
from pdfmajor.parser import _save_or_get_object
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
@@ -11,5 +13,7 @@ class Standard(TestCase):
|
||||
def test_char_doc(self):
|
||||
with (CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf").open("rb") as fp:
|
||||
buffer = BufferStream(fp)
|
||||
for xref in iter_over_xref(buffer):
|
||||
print(xref)
|
||||
db = XRefDB()
|
||||
db.update_from_xrefiter(iter_over_xref(buffer))
|
||||
for obj, gen in db.xrefs.keys():
|
||||
print(_save_or_get_object(buffer, db, obj, gen))
|
||||
Reference in new issue
Block a user