better parsing of indirects
This commit is contained in:
1 parent
343a0ae4de
commit
68dd42fea1
4 files changed
+170
-201
No files matched your search
+19
-163
@@ -1,7 +1,7 @@
|
||||
from pdfmajor.parser.objects.primitives import PDFInteger
|
||||
from pdfmajor.parser.stream.PDFStream import PDFStream
|
||||
from pdfmajor.parser.objects.primitives import get_obj_from_token_primitive
|
||||
from typing import Iterator, Optional
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
|
||||
from pdfmajor.lexer import iter_tokens
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
@@ -10,20 +10,20 @@ from pdfmajor.lexer.token import (
|
||||
TokenDictionary,
|
||||
TokenKeyword,
|
||||
)
|
||||
from pdfmajor.lexer import iter_tokens
|
||||
from .objects.indirect import IndirectObject, ObjectRef
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
|
||||
from .exceptions import InvalidKeywordPos
|
||||
from .objects.base import PDFObject
|
||||
from .objects.collections import PDFArray, PDFDictionary
|
||||
from .objects.comment import PDFComment
|
||||
from .state import ParsingState
|
||||
from .parsers import attempt_parse_prim, deal_with_collection_object
|
||||
from .exceptions import (
|
||||
EarlyStop,
|
||||
InvalidKeywordPos,
|
||||
BrokenFile,
|
||||
InvalidKeywordPos,
|
||||
ParserError,
|
||||
from .parsers import (
|
||||
deal_with_collection_object,
|
||||
on_endobj,
|
||||
on_indirect_ref_close,
|
||||
on_primitive,
|
||||
on_stream,
|
||||
)
|
||||
from .state import ParsingState
|
||||
|
||||
|
||||
def get_first_object(
|
||||
@@ -44,9 +44,9 @@ def iter_objects(
|
||||
"""
|
||||
state = ParsingState([], []) if state is None else state
|
||||
for token in iter_tokens(buffer):
|
||||
attempt_prim = attempt_parse_prim(token, state)
|
||||
if attempt_prim.success:
|
||||
yield from state.feed_or_yield_objs(attempt_prim.next_objs)
|
||||
prim_obj = get_obj_from_token_primitive(token)
|
||||
if prim_obj is not None:
|
||||
yield from on_primitive(prim_obj, state)
|
||||
elif isinstance(token, TokenComment):
|
||||
yield state.set_last_obj(
|
||||
PDFComment(
|
||||
@@ -56,14 +56,14 @@ def iter_objects(
|
||||
)
|
||||
elif isinstance(token, TokenKeyword):
|
||||
if token.value == b"R":
|
||||
yield from _on_indirect_ref_close(state)
|
||||
yield from on_indirect_ref_close(state)
|
||||
elif token.value == b"obj":
|
||||
state.initialize_indirect_obj(token.start_loc)
|
||||
elif token.value == b"endobj":
|
||||
obj = _on_endobj(state, token)
|
||||
obj = on_endobj(state, token)
|
||||
yield obj
|
||||
elif token.value == b"stream":
|
||||
yield from _on_stream(
|
||||
yield from on_stream(
|
||||
buffer,
|
||||
state,
|
||||
token,
|
||||
@@ -83,147 +83,3 @@ def iter_objects(
|
||||
cls_const=PDFDictionary,
|
||||
open_t=TDictValue.OPEN,
|
||||
)
|
||||
|
||||
|
||||
def _on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject:
|
||||
"""finish off the current context, assuming we are handling an indirect-object
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
BrokenFile: if the object fif not get initilized with two integers
|
||||
InvalidKeywordPos: If the context is not an indirect object
|
||||
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
last_ctx = state.context_stack.pop()
|
||||
if isinstance(last_ctx, IndirectObject):
|
||||
if len(state.int_collection) > 1:
|
||||
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
|
||||
for num in state.flush_int_collections():
|
||||
last_ctx.pass_item(num)
|
||||
return last_ctx
|
||||
else:
|
||||
raise ParserError(f"Recived an endobj while not procesing an indirect object")
|
||||
|
||||
|
||||
def _on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
|
||||
"""Initilize a PDFStream
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
ParserError: if the current context is not an indirect object
|
||||
"""
|
||||
last_ctx = state.current_context
|
||||
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
|
||||
obj = last_ctx.get_object()
|
||||
if not isinstance(obj, PDFDictionary):
|
||||
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
|
||||
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
|
||||
if isinstance(stream.length, PDFInteger):
|
||||
buffer.seek(stream.offset + stream.length.to_python())
|
||||
next_token = None
|
||||
for i, next_token in enumerate(iter_tokens(buffer)):
|
||||
if (
|
||||
isinstance(next_token, TokenKeyword)
|
||||
and next_token.value == b"endstream"
|
||||
):
|
||||
if i > 0:
|
||||
stream.length.value = next_token.start_loc - stream.offset
|
||||
break
|
||||
if not (
|
||||
isinstance(next_token, TokenKeyword)
|
||||
and next_token.value == b"endstream"
|
||||
):
|
||||
raise BrokenFile(
|
||||
f"stream did not contain 'endstream', instead the token {next_token} was found"
|
||||
)
|
||||
elif isinstance(stream.length, ObjectRef):
|
||||
yield last_ctx
|
||||
raise EarlyStop(
|
||||
"cannot proceed parsing as the stream.length object is an indirect object"
|
||||
)
|
||||
else:
|
||||
raise ParserError(f"PDFStream is missing initilization dictionary")
|
||||
|
||||
|
||||
def _on_indirect_ref_close(state: ParsingState):
|
||||
obj_num, gen_num = state.get_indobj_values()
|
||||
obj = ObjectRef(obj_num, gen_num)
|
||||
cur_ctx = state.current_context
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
yield obj
|
||||
|
||||
|
||||
# def _save_or_get_object(
|
||||
# buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int
|
||||
# ) -> PDFObject:
|
||||
# """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
|
||||
|
||||
# Args:
|
||||
# obj_num (int): object number
|
||||
# gen_num (int): generation number
|
||||
|
||||
# Returns:
|
||||
# PDFObject
|
||||
# """
|
||||
# indobj = xrefdb.objs.get((obj_num, gen_num), None)
|
||||
# print("trying to read", obj_num, gen_num)
|
||||
# if indobj is None:
|
||||
# xref = xrefdb.xrefs.get((obj_num, gen_num), None)
|
||||
# if xref is None:
|
||||
# raise InvalidXref(
|
||||
# f"xref not found for object retrieval {obj_num}, {gen_num}"
|
||||
# )
|
||||
# return _read_object_from_xref(xref, xrefdb, buffer)
|
||||
# else:
|
||||
# return _read_object_from_db_or_buffer(indobj, xrefdb, buffer)
|
||||
|
||||
|
||||
# def _read_object_from_xref(
|
||||
# xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream
|
||||
# ) -> PDFObject:
|
||||
# with buffer.get_window():
|
||||
# buffer.seek(xref.offset)
|
||||
# obj = next(iter_objects(buffer, xrefdb))
|
||||
# if (
|
||||
# not isinstance(obj, IndirectObject)
|
||||
# or obj.gen_num != xref.gen_num
|
||||
# or obj.obj_num != xref.obj_num
|
||||
# ):
|
||||
# raise ParserError(
|
||||
# f"Invalid offset for indirect object, expected {xref} but got {obj}"
|
||||
# )
|
||||
# return obj
|
||||
|
||||
|
||||
# def _read_object_from_db_or_buffer(
|
||||
# indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream
|
||||
# ) -> PDFObject:
|
||||
# if indobj.read:
|
||||
# return indobj
|
||||
# with buffer.get_window():
|
||||
# buffer.seek(indobj.offset)
|
||||
# next_token = next(iter_tokens(buffer))
|
||||
# if next_token.value != b"obj":
|
||||
# if isinstance(next_token, TokenKeyword):
|
||||
# raise InvalidKeywordPos(next_token, [b"obj"])
|
||||
# else:
|
||||
# raise ParserError(f"expected a keyword instead got {next_token}")
|
||||
# obj = next(
|
||||
# iter_objects(
|
||||
# buffer,
|
||||
# xrefdb,
|
||||
# ParsingState([indobj], []),
|
||||
# )
|
||||
# )
|
||||
# if obj != indobj:
|
||||
# raise ParserError(f"got a different indobj {obj} != {indobj}")
|
||||
# indobj.read = True
|
||||
# return obj
|
||||
+120
-31
@@ -1,45 +1,50 @@
|
||||
from typing import Iterator, List, NamedTuple, Type
|
||||
|
||||
from pdfmajor.lexer.token import Token, TokenComplexType, TokenComplexTypeVal
|
||||
from pdfmajor.lexer import iter_tokens
|
||||
from pdfmajor.lexer.token import (
|
||||
Token,
|
||||
TokenComplexType,
|
||||
TokenComplexTypeVal,
|
||||
TokenKeyword,
|
||||
)
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
|
||||
from .state import ParsingState
|
||||
from .exceptions import ParserError
|
||||
from .objects.base import PDFObject
|
||||
from .exceptions import BrokenFile, EarlyStop, ParserError
|
||||
from .stream.PDFStream import PDFStream
|
||||
from .objects.base import PDFContextualObject, PDFObject
|
||||
from .objects.primitives import PDFInteger, get_obj_from_token_primitive
|
||||
from .objects.collections import PDFDictionary
|
||||
from .objects.indirect import IndirectObject, ObjectRef
|
||||
from .objects.primitives import (
|
||||
PDFInteger,
|
||||
PDFPrimitiveObject,
|
||||
get_obj_from_token_primitive,
|
||||
)
|
||||
from .state import ParsingState
|
||||
|
||||
|
||||
class ParsingAttempt(NamedTuple):
|
||||
success: bool
|
||||
next_objs: List[PDFObject]
|
||||
|
||||
|
||||
def attempt_parse_prim(token: Token, state: ParsingState) -> ParsingAttempt:
|
||||
"""Attempt to parse out a primitive value from the token
|
||||
def on_primitive(pobj: PDFPrimitiveObject, state: ParsingState) -> Iterator[PDFObject]:
|
||||
"""Parses out a primitive value from the token
|
||||
|
||||
Args:
|
||||
token (Token): [description]
|
||||
state (ParsingState): [description]
|
||||
pobj (PDFPrimitiveObject)
|
||||
state (ParsingState)
|
||||
|
||||
Returns:
|
||||
ParsingAttempt
|
||||
Yields:
|
||||
Iterator[PDFObject]
|
||||
"""
|
||||
prim_obj = get_obj_from_token_primitive(token)
|
||||
if prim_obj is not None:
|
||||
out: List[PDFObject] = []
|
||||
state.set_last_obj(prim_obj)
|
||||
if isinstance(prim_obj, PDFInteger):
|
||||
if len(state.int_collection) <= 1:
|
||||
state.int_collection.append(prim_obj)
|
||||
else:
|
||||
for obj in state.flush_int_collections():
|
||||
out.append(obj)
|
||||
state.set_last_obj(pobj)
|
||||
if isinstance(pobj, PDFInteger):
|
||||
for obj in state.flush_int_collections(1):
|
||||
yield obj
|
||||
state.int_collection.append(pobj)
|
||||
else:
|
||||
for obj in state.flush_int_collections():
|
||||
yield obj
|
||||
cur_ctx = state.current_context
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(pobj)
|
||||
else:
|
||||
for obj in state.flush_int_collections():
|
||||
out.append(obj)
|
||||
out.append(prim_obj)
|
||||
return ParsingAttempt(True, out)
|
||||
return ParsingAttempt(False, [])
|
||||
yield pobj
|
||||
|
||||
|
||||
def deal_with_collection_object(
|
||||
@@ -75,3 +80,87 @@ def deal_with_collection_object(
|
||||
raise ParserError(
|
||||
f"Invalid end of {cls_const} specified at {token} with no start"
|
||||
)
|
||||
|
||||
|
||||
def on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject:
|
||||
"""finish off the current context, assuming we are handling an indirect-object
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
BrokenFile: if the object fif not get initilized with two integers
|
||||
InvalidKeywordPos: If the context is not an indirect object
|
||||
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
last_ctx = state.context_stack.pop()
|
||||
if isinstance(last_ctx, IndirectObject):
|
||||
if len(state.int_collection) > 1:
|
||||
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
|
||||
for num in state.flush_int_collections():
|
||||
last_ctx.pass_item(num)
|
||||
return last_ctx
|
||||
else:
|
||||
raise ParserError(f"Recived an endobj while not procesing an indirect object")
|
||||
|
||||
|
||||
def on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
|
||||
"""Initilize a PDFStream
|
||||
|
||||
Args:
|
||||
token (TokenKeyword)
|
||||
|
||||
Raises:
|
||||
ParserError: if the current context is not an indirect object
|
||||
"""
|
||||
last_ctx = state.current_context
|
||||
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
|
||||
obj = last_ctx.get_object()
|
||||
if not isinstance(obj, PDFDictionary):
|
||||
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
|
||||
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
|
||||
if isinstance(stream.length, PDFInteger):
|
||||
buffer.seek(stream.offset + stream.length.to_python())
|
||||
next_token = None
|
||||
for i, next_token in enumerate(iter_tokens(buffer)):
|
||||
if (
|
||||
isinstance(next_token, TokenKeyword)
|
||||
and next_token.value == b"endstream"
|
||||
):
|
||||
if i > 0:
|
||||
stream.length.value = next_token.start_loc - stream.offset
|
||||
state.health_report.write(
|
||||
issue_name="InvalidPDFStreamLength",
|
||||
additional_info=f"""
|
||||
start_token={token}
|
||||
end_token={next_token}
|
||||
object_dictionary={obj}
|
||||
""",
|
||||
)
|
||||
break
|
||||
if not (
|
||||
isinstance(next_token, TokenKeyword)
|
||||
and next_token.value == b"endstream"
|
||||
):
|
||||
raise BrokenFile(
|
||||
f"stream did not contain 'endstream', instead the token {next_token} was found"
|
||||
)
|
||||
elif isinstance(stream.length, ObjectRef):
|
||||
yield last_ctx
|
||||
raise EarlyStop(
|
||||
"cannot proceed parsing as the stream.length object is an indirect object"
|
||||
)
|
||||
else:
|
||||
raise ParserError(f"PDFStream is missing initilization dictionary")
|
||||
|
||||
|
||||
def on_indirect_ref_close(state: ParsingState):
|
||||
obj_num, gen_num = state.get_indobj_values()
|
||||
obj = ObjectRef(obj_num, gen_num)
|
||||
cur_ctx = state.current_context
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
yield obj
|
||||
@@ -1,25 +1,31 @@
|
||||
from typing import Iterator, List, Optional, Tuple, Union
|
||||
|
||||
from pdfmajor.healthlog import PDFHealthReport
|
||||
|
||||
from .exceptions import ParserError
|
||||
from .objects.base import PDFContextualObject, PDFObject
|
||||
from .objects.primitives import PDFInteger
|
||||
from .objects.comment import PDFComment
|
||||
from .objects.indirect import IndirectObject
|
||||
from .objects.primitives import PDFInteger
|
||||
|
||||
|
||||
class ParsingState:
|
||||
|
||||
__slots__ = ["context_stack", "int_collection", "last_obj"]
|
||||
__slots__ = ["context_stack", "int_collection", "last_obj", "health_report"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
context_stack: List[PDFContextualObject],
|
||||
int_collection: List[Union[PDFInteger, int]],
|
||||
last_obj: Optional[PDFObject] = None,
|
||||
health_report: Optional[PDFHealthReport] = None,
|
||||
) -> None:
|
||||
self.context_stack = context_stack
|
||||
self.int_collection = int_collection
|
||||
self.last_obj = last_obj
|
||||
self.health_report = (
|
||||
health_report if health_report is not None else PDFHealthReport()
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj})".format(
|
||||
@@ -87,7 +93,7 @@ class ParsingState:
|
||||
IndirectObject(obj_num=obj_num, gen_num=gen_num, offset=offset)
|
||||
)
|
||||
|
||||
def flush_int_collections(self) -> Iterator[PDFObject]:
|
||||
def flush_int_collections(self, up_to: int = 0) -> Iterator[PDFObject]:
|
||||
"""clear out the int-collection
|
||||
|
||||
Yields:
|
||||
@@ -95,20 +101,21 @@ class ParsingState:
|
||||
"""
|
||||
cur_ctx = self.current_context
|
||||
if cur_ctx is not None:
|
||||
for obj in self.int_collection:
|
||||
while len(self.int_collection) > up_to:
|
||||
obj = self.int_collection.pop(0)
|
||||
if not isinstance(obj, PDFInteger):
|
||||
raise ParserError(
|
||||
f"Invalid int-collection contains none PDFInteger types {obj}"
|
||||
)
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
for obj in self.int_collection:
|
||||
while len(self.int_collection) > up_to:
|
||||
obj = self.int_collection.pop(0)
|
||||
if not isinstance(obj, PDFInteger):
|
||||
raise ParserError(
|
||||
f"Invalid int-collection contains none PDFInteger types {obj}"
|
||||
)
|
||||
yield obj
|
||||
self.int_collection = []
|
||||
|
||||
def feed_or_yield_objs(self, next_objs: List[PDFObject]) -> Iterator[PDFObject]:
|
||||
"""If there context, then feed the objects into it, otherwise yield them
|
||||
|
||||
+18
-1
@@ -1,6 +1,6 @@
|
||||
from pathlib import Path
|
||||
from pdfmajor.parser import iter_objects
|
||||
from pdfmajor.parser.objects.collections import PDFDictionary
|
||||
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
|
||||
from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString
|
||||
|
||||
from typing import Any, Callable, List, Optional
|
||||
@@ -117,6 +117,23 @@ class IndirectObjects(TestCase):
|
||||
with self.assertRaises((StopIteration, EOFError)):
|
||||
next(it)
|
||||
|
||||
def test_array_ofindirects(self):
|
||||
self.run_test(
|
||||
raw=br"""[0 1 3 R 2 3 23 4 R 31 2 R]""",
|
||||
expected=[
|
||||
PDFArray.from_list(
|
||||
[
|
||||
PDFInteger(0, 0, 0),
|
||||
ObjectRef(1, 3),
|
||||
PDFInteger(2, 0, 0),
|
||||
PDFInteger(3, 0, 0),
|
||||
ObjectRef(23, 4),
|
||||
ObjectRef(31, 2),
|
||||
]
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
def test_parse_simple_indobj(self):
|
||||
self.run_test(
|
||||
raw=br"""
|
||||
|
||||
Reference in new issue
Block a user