better parsing of indirects

This commit is contained in:
ari committed 2020-10-30 14:12:34 -04:00
1 parent 343a0ae4de
commit 68dd42fea1
4 files changed
+170 -201

No files matched your search

+19 -163
View File
@@ -1,7 +1,7 @@
from pdfmajor.parser.objects.primitives import PDFInteger
from pdfmajor.parser.stream.PDFStream import PDFStream
from pdfmajor.parser.objects.primitives import get_obj_from_token_primitive
from typing import Iterator, Optional
from pdfmajor.streambuffer import BufferStream
from pdfmajor.lexer import iter_tokens
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
@@ -10,20 +10,20 @@ from pdfmajor.lexer.token import (
TokenDictionary,
TokenKeyword,
)
from pdfmajor.lexer import iter_tokens
from .objects.indirect import IndirectObject, ObjectRef
from pdfmajor.streambuffer import BufferStream
from .exceptions import InvalidKeywordPos
from .objects.base import PDFObject
from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import (
EarlyStop,
InvalidKeywordPos,
BrokenFile,
InvalidKeywordPos,
ParserError,
from .parsers import (
deal_with_collection_object,
on_endobj,
on_indirect_ref_close,
on_primitive,
on_stream,
)
from .state import ParsingState
def get_first_object(
@@ -44,9 +44,9 @@ def iter_objects(
"""
state = ParsingState([], []) if state is None else state
for token in iter_tokens(buffer):
attempt_prim = attempt_parse_prim(token, state)
if attempt_prim.success:
yield from state.feed_or_yield_objs(attempt_prim.next_objs)
prim_obj = get_obj_from_token_primitive(token)
if prim_obj is not None:
yield from on_primitive(prim_obj, state)
elif isinstance(token, TokenComment):
yield state.set_last_obj(
PDFComment(
@@ -56,14 +56,14 @@ def iter_objects(
)
elif isinstance(token, TokenKeyword):
if token.value == b"R":
yield from _on_indirect_ref_close(state)
yield from on_indirect_ref_close(state)
elif token.value == b"obj":
state.initialize_indirect_obj(token.start_loc)
elif token.value == b"endobj":
obj = _on_endobj(state, token)
obj = on_endobj(state, token)
yield obj
elif token.value == b"stream":
yield from _on_stream(
yield from on_stream(
buffer,
state,
token,
@@ -83,147 +83,3 @@ def iter_objects(
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
)
def _on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject:
"""finish off the current context, assuming we are handling an indirect-object
Args:
token (TokenKeyword)
Raises:
BrokenFile: if the object fif not get initilized with two integers
InvalidKeywordPos: If the context is not an indirect object
Yields:
PDFObject
"""
last_ctx = state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
if len(state.int_collection) > 1:
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
for num in state.flush_int_collections():
last_ctx.pass_item(num)
return last_ctx
else:
raise ParserError(f"Recived an endobj while not procesing an indirect object")
def _on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
"""Initilize a PDFStream
Args:
token (TokenKeyword)
Raises:
ParserError: if the current context is not an indirect object
"""
last_ctx = state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
obj = last_ctx.get_object()
if not isinstance(obj, PDFDictionary):
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
if isinstance(stream.length, PDFInteger):
buffer.seek(stream.offset + stream.length.to_python())
next_token = None
for i, next_token in enumerate(iter_tokens(buffer)):
if (
isinstance(next_token, TokenKeyword)
and next_token.value == b"endstream"
):
if i > 0:
stream.length.value = next_token.start_loc - stream.offset
break
if not (
isinstance(next_token, TokenKeyword)
and next_token.value == b"endstream"
):
raise BrokenFile(
f"stream did not contain 'endstream', instead the token {next_token} was found"
)
elif isinstance(stream.length, ObjectRef):
yield last_ctx
raise EarlyStop(
"cannot proceed parsing as the stream.length object is an indirect object"
)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
def _on_indirect_ref_close(state: ParsingState):
obj_num, gen_num = state.get_indobj_values()
obj = ObjectRef(obj_num, gen_num)
cur_ctx = state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
# def _save_or_get_object(
# buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int
# ) -> PDFObject:
# """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
# Args:
# obj_num (int): object number
# gen_num (int): generation number
# Returns:
# PDFObject
# """
# indobj = xrefdb.objs.get((obj_num, gen_num), None)
# print("trying to read", obj_num, gen_num)
# if indobj is None:
# xref = xrefdb.xrefs.get((obj_num, gen_num), None)
# if xref is None:
# raise InvalidXref(
# f"xref not found for object retrieval {obj_num}, {gen_num}"
# )
# return _read_object_from_xref(xref, xrefdb, buffer)
# else:
# return _read_object_from_db_or_buffer(indobj, xrefdb, buffer)
# def _read_object_from_xref(
# xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream
# ) -> PDFObject:
# with buffer.get_window():
# buffer.seek(xref.offset)
# obj = next(iter_objects(buffer, xrefdb))
# if (
# not isinstance(obj, IndirectObject)
# or obj.gen_num != xref.gen_num
# or obj.obj_num != xref.obj_num
# ):
# raise ParserError(
# f"Invalid offset for indirect object, expected {xref} but got {obj}"
# )
# return obj
# def _read_object_from_db_or_buffer(
# indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream
# ) -> PDFObject:
# if indobj.read:
# return indobj
# with buffer.get_window():
# buffer.seek(indobj.offset)
# next_token = next(iter_tokens(buffer))
# if next_token.value != b"obj":
# if isinstance(next_token, TokenKeyword):
# raise InvalidKeywordPos(next_token, [b"obj"])
# else:
# raise ParserError(f"expected a keyword instead got {next_token}")
# obj = next(
# iter_objects(
# buffer,
# xrefdb,
# ParsingState([indobj], []),
# )
# )
# if obj != indobj:
# raise ParserError(f"got a different indobj {obj} != {indobj}")
# indobj.read = True
# return obj
+120 -31
View File
@@ -1,45 +1,50 @@
from typing import Iterator, List, NamedTuple, Type
from pdfmajor.lexer.token import Token, TokenComplexType, TokenComplexTypeVal
from pdfmajor.lexer import iter_tokens
from pdfmajor.lexer.token import (
Token,
TokenComplexType,
TokenComplexTypeVal,
TokenKeyword,
)
from pdfmajor.streambuffer import BufferStream
from .state import ParsingState
from .exceptions import ParserError
from .objects.base import PDFObject
from .exceptions import BrokenFile, EarlyStop, ParserError
from .stream.PDFStream import PDFStream
from .objects.base import PDFContextualObject, PDFObject
from .objects.primitives import PDFInteger, get_obj_from_token_primitive
from .objects.collections import PDFDictionary
from .objects.indirect import IndirectObject, ObjectRef
from .objects.primitives import (
PDFInteger,
PDFPrimitiveObject,
get_obj_from_token_primitive,
)
from .state import ParsingState
class ParsingAttempt(NamedTuple):
success: bool
next_objs: List[PDFObject]
def attempt_parse_prim(token: Token, state: ParsingState) -> ParsingAttempt:
"""Attempt to parse out a primitive value from the token
def on_primitive(pobj: PDFPrimitiveObject, state: ParsingState) -> Iterator[PDFObject]:
"""Parses out a primitive value from the token
Args:
token (Token): [description]
state (ParsingState): [description]
pobj (PDFPrimitiveObject)
state (ParsingState)
Returns:
ParsingAttempt
Yields:
Iterator[PDFObject]
"""
prim_obj = get_obj_from_token_primitive(token)
if prim_obj is not None:
out: List[PDFObject] = []
state.set_last_obj(prim_obj)
if isinstance(prim_obj, PDFInteger):
if len(state.int_collection) <= 1:
state.int_collection.append(prim_obj)
else:
for obj in state.flush_int_collections():
out.append(obj)
state.set_last_obj(pobj)
if isinstance(pobj, PDFInteger):
for obj in state.flush_int_collections(1):
yield obj
state.int_collection.append(pobj)
else:
for obj in state.flush_int_collections():
yield obj
cur_ctx = state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(pobj)
else:
for obj in state.flush_int_collections():
out.append(obj)
out.append(prim_obj)
return ParsingAttempt(True, out)
return ParsingAttempt(False, [])
yield pobj
def deal_with_collection_object(
@@ -75,3 +80,87 @@ def deal_with_collection_object(
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
def on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject:
"""finish off the current context, assuming we are handling an indirect-object
Args:
token (TokenKeyword)
Raises:
BrokenFile: if the object fif not get initilized with two integers
InvalidKeywordPos: If the context is not an indirect object
Yields:
PDFObject
"""
last_ctx = state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
if len(state.int_collection) > 1:
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
for num in state.flush_int_collections():
last_ctx.pass_item(num)
return last_ctx
else:
raise ParserError(f"Recived an endobj while not procesing an indirect object")
def on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
"""Initilize a PDFStream
Args:
token (TokenKeyword)
Raises:
ParserError: if the current context is not an indirect object
"""
last_ctx = state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
obj = last_ctx.get_object()
if not isinstance(obj, PDFDictionary):
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
if isinstance(stream.length, PDFInteger):
buffer.seek(stream.offset + stream.length.to_python())
next_token = None
for i, next_token in enumerate(iter_tokens(buffer)):
if (
isinstance(next_token, TokenKeyword)
and next_token.value == b"endstream"
):
if i > 0:
stream.length.value = next_token.start_loc - stream.offset
state.health_report.write(
issue_name="InvalidPDFStreamLength",
additional_info=f"""
start_token={token}
end_token={next_token}
object_dictionary={obj}
""",
)
break
if not (
isinstance(next_token, TokenKeyword)
and next_token.value == b"endstream"
):
raise BrokenFile(
f"stream did not contain 'endstream', instead the token {next_token} was found"
)
elif isinstance(stream.length, ObjectRef):
yield last_ctx
raise EarlyStop(
"cannot proceed parsing as the stream.length object is an indirect object"
)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
def on_indirect_ref_close(state: ParsingState):
obj_num, gen_num = state.get_indobj_values()
obj = ObjectRef(obj_num, gen_num)
cur_ctx = state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
+13 -6
View File
@@ -1,25 +1,31 @@
from typing import Iterator, List, Optional, Tuple, Union
from pdfmajor.healthlog import PDFHealthReport
from .exceptions import ParserError
from .objects.base import PDFContextualObject, PDFObject
from .objects.primitives import PDFInteger
from .objects.comment import PDFComment
from .objects.indirect import IndirectObject
from .objects.primitives import PDFInteger
class ParsingState:
__slots__ = ["context_stack", "int_collection", "last_obj"]
__slots__ = ["context_stack", "int_collection", "last_obj", "health_report"]
def __init__(
self,
context_stack: List[PDFContextualObject],
int_collection: List[Union[PDFInteger, int]],
last_obj: Optional[PDFObject] = None,
health_report: Optional[PDFHealthReport] = None,
) -> None:
self.context_stack = context_stack
self.int_collection = int_collection
self.last_obj = last_obj
self.health_report = (
health_report if health_report is not None else PDFHealthReport()
)
def __repr__(self) -> str:
return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj})".format(
@@ -87,7 +93,7 @@ class ParsingState:
IndirectObject(obj_num=obj_num, gen_num=gen_num, offset=offset)
)
def flush_int_collections(self) -> Iterator[PDFObject]:
def flush_int_collections(self, up_to: int = 0) -> Iterator[PDFObject]:
"""clear out the int-collection
Yields:
@@ -95,20 +101,21 @@ class ParsingState:
"""
cur_ctx = self.current_context
if cur_ctx is not None:
for obj in self.int_collection:
while len(self.int_collection) > up_to:
obj = self.int_collection.pop(0)
if not isinstance(obj, PDFInteger):
raise ParserError(
f"Invalid int-collection contains none PDFInteger types {obj}"
)
cur_ctx.pass_item(obj)
else:
for obj in self.int_collection:
while len(self.int_collection) > up_to:
obj = self.int_collection.pop(0)
if not isinstance(obj, PDFInteger):
raise ParserError(
f"Invalid int-collection contains none PDFInteger types {obj}"
)
yield obj
self.int_collection = []
def feed_or_yield_objs(self, next_objs: List[PDFObject]) -> Iterator[PDFObject]:
"""If there context, then feed the objects into it, otherwise yield them
+18 -1
View File
@@ -1,6 +1,6 @@
from pathlib import Path
from pdfmajor.parser import iter_objects
from pdfmajor.parser.objects.collections import PDFDictionary
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString
from typing import Any, Callable, List, Optional
@@ -117,6 +117,23 @@ class IndirectObjects(TestCase):
with self.assertRaises((StopIteration, EOFError)):
next(it)
def test_array_ofindirects(self):
self.run_test(
raw=br"""[0 1 3 R 2 3 23 4 R 31 2 R]""",
expected=[
PDFArray.from_list(
[
PDFInteger(0, 0, 0),
ObjectRef(1, 3),
PDFInteger(2, 0, 0),
PDFInteger(3, 0, 0),
ObjectRef(23, 4),
ObjectRef(31, 2),
]
)
],
)
def test_parse_simple_indobj(self):
self.run_test(
raw=br"""