split out xref from parser

This commit is contained in:
ari committed 2020-10-27 20:12:07 -04:00
1 parent 5af06a3f1e
commit 78175d3a64
14 files changed
+290 -154

No files matched your search

+4
View File
@@ -1,2 +1,6 @@
class PDFMajorException(Exception):
pass
class BrokenFile(PDFMajorException):
pass
+69 -58
View File
@@ -1,6 +1,4 @@
from pdfmajor.parser.stream.PDFStream import PDFStream
from pdfmajor.parser.xref import XRefRow
from pdfmajor.parser.xrefdb import XRefDB
from typing import Iterator, Optional
from pdfmajor.streambuffer import BufferStream
from pdfmajor.lexer.token import (
@@ -12,7 +10,7 @@ from pdfmajor.lexer.token import (
TokenKeyword,
)
from pdfmajor.lexer import iter_tokens
from .objects.indirect import IndirectObject
from .objects.indirect import IndirectObject, ObjectRef
from .objects.base import PDFObject
from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment
@@ -22,14 +20,12 @@ from .exceptions import (
InvalidKeywordPos,
BrokenFile,
InvalidKeywordPos,
InvalidXref,
ParserError,
)
def iter_objects(
buffer: BufferStream,
xrefdb: XRefDB,
state: Optional[ParsingState] = None,
) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream
@@ -51,18 +47,16 @@ def iter_objects(
)
elif isinstance(token, TokenKeyword):
if token.value == b"R":
yield from _on_indirect_ref_close(buffer, state, xrefdb)
yield from _on_indirect_ref_close(state)
elif token.value == b"obj":
state.initialize_indirect_obj(token.start_loc)
elif token.value == b"endobj":
obj = _on_endobj(state, token)
xrefdb.save_indobject(obj)
yield obj
elif token.value == b"stream":
_on_stream(
buffer,
state,
xrefdb,
token,
)
else:
@@ -106,9 +100,7 @@ def _on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject:
raise ParserError(f"Recived an endobj while not procesing an indirect object")
def _on_stream(
buffer: BufferStream, state: ParsingState, xrefdb: XRefDB, token: TokenKeyword
):
def _on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
"""Initilize a PDFStream
Args:
@@ -119,10 +111,10 @@ def _on_stream(
"""
last_ctx = state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
xrefdb.save_indobject(last_ctx)
# stream = PDFStream(token.end_loc + 1, 0)
obj = _save_or_get_object(buffer, xrefdb, last_ctx.obj_num, last_ctx.gen_num)
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc, obj)
obj = last_ctx.get_object()
if not isinstance(obj, PDFDictionary):
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
buffer.seek(stream.offset + stream.length)
next_token = next(iter_tokens(buffer))
if not (
@@ -135,9 +127,9 @@ def _on_stream(
raise ParserError(f"PDFStream is missing initilization dictionary")
def _on_indirect_ref_close(buffer: BufferStream, state: ParsingState, xrefdb: XRefDB):
def _on_indirect_ref_close(state: ParsingState):
obj_num, gen_num = state.get_indobj_values()
obj = _save_or_get_object(buffer, xrefdb, obj_num, gen_num)
obj = ObjectRef(obj_num, gen_num)
cur_ctx = state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
@@ -145,50 +137,69 @@ def _on_indirect_ref_close(buffer: BufferStream, state: ParsingState, xrefdb: XR
yield obj
def _save_or_get_object(
buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int
) -> PDFObject:
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
# def _save_or_get_object(
# buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int
# ) -> PDFObject:
# """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
Args:
obj_num (int): object number
gen_num (int): generation number
# Args:
# obj_num (int): object number
# gen_num (int): generation number
Returns:
PDFObject
"""
indobj = xrefdb.objs.get((obj_num, gen_num), None)
if indobj is None:
xref = xrefdb.xrefs.get((obj_num, gen_num), None)
if xref is None:
raise InvalidXref(
f"xref not found for object retrieval {obj_num}, {gen_num}"
)
return _read_object_from_xref(xref, xrefdb, buffer)
else:
return _read_object_from_db_or_buffer(indobj, xrefdb, buffer)
# Returns:
# PDFObject
# """
# indobj = xrefdb.objs.get((obj_num, gen_num), None)
# print("trying to read", obj_num, gen_num)
# if indobj is None:
# xref = xrefdb.xrefs.get((obj_num, gen_num), None)
# if xref is None:
# raise InvalidXref(
# f"xref not found for object retrieval {obj_num}, {gen_num}"
# )
# return _read_object_from_xref(xref, xrefdb, buffer)
# else:
# return _read_object_from_db_or_buffer(indobj, xrefdb, buffer)
def _read_object_from_xref(
xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream
) -> PDFObject:
indobj = IndirectObject(xref.obj_num, xref.gen_num, xref.offset)
with buffer.get_window():
buffer.seek(indobj.offset)
obj = next(iter_objects(buffer, xrefdb))
indobj.save_object(obj)
xrefdb.save_indobject(indobj)
return obj
# def _read_object_from_xref(
# xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream
# ) -> PDFObject:
# with buffer.get_window():
# buffer.seek(xref.offset)
# obj = next(iter_objects(buffer, xrefdb))
# if (
# not isinstance(obj, IndirectObject)
# or obj.gen_num != xref.gen_num
# or obj.obj_num != xref.obj_num
# ):
# raise ParserError(
# f"Invalid offset for indirect object, expected {xref} but got {obj}"
# )
# return obj
def _read_object_from_db_or_buffer(
indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream
) -> PDFObject:
obj = indobj.get_object()
if obj is not None:
return obj
with buffer.get_window():
buffer.seek(indobj.offset)
obj = next(iter_objects(buffer, xrefdb))
indobj.save_object(obj)
return obj
# def _read_object_from_db_or_buffer(
# indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream
# ) -> PDFObject:
# if indobj.read:
# return indobj
# with buffer.get_window():
# buffer.seek(indobj.offset)
# next_token = next(iter_tokens(buffer))
# if next_token.value != b"obj":
# if isinstance(next_token, TokenKeyword):
# raise InvalidKeywordPos(next_token, [b"obj"])
# else:
# raise ParserError(f"expected a keyword instead got {next_token}")
# obj = next(
# iter_objects(
# buffer,
# xrefdb,
# ParsingState([indobj], []),
# )
# )
# if obj != indobj:
# raise ParserError(f"got a different indobj {obj} != {indobj}")
# indobj.read = True
# return obj
+5 -17
View File
@@ -1,21 +1,21 @@
from typing import List
from pdfmajor.lexer.token import TokenKeyword
from pdfmajor.exceptions import PDFMajorException
from pdfmajor.exceptions import PDFMajorException, BrokenFile
class ParserError(PDFMajorException):
pass
class BrokenFile(ParserError):
class BrokenFileParserError(BrokenFile):
pass
class UnexpectedEOF(BrokenFile, EOFError):
class UnexpectedEOF(BrokenFileParserError, EOFError):
pass
class InvalidKeywordPos(BrokenFile):
class InvalidKeywordPos(BrokenFileParserError):
def __init__(self, token: TokenKeyword, expected_token: List[bytes]) -> None:
self.token = token
super().__init__(
@@ -23,21 +23,9 @@ class InvalidKeywordPos(BrokenFile):
)
class InvalidIndirectObjAccess(ParserError):
pass
class DecodeFailed(BrokenFile):
class DecodeFailed(BrokenFileParserError):
pass
class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError):
pass
class PDFNoValidXRef(BrokenFile, EOFError):
pass
class InvalidXref(ParserError, KeyError):
pass
+17 -5
View File
@@ -4,9 +4,20 @@ from pdfmajor.parser.stream.PDFStream import PDFStream
from .base import PDFContextualObject, PDFObject
class IndirectObject(PDFContextualObject):
class ObjectRef(PDFObject):
"""A class representing a reference for other objects"""
def __init__(self, obj_num: int, gen_num: int) -> None:
self.obj_num = obj_num
self.gen_num = gen_num
def to_python(self):
return (self.obj_num, self.gen_num)
class IndirectObject(PDFContextualObject):
"""A class representing a referencable object"""
def __init__(
self,
obj_num: int,
@@ -20,6 +31,7 @@ class IndirectObject(PDFContextualObject):
self.gen_num = gen_num
self.__data = data
self.stream = stream
self.read: bool = False
def __repr__(self) -> str:
return f"IndirectObject(obj_num={self.obj_num}, gen_num={self.gen_num}, offset={self.offset}, data={self.get_object()})"
@@ -37,10 +49,10 @@ class IndirectObject(PDFContextualObject):
self.__data = obj
def to_python(self):
obj = self.get_object()
if obj is None:
return None
return obj.to_python()
data = self.get_object()
if data is not None:
data = data.to_python()
return (self.obj_num, self.gen_num, self.offset, data)
def pass_item(self, item: PDFObject):
"""This method will either propogate the item to it's child (in case the child is a contexual object) or override it's own data with the incoming
+2 -4
View File
@@ -32,10 +32,8 @@ def attempt_parse_prim(token: Token, state: ParsingState) -> ParsingAttempt:
if len(state.int_collection) <= 1:
state.int_collection.append(prim_obj)
else:
for last_num in state.int_collection:
out.append(last_num)
state.int_collection = []
out.append(prim_obj)
for obj in state.flush_int_collections():
out.append(obj)
else:
for obj in state.flush_int_collections():
out.append(obj)
+15 -3
View File
@@ -1,4 +1,4 @@
from typing import Iterator, List, Optional, Tuple
from typing import Iterator, List, Optional, Tuple, Union
from .exceptions import ParserError
from .objects.base import PDFContextualObject, PDFObject
@@ -14,7 +14,7 @@ class ParsingState:
def __init__(
self,
context_stack: List[PDFContextualObject],
int_collection: List[PDFInteger],
int_collection: List[Union[PDFInteger, int]],
last_obj: Optional[PDFObject] = None,
) -> None:
self.context_stack = context_stack
@@ -68,8 +68,12 @@ class ParsingState:
"""
if len(self.int_collection) == 2:
gen_num = self.int_collection.pop()
if isinstance(gen_num, PDFInteger):
gen_num = gen_num.to_python()
obj_num = self.int_collection.pop()
return (obj_num.to_python(), gen_num.to_python())
if isinstance(obj_num, PDFInteger):
obj_num = obj_num.to_python()
return (obj_num, gen_num)
raise ParserError("missing leading tokens for object reference")
def initialize_indirect_obj(self, offset: int):
@@ -92,9 +96,17 @@ class ParsingState:
cur_ctx = self.current_context
if cur_ctx is not None:
for obj in self.int_collection:
if not isinstance(obj, PDFInteger):
raise ParserError(
f"Invalid int-collection contains none PDFInteger types {obj}"
)
cur_ctx.pass_item(obj)
else:
for obj in self.int_collection:
if not isinstance(obj, PDFInteger):
raise ParserError(
f"Invalid int-collection contains none PDFInteger types {obj}"
)
yield obj
self.int_collection = []
+1 -3
View File
@@ -40,9 +40,7 @@ class PDFStream:
self.dl: Optional[int] = dl
@classmethod
def from_pdfdict(cls, offset: int, item: PDFObject) -> "PDFStream":
if not isinstance(item, PDFDictionary):
raise ParserError(f"Cannot initilize a pdfstream with {type(item)}")
def from_pdfdict(cls, offset: int, item: PDFDictionary) -> "PDFStream":
data = item.to_python()
stream = cls(offset, data["Length"])
stream.filter = get_single_or_list(data.get("Filter", stream.filter))
-27
View File
@@ -1,27 +0,0 @@
from pdfmajor.parser.xref import XRefRow
from typing import Dict, Iterator, Tuple
from pdfmajor.parser.exceptions import InvalidXref
from pdfmajor.parser.objects.base import PDFObject
from .objects.indirect import IndirectObject
from .objects.base import PDFObject
class XRefDB:
"""A class for collection references to objects"""
def __init__(self) -> None:
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
self.xrefs: Dict[Tuple[int, int], XRefRow] = {}
def update_from_xrefiter(self, it_xref: Iterator[XRefRow]):
for xref in it_xref:
if xref.use == b"n":
self.xrefs[(xref.obj_num, xref.gen_num)] = xref
def save_indobject(self, indobj: IndirectObject):
if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None:
self.objs[(indobj.obj_num, indobj.gen_num)] = indobj
else:
print("WARN: saving twice", indobj)
+21
View File
@@ -0,0 +1,21 @@
from pdfmajor.exceptions import PDFMajorException, BrokenFile
class XRefError(PDFMajorException):
pass
class PDFNoValidXRef(XRefError, BrokenFile, EOFError):
pass
class InvalidXref(XRefError, KeyError):
pass
class InvalidIndirectObjAccess(XRefError):
pass
class UnexpectedEOF(BrokenFile, EOFError):
pass
@@ -1,8 +1,8 @@
# from pdfmajor.parser.objects.indirect import IndirectObjectCollection
from typing import Iterator, Literal, NamedTuple, Union
from pdfmajor.lexer.token import TokenInteger, TokenKeyword
from pdfmajor.lexer import iter_tokens
from typing import Iterator, Literal, NamedTuple, Union
from pdfmajor.streambuffer import BufferStream
from .exceptions import PDFNoValidXRef, BrokenFile, UnexpectedEOF
+51
View File
@@ -0,0 +1,51 @@
from typing import Dict, Iterator, Tuple
from pdfmajor.parser.objects.indirect import IndirectObject
from .xref import XRefRow
from .exceptions import InvalidIndirectObjAccess
class XRefDB:
"""A class for collection references to objects"""
def __init__(self) -> None:
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
self.xrefs: Dict[Tuple[int, int], XRefRow] = {}
@classmethod
def from_xrefiter(cls, it_xref: Iterator[XRefRow]) -> "XRefDB":
db = cls()
db.update_from_xrefiter(it_xref)
return db
def has_xref(self, obj_num: int, gen_num: int) -> bool:
return (obj_num, gen_num) in self.xrefs.keys()
def has_obj(self, obj_num: int, gen_num: int) -> bool:
return (obj_num, gen_num) in self.objs.keys()
def update_from_xrefiter(self, it_xref: Iterator[XRefRow]):
for xref in it_xref:
if xref.use == b"n":
self.xrefs[(xref.obj_num, xref.gen_num)] = xref
def create_indobject(
self, obj_num: int, gen_num: int, offset: int
) -> IndirectObject:
if self.has_obj(obj_num, gen_num):
raise InvalidIndirectObjAccess(
f"Attempted to recreate {(obj_num, gen_num)}"
)
cur_obj = self.objs[(obj_num, gen_num)] = IndirectObject(
obj_num, gen_num, offset
)
return cur_obj
def get_or_create_indobject(self, obj_num: int, gen_num: int) -> IndirectObject:
cur_obj = self.objs.get((obj_num, gen_num), None)
if cur_obj is None:
print("WARN: got a non-existent reference")
cur_obj = self.objs[(obj_num, gen_num)] = IndirectObject(
obj_num, gen_num, -1
)
return cur_obj