split out xref from parser
This commit is contained in:
1 parent
5af06a3f1e
commit
78175d3a64
14 files changed
+290
-154
No files matched your search
@@ -1,2 +1,6 @@
|
||||
class PDFMajorException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class BrokenFile(PDFMajorException):
|
||||
pass
|
||||
+69
-58
@@ -1,6 +1,4 @@
|
||||
from pdfmajor.parser.stream.PDFStream import PDFStream
|
||||
from pdfmajor.parser.xref import XRefRow
|
||||
from pdfmajor.parser.xrefdb import XRefDB
|
||||
from typing import Iterator, Optional
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.lexer.token import (
|
||||
@@ -12,7 +10,7 @@ from pdfmajor.lexer.token import (
|
||||
TokenKeyword,
|
||||
)
|
||||
from pdfmajor.lexer import iter_tokens
|
||||
from .objects.indirect import IndirectObject
|
||||
from .objects.indirect import IndirectObject, ObjectRef
|
||||
from .objects.base import PDFObject
|
||||
from .objects.collections import PDFArray, PDFDictionary
|
||||
from .objects.comment import PDFComment
|
||||
@@ -22,14 +20,12 @@ from .exceptions import (
|
||||
InvalidKeywordPos,
|
||||
BrokenFile,
|
||||
InvalidKeywordPos,
|
||||
InvalidXref,
|
||||
ParserError,
|
||||
)
|
||||
|
||||
|
||||
def iter_objects(
|
||||
buffer: BufferStream,
|
||||
xrefdb: XRefDB,
|
||||
state: Optional[ParsingState] = None,
|
||||
) -> Iterator[PDFObject]:
|
||||
"""Iterates over the objects in the current byte stream
|
||||
@@ -51,18 +47,16 @@ def iter_objects(
|
||||
)
|
||||
elif isinstance(token, TokenKeyword):
|
||||
if token.value == b"R":
|
||||
yield from _on_indirect_ref_close(buffer, state, xrefdb)
|
||||
yield from _on_indirect_ref_close(state)
|
||||
elif token.value == b"obj":
|
||||
state.initialize_indirect_obj(token.start_loc)
|
||||
elif token.value == b"endobj":
|
||||
obj = _on_endobj(state, token)
|
||||
xrefdb.save_indobject(obj)
|
||||
yield obj
|
||||
elif token.value == b"stream":
|
||||
_on_stream(
|
||||
buffer,
|
||||
state,
|
||||
xrefdb,
|
||||
token,
|
||||
)
|
||||
else:
|
||||
@@ -106,9 +100,7 @@ def _on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject:
|
||||
raise ParserError(f"Recived an endobj while not procesing an indirect object")
|
||||
|
||||
|
||||
def _on_stream(
|
||||
buffer: BufferStream, state: ParsingState, xrefdb: XRefDB, token: TokenKeyword
|
||||
):
|
||||
def _on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
|
||||
"""Initilize a PDFStream
|
||||
|
||||
Args:
|
||||
@@ -119,10 +111,10 @@ def _on_stream(
|
||||
"""
|
||||
last_ctx = state.current_context
|
||||
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
|
||||
xrefdb.save_indobject(last_ctx)
|
||||
# stream = PDFStream(token.end_loc + 1, 0)
|
||||
obj = _save_or_get_object(buffer, xrefdb, last_ctx.obj_num, last_ctx.gen_num)
|
||||
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc, obj)
|
||||
obj = last_ctx.get_object()
|
||||
if not isinstance(obj, PDFDictionary):
|
||||
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
|
||||
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
|
||||
buffer.seek(stream.offset + stream.length)
|
||||
next_token = next(iter_tokens(buffer))
|
||||
if not (
|
||||
@@ -135,9 +127,9 @@ def _on_stream(
|
||||
raise ParserError(f"PDFStream is missing initilization dictionary")
|
||||
|
||||
|
||||
def _on_indirect_ref_close(buffer: BufferStream, state: ParsingState, xrefdb: XRefDB):
|
||||
def _on_indirect_ref_close(state: ParsingState):
|
||||
obj_num, gen_num = state.get_indobj_values()
|
||||
obj = _save_or_get_object(buffer, xrefdb, obj_num, gen_num)
|
||||
obj = ObjectRef(obj_num, gen_num)
|
||||
cur_ctx = state.current_context
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
@@ -145,50 +137,69 @@ def _on_indirect_ref_close(buffer: BufferStream, state: ParsingState, xrefdb: XR
|
||||
yield obj
|
||||
|
||||
|
||||
def _save_or_get_object(
|
||||
buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int
|
||||
) -> PDFObject:
|
||||
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
|
||||
# def _save_or_get_object(
|
||||
# buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int
|
||||
# ) -> PDFObject:
|
||||
# """get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
|
||||
|
||||
Args:
|
||||
obj_num (int): object number
|
||||
gen_num (int): generation number
|
||||
# Args:
|
||||
# obj_num (int): object number
|
||||
# gen_num (int): generation number
|
||||
|
||||
Returns:
|
||||
PDFObject
|
||||
"""
|
||||
indobj = xrefdb.objs.get((obj_num, gen_num), None)
|
||||
if indobj is None:
|
||||
xref = xrefdb.xrefs.get((obj_num, gen_num), None)
|
||||
if xref is None:
|
||||
raise InvalidXref(
|
||||
f"xref not found for object retrieval {obj_num}, {gen_num}"
|
||||
)
|
||||
return _read_object_from_xref(xref, xrefdb, buffer)
|
||||
else:
|
||||
return _read_object_from_db_or_buffer(indobj, xrefdb, buffer)
|
||||
# Returns:
|
||||
# PDFObject
|
||||
# """
|
||||
# indobj = xrefdb.objs.get((obj_num, gen_num), None)
|
||||
# print("trying to read", obj_num, gen_num)
|
||||
# if indobj is None:
|
||||
# xref = xrefdb.xrefs.get((obj_num, gen_num), None)
|
||||
# if xref is None:
|
||||
# raise InvalidXref(
|
||||
# f"xref not found for object retrieval {obj_num}, {gen_num}"
|
||||
# )
|
||||
# return _read_object_from_xref(xref, xrefdb, buffer)
|
||||
# else:
|
||||
# return _read_object_from_db_or_buffer(indobj, xrefdb, buffer)
|
||||
|
||||
|
||||
def _read_object_from_xref(
|
||||
xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream
|
||||
) -> PDFObject:
|
||||
indobj = IndirectObject(xref.obj_num, xref.gen_num, xref.offset)
|
||||
with buffer.get_window():
|
||||
buffer.seek(indobj.offset)
|
||||
obj = next(iter_objects(buffer, xrefdb))
|
||||
indobj.save_object(obj)
|
||||
xrefdb.save_indobject(indobj)
|
||||
return obj
|
||||
# def _read_object_from_xref(
|
||||
# xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream
|
||||
# ) -> PDFObject:
|
||||
# with buffer.get_window():
|
||||
# buffer.seek(xref.offset)
|
||||
# obj = next(iter_objects(buffer, xrefdb))
|
||||
# if (
|
||||
# not isinstance(obj, IndirectObject)
|
||||
# or obj.gen_num != xref.gen_num
|
||||
# or obj.obj_num != xref.obj_num
|
||||
# ):
|
||||
# raise ParserError(
|
||||
# f"Invalid offset for indirect object, expected {xref} but got {obj}"
|
||||
# )
|
||||
# return obj
|
||||
|
||||
|
||||
def _read_object_from_db_or_buffer(
|
||||
indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream
|
||||
) -> PDFObject:
|
||||
obj = indobj.get_object()
|
||||
if obj is not None:
|
||||
return obj
|
||||
with buffer.get_window():
|
||||
buffer.seek(indobj.offset)
|
||||
obj = next(iter_objects(buffer, xrefdb))
|
||||
indobj.save_object(obj)
|
||||
return obj
|
||||
# def _read_object_from_db_or_buffer(
|
||||
# indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream
|
||||
# ) -> PDFObject:
|
||||
# if indobj.read:
|
||||
# return indobj
|
||||
# with buffer.get_window():
|
||||
# buffer.seek(indobj.offset)
|
||||
# next_token = next(iter_tokens(buffer))
|
||||
# if next_token.value != b"obj":
|
||||
# if isinstance(next_token, TokenKeyword):
|
||||
# raise InvalidKeywordPos(next_token, [b"obj"])
|
||||
# else:
|
||||
# raise ParserError(f"expected a keyword instead got {next_token}")
|
||||
# obj = next(
|
||||
# iter_objects(
|
||||
# buffer,
|
||||
# xrefdb,
|
||||
# ParsingState([indobj], []),
|
||||
# )
|
||||
# )
|
||||
# if obj != indobj:
|
||||
# raise ParserError(f"got a different indobj {obj} != {indobj}")
|
||||
# indobj.read = True
|
||||
# return obj
|
||||
@@ -1,21 +1,21 @@
|
||||
from typing import List
|
||||
from pdfmajor.lexer.token import TokenKeyword
|
||||
from pdfmajor.exceptions import PDFMajorException
|
||||
from pdfmajor.exceptions import PDFMajorException, BrokenFile
|
||||
|
||||
|
||||
class ParserError(PDFMajorException):
|
||||
pass
|
||||
|
||||
|
||||
class BrokenFile(ParserError):
|
||||
class BrokenFileParserError(BrokenFile):
|
||||
pass
|
||||
|
||||
|
||||
class UnexpectedEOF(BrokenFile, EOFError):
|
||||
class UnexpectedEOF(BrokenFileParserError, EOFError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidKeywordPos(BrokenFile):
|
||||
class InvalidKeywordPos(BrokenFileParserError):
|
||||
def __init__(self, token: TokenKeyword, expected_token: List[bytes]) -> None:
|
||||
self.token = token
|
||||
super().__init__(
|
||||
@@ -23,21 +23,9 @@ class InvalidKeywordPos(BrokenFile):
|
||||
)
|
||||
|
||||
|
||||
class InvalidIndirectObjAccess(ParserError):
|
||||
pass
|
||||
|
||||
|
||||
class DecodeFailed(BrokenFile):
|
||||
class DecodeFailed(BrokenFileParserError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError):
|
||||
pass
|
||||
|
||||
|
||||
class PDFNoValidXRef(BrokenFile, EOFError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidXref(ParserError, KeyError):
|
||||
pass
|
||||
@@ -4,9 +4,20 @@ from pdfmajor.parser.stream.PDFStream import PDFStream
|
||||
from .base import PDFContextualObject, PDFObject
|
||||
|
||||
|
||||
class IndirectObject(PDFContextualObject):
|
||||
class ObjectRef(PDFObject):
|
||||
"""A class representing a reference for other objects"""
|
||||
|
||||
def __init__(self, obj_num: int, gen_num: int) -> None:
|
||||
self.obj_num = obj_num
|
||||
self.gen_num = gen_num
|
||||
|
||||
def to_python(self):
|
||||
return (self.obj_num, self.gen_num)
|
||||
|
||||
|
||||
class IndirectObject(PDFContextualObject):
|
||||
"""A class representing a referencable object"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
obj_num: int,
|
||||
@@ -20,6 +31,7 @@ class IndirectObject(PDFContextualObject):
|
||||
self.gen_num = gen_num
|
||||
self.__data = data
|
||||
self.stream = stream
|
||||
self.read: bool = False
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"IndirectObject(obj_num={self.obj_num}, gen_num={self.gen_num}, offset={self.offset}, data={self.get_object()})"
|
||||
@@ -37,10 +49,10 @@ class IndirectObject(PDFContextualObject):
|
||||
self.__data = obj
|
||||
|
||||
def to_python(self):
|
||||
obj = self.get_object()
|
||||
if obj is None:
|
||||
return None
|
||||
return obj.to_python()
|
||||
data = self.get_object()
|
||||
if data is not None:
|
||||
data = data.to_python()
|
||||
return (self.obj_num, self.gen_num, self.offset, data)
|
||||
|
||||
def pass_item(self, item: PDFObject):
|
||||
"""This method will either propogate the item to it's child (in case the child is a contexual object) or override it's own data with the incoming
|
||||
|
||||
@@ -32,10 +32,8 @@ def attempt_parse_prim(token: Token, state: ParsingState) -> ParsingAttempt:
|
||||
if len(state.int_collection) <= 1:
|
||||
state.int_collection.append(prim_obj)
|
||||
else:
|
||||
for last_num in state.int_collection:
|
||||
out.append(last_num)
|
||||
state.int_collection = []
|
||||
out.append(prim_obj)
|
||||
for obj in state.flush_int_collections():
|
||||
out.append(obj)
|
||||
else:
|
||||
for obj in state.flush_int_collections():
|
||||
out.append(obj)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Iterator, List, Optional, Tuple
|
||||
from typing import Iterator, List, Optional, Tuple, Union
|
||||
|
||||
from .exceptions import ParserError
|
||||
from .objects.base import PDFContextualObject, PDFObject
|
||||
@@ -14,7 +14,7 @@ class ParsingState:
|
||||
def __init__(
|
||||
self,
|
||||
context_stack: List[PDFContextualObject],
|
||||
int_collection: List[PDFInteger],
|
||||
int_collection: List[Union[PDFInteger, int]],
|
||||
last_obj: Optional[PDFObject] = None,
|
||||
) -> None:
|
||||
self.context_stack = context_stack
|
||||
@@ -68,8 +68,12 @@ class ParsingState:
|
||||
"""
|
||||
if len(self.int_collection) == 2:
|
||||
gen_num = self.int_collection.pop()
|
||||
if isinstance(gen_num, PDFInteger):
|
||||
gen_num = gen_num.to_python()
|
||||
obj_num = self.int_collection.pop()
|
||||
return (obj_num.to_python(), gen_num.to_python())
|
||||
if isinstance(obj_num, PDFInteger):
|
||||
obj_num = obj_num.to_python()
|
||||
return (obj_num, gen_num)
|
||||
raise ParserError("missing leading tokens for object reference")
|
||||
|
||||
def initialize_indirect_obj(self, offset: int):
|
||||
@@ -92,9 +96,17 @@ class ParsingState:
|
||||
cur_ctx = self.current_context
|
||||
if cur_ctx is not None:
|
||||
for obj in self.int_collection:
|
||||
if not isinstance(obj, PDFInteger):
|
||||
raise ParserError(
|
||||
f"Invalid int-collection contains none PDFInteger types {obj}"
|
||||
)
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
for obj in self.int_collection:
|
||||
if not isinstance(obj, PDFInteger):
|
||||
raise ParserError(
|
||||
f"Invalid int-collection contains none PDFInteger types {obj}"
|
||||
)
|
||||
yield obj
|
||||
self.int_collection = []
|
||||
|
||||
|
||||
@@ -40,9 +40,7 @@ class PDFStream:
|
||||
self.dl: Optional[int] = dl
|
||||
|
||||
@classmethod
|
||||
def from_pdfdict(cls, offset: int, item: PDFObject) -> "PDFStream":
|
||||
if not isinstance(item, PDFDictionary):
|
||||
raise ParserError(f"Cannot initilize a pdfstream with {type(item)}")
|
||||
def from_pdfdict(cls, offset: int, item: PDFDictionary) -> "PDFStream":
|
||||
data = item.to_python()
|
||||
stream = cls(offset, data["Length"])
|
||||
stream.filter = get_single_or_list(data.get("Filter", stream.filter))
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
from pdfmajor.parser.xref import XRefRow
|
||||
|
||||
from typing import Dict, Iterator, Tuple
|
||||
|
||||
from pdfmajor.parser.exceptions import InvalidXref
|
||||
from pdfmajor.parser.objects.base import PDFObject
|
||||
from .objects.indirect import IndirectObject
|
||||
from .objects.base import PDFObject
|
||||
|
||||
|
||||
class XRefDB:
|
||||
"""A class for collection references to objects"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
|
||||
self.xrefs: Dict[Tuple[int, int], XRefRow] = {}
|
||||
|
||||
def update_from_xrefiter(self, it_xref: Iterator[XRefRow]):
|
||||
for xref in it_xref:
|
||||
if xref.use == b"n":
|
||||
self.xrefs[(xref.obj_num, xref.gen_num)] = xref
|
||||
|
||||
def save_indobject(self, indobj: IndirectObject):
|
||||
if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None:
|
||||
self.objs[(indobj.obj_num, indobj.gen_num)] = indobj
|
||||
else:
|
||||
print("WARN: saving twice", indobj)
|
||||
@@ -0,0 +1,21 @@
|
||||
from pdfmajor.exceptions import PDFMajorException, BrokenFile
|
||||
|
||||
|
||||
class XRefError(PDFMajorException):
|
||||
pass
|
||||
|
||||
|
||||
class PDFNoValidXRef(XRefError, BrokenFile, EOFError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidXref(XRefError, KeyError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidIndirectObjAccess(XRefError):
|
||||
pass
|
||||
|
||||
|
||||
class UnexpectedEOF(BrokenFile, EOFError):
|
||||
pass
|
||||
@@ -1,8 +1,8 @@
|
||||
# from pdfmajor.parser.objects.indirect import IndirectObjectCollection
|
||||
from typing import Iterator, Literal, NamedTuple, Union
|
||||
from pdfmajor.lexer.token import TokenInteger, TokenKeyword
|
||||
from pdfmajor.lexer import iter_tokens
|
||||
from typing import Iterator, Literal, NamedTuple, Union
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
|
||||
from .exceptions import PDFNoValidXRef, BrokenFile, UnexpectedEOF
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
from typing import Dict, Iterator, Tuple
|
||||
|
||||
from pdfmajor.parser.objects.indirect import IndirectObject
|
||||
from .xref import XRefRow
|
||||
from .exceptions import InvalidIndirectObjAccess
|
||||
|
||||
|
||||
class XRefDB:
|
||||
"""A class for collection references to objects"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
|
||||
self.xrefs: Dict[Tuple[int, int], XRefRow] = {}
|
||||
|
||||
@classmethod
|
||||
def from_xrefiter(cls, it_xref: Iterator[XRefRow]) -> "XRefDB":
|
||||
db = cls()
|
||||
db.update_from_xrefiter(it_xref)
|
||||
return db
|
||||
|
||||
def has_xref(self, obj_num: int, gen_num: int) -> bool:
|
||||
return (obj_num, gen_num) in self.xrefs.keys()
|
||||
|
||||
def has_obj(self, obj_num: int, gen_num: int) -> bool:
|
||||
return (obj_num, gen_num) in self.objs.keys()
|
||||
|
||||
def update_from_xrefiter(self, it_xref: Iterator[XRefRow]):
|
||||
for xref in it_xref:
|
||||
if xref.use == b"n":
|
||||
self.xrefs[(xref.obj_num, xref.gen_num)] = xref
|
||||
|
||||
def create_indobject(
|
||||
self, obj_num: int, gen_num: int, offset: int
|
||||
) -> IndirectObject:
|
||||
if self.has_obj(obj_num, gen_num):
|
||||
raise InvalidIndirectObjAccess(
|
||||
f"Attempted to recreate {(obj_num, gen_num)}"
|
||||
)
|
||||
cur_obj = self.objs[(obj_num, gen_num)] = IndirectObject(
|
||||
obj_num, gen_num, offset
|
||||
)
|
||||
return cur_obj
|
||||
|
||||
def get_or_create_indobject(self, obj_num: int, gen_num: int) -> IndirectObject:
|
||||
cur_obj = self.objs.get((obj_num, gen_num), None)
|
||||
if cur_obj is None:
|
||||
print("WARN: got a non-existent reference")
|
||||
cur_obj = self.objs[(obj_num, gen_num)] = IndirectObject(
|
||||
obj_num, gen_num, -1
|
||||
)
|
||||
return cur_obj
|
||||
Reference in new issue
Block a user