working xref db

This commit is contained in:
ari committed 2020-10-23 22:17:50 -04:00
1 parent cfc2dfca47
commit e2a92517cd
12 files changed
+517 -227

No files matched your search

+13 -2
View File
@@ -19,8 +19,19 @@ from pdfmajor.streambuffer import BufferStream
class PDFLexer: class PDFLexer:
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None: def __init__(self, buffer: BufferStream) -> None:
self.buffer = BufferStream(fp, buffer_size=buffer_size) self.buffer = buffer
@classmethod
def from_iobuffer(cls, fp: io.BufferedIOBase, buffer_size: int = 4096):
return cls(BufferStream(fp, buffer_size=buffer_size))
@classmethod
def from_bytes(cls, data: bytes, buffer_size: int = 4096):
return cls(BufferStream(io.BytesIO(data), buffer_size=buffer_size))
def tell(self) -> int:
return self.buffer.tell()
def seek(self, pos: int) -> int: def seek(self, pos: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset """moves the offset of the reader to a certain spot, and returns the new offset
+162 -74
View File
@@ -1,7 +1,12 @@
import io import io
from pdfmajor.parser.objects.stream import PDFStream
from typing import Iterator, List, Optional, Union
from typing import Dict, Iterator, Optional, Tuple
from pdfmajor.streambuffer import BufferStream
from pdfmajor.parser.objects.stream import PDFStream
from pdfmajor.parser.exceptions import InvalidXref
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.lexer.token import TokenKeyword
from pdfmajor.lexer import PDFLexer from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.token import ( from pdfmajor.lexer.token import (
TArrayValue, TArrayValue,
@@ -11,15 +16,54 @@ from pdfmajor.lexer.token import (
TokenDictionary, TokenDictionary,
TokenKeyword, TokenKeyword,
) )
from .objects.indirect import IndirectObject
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import InvalidKeywordPos, ParserError
from .objects.base import PDFContextualObject, PDFObject from .objects.base import PDFContextualObject, PDFObject
from .objects.collections import PDFArray, PDFDictionary from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment from .objects.comment import PDFComment
from .objects.indirect import IndirectObject from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import InvalidKeywordPos, BrokenFile, InvalidKeywordPos, ParserError
class XRefDB:
"""A class for collection references to objects"""
def __init__(self, parser: "PDFParser") -> None:
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
self.parser = parser
def save_indobject(self, indobj: IndirectObject):
if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None:
self.objs[(indobj.obj_num, indobj.gen_num)] = indobj
def get_object_from_indobj(self, indobj: IndirectObject) -> PDFObject:
return self.get_object(indobj.obj_num, indobj.gen_num)
def get_object(self, obj_num: int, gen_num: int) -> PDFObject:
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
Args:
obj_num (int): object number
gen_num (int): generation number
Returns:
PDFObject
"""
indobj = self.objs.get((obj_num, gen_num), None)
if indobj is None:
raise InvalidXref(
f"xref not found for object retrieval {obj_num}, {gen_num}"
)
else:
obj = indobj.get_object()
if obj is not None:
return obj
cur_pos = self.parser.tell()
self.parser.seek(indobj.offset)
obj = next(self.parser.iter_objects())
indobj.save_object(obj)
self.parser.seek(cur_pos)
return obj
class PDFParser: class PDFParser:
@@ -27,11 +71,9 @@ class PDFParser:
def __init__( def __init__(
self, self,
fp: io.BufferedIOBase, buffer: BufferStream,
buffer_size: int = 4096, db: Optional[XRefDB] = None,
state: Optional[ParsingState] = None,
strict: bool = True, strict: bool = True,
retain_obj_on_seek: bool = False,
) -> None: ) -> None:
""" """
@@ -40,48 +82,62 @@ class PDFParser:
buffer_size (int, optional): the size of the reads. Defaults to 4096. buffer_size (int, optional): the size of the reads. Defaults to 4096.
state (Optional[ParsingState], optional): the parsing state. Defaults to None. state (Optional[ParsingState], optional): the parsing state. Defaults to None.
strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True. strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True.
retain_obj_on_seek (bool, optional): whether we want to ensure we keep the object collection when changing the offset using the self.seek methodm this does not apply if you specify the state when using the seek method!. Defaults to False.
""" """
self.lexer = PDFLexer(fp, buffer_size) self.lexer = PDFLexer(buffer)
self.db = XRefDB(self)
self.strict = strict self.strict = strict
self.state = ParsingState([], []) if state is None else state
self.retain_obj_on_seek = retain_obj_on_seek
def seek(self, offset: int, state: Optional[ParsingState] = None) -> int: @classmethod
def from_iobuffer(
cls,
fp: io.BufferedIOBase,
buffer_size: int = 4096,
strict: bool = True,
):
return cls(
BufferStream(fp, buffer_size=buffer_size),
strict=strict,
)
@classmethod
def from_bytes(
cls,
data: bytes,
buffer_size: int = 4096,
strict: bool = True,
):
return cls(
BufferStream(io.BytesIO(data), buffer_size=buffer_size),
strict=strict,
)
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset """moves the offset of the reader to a certain spot, and returns the new offset
Args: Args:
offset (int) offset (int)
state (Optional[ParsingState]): this will override our current state
Returns: Returns:
int: new offset int: new offset
""" """
self.state = (
ParsingState(
[],
[],
indirect_object_collection=self.state.indirect_object_collection
if not self.retain_obj_on_seek
else None,
)
if state is None
else state
)
return self.lexer.seek(offset) return self.lexer.seek(offset)
def tell(self) -> int:
return self.lexer.tell()
def iter_objects(self) -> Iterator[PDFObject]: def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream """Iterates over the objects in the current byte stream
Yields: Yields:
PDFObject PDFObject
""" """
state = ParsingState([], [], strict=self.strict)
for token in self.lexer.iter_tokens(): for token in self.lexer.iter_tokens():
attempt_prim = attempt_parse_prim(token, self.state) attempt_prim = attempt_parse_prim(token, state)
if attempt_prim.success: if attempt_prim.success:
yield from self._feed_or_yield_objs(attempt_prim.next_objs) yield from state.feed_or_yield_objs(attempt_prim.next_objs)
elif isinstance(token, TokenComment): elif isinstance(token, TokenComment):
yield self.state.set_last_obj( yield state.set_last_obj(
PDFComment( PDFComment(
token.value, token.value,
(token.start_loc, token.end_loc), (token.start_loc, token.end_loc),
@@ -89,67 +145,99 @@ class PDFParser:
) )
elif isinstance(token, TokenKeyword): elif isinstance(token, TokenKeyword):
if token.value == b"R": if token.value == b"R":
yield from self._on_indirect_ref_close() yield from self._on_indirect_ref_close(state)
elif token.value == b"obj": elif token.value == b"obj":
self.state.initialize_indirect_obj() state.initialize_indirect_obj(token.start_loc)
elif token.value == b"endobj": elif token.value == b"endobj":
yield from self._on_endobj(token) yield self._on_endobj(state, token)
elif token.value == b"stream": elif token.value == b"stream":
self._on_stream(token) self._on_stream(
state,
token,
)
elif token.value == b"endstream": elif token.value == b"endstream":
self._on_endstream(token) self._on_endstream(state, token)
else: else:
raise InvalidKeywordPos(token) raise InvalidKeywordPos(token)
elif isinstance(token, TokenArray): elif isinstance(token, TokenArray):
yield from self.state.flush_int_collections() yield from state.flush_int_collections()
yield from deal_with_collection_object( yield from deal_with_collection_object(
self.state, token, PDFArray, TArrayValue.OPEN, strict=self.strict state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
) )
elif isinstance(token, TokenDictionary): elif isinstance(token, TokenDictionary):
yield from self.state.flush_int_collections() yield from state.flush_int_collections()
yield from deal_with_collection_object( yield from deal_with_collection_object(
self.state, state,
token=token, token=token,
cls_const=PDFDictionary, cls_const=PDFDictionary,
open_t=TDictValue.OPEN, open_t=TDictValue.OPEN,
strict=self.strict, strict=self.strict,
) )
def _feed_or_yield_objs(self, next_objs: List[PDFObject]): def _on_endobj(self, state: ParsingState, token: TokenKeyword) -> IndirectObject:
cur_ctx = self.state.current_context """finish off the current context, assuming we are handling an indirect-object
for obj in next_objs:
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
def _on_indirect_ref_close(self): Args:
obj = self.state.get_indobjects() token (TokenKeyword)
cur_ctx = self.state.current_context
Raises:
BrokenFile: if the object fif not get initilized with two integers
InvalidKeywordPos: If the context is not an indirect object
Yields:
PDFObject
"""
last_ctx = state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
if len(state.int_collection) > 1:
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
for num in state.flush_int_collections():
last_ctx.pass_item(num)
obj = last_ctx.into_readonly_copy()
self.db.save_indobject(obj)
return obj
else:
raise InvalidKeywordPos(token)
def _on_stream(self, state: ParsingState, token: TokenKeyword):
"""Initilize a PDFStream
Args:
token (TokenKeyword)
Raises:
ParserError: if the current context is not an indirect object
"""
last_ctx = state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
self.db.save_indobject(last_ctx)
stream = PDFStream(token.end_loc, 0)
obj = self.db.get_object_from_indobj(last_ctx)
stream.pass_item(obj)
last_ctx.save_object(PDFStream(token.end_loc, 0))
state.context_stack.append(stream)
self.seek(stream.offset + stream.length)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
def _on_endstream(self, state: ParsingState, token: TokenKeyword):
"""Finish a PDFStream
Args:
token (TokenKeyword)
Raises:
InvalidKeywordPos: if the current context is not an indirect stream
"""
stream: PDFContextualObject = state.context_stack.pop()
if not isinstance(stream, PDFStream):
raise InvalidKeywordPos(token)
def _on_indirect_ref_close(self, state: ParsingState):
obj_num, gen_num = state.get_indobj_values()
obj = self.db.get_object(obj_num, gen_num)
cur_ctx = state.current_context
if cur_ctx is not None: if cur_ctx is not None:
cur_ctx.pass_item(obj) cur_ctx.pass_item(obj)
else: else:
yield obj yield obj
def _on_endobj(self, token: TokenKeyword):
last_ctx = self.state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
yield last_ctx.clone()
elif self.strict:
raise InvalidKeywordPos(token)
def _on_stream(self, token: TokenKeyword):
last_ctx = self.state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
stream = PDFStream(token.end_loc, 0)
stream.pass_item(last_ctx.get_object())
last_ctx.save_object(PDFStream(token.end_loc, 0))
self.state.context_stack.append(stream)
self.lexer.seek(stream.offset + stream.length)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
def _on_endstream(self, token: TokenKeyword):
stream: PDFContextualObject = self.state.context_stack.pop()
if not isinstance(stream, PDFStream):
raise InvalidKeywordPos(token)
+7 -11
View File
@@ -1,7 +1,6 @@
import io import io
from pdfmajor.streambuffer import BufferStream
from .objects.stream import PDFStream from .objects.stream import PDFStream
from .state import ParsingState
from .PDFParser import PDFParser from .PDFParser import PDFParser
from .fliters import process_filters_on_data from .fliters import process_filters_on_data
@@ -10,20 +9,17 @@ class PDFStreamPasrser(PDFParser):
def __init__( def __init__(
self, self,
stream: PDFStream, stream: PDFStream,
fp: io.BufferedIOBase, buffer: BufferStream,
state: ParsingState,
buffer_size: int = 4096,
strict: bool = True, strict: bool = True,
) -> None: ) -> None:
fp.seek(stream.offset) cur_pos = buffer.tell()
data = fp.read(stream.length) buffer.seek(stream.offset)
data = buffer.read(stream.length).data
buffer.seek(cur_pos)
data = process_filters_on_data(data, stream.filter, stream.decode_parms) data = process_filters_on_data(data, stream.filter, stream.decode_parms)
data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms) data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms)
self.stream = stream self.stream = stream
super().__init__( super().__init__(
fp=io.BytesIO(data), buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length),
buffer_size=buffer_size,
state=state,
strict=strict, strict=strict,
retain_obj_on_seek=True,
) )
+18 -2
View File
@@ -6,7 +6,15 @@ class ParserError(PDFMajorException):
pass pass
class InvalidKeywordPos(ParserError): class BrokenFile(ParserError):
pass
class UnexpectedEOF(BrokenFile, EOFError):
pass
class InvalidKeywordPos(BrokenFile):
def __init__(self, token: TokenKeyword) -> None: def __init__(self, token: TokenKeyword) -> None:
self.token = token self.token = token
super().__init__( super().__init__(
@@ -18,9 +26,17 @@ class InvalidIndirectObjAccess(ParserError):
pass pass
class DecodeFailed(ParserError): class DecodeFailed(BrokenFile):
pass pass
class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError): class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError):
pass pass
class PDFNoValidXRef(BrokenFile, EOFError):
pass
class InvalidXref(ParserError, KeyError):
pass
+7
View File
@@ -23,6 +23,13 @@ class PDFDictionary(Dict[str, PDFObject], PDFContextualObject):
self.last_name: Optional[PDFName] = None self.last_name: Optional[PDFName] = None
self.strict = strict self.strict = strict
@classmethod
def from_dict(cls, d: Dict[str, PDFObject]):
new_obj = cls()
for key, val in d.items():
new_obj[key] = val
return new_obj
@property @property
def last_item_value(self): def last_item_value(self):
return self[self.last_name] return self[self.last_name]
+27 -60
View File
@@ -1,88 +1,55 @@
from typing import Dict, Tuple from typing import Optional
from ..exceptions import InvalidIndirectObjAccess, ParserError from ..exceptions import InvalidIndirectObjAccess
from .base import PDFContextualObject, PDFObject from .base import PDFContextualObject, PDFObject
from .primitives import PDFNull from .primitives import PDFNull
class IndirectObjectCollection:
"""A class for collection references to objects"""
def __init__(self) -> None:
self.objs: Dict[Tuple[int, int], PDFObject] = {}
def create_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject":
"""Create an indirect object and store its value in the collection
return the reference to where this object lies at
Returns:
IndirectObject:
"""
return IndirectObject(self, obj_num, gen_num, read_only=False)
def get_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject":
"""creates a reference to an object
Returns:
IndirectObject
"""
return IndirectObject(self, obj_num, gen_num, read_only=True)
def get_object(self, obj_num: int, gen_num: int) -> PDFObject:
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
Args:
obj_num (int): object number
gen_num (int): generation number
Returns:
PDFObject
"""
return self.objs.get((obj_num, gen_num), PDFNull())
class IndirectObject(PDFContextualObject): class IndirectObject(PDFContextualObject):
"""A class representing a reference for other objects""" """A class representing a reference for other objects"""
def __init__( def __init__(
self, self,
collection: IndirectObjectCollection,
obj_num: int, obj_num: int,
gen_num: int, gen_num: int,
read_only: bool = True, offset: int,
data: Optional[PDFObject] = None,
) -> None: ) -> None:
self.offset = offset
self.obj_num = obj_num self.obj_num = obj_num
self.gen_num = gen_num self.gen_num = gen_num
self.__read_only = read_only self.__data = data
self.__col = collection
def __repr__(self) -> str: def __repr__(self) -> str:
return f"IndirectObject({self.obj_num}, {self.gen_num})" return f"IndirectObject(obj_num={self.obj_num}, gen_num={self.gen_num}, offset={self.offset}, data={self.get_object()})"
def get_object(self) -> PDFObject: def get_object(self) -> Optional[PDFObject]:
return self.__col.get_object(self.obj_num, self.gen_num) return self.__data
def save_object(self, obj: PDFObject): def save_object(self, obj: PDFObject):
if self.__read_only: # if self.__data is not None:
raise InvalidIndirectObjAccess( # raise InvalidIndirectObjAccess(
f"attempted to override a read-only reference of an object {obj}" # f"attempted to override an initilized reference of an object {obj}"
) # )
self.__col.objs[(self.obj_num, self.gen_num)] = obj self.__data = obj
def clone(self): def into_readonly_copy(self):
return IndirectObject(self.__col, self.obj_num, self.gen_num, True) return IndirectObject(
self.obj_num, self.gen_num, self.offset, self.get_object()
)
def to_python(self): def to_python(self):
return self.get_object().to_python() obj = self.get_object()
if obj is None:
return None
return obj.to_python()
def pass_item(self, item: PDFObject): def pass_item(self, item: PDFObject):
if self.__read_only: # if self.__read_only:
raise InvalidIndirectObjAccess( # raise InvalidIndirectObjAccess(
f"attempted to add to an read-only reference of an object {item}" # f"attempted to add to an read-only reference of an object {item}"
) # )
cur_obj = self.get_object() cur_obj = self.get_object()
if isinstance(cur_obj, PDFContextualObject): if isinstance(cur_obj, PDFContextualObject):
cur_obj.pass_item(item) cur_obj.pass_item(item)
elif isinstance(cur_obj, PDFNull):
self.save_object(item)
else: else:
raise InvalidIndirectObjAccess(f"Invalid redefinition of an object {item}") self.save_object(item)
+33 -25
View File
@@ -4,33 +4,32 @@ from .exceptions import ParserError
from .objects.base import PDFContextualObject, PDFObject from .objects.base import PDFContextualObject, PDFObject
from .objects.primitives import PDFInteger from .objects.primitives import PDFInteger
from .objects.comment import PDFComment from .objects.comment import PDFComment
from .objects.indirect import IndirectObject, IndirectObjectCollection from .objects.indirect import IndirectObject
class ParsingState: class ParsingState:
__slots__ = [ __slots__ = ["context_stack", "int_collection", "last_obj", "strict"]
"context_stack",
"int_collection",
"last_obj",
"indirect_object_collection",
]
def __init__( def __init__(
self, self,
context_stack: List[PDFContextualObject], context_stack: List[PDFContextualObject],
int_collection: List[PDFInteger], int_collection: List[PDFInteger],
last_obj: Optional[PDFObject] = None, last_obj: Optional[PDFObject] = None,
indirect_object_collection: Optional[IndirectObjectCollection] = None, strict: bool = False,
) -> None: ) -> None:
self.context_stack = context_stack self.context_stack = context_stack
self.int_collection = int_collection self.int_collection = int_collection
self.int_collection = int_collection
self.last_obj = last_obj self.last_obj = last_obj
self.indirect_object_collection = ( self.strict = strict
indirect_object_collection
if indirect_object_collection is not None def __repr__(self) -> str:
else IndirectObjectCollection() return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj}, strict={strict})".format(
stackc=len(self.context_stack),
cur_stack=self.current_context,
ints=self.int_collection,
lobj=self.last_obj,
strict=self.strict,
) )
@property @property
@@ -76,22 +75,15 @@ class ParsingState:
return (obj_num.to_python(), gen_num.to_python()) return (obj_num.to_python(), gen_num.to_python())
raise ParserError("missing leading tokens for object reference") raise ParserError("missing leading tokens for object reference")
def get_indobjects(self) -> IndirectObject: def initialize_indirect_obj(self, offset: int):
"""Gets an indirect object, based on the values in 'self.int_collection' """Initilizes an indirect object based off the values in self.int_collection
Raises: Args:
ParserError: if the int_collection does not have two elements offset (int): position for the 'obj' keyword
Returns:
IndirectObject
""" """
obj_num, gen_num = self.get_indobj_values()
return self.indirect_object_collection.get_indobject(obj_num, gen_num)
def initialize_indirect_obj(self):
obj_num, gen_num = self.get_indobj_values() obj_num, gen_num = self.get_indobj_values()
self.context_stack.append( self.context_stack.append(
self.indirect_object_collection.create_indobject(obj_num, gen_num) IndirectObject(obj_num=obj_num, gen_num=gen_num, offset=offset)
) )
def flush_int_collections(self) -> Iterator[PDFObject]: def flush_int_collections(self) -> Iterator[PDFObject]:
@@ -108,3 +100,19 @@ class ParsingState:
for obj in self.int_collection: for obj in self.int_collection:
yield obj yield obj
self.int_collection = [] self.int_collection = []
def feed_or_yield_objs(self, next_objs: List[PDFObject]) -> Iterator[PDFObject]:
"""If there context, then feed the objects into it, otherwise yield them
Args:
next_objs (List[PDFObject])
Yields:
PDFObject
"""
cur_ctx = self.current_context
for obj in next_objs:
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
+128
View File
@@ -1,3 +1,131 @@
# from pdfmajor.parser.objects.indirect import IndirectObjectCollection
from pdfmajor.lexer.token import TokenInteger, TokenKeyword
from pdfmajor.lexer import PDFLexer
from typing import Iterator, Literal, NamedTuple, Union
from pdfmajor.streambuffer import BufferStream
from .exceptions import PDFNoValidXRef, BrokenFile, UnexpectedEOF
class XRefRow(NamedTuple):
"""a parsed version of a row in the xref table
as defined in PDF spec 1.7 section 7.5.4
"""
offset: int
obj_num: int
gen_num: int
use: Union[Literal["n"], Literal["f"]]
def find_start_of_xref(buffer: BufferStream, strict: bool = False) -> int:
"""Internal function used to locate the start of the XRef table"""
prev = None
found_eof = False
for _, line in buffer.into_reverse_reader_iter():
line = line.strip()
found_eof |= line.strip() == b"%%EOF"
if line == b"startxref":
break
if line:
prev = line
if prev is None:
raise PDFNoValidXRef()
if strict and not found_eof:
raise BrokenFile(
"Failed to find %%EOF at end of file, this can indicate the file is incomplete or broken."
)
return int(prev)
def iter_over_xref(buffer: BufferStream, strict: bool = False) -> Iterator[XRefRow]:
start_offset = find_start_of_xref(buffer, strict)
buffer.seek(start_offset)
lexer = PDFLexer(buffer)
first_token = next(lexer.iter_tokens())
if isinstance(first_token, TokenInteger):
raise NotImplementedError("XRefStream: PDF-1.5")
elif isinstance(first_token, TokenKeyword):
if first_token.value == b"xref":
return iter_over_standard_xref(buffer, strict)
raise BrokenFile("Missing a valid XRef table")
# # read xref table
# def read_xref_from(self, parser, start, xrefs):
# """Reads XRefs from the given location."""
# parser.seek(start)
# parser.reset()
# try:
# (pos, token) = parser.nexttoken()
# except PSEOF:
# raise PDFNoValidXRef("Unexpected EOF")
# log.info("read_xref_from: start=%d, token=%r", start, token)
# if isinstance(token, int):
# # XRefStream: PDF-1.5
# parser.seek(pos)
# parser.reset()
# xref = PDFXRefStream()
# xref.load(parser)
# else:
# if token is parser.KEYWORD_XREF:
# parser.nextline()
# xref = PDFXRef()
# xref.load(parser)
# xrefs.append(xref)
# trailer = xref.get_trailer()
# log.info("trailer: %r", trailer)
# if "XRefStm" in trailer:
# pos = int_value(trailer["XRefStm"])
# self.read_xref_from(parser, pos, xrefs)
# if "Prev" in trailer:
# # find previous xref
# pos = int_value(trailer["Prev"])
# self.read_xref_from(parser, pos, xrefs)
# return
def iter_over_standard_xref(
buffer: BufferStream, strict: bool = False
) -> Iterator[XRefRow]:
while True:
bbline = buffer.get_firstnonempty_line()
if bbline is None:
raise UnexpectedEOF
pos, line = bbline
if line.startswith(b"trailer"):
buffer.seek(pos)
break
descriptors = line.strip().split(b" ")
if len(descriptors) != 2:
raise PDFNoValidXRef(f"Trailer not found: pos={pos} line={line!r}")
try:
(start, nobjs) = map(int, descriptors)
except ValueError:
raise PDFNoValidXRef(
f"Failed to decipher xref row: pos={pos} line={line!r}"
)
for obj_num in range(start, start + nobjs):
_, line = buffer.readline()
f = line.strip().split(b" ")
if len(f) != 3:
raise PDFNoValidXRef(f"Invalid XRef format: pos={pos} line={line!r}")
try:
pos = int(f[0])
gen_num = int(f[1])
use = f[2]
except ValueError:
raise PDFNoValidXRef(
f"Failed to decipher xref row: pos={pos} line={line!r}"
)
if use not in [b"n", b"f"] and not strict:
raise PDFNoValidXRef(
f"Invalid 'use' value for xref {use!r}: pos={pos} line={line!r}"
)
yield XRefRow(offset=pos, obj_num=obj_num, gen_num=gen_num, use=use) # type: ignore
# def construct_indobj_collection(
# buffer: BufferStream, it_xref: Iterator[XRefRow]
# ) -> IndirectObjectCollection:
# collection = IndirectObjectCollection()
# parser =
+65 -1
View File
@@ -1,5 +1,5 @@
import io import io
from typing import Iterator, NamedTuple from typing import Iterator, NamedTuple, Optional
from pdfmajor.exceptions import PDFMajorException from pdfmajor.exceptions import PDFMajorException
@@ -19,9 +19,35 @@ class BufferStream:
__slots__ = ["fp", "buffer_size"] __slots__ = ["fp", "buffer_size"]
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None: def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
"""[summary]
Args:
fp (io.BufferedIOBase)
buffer_size (int, optional): maximum length of the bytes this will read at each iteration. Defaults to 4096.
"""
self.fp = fp self.fp = fp
self.buffer_size = buffer_size self.buffer_size = buffer_size
def get_slice(
self, offset: int, length: int, buffer_size: Optional[int] = None
) -> "BufferStream":
"""Creates a slice of the current byte-stream and returns a BufferStream based on that slice
Args:
offset (int): where the slice starts
length (int): length of the slice
buffer_size (int, optional) buffer size for new slice. Defaults to self.buffer_size
Returns:
BufferStream
"""
cur_pos = self.tell()
self.seek(offset)
bslice = self.fp.read(length)
self.seek(cur_pos)
return BufferStream(
io.BytesIO(bslice),
buffer_size=buffer_size if buffer_size is not None else self.buffer_size,
)
def seekd(self, step: int) -> int: def seekd(self, step: int) -> int:
"""similar to self.fp.seek but you move the offset relative to where you are now """similar to self.fp.seek but you move the offset relative to where you are now
@@ -63,6 +89,27 @@ class BufferStream:
buf = self.fp.read(size) buf = self.fp.read(size)
return BufferedBytes(pos, buf) return BufferedBytes(pos, buf)
def readline(self) -> BufferedBytes:
"""Reads upto a line-end
Returns:
BufferedBytes
"""
pos = self.tell()
buf = self.fp.readline()
return BufferedBytes(pos, buf)
def get_firstnonempty_line(self) -> Optional[BufferedBytes]:
"""reads line-by-line until we find a none-empty line
Returns:
Optional[BufferedBytes]
"""
for bbline in self.into_line_iter():
if bbline.data.strip():
return bbline
return None
def __iter__(self): def __iter__(self):
if self.fp.read(1) == b"": if self.fp.read(1) == b"":
raise StreamEOF raise StreamEOF
@@ -84,6 +131,23 @@ class BufferStream:
def __exit__(self): def __exit__(self):
self.close() self.close()
def into_line_iter(self) -> Iterator[BufferedBytes]:
"""reads the stream line by line
Raises:
StreamEOF
Yields:
BufferedBytes
"""
last_pos = self.tell()
while True:
bbyte = self.readline()
if self.tell() == last_pos:
raise StreamEOF
yield bbyte
last_pos = bbyte.pos
def into_reverse_reader_iter(self) -> Iterator[BufferedBytes]: def into_reverse_reader_iter(self) -> Iterator[BufferedBytes]:
"""reads the file in reverse, outputting complete lines """reads the file in reverse, outputting complete lines
+1 -1
View File
@@ -222,7 +222,7 @@ class Basics(TestCase):
class Lexer(TestCase): class Lexer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]): def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PDFLexer(io.BytesIO(raw)) tokenizer = PDFLexer.from_bytes(raw)
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1): for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1):
self.assertEqual(token, etoken, f"Failed at example #{i}") self.assertEqual(token, etoken, f"Failed at example #{i}")
try: try:
+41 -51
View File
@@ -1,12 +1,13 @@
import io
from pathlib import Path from pathlib import Path
from pdfmajor.parser.objects.collections import PDFDictionary
from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString
from typing import Callable, List from typing import List
from decimal import Decimal from decimal import Decimal
from unittest import TestCase from unittest import TestCase
from pdfmajor.streambuffer import StreamEOF from pdfmajor.streambuffer import StreamEOF
from pdfmajor.parser.objects.indirect import IndirectObject, IndirectObjectCollection from pdfmajor.parser.objects.indirect import IndirectObject
from pdfmajor.parser.objects.base import PDFObject from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.comment import PDFComment from pdfmajor.parser.objects.comment import PDFComment
from pdfmajor.parser.objects.stream import PDFStream from pdfmajor.parser.objects.stream import PDFStream
@@ -17,9 +18,8 @@ CURRENT_FOLDER = Path(__file__).parent
class Collections(TestCase): class Collections(TestCase):
def test_parse_dict(self): def test_parse_dict(self):
parser = PDFParser( parser = PDFParser.from_bytes(
io.BytesIO( br"""<< /Type /Example
br"""<< /Type /Example
/Subtype /DictionaryExample /Subtype /DictionaryExample
/Version 0.01 /Version 0.01
/IntegerItem 12 /IntegerItem 12
@@ -29,8 +29,7 @@ class Collections(TestCase):
/Item2 true /Item2 true
/LastItem ( not! ) /LastItem ( not! )
/VeryLastItem ( OK ) >> /VeryLastItem ( OK ) >>
>>""" >>""",
),
strict=True, strict=True,
) )
obj = next(parser.iter_objects()) obj = next(parser.iter_objects())
@@ -52,8 +51,8 @@ class Collections(TestCase):
) )
def test_parse_array(self): def test_parse_array(self):
parser = PDFParser( parser = PDFParser.from_bytes(
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]"""), br"""[ 549 3.14 false ( Ralph ) /SomeName ]""",
strict=True, strict=True,
) )
obj = next(parser.iter_objects()) obj = next(parser.iter_objects())
@@ -63,13 +62,11 @@ class Collections(TestCase):
) )
def test_parse_nested_array(self): def test_parse_nested_array(self):
parser = PDFParser( parser = PDFParser.from_bytes(
io.BytesIO( br"""[ [10 30] [22 33] [
br"""[ [10 30] [22 33] [
<< /x 0 /y 5 >> << /x 0 /y 5 >>
<< /x 10 /y 5 >> << /x 10 /y 5 >>
] ]""" ] ]""",
),
strict=True, strict=True,
) )
obj = next(parser.iter_objects()) obj = next(parser.iter_objects())
@@ -88,9 +85,8 @@ class Collections(TestCase):
class IndirectObjects(TestCase): class IndirectObjects(TestCase):
def test_parse_simple_indobj(self): def test_parse_simple_indobj(self):
parser = PDFParser( parser = PDFParser.from_bytes(
io.BytesIO( br"""
br"""
12 1 obj (Bring) endobj 12 1 obj (Bring) endobj
100 0 obj 100 0 obj
<33> <33>
@@ -98,47 +94,45 @@ class IndirectObjects(TestCase):
13 8 13 8
obj obj
<< /x 1 /y 3>> endobj << /x 1 /y 3>> endobj
""" """,
),
strict=True, strict=True,
) )
expected = [ expected = [
(12, 1, "Bring"), IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)),
(100, 0, b"3"), IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)),
(13, 8, {"x": 1, "y": 3}), IndirectObject(
13,
8,
144,
PDFDictionary.from_dict(
{"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)}
),
),
] ]
for obj, eobj in zip(parser.iter_objects(), expected): for obj, eobj in zip(parser.iter_objects(), expected):
self.assertIsInstance(obj, IndirectObject) self.assertIsInstance(obj, IndirectObject)
self.assertEqual( self.assertEqual(obj, eobj)
obj,
parser.state.indirect_object_collection.get_indobject(eobj[0], eobj[1]),
)
self.assertEqual(obj.to_python(), eobj[2])
with self.assertRaises(StreamEOF): with self.assertRaises(StreamEOF):
next(parser.iter_objects()) next(parser.iter_objects())
def test_parse_obj_with_references(self): def test_parse_obj_with_references(self):
parser = PDFParser( parser = PDFParser.from_bytes(
io.BytesIO( br"""
br"""
12 1 obj (Bring) endobj 12 1 obj (Bring) endobj
10 2 obj << /lastone 12 1 R >> endobj 10 2 obj << /lastone 12 1 R >> endobj
""" """,
),
strict=True, strict=True,
) )
expected = [ expected = [
parser.state.indirect_object_collection.get_indobject(12, 1), "Bring",
parser.state.indirect_object_collection.get_indobject(10, 2), {"lastone": "Bring"},
] ]
for obj, exp in zip(parser.iter_objects(), expected): for obj, exp in zip(parser.iter_objects(), expected):
self.assertEqual(obj, exp) self.assertEqual(obj.to_python(), exp)
with self.assertRaises(StreamEOF): with self.assertRaises(StreamEOF):
next(parser.iter_objects()) next(parser.iter_objects())
sec_obj = parser.state.indirect_object_collection.get_indobject( sec_obj = parser.db.get_object(10, 2)
10, 2
).get_object()
self.assertEqual( self.assertEqual(
sec_obj.to_python(), sec_obj.to_python(),
{"lastone": "Bring"}, {"lastone": "Bring"},
@@ -147,13 +141,11 @@ class IndirectObjects(TestCase):
def run_test( def run_test(
self, self,
raw: bytes, raw: bytes,
expected: Callable[[IndirectObjectCollection], List[PDFObject]], expected: List[PDFObject],
): ):
parser = PDFParser(io.BytesIO(raw), strict=True) parser = PDFParser.from_bytes(raw, strict=True)
for i, (eobj, obj) in enumerate( for i, (eobj, obj) in enumerate(
zip( zip(expected, parser.iter_objects()),
expected(parser.state.indirect_object_collection), parser.iter_objects()
),
1, 1,
): ):
self.assertEqual(obj, eobj, f"Failed at example #{i}") self.assertEqual(obj, eobj, f"Failed at example #{i}")
@@ -172,20 +164,19 @@ class IndirectObjects(TestCase):
endobj %closing the object endobj %closing the object
11 1 obj 2 endobj 11 1 obj 2 endobj
""", """,
expected=lambda indobjc: [ expected=[
PDFComment(b" object references", (22, 41)), PDFComment(b" object references", (22, 41)),
PDFComment(b" start of object", (62, 79)), PDFComment(b" start of object", (62, 79)),
PDFComment(b" object content", (110, 126)), PDFComment(b" object content", (110, 126)),
indobjc.get_indobject(10, 0), IndirectObject(10, 0, 58, PDFString("testing", 0, 0)),
PDFComment(b"closing the object", (150, 169)), PDFComment(b"closing the object", (150, 169)),
indobjc.get_indobject(11, 1), IndirectObject(11, 1, 191, PDFInteger(2, 0, 0)),
], ],
) )
def test_parse_stream(self): def test_parse_stream(self):
parser = PDFParser( parser = PDFParser.from_bytes(
io.BytesIO( br"""
br"""
5 0 obj 5 0 obj
<< /Type /XRef /Length 124 /Filter /FlateDecode >> << /Type /XRef /Length 124 /Filter /FlateDecode >>
stream stream
@@ -193,7 +184,6 @@ stream
<80>endstream <80>endstream
endobj endobj
""" """
)
) )
for obj in parser.iter_objects(): for obj in parser.iter_objects():
self.assertIsInstance(obj, IndirectObject) self.assertIsInstance(obj, IndirectObject)
+15
View File
@@ -0,0 +1,15 @@
from pathlib import Path
from unittest import TestCase
from pdfmajor.streambuffer import BufferStream
from pdfmajor.parser.xref import iter_over_xref
CURRENT_FOLDER = Path(__file__).parent
class Standard(TestCase):
def test_char_doc(self):
with (CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf").open("rb") as fp:
buffer = BufferStream(fp)
for xref in iter_over_xref(buffer):
print(xref)