working xref db

This commit is contained in:
ari committed 2020-10-23 22:17:50 -04:00
1 parent cfc2dfca47
commit e2a92517cd
12 files changed
+517 -227

No files matched your search

+13 -2
View File
@@ -19,8 +19,19 @@ from pdfmajor.streambuffer import BufferStream
class PDFLexer:
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
self.buffer = BufferStream(fp, buffer_size=buffer_size)
def __init__(self, buffer: BufferStream) -> None:
self.buffer = buffer
@classmethod
def from_iobuffer(cls, fp: io.BufferedIOBase, buffer_size: int = 4096):
return cls(BufferStream(fp, buffer_size=buffer_size))
@classmethod
def from_bytes(cls, data: bytes, buffer_size: int = 4096):
return cls(BufferStream(io.BytesIO(data), buffer_size=buffer_size))
def tell(self) -> int:
return self.buffer.tell()
def seek(self, pos: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
+162 -74
View File
@@ -1,7 +1,12 @@
import io
from pdfmajor.parser.objects.stream import PDFStream
from typing import Iterator, List, Optional, Union
from typing import Dict, Iterator, Optional, Tuple
from pdfmajor.streambuffer import BufferStream
from pdfmajor.parser.objects.stream import PDFStream
from pdfmajor.parser.exceptions import InvalidXref
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.lexer.token import TokenKeyword
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.token import (
TArrayValue,
@@ -11,15 +16,54 @@ from pdfmajor.lexer.token import (
TokenDictionary,
TokenKeyword,
)
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import InvalidKeywordPos, ParserError
from .objects.indirect import IndirectObject
from .objects.base import PDFContextualObject, PDFObject
from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment
from .objects.indirect import IndirectObject
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import InvalidKeywordPos, BrokenFile, InvalidKeywordPos, ParserError
class XRefDB:
"""A class for collection references to objects"""
def __init__(self, parser: "PDFParser") -> None:
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
self.parser = parser
def save_indobject(self, indobj: IndirectObject):
if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None:
self.objs[(indobj.obj_num, indobj.gen_num)] = indobj
def get_object_from_indobj(self, indobj: IndirectObject) -> PDFObject:
return self.get_object(indobj.obj_num, indobj.gen_num)
def get_object(self, obj_num: int, gen_num: int) -> PDFObject:
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
Args:
obj_num (int): object number
gen_num (int): generation number
Returns:
PDFObject
"""
indobj = self.objs.get((obj_num, gen_num), None)
if indobj is None:
raise InvalidXref(
f"xref not found for object retrieval {obj_num}, {gen_num}"
)
else:
obj = indobj.get_object()
if obj is not None:
return obj
cur_pos = self.parser.tell()
self.parser.seek(indobj.offset)
obj = next(self.parser.iter_objects())
indobj.save_object(obj)
self.parser.seek(cur_pos)
return obj
class PDFParser:
@@ -27,11 +71,9 @@ class PDFParser:
def __init__(
self,
fp: io.BufferedIOBase,
buffer_size: int = 4096,
state: Optional[ParsingState] = None,
buffer: BufferStream,
db: Optional[XRefDB] = None,
strict: bool = True,
retain_obj_on_seek: bool = False,
) -> None:
"""
@@ -40,48 +82,62 @@ class PDFParser:
buffer_size (int, optional): the size of the reads. Defaults to 4096.
state (Optional[ParsingState], optional): the parsing state. Defaults to None.
strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True.
retain_obj_on_seek (bool, optional): whether we want to ensure we keep the object collection when changing the offset using the self.seek methodm this does not apply if you specify the state when using the seek method!. Defaults to False.
"""
self.lexer = PDFLexer(fp, buffer_size)
self.lexer = PDFLexer(buffer)
self.db = XRefDB(self)
self.strict = strict
self.state = ParsingState([], []) if state is None else state
self.retain_obj_on_seek = retain_obj_on_seek
def seek(self, offset: int, state: Optional[ParsingState] = None) -> int:
@classmethod
def from_iobuffer(
cls,
fp: io.BufferedIOBase,
buffer_size: int = 4096,
strict: bool = True,
):
return cls(
BufferStream(fp, buffer_size=buffer_size),
strict=strict,
)
@classmethod
def from_bytes(
cls,
data: bytes,
buffer_size: int = 4096,
strict: bool = True,
):
return cls(
BufferStream(io.BytesIO(data), buffer_size=buffer_size),
strict=strict,
)
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
state (Optional[ParsingState]): this will override our current state
Returns:
int: new offset
"""
self.state = (
ParsingState(
[],
[],
indirect_object_collection=self.state.indirect_object_collection
if not self.retain_obj_on_seek
else None,
)
if state is None
else state
)
return self.lexer.seek(offset)
def tell(self) -> int:
return self.lexer.tell()
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream
Yields:
PDFObject
"""
state = ParsingState([], [], strict=self.strict)
for token in self.lexer.iter_tokens():
attempt_prim = attempt_parse_prim(token, self.state)
attempt_prim = attempt_parse_prim(token, state)
if attempt_prim.success:
yield from self._feed_or_yield_objs(attempt_prim.next_objs)
yield from state.feed_or_yield_objs(attempt_prim.next_objs)
elif isinstance(token, TokenComment):
yield self.state.set_last_obj(
yield state.set_last_obj(
PDFComment(
token.value,
(token.start_loc, token.end_loc),
@@ -89,67 +145,99 @@ class PDFParser:
)
elif isinstance(token, TokenKeyword):
if token.value == b"R":
yield from self._on_indirect_ref_close()
yield from self._on_indirect_ref_close(state)
elif token.value == b"obj":
self.state.initialize_indirect_obj()
state.initialize_indirect_obj(token.start_loc)
elif token.value == b"endobj":
yield from self._on_endobj(token)
yield self._on_endobj(state, token)
elif token.value == b"stream":
self._on_stream(token)
self._on_stream(
state,
token,
)
elif token.value == b"endstream":
self._on_endstream(token)
self._on_endstream(state, token)
else:
raise InvalidKeywordPos(token)
elif isinstance(token, TokenArray):
yield from self.state.flush_int_collections()
yield from state.flush_int_collections()
yield from deal_with_collection_object(
self.state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
)
elif isinstance(token, TokenDictionary):
yield from self.state.flush_int_collections()
yield from state.flush_int_collections()
yield from deal_with_collection_object(
self.state,
state,
token=token,
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
strict=self.strict,
)
def _feed_or_yield_objs(self, next_objs: List[PDFObject]):
cur_ctx = self.state.current_context
for obj in next_objs:
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
def _on_endobj(self, state: ParsingState, token: TokenKeyword) -> IndirectObject:
"""finish off the current context, assuming we are handling an indirect-object
def _on_indirect_ref_close(self):
obj = self.state.get_indobjects()
cur_ctx = self.state.current_context
Args:
token (TokenKeyword)
Raises:
BrokenFile: if the object fif not get initilized with two integers
InvalidKeywordPos: If the context is not an indirect object
Yields:
PDFObject
"""
last_ctx = state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
if len(state.int_collection) > 1:
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
for num in state.flush_int_collections():
last_ctx.pass_item(num)
obj = last_ctx.into_readonly_copy()
self.db.save_indobject(obj)
return obj
else:
raise InvalidKeywordPos(token)
def _on_stream(self, state: ParsingState, token: TokenKeyword):
"""Initilize a PDFStream
Args:
token (TokenKeyword)
Raises:
ParserError: if the current context is not an indirect object
"""
last_ctx = state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
self.db.save_indobject(last_ctx)
stream = PDFStream(token.end_loc, 0)
obj = self.db.get_object_from_indobj(last_ctx)
stream.pass_item(obj)
last_ctx.save_object(PDFStream(token.end_loc, 0))
state.context_stack.append(stream)
self.seek(stream.offset + stream.length)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
def _on_endstream(self, state: ParsingState, token: TokenKeyword):
"""Finish a PDFStream
Args:
token (TokenKeyword)
Raises:
InvalidKeywordPos: if the current context is not an indirect stream
"""
stream: PDFContextualObject = state.context_stack.pop()
if not isinstance(stream, PDFStream):
raise InvalidKeywordPos(token)
def _on_indirect_ref_close(self, state: ParsingState):
obj_num, gen_num = state.get_indobj_values()
obj = self.db.get_object(obj_num, gen_num)
cur_ctx = state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
def _on_endobj(self, token: TokenKeyword):
last_ctx = self.state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
yield last_ctx.clone()
elif self.strict:
raise InvalidKeywordPos(token)
def _on_stream(self, token: TokenKeyword):
last_ctx = self.state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
stream = PDFStream(token.end_loc, 0)
stream.pass_item(last_ctx.get_object())
last_ctx.save_object(PDFStream(token.end_loc, 0))
self.state.context_stack.append(stream)
self.lexer.seek(stream.offset + stream.length)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
def _on_endstream(self, token: TokenKeyword):
stream: PDFContextualObject = self.state.context_stack.pop()
if not isinstance(stream, PDFStream):
raise InvalidKeywordPos(token)
+7 -11
View File
@@ -1,7 +1,6 @@
import io
from pdfmajor.streambuffer import BufferStream
from .objects.stream import PDFStream
from .state import ParsingState
from .PDFParser import PDFParser
from .fliters import process_filters_on_data
@@ -10,20 +9,17 @@ class PDFStreamPasrser(PDFParser):
def __init__(
self,
stream: PDFStream,
fp: io.BufferedIOBase,
state: ParsingState,
buffer_size: int = 4096,
buffer: BufferStream,
strict: bool = True,
) -> None:
fp.seek(stream.offset)
data = fp.read(stream.length)
cur_pos = buffer.tell()
buffer.seek(stream.offset)
data = buffer.read(stream.length).data
buffer.seek(cur_pos)
data = process_filters_on_data(data, stream.filter, stream.decode_parms)
data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms)
self.stream = stream
super().__init__(
fp=io.BytesIO(data),
buffer_size=buffer_size,
state=state,
buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length),
strict=strict,
retain_obj_on_seek=True,
)
+18 -2
View File
@@ -6,7 +6,15 @@ class ParserError(PDFMajorException):
pass
class InvalidKeywordPos(ParserError):
class BrokenFile(ParserError):
pass
class UnexpectedEOF(BrokenFile, EOFError):
pass
class InvalidKeywordPos(BrokenFile):
def __init__(self, token: TokenKeyword) -> None:
self.token = token
super().__init__(
@@ -18,9 +26,17 @@ class InvalidIndirectObjAccess(ParserError):
pass
class DecodeFailed(ParserError):
class DecodeFailed(BrokenFile):
pass
class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError):
pass
class PDFNoValidXRef(BrokenFile, EOFError):
pass
class InvalidXref(ParserError, KeyError):
pass
+7
View File
@@ -23,6 +23,13 @@ class PDFDictionary(Dict[str, PDFObject], PDFContextualObject):
self.last_name: Optional[PDFName] = None
self.strict = strict
@classmethod
def from_dict(cls, d: Dict[str, PDFObject]):
new_obj = cls()
for key, val in d.items():
new_obj[key] = val
return new_obj
@property
def last_item_value(self):
return self[self.last_name]
+27 -60
View File
@@ -1,88 +1,55 @@
from typing import Dict, Tuple
from ..exceptions import InvalidIndirectObjAccess, ParserError
from typing import Optional
from ..exceptions import InvalidIndirectObjAccess
from .base import PDFContextualObject, PDFObject
from .primitives import PDFNull
class IndirectObjectCollection:
"""A class for collection references to objects"""
def __init__(self) -> None:
self.objs: Dict[Tuple[int, int], PDFObject] = {}
def create_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject":
"""Create an indirect object and store its value in the collection
return the reference to where this object lies at
Returns:
IndirectObject:
"""
return IndirectObject(self, obj_num, gen_num, read_only=False)
def get_indobject(self, obj_num: int, gen_num: int) -> "IndirectObject":
"""creates a reference to an object
Returns:
IndirectObject
"""
return IndirectObject(self, obj_num, gen_num, read_only=True)
def get_object(self, obj_num: int, gen_num: int) -> PDFObject:
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
Args:
obj_num (int): object number
gen_num (int): generation number
Returns:
PDFObject
"""
return self.objs.get((obj_num, gen_num), PDFNull())
class IndirectObject(PDFContextualObject):
"""A class representing a reference for other objects"""
def __init__(
self,
collection: IndirectObjectCollection,
obj_num: int,
gen_num: int,
read_only: bool = True,
offset: int,
data: Optional[PDFObject] = None,
) -> None:
self.offset = offset
self.obj_num = obj_num
self.gen_num = gen_num
self.__read_only = read_only
self.__col = collection
self.__data = data
def __repr__(self) -> str:
return f"IndirectObject({self.obj_num}, {self.gen_num})"
return f"IndirectObject(obj_num={self.obj_num}, gen_num={self.gen_num}, offset={self.offset}, data={self.get_object()})"
def get_object(self) -> PDFObject:
return self.__col.get_object(self.obj_num, self.gen_num)
def get_object(self) -> Optional[PDFObject]:
return self.__data
def save_object(self, obj: PDFObject):
if self.__read_only:
raise InvalidIndirectObjAccess(
f"attempted to override a read-only reference of an object {obj}"
)
self.__col.objs[(self.obj_num, self.gen_num)] = obj
# if self.__data is not None:
# raise InvalidIndirectObjAccess(
# f"attempted to override an initilized reference of an object {obj}"
# )
self.__data = obj
def clone(self):
return IndirectObject(self.__col, self.obj_num, self.gen_num, True)
def into_readonly_copy(self):
return IndirectObject(
self.obj_num, self.gen_num, self.offset, self.get_object()
)
def to_python(self):
return self.get_object().to_python()
obj = self.get_object()
if obj is None:
return None
return obj.to_python()
def pass_item(self, item: PDFObject):
if self.__read_only:
raise InvalidIndirectObjAccess(
f"attempted to add to an read-only reference of an object {item}"
)
# if self.__read_only:
# raise InvalidIndirectObjAccess(
# f"attempted to add to an read-only reference of an object {item}"
# )
cur_obj = self.get_object()
if isinstance(cur_obj, PDFContextualObject):
cur_obj.pass_item(item)
elif isinstance(cur_obj, PDFNull):
self.save_object(item)
else:
raise InvalidIndirectObjAccess(f"Invalid redefinition of an object {item}")
self.save_object(item)
+33 -25
View File
@@ -4,33 +4,32 @@ from .exceptions import ParserError
from .objects.base import PDFContextualObject, PDFObject
from .objects.primitives import PDFInteger
from .objects.comment import PDFComment
from .objects.indirect import IndirectObject, IndirectObjectCollection
from .objects.indirect import IndirectObject
class ParsingState:
__slots__ = [
"context_stack",
"int_collection",
"last_obj",
"indirect_object_collection",
]
__slots__ = ["context_stack", "int_collection", "last_obj", "strict"]
def __init__(
self,
context_stack: List[PDFContextualObject],
int_collection: List[PDFInteger],
last_obj: Optional[PDFObject] = None,
indirect_object_collection: Optional[IndirectObjectCollection] = None,
strict: bool = False,
) -> None:
self.context_stack = context_stack
self.int_collection = int_collection
self.int_collection = int_collection
self.last_obj = last_obj
self.indirect_object_collection = (
indirect_object_collection
if indirect_object_collection is not None
else IndirectObjectCollection()
self.strict = strict
def __repr__(self) -> str:
return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj}, strict={strict})".format(
stackc=len(self.context_stack),
cur_stack=self.current_context,
ints=self.int_collection,
lobj=self.last_obj,
strict=self.strict,
)
@property
@@ -76,22 +75,15 @@ class ParsingState:
return (obj_num.to_python(), gen_num.to_python())
raise ParserError("missing leading tokens for object reference")
def get_indobjects(self) -> IndirectObject:
"""Gets an indirect object, based on the values in 'self.int_collection'
def initialize_indirect_obj(self, offset: int):
"""Initilizes an indirect object based off the values in self.int_collection
Raises:
ParserError: if the int_collection does not have two elements
Returns:
IndirectObject
Args:
offset (int): position for the 'obj' keyword
"""
obj_num, gen_num = self.get_indobj_values()
return self.indirect_object_collection.get_indobject(obj_num, gen_num)
def initialize_indirect_obj(self):
obj_num, gen_num = self.get_indobj_values()
self.context_stack.append(
self.indirect_object_collection.create_indobject(obj_num, gen_num)
IndirectObject(obj_num=obj_num, gen_num=gen_num, offset=offset)
)
def flush_int_collections(self) -> Iterator[PDFObject]:
@@ -108,3 +100,19 @@ class ParsingState:
for obj in self.int_collection:
yield obj
self.int_collection = []
def feed_or_yield_objs(self, next_objs: List[PDFObject]) -> Iterator[PDFObject]:
"""If there context, then feed the objects into it, otherwise yield them
Args:
next_objs (List[PDFObject])
Yields:
PDFObject
"""
cur_ctx = self.current_context
for obj in next_objs:
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
+128
View File
@@ -1,3 +1,131 @@
# from pdfmajor.parser.objects.indirect import IndirectObjectCollection
from pdfmajor.lexer.token import TokenInteger, TokenKeyword
from pdfmajor.lexer import PDFLexer
from typing import Iterator, Literal, NamedTuple, Union
from pdfmajor.streambuffer import BufferStream
from .exceptions import PDFNoValidXRef, BrokenFile, UnexpectedEOF
class XRefRow(NamedTuple):
"""a parsed version of a row in the xref table
as defined in PDF spec 1.7 section 7.5.4
"""
offset: int
obj_num: int
gen_num: int
use: Union[Literal["n"], Literal["f"]]
def find_start_of_xref(buffer: BufferStream, strict: bool = False) -> int:
"""Internal function used to locate the start of the XRef table"""
prev = None
found_eof = False
for _, line in buffer.into_reverse_reader_iter():
line = line.strip()
found_eof |= line.strip() == b"%%EOF"
if line == b"startxref":
break
if line:
prev = line
if prev is None:
raise PDFNoValidXRef()
if strict and not found_eof:
raise BrokenFile(
"Failed to find %%EOF at end of file, this can indicate the file is incomplete or broken."
)
return int(prev)
def iter_over_xref(buffer: BufferStream, strict: bool = False) -> Iterator[XRefRow]:
start_offset = find_start_of_xref(buffer, strict)
buffer.seek(start_offset)
lexer = PDFLexer(buffer)
first_token = next(lexer.iter_tokens())
if isinstance(first_token, TokenInteger):
raise NotImplementedError("XRefStream: PDF-1.5")
elif isinstance(first_token, TokenKeyword):
if first_token.value == b"xref":
return iter_over_standard_xref(buffer, strict)
raise BrokenFile("Missing a valid XRef table")
# # read xref table
# def read_xref_from(self, parser, start, xrefs):
# """Reads XRefs from the given location."""
# parser.seek(start)
# parser.reset()
# try:
# (pos, token) = parser.nexttoken()
# except PSEOF:
# raise PDFNoValidXRef("Unexpected EOF")
# log.info("read_xref_from: start=%d, token=%r", start, token)
# if isinstance(token, int):
# # XRefStream: PDF-1.5
# parser.seek(pos)
# parser.reset()
# xref = PDFXRefStream()
# xref.load(parser)
# else:
# if token is parser.KEYWORD_XREF:
# parser.nextline()
# xref = PDFXRef()
# xref.load(parser)
# xrefs.append(xref)
# trailer = xref.get_trailer()
# log.info("trailer: %r", trailer)
# if "XRefStm" in trailer:
# pos = int_value(trailer["XRefStm"])
# self.read_xref_from(parser, pos, xrefs)
# if "Prev" in trailer:
# # find previous xref
# pos = int_value(trailer["Prev"])
# self.read_xref_from(parser, pos, xrefs)
# return
def iter_over_standard_xref(
buffer: BufferStream, strict: bool = False
) -> Iterator[XRefRow]:
while True:
bbline = buffer.get_firstnonempty_line()
if bbline is None:
raise UnexpectedEOF
pos, line = bbline
if line.startswith(b"trailer"):
buffer.seek(pos)
break
descriptors = line.strip().split(b" ")
if len(descriptors) != 2:
raise PDFNoValidXRef(f"Trailer not found: pos={pos} line={line!r}")
try:
(start, nobjs) = map(int, descriptors)
except ValueError:
raise PDFNoValidXRef(
f"Failed to decipher xref row: pos={pos} line={line!r}"
)
for obj_num in range(start, start + nobjs):
_, line = buffer.readline()
f = line.strip().split(b" ")
if len(f) != 3:
raise PDFNoValidXRef(f"Invalid XRef format: pos={pos} line={line!r}")
try:
pos = int(f[0])
gen_num = int(f[1])
use = f[2]
except ValueError:
raise PDFNoValidXRef(
f"Failed to decipher xref row: pos={pos} line={line!r}"
)
if use not in [b"n", b"f"] and not strict:
raise PDFNoValidXRef(
f"Invalid 'use' value for xref {use!r}: pos={pos} line={line!r}"
)
yield XRefRow(offset=pos, obj_num=obj_num, gen_num=gen_num, use=use) # type: ignore
# def construct_indobj_collection(
# buffer: BufferStream, it_xref: Iterator[XRefRow]
# ) -> IndirectObjectCollection:
# collection = IndirectObjectCollection()
# parser =
+65 -1
View File
@@ -1,5 +1,5 @@
import io
from typing import Iterator, NamedTuple
from typing import Iterator, NamedTuple, Optional
from pdfmajor.exceptions import PDFMajorException
@@ -19,9 +19,35 @@ class BufferStream:
__slots__ = ["fp", "buffer_size"]
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
"""[summary]
Args:
fp (io.BufferedIOBase)
buffer_size (int, optional): maximum length of the bytes this will read at each iteration. Defaults to 4096.
"""
self.fp = fp
self.buffer_size = buffer_size
def get_slice(
self, offset: int, length: int, buffer_size: Optional[int] = None
) -> "BufferStream":
"""Creates a slice of the current byte-stream and returns a BufferStream based on that slice
Args:
offset (int): where the slice starts
length (int): length of the slice
buffer_size (int, optional) buffer size for new slice. Defaults to self.buffer_size
Returns:
BufferStream
"""
cur_pos = self.tell()
self.seek(offset)
bslice = self.fp.read(length)
self.seek(cur_pos)
return BufferStream(
io.BytesIO(bslice),
buffer_size=buffer_size if buffer_size is not None else self.buffer_size,
)
def seekd(self, step: int) -> int:
"""similar to self.fp.seek but you move the offset relative to where you are now
@@ -63,6 +89,27 @@ class BufferStream:
buf = self.fp.read(size)
return BufferedBytes(pos, buf)
def readline(self) -> BufferedBytes:
"""Reads upto a line-end
Returns:
BufferedBytes
"""
pos = self.tell()
buf = self.fp.readline()
return BufferedBytes(pos, buf)
def get_firstnonempty_line(self) -> Optional[BufferedBytes]:
"""reads line-by-line until we find a none-empty line
Returns:
Optional[BufferedBytes]
"""
for bbline in self.into_line_iter():
if bbline.data.strip():
return bbline
return None
def __iter__(self):
if self.fp.read(1) == b"":
raise StreamEOF
@@ -84,6 +131,23 @@ class BufferStream:
def __exit__(self):
self.close()
def into_line_iter(self) -> Iterator[BufferedBytes]:
"""reads the stream line by line
Raises:
StreamEOF
Yields:
BufferedBytes
"""
last_pos = self.tell()
while True:
bbyte = self.readline()
if self.tell() == last_pos:
raise StreamEOF
yield bbyte
last_pos = bbyte.pos
def into_reverse_reader_iter(self) -> Iterator[BufferedBytes]:
"""reads the file in reverse, outputting complete lines
+1 -1
View File
@@ -222,7 +222,7 @@ class Basics(TestCase):
class Lexer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PDFLexer(io.BytesIO(raw))
tokenizer = PDFLexer.from_bytes(raw)
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1):
self.assertEqual(token, etoken, f"Failed at example #{i}")
try:
+41 -51
View File
@@ -1,12 +1,13 @@
import io
from pathlib import Path
from pdfmajor.parser.objects.collections import PDFDictionary
from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString
from typing import Callable, List
from typing import List
from decimal import Decimal
from unittest import TestCase
from pdfmajor.streambuffer import StreamEOF
from pdfmajor.parser.objects.indirect import IndirectObject, IndirectObjectCollection
from pdfmajor.parser.objects.indirect import IndirectObject
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.comment import PDFComment
from pdfmajor.parser.objects.stream import PDFStream
@@ -17,9 +18,8 @@ CURRENT_FOLDER = Path(__file__).parent
class Collections(TestCase):
def test_parse_dict(self):
parser = PDFParser(
io.BytesIO(
br"""<< /Type /Example
parser = PDFParser.from_bytes(
br"""<< /Type /Example
/Subtype /DictionaryExample
/Version 0.01
/IntegerItem 12
@@ -29,8 +29,7 @@ class Collections(TestCase):
/Item2 true
/LastItem ( not! )
/VeryLastItem ( OK ) >>
>>"""
),
>>""",
strict=True,
)
obj = next(parser.iter_objects())
@@ -52,8 +51,8 @@ class Collections(TestCase):
)
def test_parse_array(self):
parser = PDFParser(
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]"""),
parser = PDFParser.from_bytes(
br"""[ 549 3.14 false ( Ralph ) /SomeName ]""",
strict=True,
)
obj = next(parser.iter_objects())
@@ -63,13 +62,11 @@ class Collections(TestCase):
)
def test_parse_nested_array(self):
parser = PDFParser(
io.BytesIO(
br"""[ [10 30] [22 33] [
parser = PDFParser.from_bytes(
br"""[ [10 30] [22 33] [
<< /x 0 /y 5 >>
<< /x 10 /y 5 >>
] ]"""
),
] ]""",
strict=True,
)
obj = next(parser.iter_objects())
@@ -88,9 +85,8 @@ class Collections(TestCase):
class IndirectObjects(TestCase):
def test_parse_simple_indobj(self):
parser = PDFParser(
io.BytesIO(
br"""
parser = PDFParser.from_bytes(
br"""
12 1 obj (Bring) endobj
100 0 obj
<33>
@@ -98,47 +94,45 @@ class IndirectObjects(TestCase):
13 8
obj
<< /x 1 /y 3>> endobj
"""
),
""",
strict=True,
)
expected = [
(12, 1, "Bring"),
(100, 0, b"3"),
(13, 8, {"x": 1, "y": 3}),
IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)),
IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)),
IndirectObject(
13,
8,
144,
PDFDictionary.from_dict(
{"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)}
),
),
]
for obj, eobj in zip(parser.iter_objects(), expected):
self.assertIsInstance(obj, IndirectObject)
self.assertEqual(
obj,
parser.state.indirect_object_collection.get_indobject(eobj[0], eobj[1]),
)
self.assertEqual(obj.to_python(), eobj[2])
self.assertEqual(obj, eobj)
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
def test_parse_obj_with_references(self):
parser = PDFParser(
io.BytesIO(
br"""
parser = PDFParser.from_bytes(
br"""
12 1 obj (Bring) endobj
10 2 obj << /lastone 12 1 R >> endobj
"""
),
""",
strict=True,
)
expected = [
parser.state.indirect_object_collection.get_indobject(12, 1),
parser.state.indirect_object_collection.get_indobject(10, 2),
"Bring",
{"lastone": "Bring"},
]
for obj, exp in zip(parser.iter_objects(), expected):
self.assertEqual(obj, exp)
self.assertEqual(obj.to_python(), exp)
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
sec_obj = parser.state.indirect_object_collection.get_indobject(
10, 2
).get_object()
sec_obj = parser.db.get_object(10, 2)
self.assertEqual(
sec_obj.to_python(),
{"lastone": "Bring"},
@@ -147,13 +141,11 @@ class IndirectObjects(TestCase):
def run_test(
self,
raw: bytes,
expected: Callable[[IndirectObjectCollection], List[PDFObject]],
expected: List[PDFObject],
):
parser = PDFParser(io.BytesIO(raw), strict=True)
parser = PDFParser.from_bytes(raw, strict=True)
for i, (eobj, obj) in enumerate(
zip(
expected(parser.state.indirect_object_collection), parser.iter_objects()
),
zip(expected, parser.iter_objects()),
1,
):
self.assertEqual(obj, eobj, f"Failed at example #{i}")
@@ -172,20 +164,19 @@ class IndirectObjects(TestCase):
endobj %closing the object
11 1 obj 2 endobj
""",
expected=lambda indobjc: [
expected=[
PDFComment(b" object references", (22, 41)),
PDFComment(b" start of object", (62, 79)),
PDFComment(b" object content", (110, 126)),
indobjc.get_indobject(10, 0),
IndirectObject(10, 0, 58, PDFString("testing", 0, 0)),
PDFComment(b"closing the object", (150, 169)),
indobjc.get_indobject(11, 1),
IndirectObject(11, 1, 191, PDFInteger(2, 0, 0)),
],
)
def test_parse_stream(self):
parser = PDFParser(
io.BytesIO(
br"""
parser = PDFParser.from_bytes(
br"""
5 0 obj
<< /Type /XRef /Length 124 /Filter /FlateDecode >>
stream
@@ -193,7 +184,6 @@ stream
<80>endstream
endobj
"""
)
)
for obj in parser.iter_objects():
self.assertIsInstance(obj, IndirectObject)
+15
View File
@@ -0,0 +1,15 @@
from pathlib import Path
from unittest import TestCase
from pdfmajor.streambuffer import BufferStream
from pdfmajor.parser.xref import iter_over_xref
CURRENT_FOLDER = Path(__file__).parent
class Standard(TestCase):
def test_char_doc(self):
with (CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf").open("rb") as fp:
buffer = BufferStream(fp)
for xref in iter_over_xref(buffer):
print(xref)