converted parser to functional

fixed a bufsize issue on the lexer
This commit is contained in:
ari committed 2020-10-27 09:46:51 -04:00
1 parent 087c1c2fe9
commit 3d39bd575f
13 files changed
+370 -370

No files matched your search

+6 -6
View File
@@ -30,9 +30,7 @@ def iter_tokens(buffer: BufferStream) -> Iterator[Token]:
else:
j = m.start(0)
next_char = buf[j : j + 1]
if next_char == b"~":
continue
elif next_char == b"%":
if next_char == b"%":
buffer.seek(bufpos + j + 1)
yield parse_comment(buffer)
elif next_char == b"/":
@@ -54,10 +52,12 @@ def iter_tokens(buffer: BufferStream) -> Iterator[Token]:
buffer.seek(bufpos + j + 1)
yield parse_double_angled_bracket(buffer)
elif next_char == b">":
buffer.seek(bufpos + j + 1)
if buf[j : j + 2] == b">>":
buffer.seekd(1)
buffer.seek(bufpos + j)
bufpos, buf = buffer.read(2)
if buf == b">>":
yield TokenDictionary(bufpos + j, bufpos + j + 2, TDictValue.CLOSE)
else:
buffer.seekd(-1)
elif next_char == b"[":
buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
-243
View File
@@ -1,243 +0,0 @@
import io
from typing import Dict, Iterator, Optional, Tuple
from pdfmajor.streambuffer import BufferStream
from pdfmajor.parser.objects.stream import PDFStream
from pdfmajor.parser.exceptions import InvalidXref
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.lexer.token import TokenKeyword
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenComment,
TokenDictionary,
TokenKeyword,
)
from .objects.indirect import IndirectObject
from .objects.base import PDFContextualObject, PDFObject
from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import InvalidKeywordPos, BrokenFile, InvalidKeywordPos, ParserError
class XRefDB:
"""A class for collection references to objects"""
def __init__(self, parser: "PDFParser") -> None:
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
self.parser = parser
def save_indobject(self, indobj: IndirectObject):
if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None:
self.objs[(indobj.obj_num, indobj.gen_num)] = indobj
def get_object_from_indobj(self, indobj: IndirectObject) -> PDFObject:
return self.get_object(indobj.obj_num, indobj.gen_num)
def get_object(self, obj_num: int, gen_num: int) -> PDFObject:
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
Args:
obj_num (int): object number
gen_num (int): generation number
Returns:
PDFObject
"""
indobj = self.objs.get((obj_num, gen_num), None)
if indobj is None:
raise InvalidXref(
f"xref not found for object retrieval {obj_num}, {gen_num}"
)
else:
obj = indobj.get_object()
if obj is not None:
return obj
cur_pos = self.parser.tell()
self.parser.seek(indobj.offset)
obj = next(self.parser.iter_objects())
indobj.save_object(obj)
self.parser.seek(cur_pos)
return obj
class PDFParser:
"""PDF Parser"""
def __init__(
self,
buffer: BufferStream,
db: Optional[XRefDB] = None,
strict: bool = True,
) -> None:
"""
Args:
fp (io.BufferedIOBase)
buffer_size (int, optional): the size of the reads. Defaults to 4096.
state (Optional[ParsingState], optional): the parsing state. Defaults to None.
strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True.
"""
self.lexer = PDFLexer(buffer)
self.db = XRefDB(self)
self.strict = strict
@classmethod
def from_iobuffer(
cls,
fp: io.BufferedIOBase,
buffer_size: int = 4096,
strict: bool = True,
):
return cls(
BufferStream(fp, buffer_size=buffer_size),
strict=strict,
)
@classmethod
def from_bytes(
cls,
data: bytes,
buffer_size: int = 4096,
strict: bool = True,
):
return cls(
BufferStream(io.BytesIO(data), buffer_size=buffer_size),
strict=strict,
)
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.lexer.seek(offset)
def tell(self) -> int:
return self.lexer.tell()
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream
Yields:
PDFObject
"""
state = ParsingState([], [], strict=self.strict)
for token in self.lexer.iter_tokens():
attempt_prim = attempt_parse_prim(token, state)
if attempt_prim.success:
yield from state.feed_or_yield_objs(attempt_prim.next_objs)
elif isinstance(token, TokenComment):
yield state.set_last_obj(
PDFComment(
token.value,
(token.start_loc, token.end_loc),
)
)
elif isinstance(token, TokenKeyword):
if token.value == b"R":
yield from self._on_indirect_ref_close(state)
elif token.value == b"obj":
state.initialize_indirect_obj(token.start_loc)
elif token.value == b"endobj":
yield self._on_endobj(state, token)
elif token.value == b"stream":
self._on_stream(
state,
token,
)
elif token.value == b"endstream":
self._on_endstream(state, token)
else:
raise InvalidKeywordPos(token)
elif isinstance(token, TokenArray):
yield from state.flush_int_collections()
yield from deal_with_collection_object(
state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
)
elif isinstance(token, TokenDictionary):
yield from state.flush_int_collections()
yield from deal_with_collection_object(
state,
token=token,
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
strict=self.strict,
)
def _on_endobj(self, state: ParsingState, token: TokenKeyword) -> IndirectObject:
"""finish off the current context, assuming we are handling an indirect-object
Args:
token (TokenKeyword)
Raises:
BrokenFile: if the object fif not get initilized with two integers
InvalidKeywordPos: If the context is not an indirect object
Yields:
PDFObject
"""
last_ctx = state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
if len(state.int_collection) > 1:
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
for num in state.flush_int_collections():
last_ctx.pass_item(num)
obj = last_ctx.into_readonly_copy()
self.db.save_indobject(obj)
return obj
else:
raise InvalidKeywordPos(token)
def _on_stream(self, state: ParsingState, token: TokenKeyword):
"""Initilize a PDFStream
Args:
token (TokenKeyword)
Raises:
ParserError: if the current context is not an indirect object
"""
last_ctx = state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
self.db.save_indobject(last_ctx)
stream = PDFStream(token.end_loc, 0)
obj = self.db.get_object_from_indobj(last_ctx)
stream.pass_item(obj)
last_ctx.save_object(PDFStream(token.end_loc, 0))
state.context_stack.append(stream)
self.seek(stream.offset + stream.length)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
def _on_endstream(self, state: ParsingState, token: TokenKeyword):
"""Finish a PDFStream
Args:
token (TokenKeyword)
Raises:
InvalidKeywordPos: if the current context is not an indirect stream
"""
stream: PDFContextualObject = state.context_stack.pop()
if not isinstance(stream, PDFStream):
raise InvalidKeywordPos(token)
def _on_indirect_ref_close(self, state: ParsingState):
obj_num, gen_num = state.get_indobj_values()
obj = self.db.get_object(obj_num, gen_num)
cur_ctx = state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
+18 -19
View File
@@ -1,25 +1,24 @@
import io
from pdfmajor.streambuffer import BufferStream
from .objects.stream import PDFStream
from .PDFParser import PDFParser
from .fliters import process_filters_on_data
class PDFStreamPasrser(PDFParser):
def __init__(
self,
stream: PDFStream,
buffer: BufferStream,
strict: bool = True,
) -> None:
cur_pos = buffer.tell()
buffer.seek(stream.offset)
data = buffer.read(stream.length).data
buffer.seek(cur_pos)
data = process_filters_on_data(data, stream.filter, stream.decode_parms)
data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms)
self.stream = stream
super().__init__(
buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length),
strict=strict,
)
# class PDFStreamPasrser(PDFParser):
# def __init__(
# self,
# stream: PDFStream,
# buffer: BufferStream,
# strict: bool = True,
# ) -> None:
# cur_pos = buffer.tell()
# buffer.seek(stream.offset)
# data = buffer.read(stream.length).data
# buffer.seek(cur_pos)
# data = process_filters_on_data(data, stream.filter, stream.decode_parms)
# data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms)
# self.stream = stream
# super().__init__(
# buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length),
# strict=strict,
# )
+197
View File
@@ -0,0 +1,197 @@
from pdfmajor.parser.xref import XRefRow
from pdfmajor.parser.xrefdb import XRefDB
from typing import Iterator, Optional
from pdfmajor.streambuffer import BufferStream
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenComment,
TokenDictionary,
TokenKeyword,
)
from pdfmajor.lexer import iter_tokens
from .objects.stream import PDFStream
from .objects.indirect import IndirectObject
from .objects.base import PDFContextualObject, PDFObject
from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import (
InvalidKeywordPos,
BrokenFile,
InvalidKeywordPos,
InvalidXref,
ParserError,
)
def iter_objects(
buffer: BufferStream,
xrefdb: XRefDB,
state: Optional[ParsingState] = None,
) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream
Yields:
PDFObject
"""
state = ParsingState([], []) if state is None else state
for token in iter_tokens(buffer):
attempt_prim = attempt_parse_prim(token, state)
if attempt_prim.success:
yield from state.feed_or_yield_objs(attempt_prim.next_objs)
elif isinstance(token, TokenComment):
yield state.set_last_obj(
PDFComment(
token.value,
(token.start_loc, token.end_loc),
)
)
elif isinstance(token, TokenKeyword):
if token.value == b"R":
yield from _on_indirect_ref_close(buffer, state, xrefdb)
elif token.value == b"obj":
state.initialize_indirect_obj(token.start_loc)
elif token.value == b"endobj":
obj = _on_endobj(state, token)
xrefdb.save_indobject(obj)
yield obj
elif token.value == b"stream":
_on_stream(
buffer,
state,
xrefdb,
token,
)
else:
raise InvalidKeywordPos(token, [b"R", b"obj", b"endobj", b"stream"])
elif isinstance(token, TokenArray):
yield from state.flush_int_collections()
yield from deal_with_collection_object(
state, token, PDFArray, TArrayValue.OPEN
)
elif isinstance(token, TokenDictionary):
yield from state.flush_int_collections()
yield from deal_with_collection_object(
state,
token=token,
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
)
def _on_endobj(state: ParsingState, token: TokenKeyword) -> IndirectObject:
"""finish off the current context, assuming we are handling an indirect-object
Args:
token (TokenKeyword)
Raises:
BrokenFile: if the object fif not get initilized with two integers
InvalidKeywordPos: If the context is not an indirect object
Yields:
PDFObject
"""
last_ctx = state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
if len(state.int_collection) > 1:
raise BrokenFile(f"Invalid number of integers in obj defition {token}")
for num in state.flush_int_collections():
last_ctx.pass_item(num)
return last_ctx.into_readonly_copy()
else:
raise ParserError(f"Recived an endobj while not procesing an indirect object")
def _on_stream(
buffer: BufferStream, state: ParsingState, xrefdb: XRefDB, token: TokenKeyword
):
"""Initilize a PDFStream
Args:
token (TokenKeyword)
Raises:
ParserError: if the current context is not an indirect object
"""
last_ctx = state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
xrefdb.save_indobject(last_ctx)
stream = PDFStream(token.end_loc + 1, 0)
obj = _save_or_get_object(buffer, xrefdb, last_ctx.obj_num, last_ctx.gen_num)
with buffer.get_window():
next_token = next(iter_tokens(buffer))
if not (
isinstance(next_token, TokenKeyword) and next_token.value == b"endstream"
):
raise BrokenFile(
f"stream did not contain 'endstream', instead the token {next_token} was found"
)
stream.pass_item(obj)
buffer.seek(stream.offset + stream.length)
last_ctx.save_object(stream)
state.context_stack.append(stream)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
def _on_indirect_ref_close(buffer: BufferStream, state: ParsingState, xrefdb: XRefDB):
obj_num, gen_num = state.get_indobj_values()
obj = _save_or_get_object(buffer, xrefdb, obj_num, gen_num)
cur_ctx = state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
def _save_or_get_object(
buffer: BufferStream, xrefdb: XRefDB, obj_num: int, gen_num: int
) -> PDFObject:
"""get the actual value of the reference, if the value does not exists return the Null object as specified in PDF spec 1.7 section 7.3.10
Args:
obj_num (int): object number
gen_num (int): generation number
Returns:
PDFObject
"""
indobj = xrefdb.objs.get((obj_num, gen_num), None)
if indobj is None:
xref = xrefdb.xrefs.get((obj_num, gen_num), None)
if xref is None:
raise InvalidXref(
f"xref not found for object retrieval {obj_num}, {gen_num}"
)
return _read_object_from_xref(xref, xrefdb, buffer)
else:
return _read_object_from_db_or_buffer(indobj, xrefdb, buffer)
def _read_object_from_xref(
xref: XRefRow, xrefdb: XRefDB, buffer: BufferStream
) -> PDFObject:
indobj = IndirectObject(xref.obj_num, xref.gen_num, xref.offset)
with buffer.get_window():
buffer.seek(indobj.offset)
obj = next(iter_objects(buffer, xrefdb))
indobj.save_object(obj)
xrefdb.save_indobject(indobj)
return obj
def _read_object_from_db_or_buffer(
indobj: IndirectObject, xrefdb: XRefDB, buffer: BufferStream
) -> PDFObject:
obj = indobj.get_object()
if obj is not None:
return obj
with buffer.get_window():
buffer.seek(indobj.offset)
obj = next(iter_objects(buffer, xrefdb))
indobj.save_object(obj)
return obj
+3 -2
View File
@@ -1,3 +1,4 @@
from typing import List
from pdfmajor.lexer.token import TokenKeyword
from pdfmajor.exceptions import PDFMajorException
@@ -15,10 +16,10 @@ class UnexpectedEOF(BrokenFile, EOFError):
class InvalidKeywordPos(BrokenFile):
def __init__(self, token: TokenKeyword) -> None:
def __init__(self, token: TokenKeyword, expected_token: List[bytes]) -> None:
self.token = token
super().__init__(
f"keyword '{token.value!r}' was found at {token.start_loc}:{token.end_loc}"
f"keyword '{token.value!r}' was found at {token.start_loc}:{token.end_loc} when expecting {expected_token}"
)
+11 -8
View File
@@ -26,10 +26,12 @@ class IndirectObject(PDFContextualObject):
return self.__data
def save_object(self, obj: PDFObject):
# if self.__data is not None:
# raise InvalidIndirectObjAccess(
# f"attempted to override an initilized reference of an object {obj}"
# )
"""This method will override the current stored object
Args:
obj (PDFObject)
"""
self.__data = obj
def into_readonly_copy(self):
@@ -44,10 +46,11 @@ class IndirectObject(PDFContextualObject):
return obj.to_python()
def pass_item(self, item: PDFObject):
# if self.__read_only:
# raise InvalidIndirectObjAccess(
# f"attempted to add to an read-only reference of an object {item}"
# )
"""This method will either propogate the item to it's child (in case the child is a contexual object) or override it's own data with the incoming
Args:
item (PDFObject): [description]
"""
cur_obj = self.get_object()
if isinstance(cur_obj, PDFContextualObject):
cur_obj.pass_item(item)
+3 -4
View File
@@ -1,4 +1,4 @@
from typing import Iterator, List, Optional, NamedTuple, Type
from typing import Iterator, List, NamedTuple, Type
from pdfmajor.lexer.token import Token, TokenComplexType, TokenComplexTypeVal
@@ -49,7 +49,6 @@ def deal_with_collection_object(
token: TokenComplexType,
cls_const: Type[PDFContextualObject],
open_t: TokenComplexTypeVal,
strict: bool = False,
) -> Iterator[PDFObject]:
"""generic method for processing things that have a start and end token
@@ -72,9 +71,9 @@ def deal_with_collection_object(
return
else:
cur_ctx.pass_item(last_ctx)
elif strict:
else:
raise ParserError(f"Invalid end of {cls_const} specified at {token}")
elif strict:
else:
raise ParserError(
f"Invalid end of {cls_const} specified at {token} with no start"
)
+2 -5
View File
@@ -9,27 +9,24 @@ from .objects.indirect import IndirectObject
class ParsingState:
__slots__ = ["context_stack", "int_collection", "last_obj", "strict"]
__slots__ = ["context_stack", "int_collection", "last_obj"]
def __init__(
self,
context_stack: List[PDFContextualObject],
int_collection: List[PDFInteger],
last_obj: Optional[PDFObject] = None,
strict: bool = False,
) -> None:
self.context_stack = context_stack
self.int_collection = int_collection
self.last_obj = last_obj
self.strict = strict
def __repr__(self) -> str:
return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj}, strict={strict})".format(
return "ParsingState:[{stackc}]({cur_stack}, ints={ints}, last={lobj})".format(
stackc=len(self.context_stack),
cur_stack=self.current_context,
ints=self.int_collection,
lobj=self.last_obj,
strict=self.strict,
)
@property
+2 -3
View File
@@ -1,6 +1,6 @@
# from pdfmajor.parser.objects.indirect import IndirectObjectCollection
from pdfmajor.lexer.token import TokenInteger, TokenKeyword
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer import iter_tokens
from typing import Iterator, Literal, NamedTuple, Union
from pdfmajor.streambuffer import BufferStream
from .exceptions import PDFNoValidXRef, BrokenFile, UnexpectedEOF
@@ -40,8 +40,7 @@ def find_start_of_xref(buffer: BufferStream, strict: bool = False) -> int:
def iter_over_xref(buffer: BufferStream, strict: bool = False) -> Iterator[XRefRow]:
start_offset = find_start_of_xref(buffer, strict)
buffer.seek(start_offset)
lexer = PDFLexer(buffer)
first_token = next(lexer.iter_tokens())
first_token = next(iter_tokens(buffer))
if isinstance(first_token, TokenInteger):
raise NotImplementedError("XRefStream: PDF-1.5")
elif isinstance(first_token, TokenKeyword):
+27
View File
@@ -0,0 +1,27 @@
from pdfmajor.parser.xref import XRefRow
from typing import Dict, Iterator, Tuple
from pdfmajor.parser.exceptions import InvalidXref
from pdfmajor.parser.objects.base import PDFObject
from .objects.indirect import IndirectObject
from .objects.base import PDFObject
class XRefDB:
"""A class for collection references to objects"""
def __init__(self) -> None:
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
self.xrefs: Dict[Tuple[int, int], XRefRow] = {}
def update_from_xrefiter(self, it_xref: Iterator[XRefRow]):
for xref in it_xref:
if xref.use == b"n":
self.xrefs[(xref.obj_num, xref.gen_num)] = xref
def save_indobject(self, indobj: IndirectObject):
if self.objs.get((indobj.obj_num, indobj.gen_num), None) is None:
self.objs[(indobj.obj_num, indobj.gen_num)] = indobj
else:
print("WARN: saving twice", indobj)
+11
View File
@@ -1,5 +1,6 @@
import io
from typing import Iterator, NamedTuple, Optional
from contextlib import contextmanager
from pdfmajor.exceptions import PDFMajorException
@@ -28,6 +29,10 @@ class BufferStream:
self.fp = fp
self.buffer_size = buffer_size
@classmethod
def from_bytes(cls, raw: bytes, buffer_size: int = 4096):
return cls(io.BytesIO(raw), buffer_size)
def get_slice(
self, offset: int, length: int, buffer_size: Optional[int] = None
) -> "BufferStream":
@@ -175,3 +180,9 @@ class BufferStream:
buf = b""
if len(buf) > 0 and pos == 0 and n == -1:
yield BufferedBytes(pos=n + 1, data=buf)
@contextmanager
def get_window(self):
cur_pos = self.tell()
yield self
self.seek(cur_pos)
+84 -78
View File
@@ -1,25 +1,46 @@
from pathlib import Path
from pdfmajor.parser.xrefdb import XRefDB
from pdfmajor.parser import iter_objects
from pdfmajor.parser.objects.collections import PDFDictionary
from pdfmajor.parser.objects.primitives import PDFHexString, PDFInteger, PDFString
from typing import List
from typing import Any, List
from decimal import Decimal
from unittest import TestCase
from pdfmajor.streambuffer import StreamEOF
from pdfmajor.streambuffer import BufferStream
from pdfmajor.parser.objects.indirect import IndirectObject
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.comment import PDFComment
from pdfmajor.parser.objects.stream import PDFStream
from pdfmajor.parser.PDFParser import PDFParser
CURRENT_FOLDER = Path(__file__).parent
class Collections(TestCase):
def run_test(self, raw: bytes, expected: Any):
size = len(raw)
for buf_size in [
2,
3,
size // 4,
size // 3,
size // 2,
3 * size // 4,
size,
2 * size,
4 * size,
]:
buffer = BufferStream.from_bytes(raw, buffer_size=buf_size)
it = iter_objects(buffer, XRefDB())
obj = next(it)
self.assertEqual(obj.to_python(), expected)
with self.assertRaises(StopIteration):
next(it)
def test_parse_dict(self):
parser = PDFParser.from_bytes(
br"""<< /Type /Example
self.run_test(
raw=br"""<< /Type /Example
/Subtype /DictionaryExample
/Version 0.01
/IntegerItem 12
@@ -30,12 +51,7 @@ class Collections(TestCase):
/LastItem ( not! )
/VeryLastItem ( OK ) >>
>>""",
strict=True,
)
obj = next(parser.iter_objects())
self.assertDictEqual(
obj.to_python(),
{
expected={
"Type": "/Example",
"Subtype": "/DictionaryExample",
"Version": Decimal("0.01"),
@@ -51,28 +67,18 @@ class Collections(TestCase):
)
def test_parse_array(self):
parser = PDFParser.from_bytes(
br"""[ 549 3.14 false ( Ralph ) /SomeName ]""",
strict=True,
)
obj = next(parser.iter_objects())
self.assertListEqual(
obj.to_python(),
[549, Decimal("3.14"), False, " Ralph ", "/SomeName"],
self.run_test(
raw=br"""[ 549 3.14 false ( Ralph ) /SomeName ]""",
expected=[549, Decimal("3.14"), False, " Ralph ", "/SomeName"],
)
def test_parse_nested_array(self):
parser = PDFParser.from_bytes(
br"""[ [10 30] [22 33] [
self.run_test(
raw=br"""[ [10 30] [22 33] [
<< /x 0 /y 5 >>
<< /x 10 /y 5 >>
] ]""",
strict=True,
)
obj = next(parser.iter_objects())
self.assertListEqual(
obj.to_python(),
[
expected=[
[10, 30],
[22, 33],
[
@@ -84,9 +90,32 @@ class Collections(TestCase):
class IndirectObjects(TestCase):
def run_test(self, raw: bytes, expected: List[PDFObject]) -> XRefDB:
db: XRefDB = XRefDB()
size = len(raw)
for buf_size in [
2,
3,
size // 4,
size // 3,
size // 2,
3 * size // 4,
size,
2 * size,
4 * size,
]:
db = XRefDB()
buffer = BufferStream.from_bytes(raw, buffer_size=buf_size)
it = iter_objects(buffer, db)
for obj, eobj in zip(it, expected):
self.assertEqual(obj, eobj)
with self.assertRaises((StopIteration, EOFError)):
next(it)
return db
def test_parse_simple_indobj(self):
parser = PDFParser.from_bytes(
br"""
self.run_test(
raw=br"""
12 1 obj (Bring) endobj
100 0 obj
<33>
@@ -95,66 +124,43 @@ class IndirectObjects(TestCase):
obj
<< /x 1 /y 3>> endobj
""",
strict=True,
)
expected = [
IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)),
IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)),
IndirectObject(
13,
8,
144,
PDFDictionary.from_dict(
{"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)}
expected=[
IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)),
IndirectObject(100, 0, 25, PDFHexString(b"3", 0, 0)),
IndirectObject(
13,
8,
144,
PDFDictionary.from_dict(
{"x": PDFInteger(1, 0, 0), "y": PDFInteger(3, 0, 0)}
),
),
),
]
for obj, eobj in zip(parser.iter_objects(), expected):
self.assertIsInstance(obj, IndirectObject)
self.assertEqual(obj, eobj)
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
],
)
def test_parse_obj_with_references(self):
parser = PDFParser.from_bytes(
br"""
db = self.run_test(
raw=br"""
12 1 obj (Bring) endobj
10 2 obj << /lastone 12 1 R >> endobj
""",
strict=True,
expected=[
IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)),
IndirectObject(
0,
2,
54,
data=PDFDictionary.from_dict({"lastone": PDFString("Bring", 0, 0)}),
),
],
)
expected = [
"Bring",
{"lastone": "Bring"},
]
for obj, exp in zip(parser.iter_objects(), expected):
self.assertEqual(obj.to_python(), exp)
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
sec_obj = parser.db.get_object(10, 2)
sec_obj = db.objs[(10, 2)]
self.assertEqual(
sec_obj.to_python(),
{"lastone": "Bring"},
)
def run_test(
self,
raw: bytes,
expected: List[PDFObject],
):
parser = PDFParser.from_bytes(raw, strict=True)
for i, (eobj, obj) in enumerate(
zip(expected, parser.iter_objects()),
1,
):
self.assertEqual(obj, eobj, f"Failed at example #{i}")
try:
next_obj = next(parser.iter_objects())
self.assertIsNone(next_obj)
except (EOFError, StopIteration) as e:
self.assertIsNotNone(e)
def test_parse_comments(self):
self.run_test(
br"""
@@ -175,7 +181,7 @@ class IndirectObjects(TestCase):
)
def test_parse_stream(self):
parser = PDFParser.from_bytes(
buffer = BufferStream.from_bytes(
br"""
5 0 obj
<< /Type /XRef /Length 124 /Filter /FlateDecode >>
@@ -185,7 +191,7 @@ stream
endobj
"""
)
for obj in parser.iter_objects():
for obj in iter_objects(buffer, XRefDB()):
self.assertIsInstance(obj, IndirectObject)
self.assertIsInstance(obj.get_object(), PDFStream)
self.assertDictEqual(
+6 -2
View File
@@ -1,4 +1,6 @@
from pathlib import Path
from pdfmajor.parser.xrefdb import XRefDB
from pdfmajor.parser import _save_or_get_object
from unittest import TestCase
from pdfmajor.streambuffer import BufferStream
@@ -11,5 +13,7 @@ class Standard(TestCase):
def test_char_doc(self):
with (CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf").open("rb") as fp:
buffer = BufferStream(fp)
for xref in iter_over_xref(buffer):
print(xref)
db = XRefDB()
db.update_from_xrefiter(iter_over_xref(buffer))
for obj, gen in db.xrefs.keys():
print(_save_or_get_object(buffer, db, obj, gen))