added parsing for object types

This commit is contained in:
ari committed 2020-10-28 10:07:30 -04:00
1 parent 78175d3a64
commit 4e3b3e10b7
8 files changed
+324 -135

No files matched your search

+38
View File
@@ -0,0 +1,38 @@
from datetime import datetime
from pdfmajor.lexer.token import TokenComment
from typing import Any, Dict, NamedTuple, Optional, Tuple
from pdfmajor.lexer import iter_tokens
from pdfmajor.streambuffer import BufferStream
class PDFDocumentCatalog(NamedTuple):
"""A Parsed version of the PDF Catalog object (see PDF spec 1.7 section 7.7.2 for more detail)
Args:
NamedTuple ([type]): [description]
"""
title: str
author: str
subject: str
keywords: str
creator: str
producer: str
creation_date: datetime
mod_date: datetime
trapped: str
mediabox: Tuple[int, int, int, int]
raw: Dict[str, Any]
@classmethod
def from_bufferstream(cls, buffer: BufferStream):
version = _get_version(buffer)
def _get_version(buffer: BufferStream) -> Optional[str]:
with buffer.get_window():
token = next(iter_tokens(buffer))
if isinstance(token, TokenComment):
return token.value.decode()
return None
+38
View File
@@ -0,0 +1,38 @@
from datetime import datetime
from pdfmajor.lexer.token import TokenComment
from typing import Any, Dict, NamedTuple, Optional, Tuple
from pdfmajor.lexer import iter_tokens
from pdfmajor.streambuffer import BufferStream
class PDFDocumentParsedMetadata(NamedTuple):
"""A Parsed version of the PDF Metadata object (see PDF spec 1.7 section 14.3.3 for more detail)
Args:
NamedTuple ([type]): [description]
"""
title: str
author: str
subject: str
keywords: str
creator: str
producer: str
creation_date: datetime
mod_date: datetime
trapped: str
mediabox: Tuple[int, int, int, int]
raw_metadata: Dict[str, Any]
@classmethod
def from_bufferstream(cls, buffer: BufferStream):
version = _get_version(buffer)
def _get_version(buffer: BufferStream) -> Optional[str]:
with buffer.get_window():
token = next(iter_tokens(buffer))
if isinstance(token, TokenComment):
return token.value.decode()
return None
+25 -8
View File
@@ -1,3 +1,4 @@
from pdfmajor.parser.objects.primitives import PDFInteger
from pdfmajor.parser.stream.PDFStream import PDFStream
from typing import Iterator, Optional
from pdfmajor.streambuffer import BufferStream
@@ -17,6 +18,7 @@ from .objects.comment import PDFComment
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import (
EarlyStop,
InvalidKeywordPos,
BrokenFile,
InvalidKeywordPos,
@@ -24,6 +26,14 @@ from .exceptions import (
)
def get_first_object(
buffer: BufferStream, offset: int, state: Optional[ParsingState] = None
) -> PDFObject:
with buffer.get_window():
buffer.seek(offset)
return next(iter_objects(buffer, state))
def iter_objects(
buffer: BufferStream,
state: Optional[ParsingState] = None,
@@ -54,7 +64,7 @@ def iter_objects(
obj = _on_endobj(state, token)
yield obj
elif token.value == b"stream":
_on_stream(
yield from _on_stream(
buffer,
state,
token,
@@ -115,13 +125,20 @@ def _on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
if not isinstance(obj, PDFDictionary):
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
buffer.seek(stream.offset + stream.length)
next_token = next(iter_tokens(buffer))
if not (
isinstance(next_token, TokenKeyword) and next_token.value == b"endstream"
):
raise BrokenFile(
f"stream did not contain 'endstream', instead the token {next_token} was found"
if isinstance(stream.length, PDFInteger):
buffer.seek(stream.offset + stream.length.to_python())
next_token = next(iter_tokens(buffer))
if not (
isinstance(next_token, TokenKeyword)
and next_token.value == b"endstream"
):
raise BrokenFile(
f"stream did not contain 'endstream', instead the token {next_token} was found"
)
elif isinstance(stream.length, ObjectRef):
yield last_ctx
raise EarlyStop(
"cannot proceed parsing as the stream.length object is an indirect object"
)
else:
raise ParserError(f"PDFStream is missing initilization dictionary")
+4
View File
@@ -7,6 +7,10 @@ class ParserError(PDFMajorException):
pass
class EarlyStop(ParserError, StopIteration):
pass
class BrokenFileParserError(BrokenFile):
pass
+45 -35
View File
@@ -1,8 +1,8 @@
from pdfmajor.parser.exceptions import ParserError
from typing import Any, Dict, List, Optional, Union
from pdfmajor.parser.objects.base import PDFObject
from typing import Any, List, Optional, TypeVar, Union
from ..objects.base import PDFObject
from ..objects.collections import PDFDictionary
from ..objects.primitives import PDFNull
class PDFStream:
@@ -11,18 +11,18 @@ class PDFStream:
def __init__(
self,
offset: int = 0,
length: int = 0,
filter: Optional[List[str]] = None,
decode_parms: Optional[List[Dict[str, Any]]] = None,
f: Optional[Any] = None, # file-specifications
ffilter: Optional[List[str]] = None,
fdecode_parms: Optional[List[Dict[str, Any]]] = None,
dl: Optional[int] = None,
length: PDFObject = PDFNull(),
filter: Optional[List[PDFObject]] = None,
decode_parms: Optional[List[PDFObject]] = None,
f: Optional[Any] = None, # TODO: file-specifications
ffilter: Optional[List[PDFObject]] = None,
fdecode_parms: Optional[List[PDFObject]] = None,
dl: Optional[PDFObject] = None,
) -> None:
"""
Args:
offset (int): the offset where the stream itself starts
length (int): the length of the stream
length (PotentialInt): the length of the stream
filter (Optional[List[str]], optional): The names of a filter that shall be applied in processing the stream data. Defaults to None.
decode_parms (Optional[List[Dict[str, Any]]], optional): Parameter dictionaries used by the filters specified by filter. Defaults to None.
f (Optional[Any], optional): The file containing the stream data. If this entry is present, the bytes between stream and endstream shall be ignored. However, the Length entry should still specify the number of those bytes (usually, there are no bytes and Length is 0).. Defaults to None.
@@ -32,49 +32,59 @@ class PDFStream:
"""
self.offset = offset
self.length = length
self.filter: Optional[List[str]] = filter
self.decode_parms: Optional[List[Dict[str, Any]]] = decode_parms
self.f: Optional[Any] = f
self.ffilter: Optional[List[str]] = ffilter
self.fdecode_parms: Optional[List[Dict[str, Any]]] = fdecode_parms
self.dl: Optional[int] = dl
self.filter = filter
self.decode_parms = decode_parms
self.f = f
self.ffilter = ffilter
self.fdecode_parms = fdecode_parms
self.dl = dl
@classmethod
def from_pdfdict(cls, offset: int, item: PDFDictionary) -> "PDFStream":
data = item.to_python()
stream = cls(offset, data["Length"])
stream.filter = get_single_or_list(data.get("Filter", stream.filter))
stream = cls(offset, item["Length"])
stream.filter = get_single_or_list(item.get("Filter", stream.filter))
stream.decode_parms = get_single_or_list(
data.get("DecodeParms", stream.decode_parms)
item.get("DecodeParms", stream.decode_parms)
)
stream.f = data.get("F", stream.f)
stream.ffilter = get_single_or_list(data.get("FFilter", stream.ffilter))
stream.f = item.get("F", stream.f)
stream.ffilter = get_single_or_list(item.get("FFilter", stream.ffilter))
stream.fdecode_parms = get_single_or_list(
data.get("FDecodeParms", stream.fdecode_parms)
item.get("FDecodeParms", stream.fdecode_parms)
)
return stream
def to_python(self):
return {
"offset": self.offset,
"length": self.length,
"filter": self.filter,
"decode_parms": None
if self.decode_parms is None
else [x.to_python() for x in self.decode_parms],
"length": self.length.to_python(),
"filter": to_python_list(self.filter),
"decode_parms": to_python_list(self.decode_parms),
"f": self.f,
"ffilter": self.ffilter,
"fdecode_parms": None
if self.fdecode_parms is None
else [x.to_python() for x in self.fdecode_parms],
"dl": self.dl,
"ffilter": to_python_list(self.ffilter),
"fdecode_parms": to_python_list(self.fdecode_parms),
"dl": to_python(self.dl),
}
def get_single_or_list(obj: Optional[Union[List[Any], Any]]) -> List[Any]:
T = TypeVar("T")
def get_single_or_list(obj: Optional[Union[List[T], T]]) -> List[T]:
if obj is None:
return []
elif isinstance(obj, list):
return obj
else:
return [obj]
def to_python(obj: Optional[PDFObject]) -> Any:
if obj is None:
return None
return obj.to_python()
def to_python_list(obj: Optional[List[PDFObject]]) -> List[Any]:
if obj is None:
return []
return [x.to_python() for x in obj]
+27 -20
View File
@@ -159,30 +159,37 @@ class BufferStream:
Yields:
BufferedBytes
"""
self.fp.seek(0, 2)
pos = self.fp.tell()
buf = b""
n = -1
while 0 < pos:
prevpos = pos
pos = max(0, pos - self.buffer_size)
self.fp.seek(pos)
s = self.fp.read(prevpos - pos)
if not s:
break
while True:
n = max(s.rfind(b"\r"), s.rfind(b"\n"))
if n == -1:
buf = s + buf
with self.get_window():
self.fp.seek(0, 2)
pos = self.fp.tell()
buf = b""
n = -1
while 0 < pos:
prevpos = pos
pos = max(0, pos - self.buffer_size)
self.fp.seek(pos)
s = self.fp.read(prevpos - pos)
if not s:
break
yield BufferedBytes(pos=n + 1, data=(s[n + 1 :] + buf))
s = s[:n]
buf = b""
if len(buf) > 0 and pos == 0 and n == -1:
yield BufferedBytes(pos=n + 1, data=buf)
while True:
n = max(s.rfind(b"\r"), s.rfind(b"\n"))
if n == -1:
buf = s + buf
break
yield BufferedBytes(pos=n + 1, data=(s[n + 1 :] + buf))
s = s[:n]
buf = b""
if len(buf) > 0 and pos == 0 and n == -1:
yield BufferedBytes(pos=n + 1, data=buf)
@contextmanager
def get_window(self):
"""Creates a context for the buffer
when this context is generated we save the original position of the buffer
whent the context exits we revert back to the original position.
This ensures that we can burrow the buffer during other processing of the item.
*NOTE*: using this in async processes will not ensure the safety of the buffer. This method should only be used in sync operations.
"""
cur_pos = self.tell()
yield self
self.seek(cur_pos)
+74 -39
View File
@@ -1,51 +1,86 @@
from typing import Dict, Iterator, Tuple
from pdfmajor.parser.objects.collections import PDFDictionary
from pdfmajor.streambuffer import BufferStream
from typing import Dict, Iterator, List, Optional, Tuple
from pdfmajor.parser.objects.indirect import IndirectObject
from .xref import XRefRow
from .exceptions import InvalidIndirectObjAccess
from pdfmajor.parser import get_first_object
from .xref import XRefRow, iter_over_xref
from .exceptions import InvalidIndirectObjAccess, InvalidXref, XRefError
# simple type aliases for better readability
ObjNum = int
GenNum = int
RawRef = Tuple[ObjNum, GenNum]
ObjType = str
class XRefDB:
"""A class for collection references to objects"""
def __init__(self) -> None:
self.objs: Dict[Tuple[int, int], IndirectObject] = {}
self.xrefs: Dict[Tuple[int, int], XRefRow] = {}
def __init__(self, buffer: BufferStream) -> None:
self.objs: Dict[RawRef, IndirectObject] = {}
self.types: Dict[RawRef, ObjType] = {}
self.xrefs: Dict[RawRef, XRefRow] = {}
@classmethod
def from_xrefiter(cls, it_xref: Iterator[XRefRow]) -> "XRefDB":
db = cls()
db.update_from_xrefiter(it_xref)
return db
with buffer.get_window():
for xref in iter_over_xref(buffer):
if xref.use == b"n":
self.xrefs[(xref.obj_num, xref.gen_num)] = xref
def has_xref(self, obj_num: int, gen_num: int) -> bool:
def get_obj_by_type(
self, buffer: BufferStream, obj_type: ObjType
) -> Iterator[IndirectObject]:
"""Iterates over the objects in memeory and filters out for the ones that match the requested type
If the object is not loaded into memory this function will load it.
Args:
buffer (BufferStream)
obj_type (ObjType)
Raises:
InvalidXref: if the object could not be found in memory or the buffer
XRefError: if the object type if found but the object was never loaded into memory
Yields:
Iterator[IndirectObject]
"""
for raw_ref in self.xrefs.keys():
ftype = self.get_type_for(*raw_ref, buffer=buffer)
if ftype == obj_type:
yield self.objs[raw_ref]
def get_type_for(
self, obj_num: ObjNum, gen_num: GenNum, buffer: BufferStream
) -> str:
raw_ref, xref = (obj_num, gen_num), self.xrefs[(obj_num, gen_num)]
found_obj_type: Optional[str] = self.types.get(raw_ref, None)
if found_obj_type is None:
obj = self.objs.get(raw_ref, None)
if obj is None:
obj = get_first_object(buffer, xref.offset)
if not isinstance(obj, IndirectObject):
raise InvalidXref(
f"Offset for {xref} returned an invalid object {obj}"
)
obj_dict = obj.get_object()
if isinstance(obj_dict, PDFDictionary):
type_name = obj_dict.get("Type", None)
if type_name is not None:
found_obj_type = type_name.to_python()
else:
found_obj_type = "{{PDFUnknownComplexType}}"
else:
found_obj_type = "{{PDFPrimitive}}"
else:
obj = self.objs.get(raw_ref, None)
if obj is None:
raise XRefError(f"Missing valid obj for {xref}")
if found_obj_type is None:
raise XRefError("WARN: Failed to find a valid type for", xref)
return found_obj_type
def has_xref(self, obj_num: ObjNum, gen_num: GenNum) -> bool:
return (obj_num, gen_num) in self.xrefs.keys()
def has_obj(self, obj_num: int, gen_num: int) -> bool:
def has_obj(self, obj_num: ObjNum, gen_num: GenNum) -> bool:
return (obj_num, gen_num) in self.objs.keys()
def update_from_xrefiter(self, it_xref: Iterator[XRefRow]):
for xref in it_xref:
if xref.use == b"n":
self.xrefs[(xref.obj_num, xref.gen_num)] = xref
def create_indobject(
self, obj_num: int, gen_num: int, offset: int
) -> IndirectObject:
if self.has_obj(obj_num, gen_num):
raise InvalidIndirectObjAccess(
f"Attempted to recreate {(obj_num, gen_num)}"
)
cur_obj = self.objs[(obj_num, gen_num)] = IndirectObject(
obj_num, gen_num, offset
)
return cur_obj
def get_or_create_indobject(self, obj_num: int, gen_num: int) -> IndirectObject:
cur_obj = self.objs.get((obj_num, gen_num), None)
if cur_obj is None:
print("WARN: got a non-existent reference")
cur_obj = self.objs[(obj_num, gen_num)] = IndirectObject(
obj_num, gen_num, -1
)
return cur_obj
+73 -33
View File
@@ -10,43 +10,83 @@ CURRENT_FOLDER = Path(__file__).parent
class Standard(TestCase):
def test_char_doc(self):
expected_xrefs = {
(1, 0): XRefRow(offset=27259, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
(3, 0): XRefRow(offset=344, obj_num=3, gen_num=0, use=b"n"),
(4, 0): XRefRow(offset=27402, obj_num=4, gen_num=0, use=b"n"),
(5, 0): XRefRow(offset=364, obj_num=5, gen_num=0, use=b"n"),
(6, 0): XRefRow(offset=714, obj_num=6, gen_num=0, use=b"n"),
(7, 0): XRefRow(offset=27545, obj_num=7, gen_num=0, use=b"n"),
(8, 0): XRefRow(offset=734, obj_num=8, gen_num=0, use=b"n"),
(9, 0): XRefRow(offset=5949, obj_num=9, gen_num=0, use=b"n"),
(10, 0): XRefRow(offset=5970, obj_num=10, gen_num=0, use=b"n"),
(11, 0): XRefRow(offset=6176, obj_num=11, gen_num=0, use=b"n"),
(12, 0): XRefRow(offset=6503, obj_num=12, gen_num=0, use=b"n"),
(13, 0): XRefRow(offset=6700, obj_num=13, gen_num=0, use=b"n"),
(14, 0): XRefRow(offset=11679, obj_num=14, gen_num=0, use=b"n"),
(15, 0): XRefRow(offset=11701, obj_num=15, gen_num=0, use=b"n"),
(16, 0): XRefRow(offset=11905, obj_num=16, gen_num=0, use=b"n"),
(17, 0): XRefRow(offset=12232, obj_num=17, gen_num=0, use=b"n"),
(18, 0): XRefRow(offset=12427, obj_num=18, gen_num=0, use=b"n"),
(19, 0): XRefRow(offset=17804, obj_num=19, gen_num=0, use=b"n"),
(20, 0): XRefRow(offset=17826, obj_num=20, gen_num=0, use=b"n"),
(21, 0): XRefRow(offset=18037, obj_num=21, gen_num=0, use=b"n"),
(22, 0): XRefRow(offset=18364, obj_num=22, gen_num=0, use=b"n"),
(23, 0): XRefRow(offset=18565, obj_num=23, gen_num=0, use=b"n"),
(24, 0): XRefRow(offset=26258, obj_num=24, gen_num=0, use=b"n"),
(25, 0): XRefRow(offset=26280, obj_num=25, gen_num=0, use=b"n"),
(26, 0): XRefRow(offset=26477, obj_num=26, gen_num=0, use=b"n"),
(27, 0): XRefRow(offset=26882, obj_num=27, gen_num=0, use=b"n"),
(28, 0): XRefRow(offset=27141, obj_num=28, gen_num=0, use=b"n"),
(29, 0): XRefRow(offset=27204, obj_num=29, gen_num=0, use=b"n"),
(30, 0): XRefRow(offset=27650, obj_num=30, gen_num=0, use=b"n"),
(31, 0): XRefRow(offset=27747, obj_num=31, gen_num=0, use=b"n"),
}
expected_types = {
(1, 0): "/Page",
(2, 0): "{{PDFUnknownComplexType}}",
(3, 0): "{{PDFPrimitive}}",
(4, 0): "/Page",
(5, 0): "{{PDFUnknownComplexType}}",
(6, 0): "{{PDFPrimitive}}",
(7, 0): "/Pages",
(8, 0): "{{PDFUnknownComplexType}}",
(9, 0): "{{PDFPrimitive}}",
(10, 0): "/FontDescriptor",
(11, 0): "{{PDFUnknownComplexType}}",
(12, 0): "/Font",
(13, 0): "{{PDFUnknownComplexType}}",
(14, 0): "{{PDFPrimitive}}",
(15, 0): "/FontDescriptor",
(16, 0): "{{PDFUnknownComplexType}}",
(17, 0): "/Font",
(18, 0): "{{PDFUnknownComplexType}}",
(19, 0): "{{PDFPrimitive}}",
(20, 0): "/FontDescriptor",
(21, 0): "{{PDFUnknownComplexType}}",
(22, 0): "/Font",
(23, 0): "{{PDFUnknownComplexType}}",
(24, 0): "{{PDFPrimitive}}",
(25, 0): "/FontDescriptor",
(26, 0): "{{PDFUnknownComplexType}}",
(27, 0): "/Font",
(28, 0): "{{PDFUnknownComplexType}}",
(29, 0): "{{PDFUnknownComplexType}}",
(30, 0): "/Catalog",
(31, 0): "{{PDFUnknownComplexType}}",
}
with (CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf").open("rb") as fp:
buffer = BufferStream(fp)
db = XRefDB()
db.update_from_xrefiter(iter_over_xref(buffer))
db = XRefDB(buffer)
self.assertDictEqual(
db.xrefs,
expected_xrefs,
)
self.assertDictEqual(
{
(1, 0): XRefRow(offset=27259, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
(3, 0): XRefRow(offset=344, obj_num=3, gen_num=0, use=b"n"),
(4, 0): XRefRow(offset=27402, obj_num=4, gen_num=0, use=b"n"),
(5, 0): XRefRow(offset=364, obj_num=5, gen_num=0, use=b"n"),
(6, 0): XRefRow(offset=714, obj_num=6, gen_num=0, use=b"n"),
(7, 0): XRefRow(offset=27545, obj_num=7, gen_num=0, use=b"n"),
(8, 0): XRefRow(offset=734, obj_num=8, gen_num=0, use=b"n"),
(9, 0): XRefRow(offset=5949, obj_num=9, gen_num=0, use=b"n"),
(10, 0): XRefRow(offset=5970, obj_num=10, gen_num=0, use=b"n"),
(11, 0): XRefRow(offset=6176, obj_num=11, gen_num=0, use=b"n"),
(12, 0): XRefRow(offset=6503, obj_num=12, gen_num=0, use=b"n"),
(13, 0): XRefRow(offset=6700, obj_num=13, gen_num=0, use=b"n"),
(14, 0): XRefRow(offset=11679, obj_num=14, gen_num=0, use=b"n"),
(15, 0): XRefRow(offset=11701, obj_num=15, gen_num=0, use=b"n"),
(16, 0): XRefRow(offset=11905, obj_num=16, gen_num=0, use=b"n"),
(17, 0): XRefRow(offset=12232, obj_num=17, gen_num=0, use=b"n"),
(18, 0): XRefRow(offset=12427, obj_num=18, gen_num=0, use=b"n"),
(19, 0): XRefRow(offset=17804, obj_num=19, gen_num=0, use=b"n"),
(20, 0): XRefRow(offset=17826, obj_num=20, gen_num=0, use=b"n"),
(21, 0): XRefRow(offset=18037, obj_num=21, gen_num=0, use=b"n"),
(22, 0): XRefRow(offset=18364, obj_num=22, gen_num=0, use=b"n"),
(23, 0): XRefRow(offset=18565, obj_num=23, gen_num=0, use=b"n"),
(24, 0): XRefRow(offset=26258, obj_num=24, gen_num=0, use=b"n"),
(25, 0): XRefRow(offset=26280, obj_num=25, gen_num=0, use=b"n"),
(26, 0): XRefRow(offset=26477, obj_num=26, gen_num=0, use=b"n"),
(27, 0): XRefRow(offset=26882, obj_num=27, gen_num=0, use=b"n"),
(28, 0): XRefRow(offset=27141, obj_num=28, gen_num=0, use=b"n"),
(29, 0): XRefRow(offset=27204, obj_num=29, gen_num=0, use=b"n"),
(30, 0): XRefRow(offset=27650, obj_num=30, gen_num=0, use=b"n"),
(31, 0): XRefRow(offset=27747, obj_num=31, gen_num=0, use=b"n"),
raw_ref: db.get_type_for(*raw_ref, buffer)
for raw_ref in db.xrefs.keys()
},
expected_types,
)