parsing pdf page leafs

This commit is contained in:
ari committed 2020-11-02 13:11:31 -05:00
1 parent ffb7b843ea
commit b48d635ec7
8 files changed
+371 -23

No files matched your search

+2 -2
View File
@@ -1,7 +1,7 @@
from pathlib import Path
from typing import BinaryIO, List
from pdfmajor.document.pages import PDFPageTreeNode
from pdfmajor.document.pages import PDFPage
from pdfmajor.document.parsers.root import get_catalog, get_info
from pdfmajor.document.PDFParsingContext import PDFParsingContext
from pdfmajor.streambuffer import BufferStream
@@ -26,7 +26,7 @@ class PDFDocument:
self.__parser = PDFParsingContext(buffer)
self.catalog = get_catalog(self.__parser)
self.info = get_info(self.__parser)
self.pages: List[PDFPageTreeNode] = []
self.pages: List[PDFPage] = []
@property
def health_report(self):
+48 -4
View File
@@ -1,12 +1,14 @@
from pathlib import Path
from typing import BinaryIO, Dict, Iterator, Optional, Tuple, Type, TypeVar
from typing import Any, BinaryIO, Dict, Iterator, Optional, Tuple, Type, TypeVar, Union
from pdfmajor.document.structures import PDFRectangle
from pdfmajor.healthlog import PDFHealthReport
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.collections import PDFDictionary
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef
from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal
from pdfmajor.streambuffer import BufferStream
from pdfmajor.util import validate_object_or_none
from pdfmajor.util import validate_number_or_none, validate_object_or_none
from pdfmajor.xref.xrefdb import GenNum, ObjNum, XRefDB
@@ -64,6 +66,24 @@ class PDFParsingContext:
)
return validate_object_or_none(obj, exptype)
def get_validate_numeric(
self, obj: Optional[PDFObject]
) -> Optional[Union[PDFInteger, PDFReal]]:
"""Validates the object is numeric, if its not None
Args:
obj (Optional[PDFObject])
Raises:
AssertionError: is raised if the object has an invalid type
Returns:
Optional[Union[PDFInteger, PDFReal]]
"""
if isinstance(obj, ObjectRef):
return validate_number_or_none(self.get_object_from_ref(obj).get_object())
return validate_number_or_none(obj)
def convert_pdfdict_to_validated_pythondict(
self,
pdfdict: PDFDictionary,
@@ -73,7 +93,7 @@ class PDFParsingContext:
Args:
pdfdict (PDFDictionary)
field_mapping (Dict[str, Tuple[Type[PDFObject]]])
field_mapping (Dict[str, Tuple[Type[PDFObject]]]): mapping of the name as it in the pdf to a tuple with a (new name, expected objcet type)
Returns:
dict
@@ -88,6 +108,30 @@ class PDFParsingContext:
self.health_report.write_error(err)
return out
def extract_and_validate_pdf_rectangles(
self, pdfdict: PDFDictionary, field_mapping: Dict[str, str]
) -> Dict[str, Optional[PDFRectangle]]:
"""Takes a pdf-dictionary and extracts the specified fields. Next it will validate and convert the fields into PDFRectangles
Args:
pdfdict (PDFDictionary)
field_mapping (Dict[str, str]): a mapping of names as they appear in the pdf to requested names
Returns:
Dict[str, Optional[PDFRectangle]]
"""
vetted_values: Any = self.convert_pdfdict_to_validated_pythondict(
pdfdict,
{
pdfname: (newname, PDFArray)
for pdfname, newname in field_mapping.items()
},
)
return {
newname: PDFRectangle.from_pdfarray(value) if value is not None else value
for newname, value in vetted_values.items()
}
def get_obj_by_type(self, obj_type: str) -> Iterator[IndirectObject]:
"""Iterates over the objects in memeory and filters out for the ones that match the requested type
If the object is not loaded into memory this function will load it.
+17 -15
View File
@@ -1,14 +1,16 @@
from decimal import Decimal
from typing import List, NamedTuple, Optional
from pdfmajor.document.exceptions import InvalidPagesNodeKids
from pdfmajor.document.structures import PDFRectangle
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
from pdfmajor.exceptions import PDFMajorException
from pdfmajor.parser.objects.collections import PDFDictionary
from pdfmajor.parser.objects.indirect import ObjectRef
from pdfmajor.parser.objects.primitives import PDFInteger
from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal
from pdfmajor.parser.stream.PDFStream import PDFStream
from pdfmajor.util import validate_object_or_none
from .utils import iter_single_ref_as_array_ref
class PDFPageTreeNode(NamedTuple):
"""A PDF Pages Node representation as it confirms with PDF spec 1.7 section 7.7.3.2"""
@@ -24,15 +26,11 @@ class PDFPageTreeNode(NamedTuple):
parent = validate_object_or_none(pdfdict.get("Parent"), ObjectRef)
kids = pdfdict["Kids"]
count = validate_object_or_none(pdfdict.get("Count"), PDFInteger)
vetted_kids: List[ObjectRef] = []
if isinstance(kids, ObjectRef):
vetted_kids = [kids]
elif isinstance(kids, PDFArray):
for kid in kids:
vetted_kid = validate_object_or_none(kid, ObjectRef)
if vetted_kid:
vetted_kids.append(vetted_kid)
else:
try:
vetted_kids: List[ObjectRef] = list(
iter_single_ref_as_array_ref(pdfdict["Kids"])
)
except PDFMajorException:
raise InvalidPagesNodeKids(f"{kids}")
return cls(
kids=vetted_kids,
@@ -48,10 +46,14 @@ class PDFPage(NamedTuple):
parent: ObjectRef
resources: PDFDictionary
mediabox: PDFRectangle
cropbox: PDFRectangle
bleedbox: PDFRectangle
trimbox: PDFRectangle
artbox: PDFRectangle
contents: List[PDFStream]
boxcolor_info: Optional[PDFDictionary]
rotate: Decimal
metadata: Optional[ObjectRef]
user_unit: Decimal
rotate: PDFInteger
metadata: Optional[PDFStream]
user_unit: PDFReal
raw: PDFDictionary
+238
View File
@@ -0,0 +1,238 @@
from decimal import Decimal
from typing import Any, Dict, List, Optional
from pdfmajor.document.exceptions import BrokenFilePDF
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef
from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal
from pdfmajor.parser.stream.PDFStream import PDFStream
from pdfmajor.util import validate_object_or_none
from ..pages import PDFPage, PDFPageTreeNode
from ..PDFParsingContext import PDFParsingContext
from ..utils import iter_single_ref_as_array_ref
PDFRawFields = Dict[str, Any]
def get_all_page_leafs(
pctx: PDFParsingContext, root_node: PDFPageTreeNode
) -> List[PDFPage]:
all_leafs: List[PDFPage] = []
current_root: List[PDFPageTreeNode] = [root_node]
current_root_aspage: List[Dict[str, Optional[PDFObject]]] = [
parse_pdfpage_fields(pctx, root_node.raw, allow_missing=True)
]
current_child_idx: List[int] = [0]
while len(current_root) > 0:
croot = current_root[-1]
croot_pagedata = current_root_aspage[-1]
ckids = croot.kids
cidx = current_child_idx[-1]
if len(ckids) < cidx + 1:
try:
current_root.pop()
current_root_aspage.pop()
current_child_idx.pop()
continue
except IndexError:
break
page_ref = ckids[cidx]
obj = pctx.get_object_from_ref(page_ref).get_object()
if isinstance(obj, PDFDictionary):
if obj["Type"].to_python() == "/Pages":
current_root.append(PDFPageTreeNode.from_pdfdict(obj))
current_root_aspage.append(
{
**croot_pagedata,
**parse_pdfpage_fields(pctx, obj, allow_missing=True),
}
)
current_child_idx[-1] += 1
current_child_idx.append(0)
continue
elif obj["Type"].to_python() == "/Page":
all_leafs.append(make_pdfpage(croot_pagedata, pctx, obj))
current_child_idx[-1] += 1
continue
# catch all other cases (the above expression will issue a 'continue')
pctx.health_report.write(
"InvalidPageobject",
f"Expecting page ref {page_ref} to be a dictionary with type 'Pages' or 'Page', but got a {obj}",
)
current_child_idx[-1] += 1
return all_leafs
def make_pdfpage(
lastpage: Optional[Dict[str, Optional[PDFObject]]],
pctx: PDFParsingContext,
pdfdict: PDFDictionary,
) -> PDFPage:
"""Creates a PDFPage object, and handle the inheritance of properties
Args:
lastpage (Optional[Dict[str, Optional[PDFObject]]]): the last inheritable properties for the pdf
pctx (PDFParsingContext)
pdfdict (PDFDictionary)
Returns:
PDFPage
"""
vetted_fields: Any = parse_pdfpage_fields(
pctx, pdfdict, lastpage=lastpage, allow_missing=False
)
parent = validate_object_or_none(pdfdict.get("Parent"), ObjectRef)
content_streams: List[PDFStream] = []
if pdfdict.get("Contents", None) is not None:
content_streams = list(_parse_page_contents(pdfdict["Contents"], pctx))
else:
pctx.health_report.write(
"MissingPDFPageContents",
f"Page with data {pdfdict} is missing the 'Contents' property."
"Will assume that the page has no content.",
)
vetted_fields["parent"] = parent
vetted_fields["contents"] = content_streams
vetted_fields["raw"] = pdfdict
vetted_fields["metadata"] = _parse_metadata(pdfdict.get("Metadata", None), pctx)
return PDFPage(**vetted_fields)
def parse_pdfpage_fields(
pctx: PDFParsingContext,
pdfdict: PDFDictionary,
allow_missing: bool = False,
lastpage: Optional[PDFRawFields] = None,
) -> PDFRawFields:
parsed_values = pctx.convert_pdfdict_to_validated_pythondict(
pdfdict,
{
"Resources": ("resources", PDFDictionary),
"BoxColorInfo": ("boxcolor_info", PDFDictionary),
"Rotate": ("rotate", PDFInteger),
},
)
mediaboxes = pctx.extract_and_validate_pdf_rectangles(
pdfdict,
{
"MediaBox": "mediabox",
"CropBox": "cropbox",
"BleedBox": "bleedbox",
"TrimBox": "trimbox",
"ArtBox": "artbox",
},
)
return _deal_with_inheritance(
{
**parsed_values, # type: ignore
**mediaboxes, # type: ignore
"user_unit": pctx.get_validate_numeric(pdfdict.get("UserUnit")), # type: ignore
},
lastpage,
allow_missing,
)
def _parse_metadata(
metadata_raw: Optional[PDFObject], pctx: PDFParsingContext
) -> Optional[PDFStream]:
if metadata_raw is None:
return None
if isinstance(metadata_raw, ObjectRef):
metadata_obj = pctx.get_object_from_ref(metadata_raw)
elif isinstance(metadata_raw, IndirectObject):
metadata_obj = metadata_raw
else:
raise BrokenFilePDF(f"Metadata object is not a valid type {metadata_raw}")
if metadata_obj.stream is None:
raise BrokenFilePDF(f"Metadata object is missing it's stream {metadata_obj}")
else:
return metadata_obj.stream
def _parse_page_contents(contents_raw: PDFObject, pctx: PDFParsingContext):
for content_ref in iter_single_ref_as_array_ref(contents_raw):
content_obj = pctx.get_object_from_ref(content_ref)
if isinstance(content_obj, PDFArray):
for subref in iter_single_ref_as_array_ref(content_obj):
sub_obj = pctx.get_object_from_ref(subref)
if isinstance(sub_obj, IndirectObject):
if sub_obj.stream is not None:
yield sub_obj.stream
continue
raise BrokenFilePDF(f"Invalid PDFStream found for page {sub_obj}")
continue
elif isinstance(content_obj, IndirectObject):
if content_obj.stream is not None:
yield content_obj.stream
continue
raise BrokenFilePDF(f"Invalid PDFStream found for page {content_obj}")
def _deal_with_inheritance(
field_dict: Dict[str, Optional[PDFObject]],
lastpage: Optional[PDFRawFields],
allow_missing: bool = False,
):
_deal_with_inheritance_for_field("resources", field_dict, lastpage, PDFDictionary())
_deal_with_inheritance_for_field(
"rotate", field_dict, lastpage, PDFInteger(0, 0, 0)
)
_deal_with_inheritance_for_field(
"user_unit", field_dict, lastpage, PDFReal(Decimal("1.0"), 0, 0)
)
_deal_with_inheritance_for_field(
"mediabox", field_dict, lastpage, allow_missing=allow_missing
)
_deal_with_inheritance_for_field(
"cropbox",
field_dict,
lastpage,
default_value=field_dict["mediabox"],
allow_missing=allow_missing,
)
_deal_with_inheritance_for_field(
"bleedbox",
field_dict,
lastpage,
default_value=field_dict["cropbox"],
allow_missing=allow_missing,
)
_deal_with_inheritance_for_field(
"trimbox",
field_dict,
lastpage,
default_value=field_dict["cropbox"],
allow_missing=allow_missing,
)
_deal_with_inheritance_for_field(
"artbox",
field_dict,
lastpage,
default_value=field_dict["cropbox"],
allow_missing=allow_missing,
)
return field_dict
def _deal_with_inheritance_for_field(
field_name: str,
field_dict: Dict[str, Optional[PDFObject]],
lastpage: Optional[PDFRawFields],
default_value: Optional[PDFObject] = None,
allow_missing: bool = False,
):
if not field_dict[field_name]:
if not lastpage:
if default_value is not None:
field_dict[field_name] = default_value
elif not allow_missing:
raise BrokenFilePDF(f"Page is missing {field_name}")
elif lastpage.get(field_name, None):
field_dict[field_name] = lastpage[field_name]
elif default_value is not None:
field_dict[field_name] = default_value
elif not allow_missing:
raise BrokenFilePDF(f"Page is missing {field_name}")
+31
View File
@@ -0,0 +1,31 @@
from typing import Iterator
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.collections import PDFArray
from pdfmajor.parser.objects.indirect import ObjectRef
from pdfmajor.util import validate_object_or_none
from .exceptions import BrokenFilePDF
def iter_single_ref_as_array_ref(kids: PDFObject) -> Iterator[ObjectRef]:
"""Validate that an object is either a list of refs or a single ref, then iterate over it
Args:
kids (PDFObject)
Raises:
BrokenFile: if the provided object is not a valid PDFObject
Yields:
Iterator[ObjectRef]
"""
if isinstance(kids, ObjectRef):
yield kids
elif isinstance(kids, PDFArray):
for kid in kids:
vetted_kid = validate_object_or_none(kid, ObjectRef)
if vetted_kid:
yield vetted_kid
else:
raise BrokenFilePDF(f"expected a list of refs and got {kids}")
+3
View File
@@ -8,6 +8,9 @@ from .primitives import PDFName
class PDFArray(List[PDFObject], PDFContextualObject):
"""A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6"""
def __repr__(self) -> str:
return f"PDFDictionary({self.to_python()})"
def pass_item(self, item: PDFObject):
self.append(item)
+23
View File
@@ -1,6 +1,7 @@
from typing import Any, List, Optional, Type, TypeVar, Union
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal
T = TypeVar("T")
@@ -50,3 +51,25 @@ def validate_object_or_none(
return obj
else:
raise AssertionError(f"obj {obj} is not of expected type {objtype}")
def validate_number_or_none(
obj: Optional[PDFObject],
) -> Optional[Union[PDFReal, PDFInteger]]:
"""Validates the object's type is some numeric type
Args:
obj (Optional[PDFObject])
Raises:
AssertionError: is raised if the object has an invalid type
Returns:
Optional[Union[PDFReal, PDFInteger]]
"""
if obj is None:
return None
if isinstance(obj, PDFInteger) or isinstance(obj, PDFReal):
return obj
else:
raise AssertionError(f"obj {obj} is not of numeric type")
+9 -2
View File
@@ -2,9 +2,11 @@ from decimal import Decimal
from typing import List
from unittest import TestCase
from pdfmajor.document.parsers.root import get_catalog, get_info
from pdfmajor.document.exceptions import BrokenFilePDF, TooManyRectField
from pdfmajor.document.pages import PDFPageTreeNode
from pdfmajor.document.parsers.pages import get_all_page_leafs
from pdfmajor.document.parsers.root import get_catalog, get_info
from pdfmajor.document.PDFDocument import PDFDocument
from pdfmajor.document.PDFDocumentCatalog import PDFDocumentCatalog
from pdfmajor.document.PDFParsingContext import PDFParsingContext
from pdfmajor.document.structures import PDFRectangle
@@ -128,9 +130,14 @@ class ParsingState(TestCase):
for file_path, buffer in all_pdf_files.items():
with self.subTest(file_path), buffer.get_window():
parser = PDFParsingContext(buffer)
get_catalog(parser)
catalog = get_catalog(parser)
self.assertEqual(len(parser.health_report), 0)
get_info(parser)
get_all_page_leafs(parser, catalog.pages)
def test_doc(self):
with all_pdf_files["bad-unicode.pdf"].get_window() as buffer:
parser = PDFDocument(buffer)
class Structures(TestCase):