From b48d635ec737719197606e9d3cc82858543d51a9 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Mon, 2 Nov 2020 13:11:31 -0500 Subject: [PATCH] parsing pdf page leafs --- pdfmajor/document/PDFDocument.py | 4 +- pdfmajor/document/PDFParsingContext.py | 52 +++++- pdfmajor/document/pages.py | 32 ++-- pdfmajor/document/parsers/pages.py | 238 +++++++++++++++++++++++++ pdfmajor/document/utils.py | 31 ++++ pdfmajor/parser/objects/collections.py | 3 + pdfmajor/util.py | 23 +++ tests/test_document.py | 11 +- 8 files changed, 371 insertions(+), 23 deletions(-) create mode 100644 pdfmajor/document/parsers/pages.py create mode 100644 pdfmajor/document/utils.py diff --git a/pdfmajor/document/PDFDocument.py b/pdfmajor/document/PDFDocument.py index 95b5090..5424243 100644 --- a/pdfmajor/document/PDFDocument.py +++ b/pdfmajor/document/PDFDocument.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import BinaryIO, List -from pdfmajor.document.pages import PDFPageTreeNode +from pdfmajor.document.pages import PDFPage from pdfmajor.document.parsers.root import get_catalog, get_info from pdfmajor.document.PDFParsingContext import PDFParsingContext from pdfmajor.streambuffer import BufferStream @@ -26,7 +26,7 @@ class PDFDocument: self.__parser = PDFParsingContext(buffer) self.catalog = get_catalog(self.__parser) self.info = get_info(self.__parser) - self.pages: List[PDFPageTreeNode] = [] + self.pages: List[PDFPage] = [] @property def health_report(self): diff --git a/pdfmajor/document/PDFParsingContext.py b/pdfmajor/document/PDFParsingContext.py index 539d3fe..e7bb725 100644 --- a/pdfmajor/document/PDFParsingContext.py +++ b/pdfmajor/document/PDFParsingContext.py @@ -1,12 +1,14 @@ from pathlib import Path -from typing import BinaryIO, Dict, Iterator, Optional, Tuple, Type, TypeVar +from typing import Any, BinaryIO, Dict, Iterator, Optional, Tuple, Type, TypeVar, Union +from pdfmajor.document.structures import PDFRectangle from pdfmajor.healthlog import PDFHealthReport from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.collections import PDFDictionary +from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef +from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal from pdfmajor.streambuffer import BufferStream -from pdfmajor.util import validate_object_or_none +from pdfmajor.util import validate_number_or_none, validate_object_or_none from pdfmajor.xref.xrefdb import GenNum, ObjNum, XRefDB @@ -64,6 +66,24 @@ class PDFParsingContext: ) return validate_object_or_none(obj, exptype) + def get_validate_numeric( + self, obj: Optional[PDFObject] + ) -> Optional[Union[PDFInteger, PDFReal]]: + """Validates the object is numeric, if its not None + + Args: + obj (Optional[PDFObject]) + + Raises: + AssertionError: is raised if the object has an invalid type + + Returns: + Optional[Union[PDFInteger, PDFReal]] + """ + if isinstance(obj, ObjectRef): + return validate_number_or_none(self.get_object_from_ref(obj).get_object()) + return validate_number_or_none(obj) + def convert_pdfdict_to_validated_pythondict( self, pdfdict: PDFDictionary, @@ -73,7 +93,7 @@ class PDFParsingContext: Args: pdfdict (PDFDictionary) - field_mapping (Dict[str, Tuple[Type[PDFObject]]]) + field_mapping (Dict[str, Tuple[Type[PDFObject]]]): mapping of the name as it in the pdf to a tuple with a (new name, expected objcet type) Returns: dict @@ -88,6 +108,30 @@ class PDFParsingContext: self.health_report.write_error(err) return out + def extract_and_validate_pdf_rectangles( + self, pdfdict: PDFDictionary, field_mapping: Dict[str, str] + ) -> Dict[str, Optional[PDFRectangle]]: + """Takes a pdf-dictionary and extracts the specified fields. Next it will validate and convert the fields into PDFRectangles + + Args: + pdfdict (PDFDictionary) + field_mapping (Dict[str, str]): a mapping of names as they appear in the pdf to requested names + + Returns: + Dict[str, Optional[PDFRectangle]] + """ + vetted_values: Any = self.convert_pdfdict_to_validated_pythondict( + pdfdict, + { + pdfname: (newname, PDFArray) + for pdfname, newname in field_mapping.items() + }, + ) + return { + newname: PDFRectangle.from_pdfarray(value) if value is not None else value + for newname, value in vetted_values.items() + } + def get_obj_by_type(self, obj_type: str) -> Iterator[IndirectObject]: """Iterates over the objects in memeory and filters out for the ones that match the requested type If the object is not loaded into memory this function will load it. diff --git a/pdfmajor/document/pages.py b/pdfmajor/document/pages.py index 40e8b7d..ccd00fe 100644 --- a/pdfmajor/document/pages.py +++ b/pdfmajor/document/pages.py @@ -1,14 +1,16 @@ -from decimal import Decimal from typing import List, NamedTuple, Optional from pdfmajor.document.exceptions import InvalidPagesNodeKids from pdfmajor.document.structures import PDFRectangle -from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary +from pdfmajor.exceptions import PDFMajorException +from pdfmajor.parser.objects.collections import PDFDictionary from pdfmajor.parser.objects.indirect import ObjectRef -from pdfmajor.parser.objects.primitives import PDFInteger +from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal from pdfmajor.parser.stream.PDFStream import PDFStream from pdfmajor.util import validate_object_or_none +from .utils import iter_single_ref_as_array_ref + class PDFPageTreeNode(NamedTuple): """A PDF Pages Node representation as it confirms with PDF spec 1.7 section 7.7.3.2""" @@ -24,15 +26,11 @@ class PDFPageTreeNode(NamedTuple): parent = validate_object_or_none(pdfdict.get("Parent"), ObjectRef) kids = pdfdict["Kids"] count = validate_object_or_none(pdfdict.get("Count"), PDFInteger) - vetted_kids: List[ObjectRef] = [] - if isinstance(kids, ObjectRef): - vetted_kids = [kids] - elif isinstance(kids, PDFArray): - for kid in kids: - vetted_kid = validate_object_or_none(kid, ObjectRef) - if vetted_kid: - vetted_kids.append(vetted_kid) - else: + try: + vetted_kids: List[ObjectRef] = list( + iter_single_ref_as_array_ref(pdfdict["Kids"]) + ) + except PDFMajorException: raise InvalidPagesNodeKids(f"{kids}") return cls( kids=vetted_kids, @@ -48,10 +46,14 @@ class PDFPage(NamedTuple): parent: ObjectRef resources: PDFDictionary mediabox: PDFRectangle + cropbox: PDFRectangle bleedbox: PDFRectangle trimbox: PDFRectangle + artbox: PDFRectangle contents: List[PDFStream] boxcolor_info: Optional[PDFDictionary] - rotate: Decimal - metadata: Optional[ObjectRef] - user_unit: Decimal + rotate: PDFInteger + metadata: Optional[PDFStream] + user_unit: PDFReal + + raw: PDFDictionary diff --git a/pdfmajor/document/parsers/pages.py b/pdfmajor/document/parsers/pages.py new file mode 100644 index 0000000..98a845f --- /dev/null +++ b/pdfmajor/document/parsers/pages.py @@ -0,0 +1,238 @@ +from decimal import Decimal +from typing import Any, Dict, List, Optional + +from pdfmajor.document.exceptions import BrokenFilePDF +from pdfmajor.parser.objects.base import PDFObject +from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary +from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef +from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal +from pdfmajor.parser.stream.PDFStream import PDFStream +from pdfmajor.util import validate_object_or_none + +from ..pages import PDFPage, PDFPageTreeNode +from ..PDFParsingContext import PDFParsingContext +from ..utils import iter_single_ref_as_array_ref + +PDFRawFields = Dict[str, Any] + + +def get_all_page_leafs( + pctx: PDFParsingContext, root_node: PDFPageTreeNode +) -> List[PDFPage]: + all_leafs: List[PDFPage] = [] + current_root: List[PDFPageTreeNode] = [root_node] + current_root_aspage: List[Dict[str, Optional[PDFObject]]] = [ + parse_pdfpage_fields(pctx, root_node.raw, allow_missing=True) + ] + current_child_idx: List[int] = [0] + while len(current_root) > 0: + croot = current_root[-1] + croot_pagedata = current_root_aspage[-1] + ckids = croot.kids + cidx = current_child_idx[-1] + if len(ckids) < cidx + 1: + try: + current_root.pop() + current_root_aspage.pop() + current_child_idx.pop() + continue + except IndexError: + break + page_ref = ckids[cidx] + obj = pctx.get_object_from_ref(page_ref).get_object() + if isinstance(obj, PDFDictionary): + if obj["Type"].to_python() == "/Pages": + current_root.append(PDFPageTreeNode.from_pdfdict(obj)) + current_root_aspage.append( + { + **croot_pagedata, + **parse_pdfpage_fields(pctx, obj, allow_missing=True), + } + ) + current_child_idx[-1] += 1 + current_child_idx.append(0) + continue + elif obj["Type"].to_python() == "/Page": + all_leafs.append(make_pdfpage(croot_pagedata, pctx, obj)) + current_child_idx[-1] += 1 + continue + # catch all other cases (the above expression will issue a 'continue') + pctx.health_report.write( + "InvalidPageobject", + f"Expecting page ref {page_ref} to be a dictionary with type 'Pages' or 'Page', but got a {obj}", + ) + current_child_idx[-1] += 1 + return all_leafs + + +def make_pdfpage( + lastpage: Optional[Dict[str, Optional[PDFObject]]], + pctx: PDFParsingContext, + pdfdict: PDFDictionary, +) -> PDFPage: + """Creates a PDFPage object, and handle the inheritance of properties + + Args: + lastpage (Optional[Dict[str, Optional[PDFObject]]]): the last inheritable properties for the pdf + pctx (PDFParsingContext) + pdfdict (PDFDictionary) + + Returns: + PDFPage + """ + vetted_fields: Any = parse_pdfpage_fields( + pctx, pdfdict, lastpage=lastpage, allow_missing=False + ) + parent = validate_object_or_none(pdfdict.get("Parent"), ObjectRef) + content_streams: List[PDFStream] = [] + if pdfdict.get("Contents", None) is not None: + content_streams = list(_parse_page_contents(pdfdict["Contents"], pctx)) + else: + pctx.health_report.write( + "MissingPDFPageContents", + f"Page with data {pdfdict} is missing the 'Contents' property." + "Will assume that the page has no content.", + ) + vetted_fields["parent"] = parent + vetted_fields["contents"] = content_streams + vetted_fields["raw"] = pdfdict + vetted_fields["metadata"] = _parse_metadata(pdfdict.get("Metadata", None), pctx) + return PDFPage(**vetted_fields) + + +def parse_pdfpage_fields( + pctx: PDFParsingContext, + pdfdict: PDFDictionary, + allow_missing: bool = False, + lastpage: Optional[PDFRawFields] = None, +) -> PDFRawFields: + parsed_values = pctx.convert_pdfdict_to_validated_pythondict( + pdfdict, + { + "Resources": ("resources", PDFDictionary), + "BoxColorInfo": ("boxcolor_info", PDFDictionary), + "Rotate": ("rotate", PDFInteger), + }, + ) + mediaboxes = pctx.extract_and_validate_pdf_rectangles( + pdfdict, + { + "MediaBox": "mediabox", + "CropBox": "cropbox", + "BleedBox": "bleedbox", + "TrimBox": "trimbox", + "ArtBox": "artbox", + }, + ) + return _deal_with_inheritance( + { + **parsed_values, # type: ignore + **mediaboxes, # type: ignore + "user_unit": pctx.get_validate_numeric(pdfdict.get("UserUnit")), # type: ignore + }, + lastpage, + allow_missing, + ) + + +def _parse_metadata( + metadata_raw: Optional[PDFObject], pctx: PDFParsingContext +) -> Optional[PDFStream]: + if metadata_raw is None: + return None + if isinstance(metadata_raw, ObjectRef): + metadata_obj = pctx.get_object_from_ref(metadata_raw) + elif isinstance(metadata_raw, IndirectObject): + metadata_obj = metadata_raw + else: + raise BrokenFilePDF(f"Metadata object is not a valid type {metadata_raw}") + if metadata_obj.stream is None: + raise BrokenFilePDF(f"Metadata object is missing it's stream {metadata_obj}") + else: + return metadata_obj.stream + + +def _parse_page_contents(contents_raw: PDFObject, pctx: PDFParsingContext): + for content_ref in iter_single_ref_as_array_ref(contents_raw): + content_obj = pctx.get_object_from_ref(content_ref) + if isinstance(content_obj, PDFArray): + for subref in iter_single_ref_as_array_ref(content_obj): + sub_obj = pctx.get_object_from_ref(subref) + if isinstance(sub_obj, IndirectObject): + if sub_obj.stream is not None: + yield sub_obj.stream + continue + raise BrokenFilePDF(f"Invalid PDFStream found for page {sub_obj}") + continue + elif isinstance(content_obj, IndirectObject): + if content_obj.stream is not None: + yield content_obj.stream + continue + raise BrokenFilePDF(f"Invalid PDFStream found for page {content_obj}") + + +def _deal_with_inheritance( + field_dict: Dict[str, Optional[PDFObject]], + lastpage: Optional[PDFRawFields], + allow_missing: bool = False, +): + _deal_with_inheritance_for_field("resources", field_dict, lastpage, PDFDictionary()) + _deal_with_inheritance_for_field( + "rotate", field_dict, lastpage, PDFInteger(0, 0, 0) + ) + _deal_with_inheritance_for_field( + "user_unit", field_dict, lastpage, PDFReal(Decimal("1.0"), 0, 0) + ) + _deal_with_inheritance_for_field( + "mediabox", field_dict, lastpage, allow_missing=allow_missing + ) + _deal_with_inheritance_for_field( + "cropbox", + field_dict, + lastpage, + default_value=field_dict["mediabox"], + allow_missing=allow_missing, + ) + _deal_with_inheritance_for_field( + "bleedbox", + field_dict, + lastpage, + default_value=field_dict["cropbox"], + allow_missing=allow_missing, + ) + _deal_with_inheritance_for_field( + "trimbox", + field_dict, + lastpage, + default_value=field_dict["cropbox"], + allow_missing=allow_missing, + ) + _deal_with_inheritance_for_field( + "artbox", + field_dict, + lastpage, + default_value=field_dict["cropbox"], + allow_missing=allow_missing, + ) + return field_dict + + +def _deal_with_inheritance_for_field( + field_name: str, + field_dict: Dict[str, Optional[PDFObject]], + lastpage: Optional[PDFRawFields], + default_value: Optional[PDFObject] = None, + allow_missing: bool = False, +): + if not field_dict[field_name]: + if not lastpage: + if default_value is not None: + field_dict[field_name] = default_value + elif not allow_missing: + raise BrokenFilePDF(f"Page is missing {field_name}") + elif lastpage.get(field_name, None): + field_dict[field_name] = lastpage[field_name] + elif default_value is not None: + field_dict[field_name] = default_value + elif not allow_missing: + raise BrokenFilePDF(f"Page is missing {field_name}") diff --git a/pdfmajor/document/utils.py b/pdfmajor/document/utils.py new file mode 100644 index 0000000..62335a7 --- /dev/null +++ b/pdfmajor/document/utils.py @@ -0,0 +1,31 @@ +from typing import Iterator + +from pdfmajor.parser.objects.base import PDFObject +from pdfmajor.parser.objects.collections import PDFArray +from pdfmajor.parser.objects.indirect import ObjectRef +from pdfmajor.util import validate_object_or_none + +from .exceptions import BrokenFilePDF + + +def iter_single_ref_as_array_ref(kids: PDFObject) -> Iterator[ObjectRef]: + """Validate that an object is either a list of refs or a single ref, then iterate over it + + Args: + kids (PDFObject) + + Raises: + BrokenFile: if the provided object is not a valid PDFObject + + Yields: + Iterator[ObjectRef] + """ + if isinstance(kids, ObjectRef): + yield kids + elif isinstance(kids, PDFArray): + for kid in kids: + vetted_kid = validate_object_or_none(kid, ObjectRef) + if vetted_kid: + yield vetted_kid + else: + raise BrokenFilePDF(f"expected a list of refs and got {kids}") diff --git a/pdfmajor/parser/objects/collections.py b/pdfmajor/parser/objects/collections.py index efe203a..a37b1c7 100644 --- a/pdfmajor/parser/objects/collections.py +++ b/pdfmajor/parser/objects/collections.py @@ -8,6 +8,9 @@ from .primitives import PDFName class PDFArray(List[PDFObject], PDFContextualObject): """A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6""" + def __repr__(self) -> str: + return f"PDFDictionary({self.to_python()})" + def pass_item(self, item: PDFObject): self.append(item) diff --git a/pdfmajor/util.py b/pdfmajor/util.py index 3c67f24..2be01f7 100644 --- a/pdfmajor/util.py +++ b/pdfmajor/util.py @@ -1,6 +1,7 @@ from typing import Any, List, Optional, Type, TypeVar, Union from pdfmajor.parser.objects.base import PDFObject +from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal T = TypeVar("T") @@ -50,3 +51,25 @@ def validate_object_or_none( return obj else: raise AssertionError(f"obj {obj} is not of expected type {objtype}") + + +def validate_number_or_none( + obj: Optional[PDFObject], +) -> Optional[Union[PDFReal, PDFInteger]]: + """Validates the object's type is some numeric type + + Args: + obj (Optional[PDFObject]) + + Raises: + AssertionError: is raised if the object has an invalid type + + Returns: + Optional[Union[PDFReal, PDFInteger]] + """ + if obj is None: + return None + if isinstance(obj, PDFInteger) or isinstance(obj, PDFReal): + return obj + else: + raise AssertionError(f"obj {obj} is not of numeric type") diff --git a/tests/test_document.py b/tests/test_document.py index 82183fe..f399163 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -2,9 +2,11 @@ from decimal import Decimal from typing import List from unittest import TestCase -from pdfmajor.document.parsers.root import get_catalog, get_info from pdfmajor.document.exceptions import BrokenFilePDF, TooManyRectField from pdfmajor.document.pages import PDFPageTreeNode +from pdfmajor.document.parsers.pages import get_all_page_leafs +from pdfmajor.document.parsers.root import get_catalog, get_info +from pdfmajor.document.PDFDocument import PDFDocument from pdfmajor.document.PDFDocumentCatalog import PDFDocumentCatalog from pdfmajor.document.PDFParsingContext import PDFParsingContext from pdfmajor.document.structures import PDFRectangle @@ -128,9 +130,14 @@ class ParsingState(TestCase): for file_path, buffer in all_pdf_files.items(): with self.subTest(file_path), buffer.get_window(): parser = PDFParsingContext(buffer) - get_catalog(parser) + catalog = get_catalog(parser) self.assertEqual(len(parser.health_report), 0) get_info(parser) + get_all_page_leafs(parser, catalog.pages) + + def test_doc(self): + with all_pdf_files["bad-unicode.pdf"].get_window() as buffer: + parser = PDFDocument(buffer) class Structures(TestCase):