parsing pdf page leafs
This commit is contained in:
1 parent
ffb7b843ea
commit
b48d635ec7
8 files changed
+371
-23
No files matched your search
@@ -1,7 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, List
|
||||
|
||||
from pdfmajor.document.pages import PDFPageTreeNode
|
||||
from pdfmajor.document.pages import PDFPage
|
||||
from pdfmajor.document.parsers.root import get_catalog, get_info
|
||||
from pdfmajor.document.PDFParsingContext import PDFParsingContext
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
@@ -26,7 +26,7 @@ class PDFDocument:
|
||||
self.__parser = PDFParsingContext(buffer)
|
||||
self.catalog = get_catalog(self.__parser)
|
||||
self.info = get_info(self.__parser)
|
||||
self.pages: List[PDFPageTreeNode] = []
|
||||
self.pages: List[PDFPage] = []
|
||||
|
||||
@property
|
||||
def health_report(self):
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, Dict, Iterator, Optional, Tuple, Type, TypeVar
|
||||
from typing import Any, BinaryIO, Dict, Iterator, Optional, Tuple, Type, TypeVar, Union
|
||||
|
||||
from pdfmajor.document.structures import PDFRectangle
|
||||
from pdfmajor.healthlog import PDFHealthReport
|
||||
from pdfmajor.parser.objects.base import PDFObject
|
||||
from pdfmajor.parser.objects.collections import PDFDictionary
|
||||
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
|
||||
from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef
|
||||
from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.util import validate_object_or_none
|
||||
from pdfmajor.util import validate_number_or_none, validate_object_or_none
|
||||
from pdfmajor.xref.xrefdb import GenNum, ObjNum, XRefDB
|
||||
|
||||
|
||||
@@ -64,6 +66,24 @@ class PDFParsingContext:
|
||||
)
|
||||
return validate_object_or_none(obj, exptype)
|
||||
|
||||
def get_validate_numeric(
|
||||
self, obj: Optional[PDFObject]
|
||||
) -> Optional[Union[PDFInteger, PDFReal]]:
|
||||
"""Validates the object is numeric, if its not None
|
||||
|
||||
Args:
|
||||
obj (Optional[PDFObject])
|
||||
|
||||
Raises:
|
||||
AssertionError: is raised if the object has an invalid type
|
||||
|
||||
Returns:
|
||||
Optional[Union[PDFInteger, PDFReal]]
|
||||
"""
|
||||
if isinstance(obj, ObjectRef):
|
||||
return validate_number_or_none(self.get_object_from_ref(obj).get_object())
|
||||
return validate_number_or_none(obj)
|
||||
|
||||
def convert_pdfdict_to_validated_pythondict(
|
||||
self,
|
||||
pdfdict: PDFDictionary,
|
||||
@@ -73,7 +93,7 @@ class PDFParsingContext:
|
||||
|
||||
Args:
|
||||
pdfdict (PDFDictionary)
|
||||
field_mapping (Dict[str, Tuple[Type[PDFObject]]])
|
||||
field_mapping (Dict[str, Tuple[Type[PDFObject]]]): mapping of the name as it in the pdf to a tuple with a (new name, expected objcet type)
|
||||
|
||||
Returns:
|
||||
dict
|
||||
@@ -88,6 +108,30 @@ class PDFParsingContext:
|
||||
self.health_report.write_error(err)
|
||||
return out
|
||||
|
||||
def extract_and_validate_pdf_rectangles(
|
||||
self, pdfdict: PDFDictionary, field_mapping: Dict[str, str]
|
||||
) -> Dict[str, Optional[PDFRectangle]]:
|
||||
"""Takes a pdf-dictionary and extracts the specified fields. Next it will validate and convert the fields into PDFRectangles
|
||||
|
||||
Args:
|
||||
pdfdict (PDFDictionary)
|
||||
field_mapping (Dict[str, str]): a mapping of names as they appear in the pdf to requested names
|
||||
|
||||
Returns:
|
||||
Dict[str, Optional[PDFRectangle]]
|
||||
"""
|
||||
vetted_values: Any = self.convert_pdfdict_to_validated_pythondict(
|
||||
pdfdict,
|
||||
{
|
||||
pdfname: (newname, PDFArray)
|
||||
for pdfname, newname in field_mapping.items()
|
||||
},
|
||||
)
|
||||
return {
|
||||
newname: PDFRectangle.from_pdfarray(value) if value is not None else value
|
||||
for newname, value in vetted_values.items()
|
||||
}
|
||||
|
||||
def get_obj_by_type(self, obj_type: str) -> Iterator[IndirectObject]:
|
||||
"""Iterates over the objects in memeory and filters out for the ones that match the requested type
|
||||
If the object is not loaded into memory this function will load it.
|
||||
|
||||
+17
-15
@@ -1,14 +1,16 @@
|
||||
from decimal import Decimal
|
||||
from typing import List, NamedTuple, Optional
|
||||
|
||||
from pdfmajor.document.exceptions import InvalidPagesNodeKids
|
||||
from pdfmajor.document.structures import PDFRectangle
|
||||
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
|
||||
from pdfmajor.exceptions import PDFMajorException
|
||||
from pdfmajor.parser.objects.collections import PDFDictionary
|
||||
from pdfmajor.parser.objects.indirect import ObjectRef
|
||||
from pdfmajor.parser.objects.primitives import PDFInteger
|
||||
from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal
|
||||
from pdfmajor.parser.stream.PDFStream import PDFStream
|
||||
from pdfmajor.util import validate_object_or_none
|
||||
|
||||
from .utils import iter_single_ref_as_array_ref
|
||||
|
||||
|
||||
class PDFPageTreeNode(NamedTuple):
|
||||
"""A PDF Pages Node representation as it confirms with PDF spec 1.7 section 7.7.3.2"""
|
||||
@@ -24,15 +26,11 @@ class PDFPageTreeNode(NamedTuple):
|
||||
parent = validate_object_or_none(pdfdict.get("Parent"), ObjectRef)
|
||||
kids = pdfdict["Kids"]
|
||||
count = validate_object_or_none(pdfdict.get("Count"), PDFInteger)
|
||||
vetted_kids: List[ObjectRef] = []
|
||||
if isinstance(kids, ObjectRef):
|
||||
vetted_kids = [kids]
|
||||
elif isinstance(kids, PDFArray):
|
||||
for kid in kids:
|
||||
vetted_kid = validate_object_or_none(kid, ObjectRef)
|
||||
if vetted_kid:
|
||||
vetted_kids.append(vetted_kid)
|
||||
else:
|
||||
try:
|
||||
vetted_kids: List[ObjectRef] = list(
|
||||
iter_single_ref_as_array_ref(pdfdict["Kids"])
|
||||
)
|
||||
except PDFMajorException:
|
||||
raise InvalidPagesNodeKids(f"{kids}")
|
||||
return cls(
|
||||
kids=vetted_kids,
|
||||
@@ -48,10 +46,14 @@ class PDFPage(NamedTuple):
|
||||
parent: ObjectRef
|
||||
resources: PDFDictionary
|
||||
mediabox: PDFRectangle
|
||||
cropbox: PDFRectangle
|
||||
bleedbox: PDFRectangle
|
||||
trimbox: PDFRectangle
|
||||
artbox: PDFRectangle
|
||||
contents: List[PDFStream]
|
||||
boxcolor_info: Optional[PDFDictionary]
|
||||
rotate: Decimal
|
||||
metadata: Optional[ObjectRef]
|
||||
user_unit: Decimal
|
||||
rotate: PDFInteger
|
||||
metadata: Optional[PDFStream]
|
||||
user_unit: PDFReal
|
||||
|
||||
raw: PDFDictionary
|
||||
@@ -0,0 +1,238 @@
|
||||
from decimal import Decimal
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pdfmajor.document.exceptions import BrokenFilePDF
|
||||
from pdfmajor.parser.objects.base import PDFObject
|
||||
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
|
||||
from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef
|
||||
from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal
|
||||
from pdfmajor.parser.stream.PDFStream import PDFStream
|
||||
from pdfmajor.util import validate_object_or_none
|
||||
|
||||
from ..pages import PDFPage, PDFPageTreeNode
|
||||
from ..PDFParsingContext import PDFParsingContext
|
||||
from ..utils import iter_single_ref_as_array_ref
|
||||
|
||||
PDFRawFields = Dict[str, Any]
|
||||
|
||||
|
||||
def get_all_page_leafs(
|
||||
pctx: PDFParsingContext, root_node: PDFPageTreeNode
|
||||
) -> List[PDFPage]:
|
||||
all_leafs: List[PDFPage] = []
|
||||
current_root: List[PDFPageTreeNode] = [root_node]
|
||||
current_root_aspage: List[Dict[str, Optional[PDFObject]]] = [
|
||||
parse_pdfpage_fields(pctx, root_node.raw, allow_missing=True)
|
||||
]
|
||||
current_child_idx: List[int] = [0]
|
||||
while len(current_root) > 0:
|
||||
croot = current_root[-1]
|
||||
croot_pagedata = current_root_aspage[-1]
|
||||
ckids = croot.kids
|
||||
cidx = current_child_idx[-1]
|
||||
if len(ckids) < cidx + 1:
|
||||
try:
|
||||
current_root.pop()
|
||||
current_root_aspage.pop()
|
||||
current_child_idx.pop()
|
||||
continue
|
||||
except IndexError:
|
||||
break
|
||||
page_ref = ckids[cidx]
|
||||
obj = pctx.get_object_from_ref(page_ref).get_object()
|
||||
if isinstance(obj, PDFDictionary):
|
||||
if obj["Type"].to_python() == "/Pages":
|
||||
current_root.append(PDFPageTreeNode.from_pdfdict(obj))
|
||||
current_root_aspage.append(
|
||||
{
|
||||
**croot_pagedata,
|
||||
**parse_pdfpage_fields(pctx, obj, allow_missing=True),
|
||||
}
|
||||
)
|
||||
current_child_idx[-1] += 1
|
||||
current_child_idx.append(0)
|
||||
continue
|
||||
elif obj["Type"].to_python() == "/Page":
|
||||
all_leafs.append(make_pdfpage(croot_pagedata, pctx, obj))
|
||||
current_child_idx[-1] += 1
|
||||
continue
|
||||
# catch all other cases (the above expression will issue a 'continue')
|
||||
pctx.health_report.write(
|
||||
"InvalidPageobject",
|
||||
f"Expecting page ref {page_ref} to be a dictionary with type 'Pages' or 'Page', but got a {obj}",
|
||||
)
|
||||
current_child_idx[-1] += 1
|
||||
return all_leafs
|
||||
|
||||
|
||||
def make_pdfpage(
|
||||
lastpage: Optional[Dict[str, Optional[PDFObject]]],
|
||||
pctx: PDFParsingContext,
|
||||
pdfdict: PDFDictionary,
|
||||
) -> PDFPage:
|
||||
"""Creates a PDFPage object, and handle the inheritance of properties
|
||||
|
||||
Args:
|
||||
lastpage (Optional[Dict[str, Optional[PDFObject]]]): the last inheritable properties for the pdf
|
||||
pctx (PDFParsingContext)
|
||||
pdfdict (PDFDictionary)
|
||||
|
||||
Returns:
|
||||
PDFPage
|
||||
"""
|
||||
vetted_fields: Any = parse_pdfpage_fields(
|
||||
pctx, pdfdict, lastpage=lastpage, allow_missing=False
|
||||
)
|
||||
parent = validate_object_or_none(pdfdict.get("Parent"), ObjectRef)
|
||||
content_streams: List[PDFStream] = []
|
||||
if pdfdict.get("Contents", None) is not None:
|
||||
content_streams = list(_parse_page_contents(pdfdict["Contents"], pctx))
|
||||
else:
|
||||
pctx.health_report.write(
|
||||
"MissingPDFPageContents",
|
||||
f"Page with data {pdfdict} is missing the 'Contents' property."
|
||||
"Will assume that the page has no content.",
|
||||
)
|
||||
vetted_fields["parent"] = parent
|
||||
vetted_fields["contents"] = content_streams
|
||||
vetted_fields["raw"] = pdfdict
|
||||
vetted_fields["metadata"] = _parse_metadata(pdfdict.get("Metadata", None), pctx)
|
||||
return PDFPage(**vetted_fields)
|
||||
|
||||
|
||||
def parse_pdfpage_fields(
|
||||
pctx: PDFParsingContext,
|
||||
pdfdict: PDFDictionary,
|
||||
allow_missing: bool = False,
|
||||
lastpage: Optional[PDFRawFields] = None,
|
||||
) -> PDFRawFields:
|
||||
parsed_values = pctx.convert_pdfdict_to_validated_pythondict(
|
||||
pdfdict,
|
||||
{
|
||||
"Resources": ("resources", PDFDictionary),
|
||||
"BoxColorInfo": ("boxcolor_info", PDFDictionary),
|
||||
"Rotate": ("rotate", PDFInteger),
|
||||
},
|
||||
)
|
||||
mediaboxes = pctx.extract_and_validate_pdf_rectangles(
|
||||
pdfdict,
|
||||
{
|
||||
"MediaBox": "mediabox",
|
||||
"CropBox": "cropbox",
|
||||
"BleedBox": "bleedbox",
|
||||
"TrimBox": "trimbox",
|
||||
"ArtBox": "artbox",
|
||||
},
|
||||
)
|
||||
return _deal_with_inheritance(
|
||||
{
|
||||
**parsed_values, # type: ignore
|
||||
**mediaboxes, # type: ignore
|
||||
"user_unit": pctx.get_validate_numeric(pdfdict.get("UserUnit")), # type: ignore
|
||||
},
|
||||
lastpage,
|
||||
allow_missing,
|
||||
)
|
||||
|
||||
|
||||
def _parse_metadata(
|
||||
metadata_raw: Optional[PDFObject], pctx: PDFParsingContext
|
||||
) -> Optional[PDFStream]:
|
||||
if metadata_raw is None:
|
||||
return None
|
||||
if isinstance(metadata_raw, ObjectRef):
|
||||
metadata_obj = pctx.get_object_from_ref(metadata_raw)
|
||||
elif isinstance(metadata_raw, IndirectObject):
|
||||
metadata_obj = metadata_raw
|
||||
else:
|
||||
raise BrokenFilePDF(f"Metadata object is not a valid type {metadata_raw}")
|
||||
if metadata_obj.stream is None:
|
||||
raise BrokenFilePDF(f"Metadata object is missing it's stream {metadata_obj}")
|
||||
else:
|
||||
return metadata_obj.stream
|
||||
|
||||
|
||||
def _parse_page_contents(contents_raw: PDFObject, pctx: PDFParsingContext):
|
||||
for content_ref in iter_single_ref_as_array_ref(contents_raw):
|
||||
content_obj = pctx.get_object_from_ref(content_ref)
|
||||
if isinstance(content_obj, PDFArray):
|
||||
for subref in iter_single_ref_as_array_ref(content_obj):
|
||||
sub_obj = pctx.get_object_from_ref(subref)
|
||||
if isinstance(sub_obj, IndirectObject):
|
||||
if sub_obj.stream is not None:
|
||||
yield sub_obj.stream
|
||||
continue
|
||||
raise BrokenFilePDF(f"Invalid PDFStream found for page {sub_obj}")
|
||||
continue
|
||||
elif isinstance(content_obj, IndirectObject):
|
||||
if content_obj.stream is not None:
|
||||
yield content_obj.stream
|
||||
continue
|
||||
raise BrokenFilePDF(f"Invalid PDFStream found for page {content_obj}")
|
||||
|
||||
|
||||
def _deal_with_inheritance(
|
||||
field_dict: Dict[str, Optional[PDFObject]],
|
||||
lastpage: Optional[PDFRawFields],
|
||||
allow_missing: bool = False,
|
||||
):
|
||||
_deal_with_inheritance_for_field("resources", field_dict, lastpage, PDFDictionary())
|
||||
_deal_with_inheritance_for_field(
|
||||
"rotate", field_dict, lastpage, PDFInteger(0, 0, 0)
|
||||
)
|
||||
_deal_with_inheritance_for_field(
|
||||
"user_unit", field_dict, lastpage, PDFReal(Decimal("1.0"), 0, 0)
|
||||
)
|
||||
_deal_with_inheritance_for_field(
|
||||
"mediabox", field_dict, lastpage, allow_missing=allow_missing
|
||||
)
|
||||
_deal_with_inheritance_for_field(
|
||||
"cropbox",
|
||||
field_dict,
|
||||
lastpage,
|
||||
default_value=field_dict["mediabox"],
|
||||
allow_missing=allow_missing,
|
||||
)
|
||||
_deal_with_inheritance_for_field(
|
||||
"bleedbox",
|
||||
field_dict,
|
||||
lastpage,
|
||||
default_value=field_dict["cropbox"],
|
||||
allow_missing=allow_missing,
|
||||
)
|
||||
_deal_with_inheritance_for_field(
|
||||
"trimbox",
|
||||
field_dict,
|
||||
lastpage,
|
||||
default_value=field_dict["cropbox"],
|
||||
allow_missing=allow_missing,
|
||||
)
|
||||
_deal_with_inheritance_for_field(
|
||||
"artbox",
|
||||
field_dict,
|
||||
lastpage,
|
||||
default_value=field_dict["cropbox"],
|
||||
allow_missing=allow_missing,
|
||||
)
|
||||
return field_dict
|
||||
|
||||
|
||||
def _deal_with_inheritance_for_field(
|
||||
field_name: str,
|
||||
field_dict: Dict[str, Optional[PDFObject]],
|
||||
lastpage: Optional[PDFRawFields],
|
||||
default_value: Optional[PDFObject] = None,
|
||||
allow_missing: bool = False,
|
||||
):
|
||||
if not field_dict[field_name]:
|
||||
if not lastpage:
|
||||
if default_value is not None:
|
||||
field_dict[field_name] = default_value
|
||||
elif not allow_missing:
|
||||
raise BrokenFilePDF(f"Page is missing {field_name}")
|
||||
elif lastpage.get(field_name, None):
|
||||
field_dict[field_name] = lastpage[field_name]
|
||||
elif default_value is not None:
|
||||
field_dict[field_name] = default_value
|
||||
elif not allow_missing:
|
||||
raise BrokenFilePDF(f"Page is missing {field_name}")
|
||||
@@ -0,0 +1,31 @@
|
||||
from typing import Iterator
|
||||
|
||||
from pdfmajor.parser.objects.base import PDFObject
|
||||
from pdfmajor.parser.objects.collections import PDFArray
|
||||
from pdfmajor.parser.objects.indirect import ObjectRef
|
||||
from pdfmajor.util import validate_object_or_none
|
||||
|
||||
from .exceptions import BrokenFilePDF
|
||||
|
||||
|
||||
def iter_single_ref_as_array_ref(kids: PDFObject) -> Iterator[ObjectRef]:
|
||||
"""Validate that an object is either a list of refs or a single ref, then iterate over it
|
||||
|
||||
Args:
|
||||
kids (PDFObject)
|
||||
|
||||
Raises:
|
||||
BrokenFile: if the provided object is not a valid PDFObject
|
||||
|
||||
Yields:
|
||||
Iterator[ObjectRef]
|
||||
"""
|
||||
if isinstance(kids, ObjectRef):
|
||||
yield kids
|
||||
elif isinstance(kids, PDFArray):
|
||||
for kid in kids:
|
||||
vetted_kid = validate_object_or_none(kid, ObjectRef)
|
||||
if vetted_kid:
|
||||
yield vetted_kid
|
||||
else:
|
||||
raise BrokenFilePDF(f"expected a list of refs and got {kids}")
|
||||
@@ -8,6 +8,9 @@ from .primitives import PDFName
|
||||
class PDFArray(List[PDFObject], PDFContextualObject):
|
||||
"""A class representing a PDF Array as specified in PDF spec 1.7 section 7.3.6"""
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"PDFDictionary({self.to_python()})"
|
||||
|
||||
def pass_item(self, item: PDFObject):
|
||||
self.append(item)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from typing import Any, List, Optional, Type, TypeVar, Union
|
||||
|
||||
from pdfmajor.parser.objects.base import PDFObject
|
||||
from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
@@ -50,3 +51,25 @@ def validate_object_or_none(
|
||||
return obj
|
||||
else:
|
||||
raise AssertionError(f"obj {obj} is not of expected type {objtype}")
|
||||
|
||||
|
||||
def validate_number_or_none(
|
||||
obj: Optional[PDFObject],
|
||||
) -> Optional[Union[PDFReal, PDFInteger]]:
|
||||
"""Validates the object's type is some numeric type
|
||||
|
||||
Args:
|
||||
obj (Optional[PDFObject])
|
||||
|
||||
Raises:
|
||||
AssertionError: is raised if the object has an invalid type
|
||||
|
||||
Returns:
|
||||
Optional[Union[PDFReal, PDFInteger]]
|
||||
"""
|
||||
if obj is None:
|
||||
return None
|
||||
if isinstance(obj, PDFInteger) or isinstance(obj, PDFReal):
|
||||
return obj
|
||||
else:
|
||||
raise AssertionError(f"obj {obj} is not of numeric type")
|
||||
@@ -2,9 +2,11 @@ from decimal import Decimal
|
||||
from typing import List
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.document.parsers.root import get_catalog, get_info
|
||||
from pdfmajor.document.exceptions import BrokenFilePDF, TooManyRectField
|
||||
from pdfmajor.document.pages import PDFPageTreeNode
|
||||
from pdfmajor.document.parsers.pages import get_all_page_leafs
|
||||
from pdfmajor.document.parsers.root import get_catalog, get_info
|
||||
from pdfmajor.document.PDFDocument import PDFDocument
|
||||
from pdfmajor.document.PDFDocumentCatalog import PDFDocumentCatalog
|
||||
from pdfmajor.document.PDFParsingContext import PDFParsingContext
|
||||
from pdfmajor.document.structures import PDFRectangle
|
||||
@@ -128,9 +130,14 @@ class ParsingState(TestCase):
|
||||
for file_path, buffer in all_pdf_files.items():
|
||||
with self.subTest(file_path), buffer.get_window():
|
||||
parser = PDFParsingContext(buffer)
|
||||
get_catalog(parser)
|
||||
catalog = get_catalog(parser)
|
||||
self.assertEqual(len(parser.health_report), 0)
|
||||
get_info(parser)
|
||||
get_all_page_leafs(parser, catalog.pages)
|
||||
|
||||
def test_doc(self):
|
||||
with all_pdf_files["bad-unicode.pdf"].get_window() as buffer:
|
||||
parser = PDFDocument(buffer)
|
||||
|
||||
|
||||
class Structures(TestCase):
|
||||
|
||||
Reference in new issue
Block a user