added metadata parser + tests
This commit is contained in:
1 parent
14cb2e31d5
commit
8813532b83
9 files changed
+104
-52
No files matched your search
@@ -1,5 +1,6 @@
|
||||
from typing import Any, Dict, NamedTuple, Optional
|
||||
|
||||
from pdfmajor.document.metadata import PDFMetadata
|
||||
from pdfmajor.document.pages import PDFPageTreeNode
|
||||
from pdfmajor.parser.objects import PDFDictionary, PDFName
|
||||
|
||||
@@ -12,6 +13,6 @@ class PDFDocumentCatalog(NamedTuple):
|
||||
page_labels: Optional[PDFDictionary]
|
||||
page_layout: Optional[PDFName]
|
||||
page_mode: Optional[PDFName]
|
||||
metadata: Optional[PDFDictionary]
|
||||
metadata: Optional[PDFMetadata]
|
||||
|
||||
raw: Dict[str, Any] # for other properties
|
||||
@@ -2,4 +2,7 @@ from typing import NamedTuple
|
||||
|
||||
|
||||
class PDFMetadata(NamedTuple):
|
||||
pass
|
||||
"""A structure representing the data in a PDF Metadata object as is specified in PDF spec 1.7 section 14.3.2"""
|
||||
|
||||
sub_type: str
|
||||
stream_data: bytes
|
||||
@@ -13,7 +13,6 @@ from pdfmajor.parser.objects import (
|
||||
PDFInteger,
|
||||
PDFObject,
|
||||
PDFReal,
|
||||
PDFStream,
|
||||
validate_number_or_none,
|
||||
validate_object_or_none,
|
||||
)
|
||||
@@ -179,7 +178,9 @@ class PDFParsingContext:
|
||||
with self.buffer.get_window() as buffer:
|
||||
return self.xrefdb.get_obj(obj_num, gen_num, buffer)
|
||||
|
||||
def validated_and_iter_stream(self, obj_ref: PDFObject) -> PDFStream:
|
||||
def validated_and_iter_stream(
|
||||
self, obj_ref: PDFObject
|
||||
) -> Tuple[PDFDictionary, bytes]:
|
||||
if isinstance(obj_ref, ObjectRef):
|
||||
obj = self.get_object_from_ref(obj_ref)
|
||||
elif isinstance(obj_ref, IndirectObject):
|
||||
@@ -190,7 +191,12 @@ class PDFParsingContext:
|
||||
if obj.stream is None:
|
||||
raise BrokenFilePDF(f"Expected a pdf-stream, instead found {obj}")
|
||||
else:
|
||||
stream_buffer = BufferStream.from_bytes(
|
||||
decode_stream(obj.stream, self.buffer)
|
||||
)
|
||||
return stream_buffer
|
||||
obj_def = obj.get_object()
|
||||
if not isinstance(obj_def, PDFDictionary):
|
||||
self.health_report.write_error(
|
||||
BrokenFilePDF(
|
||||
f"Stream object is missing a defining dictionary {obj_def} {obj_def}"
|
||||
)
|
||||
)
|
||||
obj_def = PDFDictionary()
|
||||
return obj_def, decode_stream(obj.stream, self.buffer)
|
||||
@@ -0,0 +1,17 @@
|
||||
from pdfmajor.document.metadata import PDFMetadata
|
||||
from pdfmajor.parser.objects import PDFName, PDFObject
|
||||
|
||||
from ..exceptions import BrokenFilePDF
|
||||
from ..parse_context import PDFParsingContext
|
||||
|
||||
|
||||
def get_metadata(pctx: PDFParsingContext, ref: PDFObject):
|
||||
obj_def, data = pctx.validated_and_iter_stream(ref)
|
||||
if not isinstance(obj_def["Subtype"], PDFName):
|
||||
pctx.health_report.write_error(
|
||||
BrokenFilePDF(
|
||||
f"Metadata data object is missing a valid subtype {ref} {obj_def}"
|
||||
)
|
||||
)
|
||||
return PDFMetadata("", data)
|
||||
return PDFMetadata(obj_def["Subtype"].value, data)
|
||||
@@ -1,6 +1,7 @@
|
||||
from typing import Any, Optional, Set
|
||||
|
||||
from pdfmajor.document.pages import PDFPageTreeNode
|
||||
from pdfmajor.document.parsers.metadata import get_metadata
|
||||
from pdfmajor.parser.objects import (
|
||||
IndirectObject,
|
||||
ObjectRef,
|
||||
@@ -50,7 +51,6 @@ def get_catalog(pctx: PDFParsingContext) -> PDFDocumentCatalog:
|
||||
pages_obj = cat_obj["Pages"]
|
||||
except KeyError:
|
||||
raise InvalidCatalogObj(f"Missing Pages entry {cat_obj}")
|
||||
pctx.convert_pdfdict_to_validated_pythondict
|
||||
if isinstance(pages_obj, ObjectRef):
|
||||
pages = pctx.get_object_from_ref(pages_obj).get_object()
|
||||
if not isinstance(pages, PDFDictionary):
|
||||
@@ -59,7 +59,10 @@ def get_catalog(pctx: PDFParsingContext) -> PDFDocumentCatalog:
|
||||
pages = pages_obj
|
||||
else:
|
||||
raise InvalidCatalogObj(f"Invalid pages entry {pages_obj}")
|
||||
|
||||
if cat_obj.get("Metadata"):
|
||||
validated_fields["metadata"] = get_metadata(pctx, cat_obj["Metadata"])
|
||||
else:
|
||||
validated_fields["metadata"] = None
|
||||
validated_fields["pages"] = PDFPageTreeNode.from_pdfdict(pages)
|
||||
validated_fields["raw"] = cat_obj
|
||||
return PDFDocumentCatalog(**validated_fields)
|
||||
|
||||
+19
-16
@@ -1,29 +1,32 @@
|
||||
import traceback
|
||||
from typing import List, NamedTuple
|
||||
import warnings
|
||||
from typing import List
|
||||
|
||||
|
||||
class PDFHealthLog(NamedTuple):
|
||||
"""A log of for an issue that the pdf file contains"""
|
||||
|
||||
issue_name: str
|
||||
additional_info: str
|
||||
class PDFMajorWarning(Warning):
|
||||
def __init__(self, issue_name: str, additional_info: str) -> None:
|
||||
self.issue_name = issue_name
|
||||
self.additional_info = additional_info
|
||||
super().__init__(f"health issue detected: {issue_name}: {additional_info}")
|
||||
|
||||
|
||||
class PDFHealthReport(List[PDFHealthLog]):
|
||||
class PDFHealthReport(List[PDFMajorWarning]):
|
||||
"""a list of all issues that occured during the parsing of the document"""
|
||||
|
||||
def write(self, issue_name: str, additional_info: str):
|
||||
self.append(PDFHealthLog(issue_name, additional_info))
|
||||
log = PDFMajorWarning(issue_name, additional_info)
|
||||
warnings.warn(log)
|
||||
self.append(log)
|
||||
|
||||
def write_error(self, err: Exception):
|
||||
tb = traceback.TracebackException.from_exception(err)
|
||||
self.write(err.__class__.__name__, "\n".join(tb.format()))
|
||||
|
||||
def pretty_print(self):
|
||||
print("<---HEALTH REPORT--->")
|
||||
for issue_name, info in self:
|
||||
print(" > ", issue_name)
|
||||
print(" |-> ", info)
|
||||
if len(self) == 0:
|
||||
print(" > No Issues Found.")
|
||||
print("</---HEALTH REPORT--->")
|
||||
def __repr__(self) -> str:
|
||||
out = "<---HEALTH REPORT--->"
|
||||
for log in self:
|
||||
out += "\n"
|
||||
out += f" > {log.issue_name}"
|
||||
out += f" |-> {log.additional_info}"
|
||||
out += "</---HEALTH REPORT--->"
|
||||
return out
|
||||
@@ -128,16 +128,7 @@ def on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
|
||||
isinstance(next_token, TokenKeyword)
|
||||
and next_token.value == b"endstream"
|
||||
):
|
||||
if i > 0:
|
||||
stream.length.value = next_token.start_loc - stream.offset
|
||||
state.health_report.write(
|
||||
issue_name="InvalidPDFStreamLength",
|
||||
additional_info=f"""
|
||||
start_token={token}
|
||||
end_token={next_token}
|
||||
object_dictionary={obj}
|
||||
""",
|
||||
)
|
||||
stream.length.value = next_token.start_loc - stream.offset
|
||||
break
|
||||
if not (
|
||||
isinstance(next_token, TokenKeyword)
|
||||
|
||||
@@ -22,6 +22,7 @@ with open("README.md") as readme_file:
|
||||
"bandit==1.6.2",
|
||||
"isort==5.6.4",
|
||||
"autoflake==1.4",
|
||||
"defusedxml==0.6.0",
|
||||
]
|
||||
},
|
||||
"description": "PDF parser",
|
||||
|
||||
+43
-16
@@ -3,9 +3,12 @@ from decimal import Decimal
|
||||
from typing import List
|
||||
from unittest import TestCase
|
||||
|
||||
from defusedxml.ElementTree import fromstring as xml_parse # type: ignore
|
||||
|
||||
from pdfmajor.document import PDFDocument
|
||||
from pdfmajor.document.catalog import PDFDocumentCatalog
|
||||
from pdfmajor.document.exceptions import BrokenFilePDF, TooManyRectField
|
||||
from pdfmajor.document.metadata import PDFMetadata
|
||||
from pdfmajor.document.pages import PDFPageTreeNode
|
||||
from pdfmajor.document.parse_context import PDFParsingContext
|
||||
from pdfmajor.document.parsers.pages import iter_all_page_leafs
|
||||
@@ -32,8 +35,33 @@ class ParsingState(TestCase):
|
||||
self.assertEqual(len(pdf.health_report), 0)
|
||||
cat = get_catalog(pdf)
|
||||
self.assertEqual(
|
||||
cat,
|
||||
expcat,
|
||||
cat.version,
|
||||
expcat.version,
|
||||
)
|
||||
self.assertEqual(
|
||||
cat.pages,
|
||||
expcat.pages,
|
||||
)
|
||||
self.assertEqual(
|
||||
cat.page_labels,
|
||||
expcat.page_labels,
|
||||
)
|
||||
self.assertEqual(
|
||||
cat.page_layout,
|
||||
expcat.page_layout,
|
||||
)
|
||||
self.assertEqual(
|
||||
cat.page_mode,
|
||||
expcat.page_mode,
|
||||
)
|
||||
# TODO
|
||||
# self.assertEqual(
|
||||
# cat.metadata,
|
||||
# expcat.metadata,
|
||||
# )
|
||||
self.assertEqual(
|
||||
cat.raw,
|
||||
expcat.raw,
|
||||
)
|
||||
|
||||
def test_bad_unicode(self):
|
||||
@@ -61,12 +89,9 @@ class ParsingState(TestCase):
|
||||
page_labels={"Nums": [PDFInteger(0, 0, 0), ObjectRef(140, 0)]},
|
||||
page_layout=None,
|
||||
page_mode=None,
|
||||
metadata=PDFDictionary.from_dict(
|
||||
{
|
||||
"Subtype": PDFName("XML", 0, 0),
|
||||
"Length": PDFInteger(3529, 0, 0),
|
||||
"Type": PDFName("Metadata", 0, 0),
|
||||
}
|
||||
metadata=PDFMetadata(
|
||||
"XML",
|
||||
b"",
|
||||
),
|
||||
raw=PDFDictionary.from_dict(
|
||||
{
|
||||
@@ -105,12 +130,9 @@ class ParsingState(TestCase):
|
||||
page_labels=None,
|
||||
page_layout=None,
|
||||
page_mode=None,
|
||||
metadata=PDFDictionary.from_dict(
|
||||
{
|
||||
"Type": PDFName("Metadata", 0, 0),
|
||||
"Subtype": PDFName("XML", 0, 0),
|
||||
"Length": PDFInteger(3065, 0, 0),
|
||||
}
|
||||
metadata=PDFMetadata(
|
||||
"XML",
|
||||
b"",
|
||||
),
|
||||
raw=PDFDictionary.from_dict(
|
||||
{
|
||||
@@ -140,10 +162,15 @@ class ParsingState(TestCase):
|
||||
actual_pages = len(list(parser.iter_pages()))
|
||||
self.assertEqual(actual_pages, parser.num_pages)
|
||||
|
||||
def test_doc(self):
|
||||
def test_metadata(self):
|
||||
with all_pdf_files["bad-unicode.pdf"].get_window() as buffer:
|
||||
parser = PDFDocument(buffer)
|
||||
print(parser.info)
|
||||
self.assertIsNotNone(parser.catalog.metadata)
|
||||
xml_tree = xml_parse(parser.catalog.metadata.stream_data.decode("utf-8"))
|
||||
self.assertEqual(xml_tree[0][0][0].text, "Acrobat Distiller 7.0 (Windows)")
|
||||
self.assertEqual(xml_tree[0][1][0].text, "PScript5.dll Version 5.2.2")
|
||||
self.assertEqual(xml_tree[0][1][1].text, "2009-01-15T08:49:42-07:00")
|
||||
self.assertEqual(xml_tree[0][1][2].text, "2009-01-15T08:49:42-07:00")
|
||||
|
||||
def test_warning_pagecount(self):
|
||||
with all_corrupt_pdf_files["bad-page-count.pdf"].get_window() as buffer:
|
||||
|
||||
Reference in new issue
Block a user