safe file loading

This commit is contained in:
ari committed 2020-10-30 16:43:11 -04:00
1 parent e8b31ebee0
commit 44ad73a44c
16 files changed
+439 -77

No files matched your search

-1
View File
@@ -16,7 +16,6 @@ lint:
mypy pdfmajor
isort tests
black tests
bandit -r tests
mypy tests
test:
+28
View File
@@ -0,0 +1,28 @@
from pathlib import Path
from typing import BinaryIO
from pdfmajor.healthlog import PDFHealthReport
from pdfmajor.streambuffer import BufferStream
from pdfmajor.xref.xrefdb import XRefDB
class PDFDocument:
"""A class encapsulating all of the access behavior to the document
Returns:
[type]: [description]
"""
@classmethod
def from_io(cls, fp: BinaryIO, buffer_size=4096):
return cls(BufferStream(fp, buffer_size))
@classmethod
def from_path(cls, path: Path, buffer_size=4096):
return cls(BufferStream(path.open("rb"), buffer_size))
def __init__(self, buffer: BufferStream) -> None:
self.buffer = buffer
self.health_report = PDFHealthReport()
with self.buffer.get_window():
self.xref_db = XRefDB(buffer)
+157
View File
@@ -0,0 +1,157 @@
from pathlib import Path
from typing import Any, BinaryIO, Dict, Optional, Tuple, Type, TypeVar
from pdfmajor.document.catalog import PDFDocumentCatalog
from pdfmajor.document.exceptions import InvalidCatalogObj, MissingCatalogObj
from pdfmajor.healthlog import PDFHealthReport
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef
from pdfmajor.parser.objects.primitives import PDFName
from pdfmajor.streambuffer import BufferStream
from pdfmajor.util import validate_object_or_none
from pdfmajor.xref.exceptions import InvalidNumberOfRoots, NotRootElement
from pdfmajor.xref.trailer import get_root_obj
from pdfmajor.xref.xrefdb import XRefDB
class PDFParsingContext:
"""A utility class to encapsulate various interactions with the raw document stream"""
@classmethod
def from_io(cls, fp: BinaryIO, buffer_size=4096):
return cls(BufferStream(fp, buffer_size))
@classmethod
def from_path(cls, path: Path, buffer_size=4096):
return cls(BufferStream(path.open("rb"), buffer_size))
def __init__(self, buffer: BufferStream) -> None:
self.buffer = buffer
self.health_report = PDFHealthReport()
with buffer.get_window():
self.xrefdb = XRefDB(buffer)
def get_object_from_ref(self, objref: ObjectRef) -> IndirectObject:
with self.buffer.get_window():
return self.xrefdb.get_obj(
objref.obj_num, objref.gen_num, buffer=self.buffer
)
ExpectedType = TypeVar("ExpectedType", bound=PDFObject)
def get_validate_object(
self, obj: Optional[PDFObject], exptype: Type[ExpectedType]
) -> Optional[ExpectedType]:
"""Validates the object's type if its not None
Args:
obj (Optional[PDFObject])
objtype (Type[ExpectedType]): expected type of the object
Raises:
AssertionError: is raised if the object has an invalid type
Returns:
Optional[ExpectedType]
"""
if isinstance(obj, ObjectRef):
return validate_object_or_none(
self.get_object_from_ref(obj).get_object(), exptype
)
return validate_object_or_none(obj, exptype)
def convert_pdfdict_to_validated_pythondict(
self,
pdfdict: PDFDictionary,
field_mapping: Dict[str, Tuple[str, Type[PDFObject]]],
) -> Dict[str, Optional[PDFObject]]:
"""Takes a pdf-dictionary and converts it to a regular dictionary (but first validates the types of the objects)
Args:
pdfdict (PDFDictionary)
field_mapping (Dict[str, Tuple[Type[PDFObject]]])
Returns:
dict
"""
out: Dict[str, Optional[PDFObject]] = {}
for pdf_name, (new_name, exp_type) in field_mapping.items():
try:
out[new_name] = self.get_validate_object(
pdfdict.get(pdf_name), exp_type
)
except AssertionError as err:
self.health_report.write_error(err)
return out
def get_catalog(self) -> PDFDocumentCatalog:
"""Gets the catalog for a document
Raises:
InvalidCatalogObj: if something is not correctly set in the catalog
Returns:
PDFDocumentCatalog
"""
with self.buffer.get_window():
cat_obj = _find_indirect_obj_for_root(
self.xrefdb, self.buffer, self.health_report
).get_object()
if not isinstance(cat_obj, PDFDictionary):
raise InvalidCatalogObj("not a dictionary")
validated_fields: Any = self.convert_pdfdict_to_validated_pythondict(
cat_obj,
{
"Version": ("version", PDFName),
"PageLabels": ("page_labels", PDFDictionary),
"PageLayout": ("page_layout", PDFName),
"PageMode": ("page_mode", PDFName),
"Metadata": ("metadata", PDFDictionary),
},
)
try:
pages_obj = cat_obj["Pages"]
except KeyError:
raise InvalidCatalogObj(f"Missing Pages entry {cat_obj}")
pages = []
if isinstance(pages_obj, ObjectRef):
pages = [pages_obj]
elif not isinstance(pages_obj, PDFArray):
raise InvalidCatalogObj(f"Invalid pages entry {pages_obj}")
else:
for page in pages_obj:
if isinstance(page, ObjectRef):
pages.append(page)
else:
self.health_report.write(
"InvalidPageRef", f"The page {page} has an invalid value"
)
validated_fields["pages"] = pages
validated_fields["raw"] = cat_obj
return PDFDocumentCatalog(**validated_fields)
def _find_indirect_obj_for_root(
xrefdb: XRefDB, buffer: BufferStream, health_report: PDFHealthReport
) -> IndirectObject:
try:
obj_ref = get_root_obj(xrefdb.trailers)
with buffer.get_window():
root_element = xrefdb.get_obj(
obj_num=obj_ref.obj_num, gen_num=obj_ref.gen_num, buffer=buffer
)
except InvalidNumberOfRoots as e:
health_report.write_error(e)
obj_ref = next(iter(e.roots))
with buffer.get_window():
root_element = xrefdb.get_obj(
obj_num=obj_ref.obj_num, gen_num=obj_ref.gen_num, buffer=buffer
)
except NotRootElement as e:
health_report.write_error(e)
try:
root_element = next(xrefdb.get_obj_by_type(buffer, "/Catalog"))
except StopIteration:
raise MissingCatalogObj()
return root_element
View File
Whitespace-only changes.
+18 -30
View File
@@ -1,38 +1,26 @@
from datetime import datetime
from typing import Any, Dict, NamedTuple, Optional, Tuple
from typing import Any, Dict, List, NamedTuple, Optional
from pdfmajor.lexer import iter_tokens
from pdfmajor.lexer.token import TokenComment
from pdfmajor.streambuffer import BufferStream
from pdfmajor.parser.objects.collections import PDFDictionary
from pdfmajor.parser.objects.indirect import ObjectRef
from pdfmajor.parser.objects.primitives import PDFName
class PDFDocumentCatalog(NamedTuple):
"""A Parsed version of the PDF Catalog object (see PDF spec 1.7 section 7.7.2 for more detail)
"""A Parsed version of the PDF Catalog object (see PDF spec 1.7 section 7.7.2 for more detail)"""
Args:
NamedTuple ([type]): [description]
"""
version: Optional[PDFName]
pages: List[ObjectRef]
page_labels: Optional[PDFDictionary]
page_layout: Optional[PDFName]
page_mode: Optional[PDFName]
metadata: Optional[PDFDictionary]
title: str
author: str
subject: str
keywords: str
creator: str
producer: str
creation_date: datetime
mod_date: datetime
trapped: str
mediabox: Tuple[int, int, int, int]
raw: Dict[str, Any]
@classmethod
def from_bufferstream(cls, buffer: BufferStream):
version = _get_version(buffer)
raw: Dict[str, Any] # for other properties
def _get_version(buffer: BufferStream) -> Optional[str]:
with buffer.get_window():
token = next(iter_tokens(buffer))
if isinstance(token, TokenComment):
return token.value.decode()
return None
# def _get_version(buffer: BufferStream) -> Optional[str]:
# with buffer.get_window():
# token = next(iter_tokens(buffer))
# if isinstance(token, TokenComment):
# return token.value.decode()
# return None
+13
View File
@@ -0,0 +1,13 @@
from pdfmajor.exceptions import BrokenFile, PDFMajorException
class PDFDocumentError(PDFMajorException):
pass
class MissingCatalogObj(PDFDocumentError, BrokenFile):
pass
class InvalidCatalogObj(PDFDocumentError, BrokenFile):
pass
+14 -1
View File
@@ -1,4 +1,4 @@
from dataclasses import dataclass
import traceback
from typing import List, NamedTuple
@@ -14,3 +14,16 @@ class PDFHealthReport(List[PDFHealthLog]):
def write(self, issue_name: str, additional_info: str):
self.append(PDFHealthLog(issue_name, additional_info))
def write_error(self, err: Exception):
tb = traceback.TracebackException.from_exception(err)
self.write(err.__class__.__name__, "\n".join(tb.format()))
def pretty_print(self):
print("<---HEALTH REPORT--->")
for issue_name, info in self:
print(" > ", issue_name)
print(" |-> ", info)
if len(self) == 0:
print(" > No Issues Found.")
print("</---HEALTH REPORT--->")
+3
View File
@@ -12,6 +12,9 @@ class ObjectRef(PDFObject):
self.obj_num = obj_num
self.gen_num = gen_num
def __hash__(self):
return hash(self.to_python())
def to_python(self):
return (self.obj_num, self.gen_num)
+1 -24
View File
@@ -1,6 +1,7 @@
from typing import Any, List, Optional, TypeVar, Union
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.util import get_single_or_list, to_python, to_python_list
from ..objects.collections import PDFDictionary
from ..objects.primitives import PDFNull
@@ -65,27 +66,3 @@ class PDFStream:
"fdecode_parms": to_python_list(self.fdecode_parms),
"dl": to_python(self.dl),
}
T = TypeVar("T")
def get_single_or_list(obj: Optional[Union[List[T], T]]) -> List[T]:
if obj is None:
return []
elif isinstance(obj, list):
return obj
else:
return [obj]
def to_python(obj: Optional[PDFObject]) -> Any:
if obj is None:
return None
return obj.to_python()
def to_python_list(obj: Optional[List[PDFObject]]) -> List[Any]:
if obj is None:
return []
return [x.to_python() for x in obj]
+52
View File
@@ -0,0 +1,52 @@
from typing import Any, Callable, Generic, List, Optional, Type, TypeVar, Union
from pdfmajor.parser.objects.base import PDFObject
T = TypeVar("T")
def get_single_or_list(obj: Optional[Union[List[T], T]]) -> List[T]:
if obj is None:
return []
elif isinstance(obj, list):
return obj
else:
return [obj]
def to_python(obj: Optional[PDFObject]) -> Any:
if obj is None:
return None
return obj.to_python()
def to_python_list(obj: Optional[List[PDFObject]]) -> List[Any]:
if obj is None:
return []
return [x.to_python() for x in obj]
ExpectedType = TypeVar("ExpectedType", bound=PDFObject)
def validate_object_or_none(
obj: Optional[PDFObject], objtype: Type[ExpectedType]
) -> Optional[ExpectedType]:
"""Validates the object's type if its not None
Args:
obj (Optional[PDFObject])
objtype (Type[ExpectedType]): expected type of the object
Raises:
AssertionError: is raised if the object has an invalid type
Returns:
Optional[ExpectedType]
"""
if obj is None:
return None
if isinstance(obj, objtype):
return obj
else:
raise AssertionError(f"obj {obj} is not of expected type {objtype}")
+14 -1
View File
@@ -1,4 +1,7 @@
from typing import Set
from pdfmajor.exceptions import BrokenFile, PDFMajorException
from pdfmajor.parser.objects.indirect import ObjectRef
class XRefError(PDFMajorException):
@@ -9,7 +12,7 @@ class PDFNoValidXRef(XRefError, BrokenFile, EOFError):
pass
class InvalidXref(XRefError, KeyError):
class InvalidXref(XRefError):
pass
@@ -19,3 +22,13 @@ class InvalidIndirectObjAccess(XRefError):
class UnexpectedEOF(BrokenFile, EOFError):
pass
class InvalidNumberOfRoots(XRefError):
def __init__(self, roots: Set[ObjectRef]) -> None:
super().__init__(f"Too many root elements found| total={roots}")
self.roots = roots
class NotRootElement(XRefError):
pass
+32 -3
View File
@@ -1,21 +1,47 @@
from typing import NamedTuple, Optional
from typing import List, NamedTuple, Optional, Set
from pdfmajor.lexer import iter_tokens
from pdfmajor.lexer.token import TokenKeyword
from pdfmajor.parser import get_first_object
from pdfmajor.parser.objects.base import PDFObject
from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary
from pdfmajor.parser.objects.indirect import ObjectRef
from pdfmajor.parser.objects.primitives import PDFInteger
from pdfmajor.streambuffer import BufferStream
from .exceptions import BrokenFile
from .exceptions import BrokenFile, InvalidNumberOfRoots, NotRootElement
def get_root_obj(trailers: List["PDFFileTrailer"]) -> ObjectRef:
"""attempts to find the root element of the pdf
Args:
trailers (List[)
Raises:
InvalidNumberOfRoots
NotRootElement
Returns:
ObjectRef
"""
root_elements: Set[ObjectRef] = set()
for trailer in trailers:
if trailer.root is not None:
root_elements.add(trailer.root)
if len(root_elements) == 1:
return root_elements.pop()
elif len(root_elements) > 1:
raise InvalidNumberOfRoots(root_elements)
else:
raise NotRootElement
class PDFFileTrailer(NamedTuple):
"""an object representing a PDF file trailer as is specified in PDF spec 1.7 section 7.5.5"""
size: PDFInteger
root: Optional[PDFObject]
root: Optional[ObjectRef]
prev: Optional[PDFObject]
info: Optional[PDFObject]
encrypt: Optional[PDFObject]
@@ -32,6 +58,9 @@ class PDFFileTrailer(NamedTuple):
raise BrokenFile(f"Size of trailer should be an integer instead got {size}")
root = pdfdict.get("Root", None)
if root is not None and not isinstance(root, ObjectRef):
raise BrokenFile(f"Root element is not specified as an indirect obj {root}")
encrypt = pdfdict.get("Encrypt", None)
encrypt_id = pdfdict.get("ID", None)
if encrypt is not None:
+80
View File
@@ -0,0 +1,80 @@
from pathlib import Path
from unittest import TestCase
from pdfmajor.document.catalog import PDFDocumentCatalog
from pdfmajor.document.PDFParsingContext import PDFParsingContext
from pdfmajor.parser.objects.indirect import ObjectRef
from pdfmajor.parser.objects.primitives import (
PDFBoolean,
PDFInteger,
PDFName,
PDFString,
)
from tests.util import all_pdf_files
class Catalog(TestCase):
def run_test(self, file_name: str, expcat: PDFDocumentCatalog):
with all_pdf_files[file_name].get_window() as buffer:
pdf = PDFParsingContext(buffer)
self.assertEqual(
pdf.get_catalog(),
expcat,
)
def test_bad_unicode(self):
self.run_test(
"bad-unicode.pdf",
PDFDocumentCatalog(
version=None,
pages=[ObjectRef(141, 0)],
page_labels={"Nums": [PDFInteger(0, 0, 0), ObjectRef(140, 0)]},
page_layout=None,
page_mode=None,
metadata={
"Subtype": PDFName("XML", 0, 0),
"Length": PDFInteger(3529, 0, 0),
"Type": PDFName("Metadata", 0, 0),
},
raw={
"Metadata": ObjectRef(147, 0),
"Pages": ObjectRef(141, 0),
"Type": PDFName("Catalog", 0, 0),
"PageLabels": ObjectRef(139, 0),
},
),
)
def test_bar_charts(self):
self.run_test(
"bar-charts.pdf",
PDFDocumentCatalog(
version=None,
pages=[ObjectRef(2, 0)],
page_labels=None,
page_layout=None,
page_mode=None,
metadata={
"Type": PDFName("Metadata", 0, 0),
"Subtype": PDFName("XML", 0, 0),
"Length": PDFInteger(3065, 0, 0),
},
raw={
"Type": PDFName("Catalog", 0, 0),
"Pages": ObjectRef(2, 0),
"Lang": PDFString(b"en-US", 0, 0),
"StructTreeRoot": ObjectRef(283, 0),
"MarkInfo": {"Marked": PDFBoolean(True, 0, 0)},
"Metadata": ObjectRef(646, 0),
"ViewerPreferences": ObjectRef(647, 0),
},
),
)
def test_all(self):
for file_path, buffer in all_pdf_files.items():
with self.subTest(file_path):
parser = PDFParsingContext(buffer)
parser.get_catalog()
self.assertEqual(len(parser.health_report), 0)
+1
View File
@@ -0,0 +1 @@
from unittest import TestCase
+16 -17
View File
@@ -6,18 +6,17 @@ from pdfmajor.streambuffer import BufferStream
from pdfmajor.xref.xref import XRefRow
from pdfmajor.xref.xrefdb import XRefDB
CURRENT_FOLDER = Path(__file__).parent
from tests.util import all_pdf_files
class Standard(TestCase):
def run_test(
self,
file_path: Path,
file_name: str,
expected_xrefs: Dict[Tuple[int, int], XRefRow],
expected_types: Dict[Tuple[int, int], str],
):
with file_path.open("rb") as fp:
buffer = BufferStream(fp)
with all_pdf_files[file_name].get_window() as buffer:
db = XRefDB(buffer)
self.assertDictEqual(
db.xrefs,
@@ -33,7 +32,7 @@ class Standard(TestCase):
def test_char_doc(self):
self.run_test(
file_path=(CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf"),
file_name="chars.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=27259, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -104,7 +103,7 @@ class Standard(TestCase):
def test_bar_charts(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "bar-charts.pdf",
file_name="bar-charts.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=17, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=169, obj_num=2, gen_num=0, use=b"n"),
@@ -691,7 +690,7 @@ class Standard(TestCase):
def test_colors(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "colors.pdf",
file_name="colors.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=7435, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -726,7 +725,7 @@ class Standard(TestCase):
def test_line_chart(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "line-charts.pdf",
file_name="line-charts.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=17, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=168, obj_num=2, gen_num=0, use=b"n"),
@@ -847,7 +846,7 @@ class Standard(TestCase):
def test_tables(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "tables.pdf",
file_name="tables.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=17, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=168, obj_num=2, gen_num=0, use=b"n"),
@@ -928,7 +927,7 @@ class Standard(TestCase):
def test_looseless(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "looseless.pdf",
file_name="looseless.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=1311539, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -979,7 +978,7 @@ class Standard(TestCase):
def test_shapes(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "shapes.pdf",
file_name="shapes.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=6001, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -1014,7 +1013,7 @@ class Standard(TestCase):
def test_fonts(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "fonts.pdf",
file_name="fonts.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=15601, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -1069,7 +1068,7 @@ class Standard(TestCase):
def test_jpg(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "jpg.pdf",
file_name="jpg.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=612894, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -1116,7 +1115,7 @@ class Standard(TestCase):
def test_bad_chars(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "bad-chars.pdf",
file_name="bad-chars.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=14026, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -1161,7 +1160,7 @@ class Standard(TestCase):
def test_font_color(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "font-color.pdf",
file_name="font-color.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=6019, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -1196,7 +1195,7 @@ class Standard(TestCase):
def test_chars(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "chars.pdf",
file_name="chars.pdf",
expected_xrefs={
(1, 0): XRefRow(offset=27259, obj_num=1, gen_num=0, use=b"n"),
(2, 0): XRefRow(offset=19, obj_num=2, gen_num=0, use=b"n"),
@@ -1267,7 +1266,7 @@ class Standard(TestCase):
def test_bad_unicode(self):
self.run_test(
file_path=CURRENT_FOLDER / "samples" / "pdf" / "bad-unicode.pdf",
file_name="bad-unicode.pdf",
expected_xrefs={
(149, 0): XRefRow(offset=16, obj_num=149, gen_num=0, use=b"n"),
(150, 0): XRefRow(offset=1286, obj_num=150, gen_num=0, use=b"n"),
+10
View File
@@ -0,0 +1,10 @@
from pathlib import Path
from pdfmajor.streambuffer import BufferStream
CURRENT_FOLDER = Path(__file__).parent
all_pdf_files = {
file_path.name: BufferStream(file_path.open("rb"))
for file_path in (CURRENT_FOLDER / "samples" / "pdf").iterdir()
}