From 14cb2e31d5b380d71b4805f7174a3780d9c014a8 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Mon, 2 Nov 2020 19:57:26 -0500 Subject: [PATCH] using absolute objects import --- Makefile | 4 +- pdfmajor/document/__init__.py | 2 +- pdfmajor/document/catalog.py | 3 +- pdfmajor/document/exceptions.py | 2 +- pdfmajor/document/pages.py | 13 +- pdfmajor/document/parse_context.py | 26 ++-- pdfmajor/document/parsers/pages.py | 17 ++- pdfmajor/document/parsers/root.py | 13 +- pdfmajor/document/parsers/stream.py | 42 +++++++ pdfmajor/document/structures.py | 3 +- pdfmajor/document/utils.py | 10 +- pdfmajor/filters/__init__.py | 79 ++++++++++++ .../stream/fliters => filters}/ccit.py | 14 +-- pdfmajor/filters/decoders.py | 48 +++++++ pdfmajor/filters/exceptions.py | 25 ++++ .../{parser/stream/fliters => filters}/lzw.py | 2 +- .../{parser/stream/fliters => filters}/rld.py | 0 pdfmajor/filters/types.py | 10 ++ pdfmajor/parser/__init__.py | 11 +- pdfmajor/parser/exceptions.py | 6 +- pdfmajor/parser/objects/__init__.py | 17 +++ pdfmajor/parser/objects/indirect.py | 17 +-- pdfmajor/parser/objects/ref.py | 15 +++ .../pdf_stream.py => objects/stream.py} | 51 ++++---- pdfmajor/parser/objects/util.py | 66 ++++++++++ pdfmajor/parser/parsers.py | 16 ++- pdfmajor/parser/state.py | 11 +- pdfmajor/parser/stream/PDFStreamParser.py | 23 ---- pdfmajor/parser/stream/__init__.py | 1 - pdfmajor/parser/stream/fliters/__init__.py | 117 ------------------ pdfmajor/parser/stream/fliters/types.py | 16 --- pdfmajor/util.py | 65 +--------- pdfmajor/xref/exceptions.py | 2 +- pdfmajor/xref/trailer.py | 13 +- pdfmajor/xref/xrefdb.py | 4 +- tests/test_document.py | 63 +--------- tests/test_parser.py | 17 ++- tests/test_parser_filters.py | 6 +- 38 files changed, 448 insertions(+), 402 deletions(-) create mode 100644 pdfmajor/document/parsers/stream.py create mode 100644 pdfmajor/filters/__init__.py rename pdfmajor/{parser/stream/fliters => filters}/ccit.py (99%) create mode 100644 pdfmajor/filters/decoders.py create mode 100644 pdfmajor/filters/exceptions.py rename pdfmajor/{parser/stream/fliters => filters}/lzw.py (98%) rename pdfmajor/{parser/stream/fliters => filters}/rld.py (100%) create mode 100644 pdfmajor/filters/types.py create mode 100644 pdfmajor/parser/objects/ref.py rename pdfmajor/parser/{stream/pdf_stream.py => objects/stream.py} (67%) create mode 100644 pdfmajor/parser/objects/util.py delete mode 100644 pdfmajor/parser/stream/PDFStreamParser.py delete mode 100644 pdfmajor/parser/stream/__init__.py delete mode 100644 pdfmajor/parser/stream/fliters/__init__.py delete mode 100644 pdfmajor/parser/stream/fliters/types.py diff --git a/Makefile b/Makefile index 6e6a6f3..cbc576b 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ clean: rm -rf *.egg-info lint: - autoflake --remove-all-unused-imports -r -i pdfmajor - autoflake --remove-all-unused-imports -r -i tests + autoflake --remove-all-unused-imports --ignore-init-module-imports --remove-duplicate-keys --remove-unused-variables -r -i pdfmajor + autoflake --remove-all-unused-imports --ignore-init-module-imports --remove-duplicate-keys --remove-unused-variables -r -i tests isort pdfmajor black pdfmajor isort tests diff --git a/pdfmajor/document/__init__.py b/pdfmajor/document/__init__.py index b71944a..838e72b 100644 --- a/pdfmajor/document/__init__.py +++ b/pdfmajor/document/__init__.py @@ -1 +1 @@ -from .document import PDFDocument # noqa +from .document import PDFDocument diff --git a/pdfmajor/document/catalog.py b/pdfmajor/document/catalog.py index 493b341..2f9aa52 100644 --- a/pdfmajor/document/catalog.py +++ b/pdfmajor/document/catalog.py @@ -1,8 +1,7 @@ from typing import Any, Dict, NamedTuple, Optional from pdfmajor.document.pages import PDFPageTreeNode -from pdfmajor.parser.objects.collections import PDFDictionary -from pdfmajor.parser.objects.primitives import PDFName +from pdfmajor.parser.objects import PDFDictionary, PDFName class PDFDocumentCatalog(NamedTuple): diff --git a/pdfmajor/document/exceptions.py b/pdfmajor/document/exceptions.py index 7a82dd7..d6a086c 100644 --- a/pdfmajor/document/exceptions.py +++ b/pdfmajor/document/exceptions.py @@ -2,7 +2,7 @@ from decimal import Decimal from typing import List, Set from pdfmajor.exceptions import BrokenFile, PDFMajorException -from pdfmajor.parser.objects.indirect import ObjectRef +from pdfmajor.parser.objects import ObjectRef class PDFDocumentError(PDFMajorException): diff --git a/pdfmajor/document/pages.py b/pdfmajor/document/pages.py index fe363b4..cefa293 100644 --- a/pdfmajor/document/pages.py +++ b/pdfmajor/document/pages.py @@ -3,11 +3,14 @@ from typing import List, NamedTuple, Optional from pdfmajor.document.exceptions import InvalidPagesNodeKids from pdfmajor.document.structures import PDFRectangle from pdfmajor.exceptions import PDFMajorException -from pdfmajor.parser.objects.collections import PDFDictionary -from pdfmajor.parser.objects.indirect import ObjectRef -from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal -from pdfmajor.parser.stream import PDFStream -from pdfmajor.util import validate_object_or_none +from pdfmajor.parser.objects import ( + ObjectRef, + PDFDictionary, + PDFInteger, + PDFReal, + PDFStream, + validate_object_or_none, +) from .utils import iter_single_ref_as_array_ref diff --git a/pdfmajor/document/parse_context.py b/pdfmajor/document/parse_context.py index a8003be..1feed5c 100644 --- a/pdfmajor/document/parse_context.py +++ b/pdfmajor/document/parse_context.py @@ -1,16 +1,23 @@ from pathlib import Path -from pdfmajor.document.exceptions import BrokenFilePDF -from pdfmajor.parser.stream import PDFStream from typing import Any, BinaryIO, Dict, Iterator, Optional, Tuple, Type, TypeVar, Union +from pdfmajor.document.exceptions import BrokenFilePDF +from pdfmajor.document.parsers.stream import decode_stream from pdfmajor.document.structures import PDFRectangle from pdfmajor.healthlog import PDFHealthReport -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary -from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef -from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal +from pdfmajor.parser.objects import ( + IndirectObject, + ObjectRef, + PDFArray, + PDFDictionary, + PDFInteger, + PDFObject, + PDFReal, + PDFStream, + validate_number_or_none, + validate_object_or_none, +) from pdfmajor.streambuffer import BufferStream -from pdfmajor.util import validate_number_or_none, validate_object_or_none from pdfmajor.xref.xrefdb import GenNum, ObjNum, XRefDB @@ -183,4 +190,7 @@ class PDFParsingContext: if obj.stream is None: raise BrokenFilePDF(f"Expected a pdf-stream, instead found {obj}") else: - obj.stream + stream_buffer = BufferStream.from_bytes( + decode_stream(obj.stream, self.buffer) + ) + return stream_buffer diff --git a/pdfmajor/document/parsers/pages.py b/pdfmajor/document/parsers/pages.py index 666e9b1..7cd1118 100644 --- a/pdfmajor/document/parsers/pages.py +++ b/pdfmajor/document/parsers/pages.py @@ -2,12 +2,17 @@ from decimal import Decimal from typing import Any, Dict, Iterator, List, Optional from pdfmajor.document.exceptions import BrokenFilePDF -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary -from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef -from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal -from pdfmajor.parser.stream import PDFStream -from pdfmajor.util import validate_object_or_none +from pdfmajor.parser.objects import ( + IndirectObject, + ObjectRef, + PDFArray, + PDFDictionary, + PDFInteger, + PDFObject, + PDFReal, + PDFStream, + validate_object_or_none, +) from ..pages import PDFPage, PDFPageTreeNode from ..parse_context import PDFParsingContext diff --git a/pdfmajor/document/parsers/root.py b/pdfmajor/document/parsers/root.py index 8b8bdd9..ffc058b 100644 --- a/pdfmajor/document/parsers/root.py +++ b/pdfmajor/document/parsers/root.py @@ -1,10 +1,13 @@ from typing import Any, Optional, Set from pdfmajor.document.pages import PDFPageTreeNode -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.collections import PDFDictionary -from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef -from pdfmajor.parser.objects.primitives import PDFName +from pdfmajor.parser.objects import ( + IndirectObject, + ObjectRef, + PDFDictionary, + PDFName, + PDFObject, +) from pdfmajor.xref.exceptions import InvalidNumberOfRoots, NotRootElement from pdfmajor.xref.trailer import get_root_obj @@ -47,7 +50,7 @@ def get_catalog(pctx: PDFParsingContext) -> PDFDocumentCatalog: pages_obj = cat_obj["Pages"] except KeyError: raise InvalidCatalogObj(f"Missing Pages entry {cat_obj}") - metadata = pctx.convert_pdfdict_to_validated_pythondict + pctx.convert_pdfdict_to_validated_pythondict if isinstance(pages_obj, ObjectRef): pages = pctx.get_object_from_ref(pages_obj).get_object() if not isinstance(pages, PDFDictionary): diff --git a/pdfmajor/document/parsers/stream.py b/pdfmajor/document/parsers/stream.py new file mode 100644 index 0000000..b5cdaed --- /dev/null +++ b/pdfmajor/document/parsers/stream.py @@ -0,0 +1,42 @@ +from itertools import chain +from typing import Any, Dict, List + +from pdfmajor.filters import process_filters_on_data +from pdfmajor.parser.exceptions import IncompleteStream +from pdfmajor.parser.objects import PDFDictionary, PDFName, PDFStream +from pdfmajor.streambuffer import BufferStream + + +def decode_stream(stream: PDFStream, buffer: BufferStream) -> bytes: + """Decode the bytes associated with the PDF-Stream + + Args: + stream (PDFStream) + buffer (BufferStream) + + Returns: + bytes + """ + length = stream.length.to_python() + if length is None: + raise IncompleteStream(f"Cannot decode stream as it has no specified length") + with buffer.get_window(): + buffer.seek(stream.offset) + data = buffer.read(length).data + filters: List[str] = [] + decode_parms: List[Dict[str, Any]] = [] + for f, dp in chain( + zip(stream.filter, stream.decode_parms), + zip(stream.ffilter, stream.fdecode_parms), + ): + if not isinstance(f, PDFName): + raise IncompleteStream(f"Invalid filter name {f}") + else: + filter_name = f.value + params = {} + if isinstance(dp, PDFDictionary): + params = dp.to_python() + filters.append(filter_name) + decode_parms.append(params) + data = process_filters_on_data(data, filters, decode_parms) + return data diff --git a/pdfmajor/document/structures.py b/pdfmajor/document/structures.py index a12ae6e..2f56f7c 100644 --- a/pdfmajor/document/structures.py +++ b/pdfmajor/document/structures.py @@ -2,8 +2,7 @@ from decimal import Decimal from typing import List, NamedTuple from pdfmajor.document.exceptions import BrokenFilePDF, TooManyRectField -from pdfmajor.parser.objects.collections import PDFArray -from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal +from pdfmajor.parser.objects import PDFArray, PDFInteger, PDFReal class PDFRectangle(NamedTuple): diff --git a/pdfmajor/document/utils.py b/pdfmajor/document/utils.py index 62335a7..636c3cb 100644 --- a/pdfmajor/document/utils.py +++ b/pdfmajor/document/utils.py @@ -1,9 +1,11 @@ from typing import Iterator -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.collections import PDFArray -from pdfmajor.parser.objects.indirect import ObjectRef -from pdfmajor.util import validate_object_or_none +from pdfmajor.parser.objects import ( + ObjectRef, + PDFArray, + PDFObject, + validate_object_or_none, +) from .exceptions import BrokenFilePDF diff --git a/pdfmajor/filters/__init__.py b/pdfmajor/filters/__init__.py new file mode 100644 index 0000000..fd83271 --- /dev/null +++ b/pdfmajor/filters/__init__.py @@ -0,0 +1,79 @@ +from typing import Any, Dict, List, Optional, Type + +from .decoders import ( + ASCII85Decode, + ASCIIHexDecode, + CCITTFaxDecode, + FlateDecode, + LZWDecode, + RunLengthDecode, +) +from .types import PDFFilterDecoder + +_filter_type_name_mapping: Dict[str, Optional[Type[PDFFilterDecoder]]] = { + "FlateDecode": FlateDecode, + "Fl": FlateDecode, + "LZWDecode": LZWDecode, + "LZW": LZWDecode, + "ASCII85Decode": ASCII85Decode, + "A85": ASCII85Decode, + "ASCIIHexDecode": ASCIIHexDecode, + "AHx": ASCIIHexDecode, + "RunLengthDecode": RunLengthDecode, + "RL": RunLengthDecode, + "CCITTFaxDecode": CCITTFaxDecode, + "CCF": CCITTFaxDecode, + "DCTDecode": None, # FilterType.DCTDecode, + "DCT": None, # FilterType.DCTDecode, + "Crypt": None, # FilterType.Crypt, + "JBIG2Decode": None, # FilterType.JBIG2Decode, + "JPXDecode": None, # FilterType.JPXDecode, +} + + +def process_filters_on_data( + data: bytes, + filters: Optional[List[str]], + decode_params: Optional[List[Dict[str, Any]]], +) -> bytes: + """runs the filters provided by the list with the provided params against the data + + Args: + data (bytes) + filters (Optional[List[str]]) + decode_params (Optional[List[Dict[str, Any]]]) + + Returns: + bytes + """ + if filters is not None: + decode_parms = [] + if decode_params is not None: + decode_parms = decode_params + for f in filters: + params = {} + if len(decode_parms) > 0: + params = decode_parms.pop() + decoder = detect_decoder_type(f) + data = decoder.decode(data, **params) + return data + + +def detect_decoder_type(name: str) -> Type[PDFFilterDecoder]: + """Check to see if the specified filter exists in our codebase + + Args: + name (str) + + Raises: + InvalidDecoderOrNotImplemented + + Returns: + Type[PDFFilterDecoder] + """ + decoder = _filter_type_name_mapping.get(name) + if not decoder: + raise NotImplementedError( + f"Filter {name} is not implemented yet, feel free to submit a PR and help!" + ) + return decoder diff --git a/pdfmajor/parser/stream/fliters/ccit.py b/pdfmajor/filters/ccit.py similarity index 99% rename from pdfmajor/parser/stream/fliters/ccit.py rename to pdfmajor/filters/ccit.py index 3975134..c504448 100644 --- a/pdfmajor/parser/stream/fliters/ccit.py +++ b/pdfmajor/filters/ccit.py @@ -12,19 +12,7 @@ import array from typing import Optional -from ...exceptions import DecodeFailed - - -class CCITEOFB(DecodeFailed): - pass - - -class CCITInvalidData(DecodeFailed): - pass - - -class CCITByteSkip(DecodeFailed): - pass +from .exceptions import CCITEOFB, CCITByteSkip, CCITInvalidData def get_bytes(data): diff --git a/pdfmajor/filters/decoders.py b/pdfmajor/filters/decoders.py new file mode 100644 index 0000000..7ae1d79 --- /dev/null +++ b/pdfmajor/filters/decoders.py @@ -0,0 +1,48 @@ +import zlib +from base64 import a85decode +from binascii import a2b_hex +from typing import Any + +from .ccit import ccittfaxdecode +from .lzw import lzwdecode +from .rld import rldecode +from .types import PDFFilterDecoder + + +class ASCIIHexDecode(PDFFilterDecoder): + @staticmethod + def decode(data: bytes, **decode_params: Any) -> bytes: + clean_data = b"".join(data.split(b" ")) + if len(clean_data) & 2 != 0: + clean_data += b"0" + return a2b_hex(clean_data) + + +class ASCII85Decode(PDFFilterDecoder): + @staticmethod + def decode(data: bytes, **decode_params: Any) -> bytes: + return a85decode(data, adobe=True) + + +class LZWDecode(PDFFilterDecoder): + @staticmethod + def decode(data: bytes, **decode_params: Any) -> bytes: + return lzwdecode(data) + + +class FlateDecode(PDFFilterDecoder): + @staticmethod + def decode(data: bytes, **decode_params: Any) -> bytes: + return zlib.decompress(data) + + +class RunLengthDecode(PDFFilterDecoder): + @staticmethod + def decode(data: bytes, **decode_params: Any) -> bytes: + return rldecode(data) + + +class CCITTFaxDecode(PDFFilterDecoder): + @staticmethod + def decode(data: bytes, **decode_params: Any) -> bytes: + return ccittfaxdecode(data, **decode_params) diff --git a/pdfmajor/filters/exceptions.py b/pdfmajor/filters/exceptions.py new file mode 100644 index 0000000..20f2426 --- /dev/null +++ b/pdfmajor/filters/exceptions.py @@ -0,0 +1,25 @@ +from pdfmajor.exceptions import BrokenFile + + +class FilterError(BrokenFile): + pass + + +class DecodeFailed(FilterError): + pass + + +class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError): + pass + + +class CCITEOFB(DecodeFailed, EOFError): + pass + + +class CCITInvalidData(DecodeFailed): + pass + + +class CCITByteSkip(DecodeFailed): + pass diff --git a/pdfmajor/parser/stream/fliters/lzw.py b/pdfmajor/filters/lzw.py similarity index 98% rename from pdfmajor/parser/stream/fliters/lzw.py rename to pdfmajor/filters/lzw.py index 90cc8c7..ebf62a2 100644 --- a/pdfmajor/parser/stream/fliters/lzw.py +++ b/pdfmajor/filters/lzw.py @@ -1,7 +1,7 @@ from io import BytesIO from typing import List, Optional -from ...exceptions import DecodeFailed +from .exceptions import DecodeFailed ## LZWDecoder diff --git a/pdfmajor/parser/stream/fliters/rld.py b/pdfmajor/filters/rld.py similarity index 100% rename from pdfmajor/parser/stream/fliters/rld.py rename to pdfmajor/filters/rld.py diff --git a/pdfmajor/filters/types.py b/pdfmajor/filters/types.py new file mode 100644 index 0000000..918c779 --- /dev/null +++ b/pdfmajor/filters/types.py @@ -0,0 +1,10 @@ +from abc import ABCMeta +from typing import Any + +from .exceptions import InvalidDecoderOrNotImplemented + + +class PDFFilterDecoder(metaclass=ABCMeta): + @staticmethod + def decode(data: bytes, **decode_params: Any) -> bytes: + raise InvalidDecoderOrNotImplemented diff --git a/pdfmajor/parser/__init__.py b/pdfmajor/parser/__init__.py index 0b8ce4d..c17b1e1 100644 --- a/pdfmajor/parser/__init__.py +++ b/pdfmajor/parser/__init__.py @@ -9,13 +9,16 @@ from pdfmajor.lexer.token import ( TokenDictionary, TokenKeyword, ) -from pdfmajor.parser.objects.primitives import get_obj_from_token_primitive from pdfmajor.streambuffer import BufferStream from .exceptions import InvalidKeywordPos -from .objects.base import PDFObject -from .objects.collections import PDFArray, PDFDictionary -from .objects.comment import PDFComment +from .objects import ( + PDFArray, + PDFComment, + PDFDictionary, + PDFObject, + get_obj_from_token_primitive, +) from .parsers import ( deal_with_collection_object, on_endobj, diff --git a/pdfmajor/parser/exceptions.py b/pdfmajor/parser/exceptions.py index d1eb6c8..3a1cb54 100644 --- a/pdfmajor/parser/exceptions.py +++ b/pdfmajor/parser/exceptions.py @@ -28,9 +28,5 @@ class InvalidKeywordPos(BrokenFileParserError): ) -class DecodeFailed(BrokenFileParserError): - pass - - -class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError): +class IncompleteStream(BrokenFileParserError): pass diff --git a/pdfmajor/parser/objects/__init__.py b/pdfmajor/parser/objects/__init__.py index e69de29..77491c8 100644 --- a/pdfmajor/parser/objects/__init__.py +++ b/pdfmajor/parser/objects/__init__.py @@ -0,0 +1,17 @@ +from .base import PDFContextualObject, PDFObject +from .collections import PDFArray, PDFDictionary +from .comment import PDFComment +from .indirect import IndirectObject +from .primitives import ( + PDFBoolean, + PDFInteger, + PDFName, + PDFNull, + PDFPrimitiveObject, + PDFReal, + PDFString, + get_obj_from_token_primitive, +) +from .ref import ObjectRef +from .stream import PDFStream +from .util import validate_number_or_none, validate_object_or_none diff --git a/pdfmajor/parser/objects/indirect.py b/pdfmajor/parser/objects/indirect.py index 492b441..10f9b6c 100644 --- a/pdfmajor/parser/objects/indirect.py +++ b/pdfmajor/parser/objects/indirect.py @@ -1,22 +1,7 @@ from typing import Optional -from pdfmajor.parser.stream import PDFStream - from .base import PDFContextualObject, PDFObject - - -class ObjectRef(PDFObject): - """A class representing a reference for other objects""" - - def __init__(self, obj_num: int, gen_num: int) -> None: - self.obj_num = obj_num - self.gen_num = gen_num - - def __hash__(self): - return hash(self.to_python()) - - def to_python(self): - return (self.obj_num, self.gen_num) +from .stream import PDFStream class IndirectObject(PDFContextualObject): diff --git a/pdfmajor/parser/objects/ref.py b/pdfmajor/parser/objects/ref.py new file mode 100644 index 0000000..a6836e5 --- /dev/null +++ b/pdfmajor/parser/objects/ref.py @@ -0,0 +1,15 @@ +from .base import PDFObject + + +class ObjectRef(PDFObject): + """A class representing a reference for other objects""" + + def __init__(self, obj_num: int, gen_num: int) -> None: + self.obj_num = obj_num + self.gen_num = gen_num + + def __hash__(self): + return hash(self.to_python()) + + def to_python(self): + return (self.obj_num, self.gen_num) diff --git a/pdfmajor/parser/stream/pdf_stream.py b/pdfmajor/parser/objects/stream.py similarity index 67% rename from pdfmajor/parser/stream/pdf_stream.py rename to pdfmajor/parser/objects/stream.py index 0b31e33..41d8562 100644 --- a/pdfmajor/parser/stream/pdf_stream.py +++ b/pdfmajor/parser/objects/stream.py @@ -1,10 +1,13 @@ -from typing import Any, List, Optional +from typing import Any, List, Optional, Union -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.util import get_single_or_list, to_python, to_python_list +from pdfmajor.parser.exceptions import IncompleteStream +from pdfmajor.util import get_single_or_list -from ..objects.collections import PDFDictionary -from ..objects.primitives import PDFNull +from .base import PDFObject +from .collections import PDFDictionary +from .primitives import PDFInteger +from .ref import ObjectRef +from .util import to_python, to_python_list class PDFStream: @@ -12,14 +15,14 @@ class PDFStream: def __init__( self, - offset: int = 0, - length: PDFObject = PDFNull(), - filter: Optional[List[PDFObject]] = None, - decode_parms: Optional[List[PDFObject]] = None, - f: Optional[Any] = None, # TODO: file-specifications - ffilter: Optional[List[PDFObject]] = None, - fdecode_parms: Optional[List[PDFObject]] = None, + offset: int, + length: Union[PDFInteger, ObjectRef], + filter: List[PDFObject], + decode_parms: List[PDFObject], + ffilter: List[PDFObject], + fdecode_parms: List[PDFObject], dl: Optional[PDFObject] = None, + f: Optional[Any] = None, # TODO: file-specifications ) -> None: """ Args: @@ -43,15 +46,21 @@ class PDFStream: @classmethod def from_pdfdict(cls, offset: int, item: PDFDictionary) -> "PDFStream": - stream = cls(offset, item["Length"]) - stream.filter = get_single_or_list(item.get("Filter", stream.filter)) - stream.decode_parms = get_single_or_list( - item.get("DecodeParms", stream.decode_parms) - ) - stream.f = item.get("F", stream.f) - stream.ffilter = get_single_or_list(item.get("FFilter", stream.ffilter)) - stream.fdecode_parms = get_single_or_list( - item.get("FDecodeParms", stream.fdecode_parms) + if not isinstance(item["Length"], PDFInteger) and not isinstance( + item["Length"], ObjectRef + ): + raise IncompleteStream( + f"Invalid length property {item} {type(item['Length'])}" + ) + stream = cls( + offset, + item["Length"], + filter=get_single_or_list(item.get("Filter", None)), + decode_parms=get_single_or_list(item.get("DecodeParms", None)), + ffilter=get_single_or_list(item.get("FFilter", None)), + fdecode_parms=get_single_or_list(item.get("FDecodeParms", None)), + f=item.get("F", None), + dl=item.get("DL", None), ) return stream diff --git a/pdfmajor/parser/objects/util.py b/pdfmajor/parser/objects/util.py new file mode 100644 index 0000000..e698d12 --- /dev/null +++ b/pdfmajor/parser/objects/util.py @@ -0,0 +1,66 @@ +from typing import Any, List, Optional, Type, TypeVar, Union + +from .base import PDFObject +from .primitives import PDFInteger, PDFReal + +T = TypeVar("T") + + +def to_python(obj: Optional[PDFObject]) -> Any: + if obj is None: + return None + return obj.to_python() + + +def to_python_list(obj: Optional[List[PDFObject]]) -> List[Any]: + if obj is None: + return [] + return [x.to_python() for x in obj] + + +ExpectedType = TypeVar("ExpectedType", bound=PDFObject) + + +def validate_object_or_none( + obj: Optional[PDFObject], objtype: Type[ExpectedType] +) -> Optional[ExpectedType]: + """Validates the object's type if its not None + + Args: + obj (Optional[PDFObject]) + objtype (Type[ExpectedType]): expected type of the object + + Raises: + AssertionError: is raised if the object has an invalid type + + Returns: + Optional[ExpectedType] + """ + if obj is None: + return None + if isinstance(obj, objtype): + return obj + else: + raise AssertionError(f"obj {obj} is not of expected type {objtype}") + + +def validate_number_or_none( + obj: Optional[PDFObject], +) -> Optional[Union[PDFReal, PDFInteger]]: + """Validates the object's type is some numeric type + + Args: + obj (Optional[PDFObject]) + + Raises: + AssertionError: is raised if the object has an invalid type + + Returns: + Optional[Union[PDFReal, PDFInteger]] + """ + if obj is None: + return None + if isinstance(obj, PDFInteger) or isinstance(obj, PDFReal): + return obj + else: + raise AssertionError(f"obj {obj} is not of numeric type") diff --git a/pdfmajor/parser/parsers.py b/pdfmajor/parser/parsers.py index 335da28..740e6d0 100644 --- a/pdfmajor/parser/parsers.py +++ b/pdfmajor/parser/parsers.py @@ -5,12 +5,18 @@ from pdfmajor.lexer.token import TokenComplexType, TokenComplexTypeVal, TokenKey from pdfmajor.streambuffer import BufferStream from .exceptions import BrokenFile, EarlyStop, ParserError -from .objects.base import PDFContextualObject, PDFObject -from .objects.collections import PDFDictionary -from .objects.indirect import IndirectObject, ObjectRef -from .objects.primitives import PDFInteger, PDFNull, PDFPrimitiveObject +from .objects import ( + IndirectObject, + ObjectRef, + PDFContextualObject, + PDFDictionary, + PDFInteger, + PDFNull, + PDFObject, + PDFPrimitiveObject, + PDFStream, +) from .state import ParsingState -from .stream import PDFStream def on_primitive( diff --git a/pdfmajor/parser/state.py b/pdfmajor/parser/state.py index d16aa19..18e984a 100644 --- a/pdfmajor/parser/state.py +++ b/pdfmajor/parser/state.py @@ -3,10 +3,13 @@ from typing import Iterator, List, Optional, Tuple, Union from pdfmajor.healthlog import PDFHealthReport from .exceptions import ParserError -from .objects.base import PDFContextualObject, PDFObject -from .objects.comment import PDFComment -from .objects.indirect import IndirectObject -from .objects.primitives import PDFInteger +from .objects import ( + IndirectObject, + PDFComment, + PDFContextualObject, + PDFInteger, + PDFObject, +) class ParsingState: diff --git a/pdfmajor/parser/stream/PDFStreamParser.py b/pdfmajor/parser/stream/PDFStreamParser.py deleted file mode 100644 index a51597b..0000000 --- a/pdfmajor/parser/stream/PDFStreamParser.py +++ /dev/null @@ -1,23 +0,0 @@ -from pdfmajor.streambuffer import BufferStream -from .PDFStream import PDFStream -from .fliters import process_filters_on_data - - -# class PDFStreamPasrser(PDFParser): -# def __init__( -# self, -# stream: PDFStream, -# buffer: BufferStream, -# strict: bool = True, -# ) -> None: -# cur_pos = buffer.tell() -# buffer.seek(stream.offset) -# data = buffer.read(stream.length).data -# buffer.seek(cur_pos) -# data = process_filters_on_data(data, stream.filter, stream.decode_parms) -# data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms) -# self.stream = stream -# super().__init__( -# buffer=BufferStream(io.BytesIO(data), buffer_size=stream.length), -# strict=strict, -# ) diff --git a/pdfmajor/parser/stream/__init__.py b/pdfmajor/parser/stream/__init__.py deleted file mode 100644 index d96c85c..0000000 --- a/pdfmajor/parser/stream/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .pdf_stream import PDFStream # noqa diff --git a/pdfmajor/parser/stream/fliters/__init__.py b/pdfmajor/parser/stream/fliters/__init__.py deleted file mode 100644 index 5e0c5ef..0000000 --- a/pdfmajor/parser/stream/fliters/__init__.py +++ /dev/null @@ -1,117 +0,0 @@ -import zlib -from base64 import a85decode -from binascii import a2b_hex -from typing import Any, Dict, List, Optional - -from ...exceptions import InvalidDecoderOrNotImplemented -from .ccit import ccittfaxdecode -from .lzw import lzwdecode -from .rld import rldecode -from .types import FilterType - -_filter_type_name_mapping: Dict[str, FilterType] = { - "FlateDecode": FilterType.FlateDecode, - "Fl": FilterType.FlateDecode, - "LZWDecode": FilterType.LZWDecode, - "LZW": FilterType.LZWDecode, - "ASCII85Decode": FilterType.ASCII85Decode, - "A85": FilterType.ASCII85Decode, - "ASCIIHexDecode": FilterType.ASCIIHexDecode, - "AHx": FilterType.ASCIIHexDecode, - "RunLengthDecode": FilterType.RunLengthDecode, - "RL": FilterType.RunLengthDecode, - "CCITTFaxDecode": FilterType.CCITTFaxDecode, - "CCF": FilterType.CCITTFaxDecode, - "DCTDecode": FilterType.DCTDecode, - "DCT": FilterType.DCTDecode, - "Crypt": FilterType.Crypt, - "JBIG2Decode": FilterType.JBIG2Decode, - "JPXDecode": FilterType.JPXDecode, -} - - -def process_filters_on_data( - data: bytes, - filters: Optional[List[str]], - decode_params: Optional[List[Dict[str, Any]]], -) -> bytes: - """runs the filters provided by the list with the provided params against the data - - Args: - data (bytes) - filters (Optional[List[str]]) - decode_params (Optional[List[Dict[str, Any]]]) - - Returns: - bytes - """ - if filters is not None: - decode_parms = [] - if decode_params is not None: - decode_parms = decode_params - for f in filters: - params = {} - if len(decode_parms) > 0: - params = decode_parms.pop() - filter_type = detect_filter_type(f) - data = decode_bytes(filter_type, data, **params) - return data - - -def detect_filter_type(name: str) -> FilterType: - """Check to see if the specified filter exists in our codebase - - Args: - name (str) - - Raises: - InvalidDecoderOrNotImplemented - - Returns: - FilterType - """ - ftype = _filter_type_name_mapping.get(name, None) - if ftype is None: - raise InvalidDecoderOrNotImplemented(name) - return ftype - - -def decode_bytes(filter_type: FilterType, data: bytes, **kwrgs: Any) -> bytes: - """Run the decoding function on the filter - - Args: - filter_type (FilterType) - data (bytes) - - Raises: - InvalidDecoderOrNotImplemented - - Returns: - bytes - """ - if filter_type == FilterType.ASCII85Decode: - return a85decode(data, adobe=True) - elif filter_type == FilterType.ASCIIHexDecode: - clean_data = b"".join(data.split(b" ")) - if len(clean_data) & 2 != 0: - clean_data += b"0" - return a2b_hex(clean_data) - elif filter_type == FilterType.LZWDecode: - return lzwdecode(data) - elif filter_type == FilterType.FlateDecode: - return zlib.decompress(data) - elif filter_type == FilterType.RunLengthDecode: - return rldecode(data) - elif filter_type == FilterType.CCITTFaxDecode: - return ccittfaxdecode(data, **kwrgs) - else: - raise InvalidDecoderOrNotImplemented(filter_type) - # TODO: - # elif filter_type == FilterType.JBIG2Decode: - # pass - # elif filter_type == FilterType.DCTDecode: - # pass - # elif filter_type == FilterType.JPXDecode: - # pass - # elif filter_type == FilterType.Crypt: - # pass diff --git a/pdfmajor/parser/stream/fliters/types.py b/pdfmajor/parser/stream/fliters/types.py deleted file mode 100644 index 7c93643..0000000 --- a/pdfmajor/parser/stream/fliters/types.py +++ /dev/null @@ -1,16 +0,0 @@ -import enum - - -class FilterType(enum.Enum): - """Standard PDF Filter types see PDF spec 1.7 section 7.4 table 6""" - - ASCIIHexDecode = enum.auto() - ASCII85Decode = enum.auto() - LZWDecode = enum.auto() - FlateDecode = enum.auto() - RunLengthDecode = enum.auto() - CCITTFaxDecode = enum.auto() - JBIG2Decode = enum.auto() - DCTDecode = enum.auto() - JPXDecode = enum.auto() - Crypt = enum.auto() diff --git a/pdfmajor/util.py b/pdfmajor/util.py index 2be01f7..ff729e7 100644 --- a/pdfmajor/util.py +++ b/pdfmajor/util.py @@ -1,7 +1,4 @@ -from typing import Any, List, Optional, Type, TypeVar, Union - -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.primitives import PDFInteger, PDFReal +from typing import List, Optional, TypeVar, Union T = TypeVar("T") @@ -13,63 +10,3 @@ def get_single_or_list(obj: Optional[Union[List[T], T]]) -> List[T]: return obj else: return [obj] - - -def to_python(obj: Optional[PDFObject]) -> Any: - if obj is None: - return None - return obj.to_python() - - -def to_python_list(obj: Optional[List[PDFObject]]) -> List[Any]: - if obj is None: - return [] - return [x.to_python() for x in obj] - - -ExpectedType = TypeVar("ExpectedType", bound=PDFObject) - - -def validate_object_or_none( - obj: Optional[PDFObject], objtype: Type[ExpectedType] -) -> Optional[ExpectedType]: - """Validates the object's type if its not None - - Args: - obj (Optional[PDFObject]) - objtype (Type[ExpectedType]): expected type of the object - - Raises: - AssertionError: is raised if the object has an invalid type - - Returns: - Optional[ExpectedType] - """ - if obj is None: - return None - if isinstance(obj, objtype): - return obj - else: - raise AssertionError(f"obj {obj} is not of expected type {objtype}") - - -def validate_number_or_none( - obj: Optional[PDFObject], -) -> Optional[Union[PDFReal, PDFInteger]]: - """Validates the object's type is some numeric type - - Args: - obj (Optional[PDFObject]) - - Raises: - AssertionError: is raised if the object has an invalid type - - Returns: - Optional[Union[PDFReal, PDFInteger]] - """ - if obj is None: - return None - if isinstance(obj, PDFInteger) or isinstance(obj, PDFReal): - return obj - else: - raise AssertionError(f"obj {obj} is not of numeric type") diff --git a/pdfmajor/xref/exceptions.py b/pdfmajor/xref/exceptions.py index c5a677f..cce50d9 100644 --- a/pdfmajor/xref/exceptions.py +++ b/pdfmajor/xref/exceptions.py @@ -1,7 +1,7 @@ from typing import Set from pdfmajor.exceptions import BrokenFile, PDFMajorException -from pdfmajor.parser.objects.indirect import ObjectRef +from pdfmajor.parser.objects import ObjectRef class XRefError(PDFMajorException): diff --git a/pdfmajor/xref/trailer.py b/pdfmajor/xref/trailer.py index afdb0ab..94fed48 100644 --- a/pdfmajor/xref/trailer.py +++ b/pdfmajor/xref/trailer.py @@ -3,12 +3,15 @@ from typing import List, NamedTuple, Optional, Set from pdfmajor.lexer import iter_tokens from pdfmajor.lexer.token import TokenKeyword from pdfmajor.parser import get_first_object -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary -from pdfmajor.parser.objects.indirect import ObjectRef -from pdfmajor.parser.objects.primitives import PDFInteger +from pdfmajor.parser.objects import ( + ObjectRef, + PDFArray, + PDFDictionary, + PDFInteger, + PDFObject, + validate_object_or_none, +) from pdfmajor.streambuffer import BufferStream -from pdfmajor.util import validate_object_or_none from .exceptions import BrokenFile, InvalidNumberOfRoots, NotRootElement diff --git a/pdfmajor/xref/xrefdb.py b/pdfmajor/xref/xrefdb.py index efc5a74..29b2605 100644 --- a/pdfmajor/xref/xrefdb.py +++ b/pdfmajor/xref/xrefdb.py @@ -1,9 +1,7 @@ from typing import Dict, Iterator, List, Optional, Tuple from pdfmajor.parser import get_first_object -from pdfmajor.parser.objects.collections import PDFDictionary -from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef -from pdfmajor.parser.objects.primitives import PDFInteger +from pdfmajor.parser.objects import IndirectObject, ObjectRef, PDFDictionary, PDFInteger from pdfmajor.streambuffer import BufferStream from .exceptions import BrokenFile, InvalidXref, XRefError diff --git a/tests/test_document.py b/tests/test_document.py index 8138977..ba326af 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -11,13 +11,14 @@ from pdfmajor.document.parse_context import PDFParsingContext from pdfmajor.document.parsers.pages import iter_all_page_leafs from pdfmajor.document.parsers.root import get_catalog, get_info from pdfmajor.document.structures import PDFRectangle -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary -from pdfmajor.parser.objects.indirect import ObjectRef -from pdfmajor.parser.objects.primitives import ( +from pdfmajor.parser.objects import ( + ObjectRef, + PDFArray, PDFBoolean, + PDFDictionary, PDFInteger, PDFName, + PDFObject, PDFReal, PDFString, ) @@ -127,60 +128,6 @@ class ParsingState(TestCase): ), ) - def test_bad_unicode(self): - self.run_test( - "bad-unicode.pdf", - PDFDocumentCatalog( - version=None, - pages=PDFPageTreeNode( - kids=[ - ObjectRef(142, 0), - ObjectRef(143, 0), - ObjectRef(144, 0), - ObjectRef(145, 0), - ObjectRef(146, 0), - ], - parent=None, - leaft_count=PDFInteger(47, 0, 0), - raw=PDFDictionary.from_dict( - { - "Count": PDFInteger(47, 0, 0), - "Type": PDFName("Pages", 0, 0), - "Kids": PDFArray.from_list( - [ - ObjectRef(142, 0), - ObjectRef(143, 0), - ObjectRef(144, 0), - ObjectRef(145, 0), - ObjectRef(146, 0), - ] - ), - } - ), - ), - page_labels=PDFDictionary.from_dict( - {"Nums": [PDFInteger(0, 0, 0), ObjectRef(140, 0)]} - ), - page_layout=None, - page_mode=None, - metadata=PDFDictionary.from_dict( - { - "Subtype": PDFName("XML", 0, 0), - "Length": PDFInteger(3529, 0, 0), - "Type": PDFName("Metadata", 0, 0), - } - ), - raw=PDFDictionary.from_dict( - { - "Metadata": ObjectRef(147, 0), - "Pages": ObjectRef(141, 0), - "Type": PDFName("Catalog", 0, 0), - "PageLabels": ObjectRef(139, 0), - } - ), - ), - ) - def test_all(self): for file_path, buffer in all_pdf_files.items(): with self.subTest(file_path), buffer.get_window(): diff --git a/tests/test_parser.py b/tests/test_parser.py index 2ce4fb1..37c3b39 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -4,12 +4,17 @@ from typing import Any, List from unittest import TestCase from pdfmajor.parser import iter_objects -from pdfmajor.parser.objects.base import PDFObject -from pdfmajor.parser.objects.collections import PDFArray, PDFDictionary -from pdfmajor.parser.objects.comment import PDFComment -from pdfmajor.parser.objects.indirect import IndirectObject, ObjectRef -from pdfmajor.parser.objects.primitives import PDFInteger, PDFString -from pdfmajor.parser.stream import PDFStream +from pdfmajor.parser.objects import ( + IndirectObject, + ObjectRef, + PDFArray, + PDFComment, + PDFDictionary, + PDFInteger, + PDFObject, + PDFStream, + PDFString, +) from pdfmajor.streambuffer import BufferStream CURRENT_FOLDER = Path(__file__).parent diff --git a/tests/test_parser_filters.py b/tests/test_parser_filters.py index 8ab2795..a3a2961 100644 --- a/tests/test_parser_filters.py +++ b/tests/test_parser_filters.py @@ -1,8 +1,8 @@ from unittest import TestCase -from pdfmajor.parser.stream.fliters.ccit import CCITTG4Parser -from pdfmajor.parser.stream.fliters.lzw import lzwdecode -from pdfmajor.parser.stream.fliters.rld import rldecode +from pdfmajor.filters.ccit import CCITTG4Parser +from pdfmajor.filters.lzw import lzwdecode +from pdfmajor.filters.rld import rldecode class Filters(TestCase):