diff --git a/pdfmajor/filters/__init__.py b/pdfmajor/filters/__init__.py index fd83271..6f776a1 100644 --- a/pdfmajor/filters/__init__.py +++ b/pdfmajor/filters/__init__.py @@ -6,7 +6,7 @@ from .decoders import ( CCITTFaxDecode, FlateDecode, LZWDecode, - RunLengthDecode, + RunLengthDecode ) from .types import PDFFilterDecoder @@ -47,13 +47,10 @@ def process_filters_on_data( bytes """ if filters is not None: - decode_parms = [] - if decode_params is not None: - decode_parms = decode_params - for f in filters: - params = {} - if len(decode_parms) > 0: - params = decode_parms.pop() + if decode_params is None: + decode_params = [{}] * len(filters) + for i, f in enumerate(filters): + params = decode_params[i] decoder = detect_decoder_type(f) data = decoder.decode(data, **params) return data diff --git a/pdfmajor/filters/ccit.py b/pdfmajor/filters/ccit.py index c504448..128534e 100644 --- a/pdfmajor/filters/ccit.py +++ b/pdfmajor/filters/ccit.py @@ -10,7 +10,6 @@ import array -from typing import Optional from .exceptions import CCITEOFB, CCITByteSkip, CCITInvalidData @@ -337,20 +336,20 @@ class CCITTG4Parser(BitParser): break return - def _parse_mode(self, mode): + def _parse_mode(self, mode: str): if mode == "p": self._do_pass() self._flush_line() return self.MODE elif mode == "h": self._n1 = 0 - self._accept = self._parse_horiz1 + self._accept = self._parse_horiz1 # type: ignore if self._color: return self.WHITE else: return self.BLACK elif mode == "u": - self._accept = self._parse_uncompressed + self._accept = self._parse_uncompressed # type: ignore return self.UNCOMPRESSED elif mode == "e": raise CCITEOFB @@ -552,10 +551,14 @@ class CCITTFaxDecoder(CCITTG4Parser): def ccittfaxdecode( data: bytes, - Columns: int, + # arguments for algo, with defaults based on PDF spec 1.7 section 7.4.6 + Columns: int = 1728, EncodedByteAlign: bool = False, BlackIs1: bool = False, - K: Optional[int] = None, + K: int = 0, + EndOfLine: bool = False, + Rows: int = 0, + DamagedRowsBeforeError: int = 0, ) -> bytes: if K == -1: parser = CCITTFaxDecoder(Columns, bytealign=EncodedByteAlign, reversed=BlackIs1) diff --git a/pdfmajor/lexer/__init__.py b/pdfmajor/lexer/__init__.py index 02375b1..3e899bd 100644 --- a/pdfmajor/lexer/__init__.py +++ b/pdfmajor/lexer/__init__.py @@ -13,7 +13,7 @@ from pdfmajor.lexer.token import ( Token, TokenArray, TokenDictionary, - TokenKeyword, + TokenKeyword ) from pdfmajor.streambuffer import BufferStream diff --git a/pdfmajor/parser/__init__.py b/pdfmajor/parser/__init__.py index c17b1e1..4e3261a 100644 --- a/pdfmajor/parser/__init__.py +++ b/pdfmajor/parser/__init__.py @@ -7,7 +7,7 @@ from pdfmajor.lexer.token import ( TokenArray, TokenComment, TokenDictionary, - TokenKeyword, + TokenKeyword ) from pdfmajor.streambuffer import BufferStream @@ -17,14 +17,14 @@ from .objects import ( PDFComment, PDFDictionary, PDFObject, - get_obj_from_token_primitive, + get_obj_from_token_primitive ) from .parsers import ( deal_with_collection_object, on_endobj, on_indirect_ref_close, on_primitive, - on_stream, + on_stream ) from .state import ParsingState diff --git a/pdfmajor/parser/objects/__init__.py b/pdfmajor/parser/objects/__init__.py index 77491c8..7ae83da 100644 --- a/pdfmajor/parser/objects/__init__.py +++ b/pdfmajor/parser/objects/__init__.py @@ -10,7 +10,7 @@ from .primitives import ( PDFPrimitiveObject, PDFReal, PDFString, - get_obj_from_token_primitive, + get_obj_from_token_primitive ) from .ref import ObjectRef from .stream import PDFStream diff --git a/pdfmajor/parser/objects/stream.py b/pdfmajor/parser/objects/stream.py index 41d8562..8188742 100644 --- a/pdfmajor/parser/objects/stream.py +++ b/pdfmajor/parser/objects/stream.py @@ -5,7 +5,7 @@ from pdfmajor.util import get_single_or_list from .base import PDFObject from .collections import PDFDictionary -from .primitives import PDFInteger +from .primitives import PDFInteger, PDFNull from .ref import ObjectRef from .util import to_python, to_python_list @@ -52,13 +52,21 @@ class PDFStream: raise IncompleteStream( f"Invalid length property {item} {type(item['Length'])}" ) + filters = get_single_or_list(item.get("Filter", None)) + decode_parms = get_single_or_list(item.get("DecodeParms", None)) + if len(decode_parms) == 0: + decode_parms = [PDFNull()] * len(filters) + ffilter = get_single_or_list(item.get("FFilter", None)) + fdecode_parms = get_single_or_list(item.get("FDecodeParms", None)) + if len(fdecode_parms) == 0: + fdecode_parms = [PDFNull()] * len(ffilter) stream = cls( offset, item["Length"], - filter=get_single_or_list(item.get("Filter", None)), - decode_parms=get_single_or_list(item.get("DecodeParms", None)), - ffilter=get_single_or_list(item.get("FFilter", None)), - fdecode_parms=get_single_or_list(item.get("FDecodeParms", None)), + filter=filters, + decode_parms=decode_parms, + ffilter=ffilter, + fdecode_parms=fdecode_parms, f=item.get("F", None), dl=item.get("DL", None), ) diff --git a/pdfmajor/parser/parsers.py b/pdfmajor/parser/parsers.py index fcbe076..edbdea4 100644 --- a/pdfmajor/parser/parsers.py +++ b/pdfmajor/parser/parsers.py @@ -119,7 +119,7 @@ def on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword): obj = last_ctx.get_object() if not isinstance(obj, PDFDictionary): raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}") - last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj) + last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 2, obj) if isinstance(stream.length, PDFInteger): buffer.seek(stream.offset + stream.length.to_python()) next_token = None diff --git a/tests/samples/pdf/PDF_spec.1.7.pdf b/tests/samples/pdf/PDF_spec.1.7.pdf new file mode 100644 index 0000000..778672f Binary files /dev/null and b/tests/samples/pdf/PDF_spec.1.7.pdf differ diff --git a/tests/samples/pdf/graphs.pdf b/tests/samples/pdf/graphs.pdf new file mode 100644 index 0000000..006d932 Binary files /dev/null and b/tests/samples/pdf/graphs.pdf differ diff --git a/tests/samples/pdf/vr.pdf b/tests/samples/pdf/vr.pdf new file mode 100644 index 0000000..9202d66 Binary files /dev/null and b/tests/samples/pdf/vr.pdf differ diff --git a/tests/test_document.py b/tests/test_document.py index 0cbf3ee..041bdf7 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -25,6 +25,7 @@ from pdfmajor.parser.objects import ( PDFReal, PDFString, ) +from pdfmajor.parser.objects.primitives import PDFNull from tests.util import all_corrupt_pdf_files, all_pdf_files @@ -64,6 +65,85 @@ class ParsingState(TestCase): expcat.raw, ) + def test_pdf_spec(self): + self.run_test( + "PDF_spec.1.7.pdf", + PDFDocumentCatalog( + version=None, + pages=PDFPageTreeNode( + kids=[ + ObjectRef(20, 0), + ObjectRef(21, 0), + ObjectRef(22, 0), + ObjectRef(23, 0), + ObjectRef(24, 0), + ObjectRef(25, 0), + ObjectRef(26, 0), + ObjectRef(27, 0), + ], + parent=None, + leaft_count=756, + raw=PDFDictionary.from_dict( + { + "Count": PDFInteger(756, 0, 0), + "Kids": PDFArray.from_list( + [ + ObjectRef(20, 0), + ObjectRef(21, 0), + ObjectRef(22, 0), + ObjectRef(23, 0), + ObjectRef(24, 0), + ObjectRef(25, 0), + ObjectRef(26, 0), + ObjectRef(27, 0), + ] + ), + "Type": PDFName("Pages", 0, 0), + } + ), + ), + page_labels=PDFDictionary.from_dict( + { + "Nums": PDFArray.from_list( + [ + PDFInteger(0, 0, 0), + ObjectRef(18, 0), + PDFInteger(8, 0, 0), + ObjectRef(19, 0), + ] + ) + } + ), + page_layout=None, + page_mode=PDFName("UseOutlines", 0, 0), + metadata=PDFMetadata(sub_type="XML", stream_data=b""), + raw=PDFDictionary.from_dict( + { + "MarkInfo": PDFDictionary.from_dict( + {"Marked": PDFBoolean(True, 0, 0)} + ), + "Metadata": ObjectRef(2, 0), + "Names": ObjectRef(3, 0), + "OpenAction": PDFArray.from_list( + [ + ObjectRef(4, 0), + PDFName("XYZ", 0, 0), + PDFNull(), + PDFNull(), + PDFNull(), + ] + ), + "Outlines": ObjectRef(5, 0), + "PageLabels": ObjectRef(6, 0), + "PageMode": PDFName("UseOutlines", 0, 0), + "Pages": ObjectRef(7, 0), + "StructTreeRoot": ObjectRef(8, 0), + "Type": PDFName("Catalog", 0, 0), + } + ), + ), + ) + def test_bad_unicode(self): self.run_test( "bad-unicode.pdf", @@ -152,15 +232,16 @@ class ParsingState(TestCase): def test_all(self): for file_path, buffer in all_pdf_files.items(): - with self.subTest(file_path), buffer.get_window(): - parser = PDFParsingContext(buffer) - catalog = get_catalog(parser) - self.assertEqual(len(parser.health_report), 0) - get_info(parser) - list(iter_all_page_leafs(parser, catalog.pages)) - parser = PDFDocument(buffer) - actual_pages = len(list(parser.iter_pages())) - self.assertEqual(actual_pages, parser.num_pages) + if file_path != "PDF_spec.1.7.pdf": + with self.subTest(file_path), buffer.get_window(): + parser = PDFParsingContext(buffer) + catalog = get_catalog(parser) + self.assertEqual(len(parser.health_report), 0) + get_info(parser) + list(iter_all_page_leafs(parser, catalog.pages)) + parser = PDFDocument(buffer) + actual_pages = len(list(parser.iter_pages())) + self.assertEqual(actual_pages, parser.num_pages) def test_metadata(self): with all_pdf_files["bad-unicode.pdf"].get_window() as buffer: diff --git a/tests/test_parser.py b/tests/test_parser.py index 37c3b39..38ba81e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -207,7 +207,8 @@ class IndirectObjects(TestCase): << /Type /XRef /Length 123 /Filter /FlateDecode /otherfeature [10 20]>> stream "<99>l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567 -<80>endstream +<80> +endstream endobj """ ) @@ -218,10 +219,10 @@ endobj self.assertDictEqual( obj.stream.to_python(), # type: ignore { - "offset": 100, + "offset": 101, "length": 123, "filter": ["/FlateDecode"], - "decode_parms": [], + "decode_parms": [None], "f": None, "ffilter": [], "fdecode_parms": [], @@ -256,10 +257,10 @@ endobj self.assertDictEqual( obj.stream.to_python(), # type: ignore { - "offset": 100, - "length": 123 + 5, + "offset": 101, + "length": 123 + 4, "filter": ["/FlateDecode"], - "decode_parms": [], + "decode_parms": [None], "f": None, "ffilter": [], "fdecode_parms": [], @@ -270,7 +271,7 @@ endobj obj.get_object().to_python(), { "Filter": "/FlateDecode", - "Length": 123 + 5, + "Length": 123 + 4, "Type": "/XRef", "otherfeature": [10, 20], }, diff --git a/tests/test_parser_filters.py b/tests/test_parser_filters.py index a3a2961..e2efc76 100644 --- a/tests/test_parser_filters.py +++ b/tests/test_parser_filters.py @@ -1,8 +1,11 @@ from unittest import TestCase +from pdfmajor.document.parse_context import PDFParsingContext from pdfmajor.filters.ccit import CCITTG4Parser from pdfmajor.filters.lzw import lzwdecode from pdfmajor.filters.rld import rldecode +from pdfmajor.parser.objects.ref import ObjectRef +from tests.util import all_pdf_files class Filters(TestCase): @@ -168,3 +171,107 @@ class CCIT(TestCase): parser._do_vertical(1) self.assertEqual(parser._get_bits(), "00000001") return + + +class Stream(TestCase): + def test_flatedecode(self): + with all_pdf_files["bad-unicode.pdf"].get_window() as buffer: + pctx = PDFParsingContext(buffer) + stream_def, stream_data = pctx.validated_and_iter_stream(ObjectRef(161, 0)) + self.assertListEqual( + stream_data.strip().split(b"\n"), + [ + b"/GS1 gs", + b"BT", + b"/TT2 1 Tf", + b"9.9856 0 0 9.9856 39.9 724.5601 Tm", + b"0 g", + b"0.0022 Tc", + b"-0.0036 Tw", + b"[(Map Un)-5.9(i)9.6(t)-1.3( Expla)5.6(n)-5.9(ation)6.2( )]TJ", + b"7.9885 0 0 7.9885 171.96 723.48 Tm", + b"-0.0019 Tc", + b"0.0095 Tw", + b"[(Legend: Map )15(Un)-6.9(its )7.5(for Tonto )7.5(National)-9.5( )15(Fore)-9.1(s)6(t G)9.8(e)-1.5(ol)-9.5(ogy)]TJ", + b"/TT4 1 Tf", + b"9.9856 0 0 9.9856 90.9 703.62 Tm", + b"0.0031 Tc", + b"0 Tw", + b"(Water)Tj", + b"/TT6 1 Tf", + b"7.9885 0 0 7.9885 108.96 690.48 Tm", + b"-0.0029 Tc", + b"[(Wat)-3(e)-2.2(r)]TJ", + b"ET", + b"/Cs6 cs 0 0.74902 1 scn", + b"42.48 693.66 44.04 19.98 re", + b"f", + b"BT", + b"/TT4 1 Tf", + b"13.9798 0 0 13.9798 59.52 700.26 Tm", + b"0 g", + b"0 Tc", + b"(w)Tj", + b"ET", + b"41.52 713.64 46.02 0.95996 re", + b"f", + b"86.52 692.64 1.02 21.96 re", + b"f", + b"41.52 692.64 46.02 1.02 re", + b"f", + b"41.52 692.64 0.95999 21.96 re", + b"f", + b"BT", + b"9.9856 0 0 9.9856 90.9 671.28 Tm", + b"0.0027 Tc", + b"-0.0063 Tw", + b"[(Dist)-6.8(u)6.1(r)-4.1(be)7.9(d)-6( g)-8(r)1.9(ound )-6(\\(rece)7.9(n)0(t\\))]TJ", + b"/TT6 1 Tf", + b"7.9885 0 0 7.9885 108.96 658.2001 Tm", + b"0.0031 Tc", + b"-0.0052 Tw", + b"[(H)11.7(e)-3.7(avily)7.4( distur)5.6(bed gr)5.6(ound)-7.6( )7.6(d)-7.6(u)7.4(e to)7.4( )-7.5(ag)7.4(ri)10.5(cu)7.4(lt)10.5(u)-7.6(r)13.1(e,)5.2( )7.6(exte)]TJ", + b"19.7159 0 TD", + b"0.002 Tc", + b"0.0034 Tw", + b"[(nsive )7.5(excavation,)4.1( o)6.3(r)4.5( )7.5(co)-8.7(n)6.3(s)0.6(tr)4.5(uct)9.4(i)-5.6(on)6.3( o)6.3(f)-3( ear)12(th )7.5(dam)21.2(s)]TJ", + b"ET", + b"0.82 g", + b"42.48 661.32 44.04 19.98 re", + b"f", + b"BT", + b"/TT4 1 Tf", + b"13.9798 0 0 13.9798 60.6 667.92 Tm", + b"0 g", + b"0 Tc", + b"0 Tw", + b"(d)Tj", + b"ET", + b"41.52 681.3 46.02 1.02 re", + b"f", + b"86.52 660.36 1.02 21.96 re", + b"f", + b"41.52 660.36 46.02 0.96002 re", + b"f", + b"41.52 660.36 0.95999 21.96 re", + b"f", + b"BT", + b"9.9856 0 0 9.9856 90.9 639 Tm", + b"0.0018 Tc", + b"0.0006 Tw", + b"[(Q)-7.5(u)-0.9(at)-7.7(e)7(r)-5(nar)7(y)-8.9( sediment)-7.7(ary dep)5.2(o)-2.9(sit)-7.7(s)]TJ", + b"/TT6 1 Tf", + b"7.9885 0 0 7.9885 108.96 625.92 Tm", + b"0.0005 Tc", + b"-0.0026 Tw", + b"[(T)10.5(y)4.8(p)-10.2(i)0.4(cally)4.8( non m)19.7(a)1.2(rine clas)-8.4(t)7.9(i)0.4(c sedi)-7.1(m)27.2(e)-6.3(n)4.8(t)-7.1(a)8.7(r)3(y)4.8( )-7.5(rock)4.8(s)-8.4( )7.6(d)-10.2(e)1.2(posited in late P)-6.7(l)7.9(ioc)-6.3(e)8.7(n)-10.2(e)1.2( )7.6(t)-7.1(o)4.8( )-7.5(Qu)4.8(a)-6.3(t)7.9(ern)4.8(a)-6.3(ry)4.8( tim)19.7(e, in deposition)-10.2(a)8.7(l sy)4.8(st)-7.1(em)19.7(s)6.6( )-7.5(re)8.7(fl)-7.1(ec)8.7(t)-7.1(e)1.2(d)]TJ", + b"50.7958 0 TD", + b"0.0012 Tc", + b"-0.0033 Tw", + b"[( in)5.5( )-7.5(t)8.6(h)-9.5(e m)20.4(oder)3.7(n)-2( )]TJ", + b"-50.7958 -1.1792 TD", + ], + ) + self.assertDictEqual( + stream_def.to_python(), {"Length": 1009, "Filter": "/FlateDecode"} + )