light testing for filters

This commit is contained in:
ari committed 2020-11-03 15:47:50 -05:00
1 parent 6e43a92517
commit e47475b95f
13 files changed
+238 -41

No files matched your search

+5 -8
View File
@@ -6,7 +6,7 @@ from .decoders import (
CCITTFaxDecode,
FlateDecode,
LZWDecode,
RunLengthDecode,
RunLengthDecode
)
from .types import PDFFilterDecoder
@@ -47,13 +47,10 @@ def process_filters_on_data(
bytes
"""
if filters is not None:
decode_parms = []
if decode_params is not None:
decode_parms = decode_params
for f in filters:
params = {}
if len(decode_parms) > 0:
params = decode_parms.pop()
if decode_params is None:
decode_params = [{}] * len(filters)
for i, f in enumerate(filters):
params = decode_params[i]
decoder = detect_decoder_type(f)
data = decoder.decode(data, **params)
return data
+9 -6
View File
@@ -10,7 +10,6 @@
import array
from typing import Optional
from .exceptions import CCITEOFB, CCITByteSkip, CCITInvalidData
@@ -337,20 +336,20 @@ class CCITTG4Parser(BitParser):
break
return
def _parse_mode(self, mode):
def _parse_mode(self, mode: str):
if mode == "p":
self._do_pass()
self._flush_line()
return self.MODE
elif mode == "h":
self._n1 = 0
self._accept = self._parse_horiz1
self._accept = self._parse_horiz1 # type: ignore
if self._color:
return self.WHITE
else:
return self.BLACK
elif mode == "u":
self._accept = self._parse_uncompressed
self._accept = self._parse_uncompressed # type: ignore
return self.UNCOMPRESSED
elif mode == "e":
raise CCITEOFB
@@ -552,10 +551,14 @@ class CCITTFaxDecoder(CCITTG4Parser):
def ccittfaxdecode(
data: bytes,
Columns: int,
# arguments for algo, with defaults based on PDF spec 1.7 section 7.4.6
Columns: int = 1728,
EncodedByteAlign: bool = False,
BlackIs1: bool = False,
K: Optional[int] = None,
K: int = 0,
EndOfLine: bool = False,
Rows: int = 0,
DamagedRowsBeforeError: int = 0,
) -> bytes:
if K == -1:
parser = CCITTFaxDecoder(Columns, bytealign=EncodedByteAlign, reversed=BlackIs1)
+1 -1
View File
@@ -13,7 +13,7 @@ from pdfmajor.lexer.token import (
Token,
TokenArray,
TokenDictionary,
TokenKeyword,
TokenKeyword
)
from pdfmajor.streambuffer import BufferStream
+3 -3
View File
@@ -7,7 +7,7 @@ from pdfmajor.lexer.token import (
TokenArray,
TokenComment,
TokenDictionary,
TokenKeyword,
TokenKeyword
)
from pdfmajor.streambuffer import BufferStream
@@ -17,14 +17,14 @@ from .objects import (
PDFComment,
PDFDictionary,
PDFObject,
get_obj_from_token_primitive,
get_obj_from_token_primitive
)
from .parsers import (
deal_with_collection_object,
on_endobj,
on_indirect_ref_close,
on_primitive,
on_stream,
on_stream
)
from .state import ParsingState
+1 -1
View File
@@ -10,7 +10,7 @@ from .primitives import (
PDFPrimitiveObject,
PDFReal,
PDFString,
get_obj_from_token_primitive,
get_obj_from_token_primitive
)
from .ref import ObjectRef
from .stream import PDFStream
+13 -5
View File
@@ -5,7 +5,7 @@ from pdfmajor.util import get_single_or_list
from .base import PDFObject
from .collections import PDFDictionary
from .primitives import PDFInteger
from .primitives import PDFInteger, PDFNull
from .ref import ObjectRef
from .util import to_python, to_python_list
@@ -52,13 +52,21 @@ class PDFStream:
raise IncompleteStream(
f"Invalid length property {item} {type(item['Length'])}"
)
filters = get_single_or_list(item.get("Filter", None))
decode_parms = get_single_or_list(item.get("DecodeParms", None))
if len(decode_parms) == 0:
decode_parms = [PDFNull()] * len(filters)
ffilter = get_single_or_list(item.get("FFilter", None))
fdecode_parms = get_single_or_list(item.get("FDecodeParms", None))
if len(fdecode_parms) == 0:
fdecode_parms = [PDFNull()] * len(ffilter)
stream = cls(
offset,
item["Length"],
filter=get_single_or_list(item.get("Filter", None)),
decode_parms=get_single_or_list(item.get("DecodeParms", None)),
ffilter=get_single_or_list(item.get("FFilter", None)),
fdecode_parms=get_single_or_list(item.get("FDecodeParms", None)),
filter=filters,
decode_parms=decode_parms,
ffilter=ffilter,
fdecode_parms=fdecode_parms,
f=item.get("F", None),
dl=item.get("DL", None),
)
+1 -1
View File
@@ -119,7 +119,7 @@ def on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
obj = last_ctx.get_object()
if not isinstance(obj, PDFDictionary):
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 2, obj)
if isinstance(stream.length, PDFInteger):
buffer.seek(stream.offset + stream.length.to_python())
next_token = None
Binary file not shown.
Binary file not shown.
Binary file not shown.
+90 -9
View File
@@ -25,6 +25,7 @@ from pdfmajor.parser.objects import (
PDFReal,
PDFString,
)
from pdfmajor.parser.objects.primitives import PDFNull
from tests.util import all_corrupt_pdf_files, all_pdf_files
@@ -64,6 +65,85 @@ class ParsingState(TestCase):
expcat.raw,
)
def test_pdf_spec(self):
self.run_test(
"PDF_spec.1.7.pdf",
PDFDocumentCatalog(
version=None,
pages=PDFPageTreeNode(
kids=[
ObjectRef(20, 0),
ObjectRef(21, 0),
ObjectRef(22, 0),
ObjectRef(23, 0),
ObjectRef(24, 0),
ObjectRef(25, 0),
ObjectRef(26, 0),
ObjectRef(27, 0),
],
parent=None,
leaft_count=756,
raw=PDFDictionary.from_dict(
{
"Count": PDFInteger(756, 0, 0),
"Kids": PDFArray.from_list(
[
ObjectRef(20, 0),
ObjectRef(21, 0),
ObjectRef(22, 0),
ObjectRef(23, 0),
ObjectRef(24, 0),
ObjectRef(25, 0),
ObjectRef(26, 0),
ObjectRef(27, 0),
]
),
"Type": PDFName("Pages", 0, 0),
}
),
),
page_labels=PDFDictionary.from_dict(
{
"Nums": PDFArray.from_list(
[
PDFInteger(0, 0, 0),
ObjectRef(18, 0),
PDFInteger(8, 0, 0),
ObjectRef(19, 0),
]
)
}
),
page_layout=None,
page_mode=PDFName("UseOutlines", 0, 0),
metadata=PDFMetadata(sub_type="XML", stream_data=b""),
raw=PDFDictionary.from_dict(
{
"MarkInfo": PDFDictionary.from_dict(
{"Marked": PDFBoolean(True, 0, 0)}
),
"Metadata": ObjectRef(2, 0),
"Names": ObjectRef(3, 0),
"OpenAction": PDFArray.from_list(
[
ObjectRef(4, 0),
PDFName("XYZ", 0, 0),
PDFNull(),
PDFNull(),
PDFNull(),
]
),
"Outlines": ObjectRef(5, 0),
"PageLabels": ObjectRef(6, 0),
"PageMode": PDFName("UseOutlines", 0, 0),
"Pages": ObjectRef(7, 0),
"StructTreeRoot": ObjectRef(8, 0),
"Type": PDFName("Catalog", 0, 0),
}
),
),
)
def test_bad_unicode(self):
self.run_test(
"bad-unicode.pdf",
@@ -152,15 +232,16 @@ class ParsingState(TestCase):
def test_all(self):
for file_path, buffer in all_pdf_files.items():
with self.subTest(file_path), buffer.get_window():
parser = PDFParsingContext(buffer)
catalog = get_catalog(parser)
self.assertEqual(len(parser.health_report), 0)
get_info(parser)
list(iter_all_page_leafs(parser, catalog.pages))
parser = PDFDocument(buffer)
actual_pages = len(list(parser.iter_pages()))
self.assertEqual(actual_pages, parser.num_pages)
if file_path != "PDF_spec.1.7.pdf":
with self.subTest(file_path), buffer.get_window():
parser = PDFParsingContext(buffer)
catalog = get_catalog(parser)
self.assertEqual(len(parser.health_report), 0)
get_info(parser)
list(iter_all_page_leafs(parser, catalog.pages))
parser = PDFDocument(buffer)
actual_pages = len(list(parser.iter_pages()))
self.assertEqual(actual_pages, parser.num_pages)
def test_metadata(self):
with all_pdf_files["bad-unicode.pdf"].get_window() as buffer:
+8 -7
View File
@@ -207,7 +207,8 @@ class IndirectObjects(TestCase):
<< /Type /XRef /Length 123 /Filter /FlateDecode /otherfeature [10 20]>>
stream
"<99><A3><C1>l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567
<80>endstream
<80>
endstream
endobj
"""
)
@@ -218,10 +219,10 @@ endobj
self.assertDictEqual(
obj.stream.to_python(), # type: ignore
{
"offset": 100,
"offset": 101,
"length": 123,
"filter": ["/FlateDecode"],
"decode_parms": [],
"decode_parms": [None],
"f": None,
"ffilter": [],
"fdecode_parms": [],
@@ -256,10 +257,10 @@ endobj
self.assertDictEqual(
obj.stream.to_python(), # type: ignore
{
"offset": 100,
"length": 123 + 5,
"offset": 101,
"length": 123 + 4,
"filter": ["/FlateDecode"],
"decode_parms": [],
"decode_parms": [None],
"f": None,
"ffilter": [],
"fdecode_parms": [],
@@ -270,7 +271,7 @@ endobj
obj.get_object().to_python(),
{
"Filter": "/FlateDecode",
"Length": 123 + 5,
"Length": 123 + 4,
"Type": "/XRef",
"otherfeature": [10, 20],
},
+107
View File
@@ -1,8 +1,11 @@
from unittest import TestCase
from pdfmajor.document.parse_context import PDFParsingContext
from pdfmajor.filters.ccit import CCITTG4Parser
from pdfmajor.filters.lzw import lzwdecode
from pdfmajor.filters.rld import rldecode
from pdfmajor.parser.objects.ref import ObjectRef
from tests.util import all_pdf_files
class Filters(TestCase):
@@ -168,3 +171,107 @@ class CCIT(TestCase):
parser._do_vertical(1)
self.assertEqual(parser._get_bits(), "00000001")
return
class Stream(TestCase):
def test_flatedecode(self):
with all_pdf_files["bad-unicode.pdf"].get_window() as buffer:
pctx = PDFParsingContext(buffer)
stream_def, stream_data = pctx.validated_and_iter_stream(ObjectRef(161, 0))
self.assertListEqual(
stream_data.strip().split(b"\n"),
[
b"/GS1 gs",
b"BT",
b"/TT2 1 Tf",
b"9.9856 0 0 9.9856 39.9 724.5601 Tm",
b"0 g",
b"0.0022 Tc",
b"-0.0036 Tw",
b"[(Map Un)-5.9(i)9.6(t)-1.3( Expla)5.6(n)-5.9(ation)6.2( )]TJ",
b"7.9885 0 0 7.9885 171.96 723.48 Tm",
b"-0.0019 Tc",
b"0.0095 Tw",
b"[(Legend: Map )15(Un)-6.9(its )7.5(for Tonto )7.5(National)-9.5( )15(Fore)-9.1(s)6(t G)9.8(e)-1.5(ol)-9.5(ogy)]TJ",
b"/TT4 1 Tf",
b"9.9856 0 0 9.9856 90.9 703.62 Tm",
b"0.0031 Tc",
b"0 Tw",
b"(Water)Tj",
b"/TT6 1 Tf",
b"7.9885 0 0 7.9885 108.96 690.48 Tm",
b"-0.0029 Tc",
b"[(Wat)-3(e)-2.2(r)]TJ",
b"ET",
b"/Cs6 cs 0 0.74902 1 scn",
b"42.48 693.66 44.04 19.98 re",
b"f",
b"BT",
b"/TT4 1 Tf",
b"13.9798 0 0 13.9798 59.52 700.26 Tm",
b"0 g",
b"0 Tc",
b"(w)Tj",
b"ET",
b"41.52 713.64 46.02 0.95996 re",
b"f",
b"86.52 692.64 1.02 21.96 re",
b"f",
b"41.52 692.64 46.02 1.02 re",
b"f",
b"41.52 692.64 0.95999 21.96 re",
b"f",
b"BT",
b"9.9856 0 0 9.9856 90.9 671.28 Tm",
b"0.0027 Tc",
b"-0.0063 Tw",
b"[(Dist)-6.8(u)6.1(r)-4.1(be)7.9(d)-6( g)-8(r)1.9(ound )-6(\\(rece)7.9(n)0(t\\))]TJ",
b"/TT6 1 Tf",
b"7.9885 0 0 7.9885 108.96 658.2001 Tm",
b"0.0031 Tc",
b"-0.0052 Tw",
b"[(H)11.7(e)-3.7(avily)7.4( distur)5.6(bed gr)5.6(ound)-7.6( )7.6(d)-7.6(u)7.4(e to)7.4( )-7.5(ag)7.4(ri)10.5(cu)7.4(lt)10.5(u)-7.6(r)13.1(e,)5.2( )7.6(exte)]TJ",
b"19.7159 0 TD",
b"0.002 Tc",
b"0.0034 Tw",
b"[(nsive )7.5(excavation,)4.1( o)6.3(r)4.5( )7.5(co)-8.7(n)6.3(s)0.6(tr)4.5(uct)9.4(i)-5.6(on)6.3( o)6.3(f)-3( ear)12(th )7.5(dam)21.2(s)]TJ",
b"ET",
b"0.82 g",
b"42.48 661.32 44.04 19.98 re",
b"f",
b"BT",
b"/TT4 1 Tf",
b"13.9798 0 0 13.9798 60.6 667.92 Tm",
b"0 g",
b"0 Tc",
b"0 Tw",
b"(d)Tj",
b"ET",
b"41.52 681.3 46.02 1.02 re",
b"f",
b"86.52 660.36 1.02 21.96 re",
b"f",
b"41.52 660.36 46.02 0.96002 re",
b"f",
b"41.52 660.36 0.95999 21.96 re",
b"f",
b"BT",
b"9.9856 0 0 9.9856 90.9 639 Tm",
b"0.0018 Tc",
b"0.0006 Tw",
b"[(Q)-7.5(u)-0.9(at)-7.7(e)7(r)-5(nar)7(y)-8.9( sediment)-7.7(ary dep)5.2(o)-2.9(sit)-7.7(s)]TJ",
b"/TT6 1 Tf",
b"7.9885 0 0 7.9885 108.96 625.92 Tm",
b"0.0005 Tc",
b"-0.0026 Tw",
b"[(T)10.5(y)4.8(p)-10.2(i)0.4(cally)4.8( non m)19.7(a)1.2(rine clas)-8.4(t)7.9(i)0.4(c sedi)-7.1(m)27.2(e)-6.3(n)4.8(t)-7.1(a)8.7(r)3(y)4.8( )-7.5(rock)4.8(s)-8.4( )7.6(d)-10.2(e)1.2(posited in late P)-6.7(l)7.9(ioc)-6.3(e)8.7(n)-10.2(e)1.2( )7.6(t)-7.1(o)4.8( )-7.5(Qu)4.8(a)-6.3(t)7.9(ern)4.8(a)-6.3(ry)4.8( tim)19.7(e, in deposition)-10.2(a)8.7(l sy)4.8(st)-7.1(em)19.7(s)6.6( )-7.5(re)8.7(fl)-7.1(ec)8.7(t)-7.1(e)1.2(d)]TJ",
b"50.7958 0 TD",
b"0.0012 Tc",
b"-0.0033 Tw",
b"[( in)5.5( )-7.5(t)8.6(h)-9.5(e m)20.4(oder)3.7(n)-2( )]TJ",
b"-50.7958 -1.1792 TD",
],
)
self.assertDictEqual(
stream_def.to_python(), {"Length": 1009, "Filter": "/FlateDecode"}
)