light testing for filters
This commit is contained in:
1 parent
6e43a92517
commit
e47475b95f
13 files changed
+238
-41
No files matched your search
@@ -6,7 +6,7 @@ from .decoders import (
|
||||
CCITTFaxDecode,
|
||||
FlateDecode,
|
||||
LZWDecode,
|
||||
RunLengthDecode,
|
||||
RunLengthDecode
|
||||
)
|
||||
from .types import PDFFilterDecoder
|
||||
|
||||
@@ -47,13 +47,10 @@ def process_filters_on_data(
|
||||
bytes
|
||||
"""
|
||||
if filters is not None:
|
||||
decode_parms = []
|
||||
if decode_params is not None:
|
||||
decode_parms = decode_params
|
||||
for f in filters:
|
||||
params = {}
|
||||
if len(decode_parms) > 0:
|
||||
params = decode_parms.pop()
|
||||
if decode_params is None:
|
||||
decode_params = [{}] * len(filters)
|
||||
for i, f in enumerate(filters):
|
||||
params = decode_params[i]
|
||||
decoder = detect_decoder_type(f)
|
||||
data = decoder.decode(data, **params)
|
||||
return data
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
|
||||
|
||||
import array
|
||||
from typing import Optional
|
||||
|
||||
from .exceptions import CCITEOFB, CCITByteSkip, CCITInvalidData
|
||||
|
||||
@@ -337,20 +336,20 @@ class CCITTG4Parser(BitParser):
|
||||
break
|
||||
return
|
||||
|
||||
def _parse_mode(self, mode):
|
||||
def _parse_mode(self, mode: str):
|
||||
if mode == "p":
|
||||
self._do_pass()
|
||||
self._flush_line()
|
||||
return self.MODE
|
||||
elif mode == "h":
|
||||
self._n1 = 0
|
||||
self._accept = self._parse_horiz1
|
||||
self._accept = self._parse_horiz1 # type: ignore
|
||||
if self._color:
|
||||
return self.WHITE
|
||||
else:
|
||||
return self.BLACK
|
||||
elif mode == "u":
|
||||
self._accept = self._parse_uncompressed
|
||||
self._accept = self._parse_uncompressed # type: ignore
|
||||
return self.UNCOMPRESSED
|
||||
elif mode == "e":
|
||||
raise CCITEOFB
|
||||
@@ -552,10 +551,14 @@ class CCITTFaxDecoder(CCITTG4Parser):
|
||||
|
||||
def ccittfaxdecode(
|
||||
data: bytes,
|
||||
Columns: int,
|
||||
# arguments for algo, with defaults based on PDF spec 1.7 section 7.4.6
|
||||
Columns: int = 1728,
|
||||
EncodedByteAlign: bool = False,
|
||||
BlackIs1: bool = False,
|
||||
K: Optional[int] = None,
|
||||
K: int = 0,
|
||||
EndOfLine: bool = False,
|
||||
Rows: int = 0,
|
||||
DamagedRowsBeforeError: int = 0,
|
||||
) -> bytes:
|
||||
if K == -1:
|
||||
parser = CCITTFaxDecoder(Columns, bytealign=EncodedByteAlign, reversed=BlackIs1)
|
||||
|
||||
@@ -13,7 +13,7 @@ from pdfmajor.lexer.token import (
|
||||
Token,
|
||||
TokenArray,
|
||||
TokenDictionary,
|
||||
TokenKeyword,
|
||||
TokenKeyword
|
||||
)
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from pdfmajor.lexer.token import (
|
||||
TokenArray,
|
||||
TokenComment,
|
||||
TokenDictionary,
|
||||
TokenKeyword,
|
||||
TokenKeyword
|
||||
)
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
|
||||
@@ -17,14 +17,14 @@ from .objects import (
|
||||
PDFComment,
|
||||
PDFDictionary,
|
||||
PDFObject,
|
||||
get_obj_from_token_primitive,
|
||||
get_obj_from_token_primitive
|
||||
)
|
||||
from .parsers import (
|
||||
deal_with_collection_object,
|
||||
on_endobj,
|
||||
on_indirect_ref_close,
|
||||
on_primitive,
|
||||
on_stream,
|
||||
on_stream
|
||||
)
|
||||
from .state import ParsingState
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from .primitives import (
|
||||
PDFPrimitiveObject,
|
||||
PDFReal,
|
||||
PDFString,
|
||||
get_obj_from_token_primitive,
|
||||
get_obj_from_token_primitive
|
||||
)
|
||||
from .ref import ObjectRef
|
||||
from .stream import PDFStream
|
||||
|
||||
@@ -5,7 +5,7 @@ from pdfmajor.util import get_single_or_list
|
||||
|
||||
from .base import PDFObject
|
||||
from .collections import PDFDictionary
|
||||
from .primitives import PDFInteger
|
||||
from .primitives import PDFInteger, PDFNull
|
||||
from .ref import ObjectRef
|
||||
from .util import to_python, to_python_list
|
||||
|
||||
@@ -52,13 +52,21 @@ class PDFStream:
|
||||
raise IncompleteStream(
|
||||
f"Invalid length property {item} {type(item['Length'])}"
|
||||
)
|
||||
filters = get_single_or_list(item.get("Filter", None))
|
||||
decode_parms = get_single_or_list(item.get("DecodeParms", None))
|
||||
if len(decode_parms) == 0:
|
||||
decode_parms = [PDFNull()] * len(filters)
|
||||
ffilter = get_single_or_list(item.get("FFilter", None))
|
||||
fdecode_parms = get_single_or_list(item.get("FDecodeParms", None))
|
||||
if len(fdecode_parms) == 0:
|
||||
fdecode_parms = [PDFNull()] * len(ffilter)
|
||||
stream = cls(
|
||||
offset,
|
||||
item["Length"],
|
||||
filter=get_single_or_list(item.get("Filter", None)),
|
||||
decode_parms=get_single_or_list(item.get("DecodeParms", None)),
|
||||
ffilter=get_single_or_list(item.get("FFilter", None)),
|
||||
fdecode_parms=get_single_or_list(item.get("FDecodeParms", None)),
|
||||
filter=filters,
|
||||
decode_parms=decode_parms,
|
||||
ffilter=ffilter,
|
||||
fdecode_parms=fdecode_parms,
|
||||
f=item.get("F", None),
|
||||
dl=item.get("DL", None),
|
||||
)
|
||||
|
||||
@@ -119,7 +119,7 @@ def on_stream(buffer: BufferStream, state: ParsingState, token: TokenKeyword):
|
||||
obj = last_ctx.get_object()
|
||||
if not isinstance(obj, PDFDictionary):
|
||||
raise ParserError(f"Cannot initilize a pdfstream with {type(obj)}")
|
||||
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 1, obj)
|
||||
last_ctx.stream = stream = PDFStream.from_pdfdict(token.end_loc + 2, obj)
|
||||
if isinstance(stream.length, PDFInteger):
|
||||
buffer.seek(stream.offset + stream.length.to_python())
|
||||
next_token = None
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
+90
-9
@@ -25,6 +25,7 @@ from pdfmajor.parser.objects import (
|
||||
PDFReal,
|
||||
PDFString,
|
||||
)
|
||||
from pdfmajor.parser.objects.primitives import PDFNull
|
||||
from tests.util import all_corrupt_pdf_files, all_pdf_files
|
||||
|
||||
|
||||
@@ -64,6 +65,85 @@ class ParsingState(TestCase):
|
||||
expcat.raw,
|
||||
)
|
||||
|
||||
def test_pdf_spec(self):
|
||||
self.run_test(
|
||||
"PDF_spec.1.7.pdf",
|
||||
PDFDocumentCatalog(
|
||||
version=None,
|
||||
pages=PDFPageTreeNode(
|
||||
kids=[
|
||||
ObjectRef(20, 0),
|
||||
ObjectRef(21, 0),
|
||||
ObjectRef(22, 0),
|
||||
ObjectRef(23, 0),
|
||||
ObjectRef(24, 0),
|
||||
ObjectRef(25, 0),
|
||||
ObjectRef(26, 0),
|
||||
ObjectRef(27, 0),
|
||||
],
|
||||
parent=None,
|
||||
leaft_count=756,
|
||||
raw=PDFDictionary.from_dict(
|
||||
{
|
||||
"Count": PDFInteger(756, 0, 0),
|
||||
"Kids": PDFArray.from_list(
|
||||
[
|
||||
ObjectRef(20, 0),
|
||||
ObjectRef(21, 0),
|
||||
ObjectRef(22, 0),
|
||||
ObjectRef(23, 0),
|
||||
ObjectRef(24, 0),
|
||||
ObjectRef(25, 0),
|
||||
ObjectRef(26, 0),
|
||||
ObjectRef(27, 0),
|
||||
]
|
||||
),
|
||||
"Type": PDFName("Pages", 0, 0),
|
||||
}
|
||||
),
|
||||
),
|
||||
page_labels=PDFDictionary.from_dict(
|
||||
{
|
||||
"Nums": PDFArray.from_list(
|
||||
[
|
||||
PDFInteger(0, 0, 0),
|
||||
ObjectRef(18, 0),
|
||||
PDFInteger(8, 0, 0),
|
||||
ObjectRef(19, 0),
|
||||
]
|
||||
)
|
||||
}
|
||||
),
|
||||
page_layout=None,
|
||||
page_mode=PDFName("UseOutlines", 0, 0),
|
||||
metadata=PDFMetadata(sub_type="XML", stream_data=b""),
|
||||
raw=PDFDictionary.from_dict(
|
||||
{
|
||||
"MarkInfo": PDFDictionary.from_dict(
|
||||
{"Marked": PDFBoolean(True, 0, 0)}
|
||||
),
|
||||
"Metadata": ObjectRef(2, 0),
|
||||
"Names": ObjectRef(3, 0),
|
||||
"OpenAction": PDFArray.from_list(
|
||||
[
|
||||
ObjectRef(4, 0),
|
||||
PDFName("XYZ", 0, 0),
|
||||
PDFNull(),
|
||||
PDFNull(),
|
||||
PDFNull(),
|
||||
]
|
||||
),
|
||||
"Outlines": ObjectRef(5, 0),
|
||||
"PageLabels": ObjectRef(6, 0),
|
||||
"PageMode": PDFName("UseOutlines", 0, 0),
|
||||
"Pages": ObjectRef(7, 0),
|
||||
"StructTreeRoot": ObjectRef(8, 0),
|
||||
"Type": PDFName("Catalog", 0, 0),
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def test_bad_unicode(self):
|
||||
self.run_test(
|
||||
"bad-unicode.pdf",
|
||||
@@ -152,15 +232,16 @@ class ParsingState(TestCase):
|
||||
|
||||
def test_all(self):
|
||||
for file_path, buffer in all_pdf_files.items():
|
||||
with self.subTest(file_path), buffer.get_window():
|
||||
parser = PDFParsingContext(buffer)
|
||||
catalog = get_catalog(parser)
|
||||
self.assertEqual(len(parser.health_report), 0)
|
||||
get_info(parser)
|
||||
list(iter_all_page_leafs(parser, catalog.pages))
|
||||
parser = PDFDocument(buffer)
|
||||
actual_pages = len(list(parser.iter_pages()))
|
||||
self.assertEqual(actual_pages, parser.num_pages)
|
||||
if file_path != "PDF_spec.1.7.pdf":
|
||||
with self.subTest(file_path), buffer.get_window():
|
||||
parser = PDFParsingContext(buffer)
|
||||
catalog = get_catalog(parser)
|
||||
self.assertEqual(len(parser.health_report), 0)
|
||||
get_info(parser)
|
||||
list(iter_all_page_leafs(parser, catalog.pages))
|
||||
parser = PDFDocument(buffer)
|
||||
actual_pages = len(list(parser.iter_pages()))
|
||||
self.assertEqual(actual_pages, parser.num_pages)
|
||||
|
||||
def test_metadata(self):
|
||||
with all_pdf_files["bad-unicode.pdf"].get_window() as buffer:
|
||||
|
||||
@@ -207,7 +207,8 @@ class IndirectObjects(TestCase):
|
||||
<< /Type /XRef /Length 123 /Filter /FlateDecode /otherfeature [10 20]>>
|
||||
stream
|
||||
"<99><A3><C1>l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567890l01234567
|
||||
<80>endstream
|
||||
<80>
|
||||
endstream
|
||||
endobj
|
||||
"""
|
||||
)
|
||||
@@ -218,10 +219,10 @@ endobj
|
||||
self.assertDictEqual(
|
||||
obj.stream.to_python(), # type: ignore
|
||||
{
|
||||
"offset": 100,
|
||||
"offset": 101,
|
||||
"length": 123,
|
||||
"filter": ["/FlateDecode"],
|
||||
"decode_parms": [],
|
||||
"decode_parms": [None],
|
||||
"f": None,
|
||||
"ffilter": [],
|
||||
"fdecode_parms": [],
|
||||
@@ -256,10 +257,10 @@ endobj
|
||||
self.assertDictEqual(
|
||||
obj.stream.to_python(), # type: ignore
|
||||
{
|
||||
"offset": 100,
|
||||
"length": 123 + 5,
|
||||
"offset": 101,
|
||||
"length": 123 + 4,
|
||||
"filter": ["/FlateDecode"],
|
||||
"decode_parms": [],
|
||||
"decode_parms": [None],
|
||||
"f": None,
|
||||
"ffilter": [],
|
||||
"fdecode_parms": [],
|
||||
@@ -270,7 +271,7 @@ endobj
|
||||
obj.get_object().to_python(),
|
||||
{
|
||||
"Filter": "/FlateDecode",
|
||||
"Length": 123 + 5,
|
||||
"Length": 123 + 4,
|
||||
"Type": "/XRef",
|
||||
"otherfeature": [10, 20],
|
||||
},
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.document.parse_context import PDFParsingContext
|
||||
from pdfmajor.filters.ccit import CCITTG4Parser
|
||||
from pdfmajor.filters.lzw import lzwdecode
|
||||
from pdfmajor.filters.rld import rldecode
|
||||
from pdfmajor.parser.objects.ref import ObjectRef
|
||||
from tests.util import all_pdf_files
|
||||
|
||||
|
||||
class Filters(TestCase):
|
||||
@@ -168,3 +171,107 @@ class CCIT(TestCase):
|
||||
parser._do_vertical(1)
|
||||
self.assertEqual(parser._get_bits(), "00000001")
|
||||
return
|
||||
|
||||
|
||||
class Stream(TestCase):
|
||||
def test_flatedecode(self):
|
||||
with all_pdf_files["bad-unicode.pdf"].get_window() as buffer:
|
||||
pctx = PDFParsingContext(buffer)
|
||||
stream_def, stream_data = pctx.validated_and_iter_stream(ObjectRef(161, 0))
|
||||
self.assertListEqual(
|
||||
stream_data.strip().split(b"\n"),
|
||||
[
|
||||
b"/GS1 gs",
|
||||
b"BT",
|
||||
b"/TT2 1 Tf",
|
||||
b"9.9856 0 0 9.9856 39.9 724.5601 Tm",
|
||||
b"0 g",
|
||||
b"0.0022 Tc",
|
||||
b"-0.0036 Tw",
|
||||
b"[(Map Un)-5.9(i)9.6(t)-1.3( Expla)5.6(n)-5.9(ation)6.2( )]TJ",
|
||||
b"7.9885 0 0 7.9885 171.96 723.48 Tm",
|
||||
b"-0.0019 Tc",
|
||||
b"0.0095 Tw",
|
||||
b"[(Legend: Map )15(Un)-6.9(its )7.5(for Tonto )7.5(National)-9.5( )15(Fore)-9.1(s)6(t G)9.8(e)-1.5(ol)-9.5(ogy)]TJ",
|
||||
b"/TT4 1 Tf",
|
||||
b"9.9856 0 0 9.9856 90.9 703.62 Tm",
|
||||
b"0.0031 Tc",
|
||||
b"0 Tw",
|
||||
b"(Water)Tj",
|
||||
b"/TT6 1 Tf",
|
||||
b"7.9885 0 0 7.9885 108.96 690.48 Tm",
|
||||
b"-0.0029 Tc",
|
||||
b"[(Wat)-3(e)-2.2(r)]TJ",
|
||||
b"ET",
|
||||
b"/Cs6 cs 0 0.74902 1 scn",
|
||||
b"42.48 693.66 44.04 19.98 re",
|
||||
b"f",
|
||||
b"BT",
|
||||
b"/TT4 1 Tf",
|
||||
b"13.9798 0 0 13.9798 59.52 700.26 Tm",
|
||||
b"0 g",
|
||||
b"0 Tc",
|
||||
b"(w)Tj",
|
||||
b"ET",
|
||||
b"41.52 713.64 46.02 0.95996 re",
|
||||
b"f",
|
||||
b"86.52 692.64 1.02 21.96 re",
|
||||
b"f",
|
||||
b"41.52 692.64 46.02 1.02 re",
|
||||
b"f",
|
||||
b"41.52 692.64 0.95999 21.96 re",
|
||||
b"f",
|
||||
b"BT",
|
||||
b"9.9856 0 0 9.9856 90.9 671.28 Tm",
|
||||
b"0.0027 Tc",
|
||||
b"-0.0063 Tw",
|
||||
b"[(Dist)-6.8(u)6.1(r)-4.1(be)7.9(d)-6( g)-8(r)1.9(ound )-6(\\(rece)7.9(n)0(t\\))]TJ",
|
||||
b"/TT6 1 Tf",
|
||||
b"7.9885 0 0 7.9885 108.96 658.2001 Tm",
|
||||
b"0.0031 Tc",
|
||||
b"-0.0052 Tw",
|
||||
b"[(H)11.7(e)-3.7(avily)7.4( distur)5.6(bed gr)5.6(ound)-7.6( )7.6(d)-7.6(u)7.4(e to)7.4( )-7.5(ag)7.4(ri)10.5(cu)7.4(lt)10.5(u)-7.6(r)13.1(e,)5.2( )7.6(exte)]TJ",
|
||||
b"19.7159 0 TD",
|
||||
b"0.002 Tc",
|
||||
b"0.0034 Tw",
|
||||
b"[(nsive )7.5(excavation,)4.1( o)6.3(r)4.5( )7.5(co)-8.7(n)6.3(s)0.6(tr)4.5(uct)9.4(i)-5.6(on)6.3( o)6.3(f)-3( ear)12(th )7.5(dam)21.2(s)]TJ",
|
||||
b"ET",
|
||||
b"0.82 g",
|
||||
b"42.48 661.32 44.04 19.98 re",
|
||||
b"f",
|
||||
b"BT",
|
||||
b"/TT4 1 Tf",
|
||||
b"13.9798 0 0 13.9798 60.6 667.92 Tm",
|
||||
b"0 g",
|
||||
b"0 Tc",
|
||||
b"0 Tw",
|
||||
b"(d)Tj",
|
||||
b"ET",
|
||||
b"41.52 681.3 46.02 1.02 re",
|
||||
b"f",
|
||||
b"86.52 660.36 1.02 21.96 re",
|
||||
b"f",
|
||||
b"41.52 660.36 46.02 0.96002 re",
|
||||
b"f",
|
||||
b"41.52 660.36 0.95999 21.96 re",
|
||||
b"f",
|
||||
b"BT",
|
||||
b"9.9856 0 0 9.9856 90.9 639 Tm",
|
||||
b"0.0018 Tc",
|
||||
b"0.0006 Tw",
|
||||
b"[(Q)-7.5(u)-0.9(at)-7.7(e)7(r)-5(nar)7(y)-8.9( sediment)-7.7(ary dep)5.2(o)-2.9(sit)-7.7(s)]TJ",
|
||||
b"/TT6 1 Tf",
|
||||
b"7.9885 0 0 7.9885 108.96 625.92 Tm",
|
||||
b"0.0005 Tc",
|
||||
b"-0.0026 Tw",
|
||||
b"[(T)10.5(y)4.8(p)-10.2(i)0.4(cally)4.8( non m)19.7(a)1.2(rine clas)-8.4(t)7.9(i)0.4(c sedi)-7.1(m)27.2(e)-6.3(n)4.8(t)-7.1(a)8.7(r)3(y)4.8( )-7.5(rock)4.8(s)-8.4( )7.6(d)-10.2(e)1.2(posited in late P)-6.7(l)7.9(ioc)-6.3(e)8.7(n)-10.2(e)1.2( )7.6(t)-7.1(o)4.8( )-7.5(Qu)4.8(a)-6.3(t)7.9(ern)4.8(a)-6.3(ry)4.8( tim)19.7(e, in deposition)-10.2(a)8.7(l sy)4.8(st)-7.1(em)19.7(s)6.6( )-7.5(re)8.7(fl)-7.1(ec)8.7(t)-7.1(e)1.2(d)]TJ",
|
||||
b"50.7958 0 TD",
|
||||
b"0.0012 Tc",
|
||||
b"-0.0033 Tw",
|
||||
b"[( in)5.5( )-7.5(t)8.6(h)-9.5(e m)20.4(oder)3.7(n)-2( )]TJ",
|
||||
b"-50.7958 -1.1792 TD",
|
||||
],
|
||||
)
|
||||
self.assertDictEqual(
|
||||
stream_def.to_python(), {"Length": 1009, "Filter": "/FlateDecode"}
|
||||
)
|
||||
Reference in new issue
Block a user