hardcore linting

This commit is contained in:
ari committed 2020-11-03 20:19:11 -05:00
1 parent 14df0a1600
commit 0ba835d577
24 files changed
+4659 -97

No files matched your search

+5
View File
@@ -0,0 +1,5 @@
[mypy]
python_version = 3.7
warn_return_any = True
warn_unused_configs = True
+14 -3
View File
@@ -2,7 +2,7 @@ from pathlib import Path
from typing import Any, BinaryIO, Dict, Iterator, Optional, Tuple, Type, TypeVar, Union
from pdfmajor.document.exceptions import BrokenFilePDF
from pdfmajor.document.parsers.stream import decode_stream
from pdfmajor.document.parsers.stream import decode_stream, iter_filters_in_stream
from pdfmajor.document.structures import PDFRectangle
from pdfmajor.healthlog import PDFHealthReport
from pdfmajor.pdf_parser.objects import (
@@ -212,7 +212,7 @@ class PDFParsingContext:
)
)
obj_def = PDFDictionary()
return obj_def, decode_stream(obj.stream, self.buffer)
return obj_def, self.decode_stream(obj.stream)
def decode_stream(self, stream: PDFStream) -> bytes:
"""Decode the bytes associated with the PDF-Stream
@@ -223,4 +223,15 @@ class PDFParsingContext:
Returns:
bytes
"""
return decode_stream(stream, self.buffer)
offset = stream.offset
length_ref = stream.length
if isinstance(length_ref, ObjectRef):
length_obj = self.get_object_from_ref(length_ref).get_object()
if not isinstance(length_obj, PDFInteger):
raise BrokenFilePDF(f"Length should be an int not {length_obj}")
else:
length_obj = length_ref
length = length_obj.value
return decode_stream(
offset, length, iter_filters_in_stream(stream), self.buffer
)
+3 -2
View File
@@ -7,11 +7,12 @@ from ..parse_context import PDFParsingContext
def get_metadata(pctx: PDFParsingContext, ref: PDFObject):
obj_def, data = pctx.validated_and_iter_stream(ref)
if not isinstance(obj_def["Subtype"], PDFName):
subtype = obj_def["Subtype"]
if not isinstance(subtype, PDFName):
pctx.health_report.write_error(
BrokenFilePDF(
f"Metadata data object is missing a valid subtype {ref} {obj_def}"
)
)
return PDFMetadata("", data)
return PDFMetadata(obj_def["Subtype"].value, data)
return PDFMetadata(subtype.value, data)
+1 -1
View File
@@ -187,7 +187,7 @@ def _deal_with_inheritance(
field_dict: Dict[str, Optional[PDFObject]],
lastpage: Optional[PDFRawFields],
allow_missing: bool = False,
):
) -> Dict[str, Any]:
_deal_with_inheritance_for_field("resources", field_dict, lastpage, PDFDictionary())
_deal_with_inheritance_for_field(
"rotate", field_dict, lastpage, PDFInteger(0, 0, 0)
+32 -19
View File
@@ -1,30 +1,24 @@
from itertools import chain
from typing import Any, Dict, List
from typing import Any, Dict, Iterator
from pdfmajor.filters import process_filters_on_data
from pdfmajor.filters import FilterPair, process_filters_on_data
from pdfmajor.pdf_parser.exceptions import IncompleteStream
from pdfmajor.pdf_parser.objects import PDFDictionary, PDFName, PDFStream
from pdfmajor.streambuffer import BufferStream
def decode_stream(stream: PDFStream, buffer: BufferStream) -> bytes:
"""Decode the bytes associated with the PDF-Stream
def iter_filters_in_stream(stream: PDFStream) -> Iterator[FilterPair]:
"""loops over all filters and their params in the stream
Args:
stream (PDFStream)
buffer (BufferStream)
Returns:
bytes
Raises:
IncompleteStream: if a filter-name is invalid
Yields:
Iterator[FilterPair]
"""
length = stream.length.to_python()
if length is None:
raise IncompleteStream(f"Cannot decode stream as it has no specified length")
with buffer.get_window():
buffer.seek(stream.offset)
data = buffer.read(length).data
filters: List[str] = []
decode_parms: List[Dict[str, Any]] = []
for f, dp in chain(
zip(stream.filter, stream.decode_parms),
zip(stream.ffilter, stream.fdecode_parms),
@@ -33,10 +27,29 @@ def decode_stream(stream: PDFStream, buffer: BufferStream) -> bytes:
raise IncompleteStream(f"Invalid filter name {f}")
else:
filter_name = f.value
params = {}
params: Dict[str, Any] = {}
if isinstance(dp, PDFDictionary):
params = dp.to_python()
filters.append(filter_name)
decode_parms.append(params)
data = process_filters_on_data(data, filters, decode_parms)
yield filter_name, params
def decode_stream(
offset: int, length: int, it_filters: Iterator[FilterPair], buffer: BufferStream
) -> bytes:
"""Decode the bytes associated with the PDF-Stream
Args:
offset (int)
length (int)
it_filters (Iterator[FilterPair])
buffer (BufferStream)
Returns:
bytes
"""
with buffer.get_window():
buffer.seek(offset)
data = buffer.read(length).data
data = process_filters_on_data(data, it_filters)
return data
+6 -11
View File
@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Optional, Type
from typing import Any, Dict, Iterator, List, Optional, Type
from .decoders import (
ASCII85Decode,
@@ -8,7 +8,7 @@ from .decoders import (
LZWDecode,
RunLengthDecode
)
from .types import PDFFilterDecoder
from .types import FilterPair, PDFFilterDecoder
_filter_type_name_mapping: Dict[str, Optional[Type[PDFFilterDecoder]]] = {
"FlateDecode": FlateDecode,
@@ -33,8 +33,7 @@ _filter_type_name_mapping: Dict[str, Optional[Type[PDFFilterDecoder]]] = {
def process_filters_on_data(
data: bytes,
filters: Optional[List[str]],
decode_params: Optional[List[Dict[str, Any]]],
it_filters: Iterator[FilterPair],
) -> bytes:
"""runs the filters provided by the list with the provided params against the data
@@ -46,13 +45,9 @@ def process_filters_on_data(
Returns:
bytes
"""
if filters is not None:
if decode_params is None:
decode_params = [{}] * len(filters)
for i, f in enumerate(filters):
params = decode_params[i]
decoder = detect_decoder_type(f)
data = decoder.decode(data, **params)
for f, params in it_filters:
decoder = detect_decoder_type(f)
data = decoder.decode(data, **params)
return data
+3 -3
View File
@@ -28,7 +28,7 @@ class BitParser(object):
return
@classmethod
def add(klass, root, v, bits):
def add(cls, root, v, bits):
p = root
b = None
for i in range(len(bits)):
@@ -532,10 +532,10 @@ class CCITTFaxDecoder(CCITTG4Parser):
def __init__(self, width, bytealign=False, reversed=False):
CCITTG4Parser.__init__(self, width, bytealign=bytealign)
self.reversed = reversed
self._buf = b""
self._buf: bytes = b""
return
def close(self):
def close(self) -> bytes:
return self._buf
def output_line(self, y, bits):
+6 -6
View File
@@ -12,7 +12,7 @@ class LZWDecoder(object):
self.buff: int = 0
self.bpos: int = 8
self.nbits: int = 9
self.table: Optional[List[bytes]] = None
self.table: Optional[List[Optional[bytes]]] = None
self.prevbuf: Optional[bytes] = None
return
@@ -54,15 +54,15 @@ class LZWDecoder(object):
elif not self.prevbuf:
x = self.prevbuf = self.table[code]
else:
if code < len(self.table):
if code < len(self.table): # type: ignore
x = self.table[code]
self.table.append(self.prevbuf + x[:1])
elif code == len(self.table):
elif code == len(self.table): # type: ignore
self.table.append(self.prevbuf + self.prevbuf[:1])
x = self.table[code]
else:
raise DecodeFailed("LZW")
l = len(self.table)
l = len(self.table) # type: ignore
if l == 511:
self.nbits = 10
elif l == 1023:
@@ -84,7 +84,7 @@ class LZWDecoder(object):
# lzwdecode
def lzwdecode(data: bytes):
def lzwdecode(data: bytes) -> bytes:
fp = BytesIO(data)
s = LZWDecoder(fp).run()
return b"".join(s)
return b"".join(s) # type: ignore
+4 -1
View File
@@ -1,5 +1,5 @@
from abc import ABCMeta
from typing import Any
from typing import Any, Dict, Tuple
from .exceptions import InvalidDecoderOrNotImplemented
@@ -8,3 +8,6 @@ class PDFFilterDecoder(metaclass=ABCMeta):
@staticmethod
def decode(data: bytes, **decode_params: Any) -> bytes:
raise InvalidDecoderOrNotImplemented
FilterPair = Tuple[str, Dict[str, Any]]
+27
View File
@@ -0,0 +1,27 @@
from abc import ABCMeta, abstractclassmethod
from typing import Any, Dict, Literal, Union
FontType = Union[
Literal["Type0"],
Literal["Type1"],
Literal["MMType1"],
Literal["Type3"],
Literal["TrueType"],
Literal["CIDFontType0"],
Literal["CIDFontType2"],
]
class PDFFont(metaclass=ABCMeta):
"""an object reprsenting a PDF Font as decribed in PDF Spec 1.7 section 9.5"""
def __init__(self, subtype: FontType, **kwrg: Any) -> None:
self.subtype = subtype
self.raw = kwrg
def __repr__(self) -> str:
return f"PDFFont:{self.subtype}({self.raw})"
@abstractclassmethod
def from_dict(cls, d: Dict[str, Any]) -> "PDFFont":
raise NotImplementedError
File diff suppressed because it is too large. Load diff
+43
View File
@@ -0,0 +1,43 @@
from typing import Any, Dict, Union
from .base import PDFFont
class PDFType1(PDFFont):
def __init__(
self,
base_font: str,
widths: Dict[bytes, int],
font_descriptor: Dict[str, int],
encoding: Union[str, Dict[str, Any], None],
to_unicode: Any,
) -> None:
PDFFont.__init__(
self,
"Type1",
base_font=base_font,
widths=widths,
font_descriptor=font_descriptor,
encoding=encoding,
to_unicode=to_unicode,
)
# def from_dict(cls, d: Dict[str, Any]) -> "PDFType1":
# base_font = str(d["BaseFont"])
# if base_font in STANDARD14_FONT_METRICS.keys():
# font_descriptor, widths_raw = STANDARD14_FONT_METRICS[base_font]
# widths = {c.encode(): w for c, w in widths_raw.items()}
# else:
# first_char = int(d.get("FirstChar", 0))
# last_char = int(d.get("FirstChar", 255))
# if d.get("Widths"):
# widthts_array = map(int, d["Widths"])
# else:
# widthts_array = [0] * (last_char - first_char + 1)
# font_descriptor = dict(d["FontDescriptor"])
# widths = {
# bytes([(i + first_char)]): w for (i, w) in enumerate(widthts_array)
# }
# encoding = d.get("Encoding", None)
# to_unicode = d.get("ToUnicode", None)
# return cls(base_font, widths, font_descriptor, encoding, to_unicode)
+13
View File
@@ -0,0 +1,13 @@
from typing import Any, Dict, List, NamedTuple, Union
class PDFResources(NamedTuple):
"""A class storing PDF Resources as described in PDF spec 1.7. section 7.8.3"""
extgstate: Dict[str, Any]
color_space: Dict[str, Union[str, List[str]]]
pattern: Dict[str, Any]
shading: Dict[str, Any]
xobject: Dict[str, Any]
font: Dict[str, Any]
procset: Dict[str, Any]
+1 -1
View File
@@ -16,7 +16,7 @@ class PDFObject(metaclass=ABCMeta):
def __eq__(self, o: object) -> bool:
if not isinstance(o, self.__class__):
return False
return o.to_python() == self.to_python()
return o.to_python() == self.to_python() # type: ignore
def __repr__(self) -> str:
return "{cls}({data})".format(
+1 -1
View File
@@ -46,7 +46,7 @@ class PDFDictionary(Dict[str, PDFObject], PDFContextualObject):
@property
def last_item_value(self):
return self[self.last_name]
return self[self.last_name.value]
def pass_item(self, item: PDFObject):
if self.name is None:
+7 -4
View File
@@ -1,6 +1,6 @@
from typing import Any, List, Optional, Union
from pdfmajor.pdf_parser.exceptions import IncompleteStream
from pdfmajor.pdf_parser.exceptions import BrokenFileParserError, IncompleteStream
from pdfmajor.util import get_single_or_list
from .base import PDFObject
@@ -63,13 +63,16 @@ class PDFStream:
fdecode_parms = get_single_or_list(item.get("FDecodeParms", None))
if len(fdecode_parms) == 0:
fdecode_parms = [PDFNull()] * len(ffilter)
length = item["Length"]
if not isinstance(length, PDFInteger) and not isinstance(length, ObjectRef):
raise BrokenFileParserError(f"length has an invalid type {length}")
stream = cls(
offset,
item["Length"],
length,
filter=filters,
decode_parms=decode_parms,
decode_parms=decode_parms, # type: ignore
ffilter=ffilter,
fdecode_parms=fdecode_parms,
fdecode_parms=fdecode_parms, # type: ignore
f=item.get("F", None),
dl=item.get("DL", None),
)
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import List, NamedTuple, Optional, Set
from typing import List, NamedTuple, Optional, Set, Union
from pdfmajor.lexer import iter_tokens
from pdfmajor.lexer.token import TokenKeyword
@@ -46,7 +46,7 @@ class PDFFileTrailer(NamedTuple):
size: PDFInteger
root: Optional[ObjectRef]
prev: Optional[PDFInteger]
prev: Optional[Union[PDFInteger, ObjectRef]]
info: Optional[ObjectRef]
encrypt: Optional[PDFObject]
encrypt_id: Optional[PDFArray]
+3 -3
View File
@@ -1,4 +1,4 @@
from typing import Iterator, Literal, NamedTuple, Union
from typing import Iterator, NamedTuple
from pdfmajor.lexer import iter_tokens
from pdfmajor.lexer.token import TokenInteger, TokenKeyword
@@ -15,7 +15,7 @@ class XRefRow(NamedTuple):
offset: int
obj_num: int
gen_num: int
use: Union[Literal["n"], Literal["f"]]
use: bytes
def find_start_of_xref(buffer: BufferStream, strict: bool = False) -> int:
@@ -123,7 +123,7 @@ def iter_over_standard_xref(
raise PDFNoValidXRef(
f"Invalid 'use' value for xref {use!r}: pos={pos} line={line!r}"
)
yield XRefRow(offset=pos, obj_num=obj_num, gen_num=gen_num, use=use) # type: ignore
yield XRefRow(offset=pos, obj_num=obj_num, gen_num=gen_num, use=use)
# def construct_indobj_collection(
+1 -1
View File
@@ -43,7 +43,7 @@ class XRefDB:
elif isinstance(trailer.prev, ObjectRef):
obj = self.get_obj(
trailer.prev.obj_num, trailer.prev.gen_num, buffer=buffer
)
).get_object()
if not isinstance(obj, PDFInteger):
raise BrokenFile(
f"trailer had an invalid prev options {trailer.prev}"
+1 -1
View File
@@ -12,7 +12,7 @@ with open("README.md") as readme_file:
"name": "pdfmajor",
"version": "1.3.13",
"packages": find_packages(include=["pdfmajor*"]),
"install_requires": ["pycryptodome==3.8.2", "chardet==3.0.4", "Pillow>=6.2.0"],
"install_requires": ["numpy==1.19.4", "typing-extensions==3.7.4.3"],
"extras_require": {
"dev": [
"tqdm==4.32.1",
+14 -6
View File
@@ -166,7 +166,13 @@ class ParsingState(TestCase):
}
)
),
page_labels={"Nums": [PDFInteger(0, 0, 0), ObjectRef(140, 0)]},
page_labels=PDFDictionary.from_dict(
{
"Nums": PDFArray.from_list(
[PDFInteger(0, 0, 0), ObjectRef(140, 0)]
)
}
),
page_layout=None,
page_mode=None,
metadata=PDFMetadata(
@@ -258,6 +264,8 @@ class ParsingState(TestCase):
parser = PDFDocument(buffer)
with warnings.catch_warnings(record=True) as w:
actual_pages = len(list(parser.iter_pages()))
if w is None:
raise AssertionError
self.assertEqual(len(w), 1)
self.assertNotEqual(actual_pages, parser.num_pages)
@@ -283,7 +291,7 @@ class Structures(TestCase):
[
PDFReal(Decimal("0.23"), 0, 0),
PDFInteger(10, 0, 0),
PDFInteger(Decimal("3.3"), 0, 0),
PDFReal(Decimal("3.3"), 0, 0),
PDFInteger(3, 0, 0),
],
PDFRectangle(Decimal("0.23"), Decimal(10), Decimal("3.3"), Decimal(3)),
@@ -294,10 +302,10 @@ class Structures(TestCase):
PDFRectangle.from_pdfarray(
PDFArray.from_list(
[
PDFString("h0", 0, 0),
PDFString("h0", 0, 0),
PDFString("h0", 0, 0),
PDFString("h0", 0, 0),
PDFString(b"h0", 0, 0),
PDFString(b"h0", 0, 0),
PDFString(b"h0", 0, 0),
PDFString(b"h0", 0, 0),
]
)
)
+4 -4
View File
@@ -214,7 +214,7 @@ endobj
)
for obj in iter_objects(buffer):
self.assertIsInstance(obj, IndirectObject)
self.assertIsInstance(obj.get_object(), PDFDictionary)
self.assertIsInstance(obj.get_object(), PDFDictionary) # type: ignore
self.assertIsInstance(obj.stream, PDFStream) # type: ignore
self.assertDictEqual(
obj.stream.to_python(), # type: ignore
@@ -230,7 +230,7 @@ endobj
},
)
self.assertDictEqual(
obj.get_object().to_python(),
obj.get_object().to_python(), # type: ignore
{
"Filter": "/FlateDecode",
"Length": 123,
@@ -252,7 +252,7 @@ endobj
)
for obj in iter_objects(buffer):
self.assertIsInstance(obj, IndirectObject)
self.assertIsInstance(obj.get_object(), PDFDictionary)
self.assertIsInstance(obj.get_object(), PDFDictionary) # type: ignore
self.assertIsInstance(obj.stream, PDFStream) # type: ignore
self.assertDictEqual(
obj.stream.to_python(), # type: ignore
@@ -268,7 +268,7 @@ endobj
},
)
self.assertDictEqual(
obj.get_object().to_python(),
obj.get_object().to_python(), # type: ignore
{
"Filter": "/FlateDecode",
"Length": 123 + 4,
+1 -1
View File
@@ -24,7 +24,7 @@ class Filters(TestCase):
class CCIT(TestCase):
def get_parser(self, bits):
parser = CCITTG4Parser(len(bits))
parser._curline = [int(c) for c in bits]
parser._curline = [int(c) for c in bits] # type: ignore
parser._reset_line()
return parser
+62 -27
View File
@@ -6,20 +6,56 @@ Converts PDF text content (though not images containing text) to plain text, htm
import argparse
import logging
import sys
from pdfmajor.converters import convert_file
from pdfmajor.converters import convert_file # type: ignore
def make_argparser():
parser = argparse.ArgumentParser(description=__doc__, add_help=True)
parser.add_argument("files", type=str, default=None, nargs="+", help="File to process.")
parser.add_argument("-d", "--debug", default=False, action="store_true", help="Debug output.")
parser.add_argument("-p", "--pagenos", type=str, help="Comma-separated list of page numbers to parse. Included for legacy applications, use --page-numbers for more idiomatic argument entry.")
parser.add_argument("-m", "--maxpages", type=int, default=0, help="Maximum pages to parse")
parser.add_argument("-P", "--password", type=str, default=None, help="Decryption password for PDF")
parser.add_argument("-c", "--codec", type=str, default="utf-8", help="Text encoding")
parser.add_argument("-t", "--output-type", type=str, default="text", help="Output type: text|html|xml|tag (default is text)")
parser.add_argument("-o", "--output-file", type=str, default="-", help="Output file (default \"-\" is stdout)")
parser.add_argument("-O", "--output-dir", default=None, help="Output directory for images")
parser.add_argument("-C", "--disable-caching", default=False, action="store_true", help="Disable caching")
parser.add_argument(
"files", type=str, default=None, nargs="+", help="File to process."
)
parser.add_argument(
"-d", "--debug", default=False, action="store_true", help="Debug output."
)
parser.add_argument(
"-p",
"--pagenos",
type=str,
help="Comma-separated list of page numbers to parse. Included for legacy applications, use --page-numbers for more idiomatic argument entry.",
)
parser.add_argument(
"-m", "--maxpages", type=int, default=0, help="Maximum pages to parse"
)
parser.add_argument(
"-P", "--password", type=str, default=None, help="Decryption password for PDF"
)
parser.add_argument(
"-c", "--codec", type=str, default="utf-8", help="Text encoding"
)
parser.add_argument(
"-t",
"--output-type",
type=str,
default="text",
help="Output type: text|html|xml|tag (default is text)",
)
parser.add_argument(
"-o",
"--output-file",
type=str,
default="-",
help='Output file (default "-" is stdout)',
)
parser.add_argument(
"-O", "--output-dir", default=None, help="Output directory for images"
)
parser.add_argument(
"-C",
"--disable-caching",
default=False,
action="store_true",
help="Disable caching",
)
return parser
@@ -32,35 +68,34 @@ def main(args=None):
parsed_args = argparser.parse_args(args=args)
func_args = {
'maxpages': parsed_args.maxpages,
'password': parsed_args.password,
'codec': parsed_args.codec,
'caching': not parsed_args.disable_caching,
"maxpages": parsed_args.maxpages,
"password": parsed_args.password,
"codec": parsed_args.codec,
"caching": not parsed_args.disable_caching,
}
if parsed_args.pagenos:
func_args['pagenos'] = set([int(x)-1 for x in parsed_args.pagenos.split(",")])
func_args["pagenos"] = set([int(x) - 1 for x in parsed_args.pagenos.split(",")])
if parsed_args.output_type == "text" and parsed_args.output_file != "-":
for override, alttype in ( (".htm", "html"),
(".html", "html"),
(".xml", "xml" ),
(".tag", "tag" ) ):
for override, alttype in (
(".htm", "html"),
(".html", "html"),
(".xml", "xml"),
(".tag", "tag"),
):
if parsed_args.output_file.endswith(override):
func_args['out_type'] = alttype
func_args["out_type"] = alttype
if len(parsed_args.files) == 1:
if parsed_args.output_file != "-":
func_args['output_file'] = open(parsed_args.output_file, "wb")
convert_file(
parsed_args.files[0],
**func_args
)
func_args["output_file"] = open(parsed_args.output_file, "wb")
convert_file(parsed_args.files[0], **func_args)
else:
print("Multiple files not supported yet.")
return 1
return 0
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())