From 9e923de7ec9062418175ba2f8ac378f49c8a495c Mon Sep 17 00:00:00 2001 From: Ariel Sosnovsky Date: Mon, 10 Jun 2019 16:26:52 -0400 Subject: [PATCH] - added better mypy support - updated most unit-test to use the unitttest module - added a single unit-test run file --- .gitignore | 4 +- .version | 2 +- pdfmajor/__init__.py | 2 +- pdfmajor/converters/__init__.py | 10 +- pdfmajor/converters/html.py | 6 +- pdfmajor/converters/json.py | 8 +- pdfmajor/converters/writers/html.py | 9 +- pdfmajor/converters/writers/json.py | 4 +- .../state/PDFGraphicState/PDFColor.py | 8 +- .../state/PDFGraphicState/PDFColorSpace.py | 4 +- pdfmajor/parser/PDFDocument/PDFXRef.py | 6 +- setup.py | 13 +- test.py | 4 + tests/test_caching_err.py | 27 +-- tests/test_cols.py | 183 +++++++++--------- tests/test_conversions.py | 61 +++--- tests/test_html_writer.py | 82 +++++--- tests/test_image_writer.py | 35 ++-- tests/test_interpreter.py | 27 ++- tests/test_pdfmajor_ccitt.py | 93 +++++---- tests/test_pdfmajor_crypto.py | 26 ++- tests/test_pdfmajor_psparser.py | 77 ++++---- 22 files changed, 372 insertions(+), 319 deletions(-) create mode 100644 test.py diff --git a/.gitignore b/.gitignore index 0e1259f..7ead752 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ tests/*.txt .vscode tests/samples/output samples -**__pycache** \ No newline at end of file +**__pycache**\ +.mypy_cache +.ipynb_checkpoints \ No newline at end of file diff --git a/.version b/.version index ab74b63..4c8a05a 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -v1.3.7 \ No newline at end of file +v1.3.8 \ No newline at end of file diff --git a/pdfmajor/__init__.py b/pdfmajor/__init__.py index eac0194..d15bdf3 100644 --- a/pdfmajor/__init__.py +++ b/pdfmajor/__init__.py @@ -97,7 +97,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = (1, 3, 7) +__version__ = (1, 3, 8) if __name__ == '__main__': print( diff --git a/pdfmajor/converters/__init__.py b/pdfmajor/converters/__init__.py index e49e85c..762e273 100644 --- a/pdfmajor/converters/__init__.py +++ b/pdfmajor/converters/__init__.py @@ -2,7 +2,7 @@ import io import os import re -from typing import List +from typing import List, Optional from .html import convert_to_html from .xml import convert_to_xml @@ -13,14 +13,14 @@ from ..utils import logging def convert_file( input_file: str, - output_file: str = None, - image_folder_path: str = None, + output_file: Optional[str] = None, + image_folder_path: Optional[str] = None, codec: str = 'utf-8', maxpages: int = 0, - password: str = None, + password: Optional[str] = None, caching: bool = True, check_extractable: bool = True, - pagenos: List[int] = None, + pagenos: Optional[List[int]] = None, dont_export_images: bool = False, debug_level: int = logging.WARNING, out_type: str = 'html', diff --git a/pdfmajor/converters/html.py b/pdfmajor/converters/html.py index a9f2381..41e97ed 100644 --- a/pdfmajor/converters/html.py +++ b/pdfmajor/converters/html.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from .writers.html import HTMLMaker from ..interpreter import PDFInterpreter, PageInterpreter, logging from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject @@ -10,11 +10,11 @@ INF = (1<<31) - 1 def convert_to_html( input_file_path: str, output_file_path: str, - image_folder_path: str = None, + image_folder_path: Optional[str] = None, dont_export_images: bool = False, codec: str = 'utf-8', maxpages: int = 0, - password: str = None, + password: Optional[str] = None, caching: bool = True, check_extractable: bool = True, pagenos: List[int] = None, diff --git a/pdfmajor/converters/json.py b/pdfmajor/converters/json.py index 2a8c3c4..9485d7f 100644 --- a/pdfmajor/converters/json.py +++ b/pdfmajor/converters/json.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray from ..interpreter import PDFInterpreter, PageInterpreter, logging @@ -21,14 +21,14 @@ def normalize_color_values(values: list): def convert_to_json( input_file_path: str, output_file_path: str, - image_folder_path: str = None, + image_folder_path: Optional[str] = None, dont_export_images: bool = False, codec: str = 'utf-8', maxpages: int = 0, - password: str = None, + password: str = Optional[None], caching: bool = True, check_extractable: bool = True, - pagenos: List[int] = None, + pagenos: Optional[List[int]] = None, debug_level: int = logging.WARNING, ): intepreter = PDFInterpreter(input_file_path, diff --git a/pdfmajor/converters/writers/html.py b/pdfmajor/converters/writers/html.py index f57d4d4..78438b0 100644 --- a/pdfmajor/converters/writers/html.py +++ b/pdfmajor/converters/writers/html.py @@ -1,3 +1,4 @@ +from typing import Optional from io import TextIOWrapper from contextlib import contextmanager @@ -6,7 +7,7 @@ class HTMLMaker: def __init__(self, file_path: str, codec: str = 'utf-8'): self.file_path = file_path self.codec = codec - self.outfile: TextIOWrapper = None + self.outfile: Optional[TextIOWrapper] = None def __enter__(self): self.outfile = open(self.file_path, 'wb') @@ -42,18 +43,18 @@ class HTMLMaker: f'{name}: {value}' for name, value in css.items() ]) - attrs = " ".join([ + attrs_str = " ".join([ f'{name}="{value}"' for name, value in attrs.items() ]) if not singleton: - self.write(f"<{tag_name} {attrs}>", lineend=lineend) + self.write(f"<{tag_name} {attrs_str}>", lineend=lineend) self.__levels_deep += 1 yield self.__levels_deep -= 1 self.write(f"", deep_space=deep_space, lineend='\n' if not nolineend else '') else: - self.write(f"<{tag_name} {attrs} />", lineend=lineend, deep_space=deep_space) + self.write(f"<{tag_name} {attrs_str} />", lineend=lineend, deep_space=deep_space) yield def singleton(self, tag_name: str, attrs: dict = None, css: dict = None, singleton: bool = True): diff --git a/pdfmajor/converters/writers/json.py b/pdfmajor/converters/writers/json.py index 4315800..6fd6aec 100644 --- a/pdfmajor/converters/writers/json.py +++ b/pdfmajor/converters/writers/json.py @@ -1,3 +1,4 @@ +from typing import Set from json import dumps as json_dump from io import TextIOWrapper from contextlib import contextmanager @@ -8,14 +9,13 @@ def JSONMaker(file_path: str, codec: str = 'utf-8'): with JSONMakerObject(outf, codec=codec) as obj: yield obj - class JSONMakerObject: class AccessError(Exception): pass def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = 0): self.outfile: TextIOWrapper = outfile self.codec = codec self.levels_deep = levels_deep - self.keys = set() + self.keys: Set[str] = set() def __enter__(self): self.write_raw('{') diff --git a/pdfmajor/interpreter/commands/state/PDFGraphicState/PDFColor.py b/pdfmajor/interpreter/commands/state/PDFGraphicState/PDFColor.py index 1046255..64c3a77 100644 --- a/pdfmajor/interpreter/commands/state/PDFGraphicState/PDFColor.py +++ b/pdfmajor/interpreter/commands/state/PDFGraphicState/PDFColor.py @@ -1,10 +1,10 @@ -from typing import List +from typing import Tuple, Optional from .PDFColorSpace import PDFColorSpace class PDFColor: - def __init__(self, color_space: PDFColorSpace, *values: List[float]): - self.color_space: PDFColorSpace = color_space - self.values: list = values + def __init__(self, color_space: Optional[PDFColorSpace], *values: float): + self.color_space: Optional[PDFColorSpace] = color_space + self.values: Tuple[float, ...] = values def copy(self): return self.__class__( diff --git a/pdfmajor/interpreter/commands/state/PDFGraphicState/PDFColorSpace.py b/pdfmajor/interpreter/commands/state/PDFGraphicState/PDFColorSpace.py index 5bac22c..0560259 100644 --- a/pdfmajor/interpreter/commands/state/PDFGraphicState/PDFColorSpace.py +++ b/pdfmajor/interpreter/commands/state/PDFGraphicState/PDFColorSpace.py @@ -1,3 +1,4 @@ +from typing import Dict from collections import OrderedDict class PDFColorSpace(object): @@ -5,13 +6,12 @@ class PDFColorSpace(object): def __init__(self, name: str, ncomponents: int): self.name: str = name self.ncomponents: int = ncomponents - return def __repr__(self): return '' % (self.name, self.ncomponents) -PREDEFINED_COLORSPACE = OrderedDict() +PREDEFINED_COLORSPACE: Dict[str, PDFColorSpace] = OrderedDict() for (name, n) in [ ('DeviceGray', 1), # default value first diff --git a/pdfmajor/parser/PDFDocument/PDFXRef.py b/pdfmajor/parser/PDFDocument/PDFXRef.py index 254e923..36c7aea 100644 --- a/pdfmajor/parser/PDFDocument/PDFXRef.py +++ b/pdfmajor/parser/PDFDocument/PDFXRef.py @@ -1,5 +1,6 @@ import re from logging import getLogger +from typing import Tuple, Optional, Dict from ...utils import settings, choplist, nunpack from ..PSStackParser import PSEOF @@ -13,6 +14,7 @@ from .exceptions import PDFNoValidXRef, PDFSyntaxError log = getLogger(__name__) +Pos = Tuple[Optional[int], int, int] ## XRefs ## @@ -27,7 +29,7 @@ class PDFBaseXRef(object): # Must return # (strmid, index, genno) # or (None, pos, genno) - def get_pos(self, objid): + def get_pos(self, objid) -> Pos: raise KeyError(objid) @@ -36,7 +38,7 @@ class PDFBaseXRef(object): class PDFXRef(PDFBaseXRef): def __init__(self): - self.offsets = {} + self.offsets: Dict[int, Pos] = {} self.trailer = {} return diff --git a/setup.py b/setup.py index 369bde2..1bcbeec 100644 --- a/setup.py +++ b/setup.py @@ -10,12 +10,19 @@ import sys with open('README.md') as readme_file: SETUP = { "name": "pdfmajor", - "version": "1.3.7", + "version": "1.3.8", "packages": find_packages(include=['pdfmajor*']), "install_requires": [ - 'pycryptodome', - 'chardet' + 'pycryptodome==3.8.2', + 'chardet==3.0.4', + 'Pillow==6.0.0' ], + 'extras_require': { + 'dev': [ + 'tqdm==4.32.1', + 'nose==1.3.7' + ] + }, "description": 'PDF parser', "long_description": readme_file.read(), 'long_description_content_type': 'text/markdown', diff --git a/test.py b/test.py new file mode 100644 index 0000000..195ca73 --- /dev/null +++ b/test.py @@ -0,0 +1,4 @@ +from unittest import TestLoader, TextTestRunner + +tests = TestLoader().discover('./tests') +result = TextTestRunner(verbosity=20).run(tests) \ No newline at end of file diff --git a/tests/test_caching_err.py b/tests/test_caching_err.py index 65adc53..f1102c4 100644 --- a/tests/test_caching_err.py +++ b/tests/test_caching_err.py @@ -1,6 +1,5 @@ import os -from tqdm import tqdm - +from unittest import TestCase, main # from pdfmajor.extractor import extract_items_from_pdf from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging from pdfmajor.interpreter import PageInterpreter @@ -23,13 +22,17 @@ def count_elms(file_path: str): count += 1 return count -with tqdm(desc='Cache Test', total=len(FILES)*NUM_LOOPS) as waiter: - for file_name in FILES: - waiter.set_description(f"Cache Test - [{file_name}]") - file_path = os.path.join(INPUT_FOLDER, file_name) - last_count = count_elms(file_path) - for _ in range(NUM_LOOPS): - new_count = count_elms(file_path) - assert new_count == last_count - last_count = new_count - waiter.update() +class CachingTester(TestCase): + def test_cache(self): + for file_name in FILES: + self.subTest(name = file_name) + file_path = os.path.join(INPUT_FOLDER, file_name) + last_count = count_elms(file_path) + for _ in range(NUM_LOOPS): + new_count = count_elms(file_path) + self.assertEqual( new_count , last_count ) + last_count = new_count + +if __name__ == '__main__': + # Run Tests + main() diff --git a/tests/test_cols.py b/tests/test_cols.py index 67c4a16..bab0dd2 100644 --- a/tests/test_cols.py +++ b/tests/test_cols.py @@ -1,6 +1,6 @@ import os from typing import List -from tqdm import tqdm +from unittest import TestCase, main # from pdfmajor.extractor import extract_items_from_pdf from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging @@ -15,96 +15,101 @@ INPUT_FOLDER = os.path.join( FILES = os.listdir(INPUT_FOLDER) NUM_LOOPS = 10 -file_path = os.path.join(INPUT_FOLDER, "lorem-v4.pdf") +class ColorTest(TestCase): + def run_test(self, file_path: str, curve_defs: List[List[PDFColor]]): + def run_once(item): + if isinstance(item, LTCurve): + t_stroke, t_fill = curve_defs.pop(0) + self.assertEqual( item.stroke.values, t_stroke.values ) + self.assertEqual( item.stroke.color_space, t_stroke.color_space ) + self.assertEqual( item.fill.values, t_fill.values ) + self.assertEqual( item.fill.color_space, t_fill.color_space ) + elif isinstance(item, LTXObject): + for c in item: + run_once(c) -def run_test(file_path: str, curve_defs: List[List[PDFColor]]): - def run_once(item): - if isinstance(item, LTCurve): - t_stroke, t_fill = curve_defs.pop(0) - # print([item.stroke, item.fill]) - # print('...>', [t_stroke, t_fill]) - assert item.stroke.values == t_stroke.values - assert item.stroke.color_space == t_stroke.color_space - assert item.fill.values == t_fill.values - assert item.fill.color_space == t_fill.color_space - elif isinstance(item, LTXObject): - for c in item: - run_once(c) + for page in PDFInterpreter(file_path): + for item in page: + run_once(item) - for page in PDFInterpreter(file_path): - for item in page: - run_once(item) + def test_lorem_v3(self): + self.run_test( + os.path.join(INPUT_FOLDER, "lorem-v3.pdf"), + [ + [ + PDFColor(None), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), + ], + [ + PDFColor(None), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207), + ], + [ + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207), + ], + [ + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207), + ], + [ + PDFColor(None), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), + ], + [ + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 0, 0), + ] + ] + + ([[ + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), + ]]*24)+ + ([[ + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0), + ]]*11)+ + ([[ + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0), + ]]*5)+ + [[ + PDFColor(None), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), + ]] + ) -run_test( - os.path.join(INPUT_FOLDER, "lorem-v3.pdf"), - [ - [ - PDFColor(None), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), - ], - [ - PDFColor(None), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207), - ], - [ - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207), - ], - [ - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207), - ], - [ - PDFColor(None), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), - ], - [ - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 0, 0), - ] - ] + - ([[ - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), - ]]*24)+ - ([[ - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0), - ]]*11)+ - ([[ - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0), - ]]*5)+ - [[ - PDFColor(None), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), - ]] -) + def test_lorem_v4(self): + self.run_test( + os.path.join(INPUT_FOLDER, "lorem-v4.pdf"), + [ + [ + PDFColor(None), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), + ], + [ + PDFColor(None), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 64, 64, 64), + ] + ] + ) -run_test( - os.path.join(INPUT_FOLDER, "lorem-v4.pdf"), - [ - [ - PDFColor(None), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), - ], - [ - PDFColor(None), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 64, 64, 64), - ] - ] -) + def test_lorem_v5(self): + self.run_test( + os.path.join(INPUT_FOLDER, "lorem-v5.pdf"), + [ + [ + PDFColor(None), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), + ], + [ + PDFColor(None), + PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128), + ] + ] + ) -run_test( - os.path.join(INPUT_FOLDER, "lorem-v5.pdf"), - [ - [ - PDFColor(None), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), - ], - [ - PDFColor(None), - PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128), - ] - ] -) + +if __name__ == '__main__': + # Run Tests + main() diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 0fae8c9..d051e71 100755 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -3,13 +3,14 @@ import re import os import logging +from unittest import TestCase, main from typing import List from pdfmajor.converters import convert_file from xml.etree import ElementTree from json import load as json_load from tqdm import tqdm -def rel_path(*arg: List[str]) -> str: +def rel_path(*arg: str) -> str: return os.path.normpath(os.path.join(os.path.dirname(__file__), *arg)) EXTS = ['text', 'json','xml','html'] @@ -22,28 +23,38 @@ if not os.path.exists(OUTPUT_FOLDER): files = os.listdir(INPUT_FOLDER) TOTAL = len(files)*len(EXTS) -with tqdm(total=TOTAL) as waiter: - for idx, file_name in enumerate(files): - file_name_no_ext = re.sub(r'\.pdf$',"", file_name) - input_file_path = os.path.join(INPUT_FOLDER, file_name) - for ext in EXTS: - if not os.path.exists(os.path.join(OUTPUT_FOLDER, ext)): - os.makedirs(os.path.join(OUTPUT_FOLDER, ext)) - output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext) - if len(file_name) > len(file_name_no_ext): - waiter.set_description(f"Conversion test -- {file_name}") - convert_file( - input_file=input_file_path, - output_file=output_file_path, - out_type=ext, - debug_level=logging.WARN, - ) - if ext == 'xml': - waiter.set_description(f" > testing format {file_name}") - ElementTree.parse(output_file_path) - elif ext == 'json': - waiter.set_description(f" > testing format {file_name}") - json_load(open(output_file_path, 'r')) - waiter.update() - waiter.set_description("Done") \ No newline at end of file +class ConversionTest(TestCase): + def test_conversion(self): + # with tqdm(total=TOTAL) as waiter: + for idx, file_name in enumerate(files): + self.subTest(i=idx); + file_name_no_ext = re.sub(r'\.pdf$',"", file_name) + input_file_path = os.path.join(INPUT_FOLDER, file_name) + + for ext in EXTS: + if not os.path.exists(os.path.join(OUTPUT_FOLDER, ext)): + os.makedirs(os.path.join(OUTPUT_FOLDER, ext)) + output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext) + if len(file_name) > len(file_name_no_ext): + # waiter.set_description(f"Conversion test -- {file_name}") + convert_file( + input_file=input_file_path, + output_file=output_file_path, + out_type=ext, + debug_level=logging.WARN, + ) + if ext == 'xml': + # waiter.set_description(f" > testing format {file_name}") + ElementTree.parse(output_file_path) + elif ext == 'json': + # waiter.set_description(f" > testing format {file_name}") + with open(output_file_path, 'r') as outf: + json_load(outf) + # waiter.update() + # waiter.set_description("Done") + + +if __name__ == '__main__': + # Run Tests + main() \ No newline at end of file diff --git a/tests/test_html_writer.py b/tests/test_html_writer.py index 6a47fd8..6e9e91b 100644 --- a/tests/test_html_writer.py +++ b/tests/test_html_writer.py @@ -1,43 +1,60 @@ +from contextlib import contextmanager +from unittest import TestCase, main +from typing import List, Tuple, Callable from os import remove from tempfile import mktemp from pdfmajor.converters.writers.html import HTMLMaker +from tqdm import tqdm # Make temp file FILE_NAME = mktemp() +TestFunction = Callable[[HTMLMaker], str]; +TESTS: List[Tuple[TestFunction, str]] = []; -with HTMLMaker(FILE_NAME) as html: - with html.elm("html", nolineend=True): - html.write("test") -assert(open(FILE_NAME, 'r').read() == """ +class HTMLTester(TestCase): + @contextmanager + def get_html(self): + with HTMLMaker(FILE_NAME) as html: + yield html + def compare(self, expected: str): + with open(FILE_NAME, 'r') as f: + self.assertEqual(f.read() , expected) + + def test_html_basic(self): + with self.get_html() as html: + with html.elm("html", nolineend=True): + html.write("test") + self.compare(""" test """) - -with HTMLMaker(FILE_NAME) as html: - with html.elm("html", nolineend=True): - with html.elm("head"): - html.singleton("meta", { - 'http-equiv': "Content-Type", - 'content': "text/html" - }) -assert(open(FILE_NAME, 'r').read() == """ + def test_html_with_attributes(self): + with self.get_html() as html: + with html.elm("html", nolineend=True): + with html.elm("head"): + html.singleton("meta", { + 'http-equiv': "Content-Type", + 'content': "text/html" + }) + self.compare(""" """) -with HTMLMaker(FILE_NAME) as html: - with html.elm("html", nolineend=True): - with html.elm("head"): - html.singleton("meta", { - 'http-equiv': "Content-Type", - 'content': "text/html" - }) - with html.elm("body"): - with html.elm("h1", css={"color": "red"}): - html.write("Hello World") -assert(open(FILE_NAME, 'r').read() == """ + def test_html_with_children(self): + with self.get_html() as html: + with html.elm("html", nolineend=True): + with html.elm("head"): + html.singleton("meta", { + 'http-equiv': "Content-Type", + 'content': "text/html" + }) + with html.elm("body"): + with html.elm("h1", css={"color": "red"}): + html.write("Hello World") + self.compare(""" @@ -48,13 +65,18 @@ assert(open(FILE_NAME, 'r').read() == """ """) -with HTMLMaker(FILE_NAME) as html: - with html.elm('style'): - html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''') -assert(open(FILE_NAME, 'r').read() == """ """) -# Clean up -remove(FILE_NAME) \ No newline at end of file + +if __name__ == '__main__': + # Run Tests + main() + # Clean up + remove(FILE_NAME) \ No newline at end of file diff --git a/tests/test_image_writer.py b/tests/test_image_writer.py index 776ec50..2924c77 100644 --- a/tests/test_image_writer.py +++ b/tests/test_image_writer.py @@ -1,5 +1,5 @@ import os -from tqdm import tqdm +from unittest import TestCase, main from pdfmajor.interpreter import LTImage, logging from pdfmajor.interpreter import PageInterpreter, PDFInterpreter @@ -19,16 +19,23 @@ OUTPUT_FOLDER = os.path.join( if not os.path.exists(OUTPUT_FOLDER): os.makedirs(OUTPUT_FOLDER) -for file_name in tqdm(FILES, desc='Image Writer Test'): - for page in PDFInterpreter(os.path.join( - CUR_PATH, - f"tests/samples/pdf/{file_name}.pdf" - ), debug_level=logging.ERROR): - assert isinstance(page, PageInterpreter) - for item_idx, item in enumerate(page): - assert isinstance(item, LTItem) - if isinstance(item, LTImage): - item.save_image( - OUTPUT_FOLDER, - file_name=f'{file_name}_p{page.page_num}i{item_idx}' - ) \ No newline at end of file +class ImageTester(TestCase): + def test_image_extract(self): + for file_name in FILES: + self.subTest(file_name=file_name) + for page in PDFInterpreter(os.path.join( + CUR_PATH, + f"tests/samples/pdf/{file_name}.pdf" + ), debug_level=logging.ERROR): + self.assertTrue( isinstance(page, PageInterpreter) ) + for item_idx, item in enumerate(page): + self.assertTrue( isinstance(item, LTItem) ) + if isinstance(item, LTImage): + item.save_image( + OUTPUT_FOLDER, + file_name=f'{file_name}_p{page.page_num}i{item_idx}' + ) + +if __name__ == '__main__': + # Run Tests + main() diff --git a/tests/test_interpreter.py b/tests/test_interpreter.py index 002a96c..e2a5fae 100644 --- a/tests/test_interpreter.py +++ b/tests/test_interpreter.py @@ -1,5 +1,5 @@ import os -from tqdm import tqdm +from unittest import TestCase, main from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging from pdfmajor.interpreter import PageInterpreter from pdfmajor.interpreter.commands import LTItem @@ -11,12 +11,19 @@ INPUT_FOLDER = os.path.join( ) FILES = os.listdir(INPUT_FOLDER) -for file_name in tqdm(FILES, desc="Interpreter Test"): - file_path = os.path.join(INPUT_FOLDER, file_name) - for page in PDFInterpreter(file_path, debug_level=logging.ERROR): - assert isinstance(page, PageInterpreter) - for item in page: - assert isinstance(item, LTItem) - if isinstance(item, LTCharBlock): - for char in item: - assert isinstance(char, LTChar) \ No newline at end of file +class TestInterpreter(TestCase): + def test_pdfintp(self): + for file_name in FILES: + self.subTest(file_name=file_name) + file_path = os.path.join(INPUT_FOLDER, file_name) + for page in PDFInterpreter(file_path, debug_level=logging.ERROR): + self.assertTrue( isinstance(page, PageInterpreter) ) + for item in page: + self.assertTrue( isinstance(item, LTItem) ) + if isinstance(item, LTCharBlock): + for char in item: + self.assertTrue( isinstance(char, LTChar) ) + +if __name__ == '__main__': + # Run Tests + main() diff --git a/tests/test_pdfmajor_ccitt.py b/tests/test_pdfmajor_ccitt.py index 5ac9ad8..6f63e31 100644 --- a/tests/test_pdfmajor_ccitt.py +++ b/tests/test_pdfmajor_ccitt.py @@ -2,17 +2,12 @@ # -*- coding: utf-8 -*- -from nose.tools import assert_equal, assert_true, assert_false -from nose import SkipTest -import nose - -import logging - +import unittest from pdfmajor.parser.PDFStream.ccitt import CCITTG4Parser ## Test cases ## -class TestCCITTG4Parser(): +class TestCCITTG4Parser(unittest.TestCase): def get_parser(self, bits): parser = CCITTG4Parser(len(bits)) @@ -23,98 +18,98 @@ class TestCCITTG4Parser(): def test_b1(self): parser = self.get_parser('00000') parser._do_vertical(0) - assert_equal(parser._curpos, 0) + self.assertEqual(parser._curpos, 0) return def test_b2(self): parser = self.get_parser('10000') parser._do_vertical(-1) - assert_equal(parser._curpos, 0) + self.assertEqual(parser._curpos, 0) return def test_b3(self): parser = self.get_parser('000111') parser._do_pass() - assert_equal(parser._curpos, 3) - assert_equal(parser._get_bits(), '111') + self.assertEqual(parser._curpos, 3) + self.assertEqual(parser._get_bits(), '111') return def test_b4(self): parser = self.get_parser('00000') parser._do_vertical(+2) - assert_equal(parser._curpos, 2) - assert_equal(parser._get_bits(), '11') + self.assertEqual(parser._curpos, 2) + self.assertEqual(parser._get_bits(), '11') return def test_b5(self): parser = self.get_parser('11111111100') parser._do_horizontal(0, 3) - assert_equal(parser._curpos, 3) + self.assertEqual(parser._curpos, 3) parser._do_vertical(1) - assert_equal(parser._curpos, 10) - assert_equal(parser._get_bits(), '0001111111') + self.assertEqual(parser._curpos, 10) + self.assertEqual(parser._get_bits(), '0001111111') return def test_e1(self): parser = self.get_parser('10000') parser._do_vertical(0) - assert_equal(parser._curpos, 1) + self.assertEqual(parser._curpos, 1) parser._do_vertical(0) - assert_equal(parser._curpos, 5) - assert_equal(parser._get_bits(), '10000') + self.assertEqual(parser._curpos, 5) + self.assertEqual(parser._get_bits(), '10000') return def test_e2(self): parser = self.get_parser('10011') parser._do_vertical(0) - assert_equal(parser._curpos, 1) + self.assertEqual(parser._curpos, 1) parser._do_vertical(2) - assert_equal(parser._curpos, 5) - assert_equal(parser._get_bits(), '10000') + self.assertEqual(parser._curpos, 5) + self.assertEqual(parser._get_bits(), '10000') return def test_e3(self): parser = self.get_parser('011111') parser._color = 0 parser._do_vertical(0) - assert_equal(parser._color, 1) - assert_equal(parser._curpos, 1) + self.assertEqual(parser._color, 1) + self.assertEqual(parser._curpos, 1) parser._do_vertical(-2) - assert_equal(parser._color, 0) - assert_equal(parser._curpos, 4) + self.assertEqual(parser._color, 0) + self.assertEqual(parser._curpos, 4) parser._do_vertical(0) - assert_equal(parser._curpos, 6) - assert_equal(parser._get_bits(), '011100') + self.assertEqual(parser._curpos, 6) + self.assertEqual(parser._get_bits(), '011100') return def test_e4(self): parser = self.get_parser('10000') parser._do_vertical(0) - assert_equal(parser._curpos, 1) + self.assertEqual(parser._curpos, 1) parser._do_vertical(-2) - assert_equal(parser._curpos, 3) + self.assertEqual(parser._curpos, 3) parser._do_vertical(0) - assert_equal(parser._curpos, 5) - assert_equal(parser._get_bits(), '10011') + self.assertEqual(parser._curpos, 5) + self.assertEqual(parser._get_bits(), '10011') return def test_e5(self): parser = self.get_parser('011000') parser._color = 0 parser._do_vertical(0) - assert_equal(parser._curpos, 1) + self.assertEqual(parser._curpos, 1) parser._do_vertical(3) - assert_equal(parser._curpos, 6) - assert_equal(parser._get_bits(), '011111') + self.assertEqual(parser._curpos, 6) + self.assertEqual(parser._get_bits(), '011111') return def test_e6(self): parser = self.get_parser('11001') parser._do_pass() - assert_equal(parser._curpos, 4) + self.assertEqual(parser._curpos, 4) parser._do_vertical(0) - assert_equal(parser._curpos, 5) - assert_equal(parser._get_bits(), '11111') + self.assertEqual(parser._curpos, 5) + self.assertEqual(parser._get_bits(), '11111') return def test_e7(self): @@ -122,8 +117,8 @@ class TestCCITTG4Parser(): parser._curpos = 2 parser._color = 1 parser._do_horizontal(2, 6) - assert_equal(parser._curpos, 10) - assert_equal(parser._get_bits(), '1111000000') + self.assertEqual(parser._curpos, 10) + self.assertEqual(parser._get_bits(), '1111000000') return def test_e8(self): @@ -131,19 +126,19 @@ class TestCCITTG4Parser(): parser._curpos = 1 parser._color = 0 parser._do_vertical(0) - assert_equal(parser._curpos, 2) + self.assertEqual(parser._curpos, 2) parser._do_horizontal(7, 0) - assert_equal(parser._curpos, 9) - assert_equal(parser._get_bits(), '101111111') + self.assertEqual(parser._curpos, 9) + self.assertEqual(parser._get_bits(), '101111111') return def test_m1(self): parser = self.get_parser('10101') parser._do_pass() - assert_equal(parser._curpos, 2) + self.assertEqual(parser._curpos, 2) parser._do_pass() - assert_equal(parser._curpos, 4) - assert_equal(parser._get_bits(), '1111') + self.assertEqual(parser._curpos, 4) + self.assertEqual(parser._get_bits(), '1111') return def test_m2(self): @@ -152,7 +147,7 @@ class TestCCITTG4Parser(): parser._do_vertical(-1) parser._do_vertical(1) parser._do_horizontal(1, 1) - assert_equal(parser._get_bits(), '011101') + self.assertEqual(parser._get_bits(), '011101') return def test_m3(self): @@ -161,8 +156,8 @@ class TestCCITTG4Parser(): parser._do_pass() parser._do_vertical(1) parser._do_vertical(1) - assert_equal(parser._get_bits(), '00000001') + self.assertEqual(parser._get_bits(), '00000001') return if __name__ == '__main__': - nose.runmodule() \ No newline at end of file + unittest.main() \ No newline at end of file diff --git a/tests/test_pdfmajor_crypto.py b/tests/test_pdfmajor_crypto.py index ef87d25..0e4b430 100644 --- a/tests/test_pdfmajor_crypto.py +++ b/tests/test_pdfmajor_crypto.py @@ -1,9 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from nose.tools import assert_equal -from nose import SkipTest -import nose +from unittest import TestCase, main #test of various compression/encoding modules (previously in doctests): from pdfmajor.utils.ascii85 import ascii85decode, asciihexdecode @@ -14,24 +12,24 @@ import binascii def hex(b): return binascii.hexlify(b) #encode('hex') def dehex(b): return binascii.unhexlify(b) #decode('hex') -class TestAscii85(): +class TestAscii85(TestCase): def test_ascii85decode(self): #The sample string is taken from: http://en.wikipedia.org/w/index.php?title=Ascii85 - assert_equal(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished') - assert_equal(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.') + self.assertEqual(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished') + self.assertEqual(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.') def test_asciihexdecode(self): - assert_equal(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde') - assert_equal(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep') - assert_equal(asciihexdecode(b'7>'),b'p') + self.assertEqual(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde') + self.assertEqual(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep') + self.assertEqual(asciihexdecode(b'7>'),b'p') -class TestLzw(): +class TestLzw(TestCase): def test_lzwdecode(self): - assert_equal(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42') + self.assertEqual(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42') -class TestRunlength(): +class TestRunlength(TestCase): def test_rldecode(self): - assert_equal(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde') + self.assertEqual(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde') if __name__ == '__main__': - nose.runmodule() \ No newline at end of file + main() \ No newline at end of file diff --git a/tests/test_pdfmajor_psparser.py b/tests/test_pdfmajor_psparser.py index b146b68..eeab634 100644 --- a/tests/test_pdfmajor_psparser.py +++ b/tests/test_pdfmajor_psparser.py @@ -2,19 +2,14 @@ # -*- coding: utf-8 -*- -from nose.tools import assert_equal, assert_true, assert_false -from nose import SkipTest -import nose - import logging - +from unittest import TestCase, main +from io import BytesIO from pdfmajor.parser.PSStackParser import KWD, LIT, PSBaseParser, PSEOF, PSStackParser ## Simplistic Test cases ## -class TestPSBaseParser: - - TESTDATA = br'''%!PS +TESTDATA = br'''%!PS begin end " @ # /a/BCD /Some_Name /foo#5f#xbaa @@ -33,36 +28,32 @@ func/a/b{(c)do*}def [ 1 (z) ! ] << /foo (bar) >> ''' - - TOKENS = [ - (5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')), - (21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), - (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), - (65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'), - (98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'), - (143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'), - (191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'), - (226, KWD(b'func')), (230, LIT('a')), (232, LIT('b')), - (234, KWD(b'{')), (235, b'c'), (238, KWD(b'do*')), (241, KWD(b'}')), - (242, KWD(b'def')), (246, KWD(b'[')), (248, 1), (250, b'z'), (254, KWD(b'!')), - (256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'), - (272, KWD(b'>>')) - ] - - OBJS = [ - (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), - (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), - (65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'), - (98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'), - (143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'), - (191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'), - (230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']), - (258, {'foo': b'bar'}), - ] - +TOKENS = [ + (5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')), + (21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), + (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), + (65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'), + (98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'), + (143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'), + (191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'), + (226, KWD(b'func')), (230, LIT('a')), (232, LIT('b')), + (234, KWD(b'{')), (235, b'c'), (238, KWD(b'do*')), (241, KWD(b'}')), + (242, KWD(b'def')), (246, KWD(b'[')), (248, 1), (250, b'z'), (254, KWD(b'!')), + (256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'), + (272, KWD(b'>>')) +] +OBJS = [ + (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), + (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), + (65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'), + (98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'), + (143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'), + (191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'), + (230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']), + (258, {'foo': b'bar'}), +] +class TestPSBaseParser(TestCase): def get_tokens(self, s): - from io import BytesIO - class MyParser(PSStackParser): def flush(self): self.add_results(*self.popall()) @@ -76,8 +67,6 @@ func/a/b{(c)do*}def return r def get_objects(self, s): - from io import BytesIO - class MyParser(PSStackParser): def flush(self): self.add_results(*self.popall()) @@ -91,18 +80,18 @@ func/a/b{(c)do*}def return r def test_1(self): - tokens = self.get_tokens(self.TESTDATA) + tokens = self.get_tokens(TESTDATA) logging.info(tokens) - assert_equal(tokens, self.TOKENS) + self.assertEqual(tokens, TOKENS) return def test_2(self): - objs = self.get_objects(self.TESTDATA) + objs = self.get_objects(TESTDATA) logging.info(objs) - assert_equal(objs, self.OBJS) + self.assertEqual(objs, OBJS) return if __name__ == '__main__': #import logging,sys,os,six #logging.basicConfig(level=logging.DEBUG, filename='%s_%d.%d.log'%(os.path.basename(__file__),sys.version_info[0],sys.version_info[1])) - nose.runmodule() \ No newline at end of file + main() \ No newline at end of file