- added better mypy support

- updated most unit-test to use the unitttest module
- added a single unit-test run file
This commit is contained in:
ari committed 2019-06-10 16:26:52 -04:00
1 parent 290de97cb5
commit 9e923de7ec
22 files changed
+372 -319

No files matched your search

+3 -1
View File
@@ -11,4 +11,6 @@ tests/*.txt
.vscode
tests/samples/output
samples
**__pycache**
**__pycache**\
.mypy_cache
.ipynb_checkpoints
+1 -1
View File
@@ -1 +1 @@
v1.3.7
v1.3.8
+1 -1
View File
@@ -97,7 +97,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
__version__ = (1, 3, 7)
__version__ = (1, 3, 8)
if __name__ == '__main__':
print(
+5 -5
View File
@@ -2,7 +2,7 @@ import io
import os
import re
from typing import List
from typing import List, Optional
from .html import convert_to_html
from .xml import convert_to_xml
@@ -13,14 +13,14 @@ from ..utils import logging
def convert_file(
input_file: str,
output_file: str = None,
image_folder_path: str = None,
output_file: Optional[str] = None,
image_folder_path: Optional[str] = None,
codec: str = 'utf-8',
maxpages: int = 0,
password: str = None,
password: Optional[str] = None,
caching: bool = True,
check_extractable: bool = True,
pagenos: List[int] = None,
pagenos: Optional[List[int]] = None,
dont_export_images: bool = False,
debug_level: int = logging.WARNING,
out_type: str = 'html',
+3 -3
View File
@@ -1,4 +1,4 @@
from typing import List
from typing import List, Optional
from .writers.html import HTMLMaker
from ..interpreter import PDFInterpreter, PageInterpreter, logging
from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject
@@ -10,11 +10,11 @@ INF = (1<<31) - 1
def convert_to_html(
input_file_path: str,
output_file_path: str,
image_folder_path: str = None,
image_folder_path: Optional[str] = None,
dont_export_images: bool = False,
codec: str = 'utf-8',
maxpages: int = 0,
password: str = None,
password: Optional[str] = None,
caching: bool = True,
check_extractable: bool = True,
pagenos: List[int] = None,
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import List
from typing import List, Optional
from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray
from ..interpreter import PDFInterpreter, PageInterpreter, logging
@@ -21,14 +21,14 @@ def normalize_color_values(values: list):
def convert_to_json(
input_file_path: str,
output_file_path: str,
image_folder_path: str = None,
image_folder_path: Optional[str] = None,
dont_export_images: bool = False,
codec: str = 'utf-8',
maxpages: int = 0,
password: str = None,
password: str = Optional[None],
caching: bool = True,
check_extractable: bool = True,
pagenos: List[int] = None,
pagenos: Optional[List[int]] = None,
debug_level: int = logging.WARNING,
):
intepreter = PDFInterpreter(input_file_path,
+5 -4
View File
@@ -1,3 +1,4 @@
from typing import Optional
from io import TextIOWrapper
from contextlib import contextmanager
@@ -6,7 +7,7 @@ class HTMLMaker:
def __init__(self, file_path: str, codec: str = 'utf-8'):
self.file_path = file_path
self.codec = codec
self.outfile: TextIOWrapper = None
self.outfile: Optional[TextIOWrapper] = None
def __enter__(self):
self.outfile = open(self.file_path, 'wb')
@@ -42,18 +43,18 @@ class HTMLMaker:
f'{name}: {value}'
for name, value in css.items()
])
attrs = " ".join([
attrs_str = " ".join([
f'{name}="{value}"'
for name, value in attrs.items()
])
if not singleton:
self.write(f"<{tag_name} {attrs}>", lineend=lineend)
self.write(f"<{tag_name} {attrs_str}>", lineend=lineend)
self.__levels_deep += 1
yield
self.__levels_deep -= 1
self.write(f"</{tag_name}>", deep_space=deep_space, lineend='\n' if not nolineend else '')
else:
self.write(f"<{tag_name} {attrs} />", lineend=lineend, deep_space=deep_space)
self.write(f"<{tag_name} {attrs_str} />", lineend=lineend, deep_space=deep_space)
yield
def singleton(self, tag_name: str, attrs: dict = None, css: dict = None, singleton: bool = True):
+2 -2
View File
@@ -1,3 +1,4 @@
from typing import Set
from json import dumps as json_dump
from io import TextIOWrapper
from contextlib import contextmanager
@@ -8,14 +9,13 @@ def JSONMaker(file_path: str, codec: str = 'utf-8'):
with JSONMakerObject(outf, codec=codec) as obj:
yield obj
class JSONMakerObject:
class AccessError(Exception): pass
def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = 0):
self.outfile: TextIOWrapper = outfile
self.codec = codec
self.levels_deep = levels_deep
self.keys = set()
self.keys: Set[str] = set()
def __enter__(self):
self.write_raw('{')
@@ -1,10 +1,10 @@
from typing import List
from typing import Tuple, Optional
from .PDFColorSpace import PDFColorSpace
class PDFColor:
def __init__(self, color_space: PDFColorSpace, *values: List[float]):
self.color_space: PDFColorSpace = color_space
self.values: list = values
def __init__(self, color_space: Optional[PDFColorSpace], *values: float):
self.color_space: Optional[PDFColorSpace] = color_space
self.values: Tuple[float, ...] = values
def copy(self):
return self.__class__(
@@ -1,3 +1,4 @@
from typing import Dict
from collections import OrderedDict
class PDFColorSpace(object):
@@ -5,13 +6,12 @@ class PDFColorSpace(object):
def __init__(self, name: str, ncomponents: int):
self.name: str = name
self.ncomponents: int = ncomponents
return
def __repr__(self):
return '<PDFColorSpace: %s, ncomponents=%d>' % (self.name, self.ncomponents)
PREDEFINED_COLORSPACE = OrderedDict()
PREDEFINED_COLORSPACE: Dict[str, PDFColorSpace] = OrderedDict()
for (name, n) in [
('DeviceGray', 1), # default value first
+4 -2
View File
@@ -1,5 +1,6 @@
import re
from logging import getLogger
from typing import Tuple, Optional, Dict
from ...utils import settings, choplist, nunpack
from ..PSStackParser import PSEOF
@@ -13,6 +14,7 @@ from .exceptions import PDFNoValidXRef, PDFSyntaxError
log = getLogger(__name__)
Pos = Tuple[Optional[int], int, int]
## XRefs
##
@@ -27,7 +29,7 @@ class PDFBaseXRef(object):
# Must return
# (strmid, index, genno)
# or (None, pos, genno)
def get_pos(self, objid):
def get_pos(self, objid) -> Pos:
raise KeyError(objid)
@@ -36,7 +38,7 @@ class PDFBaseXRef(object):
class PDFXRef(PDFBaseXRef):
def __init__(self):
self.offsets = {}
self.offsets: Dict[int, Pos] = {}
self.trailer = {}
return
+10 -3
View File
@@ -10,12 +10,19 @@ import sys
with open('README.md') as readme_file:
SETUP = {
"name": "pdfmajor",
"version": "1.3.7",
"version": "1.3.8",
"packages": find_packages(include=['pdfmajor*']),
"install_requires": [
'pycryptodome',
'chardet'
'pycryptodome==3.8.2',
'chardet==3.0.4',
'Pillow==6.0.0'
],
'extras_require': {
'dev': [
'tqdm==4.32.1',
'nose==1.3.7'
]
},
"description": 'PDF parser',
"long_description": readme_file.read(),
'long_description_content_type': 'text/markdown',
+4
View File
@@ -0,0 +1,4 @@
from unittest import TestLoader, TextTestRunner
tests = TestLoader().discover('./tests')
result = TextTestRunner(verbosity=20).run(tests)
+15 -12
View File
@@ -1,6 +1,5 @@
import os
from tqdm import tqdm
from unittest import TestCase, main
# from pdfmajor.extractor import extract_items_from_pdf
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
from pdfmajor.interpreter import PageInterpreter
@@ -23,13 +22,17 @@ def count_elms(file_path: str):
count += 1
return count
with tqdm(desc='Cache Test', total=len(FILES)*NUM_LOOPS) as waiter:
for file_name in FILES:
waiter.set_description(f"Cache Test - [{file_name}]")
file_path = os.path.join(INPUT_FOLDER, file_name)
last_count = count_elms(file_path)
for _ in range(NUM_LOOPS):
new_count = count_elms(file_path)
assert new_count == last_count
last_count = new_count
waiter.update()
class CachingTester(TestCase):
def test_cache(self):
for file_name in FILES:
self.subTest(name = file_name)
file_path = os.path.join(INPUT_FOLDER, file_name)
last_count = count_elms(file_path)
for _ in range(NUM_LOOPS):
new_count = count_elms(file_path)
self.assertEqual( new_count , last_count )
last_count = new_count
if __name__ == '__main__':
# Run Tests
main()
+94 -89
View File
@@ -1,6 +1,6 @@
import os
from typing import List
from tqdm import tqdm
from unittest import TestCase, main
# from pdfmajor.extractor import extract_items_from_pdf
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
@@ -15,96 +15,101 @@ INPUT_FOLDER = os.path.join(
FILES = os.listdir(INPUT_FOLDER)
NUM_LOOPS = 10
file_path = os.path.join(INPUT_FOLDER, "lorem-v4.pdf")
class ColorTest(TestCase):
def run_test(self, file_path: str, curve_defs: List[List[PDFColor]]):
def run_once(item):
if isinstance(item, LTCurve):
t_stroke, t_fill = curve_defs.pop(0)
self.assertEqual( item.stroke.values, t_stroke.values )
self.assertEqual( item.stroke.color_space, t_stroke.color_space )
self.assertEqual( item.fill.values, t_fill.values )
self.assertEqual( item.fill.color_space, t_fill.color_space )
elif isinstance(item, LTXObject):
for c in item:
run_once(c)
def run_test(file_path: str, curve_defs: List[List[PDFColor]]):
def run_once(item):
if isinstance(item, LTCurve):
t_stroke, t_fill = curve_defs.pop(0)
# print([item.stroke, item.fill])
# print('...>', [t_stroke, t_fill])
assert item.stroke.values == t_stroke.values
assert item.stroke.color_space == t_stroke.color_space
assert item.fill.values == t_fill.values
assert item.fill.color_space == t_fill.color_space
elif isinstance(item, LTXObject):
for c in item:
run_once(c)
for page in PDFInterpreter(file_path):
for item in page:
run_once(item)
for page in PDFInterpreter(file_path):
for item in page:
run_once(item)
def test_lorem_v3(self):
self.run_test(
os.path.join(INPUT_FOLDER, "lorem-v3.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 0, 0),
]
] +
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]*24)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*11)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*5)+
[[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]
)
run_test(
os.path.join(INPUT_FOLDER, "lorem-v3.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 0, 0),
]
] +
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]*24)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*11)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*5)+
[[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]
)
def test_lorem_v4(self):
self.run_test(
os.path.join(INPUT_FOLDER, "lorem-v4.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 64, 64, 64),
]
]
)
run_test(
os.path.join(INPUT_FOLDER, "lorem-v4.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 64, 64, 64),
]
]
)
def test_lorem_v5(self):
self.run_test(
os.path.join(INPUT_FOLDER, "lorem-v5.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
]
]
)
run_test(
os.path.join(INPUT_FOLDER, "lorem-v5.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
]
]
)
if __name__ == '__main__':
# Run Tests
main()
+36 -25
View File
@@ -3,13 +3,14 @@ import re
import os
import logging
from unittest import TestCase, main
from typing import List
from pdfmajor.converters import convert_file
from xml.etree import ElementTree
from json import load as json_load
from tqdm import tqdm
def rel_path(*arg: List[str]) -> str:
def rel_path(*arg: str) -> str:
return os.path.normpath(os.path.join(os.path.dirname(__file__), *arg))
EXTS = ['text', 'json','xml','html']
@@ -22,28 +23,38 @@ if not os.path.exists(OUTPUT_FOLDER):
files = os.listdir(INPUT_FOLDER)
TOTAL = len(files)*len(EXTS)
with tqdm(total=TOTAL) as waiter:
for idx, file_name in enumerate(files):
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
input_file_path = os.path.join(INPUT_FOLDER, file_name)
for ext in EXTS:
if not os.path.exists(os.path.join(OUTPUT_FOLDER, ext)):
os.makedirs(os.path.join(OUTPUT_FOLDER, ext))
output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext)
if len(file_name) > len(file_name_no_ext):
waiter.set_description(f"Conversion test -- {file_name}")
convert_file(
input_file=input_file_path,
output_file=output_file_path,
out_type=ext,
debug_level=logging.WARN,
)
if ext == 'xml':
waiter.set_description(f" > testing format {file_name}")
ElementTree.parse(output_file_path)
elif ext == 'json':
waiter.set_description(f" > testing format {file_name}")
json_load(open(output_file_path, 'r'))
waiter.update()
waiter.set_description("Done")
class ConversionTest(TestCase):
def test_conversion(self):
# with tqdm(total=TOTAL) as waiter:
for idx, file_name in enumerate(files):
self.subTest(i=idx);
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
input_file_path = os.path.join(INPUT_FOLDER, file_name)
for ext in EXTS:
if not os.path.exists(os.path.join(OUTPUT_FOLDER, ext)):
os.makedirs(os.path.join(OUTPUT_FOLDER, ext))
output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext)
if len(file_name) > len(file_name_no_ext):
# waiter.set_description(f"Conversion test -- {file_name}")
convert_file(
input_file=input_file_path,
output_file=output_file_path,
out_type=ext,
debug_level=logging.WARN,
)
if ext == 'xml':
# waiter.set_description(f" > testing format {file_name}")
ElementTree.parse(output_file_path)
elif ext == 'json':
# waiter.set_description(f" > testing format {file_name}")
with open(output_file_path, 'r') as outf:
json_load(outf)
# waiter.update()
# waiter.set_description("Done")
if __name__ == '__main__':
# Run Tests
main()
+52 -30
View File
@@ -1,43 +1,60 @@
from contextlib import contextmanager
from unittest import TestCase, main
from typing import List, Tuple, Callable
from os import remove
from tempfile import mktemp
from pdfmajor.converters.writers.html import HTMLMaker
from tqdm import tqdm
# Make temp file
FILE_NAME = mktemp()
TestFunction = Callable[[HTMLMaker], str];
TESTS: List[Tuple[TestFunction, str]] = [];
with HTMLMaker(FILE_NAME) as html:
with html.elm("html", nolineend=True):
html.write("test")
assert(open(FILE_NAME, 'r').read() == """<html >
class HTMLTester(TestCase):
@contextmanager
def get_html(self):
with HTMLMaker(FILE_NAME) as html:
yield html
def compare(self, expected: str):
with open(FILE_NAME, 'r') as f:
self.assertEqual(f.read() , expected)
def test_html_basic(self):
with self.get_html() as html:
with html.elm("html", nolineend=True):
html.write("test")
self.compare("""<html >
test
</html>""")
with HTMLMaker(FILE_NAME) as html:
with html.elm("html", nolineend=True):
with html.elm("head"):
html.singleton("meta", {
'http-equiv': "Content-Type",
'content': "text/html"
})
assert(open(FILE_NAME, 'r').read() == """<html >
def test_html_with_attributes(self):
with self.get_html() as html:
with html.elm("html", nolineend=True):
with html.elm("head"):
html.singleton("meta", {
'http-equiv': "Content-Type",
'content': "text/html"
})
self.compare("""<html >
<head >
<meta http-equiv="Content-Type" content="text/html" />
</head>
</html>""")
with HTMLMaker(FILE_NAME) as html:
with html.elm("html", nolineend=True):
with html.elm("head"):
html.singleton("meta", {
'http-equiv': "Content-Type",
'content': "text/html"
})
with html.elm("body"):
with html.elm("h1", css={"color": "red"}):
html.write("Hello World")
assert(open(FILE_NAME, 'r').read() == """<html >
def test_html_with_children(self):
with self.get_html() as html:
with html.elm("html", nolineend=True):
with html.elm("head"):
html.singleton("meta", {
'http-equiv': "Content-Type",
'content': "text/html"
})
with html.elm("body"):
with html.elm("h1", css={"color": "red"}):
html.write("Hello World")
self.compare("""<html >
<head >
<meta http-equiv="Content-Type" content="text/html" />
</head>
@@ -48,13 +65,18 @@ assert(open(FILE_NAME, 'r').read() == """<html >
</body>
</html>""")
with HTMLMaker(FILE_NAME) as html:
with html.elm('style'):
html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''')
assert(open(FILE_NAME, 'r').read() == """<style >
def test_html_with_css(self):
with self.get_html() as html:
with html.elm('style'):
html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''')
self.compare("""<style >
.page { position: relative; overflow: hidden;border: 1px black solid;}
</style>
""")
# Clean up
remove(FILE_NAME)
if __name__ == '__main__':
# Run Tests
main()
# Clean up
remove(FILE_NAME)
+21 -14
View File
@@ -1,5 +1,5 @@
import os
from tqdm import tqdm
from unittest import TestCase, main
from pdfmajor.interpreter import LTImage, logging
from pdfmajor.interpreter import PageInterpreter, PDFInterpreter
@@ -19,16 +19,23 @@ OUTPUT_FOLDER = os.path.join(
if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER)
for file_name in tqdm(FILES, desc='Image Writer Test'):
for page in PDFInterpreter(os.path.join(
CUR_PATH,
f"tests/samples/pdf/{file_name}.pdf"
), debug_level=logging.ERROR):
assert isinstance(page, PageInterpreter)
for item_idx, item in enumerate(page):
assert isinstance(item, LTItem)
if isinstance(item, LTImage):
item.save_image(
OUTPUT_FOLDER,
file_name=f'{file_name}_p{page.page_num}i{item_idx}'
)
class ImageTester(TestCase):
def test_image_extract(self):
for file_name in FILES:
self.subTest(file_name=file_name)
for page in PDFInterpreter(os.path.join(
CUR_PATH,
f"tests/samples/pdf/{file_name}.pdf"
), debug_level=logging.ERROR):
self.assertTrue( isinstance(page, PageInterpreter) )
for item_idx, item in enumerate(page):
self.assertTrue( isinstance(item, LTItem) )
if isinstance(item, LTImage):
item.save_image(
OUTPUT_FOLDER,
file_name=f'{file_name}_p{page.page_num}i{item_idx}'
)
if __name__ == '__main__':
# Run Tests
main()
+17 -10
View File
@@ -1,5 +1,5 @@
import os
from tqdm import tqdm
from unittest import TestCase, main
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
from pdfmajor.interpreter import PageInterpreter
from pdfmajor.interpreter.commands import LTItem
@@ -11,12 +11,19 @@ INPUT_FOLDER = os.path.join(
)
FILES = os.listdir(INPUT_FOLDER)
for file_name in tqdm(FILES, desc="Interpreter Test"):
file_path = os.path.join(INPUT_FOLDER, file_name)
for page in PDFInterpreter(file_path, debug_level=logging.ERROR):
assert isinstance(page, PageInterpreter)
for item in page:
assert isinstance(item, LTItem)
if isinstance(item, LTCharBlock):
for char in item:
assert isinstance(char, LTChar)
class TestInterpreter(TestCase):
def test_pdfintp(self):
for file_name in FILES:
self.subTest(file_name=file_name)
file_path = os.path.join(INPUT_FOLDER, file_name)
for page in PDFInterpreter(file_path, debug_level=logging.ERROR):
self.assertTrue( isinstance(page, PageInterpreter) )
for item in page:
self.assertTrue( isinstance(item, LTItem) )
if isinstance(item, LTCharBlock):
for char in item:
self.assertTrue( isinstance(char, LTChar) )
if __name__ == '__main__':
# Run Tests
main()
+44 -49
View File
@@ -2,17 +2,12 @@
# -*- coding: utf-8 -*-
from nose.tools import assert_equal, assert_true, assert_false
from nose import SkipTest
import nose
import logging
import unittest
from pdfmajor.parser.PDFStream.ccitt import CCITTG4Parser
## Test cases
##
class TestCCITTG4Parser():
class TestCCITTG4Parser(unittest.TestCase):
def get_parser(self, bits):
parser = CCITTG4Parser(len(bits))
@@ -23,98 +18,98 @@ class TestCCITTG4Parser():
def test_b1(self):
parser = self.get_parser('00000')
parser._do_vertical(0)
assert_equal(parser._curpos, 0)
self.assertEqual(parser._curpos, 0)
return
def test_b2(self):
parser = self.get_parser('10000')
parser._do_vertical(-1)
assert_equal(parser._curpos, 0)
self.assertEqual(parser._curpos, 0)
return
def test_b3(self):
parser = self.get_parser('000111')
parser._do_pass()
assert_equal(parser._curpos, 3)
assert_equal(parser._get_bits(), '111')
self.assertEqual(parser._curpos, 3)
self.assertEqual(parser._get_bits(), '111')
return
def test_b4(self):
parser = self.get_parser('00000')
parser._do_vertical(+2)
assert_equal(parser._curpos, 2)
assert_equal(parser._get_bits(), '11')
self.assertEqual(parser._curpos, 2)
self.assertEqual(parser._get_bits(), '11')
return
def test_b5(self):
parser = self.get_parser('11111111100')
parser._do_horizontal(0, 3)
assert_equal(parser._curpos, 3)
self.assertEqual(parser._curpos, 3)
parser._do_vertical(1)
assert_equal(parser._curpos, 10)
assert_equal(parser._get_bits(), '0001111111')
self.assertEqual(parser._curpos, 10)
self.assertEqual(parser._get_bits(), '0001111111')
return
def test_e1(self):
parser = self.get_parser('10000')
parser._do_vertical(0)
assert_equal(parser._curpos, 1)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(0)
assert_equal(parser._curpos, 5)
assert_equal(parser._get_bits(), '10000')
self.assertEqual(parser._curpos, 5)
self.assertEqual(parser._get_bits(), '10000')
return
def test_e2(self):
parser = self.get_parser('10011')
parser._do_vertical(0)
assert_equal(parser._curpos, 1)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(2)
assert_equal(parser._curpos, 5)
assert_equal(parser._get_bits(), '10000')
self.assertEqual(parser._curpos, 5)
self.assertEqual(parser._get_bits(), '10000')
return
def test_e3(self):
parser = self.get_parser('011111')
parser._color = 0
parser._do_vertical(0)
assert_equal(parser._color, 1)
assert_equal(parser._curpos, 1)
self.assertEqual(parser._color, 1)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(-2)
assert_equal(parser._color, 0)
assert_equal(parser._curpos, 4)
self.assertEqual(parser._color, 0)
self.assertEqual(parser._curpos, 4)
parser._do_vertical(0)
assert_equal(parser._curpos, 6)
assert_equal(parser._get_bits(), '011100')
self.assertEqual(parser._curpos, 6)
self.assertEqual(parser._get_bits(), '011100')
return
def test_e4(self):
parser = self.get_parser('10000')
parser._do_vertical(0)
assert_equal(parser._curpos, 1)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(-2)
assert_equal(parser._curpos, 3)
self.assertEqual(parser._curpos, 3)
parser._do_vertical(0)
assert_equal(parser._curpos, 5)
assert_equal(parser._get_bits(), '10011')
self.assertEqual(parser._curpos, 5)
self.assertEqual(parser._get_bits(), '10011')
return
def test_e5(self):
parser = self.get_parser('011000')
parser._color = 0
parser._do_vertical(0)
assert_equal(parser._curpos, 1)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(3)
assert_equal(parser._curpos, 6)
assert_equal(parser._get_bits(), '011111')
self.assertEqual(parser._curpos, 6)
self.assertEqual(parser._get_bits(), '011111')
return
def test_e6(self):
parser = self.get_parser('11001')
parser._do_pass()
assert_equal(parser._curpos, 4)
self.assertEqual(parser._curpos, 4)
parser._do_vertical(0)
assert_equal(parser._curpos, 5)
assert_equal(parser._get_bits(), '11111')
self.assertEqual(parser._curpos, 5)
self.assertEqual(parser._get_bits(), '11111')
return
def test_e7(self):
@@ -122,8 +117,8 @@ class TestCCITTG4Parser():
parser._curpos = 2
parser._color = 1
parser._do_horizontal(2, 6)
assert_equal(parser._curpos, 10)
assert_equal(parser._get_bits(), '1111000000')
self.assertEqual(parser._curpos, 10)
self.assertEqual(parser._get_bits(), '1111000000')
return
def test_e8(self):
@@ -131,19 +126,19 @@ class TestCCITTG4Parser():
parser._curpos = 1
parser._color = 0
parser._do_vertical(0)
assert_equal(parser._curpos, 2)
self.assertEqual(parser._curpos, 2)
parser._do_horizontal(7, 0)
assert_equal(parser._curpos, 9)
assert_equal(parser._get_bits(), '101111111')
self.assertEqual(parser._curpos, 9)
self.assertEqual(parser._get_bits(), '101111111')
return
def test_m1(self):
parser = self.get_parser('10101')
parser._do_pass()
assert_equal(parser._curpos, 2)
self.assertEqual(parser._curpos, 2)
parser._do_pass()
assert_equal(parser._curpos, 4)
assert_equal(parser._get_bits(), '1111')
self.assertEqual(parser._curpos, 4)
self.assertEqual(parser._get_bits(), '1111')
return
def test_m2(self):
@@ -152,7 +147,7 @@ class TestCCITTG4Parser():
parser._do_vertical(-1)
parser._do_vertical(1)
parser._do_horizontal(1, 1)
assert_equal(parser._get_bits(), '011101')
self.assertEqual(parser._get_bits(), '011101')
return
def test_m3(self):
@@ -161,8 +156,8 @@ class TestCCITTG4Parser():
parser._do_pass()
parser._do_vertical(1)
parser._do_vertical(1)
assert_equal(parser._get_bits(), '00000001')
self.assertEqual(parser._get_bits(), '00000001')
return
if __name__ == '__main__':
nose.runmodule()
unittest.main()
+12 -14
View File
@@ -1,9 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from nose.tools import assert_equal
from nose import SkipTest
import nose
from unittest import TestCase, main
#test of various compression/encoding modules (previously in doctests):
from pdfmajor.utils.ascii85 import ascii85decode, asciihexdecode
@@ -14,24 +12,24 @@ import binascii
def hex(b): return binascii.hexlify(b) #encode('hex')
def dehex(b): return binascii.unhexlify(b) #decode('hex')
class TestAscii85():
class TestAscii85(TestCase):
def test_ascii85decode(self):
#The sample string is taken from: http://en.wikipedia.org/w/index.php?title=Ascii85
assert_equal(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished')
assert_equal(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.')
self.assertEqual(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished')
self.assertEqual(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.')
def test_asciihexdecode(self):
assert_equal(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde')
assert_equal(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep')
assert_equal(asciihexdecode(b'7>'),b'p')
self.assertEqual(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde')
self.assertEqual(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep')
self.assertEqual(asciihexdecode(b'7>'),b'p')
class TestLzw():
class TestLzw(TestCase):
def test_lzwdecode(self):
assert_equal(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42')
self.assertEqual(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42')
class TestRunlength():
class TestRunlength(TestCase):
def test_rldecode(self):
assert_equal(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde')
self.assertEqual(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde')
if __name__ == '__main__':
nose.runmodule()
main()
+33 -44
View File
@@ -2,19 +2,14 @@
# -*- coding: utf-8 -*-
from nose.tools import assert_equal, assert_true, assert_false
from nose import SkipTest
import nose
import logging
from unittest import TestCase, main
from io import BytesIO
from pdfmajor.parser.PSStackParser import KWD, LIT, PSBaseParser, PSEOF, PSStackParser
## Simplistic Test cases
##
class TestPSBaseParser:
TESTDATA = br'''%!PS
TESTDATA = br'''%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#xbaa
@@ -33,36 +28,32 @@ func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>
'''
TOKENS = [
(5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')),
(21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
(65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'),
(98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'),
(143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'),
(191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'),
(226, KWD(b'func')), (230, LIT('a')), (232, LIT('b')),
(234, KWD(b'{')), (235, b'c'), (238, KWD(b'do*')), (241, KWD(b'}')),
(242, KWD(b'def')), (246, KWD(b'[')), (248, 1), (250, b'z'), (254, KWD(b'!')),
(256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'),
(272, KWD(b'>>'))
]
OBJS = [
(23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
(65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'),
(98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'),
(143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'),
(191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'),
(230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']),
(258, {'foo': b'bar'}),
]
TOKENS = [
(5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')),
(21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
(65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'),
(98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'),
(143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'),
(191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'),
(226, KWD(b'func')), (230, LIT('a')), (232, LIT('b')),
(234, KWD(b'{')), (235, b'c'), (238, KWD(b'do*')), (241, KWD(b'}')),
(242, KWD(b'def')), (246, KWD(b'[')), (248, 1), (250, b'z'), (254, KWD(b'!')),
(256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'),
(272, KWD(b'>>'))
]
OBJS = [
(23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
(65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'),
(98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'),
(143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'),
(191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'),
(230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']),
(258, {'foo': b'bar'}),
]
class TestPSBaseParser(TestCase):
def get_tokens(self, s):
from io import BytesIO
class MyParser(PSStackParser):
def flush(self):
self.add_results(*self.popall())
@@ -76,8 +67,6 @@ func/a/b{(c)do*}def
return r
def get_objects(self, s):
from io import BytesIO
class MyParser(PSStackParser):
def flush(self):
self.add_results(*self.popall())
@@ -91,18 +80,18 @@ func/a/b{(c)do*}def
return r
def test_1(self):
tokens = self.get_tokens(self.TESTDATA)
tokens = self.get_tokens(TESTDATA)
logging.info(tokens)
assert_equal(tokens, self.TOKENS)
self.assertEqual(tokens, TOKENS)
return
def test_2(self):
objs = self.get_objects(self.TESTDATA)
objs = self.get_objects(TESTDATA)
logging.info(objs)
assert_equal(objs, self.OBJS)
self.assertEqual(objs, OBJS)
return
if __name__ == '__main__':
#import logging,sys,os,six
#logging.basicConfig(level=logging.DEBUG, filename='%s_%d.%d.log'%(os.path.basename(__file__),sys.version_info[0],sys.version_info[1]))
nose.runmodule()
main()