- added better mypy support

- updated most unit-test to use the unitttest module
- added a single unit-test run file
This commit is contained in:
ari committed 2019-06-10 16:26:52 -04:00
1 parent 290de97cb5
commit 9e923de7ec
22 files changed
+372 -319

No files matched your search

+3 -1
View File
@@ -11,4 +11,6 @@ tests/*.txt
.vscode .vscode
tests/samples/output tests/samples/output
samples samples
**__pycache** **__pycache**\
.mypy_cache
.ipynb_checkpoints
+1 -1
View File
@@ -1 +1 @@
v1.3.7 v1.3.8
+1 -1
View File
@@ -97,7 +97,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
""" """
__version__ = (1, 3, 7) __version__ = (1, 3, 8)
if __name__ == '__main__': if __name__ == '__main__':
print( print(
+5 -5
View File
@@ -2,7 +2,7 @@ import io
import os import os
import re import re
from typing import List from typing import List, Optional
from .html import convert_to_html from .html import convert_to_html
from .xml import convert_to_xml from .xml import convert_to_xml
@@ -13,14 +13,14 @@ from ..utils import logging
def convert_file( def convert_file(
input_file: str, input_file: str,
output_file: str = None, output_file: Optional[str] = None,
image_folder_path: str = None, image_folder_path: Optional[str] = None,
codec: str = 'utf-8', codec: str = 'utf-8',
maxpages: int = 0, maxpages: int = 0,
password: str = None, password: Optional[str] = None,
caching: bool = True, caching: bool = True,
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, pagenos: Optional[List[int]] = None,
dont_export_images: bool = False, dont_export_images: bool = False,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
out_type: str = 'html', out_type: str = 'html',
+3 -3
View File
@@ -1,4 +1,4 @@
from typing import List from typing import List, Optional
from .writers.html import HTMLMaker from .writers.html import HTMLMaker
from ..interpreter import PDFInterpreter, PageInterpreter, logging from ..interpreter import PDFInterpreter, PageInterpreter, logging
from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject
@@ -10,11 +10,11 @@ INF = (1<<31) - 1
def convert_to_html( def convert_to_html(
input_file_path: str, input_file_path: str,
output_file_path: str, output_file_path: str,
image_folder_path: str = None, image_folder_path: Optional[str] = None,
dont_export_images: bool = False, dont_export_images: bool = False,
codec: str = 'utf-8', codec: str = 'utf-8',
maxpages: int = 0, maxpages: int = 0,
password: str = None, password: Optional[str] = None,
caching: bool = True, caching: bool = True,
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, pagenos: List[int] = None,
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import List from typing import List, Optional
from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray
from ..interpreter import PDFInterpreter, PageInterpreter, logging from ..interpreter import PDFInterpreter, PageInterpreter, logging
@@ -21,14 +21,14 @@ def normalize_color_values(values: list):
def convert_to_json( def convert_to_json(
input_file_path: str, input_file_path: str,
output_file_path: str, output_file_path: str,
image_folder_path: str = None, image_folder_path: Optional[str] = None,
dont_export_images: bool = False, dont_export_images: bool = False,
codec: str = 'utf-8', codec: str = 'utf-8',
maxpages: int = 0, maxpages: int = 0,
password: str = None, password: str = Optional[None],
caching: bool = True, caching: bool = True,
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, pagenos: Optional[List[int]] = None,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
): ):
intepreter = PDFInterpreter(input_file_path, intepreter = PDFInterpreter(input_file_path,
+5 -4
View File
@@ -1,3 +1,4 @@
from typing import Optional
from io import TextIOWrapper from io import TextIOWrapper
from contextlib import contextmanager from contextlib import contextmanager
@@ -6,7 +7,7 @@ class HTMLMaker:
def __init__(self, file_path: str, codec: str = 'utf-8'): def __init__(self, file_path: str, codec: str = 'utf-8'):
self.file_path = file_path self.file_path = file_path
self.codec = codec self.codec = codec
self.outfile: TextIOWrapper = None self.outfile: Optional[TextIOWrapper] = None
def __enter__(self): def __enter__(self):
self.outfile = open(self.file_path, 'wb') self.outfile = open(self.file_path, 'wb')
@@ -42,18 +43,18 @@ class HTMLMaker:
f'{name}: {value}' f'{name}: {value}'
for name, value in css.items() for name, value in css.items()
]) ])
attrs = " ".join([ attrs_str = " ".join([
f'{name}="{value}"' f'{name}="{value}"'
for name, value in attrs.items() for name, value in attrs.items()
]) ])
if not singleton: if not singleton:
self.write(f"<{tag_name} {attrs}>", lineend=lineend) self.write(f"<{tag_name} {attrs_str}>", lineend=lineend)
self.__levels_deep += 1 self.__levels_deep += 1
yield yield
self.__levels_deep -= 1 self.__levels_deep -= 1
self.write(f"</{tag_name}>", deep_space=deep_space, lineend='\n' if not nolineend else '') self.write(f"</{tag_name}>", deep_space=deep_space, lineend='\n' if not nolineend else '')
else: else:
self.write(f"<{tag_name} {attrs} />", lineend=lineend, deep_space=deep_space) self.write(f"<{tag_name} {attrs_str} />", lineend=lineend, deep_space=deep_space)
yield yield
def singleton(self, tag_name: str, attrs: dict = None, css: dict = None, singleton: bool = True): def singleton(self, tag_name: str, attrs: dict = None, css: dict = None, singleton: bool = True):
+2 -2
View File
@@ -1,3 +1,4 @@
from typing import Set
from json import dumps as json_dump from json import dumps as json_dump
from io import TextIOWrapper from io import TextIOWrapper
from contextlib import contextmanager from contextlib import contextmanager
@@ -8,14 +9,13 @@ def JSONMaker(file_path: str, codec: str = 'utf-8'):
with JSONMakerObject(outf, codec=codec) as obj: with JSONMakerObject(outf, codec=codec) as obj:
yield obj yield obj
class JSONMakerObject: class JSONMakerObject:
class AccessError(Exception): pass class AccessError(Exception): pass
def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = 0): def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = 0):
self.outfile: TextIOWrapper = outfile self.outfile: TextIOWrapper = outfile
self.codec = codec self.codec = codec
self.levels_deep = levels_deep self.levels_deep = levels_deep
self.keys = set() self.keys: Set[str] = set()
def __enter__(self): def __enter__(self):
self.write_raw('{') self.write_raw('{')
@@ -1,10 +1,10 @@
from typing import List from typing import Tuple, Optional
from .PDFColorSpace import PDFColorSpace from .PDFColorSpace import PDFColorSpace
class PDFColor: class PDFColor:
def __init__(self, color_space: PDFColorSpace, *values: List[float]): def __init__(self, color_space: Optional[PDFColorSpace], *values: float):
self.color_space: PDFColorSpace = color_space self.color_space: Optional[PDFColorSpace] = color_space
self.values: list = values self.values: Tuple[float, ...] = values
def copy(self): def copy(self):
return self.__class__( return self.__class__(
@@ -1,3 +1,4 @@
from typing import Dict
from collections import OrderedDict from collections import OrderedDict
class PDFColorSpace(object): class PDFColorSpace(object):
@@ -5,13 +6,12 @@ class PDFColorSpace(object):
def __init__(self, name: str, ncomponents: int): def __init__(self, name: str, ncomponents: int):
self.name: str = name self.name: str = name
self.ncomponents: int = ncomponents self.ncomponents: int = ncomponents
return
def __repr__(self): def __repr__(self):
return '<PDFColorSpace: %s, ncomponents=%d>' % (self.name, self.ncomponents) return '<PDFColorSpace: %s, ncomponents=%d>' % (self.name, self.ncomponents)
PREDEFINED_COLORSPACE = OrderedDict() PREDEFINED_COLORSPACE: Dict[str, PDFColorSpace] = OrderedDict()
for (name, n) in [ for (name, n) in [
('DeviceGray', 1), # default value first ('DeviceGray', 1), # default value first
+4 -2
View File
@@ -1,5 +1,6 @@
import re import re
from logging import getLogger from logging import getLogger
from typing import Tuple, Optional, Dict
from ...utils import settings, choplist, nunpack from ...utils import settings, choplist, nunpack
from ..PSStackParser import PSEOF from ..PSStackParser import PSEOF
@@ -13,6 +14,7 @@ from .exceptions import PDFNoValidXRef, PDFSyntaxError
log = getLogger(__name__) log = getLogger(__name__)
Pos = Tuple[Optional[int], int, int]
## XRefs ## XRefs
## ##
@@ -27,7 +29,7 @@ class PDFBaseXRef(object):
# Must return # Must return
# (strmid, index, genno) # (strmid, index, genno)
# or (None, pos, genno) # or (None, pos, genno)
def get_pos(self, objid): def get_pos(self, objid) -> Pos:
raise KeyError(objid) raise KeyError(objid)
@@ -36,7 +38,7 @@ class PDFBaseXRef(object):
class PDFXRef(PDFBaseXRef): class PDFXRef(PDFBaseXRef):
def __init__(self): def __init__(self):
self.offsets = {} self.offsets: Dict[int, Pos] = {}
self.trailer = {} self.trailer = {}
return return
+10 -3
View File
@@ -10,12 +10,19 @@ import sys
with open('README.md') as readme_file: with open('README.md') as readme_file:
SETUP = { SETUP = {
"name": "pdfmajor", "name": "pdfmajor",
"version": "1.3.7", "version": "1.3.8",
"packages": find_packages(include=['pdfmajor*']), "packages": find_packages(include=['pdfmajor*']),
"install_requires": [ "install_requires": [
'pycryptodome', 'pycryptodome==3.8.2',
'chardet' 'chardet==3.0.4',
'Pillow==6.0.0'
], ],
'extras_require': {
'dev': [
'tqdm==4.32.1',
'nose==1.3.7'
]
},
"description": 'PDF parser', "description": 'PDF parser',
"long_description": readme_file.read(), "long_description": readme_file.read(),
'long_description_content_type': 'text/markdown', 'long_description_content_type': 'text/markdown',
+4
View File
@@ -0,0 +1,4 @@
from unittest import TestLoader, TextTestRunner
tests = TestLoader().discover('./tests')
result = TextTestRunner(verbosity=20).run(tests)
+15 -12
View File
@@ -1,6 +1,5 @@
import os import os
from tqdm import tqdm from unittest import TestCase, main
# from pdfmajor.extractor import extract_items_from_pdf # from pdfmajor.extractor import extract_items_from_pdf
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
from pdfmajor.interpreter import PageInterpreter from pdfmajor.interpreter import PageInterpreter
@@ -23,13 +22,17 @@ def count_elms(file_path: str):
count += 1 count += 1
return count return count
with tqdm(desc='Cache Test', total=len(FILES)*NUM_LOOPS) as waiter: class CachingTester(TestCase):
for file_name in FILES: def test_cache(self):
waiter.set_description(f"Cache Test - [{file_name}]") for file_name in FILES:
file_path = os.path.join(INPUT_FOLDER, file_name) self.subTest(name = file_name)
last_count = count_elms(file_path) file_path = os.path.join(INPUT_FOLDER, file_name)
for _ in range(NUM_LOOPS): last_count = count_elms(file_path)
new_count = count_elms(file_path) for _ in range(NUM_LOOPS):
assert new_count == last_count new_count = count_elms(file_path)
last_count = new_count self.assertEqual( new_count , last_count )
waiter.update() last_count = new_count
if __name__ == '__main__':
# Run Tests
main()
+94 -89
View File
@@ -1,6 +1,6 @@
import os import os
from typing import List from typing import List
from tqdm import tqdm from unittest import TestCase, main
# from pdfmajor.extractor import extract_items_from_pdf # from pdfmajor.extractor import extract_items_from_pdf
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
@@ -15,96 +15,101 @@ INPUT_FOLDER = os.path.join(
FILES = os.listdir(INPUT_FOLDER) FILES = os.listdir(INPUT_FOLDER)
NUM_LOOPS = 10 NUM_LOOPS = 10
file_path = os.path.join(INPUT_FOLDER, "lorem-v4.pdf") class ColorTest(TestCase):
def run_test(self, file_path: str, curve_defs: List[List[PDFColor]]):
def run_once(item):
if isinstance(item, LTCurve):
t_stroke, t_fill = curve_defs.pop(0)
self.assertEqual( item.stroke.values, t_stroke.values )
self.assertEqual( item.stroke.color_space, t_stroke.color_space )
self.assertEqual( item.fill.values, t_fill.values )
self.assertEqual( item.fill.color_space, t_fill.color_space )
elif isinstance(item, LTXObject):
for c in item:
run_once(c)
def run_test(file_path: str, curve_defs: List[List[PDFColor]]): for page in PDFInterpreter(file_path):
def run_once(item): for item in page:
if isinstance(item, LTCurve): run_once(item)
t_stroke, t_fill = curve_defs.pop(0)
# print([item.stroke, item.fill])
# print('...>', [t_stroke, t_fill])
assert item.stroke.values == t_stroke.values
assert item.stroke.color_space == t_stroke.color_space
assert item.fill.values == t_fill.values
assert item.fill.color_space == t_fill.color_space
elif isinstance(item, LTXObject):
for c in item:
run_once(c)
for page in PDFInterpreter(file_path): def test_lorem_v3(self):
for item in page: self.run_test(
run_once(item) os.path.join(INPUT_FOLDER, "lorem-v3.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 0, 0),
]
] +
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]*24)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*11)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*5)+
[[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]
)
run_test( def test_lorem_v4(self):
os.path.join(INPUT_FOLDER, "lorem-v3.pdf"), self.run_test(
[ os.path.join(INPUT_FOLDER, "lorem-v4.pdf"),
[ [
PDFColor(None), [
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), PDFColor(None),
], PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
[ ],
PDFColor(None), [
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207), PDFColor(None),
], PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 64, 64, 64),
[ ]
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0), ]
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207), )
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 0, 0),
]
] +
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]*24)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*11)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*5)+
[[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]
)
run_test( def test_lorem_v5(self):
os.path.join(INPUT_FOLDER, "lorem-v4.pdf"), self.run_test(
[ os.path.join(INPUT_FOLDER, "lorem-v5.pdf"),
[ [
PDFColor(None), [
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255), PDFColor(None),
], PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
[ ],
PDFColor(None), [
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 64, 64, 64), PDFColor(None),
] PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
] ]
) ]
)
run_test(
os.path.join(INPUT_FOLDER, "lorem-v5.pdf"), if __name__ == '__main__':
[ # Run Tests
[ main()
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
]
]
)
+36 -25
View File
@@ -3,13 +3,14 @@ import re
import os import os
import logging import logging
from unittest import TestCase, main
from typing import List from typing import List
from pdfmajor.converters import convert_file from pdfmajor.converters import convert_file
from xml.etree import ElementTree from xml.etree import ElementTree
from json import load as json_load from json import load as json_load
from tqdm import tqdm from tqdm import tqdm
def rel_path(*arg: List[str]) -> str: def rel_path(*arg: str) -> str:
return os.path.normpath(os.path.join(os.path.dirname(__file__), *arg)) return os.path.normpath(os.path.join(os.path.dirname(__file__), *arg))
EXTS = ['text', 'json','xml','html'] EXTS = ['text', 'json','xml','html']
@@ -22,28 +23,38 @@ if not os.path.exists(OUTPUT_FOLDER):
files = os.listdir(INPUT_FOLDER) files = os.listdir(INPUT_FOLDER)
TOTAL = len(files)*len(EXTS) TOTAL = len(files)*len(EXTS)
with tqdm(total=TOTAL) as waiter:
for idx, file_name in enumerate(files):
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
input_file_path = os.path.join(INPUT_FOLDER, file_name)
for ext in EXTS: class ConversionTest(TestCase):
if not os.path.exists(os.path.join(OUTPUT_FOLDER, ext)): def test_conversion(self):
os.makedirs(os.path.join(OUTPUT_FOLDER, ext)) # with tqdm(total=TOTAL) as waiter:
output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext) for idx, file_name in enumerate(files):
if len(file_name) > len(file_name_no_ext): self.subTest(i=idx);
waiter.set_description(f"Conversion test -- {file_name}") file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
convert_file( input_file_path = os.path.join(INPUT_FOLDER, file_name)
input_file=input_file_path,
output_file=output_file_path, for ext in EXTS:
out_type=ext, if not os.path.exists(os.path.join(OUTPUT_FOLDER, ext)):
debug_level=logging.WARN, os.makedirs(os.path.join(OUTPUT_FOLDER, ext))
) output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext)
if ext == 'xml': if len(file_name) > len(file_name_no_ext):
waiter.set_description(f" > testing format {file_name}") # waiter.set_description(f"Conversion test -- {file_name}")
ElementTree.parse(output_file_path) convert_file(
elif ext == 'json': input_file=input_file_path,
waiter.set_description(f" > testing format {file_name}") output_file=output_file_path,
json_load(open(output_file_path, 'r')) out_type=ext,
waiter.update() debug_level=logging.WARN,
waiter.set_description("Done") )
if ext == 'xml':
# waiter.set_description(f" > testing format {file_name}")
ElementTree.parse(output_file_path)
elif ext == 'json':
# waiter.set_description(f" > testing format {file_name}")
with open(output_file_path, 'r') as outf:
json_load(outf)
# waiter.update()
# waiter.set_description("Done")
if __name__ == '__main__':
# Run Tests
main()
+52 -30
View File
@@ -1,43 +1,60 @@
from contextlib import contextmanager
from unittest import TestCase, main
from typing import List, Tuple, Callable
from os import remove from os import remove
from tempfile import mktemp from tempfile import mktemp
from pdfmajor.converters.writers.html import HTMLMaker from pdfmajor.converters.writers.html import HTMLMaker
from tqdm import tqdm
# Make temp file # Make temp file
FILE_NAME = mktemp() FILE_NAME = mktemp()
TestFunction = Callable[[HTMLMaker], str];
TESTS: List[Tuple[TestFunction, str]] = [];
with HTMLMaker(FILE_NAME) as html: class HTMLTester(TestCase):
with html.elm("html", nolineend=True): @contextmanager
html.write("test") def get_html(self):
assert(open(FILE_NAME, 'r').read() == """<html > with HTMLMaker(FILE_NAME) as html:
yield html
def compare(self, expected: str):
with open(FILE_NAME, 'r') as f:
self.assertEqual(f.read() , expected)
def test_html_basic(self):
with self.get_html() as html:
with html.elm("html", nolineend=True):
html.write("test")
self.compare("""<html >
test test
</html>""") </html>""")
def test_html_with_attributes(self):
with HTMLMaker(FILE_NAME) as html: with self.get_html() as html:
with html.elm("html", nolineend=True): with html.elm("html", nolineend=True):
with html.elm("head"): with html.elm("head"):
html.singleton("meta", { html.singleton("meta", {
'http-equiv': "Content-Type", 'http-equiv': "Content-Type",
'content': "text/html" 'content': "text/html"
}) })
assert(open(FILE_NAME, 'r').read() == """<html > self.compare("""<html >
<head > <head >
<meta http-equiv="Content-Type" content="text/html" /> <meta http-equiv="Content-Type" content="text/html" />
</head> </head>
</html>""") </html>""")
with HTMLMaker(FILE_NAME) as html: def test_html_with_children(self):
with html.elm("html", nolineend=True): with self.get_html() as html:
with html.elm("head"): with html.elm("html", nolineend=True):
html.singleton("meta", { with html.elm("head"):
'http-equiv': "Content-Type", html.singleton("meta", {
'content': "text/html" 'http-equiv': "Content-Type",
}) 'content': "text/html"
with html.elm("body"): })
with html.elm("h1", css={"color": "red"}): with html.elm("body"):
html.write("Hello World") with html.elm("h1", css={"color": "red"}):
assert(open(FILE_NAME, 'r').read() == """<html > html.write("Hello World")
self.compare("""<html >
<head > <head >
<meta http-equiv="Content-Type" content="text/html" /> <meta http-equiv="Content-Type" content="text/html" />
</head> </head>
@@ -48,13 +65,18 @@ assert(open(FILE_NAME, 'r').read() == """<html >
</body> </body>
</html>""") </html>""")
with HTMLMaker(FILE_NAME) as html: def test_html_with_css(self):
with html.elm('style'): with self.get_html() as html:
html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''') with html.elm('style'):
assert(open(FILE_NAME, 'r').read() == """<style > html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''')
self.compare("""<style >
.page { position: relative; overflow: hidden;border: 1px black solid;} .page { position: relative; overflow: hidden;border: 1px black solid;}
</style> </style>
""") """)
# Clean up
remove(FILE_NAME) if __name__ == '__main__':
# Run Tests
main()
# Clean up
remove(FILE_NAME)
+21 -14
View File
@@ -1,5 +1,5 @@
import os import os
from tqdm import tqdm from unittest import TestCase, main
from pdfmajor.interpreter import LTImage, logging from pdfmajor.interpreter import LTImage, logging
from pdfmajor.interpreter import PageInterpreter, PDFInterpreter from pdfmajor.interpreter import PageInterpreter, PDFInterpreter
@@ -19,16 +19,23 @@ OUTPUT_FOLDER = os.path.join(
if not os.path.exists(OUTPUT_FOLDER): if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER) os.makedirs(OUTPUT_FOLDER)
for file_name in tqdm(FILES, desc='Image Writer Test'): class ImageTester(TestCase):
for page in PDFInterpreter(os.path.join( def test_image_extract(self):
CUR_PATH, for file_name in FILES:
f"tests/samples/pdf/{file_name}.pdf" self.subTest(file_name=file_name)
), debug_level=logging.ERROR): for page in PDFInterpreter(os.path.join(
assert isinstance(page, PageInterpreter) CUR_PATH,
for item_idx, item in enumerate(page): f"tests/samples/pdf/{file_name}.pdf"
assert isinstance(item, LTItem) ), debug_level=logging.ERROR):
if isinstance(item, LTImage): self.assertTrue( isinstance(page, PageInterpreter) )
item.save_image( for item_idx, item in enumerate(page):
OUTPUT_FOLDER, self.assertTrue( isinstance(item, LTItem) )
file_name=f'{file_name}_p{page.page_num}i{item_idx}' if isinstance(item, LTImage):
) item.save_image(
OUTPUT_FOLDER,
file_name=f'{file_name}_p{page.page_num}i{item_idx}'
)
if __name__ == '__main__':
# Run Tests
main()
+17 -10
View File
@@ -1,5 +1,5 @@
import os import os
from tqdm import tqdm from unittest import TestCase, main
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
from pdfmajor.interpreter import PageInterpreter from pdfmajor.interpreter import PageInterpreter
from pdfmajor.interpreter.commands import LTItem from pdfmajor.interpreter.commands import LTItem
@@ -11,12 +11,19 @@ INPUT_FOLDER = os.path.join(
) )
FILES = os.listdir(INPUT_FOLDER) FILES = os.listdir(INPUT_FOLDER)
for file_name in tqdm(FILES, desc="Interpreter Test"): class TestInterpreter(TestCase):
file_path = os.path.join(INPUT_FOLDER, file_name) def test_pdfintp(self):
for page in PDFInterpreter(file_path, debug_level=logging.ERROR): for file_name in FILES:
assert isinstance(page, PageInterpreter) self.subTest(file_name=file_name)
for item in page: file_path = os.path.join(INPUT_FOLDER, file_name)
assert isinstance(item, LTItem) for page in PDFInterpreter(file_path, debug_level=logging.ERROR):
if isinstance(item, LTCharBlock): self.assertTrue( isinstance(page, PageInterpreter) )
for char in item: for item in page:
assert isinstance(char, LTChar) self.assertTrue( isinstance(item, LTItem) )
if isinstance(item, LTCharBlock):
for char in item:
self.assertTrue( isinstance(char, LTChar) )
if __name__ == '__main__':
# Run Tests
main()
+44 -49
View File
@@ -2,17 +2,12 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from nose.tools import assert_equal, assert_true, assert_false import unittest
from nose import SkipTest
import nose
import logging
from pdfmajor.parser.PDFStream.ccitt import CCITTG4Parser from pdfmajor.parser.PDFStream.ccitt import CCITTG4Parser
## Test cases ## Test cases
## ##
class TestCCITTG4Parser(): class TestCCITTG4Parser(unittest.TestCase):
def get_parser(self, bits): def get_parser(self, bits):
parser = CCITTG4Parser(len(bits)) parser = CCITTG4Parser(len(bits))
@@ -23,98 +18,98 @@ class TestCCITTG4Parser():
def test_b1(self): def test_b1(self):
parser = self.get_parser('00000') parser = self.get_parser('00000')
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 0) self.assertEqual(parser._curpos, 0)
return return
def test_b2(self): def test_b2(self):
parser = self.get_parser('10000') parser = self.get_parser('10000')
parser._do_vertical(-1) parser._do_vertical(-1)
assert_equal(parser._curpos, 0) self.assertEqual(parser._curpos, 0)
return return
def test_b3(self): def test_b3(self):
parser = self.get_parser('000111') parser = self.get_parser('000111')
parser._do_pass() parser._do_pass()
assert_equal(parser._curpos, 3) self.assertEqual(parser._curpos, 3)
assert_equal(parser._get_bits(), '111') self.assertEqual(parser._get_bits(), '111')
return return
def test_b4(self): def test_b4(self):
parser = self.get_parser('00000') parser = self.get_parser('00000')
parser._do_vertical(+2) parser._do_vertical(+2)
assert_equal(parser._curpos, 2) self.assertEqual(parser._curpos, 2)
assert_equal(parser._get_bits(), '11') self.assertEqual(parser._get_bits(), '11')
return return
def test_b5(self): def test_b5(self):
parser = self.get_parser('11111111100') parser = self.get_parser('11111111100')
parser._do_horizontal(0, 3) parser._do_horizontal(0, 3)
assert_equal(parser._curpos, 3) self.assertEqual(parser._curpos, 3)
parser._do_vertical(1) parser._do_vertical(1)
assert_equal(parser._curpos, 10) self.assertEqual(parser._curpos, 10)
assert_equal(parser._get_bits(), '0001111111') self.assertEqual(parser._get_bits(), '0001111111')
return return
def test_e1(self): def test_e1(self):
parser = self.get_parser('10000') parser = self.get_parser('10000')
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 5) self.assertEqual(parser._curpos, 5)
assert_equal(parser._get_bits(), '10000') self.assertEqual(parser._get_bits(), '10000')
return return
def test_e2(self): def test_e2(self):
parser = self.get_parser('10011') parser = self.get_parser('10011')
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(2) parser._do_vertical(2)
assert_equal(parser._curpos, 5) self.assertEqual(parser._curpos, 5)
assert_equal(parser._get_bits(), '10000') self.assertEqual(parser._get_bits(), '10000')
return return
def test_e3(self): def test_e3(self):
parser = self.get_parser('011111') parser = self.get_parser('011111')
parser._color = 0 parser._color = 0
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._color, 1) self.assertEqual(parser._color, 1)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(-2) parser._do_vertical(-2)
assert_equal(parser._color, 0) self.assertEqual(parser._color, 0)
assert_equal(parser._curpos, 4) self.assertEqual(parser._curpos, 4)
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 6) self.assertEqual(parser._curpos, 6)
assert_equal(parser._get_bits(), '011100') self.assertEqual(parser._get_bits(), '011100')
return return
def test_e4(self): def test_e4(self):
parser = self.get_parser('10000') parser = self.get_parser('10000')
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(-2) parser._do_vertical(-2)
assert_equal(parser._curpos, 3) self.assertEqual(parser._curpos, 3)
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 5) self.assertEqual(parser._curpos, 5)
assert_equal(parser._get_bits(), '10011') self.assertEqual(parser._get_bits(), '10011')
return return
def test_e5(self): def test_e5(self):
parser = self.get_parser('011000') parser = self.get_parser('011000')
parser._color = 0 parser._color = 0
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(3) parser._do_vertical(3)
assert_equal(parser._curpos, 6) self.assertEqual(parser._curpos, 6)
assert_equal(parser._get_bits(), '011111') self.assertEqual(parser._get_bits(), '011111')
return return
def test_e6(self): def test_e6(self):
parser = self.get_parser('11001') parser = self.get_parser('11001')
parser._do_pass() parser._do_pass()
assert_equal(parser._curpos, 4) self.assertEqual(parser._curpos, 4)
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 5) self.assertEqual(parser._curpos, 5)
assert_equal(parser._get_bits(), '11111') self.assertEqual(parser._get_bits(), '11111')
return return
def test_e7(self): def test_e7(self):
@@ -122,8 +117,8 @@ class TestCCITTG4Parser():
parser._curpos = 2 parser._curpos = 2
parser._color = 1 parser._color = 1
parser._do_horizontal(2, 6) parser._do_horizontal(2, 6)
assert_equal(parser._curpos, 10) self.assertEqual(parser._curpos, 10)
assert_equal(parser._get_bits(), '1111000000') self.assertEqual(parser._get_bits(), '1111000000')
return return
def test_e8(self): def test_e8(self):
@@ -131,19 +126,19 @@ class TestCCITTG4Parser():
parser._curpos = 1 parser._curpos = 1
parser._color = 0 parser._color = 0
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 2) self.assertEqual(parser._curpos, 2)
parser._do_horizontal(7, 0) parser._do_horizontal(7, 0)
assert_equal(parser._curpos, 9) self.assertEqual(parser._curpos, 9)
assert_equal(parser._get_bits(), '101111111') self.assertEqual(parser._get_bits(), '101111111')
return return
def test_m1(self): def test_m1(self):
parser = self.get_parser('10101') parser = self.get_parser('10101')
parser._do_pass() parser._do_pass()
assert_equal(parser._curpos, 2) self.assertEqual(parser._curpos, 2)
parser._do_pass() parser._do_pass()
assert_equal(parser._curpos, 4) self.assertEqual(parser._curpos, 4)
assert_equal(parser._get_bits(), '1111') self.assertEqual(parser._get_bits(), '1111')
return return
def test_m2(self): def test_m2(self):
@@ -152,7 +147,7 @@ class TestCCITTG4Parser():
parser._do_vertical(-1) parser._do_vertical(-1)
parser._do_vertical(1) parser._do_vertical(1)
parser._do_horizontal(1, 1) parser._do_horizontal(1, 1)
assert_equal(parser._get_bits(), '011101') self.assertEqual(parser._get_bits(), '011101')
return return
def test_m3(self): def test_m3(self):
@@ -161,8 +156,8 @@ class TestCCITTG4Parser():
parser._do_pass() parser._do_pass()
parser._do_vertical(1) parser._do_vertical(1)
parser._do_vertical(1) parser._do_vertical(1)
assert_equal(parser._get_bits(), '00000001') self.assertEqual(parser._get_bits(), '00000001')
return return
if __name__ == '__main__': if __name__ == '__main__':
nose.runmodule() unittest.main()
+12 -14
View File
@@ -1,9 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from nose.tools import assert_equal from unittest import TestCase, main
from nose import SkipTest
import nose
#test of various compression/encoding modules (previously in doctests): #test of various compression/encoding modules (previously in doctests):
from pdfmajor.utils.ascii85 import ascii85decode, asciihexdecode from pdfmajor.utils.ascii85 import ascii85decode, asciihexdecode
@@ -14,24 +12,24 @@ import binascii
def hex(b): return binascii.hexlify(b) #encode('hex') def hex(b): return binascii.hexlify(b) #encode('hex')
def dehex(b): return binascii.unhexlify(b) #decode('hex') def dehex(b): return binascii.unhexlify(b) #decode('hex')
class TestAscii85(): class TestAscii85(TestCase):
def test_ascii85decode(self): def test_ascii85decode(self):
#The sample string is taken from: http://en.wikipedia.org/w/index.php?title=Ascii85 #The sample string is taken from: http://en.wikipedia.org/w/index.php?title=Ascii85
assert_equal(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished') self.assertEqual(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished')
assert_equal(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.') self.assertEqual(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.')
def test_asciihexdecode(self): def test_asciihexdecode(self):
assert_equal(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde') self.assertEqual(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde')
assert_equal(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep') self.assertEqual(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep')
assert_equal(asciihexdecode(b'7>'),b'p') self.assertEqual(asciihexdecode(b'7>'),b'p')
class TestLzw(): class TestLzw(TestCase):
def test_lzwdecode(self): def test_lzwdecode(self):
assert_equal(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42') self.assertEqual(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42')
class TestRunlength(): class TestRunlength(TestCase):
def test_rldecode(self): def test_rldecode(self):
assert_equal(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde') self.assertEqual(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde')
if __name__ == '__main__': if __name__ == '__main__':
nose.runmodule() main()
+33 -44
View File
@@ -2,19 +2,14 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from nose.tools import assert_equal, assert_true, assert_false
from nose import SkipTest
import nose
import logging import logging
from unittest import TestCase, main
from io import BytesIO
from pdfmajor.parser.PSStackParser import KWD, LIT, PSBaseParser, PSEOF, PSStackParser from pdfmajor.parser.PSStackParser import KWD, LIT, PSBaseParser, PSEOF, PSStackParser
## Simplistic Test cases ## Simplistic Test cases
## ##
class TestPSBaseParser: TESTDATA = br'''%!PS
TESTDATA = br'''%!PS
begin end begin end
" @ # " @ #
/a/BCD /Some_Name /foo#5f#xbaa /a/BCD /Some_Name /foo#5f#xbaa
@@ -33,36 +28,32 @@ func/a/b{(c)do*}def
[ 1 (z) ! ] [ 1 (z) ! ]
<< /foo (bar) >> << /foo (bar) >>
''' '''
TOKENS = [
TOKENS = [ (5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')),
(5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')), (21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
(21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), (65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'),
(65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'), (98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'),
(98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'), (143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'),
(143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'), (191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'),
(191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'), (226, KWD(b'func')), (230, LIT('a')), (232, LIT('b')),
(226, KWD(b'func')), (230, LIT('a')), (232, LIT('b')), (234, KWD(b'{')), (235, b'c'), (238, KWD(b'do*')), (241, KWD(b'}')),
(234, KWD(b'{')), (235, b'c'), (238, KWD(b'do*')), (241, KWD(b'}')), (242, KWD(b'def')), (246, KWD(b'[')), (248, 1), (250, b'z'), (254, KWD(b'!')),
(242, KWD(b'def')), (246, KWD(b'[')), (248, 1), (250, b'z'), (254, KWD(b'!')), (256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'),
(256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'), (272, KWD(b'>>'))
(272, KWD(b'>>')) ]
] OBJS = [
(23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
OBJS = [ (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
(23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), (65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), (98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'),
(65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'), (143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'),
(98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'), (191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'),
(143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'), (230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']),
(191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'), (258, {'foo': b'bar'}),
(230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']), ]
(258, {'foo': b'bar'}), class TestPSBaseParser(TestCase):
]
def get_tokens(self, s): def get_tokens(self, s):
from io import BytesIO
class MyParser(PSStackParser): class MyParser(PSStackParser):
def flush(self): def flush(self):
self.add_results(*self.popall()) self.add_results(*self.popall())
@@ -76,8 +67,6 @@ func/a/b{(c)do*}def
return r return r
def get_objects(self, s): def get_objects(self, s):
from io import BytesIO
class MyParser(PSStackParser): class MyParser(PSStackParser):
def flush(self): def flush(self):
self.add_results(*self.popall()) self.add_results(*self.popall())
@@ -91,18 +80,18 @@ func/a/b{(c)do*}def
return r return r
def test_1(self): def test_1(self):
tokens = self.get_tokens(self.TESTDATA) tokens = self.get_tokens(TESTDATA)
logging.info(tokens) logging.info(tokens)
assert_equal(tokens, self.TOKENS) self.assertEqual(tokens, TOKENS)
return return
def test_2(self): def test_2(self):
objs = self.get_objects(self.TESTDATA) objs = self.get_objects(TESTDATA)
logging.info(objs) logging.info(objs)
assert_equal(objs, self.OBJS) self.assertEqual(objs, OBJS)
return return
if __name__ == '__main__': if __name__ == '__main__':
#import logging,sys,os,six #import logging,sys,os,six
#logging.basicConfig(level=logging.DEBUG, filename='%s_%d.%d.log'%(os.path.basename(__file__),sys.version_info[0],sys.version_info[1])) #logging.basicConfig(level=logging.DEBUG, filename='%s_%d.%d.log'%(os.path.basename(__file__),sys.version_info[0],sys.version_info[1]))
nose.runmodule() main()