- custom formatters done

- json formatter works now
- added some tests for writers
This commit is contained in:
ari committed 2018-12-22 22:44:53 -05:00
1 parent 3ba0b95001
commit bc20627d9f
21 files changed
+981 -552

No files matched your search

+27 -21
View File
@@ -6,40 +6,46 @@ import logging
from typing import List
from pdfmajor.converters import convert_file
from xml.etree import ElementTree
from json import load as json_load
from tqdm import tqdm
def rel_path(*arg: List[str]) -> str:
return os.path.join(os.path.dirname(__file__), *arg)
return os.path.normpath(os.path.join(os.path.dirname(__file__), *arg))
EXT = 'html'
EXTS = ['json','xml','html']
INPUT_FOLDER = rel_path("./pdf")
OUTPUT_FOLDER = rel_path(f"./output/{EXT}")
OUTPUT_FOLDER = rel_path(f"./output/")
if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER)
files = os.listdir(INPUT_FOLDER)
TOTAL = len(files)
waiter = tqdm(enumerate(files), total=TOTAL)
for idx, file_name in waiter:
TOTAL = len(files)*len(EXTS)
waiter = tqdm(total=TOTAL)
for idx, file_name in enumerate(files):
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
input_file_path = os.path.join(INPUT_FOLDER, file_name)
output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT)
if len(file_name) > len(file_name_no_ext):
waiter.set_description(f"Processing {file_name}")
with open(input_file_path, 'rb') as input_file:
with open(output_file_path, 'wb') as output_file:
convert_file(
input_file=input_file,
output_file=output_file,
out_type=EXT,
debug_level=logging.NOTSET,
)
if EXT == 'xml':
waiter.set_description(f" > testing format {file_name}")
ElementTree.parse(output_file_path)
waiter.refresh()
for ext in EXTS:
if not os.path.exists(os.path.join(OUTPUT_FOLDER, ext)):
os.makedirs(os.path.join(OUTPUT_FOLDER, ext))
output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext)
if len(file_name) > len(file_name_no_ext):
waiter.set_description(f"Processing {file_name}")
convert_file(
input_file=input_file_path,
output_file=output_file_path,
out_type=ext,
debug_level=logging.WARN,
)
if ext == 'xml':
waiter.set_description(f" > testing format {file_name}")
ElementTree.parse(output_file_path)
elif ext == 'json':
waiter.set_description(f" > testing format {file_name}")
json_load(open(output_file_path, 'r'))
waiter.refresh()
waiter.update()
waiter.set_description("Done")
waiter.close()