- better loggin system

- adde test case for bad characters
This commit is contained in:
ari committed 2018-11-23 14:43:51 -05:00
1 parent ac49ad1bcf
commit 38f8ca76a6
7 files changed
+95 -58

No files matched your search

+12 -8
View File
@@ -1,17 +1,19 @@
import logging
import re
import os
from typing import List
from io import BytesIO, StringIO, TextIOWrapper
from tqdm import tqdm
from ..layouts import PDFLayoutAnalyzer
from ..interpreter.PDFResourceManager import PDFResourceManager
from ..interpreter.PDFPage import PDFPage
from ..interpreter.PDFPageInterpreter import PDFPageInterpreter
from ..imagewriter import ImageWriter
from ..utils import set_log_level, get_logger, logging
log = logging.getLogger(__name__)
log = get_logger(__name__)
## PDFConverter
##
@@ -29,11 +31,10 @@ class PDFConverter(PDFLayoutAnalyzer):
check_extractable: bool = True,
pagenos: List[int] = None,
dont_export_images: bool = False,
debug: bool = False,
debug_level: int = logging.WARNING,
) -> TextIOWrapper:
if debug:
log.setLevel(logging.DEBUG)
set_log_level(debug_level)
if dont_export_images:
image_folder_path = None
@@ -53,9 +54,12 @@ class PDFConverter(PDFLayoutAnalyzer):
interpreter = PDFPageInterpreter(rsrcmgr, device)
log.debug("Reading pages....")
pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable)
for idx, page in enumerate(pages):
log.debug(f"Reading page #{idx}")
interpreter.process_page(page)
page_waiter = tqdm(enumerate(pages), disable=debug_level > logging.INFO)
for _, page in page_waiter:
page_waiter.set_description("Processing pdf...")
interpreter.process_page(page)
page_waiter.close()
device.close()
return output_file
+47 -41
View File
@@ -8,6 +8,7 @@ from .HTMLConverter import HTMLConverter
from .JSONConverter import JSONConverter
from .XMLConverter import XMLConverter
from .TextConverter import TextConverter
from ..utils import logging
def convert_file(
input_file: io.TextIOWrapper,
@@ -19,7 +20,8 @@ def convert_file(
caching: bool = True,
check_extractable: bool = True,
pagenos: List[int] = None,
debug: bool = False,
dont_export_images: bool = False,
debug_level: int = logging.WARNING,
out_type: str = 'html',
):
if output_file is None:
@@ -36,54 +38,58 @@ def convert_file(
raise Exception("Please make sure that the file name and output type match!")
if out_type == 'html':
return HTMLConverter.parse_file(
input_file,
output_file,
image_folder_path,
codec,
maxpages,
password,
caching,
check_extractable,
pagenos,
debug,
input_file=input_file,
output_file=output_file,
image_folder_path=image_folder_path,
codec=codec,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=check_extractable,
pagenos=pagenos,
dont_export_images=dont_export_images,
debug_level=debug_level,
)
elif out_type == 'xml':
return XMLConverter.parse_file(
input_file,
output_file,
image_folder_path,
codec,
maxpages,
password,
caching,
check_extractable,
pagenos,
debug,
input_file=input_file,
output_file=output_file,
image_folder_path=image_folder_path,
codec=codec,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=check_extractable,
pagenos=pagenos,
dont_export_images=dont_export_images,
debug_level=debug_level,
)
elif out_type == 'json':
return JSONConverter.parse_file(
input_file,
output_file,
image_folder_path,
codec,
maxpages,
password,
caching,
check_extractable,
pagenos,
debug,
input_file=input_file,
output_file=output_file,
image_folder_path=image_folder_path,
codec=codec,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=check_extractable,
pagenos=pagenos,
dont_export_images=dont_export_images,
debug_level=debug_level,
)
elif out_type == 'text':
return TextConverter.parse_file(
input_file,
output_file,
image_folder_path,
codec,
maxpages,
password,
caching,
check_extractable,
pagenos,
debug,
input_file=input_file,
output_file=output_file,
image_folder_path=image_folder_path,
codec=codec,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=check_extractable,
pagenos=pagenos,
dont_export_images=dont_export_images,
debug_level=debug_level,
)
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'")
+4 -2
View File
@@ -7,7 +7,9 @@ from .PDFStream import resolve1, int_value, list_value, dict_value
from .PDFParser import PDFParser
from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed
log = logging.getLogger(__name__)
from ..utils import get_logger
log = get_logger(__name__)
# some predefined literals and keywords.
LITERAL_PAGE = LIT('Page')
@@ -89,7 +91,7 @@ class PDFPage(object):
tree_type = tree.get('type')
if tree_type is LITERAL_PAGES and 'Kids' in tree:
log.info('Pages: Kids=%r', tree['Kids'])
# log.info('Pages: Kids=%r', tree['Kids'])
for c in list_value(tree['Kids']):
for x in search(c, tree):
yield x
+2 -1
View File
@@ -9,4 +9,5 @@ from .generic import *
from .lzw import *
from .matrix import *
from .text import *
from .types import *
from .types import *
from .log import *
+17
View File
@@ -0,0 +1,17 @@
import logging
logging.basicConfig(
level=logging.WARNING
)
LOGGERS = {}
def set_log_level(debug_level: int):
logging.basicConfig(level=debug_level)
for _, logger in LOGGERS.items():
logger.setLevel(debug_level)
def get_logger(logger_name: str) -> logging.Logger:
if logger_name not in LOGGERS.keys():
LOGGERS[logger_name] = logging.getLogger(logger_name)
return LOGGERS[logger_name]
Regular → Executable
+13 -6
View File
@@ -1,9 +1,12 @@
#!/usr/bin/env python
import re
import os
import logging
from typing import List
from pdfmajor.converters import convert_file
from xml.etree import ElementTree
from tqdm import tqdm
def rel_path(*arg: List[str]) -> str:
return os.path.join(os.path.dirname(__file__), *arg)
@@ -18,21 +21,25 @@ if not os.path.exists(OUTPUT_FOLDER):
files = os.listdir(INPUT_FOLDER)
TOTAL = len(files)
for idx, file_name in enumerate(files):
waiter = tqdm(enumerate(files), total=TOTAL)
for idx, file_name in waiter:
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
input_file_path = os.path.join(INPUT_FOLDER, file_name)
output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT)
if len(file_name) > len(file_name_no_ext):
print(f"processing {file_name} {idx}/{TOTAL}")
waiter.set_description(f"Processing {file_name}")
with open(input_file_path, 'rb') as input_file:
with open(output_file_path, 'wb') as output_file:
convert_file(
input_file=input_file,
output_file=output_file,
out_type=EXT,
debug=True,
debug_level=logging.NOTSET,
)
print(f" > testing format {file_name} {idx}/{TOTAL}")
ElementTree.parse(output_file_path)
print(f"processing {idx+1}/{TOTAL}")
if EXT == 'xml':
waiter.set_description(f" > testing format {file_name}")
ElementTree.parse(output_file_path)
waiter.refresh()
waiter.set_description("Done")
waiter.close()
Binary file not shown.