- better loggin system
- adde test case for bad characters
This commit is contained in:
1 parent
ac49ad1bcf
commit
38f8ca76a6
7 files changed
+95
-58
No files matched your search
@@ -1,17 +1,19 @@
|
||||
import logging
|
||||
import re
|
||||
import os
|
||||
|
||||
from typing import List
|
||||
from io import BytesIO, StringIO, TextIOWrapper
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
from ..layouts import PDFLayoutAnalyzer
|
||||
from ..interpreter.PDFResourceManager import PDFResourceManager
|
||||
from ..interpreter.PDFPage import PDFPage
|
||||
from ..interpreter.PDFPageInterpreter import PDFPageInterpreter
|
||||
from ..imagewriter import ImageWriter
|
||||
from ..utils import set_log_level, get_logger, logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log = get_logger(__name__)
|
||||
|
||||
## PDFConverter
|
||||
##
|
||||
@@ -29,11 +31,10 @@ class PDFConverter(PDFLayoutAnalyzer):
|
||||
check_extractable: bool = True,
|
||||
pagenos: List[int] = None,
|
||||
dont_export_images: bool = False,
|
||||
debug: bool = False,
|
||||
debug_level: int = logging.WARNING,
|
||||
) -> TextIOWrapper:
|
||||
|
||||
if debug:
|
||||
log.setLevel(logging.DEBUG)
|
||||
set_log_level(debug_level)
|
||||
|
||||
if dont_export_images:
|
||||
image_folder_path = None
|
||||
@@ -53,9 +54,12 @@ class PDFConverter(PDFLayoutAnalyzer):
|
||||
interpreter = PDFPageInterpreter(rsrcmgr, device)
|
||||
log.debug("Reading pages....")
|
||||
pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable)
|
||||
for idx, page in enumerate(pages):
|
||||
log.debug(f"Reading page #{idx}")
|
||||
interpreter.process_page(page)
|
||||
|
||||
page_waiter = tqdm(enumerate(pages), disable=debug_level > logging.INFO)
|
||||
for _, page in page_waiter:
|
||||
page_waiter.set_description("Processing pdf...")
|
||||
interpreter.process_page(page)
|
||||
page_waiter.close()
|
||||
device.close()
|
||||
return output_file
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from .HTMLConverter import HTMLConverter
|
||||
from .JSONConverter import JSONConverter
|
||||
from .XMLConverter import XMLConverter
|
||||
from .TextConverter import TextConverter
|
||||
from ..utils import logging
|
||||
|
||||
def convert_file(
|
||||
input_file: io.TextIOWrapper,
|
||||
@@ -19,7 +20,8 @@ def convert_file(
|
||||
caching: bool = True,
|
||||
check_extractable: bool = True,
|
||||
pagenos: List[int] = None,
|
||||
debug: bool = False,
|
||||
dont_export_images: bool = False,
|
||||
debug_level: int = logging.WARNING,
|
||||
out_type: str = 'html',
|
||||
):
|
||||
if output_file is None:
|
||||
@@ -36,54 +38,58 @@ def convert_file(
|
||||
raise Exception("Please make sure that the file name and output type match!")
|
||||
if out_type == 'html':
|
||||
return HTMLConverter.parse_file(
|
||||
input_file,
|
||||
output_file,
|
||||
image_folder_path,
|
||||
codec,
|
||||
maxpages,
|
||||
password,
|
||||
caching,
|
||||
check_extractable,
|
||||
pagenos,
|
||||
debug,
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
elif out_type == 'xml':
|
||||
return XMLConverter.parse_file(
|
||||
input_file,
|
||||
output_file,
|
||||
image_folder_path,
|
||||
codec,
|
||||
maxpages,
|
||||
password,
|
||||
caching,
|
||||
check_extractable,
|
||||
pagenos,
|
||||
debug,
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
elif out_type == 'json':
|
||||
return JSONConverter.parse_file(
|
||||
input_file,
|
||||
output_file,
|
||||
image_folder_path,
|
||||
codec,
|
||||
maxpages,
|
||||
password,
|
||||
caching,
|
||||
check_extractable,
|
||||
pagenos,
|
||||
debug,
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
elif out_type == 'text':
|
||||
return TextConverter.parse_file(
|
||||
input_file,
|
||||
output_file,
|
||||
image_folder_path,
|
||||
codec,
|
||||
maxpages,
|
||||
password,
|
||||
caching,
|
||||
check_extractable,
|
||||
pagenos,
|
||||
debug,
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'")
|
||||
@@ -7,7 +7,9 @@ from .PDFStream import resolve1, int_value, list_value, dict_value
|
||||
from .PDFParser import PDFParser
|
||||
from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from ..utils import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# some predefined literals and keywords.
|
||||
LITERAL_PAGE = LIT('Page')
|
||||
@@ -89,7 +91,7 @@ class PDFPage(object):
|
||||
tree_type = tree.get('type')
|
||||
|
||||
if tree_type is LITERAL_PAGES and 'Kids' in tree:
|
||||
log.info('Pages: Kids=%r', tree['Kids'])
|
||||
# log.info('Pages: Kids=%r', tree['Kids'])
|
||||
for c in list_value(tree['Kids']):
|
||||
for x in search(c, tree):
|
||||
yield x
|
||||
|
||||
@@ -9,4 +9,5 @@ from .generic import *
|
||||
from .lzw import *
|
||||
from .matrix import *
|
||||
from .text import *
|
||||
from .types import *
|
||||
from .types import *
|
||||
from .log import *
|
||||
@@ -0,0 +1,17 @@
|
||||
import logging
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.WARNING
|
||||
)
|
||||
|
||||
LOGGERS = {}
|
||||
|
||||
def set_log_level(debug_level: int):
|
||||
logging.basicConfig(level=debug_level)
|
||||
for _, logger in LOGGERS.items():
|
||||
logger.setLevel(debug_level)
|
||||
|
||||
def get_logger(logger_name: str) -> logging.Logger:
|
||||
if logger_name not in LOGGERS.keys():
|
||||
LOGGERS[logger_name] = logging.getLogger(logger_name)
|
||||
return LOGGERS[logger_name]
|
||||
Regular → Executable
+13
-6
@@ -1,9 +1,12 @@
|
||||
#!/usr/bin/env python
|
||||
import re
|
||||
import os
|
||||
import logging
|
||||
|
||||
from typing import List
|
||||
from pdfmajor.converters import convert_file
|
||||
from xml.etree import ElementTree
|
||||
from tqdm import tqdm
|
||||
|
||||
def rel_path(*arg: List[str]) -> str:
|
||||
return os.path.join(os.path.dirname(__file__), *arg)
|
||||
@@ -18,21 +21,25 @@ if not os.path.exists(OUTPUT_FOLDER):
|
||||
files = os.listdir(INPUT_FOLDER)
|
||||
|
||||
TOTAL = len(files)
|
||||
for idx, file_name in enumerate(files):
|
||||
waiter = tqdm(enumerate(files), total=TOTAL)
|
||||
for idx, file_name in waiter:
|
||||
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
|
||||
input_file_path = os.path.join(INPUT_FOLDER, file_name)
|
||||
output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT)
|
||||
|
||||
if len(file_name) > len(file_name_no_ext):
|
||||
print(f"processing {file_name} {idx}/{TOTAL}")
|
||||
waiter.set_description(f"Processing {file_name}")
|
||||
with open(input_file_path, 'rb') as input_file:
|
||||
with open(output_file_path, 'wb') as output_file:
|
||||
convert_file(
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
out_type=EXT,
|
||||
debug=True,
|
||||
debug_level=logging.NOTSET,
|
||||
)
|
||||
print(f" > testing format {file_name} {idx}/{TOTAL}")
|
||||
ElementTree.parse(output_file_path)
|
||||
print(f"processing {idx+1}/{TOTAL}")
|
||||
if EXT == 'xml':
|
||||
waiter.set_description(f" > testing format {file_name}")
|
||||
ElementTree.parse(output_file_path)
|
||||
waiter.refresh()
|
||||
waiter.set_description("Done")
|
||||
waiter.close()
|
||||
Binary file not shown.
Reference in new issue
Block a user