diff --git a/pdfmajor/converters/PDFConverter.py b/pdfmajor/converters/PDFConverter.py index c9c6955..01bbbd0 100644 --- a/pdfmajor/converters/PDFConverter.py +++ b/pdfmajor/converters/PDFConverter.py @@ -1,17 +1,19 @@ -import logging import re import os from typing import List from io import BytesIO, StringIO, TextIOWrapper +from tqdm import tqdm + from ..layouts import PDFLayoutAnalyzer from ..interpreter.PDFResourceManager import PDFResourceManager from ..interpreter.PDFPage import PDFPage from ..interpreter.PDFPageInterpreter import PDFPageInterpreter from ..imagewriter import ImageWriter +from ..utils import set_log_level, get_logger, logging -log = logging.getLogger(__name__) +log = get_logger(__name__) ## PDFConverter ## @@ -29,11 +31,10 @@ class PDFConverter(PDFLayoutAnalyzer): check_extractable: bool = True, pagenos: List[int] = None, dont_export_images: bool = False, - debug: bool = False, + debug_level: int = logging.WARNING, ) -> TextIOWrapper: - if debug: - log.setLevel(logging.DEBUG) + set_log_level(debug_level) if dont_export_images: image_folder_path = None @@ -53,9 +54,12 @@ class PDFConverter(PDFLayoutAnalyzer): interpreter = PDFPageInterpreter(rsrcmgr, device) log.debug("Reading pages....") pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable) - for idx, page in enumerate(pages): - log.debug(f"Reading page #{idx}") - interpreter.process_page(page) + + page_waiter = tqdm(enumerate(pages), disable=debug_level > logging.INFO) + for _, page in page_waiter: + page_waiter.set_description("Processing pdf...") + interpreter.process_page(page) + page_waiter.close() device.close() return output_file diff --git a/pdfmajor/converters/__init__.py b/pdfmajor/converters/__init__.py index d09fcf1..a2bf37a 100644 --- a/pdfmajor/converters/__init__.py +++ b/pdfmajor/converters/__init__.py @@ -8,6 +8,7 @@ from .HTMLConverter import HTMLConverter from .JSONConverter import JSONConverter from .XMLConverter import XMLConverter from .TextConverter import TextConverter +from ..utils import logging def convert_file( input_file: io.TextIOWrapper, @@ -19,7 +20,8 @@ def convert_file( caching: bool = True, check_extractable: bool = True, pagenos: List[int] = None, - debug: bool = False, + dont_export_images: bool = False, + debug_level: int = logging.WARNING, out_type: str = 'html', ): if output_file is None: @@ -36,54 +38,58 @@ def convert_file( raise Exception("Please make sure that the file name and output type match!") if out_type == 'html': return HTMLConverter.parse_file( - input_file, - output_file, - image_folder_path, - codec, - maxpages, - password, - caching, - check_extractable, - pagenos, - debug, + input_file=input_file, + output_file=output_file, + image_folder_path=image_folder_path, + codec=codec, + maxpages=maxpages, + password=password, + caching=caching, + check_extractable=check_extractable, + pagenos=pagenos, + dont_export_images=dont_export_images, + debug_level=debug_level, ) elif out_type == 'xml': return XMLConverter.parse_file( - input_file, - output_file, - image_folder_path, - codec, - maxpages, - password, - caching, - check_extractable, - pagenos, - debug, + input_file=input_file, + output_file=output_file, + image_folder_path=image_folder_path, + codec=codec, + maxpages=maxpages, + password=password, + caching=caching, + check_extractable=check_extractable, + pagenos=pagenos, + dont_export_images=dont_export_images, + debug_level=debug_level, ) elif out_type == 'json': return JSONConverter.parse_file( - input_file, - output_file, - image_folder_path, - codec, - maxpages, - password, - caching, - check_extractable, - pagenos, - debug, + input_file=input_file, + output_file=output_file, + image_folder_path=image_folder_path, + codec=codec, + maxpages=maxpages, + password=password, + caching=caching, + check_extractable=check_extractable, + pagenos=pagenos, + dont_export_images=dont_export_images, + debug_level=debug_level, ) elif out_type == 'text': return TextConverter.parse_file( - input_file, - output_file, - image_folder_path, - codec, - maxpages, - password, - caching, - check_extractable, - pagenos, - debug, + input_file=input_file, + output_file=output_file, + image_folder_path=image_folder_path, + codec=codec, + maxpages=maxpages, + password=password, + caching=caching, + check_extractable=check_extractable, + pagenos=pagenos, + dont_export_images=dont_export_images, + debug_level=debug_level, ) else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'") \ No newline at end of file diff --git a/pdfmajor/interpreter/PDFPage.py b/pdfmajor/interpreter/PDFPage.py index b0a7a97..73757a1 100644 --- a/pdfmajor/interpreter/PDFPage.py +++ b/pdfmajor/interpreter/PDFPage.py @@ -7,7 +7,9 @@ from .PDFStream import resolve1, int_value, list_value, dict_value from .PDFParser import PDFParser from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed -log = logging.getLogger(__name__) +from ..utils import get_logger + +log = get_logger(__name__) # some predefined literals and keywords. LITERAL_PAGE = LIT('Page') @@ -89,7 +91,7 @@ class PDFPage(object): tree_type = tree.get('type') if tree_type is LITERAL_PAGES and 'Kids' in tree: - log.info('Pages: Kids=%r', tree['Kids']) + # log.info('Pages: Kids=%r', tree['Kids']) for c in list_value(tree['Kids']): for x in search(c, tree): yield x diff --git a/pdfmajor/utils/__init__.py b/pdfmajor/utils/__init__.py index 7138820..29ca27c 100644 --- a/pdfmajor/utils/__init__.py +++ b/pdfmajor/utils/__init__.py @@ -9,4 +9,5 @@ from .generic import * from .lzw import * from .matrix import * from .text import * -from .types import * \ No newline at end of file +from .types import * +from .log import * \ No newline at end of file diff --git a/pdfmajor/utils/log.py b/pdfmajor/utils/log.py new file mode 100644 index 0000000..04c75cb --- /dev/null +++ b/pdfmajor/utils/log.py @@ -0,0 +1,17 @@ +import logging + +logging.basicConfig( + level=logging.WARNING +) + +LOGGERS = {} + +def set_log_level(debug_level: int): + logging.basicConfig(level=debug_level) + for _, logger in LOGGERS.items(): + logger.setLevel(debug_level) + +def get_logger(logger_name: str) -> logging.Logger: + if logger_name not in LOGGERS.keys(): + LOGGERS[logger_name] = logging.getLogger(logger_name) + return LOGGERS[logger_name] \ No newline at end of file diff --git a/samples/convert_samples.py b/samples/convert_samples.py old mode 100644 new mode 100755 index e8d37e7..eab7991 --- a/samples/convert_samples.py +++ b/samples/convert_samples.py @@ -1,9 +1,12 @@ +#!/usr/bin/env python import re import os +import logging from typing import List from pdfmajor.converters import convert_file from xml.etree import ElementTree +from tqdm import tqdm def rel_path(*arg: List[str]) -> str: return os.path.join(os.path.dirname(__file__), *arg) @@ -18,21 +21,25 @@ if not os.path.exists(OUTPUT_FOLDER): files = os.listdir(INPUT_FOLDER) TOTAL = len(files) -for idx, file_name in enumerate(files): +waiter = tqdm(enumerate(files), total=TOTAL) +for idx, file_name in waiter: file_name_no_ext = re.sub(r'\.pdf$',"", file_name) input_file_path = os.path.join(INPUT_FOLDER, file_name) output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT) if len(file_name) > len(file_name_no_ext): - print(f"processing {file_name} {idx}/{TOTAL}") + waiter.set_description(f"Processing {file_name}") with open(input_file_path, 'rb') as input_file: with open(output_file_path, 'wb') as output_file: convert_file( input_file=input_file, output_file=output_file, out_type=EXT, - debug=True, + debug_level=logging.NOTSET, ) - print(f" > testing format {file_name} {idx}/{TOTAL}") - ElementTree.parse(output_file_path) - print(f"processing {idx+1}/{TOTAL}") \ No newline at end of file + if EXT == 'xml': + waiter.set_description(f" > testing format {file_name}") + ElementTree.parse(output_file_path) + waiter.refresh() + waiter.set_description("Done") +waiter.close() \ No newline at end of file diff --git a/samples/pdf/bad-chars.pdf b/samples/pdf/bad-chars.pdf new file mode 100644 index 0000000..44f2ca9 Binary files /dev/null and b/samples/pdf/bad-chars.pdf differ