3 Commits
Author SHA1 Message Date
ari 2ea0b717f9 - added better logging options
- added test case for bad chars
2018-11-23 14:44:09 -05:00
ari 38f8ca76a6 - better loggin system
- adde test case for bad characters
2018-11-23 14:43:51 -05:00
ari ac49ad1bcf - sanitizing the xml chars 2018-11-21 09:19:33 -05:00
11 changed files with 99 additions and 62 deletions

No files matched your search

+1 -1
View File
@@ -1 +1 @@
v1.1.9 v1.1.11
+1 -1
View File
@@ -86,7 +86,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
""" """
__version__ = (1, 1, 9) __version__ = (1, 1, 11)
if __name__ == '__main__': if __name__ == '__main__':
print( print(
+12 -8
View File
@@ -1,17 +1,19 @@
import logging
import re import re
import os import os
from typing import List from typing import List
from io import BytesIO, StringIO, TextIOWrapper from io import BytesIO, StringIO, TextIOWrapper
from tqdm import tqdm
from ..layouts import PDFLayoutAnalyzer from ..layouts import PDFLayoutAnalyzer
from ..interpreter.PDFResourceManager import PDFResourceManager from ..interpreter.PDFResourceManager import PDFResourceManager
from ..interpreter.PDFPage import PDFPage from ..interpreter.PDFPage import PDFPage
from ..interpreter.PDFPageInterpreter import PDFPageInterpreter from ..interpreter.PDFPageInterpreter import PDFPageInterpreter
from ..imagewriter import ImageWriter from ..imagewriter import ImageWriter
from ..utils import set_log_level, get_logger, logging
log = logging.getLogger(__name__) log = get_logger(__name__)
## PDFConverter ## PDFConverter
## ##
@@ -29,11 +31,10 @@ class PDFConverter(PDFLayoutAnalyzer):
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, pagenos: List[int] = None,
dont_export_images: bool = False, dont_export_images: bool = False,
debug: bool = False, debug_level: int = logging.WARNING,
) -> TextIOWrapper: ) -> TextIOWrapper:
if debug: set_log_level(debug_level)
log.setLevel(logging.DEBUG)
if dont_export_images: if dont_export_images:
image_folder_path = None image_folder_path = None
@@ -53,9 +54,12 @@ class PDFConverter(PDFLayoutAnalyzer):
interpreter = PDFPageInterpreter(rsrcmgr, device) interpreter = PDFPageInterpreter(rsrcmgr, device)
log.debug("Reading pages....") log.debug("Reading pages....")
pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable) pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable)
for idx, page in enumerate(pages):
log.debug(f"Reading page #{idx}") page_waiter = tqdm(enumerate(pages), disable=debug_level > logging.INFO)
interpreter.process_page(page) for _, page in page_waiter:
page_waiter.set_description("Processing pdf...")
interpreter.process_page(page)
page_waiter.close()
device.close() device.close()
return output_file return output_file
+1 -1
View File
@@ -131,7 +131,7 @@ class XMLConverter(PDFConverter):
'y0': char.y0, 'y0': char.y0,
'y1': char.y1, 'y1': char.y1,
}, no_additional_char=True): }, no_additional_char=True):
self.write(char.get_text(), lineend='', deep_space="") self.write(saxutils.escape(char.get_text()), lineend='', deep_space="")
def render_char_block(self, char_block: LTCharBlock): def render_char_block(self, char_block: LTCharBlock):
attr = { attr = {
+47 -41
View File
@@ -8,6 +8,7 @@ from .HTMLConverter import HTMLConverter
from .JSONConverter import JSONConverter from .JSONConverter import JSONConverter
from .XMLConverter import XMLConverter from .XMLConverter import XMLConverter
from .TextConverter import TextConverter from .TextConverter import TextConverter
from ..utils import logging
def convert_file( def convert_file(
input_file: io.TextIOWrapper, input_file: io.TextIOWrapper,
@@ -19,7 +20,8 @@ def convert_file(
caching: bool = True, caching: bool = True,
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, pagenos: List[int] = None,
debug: bool = False, dont_export_images: bool = False,
debug_level: int = logging.WARNING,
out_type: str = 'html', out_type: str = 'html',
): ):
if output_file is None: if output_file is None:
@@ -36,54 +38,58 @@ def convert_file(
raise Exception("Please make sure that the file name and output type match!") raise Exception("Please make sure that the file name and output type match!")
if out_type == 'html': if out_type == 'html':
return HTMLConverter.parse_file( return HTMLConverter.parse_file(
input_file, input_file=input_file,
output_file, output_file=output_file,
image_folder_path, image_folder_path=image_folder_path,
codec, codec=codec,
maxpages, maxpages=maxpages,
password, password=password,
caching, caching=caching,
check_extractable, check_extractable=check_extractable,
pagenos, pagenos=pagenos,
debug, dont_export_images=dont_export_images,
debug_level=debug_level,
) )
elif out_type == 'xml': elif out_type == 'xml':
return XMLConverter.parse_file( return XMLConverter.parse_file(
input_file, input_file=input_file,
output_file, output_file=output_file,
image_folder_path, image_folder_path=image_folder_path,
codec, codec=codec,
maxpages, maxpages=maxpages,
password, password=password,
caching, caching=caching,
check_extractable, check_extractable=check_extractable,
pagenos, pagenos=pagenos,
debug, dont_export_images=dont_export_images,
debug_level=debug_level,
) )
elif out_type == 'json': elif out_type == 'json':
return JSONConverter.parse_file( return JSONConverter.parse_file(
input_file, input_file=input_file,
output_file, output_file=output_file,
image_folder_path, image_folder_path=image_folder_path,
codec, codec=codec,
maxpages, maxpages=maxpages,
password, password=password,
caching, caching=caching,
check_extractable, check_extractable=check_extractable,
pagenos, pagenos=pagenos,
debug, dont_export_images=dont_export_images,
debug_level=debug_level,
) )
elif out_type == 'text': elif out_type == 'text':
return TextConverter.parse_file( return TextConverter.parse_file(
input_file, input_file=input_file,
output_file, output_file=output_file,
image_folder_path, image_folder_path=image_folder_path,
codec, codec=codec,
maxpages, maxpages=maxpages,
password, password=password,
caching, caching=caching,
check_extractable, check_extractable=check_extractable,
pagenos, pagenos=pagenos,
debug, dont_export_images=dont_export_images,
debug_level=debug_level,
) )
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'") else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'")
+4 -2
View File
@@ -7,7 +7,9 @@ from .PDFStream import resolve1, int_value, list_value, dict_value
from .PDFParser import PDFParser from .PDFParser import PDFParser
from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed
log = logging.getLogger(__name__) from ..utils import get_logger
log = get_logger(__name__)
# some predefined literals and keywords. # some predefined literals and keywords.
LITERAL_PAGE = LIT('Page') LITERAL_PAGE = LIT('Page')
@@ -89,7 +91,7 @@ class PDFPage(object):
tree_type = tree.get('type') tree_type = tree.get('type')
if tree_type is LITERAL_PAGES and 'Kids' in tree: if tree_type is LITERAL_PAGES and 'Kids' in tree:
log.info('Pages: Kids=%r', tree['Kids']) # log.info('Pages: Kids=%r', tree['Kids'])
for c in list_value(tree['Kids']): for c in list_value(tree['Kids']):
for x in search(c, tree): for x in search(c, tree):
yield x yield x
+2 -1
View File
@@ -9,4 +9,5 @@ from .generic import *
from .lzw import * from .lzw import *
from .matrix import * from .matrix import *
from .text import * from .text import *
from .types import * from .types import *
from .log import *
+17
View File
@@ -0,0 +1,17 @@
import logging
logging.basicConfig(
level=logging.WARNING
)
LOGGERS = {}
def set_log_level(debug_level: int):
logging.basicConfig(level=debug_level)
for _, logger in LOGGERS.items():
logger.setLevel(debug_level)
def get_logger(logger_name: str) -> logging.Logger:
if logger_name not in LOGGERS.keys():
LOGGERS[logger_name] = logging.getLogger(logger_name)
return LOGGERS[logger_name]
Regular → Executable
+13 -6
View File
@@ -1,9 +1,12 @@
#!/usr/bin/env python
import re import re
import os import os
import logging
from typing import List from typing import List
from pdfmajor.converters import convert_file from pdfmajor.converters import convert_file
from xml.etree import ElementTree from xml.etree import ElementTree
from tqdm import tqdm
def rel_path(*arg: List[str]) -> str: def rel_path(*arg: List[str]) -> str:
return os.path.join(os.path.dirname(__file__), *arg) return os.path.join(os.path.dirname(__file__), *arg)
@@ -18,21 +21,25 @@ if not os.path.exists(OUTPUT_FOLDER):
files = os.listdir(INPUT_FOLDER) files = os.listdir(INPUT_FOLDER)
TOTAL = len(files) TOTAL = len(files)
for idx, file_name in enumerate(files): waiter = tqdm(enumerate(files), total=TOTAL)
for idx, file_name in waiter:
file_name_no_ext = re.sub(r'\.pdf$',"", file_name) file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
input_file_path = os.path.join(INPUT_FOLDER, file_name) input_file_path = os.path.join(INPUT_FOLDER, file_name)
output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT) output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT)
if len(file_name) > len(file_name_no_ext): if len(file_name) > len(file_name_no_ext):
print(f"processing {file_name} {idx}/{TOTAL}") waiter.set_description(f"Processing {file_name}")
with open(input_file_path, 'rb') as input_file: with open(input_file_path, 'rb') as input_file:
with open(output_file_path, 'wb') as output_file: with open(output_file_path, 'wb') as output_file:
convert_file( convert_file(
input_file=input_file, input_file=input_file,
output_file=output_file, output_file=output_file,
out_type=EXT, out_type=EXT,
debug=True, debug_level=logging.NOTSET,
) )
print(f" > testing format {file_name} {idx}/{TOTAL}") if EXT == 'xml':
ElementTree.parse(output_file_path) waiter.set_description(f" > testing format {file_name}")
print(f"processing {idx+1}/{TOTAL}") ElementTree.parse(output_file_path)
waiter.refresh()
waiter.set_description("Done")
waiter.close()
Binary file not shown.
+1 -1
View File
@@ -10,7 +10,7 @@ import sys
with open('README.md') as readme_file: with open('README.md') as readme_file:
SETUP = { SETUP = {
"name": "pdfmajor", "name": "pdfmajor",
"version": "1.1.9", "version": "1.1.11",
"packages": find_packages(include=['pdfmajor*']), "packages": find_packages(include=['pdfmajor*']),
"install_requires": [ "install_requires": [
'pycryptodome', 'pycryptodome',