Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2ea0b717f9 | ||
|
|
38f8ca76a6 | ||
|
|
ac49ad1bcf |
No files matched your search
@@ -86,7 +86,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
|||||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
__version__ = (1, 1, 9)
|
__version__ = (1, 1, 11)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
print(
|
print(
|
||||||
|
|||||||
@@ -1,17 +1,19 @@
|
|||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from typing import List
|
from typing import List
|
||||||
from io import BytesIO, StringIO, TextIOWrapper
|
from io import BytesIO, StringIO, TextIOWrapper
|
||||||
|
|
||||||
|
from tqdm import tqdm
|
||||||
|
|
||||||
from ..layouts import PDFLayoutAnalyzer
|
from ..layouts import PDFLayoutAnalyzer
|
||||||
from ..interpreter.PDFResourceManager import PDFResourceManager
|
from ..interpreter.PDFResourceManager import PDFResourceManager
|
||||||
from ..interpreter.PDFPage import PDFPage
|
from ..interpreter.PDFPage import PDFPage
|
||||||
from ..interpreter.PDFPageInterpreter import PDFPageInterpreter
|
from ..interpreter.PDFPageInterpreter import PDFPageInterpreter
|
||||||
from ..imagewriter import ImageWriter
|
from ..imagewriter import ImageWriter
|
||||||
|
from ..utils import set_log_level, get_logger, logging
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = get_logger(__name__)
|
||||||
|
|
||||||
## PDFConverter
|
## PDFConverter
|
||||||
##
|
##
|
||||||
@@ -29,11 +31,10 @@ class PDFConverter(PDFLayoutAnalyzer):
|
|||||||
check_extractable: bool = True,
|
check_extractable: bool = True,
|
||||||
pagenos: List[int] = None,
|
pagenos: List[int] = None,
|
||||||
dont_export_images: bool = False,
|
dont_export_images: bool = False,
|
||||||
debug: bool = False,
|
debug_level: int = logging.WARNING,
|
||||||
) -> TextIOWrapper:
|
) -> TextIOWrapper:
|
||||||
|
|
||||||
if debug:
|
set_log_level(debug_level)
|
||||||
log.setLevel(logging.DEBUG)
|
|
||||||
|
|
||||||
if dont_export_images:
|
if dont_export_images:
|
||||||
image_folder_path = None
|
image_folder_path = None
|
||||||
@@ -53,9 +54,12 @@ class PDFConverter(PDFLayoutAnalyzer):
|
|||||||
interpreter = PDFPageInterpreter(rsrcmgr, device)
|
interpreter = PDFPageInterpreter(rsrcmgr, device)
|
||||||
log.debug("Reading pages....")
|
log.debug("Reading pages....")
|
||||||
pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable)
|
pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable)
|
||||||
for idx, page in enumerate(pages):
|
|
||||||
log.debug(f"Reading page #{idx}")
|
page_waiter = tqdm(enumerate(pages), disable=debug_level > logging.INFO)
|
||||||
|
for _, page in page_waiter:
|
||||||
|
page_waiter.set_description("Processing pdf...")
|
||||||
interpreter.process_page(page)
|
interpreter.process_page(page)
|
||||||
|
page_waiter.close()
|
||||||
device.close()
|
device.close()
|
||||||
return output_file
|
return output_file
|
||||||
|
|
||||||
|
|||||||
@@ -131,7 +131,7 @@ class XMLConverter(PDFConverter):
|
|||||||
'y0': char.y0,
|
'y0': char.y0,
|
||||||
'y1': char.y1,
|
'y1': char.y1,
|
||||||
}, no_additional_char=True):
|
}, no_additional_char=True):
|
||||||
self.write(char.get_text(), lineend='', deep_space="")
|
self.write(saxutils.escape(char.get_text()), lineend='', deep_space="")
|
||||||
|
|
||||||
def render_char_block(self, char_block: LTCharBlock):
|
def render_char_block(self, char_block: LTCharBlock):
|
||||||
attr = {
|
attr = {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from .HTMLConverter import HTMLConverter
|
|||||||
from .JSONConverter import JSONConverter
|
from .JSONConverter import JSONConverter
|
||||||
from .XMLConverter import XMLConverter
|
from .XMLConverter import XMLConverter
|
||||||
from .TextConverter import TextConverter
|
from .TextConverter import TextConverter
|
||||||
|
from ..utils import logging
|
||||||
|
|
||||||
def convert_file(
|
def convert_file(
|
||||||
input_file: io.TextIOWrapper,
|
input_file: io.TextIOWrapper,
|
||||||
@@ -19,7 +20,8 @@ def convert_file(
|
|||||||
caching: bool = True,
|
caching: bool = True,
|
||||||
check_extractable: bool = True,
|
check_extractable: bool = True,
|
||||||
pagenos: List[int] = None,
|
pagenos: List[int] = None,
|
||||||
debug: bool = False,
|
dont_export_images: bool = False,
|
||||||
|
debug_level: int = logging.WARNING,
|
||||||
out_type: str = 'html',
|
out_type: str = 'html',
|
||||||
):
|
):
|
||||||
if output_file is None:
|
if output_file is None:
|
||||||
@@ -36,54 +38,58 @@ def convert_file(
|
|||||||
raise Exception("Please make sure that the file name and output type match!")
|
raise Exception("Please make sure that the file name and output type match!")
|
||||||
if out_type == 'html':
|
if out_type == 'html':
|
||||||
return HTMLConverter.parse_file(
|
return HTMLConverter.parse_file(
|
||||||
input_file,
|
input_file=input_file,
|
||||||
output_file,
|
output_file=output_file,
|
||||||
image_folder_path,
|
image_folder_path=image_folder_path,
|
||||||
codec,
|
codec=codec,
|
||||||
maxpages,
|
maxpages=maxpages,
|
||||||
password,
|
password=password,
|
||||||
caching,
|
caching=caching,
|
||||||
check_extractable,
|
check_extractable=check_extractable,
|
||||||
pagenos,
|
pagenos=pagenos,
|
||||||
debug,
|
dont_export_images=dont_export_images,
|
||||||
|
debug_level=debug_level,
|
||||||
)
|
)
|
||||||
elif out_type == 'xml':
|
elif out_type == 'xml':
|
||||||
return XMLConverter.parse_file(
|
return XMLConverter.parse_file(
|
||||||
input_file,
|
input_file=input_file,
|
||||||
output_file,
|
output_file=output_file,
|
||||||
image_folder_path,
|
image_folder_path=image_folder_path,
|
||||||
codec,
|
codec=codec,
|
||||||
maxpages,
|
maxpages=maxpages,
|
||||||
password,
|
password=password,
|
||||||
caching,
|
caching=caching,
|
||||||
check_extractable,
|
check_extractable=check_extractable,
|
||||||
pagenos,
|
pagenos=pagenos,
|
||||||
debug,
|
dont_export_images=dont_export_images,
|
||||||
|
debug_level=debug_level,
|
||||||
)
|
)
|
||||||
elif out_type == 'json':
|
elif out_type == 'json':
|
||||||
return JSONConverter.parse_file(
|
return JSONConverter.parse_file(
|
||||||
input_file,
|
input_file=input_file,
|
||||||
output_file,
|
output_file=output_file,
|
||||||
image_folder_path,
|
image_folder_path=image_folder_path,
|
||||||
codec,
|
codec=codec,
|
||||||
maxpages,
|
maxpages=maxpages,
|
||||||
password,
|
password=password,
|
||||||
caching,
|
caching=caching,
|
||||||
check_extractable,
|
check_extractable=check_extractable,
|
||||||
pagenos,
|
pagenos=pagenos,
|
||||||
debug,
|
dont_export_images=dont_export_images,
|
||||||
|
debug_level=debug_level,
|
||||||
)
|
)
|
||||||
elif out_type == 'text':
|
elif out_type == 'text':
|
||||||
return TextConverter.parse_file(
|
return TextConverter.parse_file(
|
||||||
input_file,
|
input_file=input_file,
|
||||||
output_file,
|
output_file=output_file,
|
||||||
image_folder_path,
|
image_folder_path=image_folder_path,
|
||||||
codec,
|
codec=codec,
|
||||||
maxpages,
|
maxpages=maxpages,
|
||||||
password,
|
password=password,
|
||||||
caching,
|
caching=caching,
|
||||||
check_extractable,
|
check_extractable=check_extractable,
|
||||||
pagenos,
|
pagenos=pagenos,
|
||||||
debug,
|
dont_export_images=dont_export_images,
|
||||||
|
debug_level=debug_level,
|
||||||
)
|
)
|
||||||
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'")
|
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'")
|
||||||
@@ -7,7 +7,9 @@ from .PDFStream import resolve1, int_value, list_value, dict_value
|
|||||||
from .PDFParser import PDFParser
|
from .PDFParser import PDFParser
|
||||||
from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed
|
from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
from ..utils import get_logger
|
||||||
|
|
||||||
|
log = get_logger(__name__)
|
||||||
|
|
||||||
# some predefined literals and keywords.
|
# some predefined literals and keywords.
|
||||||
LITERAL_PAGE = LIT('Page')
|
LITERAL_PAGE = LIT('Page')
|
||||||
@@ -89,7 +91,7 @@ class PDFPage(object):
|
|||||||
tree_type = tree.get('type')
|
tree_type = tree.get('type')
|
||||||
|
|
||||||
if tree_type is LITERAL_PAGES and 'Kids' in tree:
|
if tree_type is LITERAL_PAGES and 'Kids' in tree:
|
||||||
log.info('Pages: Kids=%r', tree['Kids'])
|
# log.info('Pages: Kids=%r', tree['Kids'])
|
||||||
for c in list_value(tree['Kids']):
|
for c in list_value(tree['Kids']):
|
||||||
for x in search(c, tree):
|
for x in search(c, tree):
|
||||||
yield x
|
yield x
|
||||||
|
|||||||
@@ -10,3 +10,4 @@ from .lzw import *
|
|||||||
from .matrix import *
|
from .matrix import *
|
||||||
from .text import *
|
from .text import *
|
||||||
from .types import *
|
from .types import *
|
||||||
|
from .log import *
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import logging
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.WARNING
|
||||||
|
)
|
||||||
|
|
||||||
|
LOGGERS = {}
|
||||||
|
|
||||||
|
def set_log_level(debug_level: int):
|
||||||
|
logging.basicConfig(level=debug_level)
|
||||||
|
for _, logger in LOGGERS.items():
|
||||||
|
logger.setLevel(debug_level)
|
||||||
|
|
||||||
|
def get_logger(logger_name: str) -> logging.Logger:
|
||||||
|
if logger_name not in LOGGERS.keys():
|
||||||
|
LOGGERS[logger_name] = logging.getLogger(logger_name)
|
||||||
|
return LOGGERS[logger_name]
|
||||||
Regular → Executable
+13
-6
@@ -1,9 +1,12 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
import re
|
import re
|
||||||
import os
|
import os
|
||||||
|
import logging
|
||||||
|
|
||||||
from typing import List
|
from typing import List
|
||||||
from pdfmajor.converters import convert_file
|
from pdfmajor.converters import convert_file
|
||||||
from xml.etree import ElementTree
|
from xml.etree import ElementTree
|
||||||
|
from tqdm import tqdm
|
||||||
|
|
||||||
def rel_path(*arg: List[str]) -> str:
|
def rel_path(*arg: List[str]) -> str:
|
||||||
return os.path.join(os.path.dirname(__file__), *arg)
|
return os.path.join(os.path.dirname(__file__), *arg)
|
||||||
@@ -18,21 +21,25 @@ if not os.path.exists(OUTPUT_FOLDER):
|
|||||||
files = os.listdir(INPUT_FOLDER)
|
files = os.listdir(INPUT_FOLDER)
|
||||||
|
|
||||||
TOTAL = len(files)
|
TOTAL = len(files)
|
||||||
for idx, file_name in enumerate(files):
|
waiter = tqdm(enumerate(files), total=TOTAL)
|
||||||
|
for idx, file_name in waiter:
|
||||||
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
|
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
|
||||||
input_file_path = os.path.join(INPUT_FOLDER, file_name)
|
input_file_path = os.path.join(INPUT_FOLDER, file_name)
|
||||||
output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT)
|
output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT)
|
||||||
|
|
||||||
if len(file_name) > len(file_name_no_ext):
|
if len(file_name) > len(file_name_no_ext):
|
||||||
print(f"processing {file_name} {idx}/{TOTAL}")
|
waiter.set_description(f"Processing {file_name}")
|
||||||
with open(input_file_path, 'rb') as input_file:
|
with open(input_file_path, 'rb') as input_file:
|
||||||
with open(output_file_path, 'wb') as output_file:
|
with open(output_file_path, 'wb') as output_file:
|
||||||
convert_file(
|
convert_file(
|
||||||
input_file=input_file,
|
input_file=input_file,
|
||||||
output_file=output_file,
|
output_file=output_file,
|
||||||
out_type=EXT,
|
out_type=EXT,
|
||||||
debug=True,
|
debug_level=logging.NOTSET,
|
||||||
)
|
)
|
||||||
print(f" > testing format {file_name} {idx}/{TOTAL}")
|
if EXT == 'xml':
|
||||||
ElementTree.parse(output_file_path)
|
waiter.set_description(f" > testing format {file_name}")
|
||||||
print(f"processing {idx+1}/{TOTAL}")
|
ElementTree.parse(output_file_path)
|
||||||
|
waiter.refresh()
|
||||||
|
waiter.set_description("Done")
|
||||||
|
waiter.close()
|
||||||
Binary file not shown.
@@ -10,7 +10,7 @@ import sys
|
|||||||
with open('README.md') as readme_file:
|
with open('README.md') as readme_file:
|
||||||
SETUP = {
|
SETUP = {
|
||||||
"name": "pdfmajor",
|
"name": "pdfmajor",
|
||||||
"version": "1.1.9",
|
"version": "1.1.11",
|
||||||
"packages": find_packages(include=['pdfmajor*']),
|
"packages": find_packages(include=['pdfmajor*']),
|
||||||
"install_requires": [
|
"install_requires": [
|
||||||
'pycryptodome',
|
'pycryptodome',
|
||||||
|
|||||||
Reference in new issue
Block a user