4 Commits
Author SHA1 Message Date
ari 924596e38d - added consistency to PDFGraphicStateCol 2018-11-29 15:39:16 -05:00
ari 2ea0b717f9 - added better logging options
- added test case for bad chars
2018-11-23 14:44:09 -05:00
ari 38f8ca76a6 - better loggin system
- adde test case for bad characters
2018-11-23 14:43:51 -05:00
ari ac49ad1bcf - sanitizing the xml chars 2018-11-21 09:19:33 -05:00
14 changed files with 128 additions and 95 deletions

No files matched your search

+1 -1
View File
@@ -1 +1 @@
v1.1.9
v1.1.12
+1 -1
View File
@@ -86,7 +86,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
__version__ = (1, 1, 9)
__version__ = (1, 1, 12)
if __name__ == '__main__':
print(
+28 -32
View File
@@ -25,7 +25,7 @@ def get_color(col: PDFGraphicStateColor):
elif color_type == 'cmyk':
return f"cmyk({','.join(map(str, color_val))})"
elif color_type == 'gray':
return f'rgba(0,0,0,{1-color_val})'
return f'rgba(0,0,0,{1-color_val[0]})'
elif color_type == "custom":
# Todo: convert these to more friendly color types
return f"custom({col.custom['type']})"
@@ -134,6 +134,8 @@ class HTMLConverter(PDFConverter):
'position': 'absolute',
'left': f'{item.x0}px',
'bottom': f'{item.y0}px',
'width': f'{item.width}px',
'height': f'{item.height}px',
}
path = ""
svg_attr = {
@@ -141,42 +143,36 @@ class HTMLConverter(PDFConverter):
'z-index': 2
}
if item.height > 0:
dy = 1.0/item.height
dx = (item.width/item.height)/(item.x1-item.x0)
svg_attr.update({
"viewBox": "0 0 %s %s" % ( item.width/item.height, 1 )
})
else:
dx = 1.0/(item.x1-item.x0)
dy = 1.0
svg_attr.update({
"viewBox": "0 0 1 1"
})
p_vals = { "x": [], "y": [] }
if item.height > 1:
svg_attr.update({
'height': f'{item.height}px',
})
else:
svg_attr.update({
'height': '1px',
})
if item.width > 1:
svg_attr.update({
'width': f'{item.width}px',
})
else:
svg_attr.update({
'width': '1px',
})
for p in item.paths:
for pt in p.points:
p_vals['x'].append(pt.x)
p_vals['y'].append(pt.y)
mx_x = max([*p_vals['x'], 1])
mn_x = min([*p_vals['x'], 0])
mx_y = max([*p_vals['y'], 1])
mn_y = min([*p_vals['y'], 0])
if mx_x == mn_x:
mn_x -= 1
if mx_y == mn_y:
mn_y -= 1
dx = 1/( mx_x - mn_x )
dy = (item.height/item.width)/( mx_y - mn_y )
ROUND_VAL = 5
svg_attr.update({
# "viewBox": f"{min(p_vals['x'])} {min(p_vals['y'])} {max(p_vals['x'])} {max(p_vals['y'])}",
"viewBox": f"{round(mn_x*dx, ROUND_VAL)} {round(mn_y*dy, ROUND_VAL)} {round(mx_x*dx , ROUND_VAL)} {round(mx_y*dy, ROUND_VAL)}",
})
for p in item.paths:
if p.method == CurvePath.METHOD.MOVE_TO:
path += f'M{" ".join( f"{(item.width - (p.x-item.x0) )*dx} {(item.height- (p.y-item.y0) )*dy}" for p in p.points )} '
path += f'M{" ".join( f"{round(dx*pt.x, ROUND_VAL)} {round(dy*pt.y, ROUND_VAL)}" for pt in p.points )} '
if p.method == CurvePath.METHOD.LINE_TO:
path += f'L{" ".join( f"{(item.width - (p.x-item.x0) )*dx} {( item.height - (p.y-item.y0) )*dy}" for p in p.points )} '
path += f'L{" ".join( f"{round(dx*pt.x, ROUND_VAL)} {round(dy*pt.y, ROUND_VAL)}" for pt in p.points )} '
if p.method == CurvePath.METHOD.CLOSE_PATH:
path += 'Z'
with self.place_elm_with_child('svg', svg_attr, css):
+12 -8
View File
@@ -1,17 +1,19 @@
import logging
import re
import os
from typing import List
from io import BytesIO, StringIO, TextIOWrapper
from tqdm import tqdm
from ..layouts import PDFLayoutAnalyzer
from ..interpreter.PDFResourceManager import PDFResourceManager
from ..interpreter.PDFPage import PDFPage
from ..interpreter.PDFPageInterpreter import PDFPageInterpreter
from ..imagewriter import ImageWriter
from ..utils import set_log_level, get_logger, logging
log = logging.getLogger(__name__)
log = get_logger(__name__)
## PDFConverter
##
@@ -29,11 +31,10 @@ class PDFConverter(PDFLayoutAnalyzer):
check_extractable: bool = True,
pagenos: List[int] = None,
dont_export_images: bool = False,
debug: bool = False,
debug_level: int = logging.WARNING,
) -> TextIOWrapper:
if debug:
log.setLevel(logging.DEBUG)
set_log_level(debug_level)
if dont_export_images:
image_folder_path = None
@@ -53,9 +54,12 @@ class PDFConverter(PDFLayoutAnalyzer):
interpreter = PDFPageInterpreter(rsrcmgr, device)
log.debug("Reading pages....")
pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable)
for idx, page in enumerate(pages):
log.debug(f"Reading page #{idx}")
interpreter.process_page(page)
page_waiter = tqdm(enumerate(pages), disable=debug_level > logging.INFO)
for _, page in page_waiter:
page_waiter.set_description("Processing pdf...")
interpreter.process_page(page)
page_waiter.close()
device.close()
return output_file
+1 -1
View File
@@ -131,7 +131,7 @@ class XMLConverter(PDFConverter):
'y0': char.y0,
'y1': char.y1,
}, no_additional_char=True):
self.write(char.get_text(), lineend='', deep_space="")
self.write(saxutils.escape(char.get_text()), lineend='', deep_space="")
def render_char_block(self, char_block: LTCharBlock):
attr = {
+47 -41
View File
@@ -8,6 +8,7 @@ from .HTMLConverter import HTMLConverter
from .JSONConverter import JSONConverter
from .XMLConverter import XMLConverter
from .TextConverter import TextConverter
from ..utils import logging
def convert_file(
input_file: io.TextIOWrapper,
@@ -19,7 +20,8 @@ def convert_file(
caching: bool = True,
check_extractable: bool = True,
pagenos: List[int] = None,
debug: bool = False,
dont_export_images: bool = False,
debug_level: int = logging.WARNING,
out_type: str = 'html',
):
if output_file is None:
@@ -36,54 +38,58 @@ def convert_file(
raise Exception("Please make sure that the file name and output type match!")
if out_type == 'html':
return HTMLConverter.parse_file(
input_file,
output_file,
image_folder_path,
codec,
maxpages,
password,
caching,
check_extractable,
pagenos,
debug,
input_file=input_file,
output_file=output_file,
image_folder_path=image_folder_path,
codec=codec,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=check_extractable,
pagenos=pagenos,
dont_export_images=dont_export_images,
debug_level=debug_level,
)
elif out_type == 'xml':
return XMLConverter.parse_file(
input_file,
output_file,
image_folder_path,
codec,
maxpages,
password,
caching,
check_extractable,
pagenos,
debug,
input_file=input_file,
output_file=output_file,
image_folder_path=image_folder_path,
codec=codec,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=check_extractable,
pagenos=pagenos,
dont_export_images=dont_export_images,
debug_level=debug_level,
)
elif out_type == 'json':
return JSONConverter.parse_file(
input_file,
output_file,
image_folder_path,
codec,
maxpages,
password,
caching,
check_extractable,
pagenos,
debug,
input_file=input_file,
output_file=output_file,
image_folder_path=image_folder_path,
codec=codec,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=check_extractable,
pagenos=pagenos,
dont_export_images=dont_export_images,
debug_level=debug_level,
)
elif out_type == 'text':
return TextConverter.parse_file(
input_file,
output_file,
image_folder_path,
codec,
maxpages,
password,
caching,
check_extractable,
pagenos,
debug,
input_file=input_file,
output_file=output_file,
image_folder_path=image_folder_path,
codec=codec,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=check_extractable,
pagenos=pagenos,
dont_export_images=dont_export_images,
debug_level=debug_level,
)
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'")
@@ -34,7 +34,7 @@ class PDFGraphicStateColor:
@property
def gray(self):
if self.__gray is not None: return float(self.__gray)
if self.__gray is not None: return [float(self.__gray)]
else: raise self.NotSet
@property
+4 -2
View File
@@ -7,7 +7,9 @@ from .PDFStream import resolve1, int_value, list_value, dict_value
from .PDFParser import PDFParser
from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed
log = logging.getLogger(__name__)
from ..utils import get_logger
log = get_logger(__name__)
# some predefined literals and keywords.
LITERAL_PAGE = LIT('Page')
@@ -89,7 +91,7 @@ class PDFPage(object):
tree_type = tree.get('type')
if tree_type is LITERAL_PAGES and 'Kids' in tree:
log.info('Pages: Kids=%r', tree['Kids'])
# log.info('Pages: Kids=%r', tree['Kids'])
for c in list_value(tree['Kids']):
for x in search(c, tree):
yield x
+2 -1
View File
@@ -9,4 +9,5 @@ from .generic import *
from .lzw import *
from .matrix import *
from .text import *
from .types import *
from .types import *
from .log import *
+17
View File
@@ -0,0 +1,17 @@
import logging
logging.basicConfig(
level=logging.WARNING
)
LOGGERS = {}
def set_log_level(debug_level: int):
logging.basicConfig(level=debug_level)
for _, logger in LOGGERS.items():
logger.setLevel(debug_level)
def get_logger(logger_name: str) -> logging.Logger:
if logger_name not in LOGGERS.keys():
LOGGERS[logger_name] = logging.getLogger(logger_name)
return LOGGERS[logger_name]
Regular → Executable
+13 -6
View File
@@ -1,9 +1,12 @@
#!/usr/bin/env python
import re
import os
import logging
from typing import List
from pdfmajor.converters import convert_file
from xml.etree import ElementTree
from tqdm import tqdm
def rel_path(*arg: List[str]) -> str:
return os.path.join(os.path.dirname(__file__), *arg)
@@ -18,21 +21,25 @@ if not os.path.exists(OUTPUT_FOLDER):
files = os.listdir(INPUT_FOLDER)
TOTAL = len(files)
for idx, file_name in enumerate(files):
waiter = tqdm(enumerate(files), total=TOTAL)
for idx, file_name in waiter:
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
input_file_path = os.path.join(INPUT_FOLDER, file_name)
output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT)
if len(file_name) > len(file_name_no_ext):
print(f"processing {file_name} {idx}/{TOTAL}")
waiter.set_description(f"Processing {file_name}")
with open(input_file_path, 'rb') as input_file:
with open(output_file_path, 'wb') as output_file:
convert_file(
input_file=input_file,
output_file=output_file,
out_type=EXT,
debug=True,
debug_level=logging.NOTSET,
)
print(f" > testing format {file_name} {idx}/{TOTAL}")
ElementTree.parse(output_file_path)
print(f"processing {idx+1}/{TOTAL}")
if EXT == 'xml':
waiter.set_description(f" > testing format {file_name}")
ElementTree.parse(output_file_path)
waiter.refresh()
waiter.set_description("Done")
waiter.close()
Binary file not shown.
Binary file not shown.
+1 -1
View File
@@ -10,7 +10,7 @@ import sys
with open('README.md') as readme_file:
SETUP = {
"name": "pdfmajor",
"version": "1.1.9",
"version": "1.1.12",
"packages": find_packages(include=['pdfmajor*']),
"install_requires": [
'pycryptodome',