Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
924596e38d | ||
|
|
2ea0b717f9 | ||
|
|
38f8ca76a6 | ||
|
|
ac49ad1bcf | ||
|
|
d74bf1eaa9 | ||
|
|
cf3c89bf25 | ||
|
|
b491cd2181 |
No files matched your search
@@ -86,7 +86,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
"""
|
||||
__version__ = (1, 1, 7)
|
||||
__version__ = (1, 1, 12)
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(
|
||||
|
||||
@@ -25,7 +25,7 @@ def get_color(col: PDFGraphicStateColor):
|
||||
elif color_type == 'cmyk':
|
||||
return f"cmyk({','.join(map(str, color_val))})"
|
||||
elif color_type == 'gray':
|
||||
return f'rgba(0,0,0,{1-color_val})'
|
||||
return f'rgba(0,0,0,{1-color_val[0]})'
|
||||
elif color_type == "custom":
|
||||
# Todo: convert these to more friendly color types
|
||||
return f"custom({col.custom['type']})"
|
||||
@@ -134,6 +134,8 @@ class HTMLConverter(PDFConverter):
|
||||
'position': 'absolute',
|
||||
'left': f'{item.x0}px',
|
||||
'bottom': f'{item.y0}px',
|
||||
'width': f'{item.width}px',
|
||||
'height': f'{item.height}px',
|
||||
}
|
||||
path = ""
|
||||
svg_attr = {
|
||||
@@ -141,42 +143,36 @@ class HTMLConverter(PDFConverter):
|
||||
'z-index': 2
|
||||
}
|
||||
|
||||
if item.height > 0:
|
||||
dy = 1.0/item.height
|
||||
dx = (item.width/item.height)/(item.x1-item.x0)
|
||||
svg_attr.update({
|
||||
"viewBox": "0 0 %s %s" % ( item.width/item.height, 1 )
|
||||
})
|
||||
else:
|
||||
dx = 1.0/(item.x1-item.x0)
|
||||
dy = 1.0
|
||||
svg_attr.update({
|
||||
"viewBox": "0 0 1 1"
|
||||
})
|
||||
p_vals = { "x": [], "y": [] }
|
||||
|
||||
if item.height > 1:
|
||||
svg_attr.update({
|
||||
'height': f'{item.height}px',
|
||||
})
|
||||
else:
|
||||
svg_attr.update({
|
||||
'height': '1px',
|
||||
})
|
||||
|
||||
if item.width > 1:
|
||||
svg_attr.update({
|
||||
'width': f'{item.width}px',
|
||||
})
|
||||
else:
|
||||
svg_attr.update({
|
||||
'width': '1px',
|
||||
})
|
||||
for p in item.paths:
|
||||
for pt in p.points:
|
||||
p_vals['x'].append(pt.x)
|
||||
p_vals['y'].append(pt.y)
|
||||
|
||||
mx_x = max([*p_vals['x'], 1])
|
||||
mn_x = min([*p_vals['x'], 0])
|
||||
mx_y = max([*p_vals['y'], 1])
|
||||
mn_y = min([*p_vals['y'], 0])
|
||||
|
||||
if mx_x == mn_x:
|
||||
mn_x -= 1
|
||||
if mx_y == mn_y:
|
||||
mn_y -= 1
|
||||
|
||||
dx = 1/( mx_x - mn_x )
|
||||
dy = (item.height/item.width)/( mx_y - mn_y )
|
||||
ROUND_VAL = 5
|
||||
svg_attr.update({
|
||||
# "viewBox": f"{min(p_vals['x'])} {min(p_vals['y'])} {max(p_vals['x'])} {max(p_vals['y'])}",
|
||||
"viewBox": f"{round(mn_x*dx, ROUND_VAL)} {round(mn_y*dy, ROUND_VAL)} {round(mx_x*dx , ROUND_VAL)} {round(mx_y*dy, ROUND_VAL)}",
|
||||
})
|
||||
|
||||
for p in item.paths:
|
||||
if p.method == CurvePath.METHOD.MOVE_TO:
|
||||
path += f'M{" ".join( f"{(item.width - (p.x-item.x0) )*dx} {(item.height- (p.y-item.y0) )*dy}" for p in p.points )} '
|
||||
path += f'M{" ".join( f"{round(dx*pt.x, ROUND_VAL)} {round(dy*pt.y, ROUND_VAL)}" for pt in p.points )} '
|
||||
if p.method == CurvePath.METHOD.LINE_TO:
|
||||
path += f'L{" ".join( f"{(item.width - (p.x-item.x0) )*dx} {( item.height - (p.y-item.y0) )*dy}" for p in p.points )} '
|
||||
path += f'L{" ".join( f"{round(dx*pt.x, ROUND_VAL)} {round(dy*pt.y, ROUND_VAL)}" for pt in p.points )} '
|
||||
if p.method == CurvePath.METHOD.CLOSE_PATH:
|
||||
path += 'Z'
|
||||
with self.place_elm_with_child('svg', svg_attr, css):
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
import logging
|
||||
import re
|
||||
import os
|
||||
|
||||
from typing import List
|
||||
from io import BytesIO, StringIO, TextIOWrapper
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
from ..layouts import PDFLayoutAnalyzer
|
||||
from ..interpreter.PDFResourceManager import PDFResourceManager
|
||||
from ..interpreter.PDFPage import PDFPage
|
||||
from ..interpreter.PDFPageInterpreter import PDFPageInterpreter
|
||||
from ..imagewriter import ImageWriter
|
||||
from ..utils import set_log_level, get_logger, logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log = get_logger(__name__)
|
||||
|
||||
## PDFConverter
|
||||
##
|
||||
@@ -29,11 +31,10 @@ class PDFConverter(PDFLayoutAnalyzer):
|
||||
check_extractable: bool = True,
|
||||
pagenos: List[int] = None,
|
||||
dont_export_images: bool = False,
|
||||
debug: bool = False,
|
||||
debug_level: int = logging.WARNING,
|
||||
) -> TextIOWrapper:
|
||||
|
||||
if debug:
|
||||
log.setLevel(logging.DEBUG)
|
||||
set_log_level(debug_level)
|
||||
|
||||
if dont_export_images:
|
||||
image_folder_path = None
|
||||
@@ -53,9 +54,12 @@ class PDFConverter(PDFLayoutAnalyzer):
|
||||
interpreter = PDFPageInterpreter(rsrcmgr, device)
|
||||
log.debug("Reading pages....")
|
||||
pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable)
|
||||
for idx, page in enumerate(pages):
|
||||
log.debug(f"Reading page #{idx}")
|
||||
interpreter.process_page(page)
|
||||
|
||||
page_waiter = tqdm(enumerate(pages), disable=debug_level > logging.INFO)
|
||||
for _, page in page_waiter:
|
||||
page_waiter.set_description("Processing pdf...")
|
||||
interpreter.process_page(page)
|
||||
page_waiter.close()
|
||||
device.close()
|
||||
return output_file
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ def get_color(col: PDFGraphicStateColor):
|
||||
return f'gray({color_val})'
|
||||
elif color_type == "custom":
|
||||
# Todo: convert these to more friendly color types
|
||||
return f"custom({col.custom['type']})"
|
||||
return f"{col.custom['type']}({col.custom['defintion']})"
|
||||
return ""
|
||||
|
||||
FIRST_CAP_RE = re.compile('(.)([A-Z][a-z]+)')
|
||||
@@ -131,7 +131,7 @@ class XMLConverter(PDFConverter):
|
||||
'y0': char.y0,
|
||||
'y1': char.y1,
|
||||
}, no_additional_char=True):
|
||||
self.write(char.get_text(), lineend='', deep_space="")
|
||||
self.write(saxutils.escape(char.get_text()), lineend='', deep_space="")
|
||||
|
||||
def render_char_block(self, char_block: LTCharBlock):
|
||||
attr = {
|
||||
|
||||
@@ -8,6 +8,7 @@ from .HTMLConverter import HTMLConverter
|
||||
from .JSONConverter import JSONConverter
|
||||
from .XMLConverter import XMLConverter
|
||||
from .TextConverter import TextConverter
|
||||
from ..utils import logging
|
||||
|
||||
def convert_file(
|
||||
input_file: io.TextIOWrapper,
|
||||
@@ -19,7 +20,8 @@ def convert_file(
|
||||
caching: bool = True,
|
||||
check_extractable: bool = True,
|
||||
pagenos: List[int] = None,
|
||||
debug: bool = False,
|
||||
dont_export_images: bool = False,
|
||||
debug_level: int = logging.WARNING,
|
||||
out_type: str = 'html',
|
||||
):
|
||||
if output_file is None:
|
||||
@@ -36,54 +38,58 @@ def convert_file(
|
||||
raise Exception("Please make sure that the file name and output type match!")
|
||||
if out_type == 'html':
|
||||
return HTMLConverter.parse_file(
|
||||
input_file,
|
||||
output_file,
|
||||
image_folder_path,
|
||||
codec,
|
||||
maxpages,
|
||||
password,
|
||||
caching,
|
||||
check_extractable,
|
||||
pagenos,
|
||||
debug,
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
elif out_type == 'xml':
|
||||
return XMLConverter.parse_file(
|
||||
input_file,
|
||||
output_file,
|
||||
image_folder_path,
|
||||
codec,
|
||||
maxpages,
|
||||
password,
|
||||
caching,
|
||||
check_extractable,
|
||||
pagenos,
|
||||
debug,
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
elif out_type == 'json':
|
||||
return JSONConverter.parse_file(
|
||||
input_file,
|
||||
output_file,
|
||||
image_folder_path,
|
||||
codec,
|
||||
maxpages,
|
||||
password,
|
||||
caching,
|
||||
check_extractable,
|
||||
pagenos,
|
||||
debug,
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
elif out_type == 'text':
|
||||
return TextConverter.parse_file(
|
||||
input_file,
|
||||
output_file,
|
||||
image_folder_path,
|
||||
codec,
|
||||
maxpages,
|
||||
password,
|
||||
caching,
|
||||
check_extractable,
|
||||
pagenos,
|
||||
debug,
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'")
|
||||
@@ -34,7 +34,7 @@ class PDFGraphicStateColor:
|
||||
|
||||
@property
|
||||
def gray(self):
|
||||
if self.__gray is not None: return float(self.__gray)
|
||||
if self.__gray is not None: return [float(self.__gray)]
|
||||
else: raise self.NotSet
|
||||
|
||||
@property
|
||||
|
||||
@@ -7,7 +7,9 @@ from .PDFStream import resolve1, int_value, list_value, dict_value
|
||||
from .PDFParser import PDFParser
|
||||
from .PDFDocument import PDFDocument, PDFTextExtractionNotAllowed
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from ..utils import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# some predefined literals and keywords.
|
||||
LITERAL_PAGE = LIT('Page')
|
||||
@@ -89,7 +91,7 @@ class PDFPage(object):
|
||||
tree_type = tree.get('type')
|
||||
|
||||
if tree_type is LITERAL_PAGES and 'Kids' in tree:
|
||||
log.info('Pages: Kids=%r', tree['Kids'])
|
||||
# log.info('Pages: Kids=%r', tree['Kids'])
|
||||
for c in list_value(tree['Kids']):
|
||||
for x in search(c, tree):
|
||||
yield x
|
||||
|
||||
@@ -351,6 +351,24 @@ class PDFPageInterpreter(object):
|
||||
#self.do_cs(LITERAL_DEVICE_CMYK)
|
||||
return
|
||||
|
||||
def __process_ICCBased(self, n: int, components: list):
|
||||
if n == 1:
|
||||
return {
|
||||
"type": "gray",
|
||||
"definition": components
|
||||
}
|
||||
elif n == 3:
|
||||
return {
|
||||
"type": "rgb",
|
||||
"definition": components
|
||||
}
|
||||
elif n == 4:
|
||||
return {
|
||||
"type": "cmyk",
|
||||
"definition": components
|
||||
}
|
||||
raise PDFInterpreterError("Unsupported ICCBased color")
|
||||
|
||||
# setcolor
|
||||
def __process_scn(self):
|
||||
if self.ncs:
|
||||
@@ -359,8 +377,14 @@ class PDFPageInterpreter(object):
|
||||
raise PDFInterpreterError('No colorspace specified!')
|
||||
components = self.pop(n)
|
||||
obj_id = components[0]
|
||||
if self.ncs.name == "ICCBased":
|
||||
return self.__process_ICCBased(n, components)
|
||||
if self.ncs.name not in self.resources.keys():
|
||||
raise PDFInterpreterError("Color space not in resources")
|
||||
log.warning("Color space not in resources")
|
||||
return {
|
||||
"type": self.ncs.name,
|
||||
"defintion": f"{n}|({','.join(map(str, components))})"
|
||||
}
|
||||
if obj_id.name not in self.resources[self.ncs.name].keys():
|
||||
raise PDFInterpreterError("Invalid Color reference")
|
||||
|
||||
@@ -374,11 +398,27 @@ class PDFPageInterpreter(object):
|
||||
"defintion": color_data
|
||||
}
|
||||
def do_SCN(self):
|
||||
self.graphicstate.scolor.custom = self.__process_scn()
|
||||
processed_col = self.__process_scn()
|
||||
if processed_col['type'] == 'rgb':
|
||||
self.graphicstate.scolor.rgb = processed_col['definition']
|
||||
elif processed_col['type'] == 'gray':
|
||||
self.graphicstate.scolor.gray = processed_col['definition'][0]
|
||||
elif processed_col['type'] == 'cmyk':
|
||||
self.graphicstate.scolor.cmyk = processed_col['definition']
|
||||
else:
|
||||
self.graphicstate.scolor.custom = processed_col
|
||||
return
|
||||
|
||||
def do_scn(self):
|
||||
self.graphicstate.ncolor.custom = self.__process_scn()
|
||||
processed_col = self.__process_scn()
|
||||
if processed_col['type'] == 'rgb':
|
||||
self.graphicstate.ncolor.rgb = processed_col['definition']
|
||||
elif processed_col['type'] == 'gray':
|
||||
self.graphicstate.ncolor.gray = processed_col['definition'][0]
|
||||
elif processed_col['type'] == 'cmyk':
|
||||
self.graphicstate.ncolor.cmyk = processed_col['definition']
|
||||
else:
|
||||
self.graphicstate.ncolor.custom = processed_col
|
||||
return
|
||||
|
||||
def do_SC(self):
|
||||
|
||||
@@ -9,4 +9,5 @@ from .generic import *
|
||||
from .lzw import *
|
||||
from .matrix import *
|
||||
from .text import *
|
||||
from .types import *
|
||||
from .types import *
|
||||
from .log import *
|
||||
@@ -0,0 +1,17 @@
|
||||
import logging
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.WARNING
|
||||
)
|
||||
|
||||
LOGGERS = {}
|
||||
|
||||
def set_log_level(debug_level: int):
|
||||
logging.basicConfig(level=debug_level)
|
||||
for _, logger in LOGGERS.items():
|
||||
logger.setLevel(debug_level)
|
||||
|
||||
def get_logger(logger_name: str) -> logging.Logger:
|
||||
if logger_name not in LOGGERS.keys():
|
||||
LOGGERS[logger_name] = logging.getLogger(logger_name)
|
||||
return LOGGERS[logger_name]
|
||||
Regular → Executable
+14
-7
@@ -1,14 +1,17 @@
|
||||
#!/usr/bin/env python
|
||||
import re
|
||||
import os
|
||||
import logging
|
||||
|
||||
from typing import List
|
||||
from pdfmajor.converters import convert_file
|
||||
from xml.etree import ElementTree
|
||||
from tqdm import tqdm
|
||||
|
||||
def rel_path(*arg: List[str]) -> str:
|
||||
return os.path.join(os.path.dirname(__file__), *arg)
|
||||
|
||||
EXT = 'xml'
|
||||
EXT = 'html'
|
||||
INPUT_FOLDER = rel_path("./pdf")
|
||||
OUTPUT_FOLDER = rel_path(f"./output/{EXT}")
|
||||
|
||||
@@ -18,21 +21,25 @@ if not os.path.exists(OUTPUT_FOLDER):
|
||||
files = os.listdir(INPUT_FOLDER)
|
||||
|
||||
TOTAL = len(files)
|
||||
for idx, file_name in enumerate(files):
|
||||
waiter = tqdm(enumerate(files), total=TOTAL)
|
||||
for idx, file_name in waiter:
|
||||
file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
|
||||
input_file_path = os.path.join(INPUT_FOLDER, file_name)
|
||||
output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT)
|
||||
|
||||
if len(file_name) > len(file_name_no_ext):
|
||||
print(f"processing {file_name} {idx}/{TOTAL}")
|
||||
waiter.set_description(f"Processing {file_name}")
|
||||
with open(input_file_path, 'rb') as input_file:
|
||||
with open(output_file_path, 'wb') as output_file:
|
||||
convert_file(
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
out_type=EXT,
|
||||
debug=True,
|
||||
debug_level=logging.NOTSET,
|
||||
)
|
||||
print(f" > testing format {file_name} {idx}/{TOTAL}")
|
||||
ElementTree.parse(output_file_path)
|
||||
print(f"processing {idx+1}/{TOTAL}")
|
||||
if EXT == 'xml':
|
||||
waiter.set_description(f" > testing format {file_name}")
|
||||
ElementTree.parse(output_file_path)
|
||||
waiter.refresh()
|
||||
waiter.set_description("Done")
|
||||
waiter.close()
|
||||
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user