From bc20627d9f68b0aa333c29946000fb40e30e4d38 Mon Sep 17 00:00:00 2001 From: Ariel Sosnovsky Date: Sat, 22 Dec 2018 22:44:53 -0500 Subject: [PATCH] - custom formatters done - json formatter works now - added some tests for writers --- pdfmajor/converters/HTMLConverter.py | 246 ------------------ pdfmajor/converters/XMLConverter.py | 192 -------------- pdfmajor/converters/__init__.py | 78 +++--- pdfmajor/converters/html.py | 180 ++++++++++++- pdfmajor/converters/json.py | 134 ++++++++++ pdfmajor/converters/writers/__init__.py | 0 pdfmajor/converters/writers/html.py | 62 +++++ pdfmajor/converters/writers/json.py | 129 +++++++++ pdfmajor/converters/writers/xml.py | 60 +++++ pdfmajor/converters/xml.py | 142 ++++++++++ pdfmajor/interpreter/PageInterpreter.py | 8 +- pdfmajor/interpreter/XObjectInterpreter.py | 4 +- pdfmajor/interpreter/commands/commands.py | 31 +-- .../state/PDFGraphicState/__init__.py | 18 +- .../interpreter/commands/state/__init__.py | 2 +- .../commands/state/layout/__init__.py | 2 +- samples/convert_samples.py | 48 ++-- samples/extractor.py | 23 -- tests/test_html_writer.py | 60 +++++ tests/test_interpreter.py | 2 +- tests/test_json_writer.py | 112 ++++++++ 21 files changed, 981 insertions(+), 552 deletions(-) delete mode 100644 pdfmajor/converters/HTMLConverter.py delete mode 100644 pdfmajor/converters/XMLConverter.py create mode 100644 pdfmajor/converters/json.py create mode 100644 pdfmajor/converters/writers/__init__.py create mode 100644 pdfmajor/converters/writers/html.py create mode 100644 pdfmajor/converters/writers/json.py create mode 100644 pdfmajor/converters/writers/xml.py create mode 100644 pdfmajor/converters/xml.py delete mode 100644 samples/extractor.py create mode 100644 tests/test_html_writer.py create mode 100644 tests/test_json_writer.py diff --git a/pdfmajor/converters/HTMLConverter.py b/pdfmajor/converters/HTMLConverter.py deleted file mode 100644 index 59d8853..0000000 --- a/pdfmajor/converters/HTMLConverter.py +++ /dev/null @@ -1,246 +0,0 @@ -import logging - -from typing import Dict - -from contextlib import contextmanager - -from ..layouts import LTPage -from ..layouts import LTItem -from ..layouts import LTCurve, LTLine, LTRect, CurvePath -from ..layouts import LTFigure -from ..layouts import LTImage -from ..layouts import LTChar, LTCharBlock -from ..utils import enc - -from .PDFConverter import PDFConverter -from ..interpreter.PDFGraphicState import PDFGraphicStateColor - -log = logging.getLogger(__name__) - -def get_color(col: PDFGraphicStateColor): - color_type, color_val = col.get_color() - if color_type is not None: - if color_type == 'rgb': - return f"rgb({','.join(map(str, color_val))})" - elif color_type == 'cmyk': - return f"cmyk({','.join(map(str, color_val))})" - elif color_type == 'gray': - return f'rgba(0,0,0,{1-color_val[0]})' - elif color_type == "custom": - # Todo: convert these to more friendly color types - return f"custom({col.custom['type']})" - return "none" - -class HTMLConverter(PDFConverter): - def __init__(self, - rsrcmgr, - outfp, - codec='utf-8', - pageno=1, - imagewriter=None - ): - PDFConverter.__init__(self, rsrcmgr, outfp, imagewriter=imagewriter, codec=codec, pageno=pageno) - self.__levels_deep = 0 - self.__current_height = None - self.__current_width = None - self.write_header() - - def write(self, text: str, lineend: str = '\n', deep_space: str = ' '): - text = deep_space*self.__levels_deep + text + lineend - self.write_raw(text) - - def write_raw(self, text): - if self.codec: - text = text.encode(self.codec) - self.outfp.write(text) - - def write_header(self): - self.write('') - self.__levels_deep += 1 - self.write('') - self.__levels_deep += 1 - meta_attr = { - 'http-equiv': "Content-Type", - 'content': "text/html", - } - if self.codec: - meta_attr['charset'] = self.codec - - self.place_elm_close('meta', attr_dict=meta_attr, as_childless=False) - with self.place_elm_with_child('style'): - self.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''') - self.__levels_deep -= 1 - self.write('') - self.write('') - self.__levels_deep += 1 - - def write_footer(self): - self.write('') - self.__levels_deep -= 1 - self.write('') - self.__levels_deep -= 1 - - @contextmanager - def place_elm_with_child(self, tag_name: str, attr_dict: Dict[str, str] = {}, css: Dict[str, str] = None, no_additional_char = False): - lineend = '\n' - deep_space = ' ' - if no_additional_char: - lineend = '' - deep_space = '' - - if css is not None: - attr_dict['style'] = ";".join([ - f'{name}: {value}' - for name, value in css.items() - ]) - attrs = " ".join([ - f'{name}="{value}"' - for name, value in attr_dict.items() - ]) - self.write(f"<{tag_name} {attrs}>", lineend=lineend) - self.__levels_deep += 1 - yield - self.__levels_deep -= 1 - self.write(f"", deep_space=deep_space) - - def place_elm_close(self, tag_name: str, attr_dict: Dict[str, str] = {}, css: Dict[str, str] = None, as_childless = True): - if css is not None: - attr_dict['style'] = ";".join([ - f'{name}: {value}' - for name, value in css.items() - ]) - attrs = " ".join([ - f'{name}="{value}"' - for name, value in attr_dict.items() - ]) - if as_childless: - self.write(f"<{tag_name} {attrs}>") - else: - self.write(f"<{tag_name} {attrs}/>") - - def place_image(self, item: LTImage): - name = item.name - if self.imagewriter is not None: - name = self.imagewriter.export_image(item) - self.place_elm_close('img', { - 'src': enc(name, None), - 'width': f"{item.width}px", - 'height': f"{item.height}px", - }) - return - - def render_curve(self, item: LTCurve): - css = { - 'position': 'absolute', - 'left': f'{item.x0}px', - 'bottom': f'{item.y0}px', - 'width': f'{item.width}px', - 'height': f'{item.height}px', - } - path = "" - svg_attr = { - 'class': 'curve', - 'z-index': 2 - } - - p_vals = { "x": [], "y": [] } - - for p in item.paths: - for pt in p.points: - p_vals['x'].append(pt.x) - p_vals['y'].append(pt.y) - - mx_x = max([*p_vals['x'], 1]) - mn_x = min([*p_vals['x'], 0]) - mx_y = max([*p_vals['y'], 1]) - mn_y = min([*p_vals['y'], 0]) - - if mx_x == mn_x: - mn_x -= 1 - if mx_y == mn_y: - mn_y -= 1 - - dx = 1/( mx_x - mn_x ) - dy = (item.height/item.width)/( mx_y - mn_y ) - ROUND_VAL = 5 - svg_attr.update({ - # "viewBox": f"{min(p_vals['x'])} {min(p_vals['y'])} {max(p_vals['x'])} {max(p_vals['y'])}", - "viewBox": f"{round(mn_x*dx, ROUND_VAL)} {round(mn_y*dy, ROUND_VAL)} {round(mx_x*dx , ROUND_VAL)} {round(mx_y*dy, ROUND_VAL)}", - }) - - for p in item.paths: - if p.method == CurvePath.METHOD.MOVE_TO: - path += f'M{" ".join( f"{round(dx*pt.x, ROUND_VAL)} {round(dy*pt.y, ROUND_VAL)}" for pt in p.points )} ' - if p.method == CurvePath.METHOD.LINE_TO: - path += f'L{" ".join( f"{round(dx*pt.x, ROUND_VAL)} {round(dy*pt.y, ROUND_VAL)}" for pt in p.points )} ' - if p.method == CurvePath.METHOD.CLOSE_PATH: - path += 'Z' - with self.place_elm_with_child('svg', svg_attr, css): - self.place_elm_close('path', { - "d": path, - 'stroke': get_color(item.stroke), - 'fill': get_color(item.fill), - }) - - def receive_layout(self, ltpage: LTPage): - def render(item: LTItem): - if isinstance(item, LTPage): - with self.place_elm_with_child('div', { "class": 'page', "id": f"page-{item.pageid}" }, { 'width': f'{item.width}px', 'height': f'{item.height}px' }): - self.__current_height = item.height - self.__current_width = item.width - for child in item: - render(child) - elif isinstance(item, LTCurve): - self.render_curve(item) - elif isinstance(item, LTFigure): - with self.place_elm_with_child('div', { "class": 'figure' }, { - 'position': 'absolute', - 'left': f'{item.x0}px', - 'bottom': f'{item.y0}px', - 'width': f'{item.width}px', - 'height': f'{item.height}px', - }): - for child in item: - render(child) - elif isinstance(item, LTImage): - self.place_image(item) - elif isinstance(item, LTChar): - with self.place_elm_with_child('span', { 'class': 'char' }, { - 'width': item.width, - 'min-width': item.width, - 'max-width': item.width, - 'height': item.height, - 'min-height': item.height, - 'max-height': item.height, - }): - self.write(item.get_text()) - elif isinstance(item, LTCharBlock): - with self.place_elm_with_child('span', { 'class': 'char-block' }, { - 'position': 'absolute', - 'left': f'{item.x0}', - 'bottom': f'{item.y0}', - 'text-align': 'justify', - 'font-size': f'{item.size}px', - 'font-family': item.fontname, - 'font-weight': item.font.font_weight, - 'color': get_color(item.graphicstate.ncolor), - 'width': item.width, - 'min-width': item.width, - 'max-width': item.width, - 'height': item.height, - 'min-height': item.height, - 'max-height': item.height, - 'transform': f'skew({item.font.italic_angle}deg, 0deg)', - 'text-align': 'center', - 'display': 'flex', - 'flex-flow': 'row nowrap', - 'justify-content': 'space-between', - # 'padding-left': item.font.leading - }): - for char in item: - render(char) - render(ltpage) - - def close(self): - self.write_footer() - \ No newline at end of file diff --git a/pdfmajor/converters/XMLConverter.py b/pdfmajor/converters/XMLConverter.py deleted file mode 100644 index dba1084..0000000 --- a/pdfmajor/converters/XMLConverter.py +++ /dev/null @@ -1,192 +0,0 @@ -import logging -import re -import json - -from typing import Dict - -from xml.sax import saxutils -from contextlib import contextmanager - -from ..layouts import LTPage -from ..layouts import LTItem -from ..layouts import LTCurve -from ..layouts import LTLine -from ..layouts import LTRect -from ..layouts import LTFigure -from ..layouts import LTImage -from ..layouts import LTChar, LTCharBlock -from ..interpreter.PDFGraphicState import PDFGraphicStateColor -from ..utils import enc - -from .PDFConverter import PDFConverter - -log = logging.getLogger(__name__) - -def get_color(col: PDFGraphicStateColor): - color_type, color_val = col.get_color() - if color_type is not None: - if color_type == 'rgb': - return f"rgb({','.join(map(str, color_val))})" - elif color_type == 'cmyk': - return f"cmyk({','.join(map(str, color_val))})" - elif color_type == 'gray': - return f'gray({color_val})' - elif color_type == "custom": - # Todo: convert these to more friendly color types - return f"{col.custom['type']}({col.custom['defintion']})" - return "" - -FIRST_CAP_RE = re.compile('(.)([A-Z][a-z]+)') -ALL_CAP_RE = re.compile('([a-z0-9])([A-Z])') -def convert_capcase_to_snakecase(text: str) -> str: - s1 = FIRST_CAP_RE.sub(r'\1-\2', text) - return ALL_CAP_RE.sub(r'\1-\2', s1).lower() - -class XMLConverter(PDFConverter): - def __init__(self, - rsrcmgr, - outfp, - codec='utf-8', - pageno=1, - imagewriter=None, - ): - PDFConverter.__init__(self, rsrcmgr, outfp, imagewriter=imagewriter, codec=codec, pageno=pageno) - self.__levels_deep = 0 - self.write_header() - - def write(self, text: str, lineend: str = '\n', deep_space: str = ' '): - text = deep_space*self.__levels_deep + text + lineend - self.write_raw(text) - - def write_raw(self, text): - if self.codec: - text = text.encode(self.codec) - self.outfp.write(text) - - def write_header(self): - if self.codec: - self.write('' % self.codec) - else: - self.write('') - - self.write('') - self.__levels_deep += 1 - - def write_footer(self): - self.write('') - - @contextmanager - def place_elm_with_child(self, tag_name: str, attr_dict: Dict[str, str] = {}, no_additional_char = False): - lineend = '\n' - deep_space = ' ' - if no_additional_char: - lineend = '' - deep_space = '' - attrs = " ".join([ - f'{name}="{saxutils.escape(str(value))}"' - for name, value in attr_dict.items() - ]) - self.write(f"<{tag_name} {attrs}>", lineend=lineend) - self.__levels_deep += 1 - yield - self.__levels_deep -= 1 - self.write(f"", deep_space=deep_space) - - def place_elm_close(self, tag_name: str, attr_dict: Dict[str, str] = {}, as_childless = True): - attrs = " ".join([ - f'{name}="{saxutils.escape(str(value))}"' - for name, value in attr_dict.items() - ]) - if as_childless: - self.write(f"<{tag_name} {attrs}>") - else: - self.write(f"<{tag_name} {attrs}/>") - - def place_full_container(self, class_name: str, item: LTItem): - return self.place_elm_with_child(class_name, { - "x0": item.x0, - "x1": item.x1, - "y0": item.y0, - "y1": item.y1, - }) - - def place_image(self, item: LTImage): - name = item.name - - if self.imagewriter is not None: - name = self.imagewriter.export_image(item) - - self.place_elm_close('img', { - 'src': enc(name, None), - "x0": item.x0, - "x1": item.x1, - "y0": item.y0, - "y1": item.y1, - }) - - def place_text(self, char: LTChar): - with self.place_elm_with_child('char', { - 'x0': char.x0, - 'x1': char.x1, - 'y0': char.y0, - 'y1': char.y1, - }, no_additional_char=True): - self.write(saxutils.escape(char.get_text()), lineend='', deep_space="") - - def render_char_block(self, char_block: LTCharBlock): - attr = { - 'size': char_block.size, - "color": get_color(char_block.graphicstate.ncolor), - "x0": char_block.x0, - "x1": char_block.x1, - "y0": char_block.y0, - "y1": char_block.y1, - } - for key, value in char_block.font.descriptor.items(): - if key != "Type" and "FontFile" not in key: - attr[convert_capcase_to_snakecase(key)] = value - with self.place_elm_with_child("char-block", attr): - for char in char_block: - self.place_text(char) - - def render_curve(self, item: LTCurve): - attr = { - "x0": item.x0, - "x1": item.x1, - "y0": item.y0, - "y1": item.y1, - "stroke-color": get_color(item.stroke), - "fill-color": get_color(item.fill), - } - if isinstance(item, LTRect): - return self.place_elm_close('rect', attr) - elif isinstance(item, LTLine): - return self.place_elm_close('line', attr) - - with self.place_elm_with_child('curve', attr): - for path in item.paths: - with self.place_elm_with_child('path', { "type": path.method.name }): - for point in path.points: - self.place_elm_close('point', { 'x': point.x, 'y': point.y }) - - def receive_layout(self, ltpage: LTPage): - def render(item: LTItem): - if isinstance(item, LTPage): - with self.place_full_container('page', item): - for child in item: - render(child) - elif isinstance(item, LTCurve): - self.render_curve(item) - elif isinstance(item, LTFigure): - with self.place_full_container('figure', item): - for child in item: - render(child) - elif isinstance(item, LTImage): - self.place_image(item) - elif isinstance(item, LTCharBlock): - self.render_char_block(item) - render(ltpage) - - def close(self): - self.write_footer() - \ No newline at end of file diff --git a/pdfmajor/converters/__init__.py b/pdfmajor/converters/__init__.py index a2bf37a..7a702f3 100644 --- a/pdfmajor/converters/__init__.py +++ b/pdfmajor/converters/__init__.py @@ -4,15 +4,18 @@ import re from typing import List -from .HTMLConverter import HTMLConverter -from .JSONConverter import JSONConverter -from .XMLConverter import XMLConverter -from .TextConverter import TextConverter +from .html import convert_to_html +from .xml import convert_to_xml +from .json import convert_to_json +# from .HTMLConverter import HTMLConverter +# from .JSONConverter import JSONConverter +# from .XMLConverter import XMLConverter +# from .TextConverter import TextConverter from ..utils import logging def convert_file( - input_file: io.TextIOWrapper, - output_file: io.TextIOWrapper = None, + input_file: str, + output_file: str = None, image_folder_path: str = None, codec: str = 'utf-8', maxpages: int = 0, @@ -25,35 +28,32 @@ def convert_file( out_type: str = 'html', ): if output_file is None: - output_file = open( - os.path.join( - os.path.dirname(input_file.name), - os.path.basename( - re.sub(r"\.\w+$", "."+out_type, input_file.name) - ) - ), - 'wb' + output_file = os.path.join( + os.path.dirname(input_file.name), + os.path.basename( + re.sub(r"\.\w+$", "."+out_type, input_file.name) + ) ) - if re.search(re.escape(out_type) + r'$', output_file.name ) is None: + if re.search(re.escape(out_type) + r'$', output_file ) is None: raise Exception("Please make sure that the file name and output type match!") if out_type == 'html': - return HTMLConverter.parse_file( - input_file=input_file, - output_file=output_file, + return convert_to_html( + input_file_path=input_file, + output_file_path=output_file, image_folder_path=image_folder_path, + dont_export_images=dont_export_images, codec=codec, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable, pagenos=pagenos, - dont_export_images=dont_export_images, debug_level=debug_level, ) elif out_type == 'xml': - return XMLConverter.parse_file( - input_file=input_file, - output_file=output_file, + return convert_to_xml( + input_file_path=input_file, + output_file_path=output_file, image_folder_path=image_folder_path, codec=codec, maxpages=maxpages, @@ -65,23 +65,9 @@ def convert_file( debug_level=debug_level, ) elif out_type == 'json': - return JSONConverter.parse_file( - input_file=input_file, - output_file=output_file, - image_folder_path=image_folder_path, - codec=codec, - maxpages=maxpages, - password=password, - caching=caching, - check_extractable=check_extractable, - pagenos=pagenos, - dont_export_images=dont_export_images, - debug_level=debug_level, - ) - elif out_type == 'text': - return TextConverter.parse_file( - input_file=input_file, - output_file=output_file, + return convert_to_json( + input_file_path=input_file, + output_file_path=output_file, image_folder_path=image_folder_path, codec=codec, maxpages=maxpages, @@ -92,4 +78,18 @@ def convert_file( dont_export_images=dont_export_images, debug_level=debug_level, ) + # elif out_type == 'text': + # return TextConverter.parse_file( + # input_file=input_file, + # output_file=output_file, + # image_folder_path=image_folder_path, + # codec=codec, + # maxpages=maxpages, + # password=password, + # caching=caching, + # check_extractable=check_extractable, + # pagenos=pagenos, + # dont_export_images=dont_export_images, + # debug_level=debug_level, + # ) else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'") \ No newline at end of file diff --git a/pdfmajor/converters/html.py b/pdfmajor/converters/html.py index bcb24a3..04df3a3 100644 --- a/pdfmajor/converters/html.py +++ b/pdfmajor/converters/html.py @@ -1,9 +1,15 @@ -from ..extractor import extract_items_from_pdf +from typing import List +from .writers.html import HTMLMaker +from ..interpreter import PDFInterpreter, PageInterpreter, logging +from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject +from ..interpreter.commands import LTItem +from ..interpreter.commands.state import CurvePath, PDFColor def convert_to_html( input_file_path: str, - output_file: io.TextIOWrapper = None, + output_file_path: str, image_folder_path: str = None, + dont_export_images: bool = False, codec: str = 'utf-8', maxpages: int = 0, password: str = None, @@ -12,4 +18,172 @@ def convert_to_html( pagenos: List[int] = None, debug_level: int = logging.WARNING, ): - \ No newline at end of file + intepreter = PDFInterpreter(input_file_path, + maxpages=maxpages, + password=password, + caching=caching, + check_extractable=check_extractable, + pagenos=pagenos, + debug_level=debug_level + ) + with HTMLMaker(output_file_path, codec=codec) as html: + with html.elm("html", nolineend=True): + with html.elm("head"): + meta_attr = { + 'http-equiv': "Content-Type", + 'content': "text/html", + } + if html.codec: + meta_attr['charset'] = html.codec + html.singleton("meta", meta_attr) + with html.elm('style'): + html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''') + with html.elm("body"): + for page in intepreter: + render_page(html, page) + +def render_page(html: HTMLMaker, ltpage: PageInterpreter): + def render(item: LTItem): + if isinstance(item, LTCurve): + render_curve(html, item) + elif isinstance(item, LTXObject): + with html.elm('div', { "class": 'xobject' }, { + 'position': 'absolute', + 'left': f'{item.x0}px', + 'bottom': f'{item.y0}px', + 'width': f'{item.width}px', + 'height': f'{item.height}px', + }): + for child in item: + render(child) + elif isinstance(item, LTImage): + name = item.name + # if html.imagewriter is not None: + # name = html.imagewriter.export_image(item) + html.singleton('img', attrs={ + 'src': name, + 'background': 'red', + 'width': f"{item.width}px", + 'height': f"{item.height}px", + }) + elif isinstance(item, LTChar): + with html.elm('span', { 'class': 'char' }, { + 'width': item.width, + 'min-width': item.width, + 'max-width': item.width, + 'height': item.height, + 'min-height': item.height, + 'max-height': item.height, + }): + html.write(item.get_text()) + elif isinstance(item, LTCharBlock): + with html.elm('span', { 'class': 'char-block' }, { + 'position': 'absolute', + 'left': f'{item.x0}', + 'bottom': f'{item.y0}', + 'font-size': f'{item.size}px', + 'font-family': item.fontname, + 'font-weight': item.font.font_weight, + 'color': get_color(item.color, default="black"), + 'width': item.width, + 'min-width': item.width, + 'max-width': item.width, + 'height': item.height, + 'min-height': item.height, + 'max-height': item.height, + 'transform': f'skew({item.font.italic_angle}deg, 0deg)', + 'text-align': 'center', + 'display': 'flex', + 'flex-flow': 'row nowrap', + 'justify-content': 'space-between', + # 'padding-left': item.font.leading + }): + for char in item: + render(char) + + with html.elm('div', { "class": 'page', "id": f"page-{ltpage.page_num}" }, { + 'width': f'{ltpage.width}px', + 'height': f'{ltpage.height}px' + }): + for child in ltpage: + render(child) + +def get_color(col: PDFColor, default: str = ""): + if col.color_space is not None: + if col.color_space.ncomponents == 1: + if type(col.values[0]) in [float, int]: + return f"rgba(0,0,0,{1 - float(col.values[0])})" + elif col.color_space.ncomponents == 3: + return f"rgb({','.join(map(str, col.values))})" + elif col.color_space.ncomponents == 4: + return f"cmyk({','.join(map(str, col.values))})" + return f"{col.color_space.name}({','.join(map(str, col.values))})" + else: + return default + +def render_curve(html: HTMLMaker, item: LTCurve): + css = { + 'position': 'absolute', + 'left': f'{item.x0}px', + 'bottom': f'{item.y0}px', + 'width': f'{item.width}px', + 'height': f'{item.height}px', + } + path = "" + svg_attr = { + 'class': 'curve', + 'z-index': 2 + } + path_css = { + + } + + # Compute ViewBox + p_vals = { "x": [], "y": [] } + + for p in item.paths: + for pt in p.points: + p_vals['x'].append(pt.x) + p_vals['y'].append(pt.y) + + mx_x = max([*p_vals['x'], 1]) + mn_x = min([*p_vals['x'], 0]) + mx_y = max([*p_vals['y'], 1]) + mn_y = min([*p_vals['y'], 0]) + + if mx_x == mn_x: + mn_x -= 1 + if mx_y == mn_y: + mn_y -= 1 + + dx = 1/( mx_x - mn_x ) + dy = (item.height/item.width)/( mx_y - mn_y ) + ROUND_VAL = 5 + svg_attr.update({ + # "viewBox": f"{min(p_vals['x'])} {min(p_vals['y'])} {max(p_vals['x'])} {max(p_vals['y'])}", + "viewBox": f"{round(mn_x*dx, ROUND_VAL)} {round(mn_y*dy, ROUND_VAL)} {round(mx_x*dx , ROUND_VAL)} {round(mx_y*dy, ROUND_VAL)}", + }) + + # Compute shape + for p in item.paths: + if p.method == CurvePath.METHOD.MOVE_TO: + path += f'M{" ".join( f"{round(dx*pt.x, ROUND_VAL)} {round(dy*pt.y, ROUND_VAL)}" for pt in p.points )} ' + if p.method == CurvePath.METHOD.LINE_TO: + path += f'L{" ".join( f"{round(dx*pt.x, ROUND_VAL)} {round(dy*pt.y, ROUND_VAL)}" for pt in p.points )} ' + if p.method == CurvePath.METHOD.CLOSE_PATH: + path += 'Z' + + stroke_col = get_color(item.stroke) + fill_col = get_color(item.fill) + if fill_col: + path_css['fill'] = fill_col + if stroke_col: + path_css['stroke'] = stroke_col + + with html.elm('svg', svg_attr, css): + html.singleton('path', { + "d": path, + **path_css + }, { + **path_css + }) diff --git a/pdfmajor/converters/json.py b/pdfmajor/converters/json.py new file mode 100644 index 0000000..756c5a0 --- /dev/null +++ b/pdfmajor/converters/json.py @@ -0,0 +1,134 @@ +from typing import List + +from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray +from ..interpreter import PDFInterpreter, PageInterpreter, logging +from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject +from ..interpreter.commands import LTItem +from ..interpreter.commands import LTRect, LTLine +from ..interpreter.commands.state import CurvePath, PDFColor +from ..parser.PSStackParser import PSLiteral +from ..utils import enc + +def normalize_color_values(values: list): + ret = [] + for v in values: + if isinstance(v, PSLiteral): + ret.append( enc(str(v.name)).decode() ) + else: + ret.append( enc(str(v)).decode() ) + return ret + +def convert_to_json( + input_file_path: str, + output_file_path: str, + image_folder_path: str = None, + dont_export_images: bool = False, + codec: str = 'utf-8', + maxpages: int = 0, + password: str = None, + caching: bool = True, + check_extractable: bool = True, + pagenos: List[int] = None, + debug_level: int = logging.WARNING, +): + intepreter = PDFInterpreter(input_file_path, + maxpages=maxpages, + password=password, + caching=caching, + check_extractable=check_extractable, + pagenos=pagenos, + debug_level=debug_level + ) + with JSONMaker(output_file_path, codec=codec) as json: + with json.array("pages") as arr: + for page in intepreter: + with arr.object() as page_obj: + render_page(page_obj, page) + +def render_page(json: JSONMakerObject, ltpage: PageInterpreter): + json.number("id", ltpage.page_num) + json.number("width", ltpage.width) + json.number("height", ltpage.height) + with json.array("children") as chil_arr: + for child in ltpage: + with chil_arr.object() as chil_obj: + render(chil_obj, child) + +def render(json: JSONMakerObject, item: LTItem): + if isinstance(item, LTCurve): + place_curve(json, item) + elif isinstance(item, LTXObject): + json.string("type", 'xobject') + json.number("x0", item.x0) + json.number("x1", item.x1) + json.number("y0", item.y0) + json.number("y1", item.y1) + with json.array("children") as chil_arr: + for child in item: + with chil_arr.object() as chil_obj: + render(chil_obj, child) + elif isinstance(item, LTImage): + # if json.imagewriter is not None: + # name = json.imagewriter.export_image(item) + json.string("type", 'img') + json.string("src", enc(item.name, None)) + json.number("x0", item.x0) + json.number("x1", item.x1) + json.number("y0", item.y0) + json.number("y1", item.y1) + elif isinstance(item, LTCharBlock): + place_char_block(json, item) + +def place_char_block(json: JSONMakerObject, char_block: LTCharBlock): + json.string("type", 'char-block') + json.number("size", char_block.size) + json.number("x0", char_block.x0) + json.number("x1", char_block.x1) + json.number("y0", char_block.y0) + json.number("y1", char_block.y1) + if char_block.color.color_space is not None: + json.place_object("color", { + "type": enc(char_block.color.color_space.name).decode(), + "value": normalize_color_values(char_block.color.values) + }) + for key, value in char_block.font.descriptor.items(): + if key != "Type" and "FontFile" not in key: + json.string(key, value) + json.string("text", + "".join([ + char.get_text() + for char in char_block + ]) + ) + +def place_curve(json: JSONMakerObject, item: LTCurve): + json.number("x0", item.x0) + json.number("x1", item.x1) + json.number("y0", item.y0) + json.number("y1", item.y1) + if item.fill.color_space is not None: + json.place_object("fill", { + "type": enc(item.fill.color_space.name).decode(), + "value": normalize_color_values(item.fill.values) + }) + if item.stroke.color_space is not None: + json.place_object("stroke", { + "type": enc(item.stroke.color_space.name).decode(), + "value": normalize_color_values(item.stroke.values) + }) + + if isinstance(item, LTRect): + json.string("type", 'rect') + elif isinstance(item, LTLine): + json.string("type", 'line') + else: + json.string("type", 'curve') + with json.array('paths') as paths_arr: + for path in item.paths: + paths_arr.place_object({ + "type": path.method.name, + "points": [ + {"x": point.x, "y": point.y} + for point in path.points + ] + }) \ No newline at end of file diff --git a/pdfmajor/converters/writers/__init__.py b/pdfmajor/converters/writers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pdfmajor/converters/writers/html.py b/pdfmajor/converters/writers/html.py new file mode 100644 index 0000000..391a727 --- /dev/null +++ b/pdfmajor/converters/writers/html.py @@ -0,0 +1,62 @@ +from io import TextIOWrapper +from contextlib import contextmanager + +class HTMLMaker: + class AccessError(Exception): pass + def __init__(self, file_path: str, codec: str = 'utf-8'): + self.file_path = file_path + self.codec = codec + self.outfile: TextIOWrapper = None + + def __enter__(self): + self.outfile = open(self.file_path, 'wb') + self.__levels_deep = 0 + return self + def __exit__(self, exc_type, exc_val, exc_tb): + self.outfile.close() + self.outfile = None + + def write_raw(self, text): + if self.outfile is not None: + if self.codec: + text = text.encode(self.codec) + self.outfile.write(text) + else: + raise self.AccessError("Attempting to write to file without an open connection") + + def write(self, text: str, lineend: str = '\n', deep_space: str = ' '): + text = deep_space*self.__levels_deep + text + lineend + self.write_raw(text) + + @contextmanager + def elm(self, tag_name: str, attrs: dict = None, css: dict = None, nolineend: bool = False, no_additional_char: bool = False, singleton: bool = False): + lineend = '\n' + deep_space = ' ' + if no_additional_char: + lineend = '' + deep_space = '' + if attrs is None: + attrs = {} + if css is not None: + attrs['style'] = ";".join([ + f'{name}: {value}' + for name, value in css.items() + ]) + attrs = " ".join([ + f'{name}="{value}"' + for name, value in attrs.items() + ]) + if not singleton: + self.write(f"<{tag_name} {attrs}>", lineend=lineend) + self.__levels_deep += 1 + yield + self.__levels_deep -= 1 + self.write(f"", deep_space=deep_space, lineend='\n' if not nolineend else '') + else: + self.write(f"<{tag_name} {attrs} />", lineend=lineend, deep_space=deep_space) + yield + + def singleton(self, tag_name: str, attrs: dict = None, css: dict = None): + with self.elm(tag_name, attrs, css, singleton=True): + pass + \ No newline at end of file diff --git a/pdfmajor/converters/writers/json.py b/pdfmajor/converters/writers/json.py new file mode 100644 index 0000000..94a2bb7 --- /dev/null +++ b/pdfmajor/converters/writers/json.py @@ -0,0 +1,129 @@ +from json import dumps as json_dump +from io import TextIOWrapper +from contextlib import contextmanager + +@contextmanager +def JSONMaker(file_path: str, codec: str = 'utf-8'): + with open(file_path, 'wb') as outf: + with JSONMakerObject(outf, codec=codec) as obj: + yield obj + + +class JSONMakerObject: + class AccessError(Exception): pass + def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = 0): + self.outfile: TextIOWrapper = outfile + self.codec = codec + self.levels_deep = levels_deep + self.keys = set() + + def __enter__(self): + self.write_raw('{') + self.levels_deep += 1 + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.levels_deep -= 1 + self.write('}', prefix='\n') + + def write_raw(self, text): + if self.outfile is not None: + if self.codec: + text = text.encode(self.codec) + self.outfile.write(text) + else: + raise self.AccessError("Attempting to write to file without an open connection") + + def write(self, text: str, deep_space: str = ' ', prefix:str = ''): + text = prefix + (deep_space*self.levels_deep) + text + self.write_raw(text) + + def write_prop(self, key: str, val: str = ""): + if key not in self.keys: + self.keys.add(key) + if len(self.keys) > 1: + self.write(f'"{key}": {val}', prefix=',\n') + else: + self.write(f'"{key}": {val}', prefix='\n') + + def number(self, key: str, val: float or int): + self.write_prop(key, val) + + def string(self, key: str, val: str): + self.write_prop(key, f'"{val}"') + + @contextmanager + def object(self, key: str): + self.write_prop(key) + with JSONMakerObject(self.outfile, codec=self.codec, levels_deep=self.levels_deep) as child: + yield child + + @contextmanager + def array(self, key: str): + self.write_prop(key) + with JSONMakerArray(self.outfile, codec=self.codec, levels_deep=self.levels_deep) as child: + yield child + + def place_object(self, key: str, obj: dict): + self.write_prop(key, json_dump(obj)) + + + +class JSONMakerArray: + class AccessError(Exception): pass + def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = 0): + self.outfile: TextIOWrapper = outfile + self.codec = codec + self.levels_deep = levels_deep + self.childrencount = 0 + + def __enter__(self): + self.write_raw('[') + self.levels_deep += 1 + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.levels_deep -= 1 + self.write(']', prefix='\n') + + def write_raw(self, text): + if self.outfile is not None: + if self.codec: + text = text.encode(self.codec) + self.outfile.write(text) + else: + raise self.AccessError("Attempting to write to file without an open connection") + + def write(self, text: str, deep_space: str = ' ', prefix:str = ''): + text = prefix + (deep_space*self.levels_deep) + text + self.write_raw(text) + + def write_prop(self, val: str = ""): + self.childrencount += 1 + if self.childrencount > 1: + self.write(f'{val}', prefix=',\n') + else: + self.write(f'{val}', prefix='\n') + + def number(self, val: float or int): + self.write_prop(val) + + def string(self, val: str): + self.write_prop(f'"{val}"') + + @contextmanager + def object(self): + self.write_prop() + with JSONMakerObject(self.outfile, codec=self.codec, levels_deep=self.levels_deep) as child: + yield child + + @contextmanager + def array(self): + self.write_prop() + with JSONMakerArray(self.outfile, codec=self.codec, levels_deep=self.levels_deep) as child: + yield child + + def place_object(self, obj: dict): + self.write_prop(json_dump(obj)) + + \ No newline at end of file diff --git a/pdfmajor/converters/writers/xml.py b/pdfmajor/converters/writers/xml.py new file mode 100644 index 0000000..e41b153 --- /dev/null +++ b/pdfmajor/converters/writers/xml.py @@ -0,0 +1,60 @@ +from io import TextIOWrapper +from contextlib import contextmanager + +class XMLMaker: + class AccessError(Exception): pass + def __init__(self, file_path: str, codec: str = 'utf-8'): + self.file_path = file_path + self.codec = codec + self.outfile: TextIOWrapper = None + def __enter__(self): + self.outfile = open(self.file_path, 'wb') + if self.codec: + self.write_raw(f'') + else: + self.write_raw(f'') + self.__levels_deep = 0 + return self + def __exit__(self, exc_type, exc_val, exc_tb): + self.outfile.close() + self.outfile = None + + def write_raw(self, text): + if self.outfile is not None: + if self.codec: + text = text.encode(self.codec) + self.outfile.write(text) + else: + raise self.AccessError("Attempting to write to file without an open connection") + + def write(self, text: str, lineend: str = '\n', deep_space: str = ' '): + text = deep_space*self.__levels_deep + text + lineend + self.write_raw(text) + + @contextmanager + def elm(self, tag_name: str, attrs: dict = None, nolineend: bool = False, no_additional_char: bool = False, singleton: bool = False): + lineend = '\n' + deep_space = ' ' + if no_additional_char: + lineend = '' + deep_space = '' + if attrs is None: + attrs = {} + attrs = " ".join([ + f'{name}="{value}"' + for name, value in attrs.items() + ]) + if not singleton: + self.write(f"<{tag_name} {attrs}>", lineend=lineend) + self.__levels_deep += 1 + yield + self.__levels_deep -= 1 + self.write(f"", deep_space=deep_space, lineend='\n' if not nolineend else '') + else: + self.write(f"<{tag_name} {attrs} />", lineend=lineend, deep_space=deep_space) + yield + + def singleton(self, tag_name: str, attrs: dict = None): + with self.elm(tag_name, attrs, singleton=True): + pass + \ No newline at end of file diff --git a/pdfmajor/converters/xml.py b/pdfmajor/converters/xml.py new file mode 100644 index 0000000..d03b518 --- /dev/null +++ b/pdfmajor/converters/xml.py @@ -0,0 +1,142 @@ +import re +from typing import List +from xml.sax import saxutils + +from .writers.xml import XMLMaker +from ..interpreter import PDFInterpreter, PageInterpreter, logging +from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject +from ..interpreter.commands import LTItem +from ..interpreter.commands import LTRect, LTLine +from ..interpreter.commands.state import CurvePath, PDFColor +from ..parser.PSStackParser import PSLiteral +from ..utils import enc + +FIRST_CAP_RE = re.compile('(.)([A-Z][a-z]+)') +ALL_CAP_RE = re.compile('([a-z0-9])([A-Z])') +def convert_capcase_to_snakecase(text: str) -> str: + s1 = FIRST_CAP_RE.sub(r'\1-\2', text) + return ALL_CAP_RE.sub(r'\1-\2', s1).lower() + +def normalize_color_values(values: list): + ret = "" + for i,v in enumerate(values): + if i > 0: + ret += ',' + if isinstance(v, PSLiteral): + ret += enc(str(v.name)).decode() + else: + ret += enc(str(v)).decode() + return ret + +def get_color(col: PDFColor, prefix: str = 'color', codec = 'utf-8'): + if col.color_space is not None: + return { + f"{prefix}-type": enc(col.color_space.name).decode(), + f"{prefix}-values": normalize_color_values(col.values) + } + return {} + +def convert_to_xml( + input_file_path: str, + output_file_path: str, + image_folder_path: str = None, + dont_export_images: bool = False, + codec: str = 'utf-8', + maxpages: int = 0, + password: str = None, + caching: bool = True, + check_extractable: bool = True, + pagenos: List[int] = None, + debug_level: int = logging.WARNING, +): + intepreter = PDFInterpreter(input_file_path, + maxpages=maxpages, + password=password, + caching=caching, + check_extractable=check_extractable, + pagenos=pagenos, + debug_level=debug_level + ) + with XMLMaker(output_file_path, codec=codec) as xml: + with xml.elm("pages"): + for page in intepreter: + render_page(xml, page) + +def render_page(xml: XMLMaker, ltpage: PageInterpreter): + def render(item: LTItem): + if isinstance(item, LTCurve): + place_curve(xml, item) + elif isinstance(item, LTXObject): + with xml.elm('xobject', { + "x0": item.x0, + "x1": item.x1, + "y0": item.y0, + "y1": item.y1, + }): + for child in item: + render(child) + elif isinstance(item, LTImage): + name = item.name + # if xml.imagewriter is not None: + # name = xml.imagewriter.export_image(item) + xml.singleton('img', attrs={ + 'src': enc(name, None), + "x0": item.x0, + "x1": item.x1, + "y0": item.y0, + "y1": item.y1, + }) + elif isinstance(item, LTCharBlock): + place_char_block(xml, item) + with xml.elm('page', { + "id": ltpage.page_num, + "width": ltpage.width, + "height": ltpage.height, + }): + for child in ltpage: + render(child) + +def place_char(xml: XMLMaker, char: LTChar): + with xml.elm('char', { + 'x0': char.x0, + 'x1': char.x1, + 'y0': char.y0, + 'y1': char.y1, + }, no_additional_char=True): + xml.write(saxutils.escape(char.get_text()), lineend='', deep_space="") + +def place_char_block(xml: XMLMaker, char_block: LTCharBlock): + attr = { + 'size': char_block.size, + "x0": char_block.x0, + "x1": char_block.x1, + "y0": char_block.y0, + "y1": char_block.y1, + **get_color(char_block.color, codec=xml.codec), + } + for key, value in char_block.font.descriptor.items(): + if key != "Type" and "FontFile" not in key: + attr[convert_capcase_to_snakecase(key)] = value + with xml.elm("char-block", attr): + for char in char_block: + place_char(xml, char) + +def place_curve(xml: XMLMaker, item: LTCurve): + attr = { + "x0": item.x0, + "x1": item.x1, + "y0": item.y0, + "y1": item.y1, + **get_color(item.stroke, "stroke", codec=xml.codec), + **get_color(item.fill, "fill", codec=xml.codec), + } + if isinstance(item, LTRect): + return xml.singleton('rect', attr) + elif isinstance(item, LTLine): + return xml.singleton('line', attr) + + with xml.elm('curve', attr): + for path in item.paths: + with xml.elm('path', { "type": path.method.name }): + for point in path.points: + xml.singleton('point', { 'x': point.x, 'y': point.y }) \ No newline at end of file diff --git a/pdfmajor/interpreter/PageInterpreter.py b/pdfmajor/interpreter/PageInterpreter.py index 54aa692..49af4fc 100644 --- a/pdfmajor/interpreter/PageInterpreter.py +++ b/pdfmajor/interpreter/PageInterpreter.py @@ -27,22 +27,26 @@ class PageInterpreter: self.font_cache = font_cache self.page = page self.page_num = page_num + self.height = y1-y0 + self.width = x1-x0 # Init State self.state: PDFStateStack = PDFStateStack() self.state.t_matrix = ctm self.state.resources = page.resources # set some global states. - self.state.graphics.ncolor.color_space = self.state.graphics.scolor.color_space = None + self.state.graphics.ncolspace = self.state.graphics.scolspace = None if self.state.colorspace_map: col_space = next(iter(self.state.colorspace_map.values())) - self.state.graphics.ncolor.color_space = self.state.graphics.scolor.color_space = col_space + self.state.graphics.ncolspace = self.state.graphics.scolspace = col_space # Init Resources init_resources(self.state, self.font_cache) def __iter__(self): parser = PDFContentParser(list_value(self.page.contents)) + history = [] for obj in parser: + history.append(obj) if isinstance(obj, PSKeyword): name = keyword_name(obj) method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q') diff --git a/pdfmajor/interpreter/XObjectInterpreter.py b/pdfmajor/interpreter/XObjectInterpreter.py index 32bbf41..bdd0fd1 100644 --- a/pdfmajor/interpreter/XObjectInterpreter.py +++ b/pdfmajor/interpreter/XObjectInterpreter.py @@ -28,10 +28,10 @@ class XObjectInterpreter: self.state.resources = resources # set some global states. - self.state.graphics.ncolor.color_space = self.state.graphics.scolor.color_space = None + self.state.graphics.ncolspace = self.state.graphics.scolspace = None if self.state.colorspace_map: col_space = next(iter(self.state.colorspace_map.values())) - self.state.graphics.ncolor.color_space = self.state.graphics.scolor.color_space = col_space + self.state.graphics.ncolspace = self.state.graphics.scolspace = col_space init_resources(self.state, self.font_cache) def __iter__(self): diff --git a/pdfmajor/interpreter/commands/commands.py b/pdfmajor/interpreter/commands/commands.py index 479f27d..67df488 100644 --- a/pdfmajor/interpreter/commands/commands.py +++ b/pdfmajor/interpreter/commands/commands.py @@ -32,14 +32,17 @@ def do_EMC(stack: PDFStateStack) -> PDFStateStack: @PDFCommands.add('q') def do_q(stack: PDFStateStack) -> PDFStateStack: - stack.graphicstack.append(stack.graphics) - stack.graphics = PDFGraphicState() + stack.gstack.append([ + stack.t_matrix, + stack.text.copy(), + stack.graphics.copy() + ]) return stack @PDFCommands.add('Q') def do_Q(stack: PDFStateStack) -> PDFStateStack: - if len(stack.graphicstack) > 0: - stack.graphicstate = stack.graphicstack.pop() + if len(stack.gstack) > 0: + stack.t_matrix, stack.text, stack.graphicstate = stack.gstack.pop() return stack # concat-matrix @@ -229,9 +232,8 @@ def do_n(stack: PDFStateStack) -> PDFStateStack: @PDFCommands.add('CS') def do_CS(stack: PDFStateStack, name: str) -> PDFStateStack: try: - stack.graphics.set_stroke_color(stack.colorspace_map[literal_name(name)]) + stack.graphics.scolspace = stack.colorspace_map[literal_name(name)] return stack - # stack.set_scolor() except KeyError: raise PDFCommands.InvalidOperation('Undefined ColorSpace: %r' % name) @@ -239,9 +241,8 @@ def do_CS(stack: PDFStateStack, name: str) -> PDFStateStack: @PDFCommands.add('cs') def do_cs(stack: PDFStateStack, name: str) -> PDFStateStack: try: - stack.graphics.set_nostroke_color(stack.colorspace_map[literal_name(name)]) + stack.graphics.ncolspace = stack.colorspace_map[literal_name(name)] return stack - # stack.set_scolor() except KeyError: raise PDFCommands.InvalidOperation('Undefined ColorSpace: %r' % name) @@ -306,22 +307,22 @@ def do_k(stack: PDFStateStack, c, m, y, k) -> PDFStateStack: @PDFCommands.add('SCN','SC') def do_SCN(stack: PDFStateStack) -> PDFStateStack: - if stack.graphics.scolor.color_space is None: + if stack.graphics.scolspace is None: raise PDFCommands.InvalidOperation('No colorspace specified!') - components = stack.pop(stack.graphics.scolor.color_space.ncomponents) + components = stack.pop(stack.graphics.scolspace.ncomponents) stack.graphics.set_stroke_color( - stack.graphics.scolor.color_space, + stack.graphics.scolspace, *components ) return stack @PDFCommands.add('scn','sc') def do_scn(stack: PDFStateStack) -> PDFStateStack: - if stack.graphics.ncolor.color_space is None: + if stack.graphics.ncolspace is None: raise PDFCommands.InvalidOperation('No colorspace specified!') - components = stack.pop(stack.graphics.ncolor.color_space.ncomponents) + components = stack.pop(stack.graphics.ncolspace.ncomponents) stack.graphics.set_nostroke_color( - stack.graphics.ncolor.color_space, + stack.graphics.ncolspace, *components ) return stack @@ -476,7 +477,7 @@ def do_Do(stack: PDFStateStack, xobjid) -> PDFStateStack: )) elif subtype is LITERAL_IMAGE and 'Width' in xobj and 'Height' in xobj: stack.complete_layout_items.append(make_image( - obj, stack.ctm + xobj, stack.t_matrix )) else: # unsupported xobject type. diff --git a/pdfmajor/interpreter/commands/state/PDFGraphicState/__init__.py b/pdfmajor/interpreter/commands/state/PDFGraphicState/__init__.py index f3fc3a1..e789ba4 100644 --- a/pdfmajor/interpreter/commands/state/PDFGraphicState/__init__.py +++ b/pdfmajor/interpreter/commands/state/PDFGraphicState/__init__.py @@ -14,14 +14,18 @@ class PDFGraphicState(object): self.flatness = None self.scolor:PDFColor = PDFColor(None) self.ncolor:PDFColor = PDFColor(None) + self.scolspace:PDFColorSpace = None + self.ncolspace:PDFColorSpace = None def set_stroke_color(self, colspace: PDFColorSpace, *values): - if self.scolor is None: - self.scolor = PDFColor(colspace, *values) + if colspace is None: + colspace = self.scolspace + self.scolor = PDFColor(colspace, *values) def set_nostroke_color(self, colspace: PDFColorSpace, *values): - if self.ncolor is None: - self.ncolor = PDFColor(colspace, *values) + if colspace is None: + colspace = self.ncolspace + self.ncolor = PDFColor(colspace, *values) def copy(self): obj = PDFGraphicState() @@ -32,8 +36,10 @@ class PDFGraphicState(object): obj.dash = self.dash obj.intent = self.intent obj.flatness = self.flatness - obj.scolor = self.scolor.copy() if self.scolor is not None else None - obj.ncolor = self.ncolor.copy() if self.ncolor is not None else None + obj.scolor = self.scolor.copy() + obj.ncolor = self.ncolor.copy() + obj.scolspace = self.scolspace + obj.ncolspace = self.ncolspace return obj def __repr__(self): diff --git a/pdfmajor/interpreter/commands/state/__init__.py b/pdfmajor/interpreter/commands/state/__init__.py index 6962cef..d6a3a3b 100644 --- a/pdfmajor/interpreter/commands/state/__init__.py +++ b/pdfmajor/interpreter/commands/state/__init__.py @@ -22,7 +22,7 @@ class PDFStateStack: self.t_matrix = MATRIX_IDENTITY self.text = PDFTextState() self.graphics = PDFGraphicState() - self.graphicstack : List[PDFGraphicState] = [] + self.gstack : list = [] self.curvestacks: List[CurvePath] = [] self.argstack: List[bytes] = [] self.complete_layout_items : List[LTItem] = [] diff --git a/pdfmajor/interpreter/commands/state/layout/__init__.py b/pdfmajor/interpreter/commands/state/layout/__init__.py index cb34455..f26d3dd 100644 --- a/pdfmajor/interpreter/commands/state/layout/__init__.py +++ b/pdfmajor/interpreter/commands/state/layout/__init__.py @@ -6,7 +6,7 @@ from ..PDFGraphicState import PDFGraphicState, PDFColor from ..PDFTextState import PDFTextState from ._base import LTItem, LTComponent, LTContainer -from .LTCharBlock import LTCharBlock +from .LTCharBlock import LTCharBlock, LTChar from .LTCurves import LTCurve, LTLine, LTHorizontalLine, LTRect, LTVerticalLine from .LTImage import LTImage from .LTXObject import LTXObject diff --git a/samples/convert_samples.py b/samples/convert_samples.py index eab7991..ac552e0 100755 --- a/samples/convert_samples.py +++ b/samples/convert_samples.py @@ -6,40 +6,46 @@ import logging from typing import List from pdfmajor.converters import convert_file from xml.etree import ElementTree +from json import load as json_load from tqdm import tqdm def rel_path(*arg: List[str]) -> str: - return os.path.join(os.path.dirname(__file__), *arg) + return os.path.normpath(os.path.join(os.path.dirname(__file__), *arg)) -EXT = 'html' +EXTS = ['json','xml','html'] INPUT_FOLDER = rel_path("./pdf") -OUTPUT_FOLDER = rel_path(f"./output/{EXT}") +OUTPUT_FOLDER = rel_path(f"./output/") if not os.path.exists(OUTPUT_FOLDER): os.makedirs(OUTPUT_FOLDER) files = os.listdir(INPUT_FOLDER) -TOTAL = len(files) -waiter = tqdm(enumerate(files), total=TOTAL) -for idx, file_name in waiter: +TOTAL = len(files)*len(EXTS) +waiter = tqdm(total=TOTAL) +for idx, file_name in enumerate(files): file_name_no_ext = re.sub(r'\.pdf$',"", file_name) input_file_path = os.path.join(INPUT_FOLDER, file_name) - output_file_path = os.path.join(OUTPUT_FOLDER, file_name_no_ext + "." + EXT) - if len(file_name) > len(file_name_no_ext): - waiter.set_description(f"Processing {file_name}") - with open(input_file_path, 'rb') as input_file: - with open(output_file_path, 'wb') as output_file: - convert_file( - input_file=input_file, - output_file=output_file, - out_type=EXT, - debug_level=logging.NOTSET, - ) - if EXT == 'xml': - waiter.set_description(f" > testing format {file_name}") - ElementTree.parse(output_file_path) - waiter.refresh() + for ext in EXTS: + if not os.path.exists(os.path.join(OUTPUT_FOLDER, ext)): + os.makedirs(os.path.join(OUTPUT_FOLDER, ext)) + output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext) + if len(file_name) > len(file_name_no_ext): + waiter.set_description(f"Processing {file_name}") + convert_file( + input_file=input_file_path, + output_file=output_file_path, + out_type=ext, + debug_level=logging.WARN, + ) + if ext == 'xml': + waiter.set_description(f" > testing format {file_name}") + ElementTree.parse(output_file_path) + elif ext == 'json': + waiter.set_description(f" > testing format {file_name}") + json_load(open(output_file_path, 'r')) + waiter.refresh() + waiter.update() waiter.set_description("Done") waiter.close() \ No newline at end of file diff --git a/samples/extractor.py b/samples/extractor.py deleted file mode 100644 index a1abd6c..0000000 --- a/samples/extractor.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -import re -import os -import logging - -from typing import List -from pdfmajor.converters import convert_file -from xml.etree import ElementTree -from tqdm import tqdm - -from pdfmajor.extractor import extract_items_from_pdf -from pdfmajor.layouts import LTPage - -def rel_path(*arg: List[str]) -> str: - return os.path.join(os.path.dirname(__file__), *arg) - -print(extract_items_from_pdf(rel_path('pdf/chars.pdf'))) -# for page in extract_items_from_pdf(rel_path('pdf/chars.pdf')): -# print('page-start', page) -# for item in page: -# print(' ', item) -# print('page-end', page) - \ No newline at end of file diff --git a/tests/test_html_writer.py b/tests/test_html_writer.py new file mode 100644 index 0000000..6a47fd8 --- /dev/null +++ b/tests/test_html_writer.py @@ -0,0 +1,60 @@ +from os import remove +from tempfile import mktemp +from pdfmajor.converters.writers.html import HTMLMaker + +# Make temp file +FILE_NAME = mktemp() + + +with HTMLMaker(FILE_NAME) as html: + with html.elm("html", nolineend=True): + html.write("test") +assert(open(FILE_NAME, 'r').read() == """ + test +""") + + +with HTMLMaker(FILE_NAME) as html: + with html.elm("html", nolineend=True): + with html.elm("head"): + html.singleton("meta", { + 'http-equiv': "Content-Type", + 'content': "text/html" + }) +assert(open(FILE_NAME, 'r').read() == """ + + + +""") + +with HTMLMaker(FILE_NAME) as html: + with html.elm("html", nolineend=True): + with html.elm("head"): + html.singleton("meta", { + 'http-equiv': "Content-Type", + 'content': "text/html" + }) + with html.elm("body"): + with html.elm("h1", css={"color": "red"}): + html.write("Hello World") +assert(open(FILE_NAME, 'r').read() == """ + + + + +

+ Hello World +

+ +""") + +with HTMLMaker(FILE_NAME) as html: + with html.elm('style'): + html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''') +assert(open(FILE_NAME, 'r').read() == """ +""") + +# Clean up +remove(FILE_NAME) \ No newline at end of file diff --git a/tests/test_interpreter.py b/tests/test_interpreter.py index c0763b1..cfba498 100644 --- a/tests/test_interpreter.py +++ b/tests/test_interpreter.py @@ -9,7 +9,7 @@ from pdfmajor.interpreter.commands import LTCharBlock, LTChar CUR_PATH = os.path.dirname(os.path.dirname(__file__)) FILE_NAME = os.path.join( "../" if CUR_PATH == "" else CUR_PATH, - "samples/pdf/tables.pdf" + "samples/pdf/long-tables.pdf" ) for page in PDFInterpreter(FILE_NAME, debug_level=logging.INFO): diff --git a/tests/test_json_writer.py b/tests/test_json_writer.py new file mode 100644 index 0000000..c3642ca --- /dev/null +++ b/tests/test_json_writer.py @@ -0,0 +1,112 @@ +from os import remove +from tempfile import mktemp +from pdfmajor.converters.writers.json import JSONMaker + +# Make temp file +FILE_NAME = mktemp() + +with JSONMaker(FILE_NAME) as obj: + obj.number("a", 2) + obj.string("hello", "world") +assert(open(FILE_NAME, 'r').read() == """{ + "a": 2, + "hello": "world" +}""") + +with JSONMaker(FILE_NAME) as obj: + with obj.object("test") as child: + child.number("A", 2) +assert(open(FILE_NAME, 'r').read() == """{ + "test": { + "A": 2 + } +}""") + +with JSONMaker(FILE_NAME) as obj: + with obj.object("hello") as child: + with child.object("nested") as nested: + with nested.object("so") as so: + so.string("really", 'nested'), + so.number("man", 2) + nested.number("he", 3) + child.string('fo', 'real') + obj.string('fo', 'real2') +assert(open(FILE_NAME, 'r').read() == """{ + "hello": { + "nested": { + "so": { + "really": "nested", + "man": 2 + }, + "he": 3 + }, + "fo": "real" + }, + "fo": "real2" +}""") + +with JSONMaker(FILE_NAME) as obj: + with obj.array("hello") as child: + child.number(1) + with child.array() as nchild: + nchild.string('wow') + child.number(3) +assert(open(FILE_NAME, 'r').read() == """{ + "hello": [ + 1, + [ + "wow" + ], + 3 + ] +}""") + +with JSONMaker(FILE_NAME) as obj: + with obj.array("pages") as pages: + with pages.object() as page: + page.number("id", 1) + page.number("height", 20) + page.number("width", 20) + with page.array("children") as children: + with children.object() as item: + item.string("type", "char-block") + item.string("text", "test") + item.string("color_type", "rgb") + with item.array("color_vals") as col_vals: + col_vals.number(0) + col_vals.number(0) + col_vals.number(0) + item.number("x0", 2) + item.number("x1", 4.2) + item.number("y0", 231.12) + item.number("y1", 331.12) +print(open(FILE_NAME, 'r').read()) +assert(open(FILE_NAME, 'r').read() == """{ + "pages": [ + { + "id": 1, + "height": 20, + "width": 20, + "children": [ + { + "type": "char-block", + "text": "test", + "color_type": "rgb", + "color_vals": [ + 0, + 0, + 0 + ], + "x0": 2, + "x1": 4.2, + "y0": 231.12, + "y1": 331.12 + } + ] + } + ] +}""") + + +# Clean up +remove(FILE_NAME) \ No newline at end of file