From e7701ce2acdf9d4947531ed23c660e69df923ea9 Mon Sep 17 00:00:00 2001 From: Ariel Sosnovsky Date: Mon, 24 Dec 2018 01:49:32 -0500 Subject: [PATCH] - correct usage of the BT ET tags --- pdfmajor/converters/html.py | 12 ++-- pdfmajor/converters/json.py | 18 +++-- pdfmajor/converters/text.py | 9 +-- pdfmajor/converters/xml.py | 17 ++++- pdfmajor/interpreter/PageInterpreter.py | 70 +++++-------------- pdfmajor/interpreter/XObjectInterpreter.py | 70 ------------------- pdfmajor/interpreter/__init__.py | 2 +- pdfmajor/interpreter/commands/__init__.py | 70 ++++++++++++++++++- pdfmajor/interpreter/commands/commands.py | 34 +++++++-- .../interpreter/commands/state/__init__.py | 3 +- .../commands/state/layout/LTTextBlock.py | 40 +++++++++++ .../commands/state/layout/__init__.py | 1 + pdfmajor/interpreter/{ => commands}/utils.py | 4 +- 13 files changed, 199 insertions(+), 151 deletions(-) delete mode 100644 pdfmajor/interpreter/XObjectInterpreter.py create mode 100644 pdfmajor/interpreter/commands/state/layout/LTTextBlock.py rename pdfmajor/interpreter/{ => commands}/utils.py (94%) diff --git a/pdfmajor/converters/html.py b/pdfmajor/converters/html.py index 04df3a3..471cd71 100644 --- a/pdfmajor/converters/html.py +++ b/pdfmajor/converters/html.py @@ -1,7 +1,7 @@ from typing import List from .writers.html import HTMLMaker from ..interpreter import PDFInterpreter, PageInterpreter, logging -from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject +from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject from ..interpreter.commands import LTItem from ..interpreter.commands.state import CurvePath, PDFColor @@ -76,8 +76,12 @@ def render_page(html: HTMLMaker, ltpage: PageInterpreter): 'max-height': item.height, }): html.write(item.get_text()) + elif isinstance(item, LTTextBlock): + for char in item: + render(char) elif isinstance(item, LTCharBlock): - with html.elm('span', { 'class': 'char-block' }, { + with html.elm('span', {'class': 'char-block'}): + with html.elm('p', { 'class': 'text-block' }, { 'position': 'absolute', 'left': f'{item.x0}', 'bottom': f'{item.y0}', @@ -98,8 +102,8 @@ def render_page(html: HTMLMaker, ltpage: PageInterpreter): 'justify-content': 'space-between', # 'padding-left': item.font.leading }): - for char in item: - render(char) + for char in item: + render(char) with html.elm('div', { "class": 'page', "id": f"page-{ltpage.page_num}" }, { 'width': f'{ltpage.width}px', diff --git a/pdfmajor/converters/json.py b/pdfmajor/converters/json.py index 756c5a0..2a8c3c4 100644 --- a/pdfmajor/converters/json.py +++ b/pdfmajor/converters/json.py @@ -2,7 +2,7 @@ from typing import List from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray from ..interpreter import PDFInterpreter, PageInterpreter, logging -from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject +from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTCurve, LTXObject from ..interpreter.commands import LTItem from ..interpreter.commands import LTRect, LTLine from ..interpreter.commands.state import CurvePath, PDFColor @@ -76,11 +76,21 @@ def render(json: JSONMakerObject, item: LTItem): json.number("x1", item.x1) json.number("y0", item.y0) json.number("y1", item.y1) - elif isinstance(item, LTCharBlock): - place_char_block(json, item) + elif isinstance(item, LTTextBlock): + place_text_block(json, item) + +def place_text_block(json: JSONMakerObject, text_block: LTTextBlock): + json.string("type", 'text-block') + json.number("x0", text_block.x0) + json.number("x1", text_block.x1) + json.number("y0", text_block.y0) + json.number("y1", text_block.y1) + with json.array("char_blocks") as cb_arr: + for cb in text_block: + with cb_arr.object() as cb_obj: + place_char_block(cb_obj, cb) def place_char_block(json: JSONMakerObject, char_block: LTCharBlock): - json.string("type", 'char-block') json.number("size", char_block.size) json.number("x0", char_block.x0) json.number("x1", char_block.x1) diff --git a/pdfmajor/converters/text.py b/pdfmajor/converters/text.py index 39eae3a..0e50448 100644 --- a/pdfmajor/converters/text.py +++ b/pdfmajor/converters/text.py @@ -1,7 +1,7 @@ from typing import List from ..interpreter import PDFInterpreter, PageInterpreter, logging -from ..interpreter import LTCharBlock, LTXObject +from ..interpreter import LTTextBlock, LTXObject def convert_to_text( @@ -28,9 +28,10 @@ def convert_to_text( with open(output_file_path, 'wb') as outfp: def process_container(container: LTXObject): for item in container: - if isinstance(item, LTCharBlock): - for char in item: - outfp.write(char.get_text().encode(codec)) + if isinstance(item, LTTextBlock): + for text in item: + for char in text: + outfp.write(char.get_text().encode(codec)) elif isinstance(item, LTXObject): process_container(item) for page in intepreter: diff --git a/pdfmajor/converters/xml.py b/pdfmajor/converters/xml.py index d03b518..40c3ca9 100644 --- a/pdfmajor/converters/xml.py +++ b/pdfmajor/converters/xml.py @@ -4,7 +4,7 @@ from xml.sax import saxutils from .writers.xml import XMLMaker from ..interpreter import PDFInterpreter, PageInterpreter, logging -from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject +from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject from ..interpreter.commands import LTItem from ..interpreter.commands import LTRect, LTLine from ..interpreter.commands.state import CurvePath, PDFColor @@ -86,8 +86,8 @@ def render_page(xml: XMLMaker, ltpage: PageInterpreter): "y0": item.y0, "y1": item.y1, }) - elif isinstance(item, LTCharBlock): - place_char_block(xml, item) + elif isinstance(item, LTTextBlock): + place_text_block(xml, item) with xml.elm('page', { "id": ltpage.page_num, "width": ltpage.width, @@ -121,6 +121,17 @@ def place_char_block(xml: XMLMaker, char_block: LTCharBlock): for char in char_block: place_char(xml, char) +def place_text_block(xml: XMLMaker, text_block: LTCharBlock): + attr = { + "x0": text_block.x0, + "x1": text_block.x1, + "y0": text_block.y0, + "y1": text_block.y1, + } + with xml.elm("text-block", attr): + for char_block in text_block: + place_char_block(xml, char_block) + def place_curve(xml: XMLMaker, item: LTCurve): attr = { "x0": item.x0, diff --git a/pdfmajor/interpreter/PageInterpreter.py b/pdfmajor/interpreter/PageInterpreter.py index 49af4fc..af87c4d 100644 --- a/pdfmajor/interpreter/PageInterpreter.py +++ b/pdfmajor/interpreter/PageInterpreter.py @@ -1,13 +1,8 @@ from pdfmajor.parser.PDFPage import PDFPage from pdfmajor.parser.PDFStream import list_value -from pdfmajor.parser.PDFContentParser import PDFContentParser -from pdfmajor.parser.PSStackParser import PSKeyword -from pdfmajor.parser.PSStackParser import keyword_name -from .commands import PDFStateStack, PDFCommands -from .commands import LTXObject -from .utils import init_resources -from .XObjectInterpreter import XObjectInterpreter +from .commands import PDFStateStack +from .commands import process_command_stream, prep_state class PageInterpreter: class PDFInterpreterError(Exception): pass @@ -31,52 +26,21 @@ class PageInterpreter: self.width = x1-x0 # Init State - self.state: PDFStateStack = PDFStateStack() - self.state.t_matrix = ctm - self.state.resources = page.resources - # set some global states. - self.state.graphics.ncolspace = self.state.graphics.scolspace = None - if self.state.colorspace_map: - col_space = next(iter(self.state.colorspace_map.values())) - self.state.graphics.ncolspace = self.state.graphics.scolspace = col_space - # Init Resources - init_resources(self.state, self.font_cache) + self.state: PDFStateStack = prep_state( + PDFStateStack(), + ctm=ctm, + resources=page.resources, + font_cache=self.font_cache + ) + def __iter__(self): - parser = PDFContentParser(list_value(self.page.contents)) - history = [] - for obj in parser: - history.append(obj) - if isinstance(obj, PSKeyword): - name = keyword_name(obj) - method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q') - if method in PDFCommands.commands.keys(): - func = PDFCommands.commands.get(method) - nargs = func.__code__.co_argcount-1 - if nargs: - args = self.state.pop(nargs) - if len(args) == nargs: - func(self.state, *args) - else: - raise self.PDFInterpreterError(f"Invalid Number of Args provided {method}") - else: - # log.debug('exec: %s', name) - func(self.state) - else: - raise self.PDFInterpreterError('Unknown operator: %r' % name) - else: - self.state.argstack.append(obj) - - for complete_item in self.state.complete_layout_items: - if isinstance(complete_item, LTXObject): - interpeter = XObjectInterpreter( - streams=[complete_item.stream], - resources=complete_item.resources, - ctm=complete_item.t_matrix - ) - for item in interpeter: - complete_item.add(item) - yield complete_item - self.state.complete_layout_items = [] + for item in process_command_stream( + streams=list_value(self.page.contents), + font_cache=self.font_cache, + state=self.state + ): + yield item + def __repr__(self) -> str: - return f"" \ No newline at end of file + return f"" \ No newline at end of file diff --git a/pdfmajor/interpreter/XObjectInterpreter.py b/pdfmajor/interpreter/XObjectInterpreter.py deleted file mode 100644 index bdd0fd1..0000000 --- a/pdfmajor/interpreter/XObjectInterpreter.py +++ /dev/null @@ -1,70 +0,0 @@ -from pdfmajor.utils import MATRIX_IDENTITY -from pdfmajor.parser.PSStackParser import literal_name -from pdfmajor.parser.PDFParser import PDFStream -from pdfmajor.parser.PSStackParser import PSEOF -from pdfmajor.parser.PSStackParser import PSKeyword -from pdfmajor.parser.PSStackParser import keyword_name -from pdfmajor.parser.PSStackParser import PSStackParser -from pdfmajor.parser.constants import LITERAL_PDF, LITERAL_TEXT, LITERAL_FONT - -from pdfmajor.parser.PDFStream import resolve1, list_value, dict_value -from pdfmajor.parser.PDFStream.PDFObjRef import PDFObjRef -from pdfmajor.parser.PDFContentParser import PDFContentParser - -from .commands.state import PDFStateStack, PDFColorSpace, PREDEFINED_COLORSPACE -from .commands import PDFCommands, LTXObject -from .utils import init_resources - -class XObjectInterpreter: - - class PDFInterpreterError(Exception): pass - - def __init__(self, streams, resources, ctm): - self.streams = streams - self.font_cache = {} - - self.state = PDFStateStack() - self.state.t_matrix = ctm - self.state.resources = resources - - # set some global states. - self.state.graphics.ncolspace = self.state.graphics.scolspace = None - if self.state.colorspace_map: - col_space = next(iter(self.state.colorspace_map.values())) - self.state.graphics.ncolspace = self.state.graphics.scolspace = col_space - init_resources(self.state, self.font_cache) - - def __iter__(self): - parser = PDFContentParser(self.streams) - for obj in parser: - if isinstance(obj, PSKeyword): - name = keyword_name(obj) - method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q') - if method in PDFCommands.commands.keys(): - func = PDFCommands.commands.get(method) - nargs = func.__code__.co_argcount-1 - if nargs: - args = self.state.pop(nargs) - if len(args) == nargs: - func(self.state, *args) - else: - raise self.PDFInterpreterError(f"Invalid Number of Args provided {method}") - else: - # log.debug('exec: %s', name) - func(self.state) - else: - raise self.PDFInterpreterError('Unknown operator: %r' % name) - else: - self.state.argstack.append(obj) - - for complete_item in self.state.complete_layout_items: - if isinstance(complete_item, LTXObject): - interpeter = self.__class__( - streams=[complete_item.stream], - resources=complete_item.resources, - ctm=complete_item.t_matrix - ) - for item in interpeter: - complete_item.add(item) - yield complete_item - self.state.complete_layout_items = [] \ No newline at end of file diff --git a/pdfmajor/interpreter/__init__.py b/pdfmajor/interpreter/__init__.py index c2db39e..af80051 100644 --- a/pdfmajor/interpreter/__init__.py +++ b/pdfmajor/interpreter/__init__.py @@ -6,7 +6,7 @@ from ..parser.PDFPage import PDFPage from ..utils import set_log_level, get_logger, logging from .commands.state import PDFStateStack, PDFColorSpace, PREDEFINED_COLORSPACE from .commands import PDFCommands -from .commands import LTCharBlock, LTChar +from .commands import LTTextBlock, LTCharBlock, LTChar from .commands import LTCurve, LTLine, LTHorizontalLine, LTVerticalLine, LTRect from .commands import LTImage from .commands import LTXObject diff --git a/pdfmajor/interpreter/commands/__init__.py b/pdfmajor/interpreter/commands/__init__.py index c132cff..b9fe649 100644 --- a/pdfmajor/interpreter/commands/__init__.py +++ b/pdfmajor/interpreter/commands/__init__.py @@ -1,9 +1,75 @@ +from typing import List + +from pdfmajor.parser.PDFContentParser import PDFContentParser, PDFStream +from pdfmajor.parser.PSStackParser import PSKeyword, keyword_name +from pdfmajor.utils import get_logger + from .commands import PDFCommands from .state import PDFStateStack, PDFGraphicState, PDFTextState from .state.PDFGraphicState import PDFColor, PDFColorSpace, PREDEFINED_COLORSPACE from .state import PDFFont, get_font -from .state import LTCharBlock, LTChar +from .state import LTTextBlock, LTCharBlock, LTChar from .state import LTCurve, LTLine, LTHorizontalLine, LTVerticalLine, LTRect from .state import LTImage from .state import LTXObject -from .state import LTItem, LTComponent, LTContainer \ No newline at end of file +from .state import LTItem, LTComponent, LTContainer +from .utils import init_resources + +log = get_logger('process_command_stream') + +class CommandProcessorError(Exception): pass + +def process_command_stream(streams: List[PDFStream], font_cache: dict = None, state: PDFStateStack = None): + if state is None: + state = PDFStateStack() + if font_cache is None: + font_cache = {} + parser = PDFContentParser(streams) + for obj in parser: + if isinstance(obj, PSKeyword): + name = keyword_name(obj) + method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q') + if method in PDFCommands.commands.keys(): + func = PDFCommands.commands.get(method) + nargs = func.__code__.co_argcount-1 + if nargs: + args = state.pop(nargs) + if len(args) == nargs: + func(state, *args) + else: + raise CommandProcessorError(f"Invalid Number of Args provided {method}") + else: + func(state) + else: + log.warning('Unknown operator: %r (i.e. %r)' % (name, method)) + else: + state.argstack.append(obj) + + for complete_item in state.complete_layout_items: + if isinstance(complete_item, LTXObject): + xobj_state = prep_state( + state=PDFGraphicState(), + ctm=complete_item.t_matrix, + resources=complete_item.resources, + font_cache=font_cache + ) + for item in process_command_stream( + [complete_item.stream], + font_cache=font_cache, + state=xobj_state + ): + complete_item.add(item) + yield complete_item + state.complete_layout_items = [] + +def prep_state(state: PDFStateStack, ctm: tuple, resources: dict, font_cache: dict) -> PDFStateStack: + state.t_matrix = ctm + state.resources = resources + + # set some global states. + state.graphics.ncolspace = state.graphics.scolspace = None + if state.colorspace_map: + col_space = next(iter(state.colorspace_map.values())) + state.graphics.ncolspace = state.graphics.scolspace = col_space + init_resources(state, font_cache) + return state \ No newline at end of file diff --git a/pdfmajor/interpreter/commands/commands.py b/pdfmajor/interpreter/commands/commands.py index 67df488..59d9c3d 100644 --- a/pdfmajor/interpreter/commands/commands.py +++ b/pdfmajor/interpreter/commands/commands.py @@ -1,33 +1,41 @@ -from pdfmajor.utils import mult_matrix, MATRIX_IDENTITY, apply_matrix_pt +from pdfmajor.utils import mult_matrix, MATRIX_IDENTITY, apply_matrix_pt, get_logger from pdfmajor.parser.PSStackParser import literal_name from pdfmajor.parser.PDFStream import PDFStream, list_value, dict_value from pdfmajor.parser.constants import LITERAL_FORM, LITERAL_IMAGE from .PDFCommands import PDFCommands -from .state import PDFStateStack, PDFGraphicState +from .state import PDFStateStack, PDFGraphicState, LTTextBlock from .state.Curves import CurveMethod, CurvePath, CurvePoint # from .state.PDFItem import PDFImage, PDFShape, PDFText, PDFXObject from .state import make_char_block, make_curve, make_image, make_xobject +log = get_logger('commands') + +class CommandError(Exception): pass + # No Support -@PDFCommands.add('W','W_a', 'sh', 'ET', 'BX', 'EX' ,'BI', 'ID', 'gs') -def do_no_support(stack: PDFStateStack) -> PDFStateStack: - return stack +# @PDFCommands.add('W','W_a', 'sh', 'ET', 'BX', 'EX' ,'BI', 'ID', 'gs') +# def do_no_support(stack: PDFStateStack) -> PDFStateStack: +# # log.warning("Unsupported command") +# return stack # marked content operators @PDFCommands.add('MP', 'DP') def do_MP(stack: PDFStateStack) -> PDFStateStack: + log.warning("Unsupported command - do_tag") # self.device.do_tag(tag) return stack @PDFCommands.add('BMC', 'BDC') def do_BMC(stack: PDFStateStack) -> PDFStateStack: + log.warning("Unsupported command - begin_tag") # self.device.begin_tag(tag) return stack @PDFCommands.add('EMC') def do_EMC(stack: PDFStateStack) -> PDFStateStack: # self.device.end_tag(tag) + log.warning("Unsupported command - end_tag") return stack @PDFCommands.add('q') @@ -332,6 +340,16 @@ def do_scn(stack: PDFStateStack) -> PDFStateStack: def do_BT(stack: PDFStateStack) -> PDFStateStack: stack.text.matrix = MATRIX_IDENTITY stack.text.linematrix = [0, 0] + if stack.current_textblock is None: + stack.current_textblock = LTTextBlock() + else: + raise PDFCommands.InvalidOperation("Last TextBlock did not clear") + return stack + +@PDFCommands.add('ET') +def do_ET(stack: PDFStateStack) -> PDFStateStack: + stack.complete_layout_items.append(stack.current_textblock) + stack.current_textblock = None return stack # setcharspace @@ -417,12 +435,14 @@ def do_T_a(stack: PDFStateStack) -> PDFStateStack: def do_TJ(stack: PDFStateStack, seq: bytearray) -> PDFStateStack: if stack.text.font is None: raise PDFCommands.InvalidOperation("No Font Specified") - stack.complete_layout_items.append(make_char_block( + if stack.current_textblock is None: + raise PDFCommands.InvalidOperation("No TextBlock initilized") + stack.current_textblock.add_char_block( seq, stack.t_matrix, stack.text.copy(), stack.graphics.ncolor.copy() - )) + ) return stack # show diff --git a/pdfmajor/interpreter/commands/state/__init__.py b/pdfmajor/interpreter/commands/state/__init__.py index d6a3a3b..7f4ec25 100644 --- a/pdfmajor/interpreter/commands/state/__init__.py +++ b/pdfmajor/interpreter/commands/state/__init__.py @@ -10,7 +10,7 @@ from .PDFTextState import PDFTextState, PDFFont, get_font from .Curves import CurvePath from .layout import LTItem, LTComponent, LTContainer -from .layout import LTCharBlock, LTChar +from .layout import LTTextBlock, LTCharBlock, LTChar from .layout import LTCurve, LTLine, LTHorizontalLine, LTVerticalLine, LTRect from .layout import LTXObject from .layout import LTImage @@ -30,6 +30,7 @@ class PDFStateStack: self.fontmap = {} self.xobjmap = {} self.resources = {} + self.current_textblock: LTTextBlock = None def pop(self, n: int) -> bytearray: if n == 0: diff --git a/pdfmajor/interpreter/commands/state/layout/LTTextBlock.py b/pdfmajor/interpreter/commands/state/layout/LTTextBlock.py new file mode 100644 index 0000000..517c8b7 --- /dev/null +++ b/pdfmajor/interpreter/commands/state/layout/LTTextBlock.py @@ -0,0 +1,40 @@ +from typing import List + +from pdfmajor.utils import apply_matrix_norm, apply_matrix_pt, matrix2str, Bbox, Point + +from ..PDFGraphicState import PDFColor +from ..PDFTextState import PDFFont +from ..PDFTextState import PDFTextState + +from ._base import LTContainer +from .LTCharBlock import LTCharBlock +from .utils.textdecoder import decode_text_seq + + +## LTTextBlock +## +class LTTextBlock(LTContainer): + def __init__(self, x0=None, y0=None, x1=None, y1=None): + LTContainer.__init__(self, + Bbox(x0, y0, x1, y1), + [] + ) + + def add_char_block(self, seq: bytearray, ctm: tuple, textstate: PDFTextState, color: PDFColor): + (char_meta_datas, textstate, color) = decode_text_seq(seq, ctm, textstate, color) + char_block = LTCharBlock( + chars=char_meta_datas, + color=color, + textstate=textstate + ) + if None in [self.bbox.x0, self.bbox.x1, self.bbox.y0, self.bbox.y1]: + self.bbox.x0 = char_block.x0 + self.bbox.x1 = char_block.x1 + self.bbox.y0 = char_block.y0 + self.bbox.y1 = char_block.y1 + else: + self.bbox.x0 = min([self.bbox.x0, char_block.x0]) + self.bbox.x1 = max([self.bbox.x1, char_block.x1]) + self.bbox.y0 = min([self.bbox.y0, char_block.y0]) + self.bbox.y1 = max([self.bbox.y1, char_block.y1]) + self.add(char_block) \ No newline at end of file diff --git a/pdfmajor/interpreter/commands/state/layout/__init__.py b/pdfmajor/interpreter/commands/state/layout/__init__.py index f26d3dd..624ce81 100644 --- a/pdfmajor/interpreter/commands/state/layout/__init__.py +++ b/pdfmajor/interpreter/commands/state/layout/__init__.py @@ -6,6 +6,7 @@ from ..PDFGraphicState import PDFGraphicState, PDFColor from ..PDFTextState import PDFTextState from ._base import LTItem, LTComponent, LTContainer +from .LTTextBlock import LTTextBlock from .LTCharBlock import LTCharBlock, LTChar from .LTCurves import LTCurve, LTLine, LTHorizontalLine, LTRect, LTVerticalLine from .LTImage import LTImage diff --git a/pdfmajor/interpreter/utils.py b/pdfmajor/interpreter/commands/utils.py similarity index 94% rename from pdfmajor/interpreter/utils.py rename to pdfmajor/interpreter/commands/utils.py index 6553ae4..1d7fce2 100644 --- a/pdfmajor/interpreter/utils.py +++ b/pdfmajor/interpreter/commands/utils.py @@ -4,8 +4,8 @@ from pdfmajor.parser.PDFStream.PDFObjRef import PDFObjRef from pdfmajor.parser.PSStackParser import literal_name from pdfmajor.parser.constants import LITERAL_PDF, LITERAL_TEXT -from .commands import PDFColorSpace, PREDEFINED_COLORSPACE -from .commands import PDFStateStack, get_font +from .state import PDFColorSpace, PREDEFINED_COLORSPACE +from .state import PDFStateStack, get_font def get_colorspace(spec): if isinstance(spec, list):