- correct usage of the BT ET tags

This commit is contained in:
ari committed 2018-12-24 01:49:32 -05:00
1 parent c239f79b2c
commit e7701ce2ac
13 files changed
+199 -151

No files matched your search

+8 -4
View File
@@ -1,7 +1,7 @@
from typing import List
from .writers.html import HTMLMaker
from ..interpreter import PDFInterpreter, PageInterpreter, logging
from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject
from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject
from ..interpreter.commands import LTItem
from ..interpreter.commands.state import CurvePath, PDFColor
@@ -76,8 +76,12 @@ def render_page(html: HTMLMaker, ltpage: PageInterpreter):
'max-height': item.height,
}):
html.write(item.get_text())
elif isinstance(item, LTTextBlock):
for char in item:
render(char)
elif isinstance(item, LTCharBlock):
with html.elm('span', { 'class': 'char-block' }, {
with html.elm('span', {'class': 'char-block'}):
with html.elm('p', { 'class': 'text-block' }, {
'position': 'absolute',
'left': f'{item.x0}',
'bottom': f'{item.y0}',
@@ -98,8 +102,8 @@ def render_page(html: HTMLMaker, ltpage: PageInterpreter):
'justify-content': 'space-between',
# 'padding-left': item.font.leading
}):
for char in item:
render(char)
for char in item:
render(char)
with html.elm('div', { "class": 'page', "id": f"page-{ltpage.page_num}" }, {
'width': f'{ltpage.width}px',
+14 -4
View File
@@ -2,7 +2,7 @@ from typing import List
from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray
from ..interpreter import PDFInterpreter, PageInterpreter, logging
from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject
from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTCurve, LTXObject
from ..interpreter.commands import LTItem
from ..interpreter.commands import LTRect, LTLine
from ..interpreter.commands.state import CurvePath, PDFColor
@@ -76,11 +76,21 @@ def render(json: JSONMakerObject, item: LTItem):
json.number("x1", item.x1)
json.number("y0", item.y0)
json.number("y1", item.y1)
elif isinstance(item, LTCharBlock):
place_char_block(json, item)
elif isinstance(item, LTTextBlock):
place_text_block(json, item)
def place_text_block(json: JSONMakerObject, text_block: LTTextBlock):
json.string("type", 'text-block')
json.number("x0", text_block.x0)
json.number("x1", text_block.x1)
json.number("y0", text_block.y0)
json.number("y1", text_block.y1)
with json.array("char_blocks") as cb_arr:
for cb in text_block:
with cb_arr.object() as cb_obj:
place_char_block(cb_obj, cb)
def place_char_block(json: JSONMakerObject, char_block: LTCharBlock):
json.string("type", 'char-block')
json.number("size", char_block.size)
json.number("x0", char_block.x0)
json.number("x1", char_block.x1)
+5 -4
View File
@@ -1,7 +1,7 @@
from typing import List
from ..interpreter import PDFInterpreter, PageInterpreter, logging
from ..interpreter import LTCharBlock, LTXObject
from ..interpreter import LTTextBlock, LTXObject
def convert_to_text(
@@ -28,9 +28,10 @@ def convert_to_text(
with open(output_file_path, 'wb') as outfp:
def process_container(container: LTXObject):
for item in container:
if isinstance(item, LTCharBlock):
for char in item:
outfp.write(char.get_text().encode(codec))
if isinstance(item, LTTextBlock):
for text in item:
for char in text:
outfp.write(char.get_text().encode(codec))
elif isinstance(item, LTXObject):
process_container(item)
for page in intepreter:
+14 -3
View File
@@ -4,7 +4,7 @@ from xml.sax import saxutils
from .writers.xml import XMLMaker
from ..interpreter import PDFInterpreter, PageInterpreter, logging
from ..interpreter import LTImage, LTCharBlock, LTChar, LTCurve, LTXObject
from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject
from ..interpreter.commands import LTItem
from ..interpreter.commands import LTRect, LTLine
from ..interpreter.commands.state import CurvePath, PDFColor
@@ -86,8 +86,8 @@ def render_page(xml: XMLMaker, ltpage: PageInterpreter):
"y0": item.y0,
"y1": item.y1,
})
elif isinstance(item, LTCharBlock):
place_char_block(xml, item)
elif isinstance(item, LTTextBlock):
place_text_block(xml, item)
with xml.elm('page', {
"id": ltpage.page_num,
"width": ltpage.width,
@@ -121,6 +121,17 @@ def place_char_block(xml: XMLMaker, char_block: LTCharBlock):
for char in char_block:
place_char(xml, char)
def place_text_block(xml: XMLMaker, text_block: LTCharBlock):
attr = {
"x0": text_block.x0,
"x1": text_block.x1,
"y0": text_block.y0,
"y1": text_block.y1,
}
with xml.elm("text-block", attr):
for char_block in text_block:
place_char_block(xml, char_block)
def place_curve(xml: XMLMaker, item: LTCurve):
attr = {
"x0": item.x0,
+17 -53
View File
@@ -1,13 +1,8 @@
from pdfmajor.parser.PDFPage import PDFPage
from pdfmajor.parser.PDFStream import list_value
from pdfmajor.parser.PDFContentParser import PDFContentParser
from pdfmajor.parser.PSStackParser import PSKeyword
from pdfmajor.parser.PSStackParser import keyword_name
from .commands import PDFStateStack, PDFCommands
from .commands import LTXObject
from .utils import init_resources
from .XObjectInterpreter import XObjectInterpreter
from .commands import PDFStateStack
from .commands import process_command_stream, prep_state
class PageInterpreter:
class PDFInterpreterError(Exception): pass
@@ -31,52 +26,21 @@ class PageInterpreter:
self.width = x1-x0
# Init State
self.state: PDFStateStack = PDFStateStack()
self.state.t_matrix = ctm
self.state.resources = page.resources
# set some global states.
self.state.graphics.ncolspace = self.state.graphics.scolspace = None
if self.state.colorspace_map:
col_space = next(iter(self.state.colorspace_map.values()))
self.state.graphics.ncolspace = self.state.graphics.scolspace = col_space
# Init Resources
init_resources(self.state, self.font_cache)
self.state: PDFStateStack = prep_state(
PDFStateStack(),
ctm=ctm,
resources=page.resources,
font_cache=self.font_cache
)
def __iter__(self):
parser = PDFContentParser(list_value(self.page.contents))
history = []
for obj in parser:
history.append(obj)
if isinstance(obj, PSKeyword):
name = keyword_name(obj)
method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q')
if method in PDFCommands.commands.keys():
func = PDFCommands.commands.get(method)
nargs = func.__code__.co_argcount-1
if nargs:
args = self.state.pop(nargs)
if len(args) == nargs:
func(self.state, *args)
else:
raise self.PDFInterpreterError(f"Invalid Number of Args provided {method}")
else:
# log.debug('exec: %s', name)
func(self.state)
else:
raise self.PDFInterpreterError('Unknown operator: %r' % name)
else:
self.state.argstack.append(obj)
for complete_item in self.state.complete_layout_items:
if isinstance(complete_item, LTXObject):
interpeter = XObjectInterpreter(
streams=[complete_item.stream],
resources=complete_item.resources,
ctm=complete_item.t_matrix
)
for item in interpeter:
complete_item.add(item)
yield complete_item
self.state.complete_layout_items = []
for item in process_command_stream(
streams=list_value(self.page.contents),
font_cache=self.font_cache,
state=self.state
):
yield item
def __repr__(self) -> str:
return f"<Page:{self.page_num}/>"
return f"<Page:{self.page_num} width={self.width} height={self.height}/>"
@@ -1,70 +0,0 @@
from pdfmajor.utils import MATRIX_IDENTITY
from pdfmajor.parser.PSStackParser import literal_name
from pdfmajor.parser.PDFParser import PDFStream
from pdfmajor.parser.PSStackParser import PSEOF
from pdfmajor.parser.PSStackParser import PSKeyword
from pdfmajor.parser.PSStackParser import keyword_name
from pdfmajor.parser.PSStackParser import PSStackParser
from pdfmajor.parser.constants import LITERAL_PDF, LITERAL_TEXT, LITERAL_FONT
from pdfmajor.parser.PDFStream import resolve1, list_value, dict_value
from pdfmajor.parser.PDFStream.PDFObjRef import PDFObjRef
from pdfmajor.parser.PDFContentParser import PDFContentParser
from .commands.state import PDFStateStack, PDFColorSpace, PREDEFINED_COLORSPACE
from .commands import PDFCommands, LTXObject
from .utils import init_resources
class XObjectInterpreter:
class PDFInterpreterError(Exception): pass
def __init__(self, streams, resources, ctm):
self.streams = streams
self.font_cache = {}
self.state = PDFStateStack()
self.state.t_matrix = ctm
self.state.resources = resources
# set some global states.
self.state.graphics.ncolspace = self.state.graphics.scolspace = None
if self.state.colorspace_map:
col_space = next(iter(self.state.colorspace_map.values()))
self.state.graphics.ncolspace = self.state.graphics.scolspace = col_space
init_resources(self.state, self.font_cache)
def __iter__(self):
parser = PDFContentParser(self.streams)
for obj in parser:
if isinstance(obj, PSKeyword):
name = keyword_name(obj)
method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q')
if method in PDFCommands.commands.keys():
func = PDFCommands.commands.get(method)
nargs = func.__code__.co_argcount-1
if nargs:
args = self.state.pop(nargs)
if len(args) == nargs:
func(self.state, *args)
else:
raise self.PDFInterpreterError(f"Invalid Number of Args provided {method}")
else:
# log.debug('exec: %s', name)
func(self.state)
else:
raise self.PDFInterpreterError('Unknown operator: %r' % name)
else:
self.state.argstack.append(obj)
for complete_item in self.state.complete_layout_items:
if isinstance(complete_item, LTXObject):
interpeter = self.__class__(
streams=[complete_item.stream],
resources=complete_item.resources,
ctm=complete_item.t_matrix
)
for item in interpeter:
complete_item.add(item)
yield complete_item
self.state.complete_layout_items = []
+1 -1
View File
@@ -6,7 +6,7 @@ from ..parser.PDFPage import PDFPage
from ..utils import set_log_level, get_logger, logging
from .commands.state import PDFStateStack, PDFColorSpace, PREDEFINED_COLORSPACE
from .commands import PDFCommands
from .commands import LTCharBlock, LTChar
from .commands import LTTextBlock, LTCharBlock, LTChar
from .commands import LTCurve, LTLine, LTHorizontalLine, LTVerticalLine, LTRect
from .commands import LTImage
from .commands import LTXObject
+68 -2
View File
@@ -1,9 +1,75 @@
from typing import List
from pdfmajor.parser.PDFContentParser import PDFContentParser, PDFStream
from pdfmajor.parser.PSStackParser import PSKeyword, keyword_name
from pdfmajor.utils import get_logger
from .commands import PDFCommands
from .state import PDFStateStack, PDFGraphicState, PDFTextState
from .state.PDFGraphicState import PDFColor, PDFColorSpace, PREDEFINED_COLORSPACE
from .state import PDFFont, get_font
from .state import LTCharBlock, LTChar
from .state import LTTextBlock, LTCharBlock, LTChar
from .state import LTCurve, LTLine, LTHorizontalLine, LTVerticalLine, LTRect
from .state import LTImage
from .state import LTXObject
from .state import LTItem, LTComponent, LTContainer
from .state import LTItem, LTComponent, LTContainer
from .utils import init_resources
log = get_logger('process_command_stream')
class CommandProcessorError(Exception): pass
def process_command_stream(streams: List[PDFStream], font_cache: dict = None, state: PDFStateStack = None):
if state is None:
state = PDFStateStack()
if font_cache is None:
font_cache = {}
parser = PDFContentParser(streams)
for obj in parser:
if isinstance(obj, PSKeyword):
name = keyword_name(obj)
method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q')
if method in PDFCommands.commands.keys():
func = PDFCommands.commands.get(method)
nargs = func.__code__.co_argcount-1
if nargs:
args = state.pop(nargs)
if len(args) == nargs:
func(state, *args)
else:
raise CommandProcessorError(f"Invalid Number of Args provided {method}")
else:
func(state)
else:
log.warning('Unknown operator: %r (i.e. %r)' % (name, method))
else:
state.argstack.append(obj)
for complete_item in state.complete_layout_items:
if isinstance(complete_item, LTXObject):
xobj_state = prep_state(
state=PDFGraphicState(),
ctm=complete_item.t_matrix,
resources=complete_item.resources,
font_cache=font_cache
)
for item in process_command_stream(
[complete_item.stream],
font_cache=font_cache,
state=xobj_state
):
complete_item.add(item)
yield complete_item
state.complete_layout_items = []
def prep_state(state: PDFStateStack, ctm: tuple, resources: dict, font_cache: dict) -> PDFStateStack:
state.t_matrix = ctm
state.resources = resources
# set some global states.
state.graphics.ncolspace = state.graphics.scolspace = None
if state.colorspace_map:
col_space = next(iter(state.colorspace_map.values()))
state.graphics.ncolspace = state.graphics.scolspace = col_space
init_resources(state, font_cache)
return state
+27 -7
View File
@@ -1,33 +1,41 @@
from pdfmajor.utils import mult_matrix, MATRIX_IDENTITY, apply_matrix_pt
from pdfmajor.utils import mult_matrix, MATRIX_IDENTITY, apply_matrix_pt, get_logger
from pdfmajor.parser.PSStackParser import literal_name
from pdfmajor.parser.PDFStream import PDFStream, list_value, dict_value
from pdfmajor.parser.constants import LITERAL_FORM, LITERAL_IMAGE
from .PDFCommands import PDFCommands
from .state import PDFStateStack, PDFGraphicState
from .state import PDFStateStack, PDFGraphicState, LTTextBlock
from .state.Curves import CurveMethod, CurvePath, CurvePoint
# from .state.PDFItem import PDFImage, PDFShape, PDFText, PDFXObject
from .state import make_char_block, make_curve, make_image, make_xobject
log = get_logger('commands')
class CommandError(Exception): pass
# No Support
@PDFCommands.add('W','W_a', 'sh', 'ET', 'BX', 'EX' ,'BI', 'ID', 'gs')
def do_no_support(stack: PDFStateStack) -> PDFStateStack:
return stack
# @PDFCommands.add('W','W_a', 'sh', 'ET', 'BX', 'EX' ,'BI', 'ID', 'gs')
# def do_no_support(stack: PDFStateStack) -> PDFStateStack:
# # log.warning("Unsupported command")
# return stack
# marked content operators
@PDFCommands.add('MP', 'DP')
def do_MP(stack: PDFStateStack) -> PDFStateStack:
log.warning("Unsupported command - do_tag")
# self.device.do_tag(tag)
return stack
@PDFCommands.add('BMC', 'BDC')
def do_BMC(stack: PDFStateStack) -> PDFStateStack:
log.warning("Unsupported command - begin_tag")
# self.device.begin_tag(tag)
return stack
@PDFCommands.add('EMC')
def do_EMC(stack: PDFStateStack) -> PDFStateStack:
# self.device.end_tag(tag)
log.warning("Unsupported command - end_tag")
return stack
@PDFCommands.add('q')
@@ -332,6 +340,16 @@ def do_scn(stack: PDFStateStack) -> PDFStateStack:
def do_BT(stack: PDFStateStack) -> PDFStateStack:
stack.text.matrix = MATRIX_IDENTITY
stack.text.linematrix = [0, 0]
if stack.current_textblock is None:
stack.current_textblock = LTTextBlock()
else:
raise PDFCommands.InvalidOperation("Last TextBlock did not clear")
return stack
@PDFCommands.add('ET')
def do_ET(stack: PDFStateStack) -> PDFStateStack:
stack.complete_layout_items.append(stack.current_textblock)
stack.current_textblock = None
return stack
# setcharspace
@@ -417,12 +435,14 @@ def do_T_a(stack: PDFStateStack) -> PDFStateStack:
def do_TJ(stack: PDFStateStack, seq: bytearray) -> PDFStateStack:
if stack.text.font is None:
raise PDFCommands.InvalidOperation("No Font Specified")
stack.complete_layout_items.append(make_char_block(
if stack.current_textblock is None:
raise PDFCommands.InvalidOperation("No TextBlock initilized")
stack.current_textblock.add_char_block(
seq,
stack.t_matrix,
stack.text.copy(),
stack.graphics.ncolor.copy()
))
)
return stack
# show
@@ -10,7 +10,7 @@ from .PDFTextState import PDFTextState, PDFFont, get_font
from .Curves import CurvePath
from .layout import LTItem, LTComponent, LTContainer
from .layout import LTCharBlock, LTChar
from .layout import LTTextBlock, LTCharBlock, LTChar
from .layout import LTCurve, LTLine, LTHorizontalLine, LTVerticalLine, LTRect
from .layout import LTXObject
from .layout import LTImage
@@ -30,6 +30,7 @@ class PDFStateStack:
self.fontmap = {}
self.xobjmap = {}
self.resources = {}
self.current_textblock: LTTextBlock = None
def pop(self, n: int) -> bytearray:
if n == 0:
@@ -0,0 +1,40 @@
from typing import List
from pdfmajor.utils import apply_matrix_norm, apply_matrix_pt, matrix2str, Bbox, Point
from ..PDFGraphicState import PDFColor
from ..PDFTextState import PDFFont
from ..PDFTextState import PDFTextState
from ._base import LTContainer
from .LTCharBlock import LTCharBlock
from .utils.textdecoder import decode_text_seq
## LTTextBlock
##
class LTTextBlock(LTContainer):
def __init__(self, x0=None, y0=None, x1=None, y1=None):
LTContainer.__init__(self,
Bbox(x0, y0, x1, y1),
[]
)
def add_char_block(self, seq: bytearray, ctm: tuple, textstate: PDFTextState, color: PDFColor):
(char_meta_datas, textstate, color) = decode_text_seq(seq, ctm, textstate, color)
char_block = LTCharBlock(
chars=char_meta_datas,
color=color,
textstate=textstate
)
if None in [self.bbox.x0, self.bbox.x1, self.bbox.y0, self.bbox.y1]:
self.bbox.x0 = char_block.x0
self.bbox.x1 = char_block.x1
self.bbox.y0 = char_block.y0
self.bbox.y1 = char_block.y1
else:
self.bbox.x0 = min([self.bbox.x0, char_block.x0])
self.bbox.x1 = max([self.bbox.x1, char_block.x1])
self.bbox.y0 = min([self.bbox.y0, char_block.y0])
self.bbox.y1 = max([self.bbox.y1, char_block.y1])
self.add(char_block)
@@ -6,6 +6,7 @@ from ..PDFGraphicState import PDFGraphicState, PDFColor
from ..PDFTextState import PDFTextState
from ._base import LTItem, LTComponent, LTContainer
from .LTTextBlock import LTTextBlock
from .LTCharBlock import LTCharBlock, LTChar
from .LTCurves import LTCurve, LTLine, LTHorizontalLine, LTRect, LTVerticalLine
from .LTImage import LTImage
@@ -4,8 +4,8 @@ from pdfmajor.parser.PDFStream.PDFObjRef import PDFObjRef
from pdfmajor.parser.PSStackParser import literal_name
from pdfmajor.parser.constants import LITERAL_PDF, LITERAL_TEXT
from .commands import PDFColorSpace, PREDEFINED_COLORSPACE
from .commands import PDFStateStack, get_font
from .state import PDFColorSpace, PREDEFINED_COLORSPACE
from .state import PDFStateStack, get_font
def get_colorspace(spec):
if isinstance(spec, list):