added support for ignore bad char flag

- updated unit tests to include
- using the textstate.to_unichr abstraction
This commit is contained in:
ari committed 2019-12-08 14:44:46 -05:00
1 parent 841576d9b8
commit a29d465ea4
9 files changed
+53 -11

No files matched your search

+4 -3
View File
@@ -6,7 +6,7 @@ from .commands import process_command_stream, prep_state
class PageInterpreter:
def __init__(self, page: PDFPage, page_num: int, font_cache: dict = {}):
def __init__(self, page: PDFPage, page_num: int, font_cache: dict = None, ignore_bad_chars = False):
(x0, y0, x1, y1) = page.mediabox
if page.rotate == 90:
ctm = [0, -1, 1, 0, -y0, x1]
@@ -18,7 +18,7 @@ class PageInterpreter:
ctm = [1, 0, 0, 1, -x0, -y0]
# Locals
self.font_cache = font_cache
self.font_cache = font_cache if font_cache is not None else {}
self.page = page
self.page_num = page_num
self.height = y1-y0
@@ -29,7 +29,8 @@ class PageInterpreter:
PDFStateStack(),
ctm=ctm,
resources=page.resources,
font_cache=self.font_cache
font_cache=self.font_cache,
ignore_bad_chars=ignore_bad_chars
)
+3 -1
View File
@@ -23,6 +23,7 @@ class PDFInterpreter:
password: str = None,
pagenos: List[int] = None,
preload: bool = False,
ignore_bad_chars: bool = False,
debug_level: int = logging.WARNING,
):
self.input_file_path = input_file_path
@@ -31,6 +32,7 @@ class PDFInterpreter:
self.check_extractable = check_extractable
self.password = password
self.pagenos = pagenos
self.ignore_bad_chars = ignore_bad_chars
self.debug_level = debug_level
self.__pages = []
if preload:
@@ -52,7 +54,7 @@ class PDFInterpreter:
check_extractable=self.check_extractable
)
for page_num, page in enumerate(pages):
self.__pages.append(PageInterpreter(page, page_num, font_cache))
self.__pages.append(PageInterpreter(page, page_num, font_cache, ignore_bad_chars=self.ignore_bad_chars))
yield self.__pages[-1]
log.info(f"Done Reading {len(self.__pages)} pages.")
set_log_level(logging.WARNING)
+2 -1
View File
@@ -71,9 +71,10 @@ def process_command_stream(streams: List[PDFStream], font_cache: dict = None, st
yield complete_item
state.complete_layout_items = []
def prep_state(state: PDFStateStack, ctm: tuple, resources: dict, font_cache: dict) -> PDFStateStack:
def prep_state(state: PDFStateStack, ctm: tuple, resources: dict, font_cache: dict, ignore_bad_chars: bool = False) -> PDFStateStack:
state.t_matrix = ctm
state.resources = resources
state.text.ignore_bad_chars = ignore_bad_chars
# set some global states.
state.graphics.ncolspace = state.graphics.scolspace = None
@@ -58,7 +58,7 @@ class PDFText(PDFItem):
if needcharspace:
textstate.linematrix[idx] += textstate.charspace
text = textstate.font.to_unichr(cid)
text = textstate.to_unichr(cid)
assert isinstance(text, str), str(type(text))
matrix = translate_matrix(textstate.matrix, textstate.linematrix)
@@ -42,7 +42,7 @@ class TextDecoder:
if needcharspace:
textstate.linematrix[idx] += textstate.charspace
text = textstate.font.to_unichr(cid)
text = textstate.to_unichr(cid)
assert isinstance(text, str), str(type(text))
matrix = translate_matrix(textstate.matrix, textstate.linematrix)
@@ -1,5 +1,6 @@
from .PDFFont import PDFFont, get_font
from pdfmajor.utils import MATRIX_IDENTITY
from .PDFFont import PDFFont, get_font
from .PDFFont.execptions import PDFUnicodeNotDefined, PDFFontError
## PDFTextState
##
@@ -16,14 +17,27 @@ class PDFTextState(object):
self.rise: int = 0
self.matrix = MATRIX_IDENTITY
self.linematrix = [0, 0]
self.ignore_bad_chars = False
def __repr__(self):
return ('<PDFTextState: font=%r, fontsize=%r, charspace=%r, wordspace=%r, '
' scaling=%r, leading=%r, render=%r, rise=%r, '
' matrix=%r, linematrix=%r>' %
' matrix=%r, linematrix=%r ignore_bad_chars=%r>' %
(self.font, self.fontsize, self.charspace, self.wordspace,
self.scaling, self.leading, self.render, self.rise,
self.matrix, self.linematrix))
self.matrix, self.linematrix, self.ignore_bad_chars))
def to_unichr(self, cid: int) -> str:
if self.font is None:
raise PDFFontError("Missing font")
try:
return self.font.to_unichr(cid)
except PDFUnicodeNotDefined as e:
if self.ignore_bad_chars:
return ""
else:
raise e
def copy(self):
obj = self.__class__()
@@ -37,6 +51,7 @@ class PDFTextState(object):
obj.rise = self.rise
obj.matrix = self.matrix
obj.linematrix = self.linematrix
obj.ignore_bad_chars = self.ignore_bad_chars
return obj
def reset_matrices(self):
@@ -33,7 +33,7 @@ def __render_string_along(idx: int, seq: bytearray,
if needcharspace:
textstate.linematrix[idx] += textstate.charspace
text = textstate.font.to_unichr(cid)
text = textstate.to_unichr(cid)
assert isinstance(text, str), str(type(text))
matrix = translate_matrix(textstate.matrix, textstate.linematrix)
Binary file not shown.
+23
View File
@@ -0,0 +1,23 @@
from pathlib import Path
from unittest import TestCase, main
from pdfmajor.interpreter import PDFInterpreter, exceptions
CURRENT_FOLDER = Path("./")
class BadUnicode(TestCase):
def test_ensure_failure(self):
pdf = PDFInterpreter(CURRENT_FOLDER / "tests" / "samples" / "pdf" / "bad-unicode.pdf", ignore_bad_chars=False)
pages = list(pdf)
with self.assertRaises(exceptions.PDFUnicodeNotDefined):
list(pages[2])
def test_ensure_no_failure(self):
pdf = PDFInterpreter(CURRENT_FOLDER / "tests" / "samples" / "pdf" / "bad-unicode.pdf", ignore_bad_chars=True)
pages = list(pdf)
list(pages[2])
if __name__ == '__main__':
# Run Tests
main()