- added yaml converter
This commit is contained in:
1 parent
b7026793d7
commit
f9f9a59793
4 files changed
+367
-1
No files matched your search
@@ -8,6 +8,7 @@ from .html import convert_to_html
|
||||
from .xml import convert_to_xml
|
||||
from .json import convert_to_json
|
||||
from .text import convert_to_text
|
||||
from .yaml import convert_to_yaml
|
||||
from ..utils import logging
|
||||
|
||||
def convert_file(
|
||||
@@ -75,6 +76,20 @@ def convert_file(
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
elif out_type == 'yaml' or out_type == 'yml':
|
||||
return convert_to_yaml(
|
||||
input_file_path=input_file,
|
||||
output_file_path=output_file,
|
||||
image_folder_path=image_folder_path,
|
||||
codec=codec,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
elif out_type == 'text':
|
||||
return convert_to_text(
|
||||
input_file_path=input_file,
|
||||
@@ -89,4 +104,4 @@ def convert_file(
|
||||
dont_export_images=dont_export_images,
|
||||
debug_level=debug_level,
|
||||
)
|
||||
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text'")
|
||||
else: raise Exception("Please specify out_type as 'html' or 'xml' or 'json' or 'text' or 'yaml'")
|
||||
@@ -0,0 +1,117 @@
|
||||
from json import dumps as json_dump
|
||||
from io import TextIOWrapper
|
||||
from contextlib import contextmanager
|
||||
|
||||
@contextmanager
|
||||
def YAMLMaker(file_path: str, codec: str = 'utf-8'):
|
||||
with open(file_path, 'wb') as outf:
|
||||
with YAMLMakerObject(outf, codec=codec) as obj:
|
||||
yield obj
|
||||
|
||||
class YAMLMakerWriter:
|
||||
class AccessError(Exception): pass
|
||||
def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = -1):
|
||||
self.outfile: TextIOWrapper = outfile
|
||||
self.codec = codec
|
||||
self.levels_deep = levels_deep
|
||||
|
||||
def __enter__(self):
|
||||
if self.levels_deep > 0:
|
||||
self.__write_raw('')
|
||||
self.levels_deep += 1
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.levels_deep -= 1
|
||||
|
||||
def __write_raw(self, text):
|
||||
if self.outfile is not None:
|
||||
if self.codec:
|
||||
text = text.encode(self.codec)
|
||||
self.outfile.write(text)
|
||||
else:
|
||||
raise self.AccessError("Attempting to write to file without an open connection")
|
||||
|
||||
def _write(self, text: str, append_deep: bool = True, deep_space: str = ' ', prefix:str = ''):
|
||||
if not append_deep:
|
||||
deep_space = ""
|
||||
text = prefix + (deep_space*self.levels_deep) + text
|
||||
self.__write_raw(text)
|
||||
|
||||
class YAMLMakerObject(YAMLMakerWriter):
|
||||
def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = -1, parent_is_array: bool = False):
|
||||
YAMLMakerWriter.__init__(self, outfile, codec, levels_deep)
|
||||
self.keys = set()
|
||||
self.parent_is_array = parent_is_array
|
||||
|
||||
def write(self, key: str, value: str):
|
||||
if key not in self.keys:
|
||||
self.keys.add(key)
|
||||
if len(self.keys) > 1:
|
||||
self._write(f'{key}: {value}', prefix='\n')
|
||||
else:
|
||||
self._write(f'{key}: {value}', prefix='',
|
||||
append_deep=not self.parent_is_array
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def object(self, key: str):
|
||||
if key not in self.keys:
|
||||
self.keys.add(key)
|
||||
if len(self.keys) > 1:
|
||||
self._write(f'{key}:\n', prefix='\n')
|
||||
else:
|
||||
self._write(f'{key}:\n', prefix='', append_deep=False)
|
||||
with YAMLMakerObject(self.outfile, codec=self.codec, levels_deep=self.levels_deep) as obj:
|
||||
yield obj
|
||||
|
||||
@contextmanager
|
||||
def array(self, key: str):
|
||||
if key not in self.keys:
|
||||
self.keys.add(key)
|
||||
if len(self.keys) > 1:
|
||||
self._write(f'{key}:\n', prefix='\n')
|
||||
else:
|
||||
self._write(f'{key}:\n', prefix='')
|
||||
with YAMLMakerArray(self.outfile, codec=self.codec, levels_deep=self.levels_deep) as obj:
|
||||
yield obj
|
||||
|
||||
def place_object(self, key: str, obj: dict):
|
||||
with self.object(key) as child:
|
||||
for key, val in obj.items():
|
||||
child.write(key, val)
|
||||
|
||||
def place_array(self, key: str, arr: list):
|
||||
with self.array(key) as child:
|
||||
for val in arr:
|
||||
child.write(val)
|
||||
|
||||
|
||||
class YAMLMakerArray(YAMLMakerWriter):
|
||||
def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = -1):
|
||||
YAMLMakerWriter.__init__(self, outfile, codec, levels_deep)
|
||||
self.childrencount = 0
|
||||
|
||||
def write(self, value: str = ""):
|
||||
self.childrencount += 1
|
||||
if self.childrencount > 1:
|
||||
self._write(f'- {value}', prefix='\n')
|
||||
else:
|
||||
self._write(f'- {value}', prefix='')
|
||||
|
||||
@contextmanager
|
||||
def object(self):
|
||||
self.childrencount += 1
|
||||
if self.childrencount > 1:
|
||||
self._write(f'- ', prefix='\n')
|
||||
else:
|
||||
self._write(f'- ', prefix='')
|
||||
with YAMLMakerObject(self.outfile, codec=self.codec, levels_deep=self.levels_deep, parent_is_array=True) as obj:
|
||||
yield obj
|
||||
|
||||
def place_object(self, obj: dict):
|
||||
with self.object() as child:
|
||||
for key, val in obj.items():
|
||||
child.write(key, val)
|
||||
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
from typing import List
|
||||
|
||||
from .writers.yaml import YAMLMaker, YAMLMakerObject, YAMLMakerArray
|
||||
from ..interpreter import PDFInterpreter, PageInterpreter, logging
|
||||
from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTCurve, LTXObject
|
||||
from ..interpreter.commands import LTItem
|
||||
from ..interpreter.commands import LTRect, LTLine
|
||||
from ..interpreter.commands.state import CurvePath, PDFColor
|
||||
from ..parser.PSStackParser import PSLiteral
|
||||
from ..utils import enc
|
||||
|
||||
def normalize_color_values(values: list):
|
||||
ret = []
|
||||
for v in values:
|
||||
if isinstance(v, PSLiteral):
|
||||
ret.append( enc(str(v.name)).decode() )
|
||||
else:
|
||||
ret.append( enc(str(v)).decode() )
|
||||
return ret
|
||||
|
||||
def convert_to_yaml(
|
||||
input_file_path: str,
|
||||
output_file_path: str,
|
||||
image_folder_path: str = None,
|
||||
dont_export_images: bool = False,
|
||||
codec: str = 'utf-8',
|
||||
maxpages: int = 0,
|
||||
password: str = None,
|
||||
caching: bool = True,
|
||||
check_extractable: bool = True,
|
||||
pagenos: List[int] = None,
|
||||
debug_level: int = logging.WARNING,
|
||||
):
|
||||
intepreter = PDFInterpreter(input_file_path,
|
||||
maxpages=maxpages,
|
||||
password=password,
|
||||
caching=caching,
|
||||
check_extractable=check_extractable,
|
||||
pagenos=pagenos,
|
||||
debug_level=debug_level
|
||||
)
|
||||
with YAMLMaker(output_file_path, codec=codec) as yaml:
|
||||
with yaml.array("pages") as arr:
|
||||
for page in intepreter:
|
||||
with arr.object() as page_obj:
|
||||
render_page(page_obj, page)
|
||||
|
||||
def render_page(yaml: YAMLMakerObject, ltpage: PageInterpreter):
|
||||
yaml.write("id", ltpage.page_num)
|
||||
yaml.write("width", ltpage.width)
|
||||
yaml.write("height", ltpage.height)
|
||||
with yaml.array("children") as chil_arr:
|
||||
for child in ltpage:
|
||||
with chil_arr.object() as chil_obj:
|
||||
render(chil_obj, child)
|
||||
|
||||
def render(yaml: YAMLMakerObject, item: LTItem):
|
||||
if isinstance(item, LTCurve):
|
||||
place_curve(yaml, item)
|
||||
elif isinstance(item, LTXObject):
|
||||
yaml.write("type", 'xobject')
|
||||
yaml.write("x0", item.x0)
|
||||
yaml.write("x1", item.x1)
|
||||
yaml.write("y0", item.y0)
|
||||
yaml.write("y1", item.y1)
|
||||
with yaml.array("children") as chil_arr:
|
||||
for child in item:
|
||||
with chil_arr.object() as chil_obj:
|
||||
render(chil_obj, child)
|
||||
elif isinstance(item, LTImage):
|
||||
# if yaml.imagewriter is not None:
|
||||
# name = yaml.imagewriter.export_image(item)
|
||||
yaml.write("type", 'img')
|
||||
yaml.write("src", enc(item.name, None))
|
||||
yaml.write("x0", item.x0)
|
||||
yaml.write("x1", item.x1)
|
||||
yaml.write("y0", item.y0)
|
||||
yaml.write("y1", item.y1)
|
||||
elif isinstance(item, LTTextBlock):
|
||||
place_text_block(yaml, item)
|
||||
|
||||
def place_text_block(yaml: YAMLMakerObject, text_block: LTTextBlock):
|
||||
yaml.write("type", 'text-block')
|
||||
yaml.write("x0", text_block.x0)
|
||||
yaml.write("x1", text_block.x1)
|
||||
yaml.write("y0", text_block.y0)
|
||||
yaml.write("y1", text_block.y1)
|
||||
with yaml.array("char_blocks") as cb_arr:
|
||||
for cb in text_block:
|
||||
with cb_arr.object() as cb_obj:
|
||||
place_char_block(cb_obj, cb)
|
||||
|
||||
def place_char_block(yaml: YAMLMakerObject, char_block: LTCharBlock):
|
||||
yaml.write("size", char_block.size)
|
||||
yaml.write("x0", char_block.x0)
|
||||
yaml.write("x1", char_block.x1)
|
||||
yaml.write("y0", char_block.y0)
|
||||
yaml.write("y1", char_block.y1)
|
||||
if char_block.color.color_space is not None:
|
||||
yaml.place_object("color", {
|
||||
"type": enc(char_block.color.color_space.name).decode(),
|
||||
"value": normalize_color_values(char_block.color.values)
|
||||
})
|
||||
for key, value in char_block.font.descriptor.items():
|
||||
if key != "Type" and "FontFile" not in key:
|
||||
yaml.write(key, value)
|
||||
yaml.write("text",
|
||||
"".join([
|
||||
char.get_text()
|
||||
for char in char_block
|
||||
])
|
||||
)
|
||||
|
||||
def place_curve(yaml: YAMLMakerObject, item: LTCurve):
|
||||
if isinstance(item, LTRect):
|
||||
yaml.write("type", 'rect')
|
||||
elif isinstance(item, LTLine):
|
||||
yaml.write("type", 'line')
|
||||
else:
|
||||
yaml.write("type", 'curve')
|
||||
with yaml.array('paths') as paths_arr:
|
||||
for path in item.paths:
|
||||
paths_arr.place_object({
|
||||
"type": path.method.name,
|
||||
"points": [
|
||||
{"x": point.x, "y": point.y}
|
||||
for point in path.points
|
||||
]
|
||||
})
|
||||
|
||||
yaml.write("x0", item.x0)
|
||||
yaml.write("x1", item.x1)
|
||||
yaml.write("y0", item.y0)
|
||||
yaml.write("y1", item.y1)
|
||||
if item.fill.color_space is not None:
|
||||
yaml.place_object("fill", {
|
||||
"type": enc(item.fill.color_space.name).decode(),
|
||||
"value": normalize_color_values(item.fill.values)
|
||||
})
|
||||
if item.stroke.color_space is not None:
|
||||
yaml.place_object("stroke", {
|
||||
"type": enc(item.stroke.color_space.name).decode(),
|
||||
"value": normalize_color_values(item.stroke.values)
|
||||
})
|
||||
@@ -0,0 +1,90 @@
|
||||
from os import remove
|
||||
from tempfile import mktemp
|
||||
from pdfmajor.converters.writers.yaml import YAMLMaker
|
||||
|
||||
# Make temp file
|
||||
FILE_NAME = mktemp()
|
||||
|
||||
|
||||
with YAMLMaker(FILE_NAME) as yml:
|
||||
yml.write("name", "bob")
|
||||
yml.write("age", "20")
|
||||
with yml.object("wife") as wife:
|
||||
wife.write("name", "kory")
|
||||
wife.write("age", "21")
|
||||
with yml.array("children") as children:
|
||||
with children.object() as child:
|
||||
child.write("name", "Mike")
|
||||
with child.array("toys") as toys:
|
||||
toys.write("lion")
|
||||
toys.write("bear")
|
||||
with yml.object("computer") as computer:
|
||||
computer.write("type", "awesome")
|
||||
print(open(FILE_NAME, 'r').read())
|
||||
assert(open(FILE_NAME, 'r').read() == """name: bob
|
||||
age: 20
|
||||
wife:
|
||||
name: kory
|
||||
age: 21
|
||||
children:
|
||||
- name: Mike
|
||||
toys:
|
||||
- lion
|
||||
- bear
|
||||
computer:
|
||||
type: awesome""")
|
||||
|
||||
with YAMLMaker(FILE_NAME) as yml:
|
||||
yml.write("name", "bob")
|
||||
yml.write("age", "20")
|
||||
yml.place_object("wife", {
|
||||
"name": "kory",
|
||||
"age": 21
|
||||
})
|
||||
with yml.array("children") as children:
|
||||
with children.object() as child:
|
||||
child.write("name", "Mike")
|
||||
child.place_array("toys", [
|
||||
"lion", "bear"
|
||||
])
|
||||
yml.place_object("computer", {"type": "awesome"})
|
||||
print(open(FILE_NAME, 'r').read())
|
||||
assert(open(FILE_NAME, 'r').read() == """name: bob
|
||||
age: 20
|
||||
wife:
|
||||
name: kory
|
||||
age: 21
|
||||
children:
|
||||
- name: Mike
|
||||
toys:
|
||||
- lion
|
||||
- bear
|
||||
computer:
|
||||
type: awesome""")
|
||||
|
||||
with YAMLMaker(FILE_NAME) as yml:
|
||||
with yml.array('pages') as pages:
|
||||
with pages.object() as page:
|
||||
page.write("id", 0)
|
||||
page.write("width", 100)
|
||||
page.write("height", 200)
|
||||
with page.array("children") as children:
|
||||
with children.object() as curve:
|
||||
with curve.object("stroke") as fill:
|
||||
fill.write("type", "DeviceRGB")
|
||||
fill.write("value", ['1','2','3'])
|
||||
curve.write("x0", 0)
|
||||
curve.write("x1", 1)
|
||||
with curve.object("fill") as fill:
|
||||
fill.write("type", "DeviceRGB")
|
||||
fill.write("value", ['1','2','3'])
|
||||
curve.write("y0", 0)
|
||||
curve.write("y1", 1)
|
||||
|
||||
print(open(FILE_NAME, 'r').read())
|
||||
assert(open(FILE_NAME, 'r').read() == """""")
|
||||
|
||||
|
||||
|
||||
# Clean up
|
||||
remove(FILE_NAME)
|
||||
Reference in new issue
Block a user