From 56dd4c7ef4276478d6170343ff77ff99bd58b5f3 Mon Sep 17 00:00:00 2001 From: Ariel Sosnovsky Date: Sun, 9 Dec 2018 16:46:43 -0500 Subject: [PATCH] - added a functional library access --- .travis.yml | 3 -- .version | 2 +- README.md | 15 ++++++ doc-src/api.md | 58 +++++++++++++++++++++- doc-src/index.md | 15 ++++++ pdfmajor/__init__.py | 17 ++++++- pdfmajor/extractor/PDFExtractor.py | 12 +++++ pdfmajor/extractor/__init__.py | 46 +++++++++++++++++ pdfmajor/interpreter/PDFPageInterpreter.py | 3 +- pdfmajor/layouts/LTCharBlock.py | 15 +++--- pdfmajor/layouts/_base.py | 4 +- samples/extractor.py | 23 +++++++++ setup.py | 2 +- 13 files changed, 195 insertions(+), 20 deletions(-) create mode 100644 pdfmajor/extractor/PDFExtractor.py create mode 100644 pdfmajor/extractor/__init__.py create mode 100644 samples/extractor.py diff --git a/.travis.yml b/.travis.yml index 1c10c19..d20ab6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,5 @@ language: python python: - "3.6.4" - "3.7" -install: - - pip install pycryptodome - - pip install chardet script: nosetests --nologcapture diff --git a/.version b/.version index 22cdb00..0408c30 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -v1.1.12 \ No newline at end of file +v1.2.0 \ No newline at end of file diff --git a/README.md b/README.md index bf90971..6773938 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,24 @@ PDF-Major is a fork of `PDFMiner.six`. It is meant to be a more light-weight implementation that makes fewer assumptions on the data. Additionally, this version intends to be more up to date with recent versions of python and make liberal use of the typing capabilities of python for improved future maintainability. +The ultimate goal is to have a functional-programming with typing fork of the original library. + ## Documentation See documentation [here](https://asosnovsky.github.io/pdfmajor/). +## Basic Example +```py +from pdfmajor.extractor import extract_items_from_pdf + +for page in extract_items_from_pdf('path/to/your/file.pdf'): + print('page-start', page) + for item in page: + print(' ', item) + print('page-end', page) + +``` + ## Why Another Fork? `PDFMiner` was designed to run in python 2.7>, `PDFMiner.six` was written to bring cross-version support for python 2 and 3. This versions intends to make use of the latest improvement in the python language while remaining fairly simple to develop with. @@ -24,6 +38,7 @@ See documentation [here](https://asosnovsky.github.io/pdfmajor/). ## Features + * Functional extraction method based on generators * Written entirely in Python. * Parse, analyze, and convert PDF documents. * PDF-1.7 specification support. (well, almost) diff --git a/doc-src/api.md b/doc-src/api.md index 16de3f0..6c795bb 100644 --- a/doc-src/api.md +++ b/doc-src/api.md @@ -2,11 +2,44 @@ The library is constructed from 4 base modules: + * **extractor**: contains a high-level function for functional extraction of pdf-assets * **interpreter**: contains low-level classes for extracting fundamental data-structures from the documents * **converters**: contains high-level classes for conversion of the fundamental pdf structures to other formats * **imagewriter**: contains a simple implementation for converting PDF Image Streams to png/bmp/img formats * **layouts**: contains an abstraction layer for the data-structures found in the pdf +## extractors +Contains a high-level function for functional extraction of pdf-assets + +### Example +```py +from pdfmajor.extractor import extract_items_from_pdf + +for page in extract_items_from_pdf('path/to/your/file.pdf'): + print('page-start', page) + for item in page: + print(' ', item) + print('page-end', page) + +``` + +### extract_items_from_pdf(input_file_path,maxpages,password,caching,check_extractable,pagenos,debug_level) -> generator[LTPage] + +This generator-function yields individual pages which contain their respected items. + +#### Arguments + +- `input_file_path`: [str](str) +- `maxpages`: [int](#) defaults to 0 +- `password`: [str](#) defaults to None +- `caching`: [bool](#) defaults to True +- `check_extractable`: [bool](#) defaults to True +- `pagenos`: [List[int]](#) defaults to None +- `debug_level`: [logging.levels](#https://docs.python.org/3/library/logging.html#levels) defaults logging.WARNING + +#### Return Value +This function returns a generator that makes `LTPage` + ## interpreter WIP @@ -100,4 +133,27 @@ WIP ## layouts -WIP +WIP: this documentation set is not yet complete. + +### Layout Items + +All layout items extend the `LTItem` class. There are two kinds of layout items: + +- LTComponent: extends the base `LTItem` class, this class will have additional values such as boundary boxes, height and width +- LTContainer: extends the `LTComponent` class, this class is used to contain elements of the pdf that would have child elements. Iterating on this element will output its child elements. + + +#### Layout Containers +All of these classes extend the LTContainer class. + +- LTPage: a layout item containing all elements of the given page. +- LTFigure: a layout item containing LTCurves and LTImages +- LTCharBlock: a layout item containing LTChars + +#### Layout Components +All of these classes extend the LTComponent class. + +- LTChar: an individual character +- LTCurves: represents a collection of svg-paths (available under `self.paths`) +- LTImage: a component containing information regarding an image + diff --git a/doc-src/index.md b/doc-src/index.md index bf90971..6773938 100644 --- a/doc-src/index.md +++ b/doc-src/index.md @@ -13,10 +13,24 @@ PDF-Major is a fork of `PDFMiner.six`. It is meant to be a more light-weight implementation that makes fewer assumptions on the data. Additionally, this version intends to be more up to date with recent versions of python and make liberal use of the typing capabilities of python for improved future maintainability. +The ultimate goal is to have a functional-programming with typing fork of the original library. + ## Documentation See documentation [here](https://asosnovsky.github.io/pdfmajor/). +## Basic Example +```py +from pdfmajor.extractor import extract_items_from_pdf + +for page in extract_items_from_pdf('path/to/your/file.pdf'): + print('page-start', page) + for item in page: + print(' ', item) + print('page-end', page) + +``` + ## Why Another Fork? `PDFMiner` was designed to run in python 2.7>, `PDFMiner.six` was written to bring cross-version support for python 2 and 3. This versions intends to make use of the latest improvement in the python language while remaining fairly simple to develop with. @@ -24,6 +38,7 @@ See documentation [here](https://asosnovsky.github.io/pdfmajor/). ## Features + * Functional extraction method based on generators * Written entirely in Python. * Parse, analyze, and convert PDF documents. * PDF-1.7 specification support. (well, almost) diff --git a/pdfmajor/__init__.py b/pdfmajor/__init__.py index beeaba4..f764a54 100644 --- a/pdfmajor/__init__.py +++ b/pdfmajor/__init__.py @@ -18,10 +18,24 @@ PDF-Major is a fork of `PDFMiner.six`. It is meant to be a more light-weight implementation that makes fewer assumptions on the data. Additionally, this version intends to be more up to date with recent versions of python and make liberal use of the typing capabilities of python for improved future maintainability. +The ultimate goal is to have a functional-programming with typing fork of the original library. + ## Documentation See documentation [here](https://asosnovsky.github.io/pdfmajor/). +## Basic Example +```py +from pdfmajor.extractor import extract_items_from_pdf + +for page in extract_items_from_pdf('path/to/your/file.pdf'): + print('page-start', page) + for item in page: + print(' ', item) + print('page-end', page) + +``` + ## Why Another Fork? `PDFMiner` was designed to run in python 2.7>, `PDFMiner.six` was written to bring cross-version support for python 2 and 3. This versions intends to make use of the latest improvement in the python language while remaining fairly simple to develop with. @@ -29,6 +43,7 @@ See documentation [here](https://asosnovsky.github.io/pdfmajor/). ## Features + * Functional extraction method based on generators * Written entirely in Python. * Parse, analyze, and convert PDF documents. * PDF-1.7 specification support. (well, almost) @@ -86,7 +101,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = (1, 1, 12) +__version__ = (1, 2, 0) if __name__ == '__main__': print( diff --git a/pdfmajor/extractor/PDFExtractor.py b/pdfmajor/extractor/PDFExtractor.py new file mode 100644 index 0000000..209629b --- /dev/null +++ b/pdfmajor/extractor/PDFExtractor.py @@ -0,0 +1,12 @@ +from ..utils import get_logger + +log = get_logger(__name__) + +from ..layouts import PDFLayoutAnalyzer, PDFPage, LTPage + +class PDFExctractor(PDFLayoutAnalyzer): + def end_page(self, page: PDFPage): + assert not self._stack, str(len(self._stack)) + assert isinstance(self.cur_item, LTPage), str(type(self.cur_item)) + self.pageno += 1 + return self.cur_item diff --git a/pdfmajor/extractor/__init__.py b/pdfmajor/extractor/__init__.py new file mode 100644 index 0000000..a77bce9 --- /dev/null +++ b/pdfmajor/extractor/__init__.py @@ -0,0 +1,46 @@ +import re +import os + +from typing import List +from io import BytesIO, StringIO, TextIOWrapper + +from tqdm import tqdm + +from ..interpreter.PDFResourceManager import PDFResourceManager +from ..interpreter.PDFPage import PDFPage +from ..interpreter.PDFPageInterpreter import PDFPageInterpreter +from ..imagewriter import ImageWriter +from ..utils import set_log_level, get_logger, logging +from .PDFExtractor import PDFExctractor + +log = get_logger(__name__) + + +def extract_items_from_pdf( + input_file_path: str, + maxpages: int = 0, + password: str = None, + caching: bool = True, + check_extractable: bool = True, + pagenos: List[int] = None, + debug_level: int = logging.WARNING, +): + set_log_level(debug_level) + + log.debug("Creating resources....") + rsrcmgr = PDFResourceManager(caching=caching) + device = PDFExctractor( + rsrcmgr=rsrcmgr, + pageno=0 + ) + interpreter = PDFPageInterpreter(rsrcmgr, device) + log.debug("Reading pages....") + with open(input_file_path, 'rb') as input_file: + pages = PDFPage.get_pages(input_file, pagenos=pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=check_extractable) + + page_waiter = tqdm(enumerate(pages), disable=debug_level > logging.INFO) + for _, page in page_waiter: + page_waiter.set_description("Processing pdf...") + yield interpreter.process_page(page) + page_waiter.close() + device.close() \ No newline at end of file diff --git a/pdfmajor/interpreter/PDFPageInterpreter.py b/pdfmajor/interpreter/PDFPageInterpreter.py index 411d263..8f50534 100644 --- a/pdfmajor/interpreter/PDFPageInterpreter.py +++ b/pdfmajor/interpreter/PDFPageInterpreter.py @@ -624,8 +624,7 @@ class PDFPageInterpreter(object): ctm = (1, 0, 0, 1, -x0, -y0) self.device.begin_page(page, ctm) self.render_contents(page.resources, page.contents, ctm=ctm) - self.device.end_page(page) - return + return self.device.end_page(page) # render_contents(resources, streams, ctm) # Render the content streams. diff --git a/pdfmajor/layouts/LTCharBlock.py b/pdfmajor/layouts/LTCharBlock.py index 05c9643..f49520a 100644 --- a/pdfmajor/layouts/LTCharBlock.py +++ b/pdfmajor/layouts/LTCharBlock.py @@ -1,6 +1,6 @@ from typing import List -from ._base import LTComponent +from ._base import LTContainer from .LTChar import LTChar from ..utils import apply_matrix_norm, apply_matrix_pt, matrix2str, Bbox, Point @@ -11,7 +11,7 @@ from ..interpreter.PDFTextState import PDFTextState ## LTCharBlock ## -class LTCharBlock(LTComponent): +class LTCharBlock(LTContainer): def __init__(self, chars: List[Point], @@ -20,7 +20,7 @@ class LTCharBlock(LTComponent): graphicstate: PDFGraphicState ): ((x0, y0), (x1, y1)) = chars[0][1] - self.chars = [ + ltchars = [ LTChar( bbox=Bbox(x0, y0, x1, y1), char= chars[0][0], @@ -36,7 +36,7 @@ class LTCharBlock(LTComponent): x1 = max([ x0, x1, nx0, nx1 ]) y1 = max([ y0, y1, ny0, ny1 ]) y0 = min([ y0, y1, ny0, ny1 ]) - self.chars.append(LTChar( + ltchars.append(LTChar( bbox=Bbox(nx0, ny0, nx1, ny1), char= char[0], textstate=textstate, @@ -44,7 +44,7 @@ class LTCharBlock(LTComponent): graphicstate=graphicstate )) - LTComponent.__init__(self, Bbox(x0, y0, x1, y1)) + LTContainer.__init__(self, Bbox(x0, y0, x1, y1), ltchars) self.font = textstate.font self.textstate = textstate @@ -60,11 +60,8 @@ class LTCharBlock(LTComponent): def __repr__(self): return f"""<{self.__class__.__name__} font="{self.fontname}" text="{self.get_text()}"/>""" - def __iter__(self): - return iter(self.chars) - def get_text(self): - return "".join([ c.get_text() for c in self.chars ]) + return "".join([ c.get_text() for c in self ]) @property def fontname(self) -> str: diff --git a/pdfmajor/layouts/_base.py b/pdfmajor/layouts/_base.py index 2a2f652..02a7c8a 100644 --- a/pdfmajor/layouts/_base.py +++ b/pdfmajor/layouts/_base.py @@ -90,9 +90,9 @@ class LTComponent(LTItem): ## class LTContainer(LTComponent): - def __init__(self, bbox: Bbox): + def __init__(self, bbox: Bbox, objs: List[LTItem] = []): LTComponent.__init__(self, bbox) - self._objs: List[LTItem] = [] + self._objs: List[LTItem] = objs return def __iter__(self): diff --git a/samples/extractor.py b/samples/extractor.py new file mode 100644 index 0000000..a1abd6c --- /dev/null +++ b/samples/extractor.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +import re +import os +import logging + +from typing import List +from pdfmajor.converters import convert_file +from xml.etree import ElementTree +from tqdm import tqdm + +from pdfmajor.extractor import extract_items_from_pdf +from pdfmajor.layouts import LTPage + +def rel_path(*arg: List[str]) -> str: + return os.path.join(os.path.dirname(__file__), *arg) + +print(extract_items_from_pdf(rel_path('pdf/chars.pdf'))) +# for page in extract_items_from_pdf(rel_path('pdf/chars.pdf')): +# print('page-start', page) +# for item in page: +# print(' ', item) +# print('page-end', page) + \ No newline at end of file diff --git a/setup.py b/setup.py index 338ad12..c9587b3 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys with open('README.md') as readme_file: SETUP = { "name": "pdfmajor", - "version": "1.1.12", + "version": "1.2.0", "packages": find_packages(include=['pdfmajor*']), "install_requires": [ 'pycryptodome',