Author SHA1 Message Date
ari 152d3aaf55 v1.3.12 2019-12-11 22:16:28 -05:00
ari 36a9371657 generated docs 2019-12-08 14:45:28 -05:00
ari a29d465ea4 added support for ignore bad char flag
- updated unit tests to include
- using the textstate.to_unichr abstraction
2019-12-08 14:44:46 -05:00
ari 841576d9b8 exposed exceptions to top module
updated dependencies
2019-12-07 23:15:55 -05:00
ari bfc0b6ba34 added venv 2019-12-07 23:15:12 -05:00
ari 336f5f1fba propograted exceptions 2019-12-07 22:58:13 -05:00
ari 1b1db13b5b updated deps
- upgraded to Pillow 6.2.0 (passes tests)
- removed unnecessary nose dep
2019-12-07 22:57:57 -05:00
ari 3f561f3488 - removed none-develop dependency on tqdm
- ensured all test pdfs are properly pushed
2019-08-18 18:00:50 -04:00
ari 989de3122e corrected gitignore to not ignore lorem-v5.pdf 2019-08-18 17:59:14 -04:00
ari 20be0aa49e removed dependency on tqdm 2019-08-18 17:55:08 -04:00
ari d37b56a685 Merge branch 'master' of github.com:asosnovsky/pdfmajor 2019-06-10 16:28:23 -04:00
ari 9e923de7ec - added better mypy support
- updated most unit-test to use the unitttest module
- added a single unit-test run file
2019-06-10 16:26:52 -04:00
Ari Sosnovsky 2c9593ac21 Update README.md 2019-04-01 17:37:13 -04:00
Ari Sosnovsky 057cc0cf28 Update README.md 2019-04-01 17:36:36 -04:00
Ari Sosnovsky 82b47f1b19 Update README.md 2019-04-01 17:35:24 -04:00
ari 290de97cb5 - fixed invisible color bug 2019-01-26 23:08:57 -05:00
ari b32aef7fda - fixed the curve ignoring of the color error 2019-01-24 21:38:47 -05:00
59 changed files with 470 additions and 208 deletions

No files matched your search

+5 -2
View File
@@ -10,5 +10,8 @@ tests/*.txt
.idea/ .idea/
.vscode .vscode
tests/samples/output tests/samples/output
samples ./samples
**__pycache** **__pycache**\
.mypy_cache
.ipynb_checkpoints
venv
+1 -1
View File
@@ -1 +1 @@
v1.3.6 v1.3.11
+4 -4
View File
@@ -11,11 +11,11 @@
</tr> </tr>
</table> </table>
PDF-Major is a fork of `PDFMiner.six`. It is meant to be a more light-weight implementation that makes fewer assumptions on the data. The ultimate goal of this project is to provide a simpler, faster and more functional library that both casual and low-level developers could build upon. PDF-Major is a **complete re-write** of `PDFMiner.six`. It is meant to be a more light-weight implementation that makes fewer assumptions on the data. The ultimate goal of this project is to provide a simpler, faster and more functional library that both casual and low-level developers could build upon.
## Documentation ## Documentation
See documentation [here](https://asosnovsky.github.io/pdfmajor/). See [full API documentation here](https://asosnovsky.github.io/pdfmajor/api/).
## Basic Example ## Basic Example
```py ```py
@@ -29,9 +29,9 @@ for page in PDFInterpreter("/path/to/pdf.pdf"):
``` ```
## Why Another Fork? ## Why a re-write?
`PDFMiner` was designed to run in python 2.7>, `PDFMiner.six` was written to bring cross-version support for python 2 and 3. However, after attempting to build and extend upon `PDFMiner.six` I found it rather difficult to extend and identify additional details regarding the items encoded in the pdf. This library will attempt to expose as much information as possible to end-users, without having them solely rely on to-xml or to-json exports. `PDFMiner` was designed to run in python 2.7, `PDFMiner.six` was written to bring cross-version support for python 2 and 3. However, after attempting to build and extend upon `PDFMiner.six` I found it rather difficult to extend and identify additional details regarding the items encoded in the pdf. This library will attempt to expose as much information as possible to end-users, without having them solely rely on to-xml or to-json exports.
## Features ## Features
+10 -4
View File
@@ -4,10 +4,16 @@ WK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $WK_DIR cd $WK_DIR
cd .. cd ..
for filename in ./tests/test*.py; do # for filename in ./tests/test*.py; do
if python $filename ; then # if python $filename ; then
echo woohoo # echo woohoo
# else
# exit 1
# fi
# done
if python ./test.py ; then
echo "PASSED TESTS"
else else
exit 1 exit 1
fi fi
done
+1
View File
@@ -37,6 +37,7 @@ This generator-function yields individual pages which contain their respected it
- `password`: [str](#) defaults to None - `password`: [str](#) defaults to None
- `caching`: [bool](#) defaults to True - `caching`: [bool](#) defaults to True
- `check_extractable`: [bool](#) defaults to True - `check_extractable`: [bool](#) defaults to True
- `ignore_bad_chars`: [bool](#) defaults to False
- `pagenos`: [List[int]](#) defaults to None - `pagenos`: [List[int]](#) defaults to None
- `debug_level`: [logging.levels](#https://docs.python.org/3/library/logging.html#levels) defaults logging.WARNING - `debug_level`: [logging.levels](#https://docs.python.org/3/library/logging.html#levels) defaults logging.WARNING
+1
View File
@@ -140,6 +140,7 @@ for page in PDFInterpreter(&quot;/path/to/pdf.pdf&quot;):
<li><code>password</code>: <a href="#">str</a> defaults to None </li> <li><code>password</code>: <a href="#">str</a> defaults to None </li>
<li><code>caching</code>: <a href="#">bool</a> defaults to True </li> <li><code>caching</code>: <a href="#">bool</a> defaults to True </li>
<li><code>check_extractable</code>: <a href="#">bool</a> defaults to True</li> <li><code>check_extractable</code>: <a href="#">bool</a> defaults to True</li>
<li><code>ignore_bad_chars</code>: <a href="#">bool</a> defaults to False</li>
<li><code>pagenos</code>: <a href="#">List[int]</a> defaults to None</li> <li><code>pagenos</code>: <a href="#">List[int]</a> defaults to None</li>
<li><code>debug_level</code>: <a href="#https://docs.python.org/3/library/logging.html#levels">logging.levels</a> defaults logging.WARNING</li> <li><code>debug_level</code>: <a href="#https://docs.python.org/3/library/logging.html#levels">logging.levels</a> defaults logging.WARNING</li>
</ul> </ul>
+1 -1
View File
@@ -240,5 +240,5 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.</p>
<!-- <!--
MkDocs version : 1.0.4 MkDocs version : 1.0.4
Build Date UTC : 2019-01-11 20:22:59 Build Date UTC : 2019-12-08 19:44:26
--> -->
+1 -1
View File
@@ -1 +1 @@
{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"PDFMajor Latest Release PDF-Major is a fork of PDFMiner.six . It is meant to be a more light-weight implementation that makes fewer assumptions on the data. The ultimate goal of this project is to provide a simpler, faster and more functional library that both casual and low-level developers could build upon. Documentation See documentation here . Basic Example from pdfmajor.interpreter import PDFInterpreter for page in PDFInterpreter(\"/path/to/pdf.pdf\"): print(\"page start\", page.page_num) for item in page: print(\" >\", item) print(\"page end\", page.page_num) Why Another Fork? PDFMiner was designed to run in python 2.7>, PDFMiner.six was written to bring cross-version support for python 2 and 3. However, after attempting to build and extend upon PDFMiner.six I found it rather difficult to extend and identify additional details regarding the items encoded in the pdf. This library will attempt to expose as much information as possible to end-users, without having them solely rely on to-xml or to-json exports. Features Functional extraction method based on generators Parse, analyze, and convert PDF documents. PDF-1.7 specification support. (well, almost) Font-Color extraction Shape fill and stroke color extraction CJK languages and vertical writing scripts support. Various font types (Type1, TrueType, Type3, and CID) support. Basic encryption (RC4) support. Note : We took out the layout-analysis process in this version (there is no more LTTextHorizontal or LTTextVertical). While the mathematics behind the grouping process was sound, the coupling of the layout-analysis process with the parsing and interpretation process produced unfriendly-code. This feature could be brought back by running your own implementation of it on the pdfmajor.interpreter.PageInterpreter class, but at the current time is not supported. How to Install Source Install Python 3.6.4 or newer. clone this repo git clone https://github.com/asosnovsky/pdfmajor * install repo python setup.py install Pypi Install Python 3.6.4 or newer. install repo pip install pdfmajor Terms and Conditions (This is so-called MIT/X License) Copyright (c) 2018-2019 Ariel Sosnovsky Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.","title":"Home"},{"location":"#pdfmajor","text":"Latest Release PDF-Major is a fork of PDFMiner.six . It is meant to be a more light-weight implementation that makes fewer assumptions on the data. The ultimate goal of this project is to provide a simpler, faster and more functional library that both casual and low-level developers could build upon.","title":"PDFMajor"},{"location":"#documentation","text":"See documentation here .","title":"Documentation"},{"location":"#basic-example","text":"from pdfmajor.interpreter import PDFInterpreter for page in PDFInterpreter(\"/path/to/pdf.pdf\"): print(\"page start\", page.page_num) for item in page: print(\" >\", item) print(\"page end\", page.page_num)","title":"Basic Example"},{"location":"#why-another-fork","text":"PDFMiner was designed to run in python 2.7>, PDFMiner.six was written to bring cross-version support for python 2 and 3. However, after attempting to build and extend upon PDFMiner.six I found it rather difficult to extend and identify additional details regarding the items encoded in the pdf. This library will attempt to expose as much information as possible to end-users, without having them solely rely on to-xml or to-json exports.","title":"Why Another Fork?"},{"location":"#features","text":"Functional extraction method based on generators Parse, analyze, and convert PDF documents. PDF-1.7 specification support. (well, almost) Font-Color extraction Shape fill and stroke color extraction CJK languages and vertical writing scripts support. Various font types (Type1, TrueType, Type3, and CID) support. Basic encryption (RC4) support. Note : We took out the layout-analysis process in this version (there is no more LTTexLine truncated {"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"PDFMajor Latest Release PDF-Major is a fork of PDFMiner.six . It is meant to be a more light-weight implementation that makes fewer assumptions on the data. The ultimate goal of this project is to provide a simpler, faster and more functional library that both casual and low-level developers could build upon. Documentation See documentation here . Basic Example from pdfmajor.interpreter import PDFInterpreter for page in PDFInterpreter(\"/path/to/pdf.pdf\"): print(\"page start\", page.page_num) for item in page: print(\" >\", item) print(\"page end\", page.page_num) Why Another Fork? PDFMiner was designed to run in python 2.7>, PDFMiner.six was written to bring cross-version support for python 2 and 3. However, after attempting to build and extend upon PDFMiner.six I found it rather difficult to extend and identify additional details regarding the items encoded in the pdf. This library will attempt to expose as much information as possible to end-users, without having them solely rely on to-xml or to-json exports. Features Functional extraction method based on generators Parse, analyze, and convert PDF documents. PDF-1.7 specification support. (well, almost) Font-Color extraction Shape fill and stroke color extraction CJK languages and vertical writing scripts support. Various font types (Type1, TrueType, Type3, and CID) support. Basic encryption (RC4) support. Note : We took out the layout-analysis process in this version (there is no more LTTextHorizontal or LTTextVertical). While the mathematics behind the grouping process was sound, the coupling of the layout-analysis process with the parsing and interpretation process produced unfriendly-code. This feature could be brought back by running your own implementation of it on the pdfmajor.interpreter.PageInterpreter class, but at the current time is not supported. How to Install Source Install Python 3.6.4 or newer. clone this repo git clone https://github.com/asosnovsky/pdfmajor * install repo python setup.py install Pypi Install Python 3.6.4 or newer. install repo pip install pdfmajor Terms and Conditions (This is so-called MIT/X License) Copyright (c) 2018-2019 Ariel Sosnovsky Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.","title":"Home"},{"location":"#pdfmajor","text":"Latest Release PDF-Major is a fork of PDFMiner.six . It is meant to be a more light-weight implementation that makes fewer assumptions on the data. The ultimate goal of this project is to provide a simpler, faster and more functional library that both casual and low-level developers could build upon.","title":"PDFMajor"},{"location":"#documentation","text":"See documentation here .","title":"Documentation"},{"location":"#basic-example","text":"from pdfmajor.interpreter import PDFInterpreter for page in PDFInterpreter(\"/path/to/pdf.pdf\"): print(\"page start\", page.page_num) for item in page: print(\" >\", item) print(\"page end\", page.page_num)","title":"Basic Example"},{"location":"#why-another-fork","text":"PDFMiner was designed to run in python 2.7>, PDFMiner.six was written to bring cross-version support for python 2 and 3. However, after attempting to build and extend upon PDFMiner.six I found it rather difficult to extend and identify additional details regarding the items encoded in the pdf. This library will attempt to expose as much information as possible to end-users, without having them solely rely on to-xml or to-json exports.","title":"Why Another Fork?"},{"location":"#features","text":"Functional extraction method based on generators Parse, analyze, and convert PDF documents. PDF-1.7 specification support. (well, almost) Font-Color extraction Shape fill and stroke color extraction CJK languages and vertical writing scripts support. Various font types (Type1, TrueType, Type3, and CID) support. Basic encryption (RC4) support. Note : We took out the layout-analysis process in this version (there is no more LTTexLine truncated
+2 -2
View File
@@ -2,12 +2,12 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url> <url>
<loc>None</loc> <loc>None</loc>
<lastmod>2019-01-11</lastmod> <lastmod>2019-12-08</lastmod>
<changefreq>daily</changefreq> <changefreq>daily</changefreq>
</url> </url>
<url> <url>
<loc>None</loc> <loc>None</loc>
<lastmod>2019-01-11</lastmod> <lastmod>2019-12-08</lastmod>
<changefreq>daily</changefreq> <changefreq>daily</changefreq>
</url> </url>
</urlset> </urlset>
Binary file not shown.
+5 -5
View File
@@ -16,11 +16,11 @@
</tr> </tr>
</table> </table>
PDF-Major is a fork of `PDFMiner.six`. It is meant to be a more light-weight implementation that makes fewer assumptions on the data. The ultimate goal of this project is to provide a simpler, faster and more functional library that both casual and low-level developers could build upon. PDF-Major is a **complete re-write** of `PDFMiner.six`. It is meant to be a more light-weight implementation that makes fewer assumptions on the data. The ultimate goal of this project is to provide a simpler, faster and more functional library that both casual and low-level developers could build upon.
## Documentation ## Documentation
See documentation [here](https://asosnovsky.github.io/pdfmajor/). See [full API documentation here](https://asosnovsky.github.io/pdfmajor/api/).
## Basic Example ## Basic Example
```py ```py
@@ -34,9 +34,9 @@ for page in PDFInterpreter("/path/to/pdf.pdf"):
``` ```
## Why Another Fork? ## Why a re-write?
`PDFMiner` was designed to run in python 2.7>, `PDFMiner.six` was written to bring cross-version support for python 2 and 3. However, after attempting to build and extend upon `PDFMiner.six` I found it rather difficult to extend and identify additional details regarding the items encoded in the pdf. This library will attempt to expose as much information as possible to end-users, without having them solely rely on to-xml or to-json exports. `PDFMiner` was designed to run in python 2.7, `PDFMiner.six` was written to bring cross-version support for python 2 and 3. However, after attempting to build and extend upon `PDFMiner.six` I found it rather difficult to extend and identify additional details regarding the items encoded in the pdf. This library will attempt to expose as much information as possible to end-users, without having them solely rely on to-xml or to-json exports.
## Features ## Features
@@ -97,7 +97,7 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
""" """
__version__ = (1, 3, 6) __version__ = (1, 3, 11)
if __name__ == '__main__': if __name__ == '__main__':
print( print(
+11 -5
View File
@@ -2,7 +2,7 @@ import io
import os import os
import re import re
from typing import List from typing import List, Optional
from .html import convert_to_html from .html import convert_to_html
from .xml import convert_to_xml from .xml import convert_to_xml
@@ -13,15 +13,16 @@ from ..utils import logging
def convert_file( def convert_file(
input_file: str, input_file: str,
output_file: str = None, output_file: Optional[str] = None,
image_folder_path: str = None, image_folder_path: Optional[str] = None,
codec: str = 'utf-8', codec: str = 'utf-8',
maxpages: int = 0, maxpages: int = 0,
password: str = None, password: Optional[str] = None,
caching: bool = True, caching: bool = True,
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, pagenos: Optional[List[int]] = None,
dont_export_images: bool = False, dont_export_images: bool = False,
ignore_bad_chars: bool = False,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
out_type: str = 'html', out_type: str = 'html',
): ):
@@ -45,6 +46,7 @@ def convert_file(
password=password, password=password,
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
ignore_bad_chars=ignore_bad_chars,
pagenos=pagenos, pagenos=pagenos,
debug_level=debug_level, debug_level=debug_level,
) )
@@ -59,6 +61,7 @@ def convert_file(
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
pagenos=pagenos, pagenos=pagenos,
ignore_bad_chars=ignore_bad_chars,
dont_export_images=dont_export_images, dont_export_images=dont_export_images,
debug_level=debug_level, debug_level=debug_level,
) )
@@ -73,6 +76,7 @@ def convert_file(
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
pagenos=pagenos, pagenos=pagenos,
ignore_bad_chars=ignore_bad_chars,
dont_export_images=dont_export_images, dont_export_images=dont_export_images,
debug_level=debug_level, debug_level=debug_level,
) )
@@ -87,6 +91,7 @@ def convert_file(
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
pagenos=pagenos, pagenos=pagenos,
ignore_bad_chars=ignore_bad_chars,
dont_export_images=dont_export_images, dont_export_images=dont_export_images,
debug_level=debug_level, debug_level=debug_level,
) )
@@ -101,6 +106,7 @@ def convert_file(
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
pagenos=pagenos, pagenos=pagenos,
ignore_bad_chars=ignore_bad_chars,
dont_export_images=dont_export_images, dont_export_images=dont_export_images,
debug_level=debug_level, debug_level=debug_level,
) )
+5 -3
View File
@@ -1,4 +1,4 @@
from typing import List from typing import List, Optional
from .writers.html import HTMLMaker from .writers.html import HTMLMaker
from ..interpreter import PDFInterpreter, PageInterpreter, logging from ..interpreter import PDFInterpreter, PageInterpreter, logging
from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject from ..interpreter import LTImage, LTTextBlock, LTCharBlock, LTChar, LTCurve, LTXObject
@@ -10,12 +10,13 @@ INF = (1<<31) - 1
def convert_to_html( def convert_to_html(
input_file_path: str, input_file_path: str,
output_file_path: str, output_file_path: str,
image_folder_path: str = None, image_folder_path: Optional[str] = None,
dont_export_images: bool = False, dont_export_images: bool = False,
codec: str = 'utf-8', codec: str = 'utf-8',
maxpages: int = 0, maxpages: int = 0,
password: str = None, password: Optional[str] = None,
caching: bool = True, caching: bool = True,
ignore_bad_chars: bool = False,
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, pagenos: List[int] = None,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
@@ -25,6 +26,7 @@ def convert_to_html(
password=password, password=password,
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
ignore_bad_chars=ignore_bad_chars,
pagenos=pagenos, pagenos=pagenos,
debug_level=debug_level debug_level=debug_level
) )
+6 -4
View File
@@ -1,4 +1,4 @@
from typing import List from typing import List, Optional
from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray from .writers.json import JSONMaker, JSONMakerObject, JSONMakerArray
from ..interpreter import PDFInterpreter, PageInterpreter, logging from ..interpreter import PDFInterpreter, PageInterpreter, logging
@@ -21,14 +21,15 @@ def normalize_color_values(values: list):
def convert_to_json( def convert_to_json(
input_file_path: str, input_file_path: str,
output_file_path: str, output_file_path: str,
image_folder_path: str = None, image_folder_path: Optional[str] = None,
dont_export_images: bool = False, dont_export_images: bool = False,
codec: str = 'utf-8', codec: str = 'utf-8',
maxpages: int = 0, maxpages: int = 0,
password: str = None, password: str = Optional[None],
caching: bool = True, caching: bool = True,
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, ignore_bad_chars: bool = False,
pagenos: Optional[List[int]] = None,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
): ):
intepreter = PDFInterpreter(input_file_path, intepreter = PDFInterpreter(input_file_path,
@@ -36,6 +37,7 @@ def convert_to_json(
password=password, password=password,
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
ignore_bad_chars=ignore_bad_chars,
pagenos=pagenos, pagenos=pagenos,
debug_level=debug_level debug_level=debug_level
) )
+2
View File
@@ -14,6 +14,7 @@ def convert_to_text(
password: str = None, password: str = None,
caching: bool = True, caching: bool = True,
check_extractable: bool = True, check_extractable: bool = True,
ignore_bad_chars: bool = False,
pagenos: List[int] = None, pagenos: List[int] = None,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
): ):
@@ -22,6 +23,7 @@ def convert_to_text(
password=password, password=password,
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
ignore_bad_chars=ignore_bad_chars,
pagenos=pagenos, pagenos=pagenos,
debug_level=debug_level debug_level=debug_level
) )
+5 -4
View File
@@ -1,3 +1,4 @@
from typing import Optional
from io import TextIOWrapper from io import TextIOWrapper
from contextlib import contextmanager from contextlib import contextmanager
@@ -6,7 +7,7 @@ class HTMLMaker:
def __init__(self, file_path: str, codec: str = 'utf-8'): def __init__(self, file_path: str, codec: str = 'utf-8'):
self.file_path = file_path self.file_path = file_path
self.codec = codec self.codec = codec
self.outfile: TextIOWrapper = None self.outfile: Optional[TextIOWrapper] = None
def __enter__(self): def __enter__(self):
self.outfile = open(self.file_path, 'wb') self.outfile = open(self.file_path, 'wb')
@@ -42,18 +43,18 @@ class HTMLMaker:
f'{name}: {value}' f'{name}: {value}'
for name, value in css.items() for name, value in css.items()
]) ])
attrs = " ".join([ attrs_str = " ".join([
f'{name}="{value}"' f'{name}="{value}"'
for name, value in attrs.items() for name, value in attrs.items()
]) ])
if not singleton: if not singleton:
self.write(f"<{tag_name} {attrs}>", lineend=lineend) self.write(f"<{tag_name} {attrs_str}>", lineend=lineend)
self.__levels_deep += 1 self.__levels_deep += 1
yield yield
self.__levels_deep -= 1 self.__levels_deep -= 1
self.write(f"</{tag_name}>", deep_space=deep_space, lineend='\n' if not nolineend else '') self.write(f"</{tag_name}>", deep_space=deep_space, lineend='\n' if not nolineend else '')
else: else:
self.write(f"<{tag_name} {attrs} />", lineend=lineend, deep_space=deep_space) self.write(f"<{tag_name} {attrs_str} />", lineend=lineend, deep_space=deep_space)
yield yield
def singleton(self, tag_name: str, attrs: dict = None, css: dict = None, singleton: bool = True): def singleton(self, tag_name: str, attrs: dict = None, css: dict = None, singleton: bool = True):
+2 -2
View File
@@ -1,3 +1,4 @@
from typing import Set
from json import dumps as json_dump from json import dumps as json_dump
from io import TextIOWrapper from io import TextIOWrapper
from contextlib import contextmanager from contextlib import contextmanager
@@ -8,14 +9,13 @@ def JSONMaker(file_path: str, codec: str = 'utf-8'):
with JSONMakerObject(outf, codec=codec) as obj: with JSONMakerObject(outf, codec=codec) as obj:
yield obj yield obj
class JSONMakerObject: class JSONMakerObject:
class AccessError(Exception): pass class AccessError(Exception): pass
def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = 0): def __init__(self, outfile: TextIOWrapper, codec: str = 'utf-8', levels_deep: int = 0):
self.outfile: TextIOWrapper = outfile self.outfile: TextIOWrapper = outfile
self.codec = codec self.codec = codec
self.levels_deep = levels_deep self.levels_deep = levels_deep
self.keys = set() self.keys: Set[str] = set()
def __enter__(self): def __enter__(self):
self.write_raw('{') self.write_raw('{')
+2
View File
@@ -45,6 +45,7 @@ def convert_to_xml(
maxpages: int = 0, maxpages: int = 0,
password: str = None, password: str = None,
caching: bool = True, caching: bool = True,
ignore_bad_chars: bool = False,
check_extractable: bool = True, check_extractable: bool = True,
pagenos: List[int] = None, pagenos: List[int] = None,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
@@ -54,6 +55,7 @@ def convert_to_xml(
password=password, password=password,
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
ignore_bad_chars=ignore_bad_chars,
pagenos=pagenos, pagenos=pagenos,
debug_level=debug_level debug_level=debug_level
) )
+2
View File
@@ -28,6 +28,7 @@ def convert_to_yaml(
password: str = None, password: str = None,
caching: bool = True, caching: bool = True,
check_extractable: bool = True, check_extractable: bool = True,
ignore_bad_chars: bool = False,
pagenos: List[int] = None, pagenos: List[int] = None,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
): ):
@@ -36,6 +37,7 @@ def convert_to_yaml(
password=password, password=password,
caching=caching, caching=caching,
check_extractable=check_extractable, check_extractable=check_extractable,
ignore_bad_chars=ignore_bad_chars,
pagenos=pagenos, pagenos=pagenos,
debug_level=debug_level debug_level=debug_level
) )
+2
View File
@@ -0,0 +1,2 @@
import pdfmajor.parser.exceptions as parser
import pdfmajor.interpreter.exceptions as interpreter
+4 -4
View File
@@ -5,9 +5,8 @@ from .commands import PDFStateStack
from .commands import process_command_stream, prep_state from .commands import process_command_stream, prep_state
class PageInterpreter: class PageInterpreter:
class PDFInterpreterError(Exception): pass
def __init__(self, page: PDFPage, page_num: int, font_cache: dict = {}): def __init__(self, page: PDFPage, page_num: int, font_cache: dict = None, ignore_bad_chars = False):
(x0, y0, x1, y1) = page.mediabox (x0, y0, x1, y1) = page.mediabox
if page.rotate == 90: if page.rotate == 90:
ctm = [0, -1, 1, 0, -y0, x1] ctm = [0, -1, 1, 0, -y0, x1]
@@ -19,7 +18,7 @@ class PageInterpreter:
ctm = [1, 0, 0, 1, -x0, -y0] ctm = [1, 0, 0, 1, -x0, -y0]
# Locals # Locals
self.font_cache = font_cache self.font_cache = font_cache if font_cache is not None else {}
self.page = page self.page = page
self.page_num = page_num self.page_num = page_num
self.height = y1-y0 self.height = y1-y0
@@ -30,7 +29,8 @@ class PageInterpreter:
PDFStateStack(), PDFStateStack(),
ctm=ctm, ctm=ctm,
resources=page.resources, resources=page.resources,
font_cache=self.font_cache font_cache=self.font_cache,
ignore_bad_chars=ignore_bad_chars
) )
+5 -5
View File
@@ -1,7 +1,5 @@
from typing import List from typing import List
from tqdm import tqdm
from ..parser.PDFPage import PDFPage from ..parser.PDFPage import PDFPage
from ..utils import set_log_level, get_logger, logging from ..utils import set_log_level, get_logger, logging
from .commands.state import PDFStateStack, PDFColorSpace, PREDEFINED_COLORSPACE from .commands.state import PDFStateStack, PDFColorSpace, PREDEFINED_COLORSPACE
@@ -25,6 +23,7 @@ class PDFInterpreter:
password: str = None, password: str = None,
pagenos: List[int] = None, pagenos: List[int] = None,
preload: bool = False, preload: bool = False,
ignore_bad_chars: bool = False,
debug_level: int = logging.WARNING, debug_level: int = logging.WARNING,
): ):
self.input_file_path = input_file_path self.input_file_path = input_file_path
@@ -33,6 +32,7 @@ class PDFInterpreter:
self.check_extractable = check_extractable self.check_extractable = check_extractable
self.password = password self.password = password
self.pagenos = pagenos self.pagenos = pagenos
self.ignore_bad_chars = ignore_bad_chars
self.debug_level = debug_level self.debug_level = debug_level
self.__pages = [] self.__pages = []
if preload: if preload:
@@ -53,8 +53,8 @@ class PDFInterpreter:
caching=self.caching, caching=self.caching,
check_extractable=self.check_extractable check_extractable=self.check_extractable
) )
for page_num, page in tqdm(enumerate(pages), desc="Reading pages...", disable=self.debug_level > logging.INFO): for page_num, page in enumerate(pages):
self.__pages.append(PageInterpreter(page, page_num, font_cache)) self.__pages.append(PageInterpreter(page, page_num, font_cache, ignore_bad_chars=self.ignore_bad_chars))
yield self.__pages[-1] yield self.__pages[-1]
log.info(f"Done Reading {len(self.__pages)} pages.") log.info(f"Done Reading {len(self.__pages)} pages.")
set_log_level(logging.WARNING) set_log_level(logging.WARNING)
@@ -64,7 +64,7 @@ class PDFInterpreter:
def __iter__(self): def __iter__(self):
if len(self.__pages) > 0: if len(self.__pages) > 0:
set_log_level(self.debug_level) set_log_level(self.debug_level)
for page in tqdm(self.__pages, desc="Reading pages from store...", disable=self.debug_level > logging.INFO): for page in self.__pages:
yield page yield page
set_log_level(logging.WARNING) set_log_level(logging.WARNING)
else: else:
+4 -3
View File
@@ -7,17 +7,18 @@ from pdfmajor.parser.constants import LITERAL_FORM, LITERAL_IMAGE
from .state import PDFStateStack, PDFColorSpace from .state import PDFStateStack, PDFColorSpace
from .state.Curves import CurveMethod, CurvePath, CurvePoint from .state.Curves import CurveMethod, CurvePath, CurvePoint
class PDFCommandsInvalidOperation(Exception): pass
class PDFCommandsRepeatedCommand(Exception): pass
class PDFCommands: class PDFCommands:
commands = {} commands = {}
class InvalidOperation(Exception): pass
class RepeatedCommand(Exception): pass
@classmethod @classmethod
def __add(cls, cmd_name: str, cmd: callable): def __add(cls, cmd_name: str, cmd: callable):
if not (cmd_name in cls.commands.keys()): if not (cmd_name in cls.commands.keys()):
cls.commands[cmd_name] = cmd cls.commands[cmd_name] = cmd
return cmd return cmd
raise cls.RepeatedCommand(cmd_name) raise PDFCommandsRepeatedCommand(cmd_name)
@classmethod @classmethod
def add(cls, *cmd_names: List[str]): def add(cls, *cmd_names: List[str]):
+13 -1
View File
@@ -23,10 +23,16 @@ def process_command_stream(streams: List[PDFStream], font_cache: dict = None, st
if font_cache is None: if font_cache is None:
font_cache = {} font_cache = {}
parser = PDFContentParser(streams) parser = PDFContentParser(streams)
# from .cmd_record import CmdRecord
# history = CmdRecord()
for obj in parser: for obj in parser:
if isinstance(obj, PSKeyword): if isinstance(obj, PSKeyword):
name = keyword_name(obj) name = keyword_name(obj)
method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q') method = name.replace('*', '_a').replace('"', '_w').replace("'", '_q')
func = None
args = []
if method == "Do":
pass
if method in PDFCommands.commands.keys(): if method in PDFCommands.commands.keys():
func = PDFCommands.commands.get(method) func = PDFCommands.commands.get(method)
nargs = func.__code__.co_argcount-1 nargs = func.__code__.co_argcount-1
@@ -40,8 +46,12 @@ def process_command_stream(streams: List[PDFStream], font_cache: dict = None, st
func(state) func(state)
else: else:
log.debug('Unknown operator: %r (i.e. %r)' % (name, method)) log.debug('Unknown operator: %r (i.e. %r)' % (name, method))
# history.write(name,
# None if func is None else func.__name__
# , args)
else: else:
state.argstack.append(obj) state.argstack.append(obj)
# history.write('arg', None, obj)
for complete_item in state.complete_layout_items: for complete_item in state.complete_layout_items:
if isinstance(complete_item, LTXObject): if isinstance(complete_item, LTXObject):
@@ -51,6 +61,7 @@ def process_command_stream(streams: List[PDFStream], font_cache: dict = None, st
resources=complete_item.resources, resources=complete_item.resources,
font_cache=font_cache font_cache=font_cache
) )
xobj_state.graphics = state.graphics.copy()
for item in process_command_stream( for item in process_command_stream(
[complete_item.stream], [complete_item.stream],
font_cache=font_cache, font_cache=font_cache,
@@ -60,9 +71,10 @@ def process_command_stream(streams: List[PDFStream], font_cache: dict = None, st
yield complete_item yield complete_item
state.complete_layout_items = [] state.complete_layout_items = []
def prep_state(state: PDFStateStack, ctm: tuple, resources: dict, font_cache: dict) -> PDFStateStack: def prep_state(state: PDFStateStack, ctm: tuple, resources: dict, font_cache: dict, ignore_bad_chars: bool = False) -> PDFStateStack:
state.t_matrix = ctm state.t_matrix = ctm
state.resources = resources state.resources = resources
state.text.ignore_bad_chars = ignore_bad_chars
# set some global states. # set some global states.
state.graphics.ncolspace = state.graphics.scolspace = None state.graphics.ncolspace = state.graphics.scolspace = None
@@ -0,0 +1,9 @@
class CmdRecord:
def __init__(self, file_name: str = "./cmd_record"):
self.file = open(file_name, 'w')
self.file.write("c_name,func_name,args\n")
def write(self, c_name, func_name, args):
self.file.write(f"{c_name},{func_name},{args}\n")
def close(self):
self.file.close()
+10 -10
View File
@@ -3,7 +3,7 @@ from pdfmajor.parser.PSStackParser import literal_name
from pdfmajor.parser.PDFStream import PDFStream, list_value, dict_value from pdfmajor.parser.PDFStream import PDFStream, list_value, dict_value
from pdfmajor.parser.constants import LITERAL_FORM, LITERAL_IMAGE from pdfmajor.parser.constants import LITERAL_FORM, LITERAL_IMAGE
from .PDFCommands import PDFCommands from .PDFCommands import PDFCommands, PDFCommandsInvalidOperation, PDFCommandsRepeatedCommand
from .state import PDFStateStack, PDFGraphicState, LTTextBlock from .state import PDFStateStack, PDFGraphicState, LTTextBlock
from .state.Curves import CurveMethod, CurvePath, CurvePoint from .state.Curves import CurveMethod, CurvePath, CurvePoint
# from .state.PDFItem import PDFImage, PDFShape, PDFText, PDFXObject # from .state.PDFItem import PDFImage, PDFShape, PDFText, PDFXObject
@@ -253,7 +253,7 @@ def set_stroke_colorspace(stack: PDFStateStack, name: str) -> PDFStateStack:
stack.graphics.scolspace = stack.colorspace_map[literal_name(name)] stack.graphics.scolspace = stack.colorspace_map[literal_name(name)]
return stack return stack
except KeyError: except KeyError:
raise PDFCommands.InvalidOperation('Undefined ColorSpace: %r' % name) raise PDFCommandsInvalidOperation('Undefined ColorSpace: %r' % name)
# setcolorspace-non-strokine # setcolorspace-non-strokine
@PDFCommands.add('cs') @PDFCommands.add('cs')
@@ -262,7 +262,7 @@ def set_nonstroke_colorspace(stack: PDFStateStack, name: str) -> PDFStateStack:
stack.graphics.ncolspace = stack.colorspace_map[literal_name(name)] stack.graphics.ncolspace = stack.colorspace_map[literal_name(name)]
return stack return stack
except KeyError: except KeyError:
raise PDFCommands.InvalidOperation('Undefined ColorSpace: %r' % name) raise PDFCommandsInvalidOperation('Undefined ColorSpace: %r' % name)
# setgray-stroking # setgray-stroking
@PDFCommands.add('G') @PDFCommands.add('G')
@@ -326,7 +326,7 @@ def set_nonstroke_cmyk(stack: PDFStateStack, c, m, y, k) -> PDFStateStack:
@PDFCommands.add('SCN','SC') @PDFCommands.add('SCN','SC')
def set_stroke_colorspace_custom(stack: PDFStateStack) -> PDFStateStack: def set_stroke_colorspace_custom(stack: PDFStateStack) -> PDFStateStack:
if stack.graphics.scolspace is None: if stack.graphics.scolspace is None:
raise PDFCommands.InvalidOperation('No colorspace specified!') raise PDFCommandsInvalidOperation('No colorspace specified!')
components = stack.pop(stack.graphics.scolspace.ncomponents) components = stack.pop(stack.graphics.scolspace.ncomponents)
stack.graphics.set_stroke_color( stack.graphics.set_stroke_color(
stack.graphics.scolspace, stack.graphics.scolspace,
@@ -337,7 +337,7 @@ def set_stroke_colorspace_custom(stack: PDFStateStack) -> PDFStateStack:
@PDFCommands.add('scn','sc') @PDFCommands.add('scn','sc')
def set_nonstroke_colorspace_custom(stack: PDFStateStack) -> PDFStateStack: def set_nonstroke_colorspace_custom(stack: PDFStateStack) -> PDFStateStack:
if stack.graphics.ncolspace is None: if stack.graphics.ncolspace is None:
raise PDFCommands.InvalidOperation('No colorspace specified!') raise PDFCommandsInvalidOperation('No colorspace specified!')
components = stack.pop(stack.graphics.ncolspace.ncomponents) components = stack.pop(stack.graphics.ncolspace.ncomponents)
stack.graphics.set_nostroke_color( stack.graphics.set_nostroke_color(
stack.graphics.ncolspace, stack.graphics.ncolspace,
@@ -353,7 +353,7 @@ def begin_text(stack: PDFStateStack) -> PDFStateStack:
if stack.current_textblock is None: if stack.current_textblock is None:
stack.current_textblock = LTTextBlock() stack.current_textblock = LTTextBlock()
else: else:
raise PDFCommands.InvalidOperation("Last TextBlock did not clear") raise PDFCommandsInvalidOperation("Last TextBlock did not clear")
return stack return stack
@PDFCommands.add('ET') @PDFCommands.add('ET')
@@ -394,7 +394,7 @@ def set_font(stack: PDFStateStack, fontid, fontsize) -> PDFStateStack:
stack.text.font = stack.fontmap[literal_name(fontid)] stack.text.font = stack.fontmap[literal_name(fontid)]
return stack return stack
except KeyError: except KeyError:
raise PDFCommands.InvalidOperation('Undefined Font id: %r' % fontid) raise PDFCommandsInvalidOperation('Undefined Font id: %r' % fontid)
# setrendering # setrendering
@PDFCommands.add('Tr') @PDFCommands.add('Tr')
@@ -444,9 +444,9 @@ def nextline(stack: PDFStateStack) -> PDFStateStack:
@PDFCommands.add('TJ') @PDFCommands.add('TJ')
def add_char_blocks(stack: PDFStateStack, seq: bytearray) -> PDFStateStack: def add_char_blocks(stack: PDFStateStack, seq: bytearray) -> PDFStateStack:
if stack.text.font is None: if stack.text.font is None:
raise PDFCommands.InvalidOperation("No Font Specified") raise PDFCommandsInvalidOperation("No Font Specified")
if stack.current_textblock is None: if stack.current_textblock is None:
raise PDFCommands.InvalidOperation("No TextBlock initilized") raise PDFCommandsInvalidOperation("No TextBlock initilized")
stack.current_textblock.add_char_block( stack.current_textblock.add_char_block(
seq, seq,
stack.t_matrix, stack.t_matrix,
@@ -487,7 +487,7 @@ def insert_xobject(stack: PDFStateStack, xobjid) -> PDFStateStack:
try: try:
xobj = PDFStream.validated_stream(stack.xobjmap[xobjid]) xobj = PDFStream.validated_stream(stack.xobjmap[xobjid])
except KeyError: except KeyError:
raise PDFCommands.InvalidOperation('Undefined xobject id: %r' % xobjid) raise PDFCommandsInvalidOperation('Undefined xobject id: %r' % xobjid)
# log.info('Processing xobj: %r', xobj) # log.info('Processing xobj: %r', xobj)
subtype = xobj.get('Subtype') subtype = xobj.get('Subtype')
if subtype is LITERAL_FORM and 'BBox' in xobj: if subtype is LITERAL_FORM and 'BBox' in xobj:
@@ -1,10 +1,10 @@
from typing import List from typing import Tuple, Optional
from .PDFColorSpace import PDFColorSpace from .PDFColorSpace import PDFColorSpace
class PDFColor: class PDFColor:
def __init__(self, color_space: PDFColorSpace, *values: List[float]): def __init__(self, color_space: Optional[PDFColorSpace], *values: float):
self.color_space: PDFColorSpace = color_space self.color_space: Optional[PDFColorSpace] = color_space
self.values: list = values self.values: Tuple[float, ...] = values
def copy(self): def copy(self):
return self.__class__( return self.__class__(
@@ -1,3 +1,4 @@
from typing import Dict
from collections import OrderedDict from collections import OrderedDict
class PDFColorSpace(object): class PDFColorSpace(object):
@@ -5,13 +6,12 @@ class PDFColorSpace(object):
def __init__(self, name: str, ncomponents: int): def __init__(self, name: str, ncomponents: int):
self.name: str = name self.name: str = name
self.ncomponents: int = ncomponents self.ncomponents: int = ncomponents
return
def __repr__(self): def __repr__(self):
return '<PDFColorSpace: %s, ncomponents=%d>' % (self.name, self.ncomponents) return '<PDFColorSpace: %s, ncomponents=%d>' % (self.name, self.ncomponents)
PREDEFINED_COLORSPACE = OrderedDict() PREDEFINED_COLORSPACE: Dict[str, PDFColorSpace] = OrderedDict()
for (name, n) in [ for (name, n) in [
('DeviceGray', 1), # default value first ('DeviceGray', 1), # default value first
@@ -2,7 +2,6 @@ from .PDFColor import PDFColor, PDFColorSpace
from .PDFColorSpace import PREDEFINED_COLORSPACE from .PDFColorSpace import PREDEFINED_COLORSPACE
class PDFGraphicState(object): class PDFGraphicState(object):
class InvalidOperation(Exception): pass
def __init__(self): def __init__(self):
self.linewidth = 0 self.linewidth = 0
@@ -58,7 +58,7 @@ class PDFText(PDFItem):
if needcharspace: if needcharspace:
textstate.linematrix[idx] += textstate.charspace textstate.linematrix[idx] += textstate.charspace
text = textstate.font.to_unichr(cid) text = textstate.to_unichr(cid)
assert isinstance(text, str), str(type(text)) assert isinstance(text, str), str(type(text))
matrix = translate_matrix(textstate.matrix, textstate.linematrix) matrix = translate_matrix(textstate.matrix, textstate.linematrix)
@@ -42,7 +42,7 @@ class TextDecoder:
if needcharspace: if needcharspace:
textstate.linematrix[idx] += textstate.charspace textstate.linematrix[idx] += textstate.charspace
text = textstate.font.to_unichr(cid) text = textstate.to_unichr(cid)
assert isinstance(text, str), str(type(text)) assert isinstance(text, str), str(type(text))
matrix = translate_matrix(textstate.matrix, textstate.linematrix) matrix = translate_matrix(textstate.matrix, textstate.linematrix)
@@ -10,10 +10,10 @@ from pdfmajor.parser.PDFStream import resolve1
from pdfmajor.parser.PDFStream import int_value from pdfmajor.parser.PDFStream import int_value
from pdfmajor.parser.PDFStream import num_value from pdfmajor.parser.PDFStream import num_value
from pdfmajor.parser.PDFStream import list_value from pdfmajor.parser.PDFStream import list_value
from pdfmajor.parser.PDFStream import PDFException
from pdfmajor.parser.PDFStream import PDFStream from pdfmajor.parser.PDFStream import PDFStream
from pdfmajor.parser.cmapdb import FileUnicodeMap from pdfmajor.parser.cmapdb import FileUnicodeMap
from pdfmajor.parser.cmapdb import CMapParser from pdfmajor.parser.cmapdb import CMapParser
from .execptions import PDFFontError, PDFUnicodeNotDefined
## Literals ## Literals
## ##
@@ -22,8 +22,6 @@ LITERAL_TYPE1C = LIT('Type1C')
class PDFFont(object): class PDFFont(object):
class PDFFontError(PDFException): pass
class PDFUnicodeNotDefined(PDFFontError): pass
def __init__(self, descriptor, widths, default_width=None): def __init__(self, descriptor, widths, default_width=None):
self.descriptor = descriptor self.descriptor = descriptor
@@ -77,7 +75,7 @@ class PDFFont(object):
except KeyError: except KeyError:
try: try:
return self.widths[self.to_unichr(cid)] * self.hscale return self.widths[self.to_unichr(cid)] * self.hscale
except (KeyError, self.PDFUnicodeNotDefined): except (KeyError, PDFUnicodeNotDefined):
return self.default_width * self.hscale return self.default_width * self.hscale
def char_disp(self, cid): def char_disp(self, cid):
@@ -137,4 +135,4 @@ class PDFSimpleFont(PDFFont):
try: try:
return self.cid2unicode[cid] return self.cid2unicode[cid]
except KeyError: except KeyError:
raise self.PDFUnicodeNotDefined(None, cid) raise PDFUnicodeNotDefined(None, cid)
@@ -4,6 +4,7 @@ from pdfmajor.parser.PDFStream import list_value
from pdfmajor.parser.PDFStream import dict_value from pdfmajor.parser.PDFStream import dict_value
from pdfmajor.parser.PDFStream import resolve1 from pdfmajor.parser.PDFStream import resolve1
from pdfmajor.parser.constants import LITERAL_FONT from pdfmajor.parser.constants import LITERAL_FONT
from .execptions import PDFFontError
from .fonts import PDFType1Font, PDFTrueTypeFont, PDFType3Font, PDFCIDFont, PDFFont from .fonts import PDFType1Font, PDFTrueTypeFont, PDFType3Font, PDFCIDFont, PDFFont
@@ -16,13 +17,13 @@ def get_font(objid: int, spec: dict, cached_fonts: dict = {}):
log.debug('get_font: create: objid=%r, spec=%r', objid, spec) log.debug('get_font: create: objid=%r, spec=%r', objid, spec)
if settings.STRICT: if settings.STRICT:
if spec['Type'] is not LITERAL_FONT: if spec['Type'] is not LITERAL_FONT:
raise PDFFont.PDFFontError('Type is not /Font') raise PDFFontError('Type is not /Font')
# Create a Font object. # Create a Font object.
if 'Subtype' in spec: if 'Subtype' in spec:
subtype = literal_name(spec['Subtype']) subtype = literal_name(spec['Subtype'])
else: else:
if settings.STRICT: if settings.STRICT:
raise PDFFont.PDFFontError('Font Subtype is not specified.') raise PDFFontError('Font Subtype is not specified.')
subtype = 'Type1' subtype = 'Type1'
if subtype in ('Type1', 'MMType1'): if subtype in ('Type1', 'MMType1'):
# Type1 Font # Type1 Font
@@ -47,7 +48,7 @@ def get_font(objid: int, spec: dict, cached_fonts: dict = {}):
font = get_font(None, subspec, cached_fonts) font = get_font(None, subspec, cached_fonts)
else: else:
if settings.STRICT: if settings.STRICT:
raise PDFFont.PDFFontError('Invalid Font spec: %r' % spec) raise PDFFontError('Invalid Font spec: %r' % spec)
font = PDFType1Font(spec) # this is so wrong! font = PDFType1Font(spec) # this is so wrong!
if objid: if objid:
cached_fonts[objid] = font cached_fonts[objid] = font
@@ -0,0 +1,4 @@
from pdfmajor.parser.PDFStream import PDFException
class PDFFontError(PDFException): pass
class PDFUnicodeNotDefined(PDFFontError): pass
@@ -15,6 +15,7 @@ from .PDFFont import PDFFont, PDFSimpleFont
from .util import FontMetricsDB, get_widths, get_widths2 from .util import FontMetricsDB, get_widths, get_widths2
from .Type1FontHeaderParser import Type1FontHeaderParser from .Type1FontHeaderParser import Type1FontHeaderParser
from .TrueTypeFont import TrueTypeFont from .TrueTypeFont import TrueTypeFont
from .execptions import PDFFontError, PDFUnicodeNotDefined
# PDFType1Font # PDFType1Font
class PDFType1Font(PDFSimpleFont): class PDFType1Font(PDFSimpleFont):
@@ -24,7 +25,7 @@ class PDFType1Font(PDFSimpleFont):
self.basefont = literal_name(spec['BaseFont']) self.basefont = literal_name(spec['BaseFont'])
except KeyError: except KeyError:
if settings.STRICT: if settings.STRICT:
raise self.PDFFontError('BaseFont is missing') raise PDFFontError('BaseFont is missing')
self.basefont = 'unknown' self.basefont = 'unknown'
try: try:
(descriptor, widths) = FontMetricsDB.get_metrics(self.basefont) (descriptor, widths) = FontMetricsDB.get_metrics(self.basefont)
@@ -86,7 +87,7 @@ class PDFCIDFont(PDFFont):
self.basefont = literal_name(spec['BaseFont']) self.basefont = literal_name(spec['BaseFont'])
except KeyError: except KeyError:
if strict: if strict:
raise self.PDFFontError('BaseFont is missing') raise PDFFontError('BaseFont is missing')
self.basefont = 'unknown' self.basefont = 'unknown'
self.cidsysteminfo = dict_value(spec.get('CIDSystemInfo', {})) self.cidsysteminfo = dict_value(spec.get('CIDSystemInfo', {}))
self.cidcoding = '%s-%s' % (resolve1(self.cidsysteminfo.get('Registry', b'unknown')).decode("latin1"), self.cidcoding = '%s-%s' % (resolve1(self.cidsysteminfo.get('Registry', b'unknown')).decode("latin1"),
@@ -95,19 +96,19 @@ class PDFCIDFont(PDFFont):
name = literal_name(spec['Encoding']) name = literal_name(spec['Encoding'])
except KeyError: except KeyError:
if strict: if strict:
raise self.PDFFontError('Encoding is unspecified') raise PDFFontError('Encoding is unspecified')
name = 'unknown' name = 'unknown'
try: try:
self.cmap = CMapDB.get_cmap(name) self.cmap = CMapDB.get_cmap(name)
except CMapDB.CMapNotFound as e: except CMapDB.CMapNotFound as e:
if strict: if strict:
raise self.PDFFontError(e) raise PDFFontError(e)
self.cmap = CMap() self.cmap = CMap()
try: try:
descriptor = dict_value(spec['FontDescriptor']) descriptor = dict_value(spec['FontDescriptor'])
except KeyError: except KeyError:
if strict: if strict:
raise self.PDFFontError('FontDescriptor is missing') raise PDFFontError('FontDescriptor is missing')
descriptor = {} descriptor = {}
ttf = None ttf = None
if 'FontFile2' in descriptor: if 'FontFile2' in descriptor:
@@ -171,4 +172,4 @@ class PDFCIDFont(PDFFont):
raise KeyError(cid) raise KeyError(cid)
return self.unicode_map.get_unichr(cid) return self.unicode_map.get_unichr(cid)
except KeyError: except KeyError:
raise self.PDFUnicodeNotDefined(self.cidcoding, cid) raise PDFUnicodeNotDefined(self.cidcoding, cid)
@@ -1,5 +1,6 @@
from .PDFFont import PDFFont, get_font
from pdfmajor.utils import MATRIX_IDENTITY from pdfmajor.utils import MATRIX_IDENTITY
from .PDFFont import PDFFont, get_font
from .PDFFont.execptions import PDFUnicodeNotDefined, PDFFontError
## PDFTextState ## PDFTextState
## ##
@@ -16,14 +17,27 @@ class PDFTextState(object):
self.rise: int = 0 self.rise: int = 0
self.matrix = MATRIX_IDENTITY self.matrix = MATRIX_IDENTITY
self.linematrix = [0, 0] self.linematrix = [0, 0]
self.ignore_bad_chars = False
def __repr__(self): def __repr__(self):
return ('<PDFTextState: font=%r, fontsize=%r, charspace=%r, wordspace=%r, ' return ('<PDFTextState: font=%r, fontsize=%r, charspace=%r, wordspace=%r, '
' scaling=%r, leading=%r, render=%r, rise=%r, ' ' scaling=%r, leading=%r, render=%r, rise=%r, '
' matrix=%r, linematrix=%r>' % ' matrix=%r, linematrix=%r ignore_bad_chars=%r>' %
(self.font, self.fontsize, self.charspace, self.wordspace, (self.font, self.fontsize, self.charspace, self.wordspace,
self.scaling, self.leading, self.render, self.rise, self.scaling, self.leading, self.render, self.rise,
self.matrix, self.linematrix)) self.matrix, self.linematrix, self.ignore_bad_chars))
def to_unichr(self, cid: int) -> str:
if self.font is None:
raise PDFFontError("Missing font")
try:
return self.font.to_unichr(cid)
except PDFUnicodeNotDefined as e:
if self.ignore_bad_chars:
return ""
else:
raise e
def copy(self): def copy(self):
obj = self.__class__() obj = self.__class__()
@@ -37,6 +51,7 @@ class PDFTextState(object):
obj.rise = self.rise obj.rise = self.rise
obj.matrix = self.matrix obj.matrix = self.matrix
obj.linematrix = self.linematrix obj.linematrix = self.linematrix
obj.ignore_bad_chars = self.ignore_bad_chars
return obj return obj
def reset_matrices(self): def reset_matrices(self):
@@ -33,7 +33,7 @@ def __render_string_along(idx: int, seq: bytearray,
if needcharspace: if needcharspace:
textstate.linematrix[idx] += textstate.charspace textstate.linematrix[idx] += textstate.charspace
text = textstate.font.to_unichr(cid) text = textstate.to_unichr(cid)
assert isinstance(text, str), str(type(text)) assert isinstance(text, str), str(type(text))
matrix = translate_matrix(textstate.matrix, textstate.linematrix) matrix = translate_matrix(textstate.matrix, textstate.linematrix)
+3
View File
@@ -0,0 +1,3 @@
from .commands import CommandProcessorError
from .commands.PDFCommands import PDFCommandsRepeatedCommand, PDFCommandsInvalidOperation
from .commands.state.PDFTextState.PDFFont.execptions import *
+4 -2
View File
@@ -1,5 +1,6 @@
import re import re
from logging import getLogger from logging import getLogger
from typing import Tuple, Optional, Dict
from ...utils import settings, choplist, nunpack from ...utils import settings, choplist, nunpack
from ..PSStackParser import PSEOF from ..PSStackParser import PSEOF
@@ -13,6 +14,7 @@ from .exceptions import PDFNoValidXRef, PDFSyntaxError
log = getLogger(__name__) log = getLogger(__name__)
Pos = Tuple[Optional[int], int, int]
## XRefs ## XRefs
## ##
@@ -27,7 +29,7 @@ class PDFBaseXRef(object):
# Must return # Must return
# (strmid, index, genno) # (strmid, index, genno)
# or (None, pos, genno) # or (None, pos, genno)
def get_pos(self, objid): def get_pos(self, objid) -> Pos:
raise KeyError(objid) raise KeyError(objid)
@@ -36,7 +38,7 @@ class PDFBaseXRef(object):
class PDFXRef(PDFBaseXRef): class PDFXRef(PDFBaseXRef):
def __init__(self): def __init__(self):
self.offsets = {} self.offsets: Dict[int, Pos] = {}
self.trailer = {} self.trailer = {}
return return
+5
View File
@@ -0,0 +1,5 @@
from .cmapdb import CMapError
from .PDFStream.exceptions import *
from .PDFDocument.exceptions import *
from .PSStackParser.exceptions import *
from .PDFParser import PDFSyntaxError
Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 576 KiB

Binary file not shown.
Binary file not shown.
+10 -3
View File
@@ -10,12 +10,19 @@ import sys
with open('README.md') as readme_file: with open('README.md') as readme_file:
SETUP = { SETUP = {
"name": "pdfmajor", "name": "pdfmajor",
"version": "1.3.6", "version": "1.3.12",
"packages": find_packages(include=['pdfmajor*']), "packages": find_packages(include=['pdfmajor*']),
"install_requires": [ "install_requires": [
'pycryptodome', 'pycryptodome==3.8.2',
'chardet' 'chardet==3.0.4',
'Pillow>=6.2.0'
], ],
'extras_require': {
'dev': [
'tqdm==4.32.1',
'mkdocs==1.0.4'
]
},
"description": 'PDF parser', "description": 'PDF parser',
"long_description": readme_file.read(), "long_description": readme_file.read(),
'long_description_content_type': 'text/markdown', 'long_description_content_type': 'text/markdown',
+4
View File
@@ -0,0 +1,4 @@
from unittest import TestLoader, TextTestRunner
tests = TestLoader().discover('./tests')
result = TextTestRunner(verbosity=20).run(tests)
Binary file not shown.
Binary file not shown.
BIN
View File
Binary file not shown.
+23
View File
@@ -0,0 +1,23 @@
from pathlib import Path
from unittest import TestCase, main
from pdfmajor.interpreter import PDFInterpreter, exceptions
CURRENT_FOLDER = Path("./")
class BadUnicode(TestCase):
def test_ensure_failure(self):
pdf = PDFInterpreter(CURRENT_FOLDER / "tests" / "samples" / "pdf" / "bad-unicode.pdf", ignore_bad_chars=False)
pages = list(pdf)
with self.assertRaises(exceptions.PDFUnicodeNotDefined):
list(pages[2])
def test_ensure_no_failure(self):
pdf = PDFInterpreter(CURRENT_FOLDER / "tests" / "samples" / "pdf" / "bad-unicode.pdf", ignore_bad_chars=True)
pages = list(pdf)
list(pages[2])
if __name__ == '__main__':
# Run Tests
main()
+10 -7
View File
@@ -1,6 +1,5 @@
import os import os
from tqdm import tqdm from unittest import TestCase, main
# from pdfmajor.extractor import extract_items_from_pdf # from pdfmajor.extractor import extract_items_from_pdf
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
from pdfmajor.interpreter import PageInterpreter from pdfmajor.interpreter import PageInterpreter
@@ -15,7 +14,7 @@ NUM_LOOPS = 10
def count_elms(file_path: str): def count_elms(file_path: str):
count = 0 count = 0
for page in PDFInterpreter(file_path): for page in PDFInterpreter(file_path, ignore_bad_chars=True):
for item in page: for item in page:
if isinstance(item, LTContainer): if isinstance(item, LTContainer):
count += len([*item]) count += len([*item])
@@ -23,13 +22,17 @@ def count_elms(file_path: str):
count += 1 count += 1
return count return count
with tqdm(desc='Cache Test', total=len(FILES)*NUM_LOOPS) as waiter: class CachingTester(TestCase):
def test_cache(self):
for file_name in FILES: for file_name in FILES:
waiter.set_description(f"Cache Test - [{file_name}]") self.subTest(name = file_name)
file_path = os.path.join(INPUT_FOLDER, file_name) file_path = os.path.join(INPUT_FOLDER, file_name)
last_count = count_elms(file_path) last_count = count_elms(file_path)
for _ in range(NUM_LOOPS): for _ in range(NUM_LOOPS):
new_count = count_elms(file_path) new_count = count_elms(file_path)
assert new_count == last_count self.assertEqual( new_count , last_count )
last_count = new_count last_count = new_count
waiter.update()
if __name__ == '__main__':
# Run Tests
main()
+115
View File
@@ -0,0 +1,115 @@
import os
from typing import List
from unittest import TestCase, main
# from pdfmajor.extractor import extract_items_from_pdf
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
from pdfmajor.interpreter import PageInterpreter
from pdfmajor.interpreter.commands import LTCurve, PDFColor, LTXObject
from pdfmajor.interpreter.commands import PREDEFINED_COLORSPACE
CUR_PATH = os.path.dirname(os.path.dirname(__file__))
INPUT_FOLDER = os.path.join(
CUR_PATH, "tests/samples/pdf"
)
FILES = os.listdir(INPUT_FOLDER)
NUM_LOOPS = 10
class ColorTest(TestCase):
def run_test(self, file_path: str, curve_defs: List[List[PDFColor]]):
def run_once(item):
if isinstance(item, LTCurve):
t_stroke, t_fill = curve_defs.pop(0)
self.assertEqual( item.stroke.values, t_stroke.values )
self.assertEqual( item.stroke.color_space, t_stroke.color_space )
self.assertEqual( item.fill.values, t_fill.values )
self.assertEqual( item.fill.color_space, t_fill.color_space )
elif isinstance(item, LTXObject):
for c in item:
run_once(c)
for page in PDFInterpreter(file_path):
for item in page:
run_once(item)
def test_lorem_v3(self):
self.run_test(
os.path.join(INPUT_FOLDER, "lorem-v3.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 0),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 114, 159, 207),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 0, 0),
]
] +
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]*24)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*11)+
([[
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 0, 0, 0),
]]*5)+
[[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
]]
)
def test_lorem_v4(self):
self.run_test(
os.path.join(INPUT_FOLDER, "lorem-v4.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 64, 64, 64),
]
]
)
def test_lorem_v5(self):
self.run_test(
os.path.join(INPUT_FOLDER, "lorem-v5.pdf"),
[
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 255, 255, 255),
],
[
PDFColor(None),
PDFColor(PREDEFINED_COLORSPACE['DeviceRGB'], 128, 128, 128),
]
]
)
if __name__ == '__main__':
# Run Tests
main()
+20 -8
View File
@@ -3,13 +3,14 @@ import re
import os import os
import logging import logging
from unittest import TestCase, main
from typing import List from typing import List
from pdfmajor.converters import convert_file from pdfmajor.converters import convert_file
from xml.etree import ElementTree from xml.etree import ElementTree
from json import load as json_load from json import load as json_load
from tqdm import tqdm from tqdm import tqdm
def rel_path(*arg: List[str]) -> str: def rel_path(*arg: str) -> str:
return os.path.normpath(os.path.join(os.path.dirname(__file__), *arg)) return os.path.normpath(os.path.join(os.path.dirname(__file__), *arg))
EXTS = ['text', 'json','xml','html'] EXTS = ['text', 'json','xml','html']
@@ -22,8 +23,12 @@ if not os.path.exists(OUTPUT_FOLDER):
files = os.listdir(INPUT_FOLDER) files = os.listdir(INPUT_FOLDER)
TOTAL = len(files)*len(EXTS) TOTAL = len(files)*len(EXTS)
with tqdm(total=TOTAL) as waiter:
class ConversionTest(TestCase):
def test_conversion(self):
# with tqdm(total=TOTAL) as waiter:
for idx, file_name in enumerate(files): for idx, file_name in enumerate(files):
self.subTest(i=idx);
file_name_no_ext = re.sub(r'\.pdf$',"", file_name) file_name_no_ext = re.sub(r'\.pdf$',"", file_name)
input_file_path = os.path.join(INPUT_FOLDER, file_name) input_file_path = os.path.join(INPUT_FOLDER, file_name)
@@ -32,18 +37,25 @@ with tqdm(total=TOTAL) as waiter:
os.makedirs(os.path.join(OUTPUT_FOLDER, ext)) os.makedirs(os.path.join(OUTPUT_FOLDER, ext))
output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext) output_file_path = os.path.join(OUTPUT_FOLDER, ext, file_name_no_ext + "." + ext)
if len(file_name) > len(file_name_no_ext): if len(file_name) > len(file_name_no_ext):
waiter.set_description(f"Conversion test -- {file_name}") # waiter.set_description(f"Conversion test -- {file_name}")
convert_file( convert_file(
input_file=input_file_path, input_file=input_file_path,
output_file=output_file_path, output_file=output_file_path,
out_type=ext, out_type=ext,
ignore_bad_chars=True,
debug_level=logging.WARN, debug_level=logging.WARN,
) )
if ext == 'xml': if ext == 'xml':
waiter.set_description(f" > testing format {file_name}") # waiter.set_description(f" > testing format {file_name}")
ElementTree.parse(output_file_path) ElementTree.parse(output_file_path)
elif ext == 'json': elif ext == 'json':
waiter.set_description(f" > testing format {file_name}") # waiter.set_description(f" > testing format {file_name}")
json_load(open(output_file_path, 'r')) with open(output_file_path, 'r') as outf:
waiter.update() json_load(outf)
waiter.set_description("Done") # waiter.update()
# waiter.set_description("Done")
if __name__ == '__main__':
# Run Tests
main()
+30 -8
View File
@@ -1,33 +1,50 @@
from contextlib import contextmanager
from unittest import TestCase, main
from typing import List, Tuple, Callable
from os import remove from os import remove
from tempfile import mktemp from tempfile import mktemp
from pdfmajor.converters.writers.html import HTMLMaker from pdfmajor.converters.writers.html import HTMLMaker
from tqdm import tqdm
# Make temp file # Make temp file
FILE_NAME = mktemp() FILE_NAME = mktemp()
TestFunction = Callable[[HTMLMaker], str];
TESTS: List[Tuple[TestFunction, str]] = [];
class HTMLTester(TestCase):
@contextmanager
def get_html(self):
with HTMLMaker(FILE_NAME) as html: with HTMLMaker(FILE_NAME) as html:
yield html
def compare(self, expected: str):
with open(FILE_NAME, 'r') as f:
self.assertEqual(f.read() , expected)
def test_html_basic(self):
with self.get_html() as html:
with html.elm("html", nolineend=True): with html.elm("html", nolineend=True):
html.write("test") html.write("test")
assert(open(FILE_NAME, 'r').read() == """<html > self.compare("""<html >
test test
</html>""") </html>""")
def test_html_with_attributes(self):
with HTMLMaker(FILE_NAME) as html: with self.get_html() as html:
with html.elm("html", nolineend=True): with html.elm("html", nolineend=True):
with html.elm("head"): with html.elm("head"):
html.singleton("meta", { html.singleton("meta", {
'http-equiv': "Content-Type", 'http-equiv': "Content-Type",
'content': "text/html" 'content': "text/html"
}) })
assert(open(FILE_NAME, 'r').read() == """<html > self.compare("""<html >
<head > <head >
<meta http-equiv="Content-Type" content="text/html" /> <meta http-equiv="Content-Type" content="text/html" />
</head> </head>
</html>""") </html>""")
with HTMLMaker(FILE_NAME) as html: def test_html_with_children(self):
with self.get_html() as html:
with html.elm("html", nolineend=True): with html.elm("html", nolineend=True):
with html.elm("head"): with html.elm("head"):
html.singleton("meta", { html.singleton("meta", {
@@ -37,7 +54,7 @@ with HTMLMaker(FILE_NAME) as html:
with html.elm("body"): with html.elm("body"):
with html.elm("h1", css={"color": "red"}): with html.elm("h1", css={"color": "red"}):
html.write("Hello World") html.write("Hello World")
assert(open(FILE_NAME, 'r').read() == """<html > self.compare("""<html >
<head > <head >
<meta http-equiv="Content-Type" content="text/html" /> <meta http-equiv="Content-Type" content="text/html" />
</head> </head>
@@ -48,13 +65,18 @@ assert(open(FILE_NAME, 'r').read() == """<html >
</body> </body>
</html>""") </html>""")
with HTMLMaker(FILE_NAME) as html: def test_html_with_css(self):
with self.get_html() as html:
with html.elm('style'): with html.elm('style'):
html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''') html.write('''.page { position: relative; overflow: hidden;border: 1px black solid;}''')
assert(open(FILE_NAME, 'r').read() == """<style > self.compare("""<style >
.page { position: relative; overflow: hidden;border: 1px black solid;} .page { position: relative; overflow: hidden;border: 1px black solid;}
</style> </style>
""") """)
if __name__ == '__main__':
# Run Tests
main()
# Clean up # Clean up
remove(FILE_NAME) remove(FILE_NAME)
+11 -4
View File
@@ -1,5 +1,5 @@
import os import os
from tqdm import tqdm from unittest import TestCase, main
from pdfmajor.interpreter import LTImage, logging from pdfmajor.interpreter import LTImage, logging
from pdfmajor.interpreter import PageInterpreter, PDFInterpreter from pdfmajor.interpreter import PageInterpreter, PDFInterpreter
@@ -19,16 +19,23 @@ OUTPUT_FOLDER = os.path.join(
if not os.path.exists(OUTPUT_FOLDER): if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER) os.makedirs(OUTPUT_FOLDER)
for file_name in tqdm(FILES, desc='Image Writer Test'): class ImageTester(TestCase):
def test_image_extract(self):
for file_name in FILES:
self.subTest(file_name=file_name)
for page in PDFInterpreter(os.path.join( for page in PDFInterpreter(os.path.join(
CUR_PATH, CUR_PATH,
f"tests/samples/pdf/{file_name}.pdf" f"tests/samples/pdf/{file_name}.pdf"
), debug_level=logging.ERROR): ), debug_level=logging.ERROR):
assert isinstance(page, PageInterpreter) self.assertTrue( isinstance(page, PageInterpreter) )
for item_idx, item in enumerate(page): for item_idx, item in enumerate(page):
assert isinstance(item, LTItem) self.assertTrue( isinstance(item, LTItem) )
if isinstance(item, LTImage): if isinstance(item, LTImage):
item.save_image( item.save_image(
OUTPUT_FOLDER, OUTPUT_FOLDER,
file_name=f'{file_name}_p{page.page_num}i{item_idx}' file_name=f'{file_name}_p{page.page_num}i{item_idx}'
) )
if __name__ == '__main__':
# Run Tests
main()
+13 -6
View File
@@ -1,5 +1,5 @@
import os import os
from tqdm import tqdm from unittest import TestCase, main
from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging from pdfmajor.interpreter import LTCharBlock, PDFInterpreter, logging
from pdfmajor.interpreter import PageInterpreter from pdfmajor.interpreter import PageInterpreter
from pdfmajor.interpreter.commands import LTItem from pdfmajor.interpreter.commands import LTItem
@@ -11,12 +11,19 @@ INPUT_FOLDER = os.path.join(
) )
FILES = os.listdir(INPUT_FOLDER) FILES = os.listdir(INPUT_FOLDER)
for file_name in tqdm(FILES, desc="Interpreter Test"): class TestInterpreter(TestCase):
def test_pdfintp(self):
for file_name in FILES:
self.subTest(file_name=file_name)
file_path = os.path.join(INPUT_FOLDER, file_name) file_path = os.path.join(INPUT_FOLDER, file_name)
for page in PDFInterpreter(file_path, debug_level=logging.ERROR): for page in PDFInterpreter(file_path, debug_level=logging.ERROR, ignore_bad_chars=True):
assert isinstance(page, PageInterpreter) self.assertTrue( isinstance(page, PageInterpreter) )
for item in page: for item in page:
assert isinstance(item, LTItem) self.assertTrue( isinstance(item, LTItem) )
if isinstance(item, LTCharBlock): if isinstance(item, LTCharBlock):
for char in item: for char in item:
assert isinstance(char, LTChar) self.assertTrue( isinstance(char, LTChar) )
if __name__ == '__main__':
# Run Tests
main()
+44 -49
View File
@@ -2,17 +2,12 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from nose.tools import assert_equal, assert_true, assert_false import unittest
from nose import SkipTest
import nose
import logging
from pdfmajor.parser.PDFStream.ccitt import CCITTG4Parser from pdfmajor.parser.PDFStream.ccitt import CCITTG4Parser
## Test cases ## Test cases
## ##
class TestCCITTG4Parser(): class TestCCITTG4Parser(unittest.TestCase):
def get_parser(self, bits): def get_parser(self, bits):
parser = CCITTG4Parser(len(bits)) parser = CCITTG4Parser(len(bits))
@@ -23,98 +18,98 @@ class TestCCITTG4Parser():
def test_b1(self): def test_b1(self):
parser = self.get_parser('00000') parser = self.get_parser('00000')
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 0) self.assertEqual(parser._curpos, 0)
return return
def test_b2(self): def test_b2(self):
parser = self.get_parser('10000') parser = self.get_parser('10000')
parser._do_vertical(-1) parser._do_vertical(-1)
assert_equal(parser._curpos, 0) self.assertEqual(parser._curpos, 0)
return return
def test_b3(self): def test_b3(self):
parser = self.get_parser('000111') parser = self.get_parser('000111')
parser._do_pass() parser._do_pass()
assert_equal(parser._curpos, 3) self.assertEqual(parser._curpos, 3)
assert_equal(parser._get_bits(), '111') self.assertEqual(parser._get_bits(), '111')
return return
def test_b4(self): def test_b4(self):
parser = self.get_parser('00000') parser = self.get_parser('00000')
parser._do_vertical(+2) parser._do_vertical(+2)
assert_equal(parser._curpos, 2) self.assertEqual(parser._curpos, 2)
assert_equal(parser._get_bits(), '11') self.assertEqual(parser._get_bits(), '11')
return return
def test_b5(self): def test_b5(self):
parser = self.get_parser('11111111100') parser = self.get_parser('11111111100')
parser._do_horizontal(0, 3) parser._do_horizontal(0, 3)
assert_equal(parser._curpos, 3) self.assertEqual(parser._curpos, 3)
parser._do_vertical(1) parser._do_vertical(1)
assert_equal(parser._curpos, 10) self.assertEqual(parser._curpos, 10)
assert_equal(parser._get_bits(), '0001111111') self.assertEqual(parser._get_bits(), '0001111111')
return return
def test_e1(self): def test_e1(self):
parser = self.get_parser('10000') parser = self.get_parser('10000')
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 5) self.assertEqual(parser._curpos, 5)
assert_equal(parser._get_bits(), '10000') self.assertEqual(parser._get_bits(), '10000')
return return
def test_e2(self): def test_e2(self):
parser = self.get_parser('10011') parser = self.get_parser('10011')
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(2) parser._do_vertical(2)
assert_equal(parser._curpos, 5) self.assertEqual(parser._curpos, 5)
assert_equal(parser._get_bits(), '10000') self.assertEqual(parser._get_bits(), '10000')
return return
def test_e3(self): def test_e3(self):
parser = self.get_parser('011111') parser = self.get_parser('011111')
parser._color = 0 parser._color = 0
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._color, 1) self.assertEqual(parser._color, 1)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(-2) parser._do_vertical(-2)
assert_equal(parser._color, 0) self.assertEqual(parser._color, 0)
assert_equal(parser._curpos, 4) self.assertEqual(parser._curpos, 4)
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 6) self.assertEqual(parser._curpos, 6)
assert_equal(parser._get_bits(), '011100') self.assertEqual(parser._get_bits(), '011100')
return return
def test_e4(self): def test_e4(self):
parser = self.get_parser('10000') parser = self.get_parser('10000')
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(-2) parser._do_vertical(-2)
assert_equal(parser._curpos, 3) self.assertEqual(parser._curpos, 3)
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 5) self.assertEqual(parser._curpos, 5)
assert_equal(parser._get_bits(), '10011') self.assertEqual(parser._get_bits(), '10011')
return return
def test_e5(self): def test_e5(self):
parser = self.get_parser('011000') parser = self.get_parser('011000')
parser._color = 0 parser._color = 0
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 1) self.assertEqual(parser._curpos, 1)
parser._do_vertical(3) parser._do_vertical(3)
assert_equal(parser._curpos, 6) self.assertEqual(parser._curpos, 6)
assert_equal(parser._get_bits(), '011111') self.assertEqual(parser._get_bits(), '011111')
return return
def test_e6(self): def test_e6(self):
parser = self.get_parser('11001') parser = self.get_parser('11001')
parser._do_pass() parser._do_pass()
assert_equal(parser._curpos, 4) self.assertEqual(parser._curpos, 4)
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 5) self.assertEqual(parser._curpos, 5)
assert_equal(parser._get_bits(), '11111') self.assertEqual(parser._get_bits(), '11111')
return return
def test_e7(self): def test_e7(self):
@@ -122,8 +117,8 @@ class TestCCITTG4Parser():
parser._curpos = 2 parser._curpos = 2
parser._color = 1 parser._color = 1
parser._do_horizontal(2, 6) parser._do_horizontal(2, 6)
assert_equal(parser._curpos, 10) self.assertEqual(parser._curpos, 10)
assert_equal(parser._get_bits(), '1111000000') self.assertEqual(parser._get_bits(), '1111000000')
return return
def test_e8(self): def test_e8(self):
@@ -131,19 +126,19 @@ class TestCCITTG4Parser():
parser._curpos = 1 parser._curpos = 1
parser._color = 0 parser._color = 0
parser._do_vertical(0) parser._do_vertical(0)
assert_equal(parser._curpos, 2) self.assertEqual(parser._curpos, 2)
parser._do_horizontal(7, 0) parser._do_horizontal(7, 0)
assert_equal(parser._curpos, 9) self.assertEqual(parser._curpos, 9)
assert_equal(parser._get_bits(), '101111111') self.assertEqual(parser._get_bits(), '101111111')
return return
def test_m1(self): def test_m1(self):
parser = self.get_parser('10101') parser = self.get_parser('10101')
parser._do_pass() parser._do_pass()
assert_equal(parser._curpos, 2) self.assertEqual(parser._curpos, 2)
parser._do_pass() parser._do_pass()
assert_equal(parser._curpos, 4) self.assertEqual(parser._curpos, 4)
assert_equal(parser._get_bits(), '1111') self.assertEqual(parser._get_bits(), '1111')
return return
def test_m2(self): def test_m2(self):
@@ -152,7 +147,7 @@ class TestCCITTG4Parser():
parser._do_vertical(-1) parser._do_vertical(-1)
parser._do_vertical(1) parser._do_vertical(1)
parser._do_horizontal(1, 1) parser._do_horizontal(1, 1)
assert_equal(parser._get_bits(), '011101') self.assertEqual(parser._get_bits(), '011101')
return return
def test_m3(self): def test_m3(self):
@@ -161,8 +156,8 @@ class TestCCITTG4Parser():
parser._do_pass() parser._do_pass()
parser._do_vertical(1) parser._do_vertical(1)
parser._do_vertical(1) parser._do_vertical(1)
assert_equal(parser._get_bits(), '00000001') self.assertEqual(parser._get_bits(), '00000001')
return return
if __name__ == '__main__': if __name__ == '__main__':
nose.runmodule() unittest.main()
+12 -14
View File
@@ -1,9 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from nose.tools import assert_equal from unittest import TestCase, main
from nose import SkipTest
import nose
#test of various compression/encoding modules (previously in doctests): #test of various compression/encoding modules (previously in doctests):
from pdfmajor.utils.ascii85 import ascii85decode, asciihexdecode from pdfmajor.utils.ascii85 import ascii85decode, asciihexdecode
@@ -14,24 +12,24 @@ import binascii
def hex(b): return binascii.hexlify(b) #encode('hex') def hex(b): return binascii.hexlify(b) #encode('hex')
def dehex(b): return binascii.unhexlify(b) #decode('hex') def dehex(b): return binascii.unhexlify(b) #decode('hex')
class TestAscii85(): class TestAscii85(TestCase):
def test_ascii85decode(self): def test_ascii85decode(self):
#The sample string is taken from: http://en.wikipedia.org/w/index.php?title=Ascii85 #The sample string is taken from: http://en.wikipedia.org/w/index.php?title=Ascii85
assert_equal(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished') self.assertEqual(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished')
assert_equal(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.') self.assertEqual(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.')
def test_asciihexdecode(self): def test_asciihexdecode(self):
assert_equal(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde') self.assertEqual(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde')
assert_equal(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep') self.assertEqual(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep')
assert_equal(asciihexdecode(b'7>'),b'p') self.assertEqual(asciihexdecode(b'7>'),b'p')
class TestLzw(): class TestLzw(TestCase):
def test_lzwdecode(self): def test_lzwdecode(self):
assert_equal(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42') self.assertEqual(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42')
class TestRunlength(): class TestRunlength(TestCase):
def test_rldecode(self): def test_rldecode(self):
assert_equal(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde') self.assertEqual(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde')
if __name__ == '__main__': if __name__ == '__main__':
nose.runmodule() main()
+8 -19
View File
@@ -2,18 +2,13 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from nose.tools import assert_equal, assert_true, assert_false
from nose import SkipTest
import nose
import logging import logging
from unittest import TestCase, main
from io import BytesIO
from pdfmajor.parser.PSStackParser import KWD, LIT, PSBaseParser, PSEOF, PSStackParser from pdfmajor.parser.PSStackParser import KWD, LIT, PSBaseParser, PSEOF, PSStackParser
## Simplistic Test cases ## Simplistic Test cases
## ##
class TestPSBaseParser:
TESTDATA = br'''%!PS TESTDATA = br'''%!PS
begin end begin end
" @ # " @ #
@@ -33,7 +28,6 @@ func/a/b{(c)do*}def
[ 1 (z) ! ] [ 1 (z) ! ]
<< /foo (bar) >> << /foo (bar) >>
''' '''
TOKENS = [ TOKENS = [
(5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')), (5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')),
(21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), (21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
@@ -48,7 +42,6 @@ func/a/b{(c)do*}def
(256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'), (256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'),
(272, KWD(b'>>')) (272, KWD(b'>>'))
] ]
OBJS = [ OBJS = [
(23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
@@ -59,10 +52,8 @@ func/a/b{(c)do*}def
(230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']), (230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']),
(258, {'foo': b'bar'}), (258, {'foo': b'bar'}),
] ]
class TestPSBaseParser(TestCase):
def get_tokens(self, s): def get_tokens(self, s):
from io import BytesIO
class MyParser(PSStackParser): class MyParser(PSStackParser):
def flush(self): def flush(self):
self.add_results(*self.popall()) self.add_results(*self.popall())
@@ -76,8 +67,6 @@ func/a/b{(c)do*}def
return r return r
def get_objects(self, s): def get_objects(self, s):
from io import BytesIO
class MyParser(PSStackParser): class MyParser(PSStackParser):
def flush(self): def flush(self):
self.add_results(*self.popall()) self.add_results(*self.popall())
@@ -91,18 +80,18 @@ func/a/b{(c)do*}def
return r return r
def test_1(self): def test_1(self):
tokens = self.get_tokens(self.TESTDATA) tokens = self.get_tokens(TESTDATA)
logging.info(tokens) logging.info(tokens)
assert_equal(tokens, self.TOKENS) self.assertEqual(tokens, TOKENS)
return return
def test_2(self): def test_2(self):
objs = self.get_objects(self.TESTDATA) objs = self.get_objects(TESTDATA)
logging.info(objs) logging.info(objs)
assert_equal(objs, self.OBJS) self.assertEqual(objs, OBJS)
return return
if __name__ == '__main__': if __name__ == '__main__':
#import logging,sys,os,six #import logging,sys,os,six
#logging.basicConfig(level=logging.DEBUG, filename='%s_%d.%d.log'%(os.path.basename(__file__),sys.version_info[0],sys.version_info[1])) #logging.basicConfig(level=logging.DEBUG, filename='%s_%d.%d.log'%(os.path.basename(__file__),sys.version_info[0],sys.version_info[1]))
nose.runmodule() main()