new attempt for parser

- now support clear indirect objects
This commit is contained in:
ari committed 2020-10-19 15:00:31 -04:00
1 parent b802f782fa
commit 07244aaf11
18 files changed
+606 -514

No files matched your search

+21 -23
View File
@@ -5,7 +5,6 @@ from typing import List
from unittest import TestCase
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.exceptions import LexerError
from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
from pdfmajor.lexer.string import parse_string
@@ -13,11 +12,9 @@ from pdfmajor.lexer.name import parse_name
from pdfmajor.lexer.keyword import parse_keyword
from pdfmajor.streambuffer import BufferStream
from pdfmajor.lexer.token import (
PDFName,
TArrayValue,
TDictValue,
TArrayValue,
Token,
TokenArray,
TokenBoolean,
TokenComment,
TokenNull,
@@ -27,6 +24,7 @@ from pdfmajor.lexer.token import (
TokenInteger,
TokenKeyword,
TokenName,
TokenArray,
TokenString,
)
from pdfmajor.lexer.comment import parse_comment
@@ -114,7 +112,7 @@ class Basics(TestCase):
token = parse_name(buffer)
self.assertIsInstance(token, TokenName)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, PDFName("Some_Name"))
self.assertEqual(token.value, "Some_Name")
self.assertEqual(token.end_loc, 9)
def test_parse_literal_names_hex(self):
@@ -124,7 +122,7 @@ class Basics(TestCase):
token = parse_name(buffer)
self.assertIsInstance(token, TokenName)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, PDFName("foo_aa"))
self.assertEqual(token.value, "foo_aa")
self.assertEqual(token.end_loc, 8)
def test_parse_string(self):
@@ -246,7 +244,7 @@ class Lexer(TestCase):
self.run_test(
b"/Length 4 0 R\n",
expected=[
TokenName(0, 7, PDFName("Length")),
TokenName(0, 7, "Length"),
TokenInteger(8, 9, 4),
TokenInteger(10, 11, 0),
TokenKeyword(12, 13, b"R"),
@@ -299,10 +297,10 @@ func/a/b{(c)do*}def
TokenKeyword(start_loc=16, end_loc=17, value=b'"'),
TokenKeyword(start_loc=19, end_loc=20, value=b"@"),
TokenKeyword(start_loc=21, end_loc=22, value=b"#"),
TokenName(start_loc=23, end_loc=25, value=PDFName("a")),
TokenName(start_loc=25, end_loc=29, value=PDFName("BCD")),
TokenName(start_loc=30, end_loc=40, value=PDFName("Some_Name")),
TokenName(start_loc=41, end_loc=53, value=PDFName("foo_naa")),
TokenName(start_loc=23, end_loc=25, value="a"),
TokenName(start_loc=25, end_loc=29, value="BCD"),
TokenName(start_loc=30, end_loc=40, value="Some_Name"),
TokenName(start_loc=41, end_loc=53, value="foo_naa"),
TokenInteger(start_loc=54, end_loc=55, value=0),
TokenInteger(start_loc=56, end_loc=58, value=1),
TokenInteger(start_loc=59, end_loc=61, value=-2),
@@ -326,8 +324,8 @@ func/a/b{(c)do*}def
start_loc=211, end_loc=225, value=b"\xab\xcd\x00\x124\x05"
),
TokenKeyword(start_loc=226, end_loc=230, value=b"func"),
TokenName(start_loc=230, end_loc=232, value=PDFName("a")),
TokenName(start_loc=232, end_loc=234, value=PDFName("b")),
TokenName(start_loc=230, end_loc=232, value="a"),
TokenName(start_loc=232, end_loc=234, value="b"),
TokenKeyword(start_loc=234, end_loc=235, value=b"{"),
TokenString(start_loc=235, end_loc=238, value="c"),
TokenKeyword(start_loc=238, end_loc=241, value=b"do*"),
@@ -339,7 +337,7 @@ func/a/b{(c)do*}def
TokenKeyword(start_loc=254, end_loc=255, value=b"!"),
TokenArray(start_loc=256, end_loc=257, value=TArrayValue.CLOSE),
TokenDictionary(start_loc=258, end_loc=260, value=TDictValue.OPEN),
TokenName(start_loc=261, end_loc=265, value=PDFName("foo")),
TokenName(start_loc=261, end_loc=265, value="foo"),
TokenString(start_loc=266, end_loc=271, value="bar"),
TokenDictionary(start_loc=272, end_loc=274, value=TDictValue.CLOSE),
],
@@ -382,15 +380,15 @@ func/a/b
TokenKeyword(start_loc=44, end_loc=45, value=b'"'),
TokenKeyword(start_loc=47, end_loc=48, value=b"@"),
TokenKeyword(start_loc=49, end_loc=50, value=b"#"),
TokenName(start_loc=51, end_loc=53, value=PDFName("a")),
TokenName(start_loc=53, end_loc=57, value=PDFName("BCD")),
TokenName(start_loc=58, end_loc=68, value=PDFName("Some_Name")),
TokenName(start_loc=69, end_loc=81, value=PDFName("foo_naa")),
TokenName(start_loc=82, end_loc=89, value=PDFName("barj")),
TokenName(start_loc=90, end_loc=95, value=PDFName("need")),
TokenName(start_loc=96, end_loc=102, value=PDFName("oh1oh")),
TokenName(start_loc=51, end_loc=53, value="a"),
TokenName(start_loc=53, end_loc=57, value="BCD"),
TokenName(start_loc=58, end_loc=68, value="Some_Name"),
TokenName(start_loc=69, end_loc=81, value="foo_naa"),
TokenName(start_loc=82, end_loc=89, value="barj"),
TokenName(start_loc=90, end_loc=95, value="need"),
TokenName(start_loc=96, end_loc=102, value="oh1oh"),
TokenKeyword(start_loc=103, end_loc=107, value=b"func"),
TokenName(start_loc=107, end_loc=109, value=PDFName("a")),
TokenName(start_loc=109, end_loc=111, value=PDFName("b")),
TokenName(start_loc=107, end_loc=109, value="a"),
TokenName(start_loc=109, end_loc=111, value="b"),
],
)
+89 -70
View File
@@ -1,18 +1,19 @@
from decimal import Decimal
import io
from typing import Callable, List
from pdfmajor.lexer.token import TokenInteger, TokenKeyword, TokenString
from pdfmajor.parser_v2.indirect_objects import IndirectObject, IndirectObjectCollection
from pdfmajor.parser_v2.l2 import PDFL2Parser
from pdfmajor.parser_v2.objects import PDFComment, PDFName, PDFObject, PDFPrimitive
from pdfmajor.parser_v2.l1 import PDFL1Parser
from pdfmajor.streambuffer import StreamEOF
from decimal import Decimal
from unittest import TestCase
from pdfmajor.streambuffer import StreamEOF
from pdfmajor.parser_v3.objects.indirect import IndirectObject, IndirectObjectCollection
from pdfmajor.parser_v3.objects.base import PDFObject
from pdfmajor.parser_v3 import PDFParser
from pdfmajor.parser_v3.objects.comment import PDFComment
class L1(TestCase):
class Collections(TestCase):
def test_parse_dict(self):
parser = PDFL1Parser(
parser = PDFParser(
io.BytesIO(
br"""<< /Type /Example
/Subtype /DictionaryExample
@@ -25,97 +26,65 @@ class L1(TestCase):
/LastItem ( not! )
/VeryLastItem ( OK ) >>
>>"""
)
),
strict=True,
)
obj = next(parser.iter_objects())
self.assertDictEqual(
obj,
obj.to_python(),
{
PDFName("Type"): PDFName("Example"),
PDFName("Subtype"): PDFName("DictionaryExample"),
PDFName("Version"): Decimal("0.01"),
PDFName("IntegerItem"): 12,
PDFName("StringItem"): " a string ",
PDFName("Subdictionary"): {
PDFName("Item1"): Decimal("0.4"),
PDFName("Item2"): True,
PDFName("LastItem"): " not! ",
PDFName("VeryLastItem"): " OK ",
"Type": "/Example",
"Subtype": "/DictionaryExample",
"Version": Decimal("0.01"),
"IntegerItem": 12,
"StringItem": " a string ",
"Subdictionary": {
"Item1": Decimal("0.4"),
"Item2": True,
"LastItem": " not! ",
"VeryLastItem": " OK ",
},
},
)
def test_parse_array(self):
parser = PDFL1Parser(
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]""")
parser = PDFParser(
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]"""),
strict=True,
)
obj = next(parser.iter_objects())
self.assertListEqual(
obj, [549, Decimal("3.14"), False, " Ralph ", PDFName("SomeName")]
obj.to_python(),
[549, Decimal("3.14"), False, " Ralph ", "/SomeName"],
)
def test_parse_nested_array(self):
parser = PDFL1Parser(
parser = PDFParser(
io.BytesIO(
br"""[ [10 30] [22 33] [
<< /x 0 /y 5 >>
<< /x 10 /y 5 >>
] ]"""
)
),
strict=True,
)
obj = next(parser.iter_objects())
self.assertListEqual(
obj,
obj.to_python(),
[
[10, 30],
[22, 33],
[
{PDFName("x"): 0, PDFName("y"): 5},
{PDFName("x"): 10, PDFName("y"): 5},
{"x": 0, "y": 5},
{"x": 10, "y": 5},
],
],
)
def test_parse_keeps_keywords(self):
parser = PDFL1Parser(
io.BytesIO(
br"""
12 1 obj (Bring) endobj
"""
)
)
expected = [
PDFPrimitive(TokenInteger(13, 15, 12)),
PDFPrimitive(TokenInteger(16, 17, 1)),
TokenKeyword(18, 21, b"obj"),
PDFPrimitive(TokenString(22, 29, "Bring")),
TokenKeyword(30, 36, b"endobj"),
]
for obj, eobj in zip(parser.iter_objects(), expected):
self.assertEqual(obj, eobj)
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
class L2(TestCase):
def run_test(
self,
raw: bytes,
expected: Callable[[IndirectObjectCollection], List[PDFObject]],
):
parser = PDFL2Parser(io.BytesIO(raw))
for i, (eobj, obj) in enumerate(
zip(expected(parser.inobjects), parser.iter_objects()), 1
):
self.assertEqual(obj, eobj, f"Failed at example #{i}")
try:
next_obj = next(parser.iter_objects())
self.assertIsNone(next_obj)
except (EOFError, StopIteration) as e:
self.assertIsNotNone(e)
class IndirectObjects(TestCase):
def test_parse_simple_indobj(self):
parser = PDFL2Parser(
parser = PDFParser(
io.BytesIO(
br"""
12 1 obj (Bring) endobj
@@ -126,20 +95,70 @@ class L2(TestCase):
obj
<< /x 1 /y 3>> endobj
"""
)
),
strict=True,
)
expected = [
(12, 1, "Bring"),
(100, 0, b"3"),
(13, 8, {PDFName("x"): 1, PDFName("y"): 3}),
(13, 8, {"x": 1, "y": 3}),
]
for obj, eobj in zip(parser.iter_objects(), expected):
self.assertIsInstance(obj, IndirectObject)
self.assertEqual(obj, parser.inobjects.get_indobject(eobj[0], eobj[1]))
self.assertEqual(
obj,
parser.state.indirect_object_collection.get_indobject(eobj[0], eobj[1]),
)
self.assertEqual(obj.to_python(), eobj[2])
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
def test_parse_obj_with_references(self):
parser = PDFParser(
io.BytesIO(
br"""
12 1 obj (Bring) endobj
10 2 obj << /lastone 12 1 R >> endobj
"""
),
strict=True,
)
expected = [
parser.state.indirect_object_collection.get_indobject(12, 1),
parser.state.indirect_object_collection.get_indobject(10, 2),
]
for obj, exp in zip(parser.iter_objects(), expected):
self.assertEqual(obj, exp)
with self.assertRaises(StreamEOF):
next(parser.iter_objects())
sec_obj = parser.state.indirect_object_collection.get_indobject(
10, 2
).get_object()
self.assertEqual(
sec_obj.to_python(),
{"lastone": "Bring"},
)
def run_test(
self,
raw: bytes,
expected: Callable[[IndirectObjectCollection], List[PDFObject]],
):
parser = PDFParser(io.BytesIO(raw), strict=True)
for i, (eobj, obj) in enumerate(
zip(
expected(parser.state.indirect_object_collection), parser.iter_objects()
),
1,
):
self.assertEqual(obj, eobj, f"Failed at example #{i}")
try:
next_obj = next(parser.iter_objects())
self.assertIsNone(next_obj)
except (EOFError, StopIteration) as e:
self.assertIsNotNone(e)
def test_parse_comments(self):
self.run_test(
br"""