new attempt for parser
- now support clear indirect objects
This commit is contained in:
1 parent
b802f782fa
commit
07244aaf11
18 files changed
+606
-514
No files matched your search
+21
-23
@@ -5,7 +5,6 @@ from typing import List
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.lexer.exceptions import LexerError
|
||||
from pdfmajor.lexer.number import parse_number
|
||||
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
|
||||
from pdfmajor.lexer.string import parse_string
|
||||
@@ -13,11 +12,9 @@ from pdfmajor.lexer.name import parse_name
|
||||
from pdfmajor.lexer.keyword import parse_keyword
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.lexer.token import (
|
||||
PDFName,
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
TArrayValue,
|
||||
Token,
|
||||
TokenArray,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenNull,
|
||||
@@ -27,6 +24,7 @@ from pdfmajor.lexer.token import (
|
||||
TokenInteger,
|
||||
TokenKeyword,
|
||||
TokenName,
|
||||
TokenArray,
|
||||
TokenString,
|
||||
)
|
||||
from pdfmajor.lexer.comment import parse_comment
|
||||
@@ -114,7 +112,7 @@ class Basics(TestCase):
|
||||
token = parse_name(buffer)
|
||||
self.assertIsInstance(token, TokenName)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, PDFName("Some_Name"))
|
||||
self.assertEqual(token.value, "Some_Name")
|
||||
self.assertEqual(token.end_loc, 9)
|
||||
|
||||
def test_parse_literal_names_hex(self):
|
||||
@@ -124,7 +122,7 @@ class Basics(TestCase):
|
||||
token = parse_name(buffer)
|
||||
self.assertIsInstance(token, TokenName)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, PDFName("foo_aa"))
|
||||
self.assertEqual(token.value, "foo_aa")
|
||||
self.assertEqual(token.end_loc, 8)
|
||||
|
||||
def test_parse_string(self):
|
||||
@@ -246,7 +244,7 @@ class Lexer(TestCase):
|
||||
self.run_test(
|
||||
b"/Length 4 0 R\n",
|
||||
expected=[
|
||||
TokenName(0, 7, PDFName("Length")),
|
||||
TokenName(0, 7, "Length"),
|
||||
TokenInteger(8, 9, 4),
|
||||
TokenInteger(10, 11, 0),
|
||||
TokenKeyword(12, 13, b"R"),
|
||||
@@ -299,10 +297,10 @@ func/a/b{(c)do*}def
|
||||
TokenKeyword(start_loc=16, end_loc=17, value=b'"'),
|
||||
TokenKeyword(start_loc=19, end_loc=20, value=b"@"),
|
||||
TokenKeyword(start_loc=21, end_loc=22, value=b"#"),
|
||||
TokenName(start_loc=23, end_loc=25, value=PDFName("a")),
|
||||
TokenName(start_loc=25, end_loc=29, value=PDFName("BCD")),
|
||||
TokenName(start_loc=30, end_loc=40, value=PDFName("Some_Name")),
|
||||
TokenName(start_loc=41, end_loc=53, value=PDFName("foo_naa")),
|
||||
TokenName(start_loc=23, end_loc=25, value="a"),
|
||||
TokenName(start_loc=25, end_loc=29, value="BCD"),
|
||||
TokenName(start_loc=30, end_loc=40, value="Some_Name"),
|
||||
TokenName(start_loc=41, end_loc=53, value="foo_naa"),
|
||||
TokenInteger(start_loc=54, end_loc=55, value=0),
|
||||
TokenInteger(start_loc=56, end_loc=58, value=1),
|
||||
TokenInteger(start_loc=59, end_loc=61, value=-2),
|
||||
@@ -326,8 +324,8 @@ func/a/b{(c)do*}def
|
||||
start_loc=211, end_loc=225, value=b"\xab\xcd\x00\x124\x05"
|
||||
),
|
||||
TokenKeyword(start_loc=226, end_loc=230, value=b"func"),
|
||||
TokenName(start_loc=230, end_loc=232, value=PDFName("a")),
|
||||
TokenName(start_loc=232, end_loc=234, value=PDFName("b")),
|
||||
TokenName(start_loc=230, end_loc=232, value="a"),
|
||||
TokenName(start_loc=232, end_loc=234, value="b"),
|
||||
TokenKeyword(start_loc=234, end_loc=235, value=b"{"),
|
||||
TokenString(start_loc=235, end_loc=238, value="c"),
|
||||
TokenKeyword(start_loc=238, end_loc=241, value=b"do*"),
|
||||
@@ -339,7 +337,7 @@ func/a/b{(c)do*}def
|
||||
TokenKeyword(start_loc=254, end_loc=255, value=b"!"),
|
||||
TokenArray(start_loc=256, end_loc=257, value=TArrayValue.CLOSE),
|
||||
TokenDictionary(start_loc=258, end_loc=260, value=TDictValue.OPEN),
|
||||
TokenName(start_loc=261, end_loc=265, value=PDFName("foo")),
|
||||
TokenName(start_loc=261, end_loc=265, value="foo"),
|
||||
TokenString(start_loc=266, end_loc=271, value="bar"),
|
||||
TokenDictionary(start_loc=272, end_loc=274, value=TDictValue.CLOSE),
|
||||
],
|
||||
@@ -382,15 +380,15 @@ func/a/b
|
||||
TokenKeyword(start_loc=44, end_loc=45, value=b'"'),
|
||||
TokenKeyword(start_loc=47, end_loc=48, value=b"@"),
|
||||
TokenKeyword(start_loc=49, end_loc=50, value=b"#"),
|
||||
TokenName(start_loc=51, end_loc=53, value=PDFName("a")),
|
||||
TokenName(start_loc=53, end_loc=57, value=PDFName("BCD")),
|
||||
TokenName(start_loc=58, end_loc=68, value=PDFName("Some_Name")),
|
||||
TokenName(start_loc=69, end_loc=81, value=PDFName("foo_naa")),
|
||||
TokenName(start_loc=82, end_loc=89, value=PDFName("barj")),
|
||||
TokenName(start_loc=90, end_loc=95, value=PDFName("need")),
|
||||
TokenName(start_loc=96, end_loc=102, value=PDFName("oh1oh")),
|
||||
TokenName(start_loc=51, end_loc=53, value="a"),
|
||||
TokenName(start_loc=53, end_loc=57, value="BCD"),
|
||||
TokenName(start_loc=58, end_loc=68, value="Some_Name"),
|
||||
TokenName(start_loc=69, end_loc=81, value="foo_naa"),
|
||||
TokenName(start_loc=82, end_loc=89, value="barj"),
|
||||
TokenName(start_loc=90, end_loc=95, value="need"),
|
||||
TokenName(start_loc=96, end_loc=102, value="oh1oh"),
|
||||
TokenKeyword(start_loc=103, end_loc=107, value=b"func"),
|
||||
TokenName(start_loc=107, end_loc=109, value=PDFName("a")),
|
||||
TokenName(start_loc=109, end_loc=111, value=PDFName("b")),
|
||||
TokenName(start_loc=107, end_loc=109, value="a"),
|
||||
TokenName(start_loc=109, end_loc=111, value="b"),
|
||||
],
|
||||
)
|
||||
+89
-70
@@ -1,18 +1,19 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
|
||||
from typing import Callable, List
|
||||
from pdfmajor.lexer.token import TokenInteger, TokenKeyword, TokenString
|
||||
from pdfmajor.parser_v2.indirect_objects import IndirectObject, IndirectObjectCollection
|
||||
from pdfmajor.parser_v2.l2 import PDFL2Parser
|
||||
from pdfmajor.parser_v2.objects import PDFComment, PDFName, PDFObject, PDFPrimitive
|
||||
from pdfmajor.parser_v2.l1 import PDFL1Parser
|
||||
from pdfmajor.streambuffer import StreamEOF
|
||||
from decimal import Decimal
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.streambuffer import StreamEOF
|
||||
from pdfmajor.parser_v3.objects.indirect import IndirectObject, IndirectObjectCollection
|
||||
from pdfmajor.parser_v3.objects.base import PDFObject
|
||||
from pdfmajor.parser_v3 import PDFParser
|
||||
from pdfmajor.parser_v3.objects.comment import PDFComment
|
||||
|
||||
class L1(TestCase):
|
||||
|
||||
class Collections(TestCase):
|
||||
def test_parse_dict(self):
|
||||
parser = PDFL1Parser(
|
||||
parser = PDFParser(
|
||||
io.BytesIO(
|
||||
br"""<< /Type /Example
|
||||
/Subtype /DictionaryExample
|
||||
@@ -25,97 +26,65 @@ class L1(TestCase):
|
||||
/LastItem ( not! )
|
||||
/VeryLastItem ( OK ) >>
|
||||
>>"""
|
||||
)
|
||||
),
|
||||
strict=True,
|
||||
)
|
||||
obj = next(parser.iter_objects())
|
||||
self.assertDictEqual(
|
||||
obj,
|
||||
obj.to_python(),
|
||||
{
|
||||
PDFName("Type"): PDFName("Example"),
|
||||
PDFName("Subtype"): PDFName("DictionaryExample"),
|
||||
PDFName("Version"): Decimal("0.01"),
|
||||
PDFName("IntegerItem"): 12,
|
||||
PDFName("StringItem"): " a string ",
|
||||
PDFName("Subdictionary"): {
|
||||
PDFName("Item1"): Decimal("0.4"),
|
||||
PDFName("Item2"): True,
|
||||
PDFName("LastItem"): " not! ",
|
||||
PDFName("VeryLastItem"): " OK ",
|
||||
"Type": "/Example",
|
||||
"Subtype": "/DictionaryExample",
|
||||
"Version": Decimal("0.01"),
|
||||
"IntegerItem": 12,
|
||||
"StringItem": " a string ",
|
||||
"Subdictionary": {
|
||||
"Item1": Decimal("0.4"),
|
||||
"Item2": True,
|
||||
"LastItem": " not! ",
|
||||
"VeryLastItem": " OK ",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def test_parse_array(self):
|
||||
parser = PDFL1Parser(
|
||||
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]""")
|
||||
parser = PDFParser(
|
||||
io.BytesIO(br"""[ 549 3.14 false ( Ralph ) /SomeName ]"""),
|
||||
strict=True,
|
||||
)
|
||||
obj = next(parser.iter_objects())
|
||||
self.assertListEqual(
|
||||
obj, [549, Decimal("3.14"), False, " Ralph ", PDFName("SomeName")]
|
||||
obj.to_python(),
|
||||
[549, Decimal("3.14"), False, " Ralph ", "/SomeName"],
|
||||
)
|
||||
|
||||
def test_parse_nested_array(self):
|
||||
parser = PDFL1Parser(
|
||||
parser = PDFParser(
|
||||
io.BytesIO(
|
||||
br"""[ [10 30] [22 33] [
|
||||
<< /x 0 /y 5 >>
|
||||
<< /x 10 /y 5 >>
|
||||
] ]"""
|
||||
)
|
||||
),
|
||||
strict=True,
|
||||
)
|
||||
obj = next(parser.iter_objects())
|
||||
self.assertListEqual(
|
||||
obj,
|
||||
obj.to_python(),
|
||||
[
|
||||
[10, 30],
|
||||
[22, 33],
|
||||
[
|
||||
{PDFName("x"): 0, PDFName("y"): 5},
|
||||
{PDFName("x"): 10, PDFName("y"): 5},
|
||||
{"x": 0, "y": 5},
|
||||
{"x": 10, "y": 5},
|
||||
],
|
||||
],
|
||||
)
|
||||
|
||||
def test_parse_keeps_keywords(self):
|
||||
parser = PDFL1Parser(
|
||||
io.BytesIO(
|
||||
br"""
|
||||
12 1 obj (Bring) endobj
|
||||
"""
|
||||
)
|
||||
)
|
||||
expected = [
|
||||
PDFPrimitive(TokenInteger(13, 15, 12)),
|
||||
PDFPrimitive(TokenInteger(16, 17, 1)),
|
||||
TokenKeyword(18, 21, b"obj"),
|
||||
PDFPrimitive(TokenString(22, 29, "Bring")),
|
||||
TokenKeyword(30, 36, b"endobj"),
|
||||
]
|
||||
for obj, eobj in zip(parser.iter_objects(), expected):
|
||||
self.assertEqual(obj, eobj)
|
||||
with self.assertRaises(StreamEOF):
|
||||
next(parser.iter_objects())
|
||||
|
||||
|
||||
class L2(TestCase):
|
||||
def run_test(
|
||||
self,
|
||||
raw: bytes,
|
||||
expected: Callable[[IndirectObjectCollection], List[PDFObject]],
|
||||
):
|
||||
parser = PDFL2Parser(io.BytesIO(raw))
|
||||
for i, (eobj, obj) in enumerate(
|
||||
zip(expected(parser.inobjects), parser.iter_objects()), 1
|
||||
):
|
||||
self.assertEqual(obj, eobj, f"Failed at example #{i}")
|
||||
try:
|
||||
next_obj = next(parser.iter_objects())
|
||||
self.assertIsNone(next_obj)
|
||||
except (EOFError, StopIteration) as e:
|
||||
self.assertIsNotNone(e)
|
||||
|
||||
class IndirectObjects(TestCase):
|
||||
def test_parse_simple_indobj(self):
|
||||
parser = PDFL2Parser(
|
||||
parser = PDFParser(
|
||||
io.BytesIO(
|
||||
br"""
|
||||
12 1 obj (Bring) endobj
|
||||
@@ -126,20 +95,70 @@ class L2(TestCase):
|
||||
obj
|
||||
<< /x 1 /y 3>> endobj
|
||||
"""
|
||||
)
|
||||
),
|
||||
strict=True,
|
||||
)
|
||||
expected = [
|
||||
(12, 1, "Bring"),
|
||||
(100, 0, b"3"),
|
||||
(13, 8, {PDFName("x"): 1, PDFName("y"): 3}),
|
||||
(13, 8, {"x": 1, "y": 3}),
|
||||
]
|
||||
for obj, eobj in zip(parser.iter_objects(), expected):
|
||||
self.assertIsInstance(obj, IndirectObject)
|
||||
self.assertEqual(obj, parser.inobjects.get_indobject(eobj[0], eobj[1]))
|
||||
self.assertEqual(
|
||||
obj,
|
||||
parser.state.indirect_object_collection.get_indobject(eobj[0], eobj[1]),
|
||||
)
|
||||
self.assertEqual(obj.to_python(), eobj[2])
|
||||
with self.assertRaises(StreamEOF):
|
||||
next(parser.iter_objects())
|
||||
|
||||
def test_parse_obj_with_references(self):
|
||||
parser = PDFParser(
|
||||
io.BytesIO(
|
||||
br"""
|
||||
12 1 obj (Bring) endobj
|
||||
10 2 obj << /lastone 12 1 R >> endobj
|
||||
|
||||
"""
|
||||
),
|
||||
strict=True,
|
||||
)
|
||||
expected = [
|
||||
parser.state.indirect_object_collection.get_indobject(12, 1),
|
||||
parser.state.indirect_object_collection.get_indobject(10, 2),
|
||||
]
|
||||
for obj, exp in zip(parser.iter_objects(), expected):
|
||||
self.assertEqual(obj, exp)
|
||||
with self.assertRaises(StreamEOF):
|
||||
next(parser.iter_objects())
|
||||
sec_obj = parser.state.indirect_object_collection.get_indobject(
|
||||
10, 2
|
||||
).get_object()
|
||||
self.assertEqual(
|
||||
sec_obj.to_python(),
|
||||
{"lastone": "Bring"},
|
||||
)
|
||||
|
||||
def run_test(
|
||||
self,
|
||||
raw: bytes,
|
||||
expected: Callable[[IndirectObjectCollection], List[PDFObject]],
|
||||
):
|
||||
parser = PDFParser(io.BytesIO(raw), strict=True)
|
||||
for i, (eobj, obj) in enumerate(
|
||||
zip(
|
||||
expected(parser.state.indirect_object_collection), parser.iter_objects()
|
||||
),
|
||||
1,
|
||||
):
|
||||
self.assertEqual(obj, eobj, f"Failed at example #{i}")
|
||||
try:
|
||||
next_obj = next(parser.iter_objects())
|
||||
self.assertIsNone(next_obj)
|
||||
except (EOFError, StopIteration) as e:
|
||||
self.assertIsNotNone(e)
|
||||
|
||||
def test_parse_comments(self):
|
||||
self.run_test(
|
||||
br"""
|
||||
|
||||
Reference in new issue
Block a user