diff --git a/pdfmajor/lexer/string.py b/pdfmajor/lexer/string.py index 291513d..77153ab 100644 --- a/pdfmajor/lexer/string.py +++ b/pdfmajor/lexer/string.py @@ -50,7 +50,7 @@ def parse_string(buffer: BufferStream) -> TokenString: return TokenString( initialpos, buffer.tell(), - state.curtoken.decode(), + state.curtoken, ) raise LexerEOF diff --git a/pdfmajor/lexer/token.py b/pdfmajor/lexer/token.py index 30a8b30..629da93 100644 --- a/pdfmajor/lexer/token.py +++ b/pdfmajor/lexer/token.py @@ -77,7 +77,7 @@ class TokenNull(Token[None]): pass -class TokenString(Token[str]): +class TokenString(Token[bytes]): """Token representing PDF literal string Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2 """ diff --git a/pdfmajor/parser/objects/primitives.py b/pdfmajor/parser/objects/primitives.py index 68fd758..5b0f0b5 100644 --- a/pdfmajor/parser/objects/primitives.py +++ b/pdfmajor/parser/objects/primitives.py @@ -38,7 +38,7 @@ class PDFPrimitiveObject(PDFObject, Generic[T]): ) -class PDFString(PDFPrimitiveObject[str]): +class PDFString(PDFPrimitiveObject[bytes]): pass diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 653a82c..a2a5b5c 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -132,7 +132,7 @@ class Basics(TestCase): token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) - self.assertEqual(token.value, "this % is not a comment.") + self.assertEqual(token.value, b"this % is not a comment.") self.assertEqual(token.end_loc, 25) def test_parse_string_nest_brackets(self): @@ -142,7 +142,7 @@ class Basics(TestCase): token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) - self.assertEqual(token.value, "x - (y*2-z-(2+3))") + self.assertEqual(token.value, b"x - (y*2-z-(2+3))") self.assertEqual(token.end_loc, 18) def test_parse_string_hex_chars(self): @@ -154,7 +154,7 @@ class Basics(TestCase): token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) - self.assertEqual(token.value, "def 4ghi") + self.assertEqual(token.value, b"def 4ghi") self.assertEqual(token.end_loc, 18) def test_parse_string_escape_char(self): @@ -164,7 +164,7 @@ class Basics(TestCase): token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) - self.assertEqual(token.value, r"bach\slask") + self.assertEqual(token.value, br"bach\slask") self.assertEqual(token.end_loc, 12) def test_parse_hex_string(self): @@ -319,17 +319,17 @@ func/a/b{(c)do*}def TokenInteger(start_loc=59, end_loc=61, value=-2), TokenReal(start_loc=62, end_loc=64, value=Decimal("0.5")), TokenReal(start_loc=65, end_loc=70, value=Decimal("1.234")), - TokenString(start_loc=71, end_loc=76, value="abc"), - TokenString(start_loc=77, end_loc=79, value=""), - TokenString(start_loc=80, end_loc=97, value="abc ( def ) ghi"), - TokenString(start_loc=98, end_loc=117, value="def 4ghi"), - TokenString(start_loc=118, end_loc=131, value=r"bach\slask"), - TokenString(start_loc=132, end_loc=142, value="foo\nbaa"), + TokenString(start_loc=71, end_loc=76, value=b"abc"), + TokenString(start_loc=77, end_loc=79, value=b""), + TokenString(start_loc=80, end_loc=97, value=b"abc ( def ) ghi"), + TokenString(start_loc=98, end_loc=117, value=b"def 4ghi"), + TokenString(start_loc=118, end_loc=131, value=br"bach\slask"), + TokenString(start_loc=132, end_loc=142, value=b"foo\nbaa"), TokenString( - start_loc=143, end_loc=169, value="this % is not a comment." + start_loc=143, end_loc=169, value=b"this % is not a comment." ), - TokenString(start_loc=170, end_loc=179, value="foo\nbaa"), - TokenString(start_loc=180, end_loc=190, value="foobaa"), + TokenString(start_loc=170, end_loc=179, value=b"foo\nbaa"), + TokenString(start_loc=180, end_loc=190, value=b"foobaa"), TokenHexString(start_loc=191, end_loc=193, value=b""), TokenHexString(start_loc=194, end_loc=198, value=b" "), TokenHexString(start_loc=199, end_loc=210, value=b"@@ "), @@ -340,18 +340,18 @@ func/a/b{(c)do*}def TokenName(start_loc=230, end_loc=232, value="a"), TokenName(start_loc=232, end_loc=234, value="b"), TokenKeyword(start_loc=234, end_loc=235, value=b"{"), - TokenString(start_loc=235, end_loc=238, value="c"), + TokenString(start_loc=235, end_loc=238, value=b"c"), TokenKeyword(start_loc=238, end_loc=241, value=b"do*"), TokenKeyword(start_loc=241, end_loc=242, value=b"}"), TokenKeyword(start_loc=242, end_loc=245, value=b"def"), TokenArray(start_loc=246, end_loc=247, value=TArrayValue.OPEN), TokenInteger(start_loc=248, end_loc=249, value=1), - TokenString(start_loc=250, end_loc=253, value="z"), + TokenString(start_loc=250, end_loc=253, value=b"z"), TokenKeyword(start_loc=254, end_loc=255, value=b"!"), TokenArray(start_loc=256, end_loc=257, value=TArrayValue.CLOSE), TokenDictionary(start_loc=258, end_loc=260, value=TDictValue.OPEN), TokenName(start_loc=261, end_loc=265, value="foo"), - TokenString(start_loc=266, end_loc=271, value="bar"), + TokenString(start_loc=266, end_loc=271, value=b"bar"), TokenDictionary(start_loc=273, end_loc=275, value=TDictValue.CLOSE), ], ) diff --git a/tests/test_parser.py b/tests/test_parser.py index 66675b6..be5e03b 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -55,12 +55,12 @@ class Collections(TestCase): "Subtype": "/DictionaryExample", "Version": Decimal("0.01"), "IntegerItem": 12, - "StringItem": " a string ", + "StringItem": b" a string ", "Subdictionary": { "Item1": Decimal("0.4"), "Item2": True, - "LastItem": " not! ", - "VeryLastItem": " OK ", + "LastItem": b" not! ", + "VeryLastItem": b" OK ", }, }, ) @@ -68,7 +68,7 @@ class Collections(TestCase): def test_parse_array(self): self.run_test( raw=br"""[ 549 3.14 false ( Ralph ) /SomeName ]""", - expected=[549, Decimal("3.14"), False, " Ralph ", "/SomeName"], + expected=[549, Decimal("3.14"), False, b" Ralph ", "/SomeName"], ) def test_parse_nested_array(self): @@ -129,7 +129,7 @@ class IndirectObjects(TestCase): << /x 1 /y 3>> endobj """, expected=[ - IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)), + IndirectObject(12, 1, 18, PDFString(b"Bring", 0, 0)), IndirectObject(100, 0, 55, PDFHexString(b"3", 0, 0)), IndirectObject( 13, @@ -150,7 +150,7 @@ class IndirectObjects(TestCase): """, expected=[ - IndirectObject(12, 1, 18, PDFString("Bring", 0, 0)), + IndirectObject(12, 1, 18, PDFString(b"Bring", 0, 0)), IndirectObject( 10, 2, @@ -173,7 +173,7 @@ class IndirectObjects(TestCase): PDFComment(b" object references", (22, 41)), PDFComment(b" start of object", (62, 79)), PDFComment(b" object content", (110, 126)), - IndirectObject(10, 0, 58, PDFString("testing", 0, 0)), + IndirectObject(10, 0, 58, PDFString(b"testing", 0, 0)), PDFComment(b"closing the object", (150, 169)), IndirectObject(11, 1, 191, PDFInteger(2, 0, 0)), ],