import io from decimal import Decimal from typing import List from unittest import TestCase from pdfmajor.lexer import PDFLexer from pdfmajor.lexer.exceptions import LexerError from pdfmajor.lexer.number import parse_number from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring from pdfmajor.lexer.string import parse_string from pdfmajor.lexer.name import parse_name from pdfmajor.lexer.keyword import parse_keyword from pdfmajor.streambuffer import BufferStream from pdfmajor.lexer.token import ( PDFName, TArrayValue, TDictValue, Token, TokenArray, TokenBoolean, TokenComment, TokenNull, TokenReal, TokenDictionary, TokenHexString, TokenInteger, TokenKeyword, TokenName, TokenString, ) from pdfmajor.lexer.comment import parse_comment def make_stream_iter(data: bytes, buf_size: int): return BufferStream(io.BytesIO(data), buf_size) class Basics(TestCase): def test_comment_parser_ln(self): for buf_size in [2, 7, 11, 150]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter( b"this is a lengthy comment that ends here\nso this is not reachable", buf_size, ) token = parse_comment(buffer) self.assertIsInstance(token, TokenComment) self.assertEqual(token.start_loc, -1) self.assertEqual( token.value, b"this is a lengthy comment that ends here" ) self.assertEqual(token.end_loc, 40) def test_comment_parser_lr(self): for buf_size in [2, 7, 11, 150]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter( b"this is a lengthy comment that ends here\rso this is not reachable", buf_size, ) token = parse_comment(buffer) self.assertIsInstance(token, TokenComment) self.assertEqual(token.start_loc, -1) self.assertEqual( token.value, b"this is a lengthy comment that ends here" ) self.assertEqual(token.end_loc, 40) def test_parse_bool_true(self): for buf_size in [2, 3, 50]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(b"true and the american way", buf_size) token = parse_keyword(buffer) self.assertIsInstance(token, TokenBoolean) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, True) self.assertEqual(token.end_loc, 4) def test_parse_bool_false(self): for buf_size in [2, 3, 50]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(b"false mannn", buf_size) token = parse_keyword(buffer) self.assertIsInstance(token, TokenBoolean) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, False) self.assertEqual(token.end_loc, 5) def test_parse_null(self): for buf_size in [2, 3, 50]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(b"null Nothing here", buf_size) token = parse_keyword(buffer) self.assertIsInstance(token, TokenNull) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, None) self.assertEqual(token.end_loc, 4) def test_parse_word(self): for buf_size in [2, 3, 50]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(b"likeing_turtles ", buf_size) token = parse_keyword(buffer) self.assertIsInstance(token, TokenKeyword) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, b"likeing_turtles") self.assertEqual(token.end_loc, 15) def test_parse_literal_names(self): for buf_size in [2, 3, 50, 444]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(br"Some_Name /foo#5f#xbaa", buf_size) token = parse_name(buffer) self.assertIsInstance(token, TokenName) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, PDFName("Some_Name")) self.assertEqual(token.end_loc, 9) def test_parse_literal_names_hex(self): for buf_size in [50, 1000]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(br"foo#5faa ", buf_size) token = parse_name(buffer) self.assertIsInstance(token, TokenName) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, PDFName("foo_aa")) self.assertEqual(token.end_loc, 8) def test_parse_string(self): for buf_size in [2, 3, 50, 1000]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(br"this % is not a comment.) ", buf_size) token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, "this % is not a comment.") self.assertEqual(token.end_loc, 25) def test_parse_string_nest_brackets(self): for buf_size in [1, 2, 3, 50, 100]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(br"x - (y*2-z-(2+3))) ", buf_size) token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, "x - (y*2-z-(2+3))") self.assertEqual(token.end_loc, 18) def test_parse_string_hex_chars(self): for buf_size in [1, 2, 3, 50, 100]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter( br"def\040\0\0404ghi) (bach\\slask) ", buf_size ) token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, "def 4ghi") self.assertEqual(token.end_loc, 18) def test_parse_string_escape_char(self): for buf_size in [1, 2, 3, 50, 100]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(br"bach\\slask) ", buf_size) token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, r"bach\slask") self.assertEqual(token.end_loc, 12) def test_parse_hex_string(self): for buf_size in [1, 2, 3, 50, 100]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(br"10 20 31a 12a>", buf_size) token = parse_hexstring(buffer) self.assertIsInstance(token, TokenHexString) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, b"\x10 1\xa1*") self.assertEqual(token.end_loc, 14) def test_parse_dict_keyword(self): for buf_size in [1, 2, 3, 50, 100]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(br"", buf_size) token = parse_double_angled_bracket(buffer) self.assertIsInstance(token, TokenHexString) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, b"\x10 1\xa1*") self.assertEqual(token.end_loc, 14) def test_parse_float(self): for buf_size in [2, 3, 50]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter(b"0.120 + some invalid text", buf_size) token = parse_number(buffer) self.assertIsInstance(token, TokenReal) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, Decimal("0.12")) self.assertEqual(token.end_loc, 5) def test_parse_int(self): for buf_size in [2, 3, 50]: with self.subTest(buf_end_loc=buf_size): buffer = make_stream_iter( b"130 oh oh 0.12 + some invalid text", buf_size ) token = parse_number(buffer) self.assertIsInstance(token, TokenInteger) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, 130) self.assertEqual(token.end_loc, 3) class Lexer(TestCase): def run_test(self, raw: bytes, expected: List[Token]): tokenizer = PDFLexer(io.BytesIO(raw)) for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1): self.assertEqual(token, etoken, f"Failed at example #{i}") try: next_token = next(tokenizer.iter_tokens()) self.assertIsNone(next_token) except (EOFError, StopIteration) as e: self.assertIsNotNone(e) def test_scase1(self): self.run_test( b"begin end\n", expected=[ TokenKeyword(0, 5, b"begin"), TokenKeyword(6, 9, b"end"), ], ) def test_scase2(self): self.run_test( b"/Length 4 0 R\n", expected=[ TokenName(0, 7, PDFName("Length")), TokenInteger(8, 9, 4), TokenInteger(10, 11, 0), TokenKeyword(12, 13, b"R"), ], ) def test_scase3(self): self.run_test( b"""endstream endobj 4 0 obj 2618 endobj """, expected=[ TokenKeyword(0, 9, b"endstream"), TokenKeyword(10, 16, b"endobj"), TokenInteger(17, 18, 4), TokenInteger(19, 20, 0), TokenKeyword(21, 24, b"obj"), TokenInteger(28, 32, 2618), TokenKeyword(33, 39, b"endobj"), ], ) def test_case1(self): self.run_test( raw=br"""%!PS begin end " @ # /a/BCD /Some_Name /foo#5f#6eaa 0 +1 -2 .5 1.234 (abc) () (abc ( def ) ghi) (def\040\0\0404ghi) (bach\\slask) (foo\nbaa) (this % is not a comment.) (foo baa) (foo\ baa) <> <20> < 40 4020 > func/a/b{(c)do*}def [ 1 (z) ! ] << /foo (bar) >>""", expected=[ TokenComment(start_loc=0, end_loc=4, value=b"!PS"), TokenKeyword(start_loc=5, end_loc=10, value=b"begin"), TokenKeyword(start_loc=11, end_loc=14, value=b"end"), TokenKeyword(start_loc=16, end_loc=17, value=b'"'), TokenKeyword(start_loc=19, end_loc=20, value=b"@"), TokenKeyword(start_loc=21, end_loc=22, value=b"#"), TokenName(start_loc=23, end_loc=25, value=PDFName("a")), TokenName(start_loc=25, end_loc=29, value=PDFName("BCD")), TokenName(start_loc=30, end_loc=40, value=PDFName("Some_Name")), TokenName(start_loc=41, end_loc=53, value=PDFName("foo_naa")), TokenInteger(start_loc=54, end_loc=55, value=0), TokenInteger(start_loc=56, end_loc=58, value=1), TokenInteger(start_loc=59, end_loc=61, value=-2), TokenReal(start_loc=62, end_loc=64, value=Decimal("0.5")), TokenReal(start_loc=65, end_loc=70, value=Decimal("1.234")), TokenString(start_loc=71, end_loc=76, value="abc"), TokenString(start_loc=77, end_loc=79, value=""), TokenString(start_loc=80, end_loc=97, value="abc ( def ) ghi"), TokenString(start_loc=98, end_loc=117, value="def 4ghi"), TokenString(start_loc=118, end_loc=131, value=r"bach\slask"), TokenString(start_loc=132, end_loc=142, value="foo\nbaa"), TokenString( start_loc=143, end_loc=169, value="this % is not a comment." ), TokenString(start_loc=170, end_loc=179, value="foo\nbaa"), TokenString(start_loc=180, end_loc=190, value="foobaa"), TokenHexString(start_loc=191, end_loc=193, value=b""), TokenHexString(start_loc=194, end_loc=198, value=b" "), TokenHexString(start_loc=199, end_loc=210, value=b"@@ "), TokenHexString( start_loc=211, end_loc=225, value=b"\xab\xcd\x00\x124\x05" ), TokenKeyword(start_loc=226, end_loc=230, value=b"func"), TokenName(start_loc=230, end_loc=232, value=PDFName("a")), TokenName(start_loc=232, end_loc=234, value=PDFName("b")), TokenKeyword(start_loc=234, end_loc=235, value=b"{"), TokenString(start_loc=235, end_loc=238, value="c"), TokenKeyword(start_loc=238, end_loc=241, value=b"do*"), TokenKeyword(start_loc=241, end_loc=242, value=b"}"), TokenKeyword(start_loc=242, end_loc=245, value=b"def"), TokenArray(start_loc=246, end_loc=247, value=TArrayValue.OPEN), TokenInteger(start_loc=248, end_loc=249, value=1), TokenString(start_loc=250, end_loc=253, value="z"), TokenKeyword(start_loc=254, end_loc=255, value=b"!"), TokenArray(start_loc=256, end_loc=257, value=TArrayValue.CLOSE), TokenDictionary(start_loc=258, end_loc=260, value=TDictValue.OPEN), TokenName(start_loc=261, end_loc=265, value=PDFName("foo")), TokenString(start_loc=266, end_loc=271, value="bar"), TokenDictionary(start_loc=272, end_loc=274, value=TDictValue.CLOSE), ], ) def test_case2(self): self.run_test( raw=br"""10.32 0 +1 -2 .5 1.234 -1.234 990 """, expected=[ TokenReal(start_loc=0, end_loc=5, value=Decimal("10.32")), TokenInteger(start_loc=6, end_loc=7, value=0), TokenInteger(start_loc=8, end_loc=10, value=1), TokenInteger(start_loc=11, end_loc=13, value=-2), TokenReal(start_loc=14, end_loc=16, value=Decimal("0.5")), TokenReal(start_loc=17, end_loc=22, value=Decimal("1.234")), TokenReal(start_loc=23, end_loc=29, value=Decimal("-1.234")), TokenInteger(start_loc=30, end_loc=33, value=990), ], ) def test_case3(self): self.run_test( raw=br"""%!PS % This is some secret sauce begin end " @ # /a/BCD /Some_Name /foo#5f#6eaa /bar#6a /need /oh1oh func/a/b """, expected=[ TokenComment(start_loc=0, end_loc=4, value=b"!PS"), TokenComment( start_loc=5, end_loc=32, value=b" This is some secret sauce" ), TokenKeyword(start_loc=33, end_loc=38, value=b"begin"), TokenKeyword(start_loc=39, end_loc=42, value=b"end"), TokenKeyword(start_loc=44, end_loc=45, value=b'"'), TokenKeyword(start_loc=47, end_loc=48, value=b"@"), TokenKeyword(start_loc=49, end_loc=50, value=b"#"), TokenName(start_loc=51, end_loc=53, value=PDFName("a")), TokenName(start_loc=53, end_loc=57, value=PDFName("BCD")), TokenName(start_loc=58, end_loc=68, value=PDFName("Some_Name")), TokenName(start_loc=69, end_loc=81, value=PDFName("foo_naa")), TokenName(start_loc=82, end_loc=89, value=PDFName("barj")), TokenName(start_loc=90, end_loc=95, value=PDFName("need")), TokenName(start_loc=96, end_loc=102, value=PDFName("oh1oh")), TokenKeyword(start_loc=103, end_loc=107, value=b"func"), TokenName(start_loc=107, end_loc=109, value=PDFName("a")), TokenName(start_loc=109, end_loc=111, value=PDFName("b")), ], )