diff --git a/pdfmajor/lexer/__init__.py b/pdfmajor/lexer/__init__.py index 8a00a61..d524b24 100644 --- a/pdfmajor/lexer/__init__.py +++ b/pdfmajor/lexer/__init__.py @@ -45,10 +45,10 @@ class PDFLexer: yield parse_number(self.buffer, next_char) elif next_char.isdigit() or next_char == b".": self.buffer.seek(bufpos + j) - yield parse_number(self.buffer) + yield parse_number(self.buffer, initialpos=self.buffer.tell()) elif next_char.isalpha(): self.buffer.seek(bufpos + j) - yield parse_keyword(self.buffer) + yield parse_keyword(self.buffer, self.buffer.tell()) elif next_char == b"(": self.buffer.seek(bufpos + j + 1) yield parse_string(self.buffer) diff --git a/pdfmajor/lexer/dict_and_hex.py b/pdfmajor/lexer/dict_and_hex.py index 7a1d94e..013269e 100644 --- a/pdfmajor/lexer/dict_and_hex.py +++ b/pdfmajor/lexer/dict_and_hex.py @@ -21,7 +21,7 @@ def parse_double_angled_bracket( first_char = step.data[0:1] if first_char == b"<": buffer.seek(step.pos + 1) - return TokenDictionary(initialpos, 2, TDictValue.OPEN) + return TokenDictionary(initialpos, buffer.tell(), TDictValue.OPEN) buffer.seek(step.pos) return parse_hexstring(buffer) diff --git a/pdfmajor/lexer/keyword.py b/pdfmajor/lexer/keyword.py index 91789cf..a3a30c2 100644 --- a/pdfmajor/lexer/keyword.py +++ b/pdfmajor/lexer/keyword.py @@ -2,20 +2,23 @@ from pdfmajor.streambuffer import BufferStream from pdfmajor.lexer.exceptions import LexerEOF from pdfmajor.lexer.token import TokenBoolean, TokenKeyword from pdfmajor.lexer.regex import END_KEYWORD -from typing import Union +from typing import Optional, Union -def parse_keyword(buffer: BufferStream) -> Union[TokenKeyword, TokenBoolean]: +def parse_keyword( + buffer: BufferStream, initialpos: Optional[int] = None +) -> Union[TokenKeyword, TokenBoolean]: """Parses input stream into a keyword or bool Args: buffer (BufferStream) + initialpos (Optional[int]) Returns: Union[TokenKeyword, TokenBoolean] """ curtoken = b"" - initialpos = buffer.tell() - 1 + initialpos = buffer.tell() - 1 if initialpos is None else initialpos for pos, buf in buffer: m = END_KEYWORD.search(buf, 0) if not m: diff --git a/pdfmajor/lexer/number.py b/pdfmajor/lexer/number.py index f4af5d7..93752e8 100644 --- a/pdfmajor/lexer/number.py +++ b/pdfmajor/lexer/number.py @@ -1,4 +1,5 @@ from decimal import Decimal +from typing import Optional from pdfmajor.streambuffer import BufferStream from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError from pdfmajor.lexer.token import ( @@ -11,12 +12,15 @@ from pdfmajor.lexer.regex import ( ) -def parse_number(buffer: BufferStream, sign: bytes = b"+") -> TokenNumber: +def parse_number( + buffer: BufferStream, sign: bytes = b"+", initialpos: Optional[int] = None +) -> TokenNumber: """Parses input stream into a number Args: buffer (BufferStream) sign (str): either a + or - + initialpos (Optional[int]) Returns: TokenNumber @@ -25,7 +29,7 @@ def parse_number(buffer: BufferStream, sign: bytes = b"+") -> TokenNumber: raise LexerError("Number was provided an invalid sign {!r}".format(sign)) curtoken = b"" + sign is_decimal = False - initialpos = buffer.tell() - 1 + initialpos = buffer.tell() - 1 if initialpos is None else initialpos for pos, buf in buffer: m = END_NUMBER.search(buf, 0) if not m: diff --git a/tests/test_lexer.py b/tests/test_lexer.py index bbd129c..caaf1cb 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -13,8 +13,10 @@ from pdfmajor.lexer.name import parse_name from pdfmajor.lexer.keyword import parse_keyword from pdfmajor.streambuffer import BufferStream from pdfmajor.lexer.token import ( + TArrayValue, TDictValue, Token, + TokenArray, TokenBoolean, TokenComment, TokenDecimal, @@ -83,6 +85,16 @@ class Basics(TestCase): self.assertEqual(token.value, False) self.assertEqual(token.end_loc, 5) + def test_parse_word(self): + for buf_size in [2, 3, 50]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(b"likeing_turtles ", buf_size) + token = parse_keyword(buffer) + self.assertIsInstance(token, TokenKeyword) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, b"likeing_turtles") + self.assertEqual(token.end_loc, 15) + def test_parse_literal_names(self): for buf_size in [2, 3, 50, 444]: with self.subTest(buf_end_loc=buf_size): @@ -163,7 +175,7 @@ class Basics(TestCase): self.assertIsInstance(token, TokenDictionary) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, TDictValue.OPEN) - self.assertEqual(token.end_loc, 2) + self.assertEqual(token.end_loc, 1) def test_parse_dd_to_hex(self): for buf_size in [1, 2, 3, 50, 100]: @@ -201,14 +213,53 @@ class Basics(TestCase): class Lexer(TestCase): def run_test(self, raw: bytes, expected: List[Token]): tokenizer = PDFLexer(io.BytesIO(raw)) - for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())): - self.assertEqual(token, etoken, f"Failed at example {i}") + for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1): + self.assertEqual(token, etoken, f"Failed at example #{i}") try: next_token = next(tokenizer.iter_tokens()) self.assertIsNone(next_token) - except LexerError as e: + except (EOFError, StopIteration) as e: self.assertIsNotNone(e) + def test_scase1(self): + self.run_test( + b"begin end\n", + expected=[ + TokenKeyword(0, 5, b"begin"), + TokenKeyword(6, 9, b"end"), + ], + ) + + def test_scase2(self): + self.run_test( + b"/Length 4 0 R\n", + expected=[ + TokenName(0, 7, "Length"), + TokenInteger(8, 9, 4), + TokenInteger(10, 11, 0), + TokenKeyword(12, 13, b"R"), + ], + ) + + def test_scase3(self): + self.run_test( + b"""endstream +endobj +4 0 obj + 2618 +endobj +""", + expected=[ + TokenKeyword(0, 9, b"endstream"), + TokenKeyword(10, 16, b"endobj"), + TokenInteger(17, 18, 4), + TokenInteger(19, 20, 0), + TokenKeyword(21, 24, b"obj"), + TokenInteger(28, 32, 2618), + TokenKeyword(33, 39, b"endobj"), + ], + ) + def test_case1(self): self.run_test( raw=br"""%!PS @@ -231,41 +282,54 @@ func/a/b{(c)do*}def << /foo (bar) >>""", expected=[ TokenComment(start_loc=0, end_loc=4, value=b"!PS"), - TokenKeyword(start_loc=5, end_loc=5, value=b"begin"), - TokenKeyword(start_loc=11, end_loc=3, value=b"end"), - TokenKeyword(start_loc=16, end_loc=1, value=b'"'), - TokenKeyword(start_loc=19, end_loc=1, value=b"@"), - TokenKeyword(start_loc=21, end_loc=1, value=b"#"), - TokenName(start_loc=24, end_loc=1, value="a"), - TokenName(start_loc=26, end_loc=3, value="BCD"), - TokenName(start_loc=31, end_loc=9, value="Some_Name"), - TokenName(start_loc=42, end_loc=11, value="foo_naa"), - TokenInteger(start_loc=54, end_loc=1, value=0), - TokenInteger(start_loc=56, end_loc=2, value=1), - TokenInteger(start_loc=59, end_loc=2, value=-2), - TokenDecimal(start_loc=62, end_loc=2, value=Decimal("0.5")), - TokenDecimal(start_loc=65, end_loc=5, value=Decimal("1.234")), - TokenString(start_loc=71, end_loc=5, value="abc"), - TokenString(start_loc=77, end_loc=2, value=""), - TokenString(start_loc=80, end_loc=17, value="abc ( def ) ghi"), - TokenString(start_loc=98, end_loc=19, value="def 4ghi"), - TokenString(start_loc=118, end_loc=13, value=r"bach\slask"), - TokenString(start_loc=132, end_loc=10, value="foo\nbaa"), + TokenKeyword(start_loc=5, end_loc=10, value=b"begin"), + TokenKeyword(start_loc=11, end_loc=14, value=b"end"), + TokenKeyword(start_loc=16, end_loc=17, value=b'"'), + TokenKeyword(start_loc=19, end_loc=20, value=b"@"), + TokenKeyword(start_loc=21, end_loc=22, value=b"#"), + TokenName(start_loc=23, end_loc=25, value="a"), + TokenName(start_loc=25, end_loc=29, value="BCD"), + TokenName(start_loc=30, end_loc=40, value="Some_Name"), + TokenName(start_loc=41, end_loc=53, value="foo_naa"), + TokenInteger(start_loc=54, end_loc=55, value=0), + TokenInteger(start_loc=56, end_loc=58, value=1), + TokenInteger(start_loc=59, end_loc=61, value=-2), + TokenDecimal(start_loc=62, end_loc=64, value=Decimal("0.5")), + TokenDecimal(start_loc=65, end_loc=70, value=Decimal("1.234")), + TokenString(start_loc=71, end_loc=76, value="abc"), + TokenString(start_loc=77, end_loc=79, value=""), + TokenString(start_loc=80, end_loc=97, value="abc ( def ) ghi"), + TokenString(start_loc=98, end_loc=117, value="def 4ghi"), + TokenString(start_loc=118, end_loc=131, value=r"bach\slask"), + TokenString(start_loc=132, end_loc=142, value="foo\nbaa"), TokenString( - start_loc=143, end_loc=26, value="this % is not a comment." + start_loc=143, end_loc=169, value="this % is not a comment." ), - TokenString(start_loc=170, end_loc=9, value="foo\nbaa"), - TokenString(start_loc=180, end_loc=10, value="foobaa"), - TokenHexString(start_loc=191, end_loc=2, value=b""), - TokenHexString(start_loc=194, end_loc=4, value=b" "), - TokenHexString(start_loc=199, end_loc=11, value=b"@@ "), + TokenString(start_loc=170, end_loc=179, value="foo\nbaa"), + TokenString(start_loc=180, end_loc=190, value="foobaa"), + TokenHexString(start_loc=191, end_loc=193, value=b""), + TokenHexString(start_loc=194, end_loc=198, value=b" "), + TokenHexString(start_loc=199, end_loc=210, value=b"@@ "), TokenHexString( - start_loc=211, end_loc=14, value=b"\xab\xcd\x00\x124\x05" + start_loc=211, end_loc=225, value=b"\xab\xcd\x00\x124\x05" ), - TokenKeyword(start_loc=226, end_loc=4, value=b"func"), - TokenName(start_loc=231, end_loc=1, value="a"), - TokenName(start_loc=233, end_loc=1, value="b"), - TokenKeyword(start_loc=234, end_loc=1, value=b"{"), + TokenKeyword(start_loc=226, end_loc=230, value=b"func"), + TokenName(start_loc=230, end_loc=232, value="a"), + TokenName(start_loc=232, end_loc=234, value="b"), + TokenKeyword(start_loc=234, end_loc=235, value=b"{"), + TokenString(start_loc=235, end_loc=238, value="c"), + TokenKeyword(start_loc=238, end_loc=241, value=b"do*"), + TokenKeyword(start_loc=241, end_loc=242, value=b"}"), + TokenKeyword(start_loc=242, end_loc=245, value=b"def"), + TokenArray(start_loc=246, end_loc=247, value=TArrayValue.OPEN), + TokenInteger(start_loc=248, end_loc=249, value=1), + TokenString(start_loc=250, end_loc=253, value="z"), + TokenKeyword(start_loc=254, end_loc=255, value=b"!"), + TokenArray(start_loc=256, end_loc=257, value=TArrayValue.CLOSE), + TokenDictionary(start_loc=258, end_loc=260, value=TDictValue.OPEN), + TokenName(start_loc=261, end_loc=265, value="foo"), + TokenString(start_loc=266, end_loc=271, value="bar"), + TokenDictionary(start_loc=273, end_loc=274, value=TDictValue.CLOSE), ], ) @@ -276,27 +340,17 @@ func/a/b{(c)do*}def -1.234 990 """, expected=[ TokenDecimal(start_loc=0, end_loc=5, value=Decimal("10.32")), - TokenInteger(start_loc=6, end_loc=1, value=0), - TokenInteger(start_loc=8, end_loc=2, value=1), - TokenInteger(start_loc=11, end_loc=2, value=-2), - TokenDecimal(start_loc=14, end_loc=2, value=Decimal("0.5")), - TokenDecimal(start_loc=17, end_loc=5, value=Decimal("1.234")), - TokenDecimal(start_loc=23, end_loc=6, value=Decimal("-1.234")), - TokenInteger(start_loc=30, end_loc=3, value=990), + TokenInteger(start_loc=6, end_loc=7, value=0), + TokenInteger(start_loc=8, end_loc=10, value=1), + TokenInteger(start_loc=11, end_loc=13, value=-2), + TokenDecimal(start_loc=14, end_loc=16, value=Decimal("0.5")), + TokenDecimal(start_loc=17, end_loc=22, value=Decimal("1.234")), + TokenDecimal(start_loc=23, end_loc=29, value=Decimal("-1.234")), + TokenInteger(start_loc=30, end_loc=33, value=990), ], ) def test_case3(self): - self.run_test( - raw=br"""/b{(c)do*}def""", - expected=[ - TokenName(start_loc=0, end_loc=2, value="b"), - TokenKeyword(start_loc=2, end_loc=3, value=b"{"), - TokenString(start_loc=4, end_loc=5, value="c"), - ], - ) - - def test_case4(self): self.run_test( raw=br"""%!PS % This is some secret sauce @@ -309,22 +363,22 @@ func/a/b expected=[ TokenComment(start_loc=0, end_loc=4, value=b"!PS"), TokenComment( - start_loc=5, end_loc=27, value=b" This is some secret sauce" + start_loc=5, end_loc=32, value=b" This is some secret sauce" ), - TokenKeyword(start_loc=33, end_loc=5, value=b"begin"), - TokenKeyword(start_loc=39, end_loc=3, value=b"end"), - TokenKeyword(start_loc=44, end_loc=1, value=b'"'), - TokenKeyword(start_loc=47, end_loc=1, value=b"@"), - TokenKeyword(start_loc=49, end_loc=1, value=b"#"), - TokenName(start_loc=51, end_loc=2, value="a"), - TokenName(start_loc=53, end_loc=4, value="BCD"), - TokenName(start_loc=58, end_loc=10, value="Some_Name"), - TokenName(start_loc=69, end_loc=12, value="foo_naa"), - TokenName(start_loc=83, end_loc=7, value="barj"), - TokenName(start_loc=90, end_loc=5, value="need"), - TokenName(start_loc=96, end_loc=6, value="oh1oh"), - TokenKeyword(start_loc=103, end_loc=4, value=b"func"), - TokenName(start_loc=107, end_loc=3, value="a"), - TokenName(start_loc=109, end_loc=3, value="b"), + TokenKeyword(start_loc=33, end_loc=38, value=b"begin"), + TokenKeyword(start_loc=39, end_loc=42, value=b"end"), + TokenKeyword(start_loc=44, end_loc=45, value=b'"'), + TokenKeyword(start_loc=47, end_loc=48, value=b"@"), + TokenKeyword(start_loc=49, end_loc=50, value=b"#"), + TokenName(start_loc=51, end_loc=53, value="a"), + TokenName(start_loc=53, end_loc=57, value="BCD"), + TokenName(start_loc=58, end_loc=68, value="Some_Name"), + TokenName(start_loc=69, end_loc=81, value="foo_naa"), + TokenName(start_loc=82, end_loc=89, value="barj"), + TokenName(start_loc=90, end_loc=95, value="need"), + TokenName(start_loc=96, end_loc=102, value="oh1oh"), + TokenKeyword(start_loc=103, end_loc=107, value=b"func"), + TokenName(start_loc=107, end_loc=109, value="a"), + TokenName(start_loc=109, end_loc=111, value="b"), ], )