From 4a6b0608a06fa847f745c2cd1a93f4eda18eaf5c Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Wed, 14 Oct 2020 21:29:46 -0400 Subject: [PATCH] strict hex format --- pdfmajor/tokenizer/__init__.py | 2 +- pdfmajor/tokenizer/exceptions.py | 9 +- pdfmajor/tokenizer/token_parsers/literal.py | 26 ++- pdfmajor/tokenizer/token_parsers/number.py | 2 +- tests/test_tokenizer.py | 217 +++++++++++--------- 5 files changed, 150 insertions(+), 106 deletions(-) diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py index f8c558b..97994c0 100644 --- a/pdfmajor/tokenizer/__init__.py +++ b/pdfmajor/tokenizer/__init__.py @@ -71,7 +71,7 @@ class PSTokenizer: elif c == b"/": self.inc_buf_skipstep(j + 1) yield self._check_token( - cur, parse_literal(bufpos + j, self.iter_buffer()) + cur, parse_literal(bufpos + j + 1, self.iter_buffer()) ) elif c in b"-+": self.inc_buf_skipstep(j + 1) diff --git a/pdfmajor/tokenizer/exceptions.py b/pdfmajor/tokenizer/exceptions.py index ce904ed..561d281 100644 --- a/pdfmajor/tokenizer/exceptions.py +++ b/pdfmajor/tokenizer/exceptions.py @@ -12,5 +12,12 @@ class TokenizerEOF(TokenizerError): class InvalidToken(TokenizerError): def __init__(self, pos: int, token: bytes) -> None: super().__init__( - "Invalid Token Found at {pos}, of value={v:r}".format(pos=pos, v=token) + "Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token)) + ) + + +class InvalidHexToken(TokenizerError): + def __init__(self, pos: int, token: bytes) -> None: + super().__init__( + "Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token)) ) diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py index 28fe579..7cc9115 100644 --- a/pdfmajor/tokenizer/token_parsers/literal.py +++ b/pdfmajor/tokenizer/token_parsers/literal.py @@ -1,5 +1,10 @@ from dataclasses import dataclass -from pdfmajor.tokenizer.exceptions import TokenizerEOF, TokenizerError +from pdfmajor.tokenizer.exceptions import ( + InvalidHexToken, + InvalidToken, + TokenizerEOF, + TokenizerError, +) from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize from pdfmajor.tokenizer.token_parsers.util import PInput @@ -29,18 +34,19 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: state = LiteralParseState(b"") for curpos, buf in inp: it = SafeBufferIt(buf) - for _ in it.into_iter(): + for subbuf in it.into_iter(): if state.hex_value is not None: - for ci in range(len(buf[it.skip :])): - c = buf[it.skip + ci : it.skip + ci + 1] + for ci in range(len(subbuf)): + c = subbuf[ci : ci + 1] if HEX.match(c) and len(state.hex_value) < 2: state.hex_value += c - else: - if state.hex_value: + it.skip += 1 + if len(state.hex_value) >= 2: state.curtoken += int2byte(int(state.hex_value, 16)) - state.hex_value = None - it.skip += ci - break + state.hex_value = None + break + else: + raise InvalidHexToken(curpos + ci, c) else: m = END_LITERAL.search(buf[it.skip :], 0) if not m: @@ -57,7 +63,7 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: else: return TokenLiteral( initialpos, - cmp_tsize(curpos, initialpos, j) - 1, + cmp_tsize(curpos, initialpos, j), state.curtoken.decode(), ) raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/number.py b/pdfmajor/tokenizer/token_parsers/number.py index 1e5093c..0a65f27 100644 --- a/pdfmajor/tokenizer/token_parsers/number.py +++ b/pdfmajor/tokenizer/token_parsers/number.py @@ -53,7 +53,7 @@ def parse_number( j += nj return TokenDecimal( initialpos, - cmp_tsize(curpos, initialpos, j), + cmp_tsize(curpos, initialpos, j + 1), Decimal(curtoken.decode()), ) try: diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 3491d88..5130e72 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -38,90 +38,119 @@ def make_stream_iter(data: bytes, initpos: int = 0): class Individual(TestCase): def test_comment_parser_ln(self): - iter_stream = make_stream_iter( - b"this is a lengthy comment that ends here\nso this is not reachable" - ) - token = parse_comment(0, iter_stream(3)) - self.assertIsInstance(token, TokenComment) - self.assertEqual(token.pos, 0) - self.assertEqual(token.value, b"this is a lengthy comment that ends here") - self.assertEqual(token.size, 40) + for buf_size in [2, 7, 11, 150]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter( + b"this is a lengthy comment that ends here\nso this is not reachable" + ) + token = parse_comment(0, iter_stream(buf_size)) + self.assertIsInstance(token, TokenComment) + self.assertEqual(token.pos, 0) + self.assertEqual( + token.value, b"this is a lengthy comment that ends here" + ) + self.assertEqual(token.size, 40) def test_comment_parser_lr(self): - iter_stream = make_stream_iter( - b"this is a lengthy comment that ends here\rso this is not reachable" - ) - token = parse_comment(0, iter_stream(7)) - self.assertIsInstance(token, TokenComment) - self.assertEqual(token.pos, 0) - self.assertEqual(token.value, b"this is a lengthy comment that ends here") - self.assertEqual(token.size, 40) + for buf_size in [2, 7, 11, 150]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter( + b"this is a lengthy comment that ends here\rso this is not reachable" + ) + token = parse_comment(0, iter_stream(buf_size)) + self.assertIsInstance(token, TokenComment) + self.assertEqual(token.pos, 0) + self.assertEqual( + token.value, b"this is a lengthy comment that ends here" + ) + self.assertEqual(token.size, 40) def test_parse_float(self): - iter_stream = make_stream_iter(b"0.120 + some invalid text", 5) - token = parse_number(5, iter_stream(3)) - self.assertIsInstance(token, TokenDecimal) - self.assertEqual(token.pos, 5) - self.assertEqual(token.value, Decimal("0.12")) - self.assertEqual(token.size, 5) + for buf_size in [2, 3, 50]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter(b"0.120 + some invalid text", 5) + token = parse_number(5, iter_stream(buf_size)) + self.assertIsInstance(token, TokenDecimal) + self.assertEqual(token.pos, 5) + self.assertEqual(token.value, Decimal("0.12")) + self.assertEqual(token.size, 5) def test_parse_int(self): - iter_stream = make_stream_iter(b"130 oh oh 0.12 + some invalid text", 11) - token = parse_number(11, iter_stream(10)) - self.assertIsInstance(token, TokenInteger) - self.assertEqual(token.pos, 11) - self.assertEqual(token.value, 130) - self.assertEqual(token.size, 3) + for buf_size in [2, 3, 50]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter( + b"130 oh oh 0.12 + some invalid text", 11 + ) + token = parse_number(11, iter_stream(buf_size)) + self.assertIsInstance(token, TokenInteger) + self.assertEqual(token.pos, 11) + self.assertEqual(token.value, 130) + self.assertEqual(token.size, 3) def test_parse_bool_true(self): - iter_stream = make_stream_iter(b"true and the american way", 91) - token = parse_keyword(91, iter_stream(33)) - self.assertIsInstance(token, TokenBoolean) - self.assertEqual(token.pos, 91) - self.assertEqual(token.value, True) - self.assertEqual(token.size, 4) + for buf_size in [2, 3, 50]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter(b"true and the american way", 91) + token = parse_keyword(91, iter_stream(buf_size)) + self.assertIsInstance(token, TokenBoolean) + self.assertEqual(token.pos, 91) + self.assertEqual(token.value, True) + self.assertEqual(token.size, 4) def test_parse_bool_false(self): - iter_stream = make_stream_iter(b"false mannn", 33) - token = parse_keyword(33, iter_stream(100)) - self.assertIsInstance(token, TokenBoolean) - self.assertEqual(token.pos, 33) - self.assertEqual(token.value, False) - self.assertEqual(token.size, 5) + for buf_size in [2, 3, 50]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter(b"false mannn", 33) + token = parse_keyword(33, iter_stream(buf_size)) + self.assertIsInstance(token, TokenBoolean) + self.assertEqual(token.pos, 33) + self.assertEqual(token.value, False) + self.assertEqual(token.size, 5) def test_parse_literals(self): - iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71) - token = parse_literal(71, iter_stream(1)) - self.assertIsInstance(token, TokenLiteral) - self.assertEqual(token.pos, 71) - self.assertEqual(token.value, "Some_Name") - self.assertEqual(token.size, 9) + for buf_size in [2, 3, 50, 444]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71) + token = parse_literal(71, iter_stream(buf_size)) + self.assertIsInstance(token, TokenLiteral) + self.assertEqual(token.pos, 71) + self.assertEqual(token.value, "Some_Name") + self.assertEqual(token.size, 9) def test_parse_literals_hex(self): - iter_stream = make_stream_iter(br"foo#5f#xbaa ", 71) - token = parse_literal(71, iter_stream(100)) - self.assertIsInstance(token, TokenLiteral) - self.assertEqual(token.pos, 71) - self.assertEqual(token.value, "foo_xbaa") - self.assertEqual(token.size, 11) + for buf_size in [2]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter(br"foo#5faa ", 71) + token = parse_literal(71, iter_stream(buf_size)) + self.assertIsInstance(token, TokenLiteral) + self.assertEqual(token.pos, 71) + self.assertEqual(token.value, "foo_aa") + self.assertEqual(token.size, 8) def test_parse_string(self): - iter_stream = make_stream_iter(br"this % is not a comment.) ", 88) - token = parse_string(88, iter_stream(452)) - self.assertIsInstance(token, TokenString) - self.assertEqual(token.pos, 88) - self.assertEqual(token.value, "this % is not a comment.") - self.assertEqual(token.size, 24) + for buf_size in [2]: + with self.subTest(buf_size=buf_size): + iter_stream = make_stream_iter(br"this % is not a comment.) ", 88) + token = parse_string(88, iter_stream(buf_size)) + self.assertIsInstance(token, TokenString) + self.assertEqual(token.pos, 88) + self.assertEqual(token.value, "this % is not a comment.") + self.assertEqual(token.size, 24) class Tokenizer(TestCase): + def run_test(self, raw: bytes, expected: List[Token]): + tokenizer = PSTokenizer(io.BytesIO(raw)) + for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())): + self.assertEqual(token, etoken, f"Failed at example {i}") + def test_case1(self): tokenizer = PSTokenizer( io.BytesIO( br"""%!PS begin end " @ # -/a/BCD /Some_Name /foo#5f#xbaa +/a/BCD /Some_Name /foo#5f#6eaa 0 +1 -2 .5 1.234 (abc) () (abc ( def ) ghi) (def\040\0\0404ghi) (bach\\slask) (foo\nbaa) @@ -142,47 +171,49 @@ func/a/b{(c)do*}def print(token) def test_case2(self): - tokenizer = PSTokenizer( - io.BytesIO( - br"""10.32 + self.run_test( + raw=br"""10.32 0 +1 -2 .5 1.234 --1.234 990""" - ) +-1.234 990""", + expected=[ + TokenDecimal(pos=0, size=5, value=Decimal("10.32")), + TokenInteger(pos=6, size=1, value=0), + TokenInteger(pos=8, size=2, value=1), + TokenInteger(pos=11, size=2, value=-2), + TokenDecimal(pos=14, size=2, value=Decimal("0.5")), + TokenDecimal(pos=17, size=5, value=Decimal("1.234")), + TokenDecimal(pos=23, size=6, value=Decimal("-1.234")), + TokenInteger(pos=30, size=3, value=990), + ], ) - for token in tokenizer.iter_tokens(): - print(token) def test_case_words1(self): - tokenizer = PSTokenizer( - io.BytesIO( - br"""%!PS + self.run_test( + raw=br"""%!PS % This is some secret sauce begin end " @ # -/a/BCD /Some_Name /foo#5f#xbaa /bar#6a +/a/BCD /Some_Name /foo#5f#6eaa /bar#6a /need /oh1oh func/a/b -""" - ) +""", + expected=[ + TokenComment(pos=0, size=4, value=b"!PS"), + TokenComment(pos=5, size=27, value=b" This is some secret sauce"), + TokenKeyword(pos=33, size=5, value=b"begin"), + TokenKeyword(pos=39, size=3, value=b"end"), + TokenKeyword(pos=44, size=1, value=b'"'), + TokenKeyword(pos=47, size=1, value=b"@"), + TokenKeyword(pos=49, size=1, value=b"#"), + TokenLiteral(pos=52, size=1, value="a"), + TokenLiteral(pos=54, size=3, value="BCD"), + TokenLiteral(pos=59, size=9, value="Some_Name"), + TokenLiteral(pos=70, size=11, value="foo_naa"), + TokenLiteral(pos=83, size=6, value="barj"), + TokenLiteral(pos=91, size=4, value="need"), + TokenLiteral(pos=97, size=5, value="oh1oh"), + TokenKeyword(pos=103, size=4, value=b"func"), + TokenLiteral(pos=108, size=1, value="a"), + TokenLiteral(pos=110, size=1, value="b"), + ], ) - expected: List[Token] = [ - TokenComment(pos=0, size=4, value=b"!PS"), - TokenComment(pos=5, size=27, value=b" This is some secret sauce"), - TokenKeyword(pos=33, size=5, value=b"begin"), - TokenKeyword(pos=39, size=3, value=b"end"), - TokenKeyword(pos=44, size=1, value=b'"'), - TokenKeyword(pos=47, size=1, value=b"@"), - TokenKeyword(pos=49, size=1, value=b"#"), - TokenLiteral(pos=51, size=1, value="a"), - TokenLiteral(pos=53, size=3, value="BCD"), - TokenLiteral(pos=58, size=9, value="Some_Name"), - TokenLiteral(pos=69, size=11, value="foo_xbaa"), - TokenLiteral(pos=82, size=6, value="barj"), - TokenLiteral(pos=90, size=4, value="need"), - TokenLiteral(pos=96, size=5, value="oh1oh"), - TokenKeyword(pos=103, size=4, value=b"func"), - TokenLiteral(pos=107, size=1, value="a"), - TokenLiteral(pos=109, size=1, value="b"), - ] - for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())): - self.assertEqual(token, etoken, f"Failed at example {i}")