From 96d68cdf68e8a1f173751f3d8d265e0e73b28552 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Wed, 14 Oct 2020 08:32:34 -0400 Subject: [PATCH] added string tokenizer --- pdfmajor/tokenizer/token_parsers/string.py | 87 ++++----- tests/test_tokenizer.py | 196 ++++++++++----------- 2 files changed, 141 insertions(+), 142 deletions(-) diff --git a/pdfmajor/tokenizer/token_parsers/string.py b/pdfmajor/tokenizer/token_parsers/string.py index 3f2faac..601d0c6 100644 --- a/pdfmajor/tokenizer/token_parsers/string.py +++ b/pdfmajor/tokenizer/token_parsers/string.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from pdfmajor.tokenizer.exceptions import TokenizerEOF -from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize +from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize from pdfmajor.utils import int2byte from pdfmajor.tokenizer.token import ( TokenString, @@ -23,47 +23,48 @@ class StringParseState: def parse_string(initialpos: int, inp: Iterator[PInput]): state = StringParseState(b"", 1) for curpos, buf in inp: - skip: int = 0 - for i in range(len(buf) + 1): - if i < len(buf): - raise TokenizerEOF("Max Iteration Reached!") - if skip >= len(buf): - break - m = END_STRING.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - c = s[j : j + 1] - if c == b"\\": - _parse_string_1 - return j + 1 - elif c == b"(": - paren += 1 - curtoken += c - return j + 1 - elif c == b")": - paren -= 1 - if paren: - curtoken += c - return j + 1 - return TokenString( - initialpos, cmp_tsize(curpos, initialpos, j), curtoken.decode() - ) + it = SafeBufferIt(buf) + for s in it.into_iter(): + if state.oct_value is not None: + it.skip += parse_string_oct(s, state) + else: + m = END_STRING.search(s, 0) + if not m: + state.curtoken += s + else: + j = m.start(0) + state.curtoken += s[:j] + c = s[j : j + 1] + if c == b"\\": + state.oct_value = b"" + it.skip += j + 1 + elif c == b"(": + state.paren += 1 + state.curtoken += c + it.skip += j + 1 + elif c == b")": + state.paren -= 1 + if state.paren: + state.curtoken += c + it.skip += j + 1 + return TokenString( + initialpos, + cmp_tsize(curpos, initialpos, j), + state.curtoken.decode(), + ) -def parse_string_oct(initialpos: int, inp: Iterator[PInput]): - curtoken: bytes = b"" - oct_val: bytes = b"" - for curpos, s in inp: - for ci in range(len(s)): - c = s[ci : ci + 1] - if OCT_STRING.match(c) and len(oct_val) < 3: - oct_val += c - elif oct_val: - curtoken += int2byte(int(oct_val, 8)) - return (curtoken, curpos + ci) - elif c in ESC_STRING: - curtoken += int2byte(ESC_STRING[c]) - return (curtoken, curpos + ci + 1) +def parse_string_oct(s: bytes, state: StringParseState) -> int: + for ci in range(len(s)): + c = s[ci : ci + 1] + if OCT_STRING.match(c) and len(state.oct_value) < 3: + state.oct_value += c + elif state.oct_value: + state.curtoken += int2byte(int(state.oct_value, 8)) + return ci + elif c in ESC_STRING: + state.curtoken += int2byte(ESC_STRING[c]) + state.oct_value = None + return ci + 1 + state.oct_value = None + return len(s) + 1 diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index bdf683e..246467f 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -1,12 +1,14 @@ from decimal import Decimal import io +from pdfmajor.tokenizer.token_parsers.string import parse_string from pdfmajor.tokenizer import PSTokenizer from pdfmajor.tokenizer.token import ( - TokenBoolean, - TokenComment, - TokenDecimal, - TokenInteger, - TokenLiteral + TokenBoolean, + TokenComment, + TokenDecimal, + TokenInteger, + TokenLiteral, + TokenString, ) from pdfmajor.tokenizer.token_parsers.util import PInput from pdfmajor.tokenizer.token_parsers.comment import parse_comment @@ -15,111 +17,105 @@ from pdfmajor.tokenizer.token_parsers.number import parse_number from pdfmajor.tokenizer.token_parsers.literal import parse_literal from unittest import TestCase -def make_stream_iter(data: bytes, initpos: int = 0): - stream = io.BytesIO(data) - def iter_stream(buf_size: int): - while True: - bufpos = stream.tell() - buf = stream.read(buf_size) - if buf: - yield PInput(initpos+bufpos, buf) - else: - raise "EOF" - return iter_stream +def make_stream_iter(data: bytes, initpos: int = 0): + stream = io.BytesIO(data) + + def iter_stream(buf_size: int): + while True: + bufpos = stream.tell() + buf = stream.read(buf_size) + if buf: + yield PInput(initpos + bufpos, buf) + else: + raise "EOF" + + return iter_stream + class Individual(TestCase): - def test_comment_parser_ln(self): - iter_stream = make_stream_iter( - b"this is a lengthy comment that ends here\nso this is not reachable" - ) - token = parse_comment(0, iter_stream(3)) - self.assertIsInstance(token, TokenComment) - self.assertEqual(token.pos, 0) - self.assertEqual(token.value, b"this is a lengthy comment that ends here") - self.assertEqual(token.size, 40) + def test_comment_parser_ln(self): + iter_stream = make_stream_iter( + b"this is a lengthy comment that ends here\nso this is not reachable" + ) + token = parse_comment(0, iter_stream(3)) + self.assertIsInstance(token, TokenComment) + self.assertEqual(token.pos, 0) + self.assertEqual(token.value, b"this is a lengthy comment that ends here") + self.assertEqual(token.size, 40) - def test_comment_parser_lr(self): - iter_stream = make_stream_iter( - b"this is a lengthy comment that ends here\rso this is not reachable" - ) - token = parse_comment(0, iter_stream(7)) - self.assertIsInstance(token, TokenComment) - self.assertEqual(token.pos, 0) - self.assertEqual(token.value, b"this is a lengthy comment that ends here") - self.assertEqual(token.size, 40) + def test_comment_parser_lr(self): + iter_stream = make_stream_iter( + b"this is a lengthy comment that ends here\rso this is not reachable" + ) + token = parse_comment(0, iter_stream(7)) + self.assertIsInstance(token, TokenComment) + self.assertEqual(token.pos, 0) + self.assertEqual(token.value, b"this is a lengthy comment that ends here") + self.assertEqual(token.size, 40) - def test_parse_float(self): - iter_stream = make_stream_iter( - b"0.120 + some invalid text", - 5 - ) - token = parse_number(5, iter_stream(3)) - self.assertIsInstance(token, TokenDecimal) - self.assertEqual(token.pos, 5) - self.assertEqual(token.value, Decimal('0.12')) - self.assertEqual(token.size, 5) + def test_parse_float(self): + iter_stream = make_stream_iter(b"0.120 + some invalid text", 5) + token = parse_number(5, iter_stream(3)) + self.assertIsInstance(token, TokenDecimal) + self.assertEqual(token.pos, 5) + self.assertEqual(token.value, Decimal("0.12")) + self.assertEqual(token.size, 5) + def test_parse_int(self): + iter_stream = make_stream_iter(b"130 oh oh 0.12 + some invalid text", 11) + token = parse_number(11, iter_stream(10)) + self.assertIsInstance(token, TokenInteger) + self.assertEqual(token.pos, 11) + self.assertEqual(token.value, 130) + self.assertEqual(token.size, 3) - def test_parse_int(self): - iter_stream = make_stream_iter( - b"130 oh oh 0.12 + some invalid text", - 11 - ) - token = parse_number(11, iter_stream(10)) - self.assertIsInstance(token, TokenInteger) - self.assertEqual(token.pos, 11) - self.assertEqual(token.value, 130) - self.assertEqual(token.size, 3) + def test_parse_bool_true(self): + iter_stream = make_stream_iter(b"true and the american way", 91) + token = parse_keyword(91, iter_stream(33)) + self.assertIsInstance(token, TokenBoolean) + self.assertEqual(token.pos, 91) + self.assertEqual(token.value, True) + self.assertEqual(token.size, 4) + def test_parse_bool_false(self): + iter_stream = make_stream_iter(b"false mannn", 33) + token = parse_keyword(33, iter_stream(100)) + self.assertIsInstance(token, TokenBoolean) + self.assertEqual(token.pos, 33) + self.assertEqual(token.value, False) + self.assertEqual(token.size, 5) - def test_parse_bool_true(self): - iter_stream = make_stream_iter( - b"true and the american way", - 91 - ) - token = parse_keyword(91, iter_stream(33)) - self.assertIsInstance(token, TokenBoolean) - self.assertEqual(token.pos, 91) - self.assertEqual(token.value, True) - self.assertEqual(token.size, 4) + def test_parse_literals(self): + iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71) + token = parse_literal(71, iter_stream(1)) + self.assertIsInstance(token, TokenLiteral) + self.assertEqual(token.pos, 71) + self.assertEqual(token.value, "Some_Name") + self.assertEqual(token.size, 9) - def test_parse_bool_false(self): - iter_stream = make_stream_iter( - b"false mannn", - 33 - ) - token = parse_keyword(33, iter_stream(100)) - self.assertIsInstance(token, TokenBoolean) - self.assertEqual(token.pos, 33) - self.assertEqual(token.value, False) - self.assertEqual(token.size, 5) + def test_parse_literals_hex(self): + iter_stream = make_stream_iter(br"foo#5f#xbaa ", 71) + token = parse_literal(71, iter_stream(100)) + self.assertIsInstance(token, TokenLiteral) + self.assertEqual(token.pos, 71) + self.assertEqual(token.value, "foo_xbaa") + self.assertEqual(token.size, 11) - def test_parse_literals(self): - iter_stream = make_stream_iter( - br"Some_Name /foo#5f#xbaa", - 71 - ) - token = parse_literal(71, iter_stream(1)) - self.assertIsInstance(token, TokenLiteral) - self.assertEqual(token.pos, 71) - self.assertEqual(token.value, 'Some_Name') - self.assertEqual(token.size, 9) + def test_parse_string(self): + iter_stream = make_stream_iter(br"this % is not a comment.) ", 88) + token = parse_string(88, iter_stream(452)) + self.assertIsInstance(token, TokenString) + self.assertEqual(token.pos, 88) + self.assertEqual(token.value, "this % is not a comment.") + self.assertEqual(token.size, 24) - def test_parse_literals_hex(self): - iter_stream = make_stream_iter( - br"foo#5f#xbaa ", - 71 - ) - token = parse_literal(71, iter_stream(100)) - self.assertIsInstance(token, TokenLiteral) - self.assertEqual(token.pos, 71) - self.assertEqual(token.value, 'foo_xbaa') - self.assertEqual(token.size, 11) class Tokenizer(TestCase): - def test_case1(self): - tokenizer = PSTokenizer(io.BytesIO(br"""%!PS + def test_case1(self): + tokenizer = PSTokenizer( + io.BytesIO( + br"""%!PS begin end " @ # /a/BCD /Some_Name /foo#5f#xbaa @@ -136,6 +132,8 @@ baa) 12345> func/a/b{(c)do*}def [ 1 (z) ! ] -<< /foo (bar) >>""")) - for token in tokenizer.iter_tokens(): - print(token) +<< /foo (bar) >>""" + ) + ) + for token in tokenizer.iter_tokens(): + print(token)