From 370a1b40a15bc56a102882c9a27f56325b3df982 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Fri, 16 Oct 2020 16:31:02 -0400 Subject: [PATCH] moved string parsing to new method --- pdfmajor/lexer/regex.py | 12 ++++++ pdfmajor/lexer/string.py | 80 ++++++++++++++++++++++++++++++++++++++++ tests/test_lexer.py | 51 ++++++++++++++++++++----- 3 files changed, 133 insertions(+), 10 deletions(-) create mode 100644 pdfmajor/lexer/string.py diff --git a/pdfmajor/lexer/regex.py b/pdfmajor/lexer/regex.py index 6a59169..1354ddb 100644 --- a/pdfmajor/lexer/regex.py +++ b/pdfmajor/lexer/regex.py @@ -4,3 +4,15 @@ EOL = re.compile(br"[\r\n]") END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]") HEX = re.compile(br"[0-9a-fA-F]") END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]") +END_STRING = re.compile(br"[()\134]") +OCT_STRING = re.compile(br"^[0-7]{1,3}$") +ESC_STRING = { + b"b": 8, + b"t": 9, + b"n": 10, + b"f": 12, + b"r": 13, + b"(": 40, + b")": 41, + b"\\": 92, +} diff --git a/pdfmajor/lexer/string.py b/pdfmajor/lexer/string.py new file mode 100644 index 0000000..bab5683 --- /dev/null +++ b/pdfmajor/lexer/string.py @@ -0,0 +1,80 @@ +from dataclasses import dataclass +from typing import Optional +from pdfmajor.lexer.regex import END_STRING, ESC_STRING, OCT_STRING +from pdfmajor.lexer.token import TokenString +from pdfmajor.lexer.exceptions import LexerEOF, LexerError +from pdfmajor.safebufiterator import SafeBufferIt + + +@dataclass +class StringParseState: + curtoken: bytes + paren: int + escaped_value: Optional[bytes] = None + + +def parse_string(buffer: SafeBufferIt) -> TokenString: + """Parses input stream into a simple string object + + Args: + buffer (SafeBufferIt) + + Returns: + TokenString + """ + initialpos = buffer.tell() - 1 + state = StringParseState(b"", 1) + for curpos, buf in buffer: + if state.escaped_value is not None: + buffer.seek(curpos) + parse_string_escape(buffer, state) + else: + m = END_STRING.search(buf, 0) + if not m: + state.curtoken += buf + else: + j = m.start(0) + state.curtoken += buf[:j] + buffer.seek(curpos + j + 1) + next_char = buf[j : j + 1] + if next_char == b"\\": + state.escaped_value = b"" + elif next_char == b"(": + state.paren += 1 + state.curtoken += next_char + elif next_char == b")": + state.paren -= 1 + if state.paren: + state.curtoken += next_char + else: + return TokenString( + initialpos, + buffer.tell(), + state.curtoken.decode(), + ) + raise LexerEOF + + +def parse_string_escape(buffer: SafeBufferIt, state: StringParseState): + if state.escaped_value is None: + raise LexerError("state.escaped_value is None") + for pos, buf in buffer: + for ci in range(len(buf)): + c = buf[ci : ci + 1] + if len(state.escaped_value) < 3: + state.escaped_value += c + buffer.seek(pos + ci + 1) + if len(state.escaped_value) == 1: + if (ci < 1) and (c in ESC_STRING): + state.curtoken += bytes([ESC_STRING[c]]) + state.escaped_value = None + return + elif OCT_STRING.match(state.escaped_value): + if len(state.escaped_value) == 3: + state.curtoken += bytes([int(state.escaped_value, 8)]) + state.escaped_value = None + return + else: + state.escaped_value = None + buffer.seekd(-1) + return diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 3f0d248..503a5bb 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -1,10 +1,11 @@ import io +from pdfmajor.lexer.string import parse_string from unittest import TestCase from pdfmajor.lexer.name import parse_name from pdfmajor.lexer.keyword import parse_keyword from pdfmajor.safebufiterator import SafeBufferIt -from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName +from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName, TokenString from pdfmajor.lexer.comment import parse_comment @@ -83,12 +84,42 @@ class Basics(TestCase): self.assertEqual(token.value, "foo_aa") self.assertEqual(token.end_loc, 8) - # def test_parse_string(self): - # for buf_size in [2, 3, 50, 1000]: - # with self.subTest(buf_end_loc=buf_size): - # iter_stream = make_stream_iter(br"this % is not a comment.) ", 88) - # token = parse_string(88, iter_stream(buf_size)) - # self.assertIsInstance(token, TokenString) - # self.assertEqual(token.start_loc, 88) - # self.assertEqual(token.value, "this % is not a comment.") - # self.assertEqual(token.end_loc, 88 + 25) + def test_parse_string(self): + for buf_size in [2, 3, 50, 1000]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(br"this % is not a comment.) ", buf_size) + token = parse_string(buffer) + self.assertIsInstance(token, TokenString) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, "this % is not a comment.") + self.assertEqual(token.end_loc, 25) + + def test_parse_string_nest_brackets(self): + for buf_size in [1, 2, 3, 50, 100]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(br"x - (y*2-z-(2+3))) ", 88) + token = parse_string(buffer) + self.assertIsInstance(token, TokenString) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, "x - (y*2-z-(2+3))") + self.assertEqual(token.end_loc, 18) + + def test_parse_string_hex_chars(self): + for buf_size in [1, 2, 3, 50, 100]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(br"def\040\0\0404ghi) (bach\\slask) ", 39) + token = parse_string(buffer) + self.assertIsInstance(token, TokenString) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, "def 4ghi") + self.assertEqual(token.end_loc, 18) + + def test_parse_string_escape_char(self): + for buf_size in [1, 2, 3, 50, 100]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(br"bach\\slask) ", 39) + token = parse_string(buffer) + self.assertIsInstance(token, TokenString) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, r"bach\slask") + self.assertEqual(token.end_loc, 12)