diff --git a/pdfmajor/tokenizer/token_parsers/string.py b/pdfmajor/tokenizer/token_parsers/string.py index 601d0c6..5d16778 100644 --- a/pdfmajor/tokenizer/token_parsers/string.py +++ b/pdfmajor/tokenizer/token_parsers/string.py @@ -1,7 +1,5 @@ from dataclasses import dataclass -from pdfmajor.tokenizer.exceptions import TokenizerEOF from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize -from pdfmajor.utils import int2byte from pdfmajor.tokenizer.token import ( TokenString, ) @@ -17,7 +15,7 @@ from typing import Iterator, Optional class StringParseState: curtoken: bytes paren: int - oct_value: Optional[bytes] = None + escaped_value: Optional[bytes] = None def parse_string(initialpos: int, inp: Iterator[PInput]): @@ -25,27 +23,28 @@ def parse_string(initialpos: int, inp: Iterator[PInput]): for curpos, buf in inp: it = SafeBufferIt(buf) for s in it.into_iter(): - if state.oct_value is not None: - it.skip += parse_string_oct(s, state) + if state.escaped_value is not None: + it.skip += parse_string_escape(s, state) else: m = END_STRING.search(s, 0) if not m: state.curtoken += s + it.skip = len(s) else: j = m.start(0) state.curtoken += s[:j] - c = s[j : j + 1] - if c == b"\\": - state.oct_value = b"" + next_char = s[j : j + 1] + if next_char == b"\\": + state.escaped_value = b"" it.skip += j + 1 - elif c == b"(": + elif next_char == b"(": state.paren += 1 - state.curtoken += c + state.curtoken += next_char it.skip += j + 1 - elif c == b")": + elif next_char == b")": state.paren -= 1 if state.paren: - state.curtoken += c + state.curtoken += next_char it.skip += j + 1 return TokenString( initialpos, @@ -54,17 +53,18 @@ def parse_string(initialpos: int, inp: Iterator[PInput]): ) -def parse_string_oct(s: bytes, state: StringParseState) -> int: +def parse_string_escape(s: bytes, state: StringParseState) -> int: for ci in range(len(s)): c = s[ci : ci + 1] - if OCT_STRING.match(c) and len(state.oct_value) < 3: - state.oct_value += c - elif state.oct_value: - state.curtoken += int2byte(int(state.oct_value, 8)) - return ci - elif c in ESC_STRING: - state.curtoken += int2byte(ESC_STRING[c]) - state.oct_value = None - return ci + 1 - state.oct_value = None + if (ci < 1) and (c in ESC_STRING): + state.curtoken += bytes([ESC_STRING[c]]) + state.escaped_value = None + return 1 + if OCT_STRING.match(c) and len(state.escaped_value) < 3: + state.escaped_value += c + elif len(state.escaped_value) == 3: + state.curtoken += bytes([int(state.escaped_value, 8)]) + return 3 + + state.escaped_value = None return len(s) + 1 diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 5130e72..ec761bd 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -1,5 +1,6 @@ -from decimal import Decimal import io +from random import randint +from decimal import Decimal from typing import List from pdfmajor.tokenizer.token_parsers.string import parse_string from pdfmajor.tokenizer import PSTokenizer @@ -13,7 +14,7 @@ from pdfmajor.tokenizer.token import ( TokenLiteral, TokenString, ) -from pdfmajor.tokenizer.token_parsers.util import PInput +from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt from pdfmajor.tokenizer.token_parsers.comment import parse_comment from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword from pdfmajor.tokenizer.token_parsers.number import parse_number @@ -36,6 +37,16 @@ def make_stream_iter(data: bytes, initpos: int = 0): return iter_stream +class Util(TestCase): + def test_safebuff(self): + it = SafeBufferIt( + b"this is a lengthy comment that ends here\nso this is not reachable" + ) + for subbuf in it.into_iter(): + self.assertEqual(len(subbuf), len(it.buffer) - it.skip) + it.skip += randint(1, max(len(subbuf) - 1, 1)) + + class Individual(TestCase): def test_comment_parser_ln(self): for buf_size in [2, 7, 11, 150]: