diff --git a/pdfmajor/tokenizer/token_parsers/keyword.py b/pdfmajor/tokenizer/token_parsers/keyword.py index ed2d6ca..9e41ee8 100644 --- a/pdfmajor/tokenizer/token_parsers/keyword.py +++ b/pdfmajor/tokenizer/token_parsers/keyword.py @@ -11,11 +11,11 @@ def parse_keyword( """Parses input stream into a keyword or bool Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) + initialpos (int): initial position where we started + inp (Iterator[PInput]) Returns: - Union[TokenKeyword, TokenBoolean] + Union[TokenKeyword, TokenBoolean] """ curtoken = b"" for curpos, s in inp: diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py index 890639f..c682a40 100644 --- a/pdfmajor/tokenizer/token_parsers/literal.py +++ b/pdfmajor/tokenizer/token_parsers/literal.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from pdfmajor.tokenizer.exceptions import TokenizerEOF, TokenizerError -from pdfmajor.tokenizer.token_parsers.util import cmp_tsize +from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize from pdfmajor.tokenizer.token_parsers.util import PInput from pdfmajor.tokenizer.token import TokenLiteral from pdfmajor.utils import int2byte @@ -28,35 +28,31 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: """ state = LiteralParseState(b"") for curpos, buf in inp: - skip: int = 0 - for i in range(len(buf) + 1): - if i < len(buf): - raise TokenizerEOF("Max Iteration Reached!") - if skip >= len(buf): - break + it = SafeBufferIt(buf) + for _ in it.into_iter(): if state.hex_value is not None: - for ci in range(len(buf[skip:])): - c = buf[skip + ci : skip + ci + 1] + for ci in range(len(buf[it.skip :])): + c = buf[it.skip + ci : it.skip + ci + 1] if HEX.match(c) and len(state.hex_value) < 2: state.hex_value += c else: if state.hex_value: state.curtoken += int2byte(int(state.hex_value, 16)) state.hex_value = None - skip += ci + it.skip += ci break else: - m = END_LITERAL.search(buf[skip:], 0) + m = END_LITERAL.search(buf[it.skip :], 0) if not m: - state.curtoken += buf[skip:] - skip = len(buf) + state.curtoken += buf[it.skip :] + it.skip = len(buf) else: - j: int = m.start(0) + skip - state.curtoken += buf[skip:j] + j: int = m.start(0) + it.skip + state.curtoken += buf[it.skip : j] c = buf[j : j + 1] if c == b"#": state.hex_value = b"" - skip = j + 1 + it.skip = j + 1 continue else: return TokenLiteral( diff --git a/pdfmajor/tokenizer/token_parsers/string.py b/pdfmajor/tokenizer/token_parsers/string.py index b137503..3f2faac 100644 --- a/pdfmajor/tokenizer/token_parsers/string.py +++ b/pdfmajor/tokenizer/token_parsers/string.py @@ -1,33 +1,34 @@ -from decimal import Decimal -from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX +from dataclasses import dataclass +from pdfmajor.tokenizer.exceptions import TokenizerEOF +from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize from pdfmajor.utils import int2byte -from pdfmajor.tokenizer.exceptions import InvalidToken from pdfmajor.tokenizer.token import ( - Token, - TokenBoolean, - TokenComment, - TokenDecimal, - TokenKeyword, - TokenLiteral, - TokenNumber, - TokenInteger, TokenString, ) from pdfmajor.tokenizer.constants import ( - END_KEYWORD, - END_NUMBER, END_STRING, - EOL, ESC_STRING, OCT_STRING, ) -from typing import Callable, Iterator, Literal, NamedTuple, Union +from typing import Iterator, Optional + + +@dataclass +class StringParseState: + curtoken: bytes + paren: int + oct_value: Optional[bytes] = None def parse_string(initialpos: int, inp: Iterator[PInput]): - curtoken = b"" - paren = 1 - for curpos, s in inp: + state = StringParseState(b"", 1) + for curpos, buf in inp: + skip: int = 0 + for i in range(len(buf) + 1): + if i < len(buf): + raise TokenizerEOF("Max Iteration Reached!") + if skip >= len(buf): + break m = END_STRING.search(s, 0) if not m: curtoken += s @@ -36,7 +37,7 @@ def parse_string(initialpos: int, inp: Iterator[PInput]): curtoken += s[:j] c = s[j : j + 1] if c == b"\\": - self._current_parse_func = self._parse_string_1 + _parse_string_1 return j + 1 elif c == b"(": paren += 1 @@ -44,7 +45,7 @@ def parse_string(initialpos: int, inp: Iterator[PInput]): return j + 1 elif c == b")": paren -= 1 - if paren: # WTF, they said balanced parens need no special treatment. + if paren: curtoken += c return j + 1 return TokenString( diff --git a/pdfmajor/tokenizer/token_parsers/util.py b/pdfmajor/tokenizer/token_parsers/util.py index ce6d0be..77b057c 100644 --- a/pdfmajor/tokenizer/token_parsers/util.py +++ b/pdfmajor/tokenizer/token_parsers/util.py @@ -1,3 +1,4 @@ +from pdfmajor.tokenizer.exceptions import TokenizerEOF from pdfmajor.tokenizer.token import Token from typing import Callable, Iterator, NamedTuple @@ -22,3 +23,19 @@ def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int: int: length of index """ return curpos - initialpos + tokenend_idx + + +class SafeBufferIt: + __slots__ = ["skip", "buffer"] + + def __init__(self, buffer: bytes, skip: int = 0) -> None: + self.skip = skip + self.buffer = buffer + + def into_iter(self) -> Iterator[bytes]: + for i in range(len(self.buffer) + 1): + if i > len(self.buffer): + raise TokenizerEOF("Max Iteration Reached!") + if self.skip >= len(self.buffer): + break + yield self.buffer[self.skip :]