generalized form buf iteration

This commit is contained in:
ari committed 2020-10-13 21:35:17 -04:00
1 parent be2778bffc
commit d8be2885b8
4 files changed
+53 -39

No files matched your search

+3 -3
View File
@@ -11,11 +11,11 @@ def parse_keyword(
"""Parses input stream into a keyword or bool """Parses input stream into a keyword or bool
Args: Args:
initialpos (int): initial position where we started initialpos (int): initial position where we started
inp (Iterator[PInput]) inp (Iterator[PInput])
Returns: Returns:
Union[TokenKeyword, TokenBoolean] Union[TokenKeyword, TokenBoolean]
""" """
curtoken = b"" curtoken = b""
for curpos, s in inp: for curpos, s in inp:
+12 -16
View File
@@ -1,7 +1,7 @@
from dataclasses import dataclass from dataclasses import dataclass
from pdfmajor.tokenizer.exceptions import TokenizerEOF, TokenizerError from pdfmajor.tokenizer.exceptions import TokenizerEOF, TokenizerError
from pdfmajor.tokenizer.token_parsers.util import cmp_tsize from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize
from pdfmajor.tokenizer.token_parsers.util import PInput from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token import TokenLiteral from pdfmajor.tokenizer.token import TokenLiteral
from pdfmajor.utils import int2byte from pdfmajor.utils import int2byte
@@ -28,35 +28,31 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
""" """
state = LiteralParseState(b"") state = LiteralParseState(b"")
for curpos, buf in inp: for curpos, buf in inp:
skip: int = 0 it = SafeBufferIt(buf)
for i in range(len(buf) + 1): for _ in it.into_iter():
if i < len(buf):
raise TokenizerEOF("Max Iteration Reached!")
if skip >= len(buf):
break
if state.hex_value is not None: if state.hex_value is not None:
for ci in range(len(buf[skip:])): for ci in range(len(buf[it.skip :])):
c = buf[skip + ci : skip + ci + 1] c = buf[it.skip + ci : it.skip + ci + 1]
if HEX.match(c) and len(state.hex_value) < 2: if HEX.match(c) and len(state.hex_value) < 2:
state.hex_value += c state.hex_value += c
else: else:
if state.hex_value: if state.hex_value:
state.curtoken += int2byte(int(state.hex_value, 16)) state.curtoken += int2byte(int(state.hex_value, 16))
state.hex_value = None state.hex_value = None
skip += ci it.skip += ci
break break
else: else:
m = END_LITERAL.search(buf[skip:], 0) m = END_LITERAL.search(buf[it.skip :], 0)
if not m: if not m:
state.curtoken += buf[skip:] state.curtoken += buf[it.skip :]
skip = len(buf) it.skip = len(buf)
else: else:
j: int = m.start(0) + skip j: int = m.start(0) + it.skip
state.curtoken += buf[skip:j] state.curtoken += buf[it.skip : j]
c = buf[j : j + 1] c = buf[j : j + 1]
if c == b"#": if c == b"#":
state.hex_value = b"" state.hex_value = b""
skip = j + 1 it.skip = j + 1
continue continue
else: else:
return TokenLiteral( return TokenLiteral(
+21 -20
View File
@@ -1,33 +1,34 @@
from decimal import Decimal from dataclasses import dataclass
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize
from pdfmajor.utils import int2byte from pdfmajor.utils import int2byte
from pdfmajor.tokenizer.exceptions import InvalidToken
from pdfmajor.tokenizer.token import ( from pdfmajor.tokenizer.token import (
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenKeyword,
TokenLiteral,
TokenNumber,
TokenInteger,
TokenString, TokenString,
) )
from pdfmajor.tokenizer.constants import ( from pdfmajor.tokenizer.constants import (
END_KEYWORD,
END_NUMBER,
END_STRING, END_STRING,
EOL,
ESC_STRING, ESC_STRING,
OCT_STRING, OCT_STRING,
) )
from typing import Callable, Iterator, Literal, NamedTuple, Union from typing import Iterator, Optional
@dataclass
class StringParseState:
curtoken: bytes
paren: int
oct_value: Optional[bytes] = None
def parse_string(initialpos: int, inp: Iterator[PInput]): def parse_string(initialpos: int, inp: Iterator[PInput]):
curtoken = b"" state = StringParseState(b"", 1)
paren = 1 for curpos, buf in inp:
for curpos, s in inp: skip: int = 0
for i in range(len(buf) + 1):
if i < len(buf):
raise TokenizerEOF("Max Iteration Reached!")
if skip >= len(buf):
break
m = END_STRING.search(s, 0) m = END_STRING.search(s, 0)
if not m: if not m:
curtoken += s curtoken += s
@@ -36,7 +37,7 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
curtoken += s[:j] curtoken += s[:j]
c = s[j : j + 1] c = s[j : j + 1]
if c == b"\\": if c == b"\\":
self._current_parse_func = self._parse_string_1 _parse_string_1
return j + 1 return j + 1
elif c == b"(": elif c == b"(":
paren += 1 paren += 1
@@ -44,7 +45,7 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
return j + 1 return j + 1
elif c == b")": elif c == b")":
paren -= 1 paren -= 1
if paren: # WTF, they said balanced parens need no special treatment. if paren:
curtoken += c curtoken += c
return j + 1 return j + 1
return TokenString( return TokenString(
+17
View File
@@ -1,3 +1,4 @@
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token import Token from pdfmajor.tokenizer.token import Token
from typing import Callable, Iterator, NamedTuple from typing import Callable, Iterator, NamedTuple
@@ -22,3 +23,19 @@ def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
int: length of index int: length of index
""" """
return curpos - initialpos + tokenend_idx return curpos - initialpos + tokenend_idx
class SafeBufferIt:
__slots__ = ["skip", "buffer"]
def __init__(self, buffer: bytes, skip: int = 0) -> None:
self.skip = skip
self.buffer = buffer
def into_iter(self) -> Iterator[bytes]:
for i in range(len(self.buffer) + 1):
if i > len(self.buffer):
raise TokenizerEOF("Max Iteration Reached!")
if self.skip >= len(self.buffer):
break
yield self.buffer[self.skip :]