generalized form buf iteration
This commit is contained in:
1 parent
be2778bffc
commit
d8be2885b8
4 files changed
+53
-39
No files matched your search
@@ -11,11 +11,11 @@ def parse_keyword(
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dataclasses import dataclass
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF, TokenizerError
|
||||
|
||||
from pdfmajor.tokenizer.token_parsers.util import cmp_tsize
|
||||
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token import TokenLiteral
|
||||
from pdfmajor.utils import int2byte
|
||||
@@ -28,35 +28,31 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
"""
|
||||
state = LiteralParseState(b"")
|
||||
for curpos, buf in inp:
|
||||
skip: int = 0
|
||||
for i in range(len(buf) + 1):
|
||||
if i < len(buf):
|
||||
raise TokenizerEOF("Max Iteration Reached!")
|
||||
if skip >= len(buf):
|
||||
break
|
||||
it = SafeBufferIt(buf)
|
||||
for _ in it.into_iter():
|
||||
if state.hex_value is not None:
|
||||
for ci in range(len(buf[skip:])):
|
||||
c = buf[skip + ci : skip + ci + 1]
|
||||
for ci in range(len(buf[it.skip :])):
|
||||
c = buf[it.skip + ci : it.skip + ci + 1]
|
||||
if HEX.match(c) and len(state.hex_value) < 2:
|
||||
state.hex_value += c
|
||||
else:
|
||||
if state.hex_value:
|
||||
state.curtoken += int2byte(int(state.hex_value, 16))
|
||||
state.hex_value = None
|
||||
skip += ci
|
||||
it.skip += ci
|
||||
break
|
||||
else:
|
||||
m = END_LITERAL.search(buf[skip:], 0)
|
||||
m = END_LITERAL.search(buf[it.skip :], 0)
|
||||
if not m:
|
||||
state.curtoken += buf[skip:]
|
||||
skip = len(buf)
|
||||
state.curtoken += buf[it.skip :]
|
||||
it.skip = len(buf)
|
||||
else:
|
||||
j: int = m.start(0) + skip
|
||||
state.curtoken += buf[skip:j]
|
||||
j: int = m.start(0) + it.skip
|
||||
state.curtoken += buf[it.skip : j]
|
||||
c = buf[j : j + 1]
|
||||
if c == b"#":
|
||||
state.hex_value = b""
|
||||
skip = j + 1
|
||||
it.skip = j + 1
|
||||
continue
|
||||
else:
|
||||
return TokenLiteral(
|
||||
|
||||
@@ -1,33 +1,34 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
|
||||
from dataclasses import dataclass
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize
|
||||
from pdfmajor.utils import int2byte
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken
|
||||
from pdfmajor.tokenizer.token import (
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenKeyword,
|
||||
TokenLiteral,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
TokenString,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_KEYWORD,
|
||||
END_NUMBER,
|
||||
END_STRING,
|
||||
EOL,
|
||||
ESC_STRING,
|
||||
OCT_STRING,
|
||||
)
|
||||
from typing import Callable, Iterator, Literal, NamedTuple, Union
|
||||
from typing import Iterator, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class StringParseState:
|
||||
curtoken: bytes
|
||||
paren: int
|
||||
oct_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
||||
curtoken = b""
|
||||
paren = 1
|
||||
for curpos, s in inp:
|
||||
state = StringParseState(b"", 1)
|
||||
for curpos, buf in inp:
|
||||
skip: int = 0
|
||||
for i in range(len(buf) + 1):
|
||||
if i < len(buf):
|
||||
raise TokenizerEOF("Max Iteration Reached!")
|
||||
if skip >= len(buf):
|
||||
break
|
||||
m = END_STRING.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
@@ -36,7 +37,7 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"\\":
|
||||
self._current_parse_func = self._parse_string_1
|
||||
_parse_string_1
|
||||
return j + 1
|
||||
elif c == b"(":
|
||||
paren += 1
|
||||
@@ -44,7 +45,7 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
|
||||
return j + 1
|
||||
elif c == b")":
|
||||
paren -= 1
|
||||
if paren: # WTF, they said balanced parens need no special treatment.
|
||||
if paren:
|
||||
curtoken += c
|
||||
return j + 1
|
||||
return TokenString(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.token import Token
|
||||
from typing import Callable, Iterator, NamedTuple
|
||||
|
||||
@@ -22,3 +23,19 @@ def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
|
||||
int: length of index
|
||||
"""
|
||||
return curpos - initialpos + tokenend_idx
|
||||
|
||||
|
||||
class SafeBufferIt:
|
||||
__slots__ = ["skip", "buffer"]
|
||||
|
||||
def __init__(self, buffer: bytes, skip: int = 0) -> None:
|
||||
self.skip = skip
|
||||
self.buffer = buffer
|
||||
|
||||
def into_iter(self) -> Iterator[bytes]:
|
||||
for i in range(len(self.buffer) + 1):
|
||||
if i > len(self.buffer):
|
||||
raise TokenizerEOF("Max Iteration Reached!")
|
||||
if self.skip >= len(self.buffer):
|
||||
break
|
||||
yield self.buffer[self.skip :]
|
||||
Reference in new issue
Block a user