generalized form buf iteration
This commit is contained in:
1 parent
be2778bffc
commit
d8be2885b8
4 files changed
+53
-39
No files matched your search
@@ -11,11 +11,11 @@ def parse_keyword(
|
|||||||
"""Parses input stream into a keyword or bool
|
"""Parses input stream into a keyword or bool
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
initialpos (int): initial position where we started
|
initialpos (int): initial position where we started
|
||||||
inp (Iterator[PInput])
|
inp (Iterator[PInput])
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Union[TokenKeyword, TokenBoolean]
|
Union[TokenKeyword, TokenBoolean]
|
||||||
"""
|
"""
|
||||||
curtoken = b""
|
curtoken = b""
|
||||||
for curpos, s in inp:
|
for curpos, s in inp:
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF, TokenizerError
|
from pdfmajor.tokenizer.exceptions import TokenizerEOF, TokenizerError
|
||||||
|
|
||||||
from pdfmajor.tokenizer.token_parsers.util import cmp_tsize
|
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize
|
||||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||||
from pdfmajor.tokenizer.token import TokenLiteral
|
from pdfmajor.tokenizer.token import TokenLiteral
|
||||||
from pdfmajor.utils import int2byte
|
from pdfmajor.utils import int2byte
|
||||||
@@ -28,35 +28,31 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
|||||||
"""
|
"""
|
||||||
state = LiteralParseState(b"")
|
state = LiteralParseState(b"")
|
||||||
for curpos, buf in inp:
|
for curpos, buf in inp:
|
||||||
skip: int = 0
|
it = SafeBufferIt(buf)
|
||||||
for i in range(len(buf) + 1):
|
for _ in it.into_iter():
|
||||||
if i < len(buf):
|
|
||||||
raise TokenizerEOF("Max Iteration Reached!")
|
|
||||||
if skip >= len(buf):
|
|
||||||
break
|
|
||||||
if state.hex_value is not None:
|
if state.hex_value is not None:
|
||||||
for ci in range(len(buf[skip:])):
|
for ci in range(len(buf[it.skip :])):
|
||||||
c = buf[skip + ci : skip + ci + 1]
|
c = buf[it.skip + ci : it.skip + ci + 1]
|
||||||
if HEX.match(c) and len(state.hex_value) < 2:
|
if HEX.match(c) and len(state.hex_value) < 2:
|
||||||
state.hex_value += c
|
state.hex_value += c
|
||||||
else:
|
else:
|
||||||
if state.hex_value:
|
if state.hex_value:
|
||||||
state.curtoken += int2byte(int(state.hex_value, 16))
|
state.curtoken += int2byte(int(state.hex_value, 16))
|
||||||
state.hex_value = None
|
state.hex_value = None
|
||||||
skip += ci
|
it.skip += ci
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
m = END_LITERAL.search(buf[skip:], 0)
|
m = END_LITERAL.search(buf[it.skip :], 0)
|
||||||
if not m:
|
if not m:
|
||||||
state.curtoken += buf[skip:]
|
state.curtoken += buf[it.skip :]
|
||||||
skip = len(buf)
|
it.skip = len(buf)
|
||||||
else:
|
else:
|
||||||
j: int = m.start(0) + skip
|
j: int = m.start(0) + it.skip
|
||||||
state.curtoken += buf[skip:j]
|
state.curtoken += buf[it.skip : j]
|
||||||
c = buf[j : j + 1]
|
c = buf[j : j + 1]
|
||||||
if c == b"#":
|
if c == b"#":
|
||||||
state.hex_value = b""
|
state.hex_value = b""
|
||||||
skip = j + 1
|
it.skip = j + 1
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
return TokenLiteral(
|
return TokenLiteral(
|
||||||
|
|||||||
@@ -1,33 +1,34 @@
|
|||||||
from decimal import Decimal
|
from dataclasses import dataclass
|
||||||
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
|
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||||
|
from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize
|
||||||
from pdfmajor.utils import int2byte
|
from pdfmajor.utils import int2byte
|
||||||
from pdfmajor.tokenizer.exceptions import InvalidToken
|
|
||||||
from pdfmajor.tokenizer.token import (
|
from pdfmajor.tokenizer.token import (
|
||||||
Token,
|
|
||||||
TokenBoolean,
|
|
||||||
TokenComment,
|
|
||||||
TokenDecimal,
|
|
||||||
TokenKeyword,
|
|
||||||
TokenLiteral,
|
|
||||||
TokenNumber,
|
|
||||||
TokenInteger,
|
|
||||||
TokenString,
|
TokenString,
|
||||||
)
|
)
|
||||||
from pdfmajor.tokenizer.constants import (
|
from pdfmajor.tokenizer.constants import (
|
||||||
END_KEYWORD,
|
|
||||||
END_NUMBER,
|
|
||||||
END_STRING,
|
END_STRING,
|
||||||
EOL,
|
|
||||||
ESC_STRING,
|
ESC_STRING,
|
||||||
OCT_STRING,
|
OCT_STRING,
|
||||||
)
|
)
|
||||||
from typing import Callable, Iterator, Literal, NamedTuple, Union
|
from typing import Iterator, Optional
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class StringParseState:
|
||||||
|
curtoken: bytes
|
||||||
|
paren: int
|
||||||
|
oct_value: Optional[bytes] = None
|
||||||
|
|
||||||
|
|
||||||
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
||||||
curtoken = b""
|
state = StringParseState(b"", 1)
|
||||||
paren = 1
|
for curpos, buf in inp:
|
||||||
for curpos, s in inp:
|
skip: int = 0
|
||||||
|
for i in range(len(buf) + 1):
|
||||||
|
if i < len(buf):
|
||||||
|
raise TokenizerEOF("Max Iteration Reached!")
|
||||||
|
if skip >= len(buf):
|
||||||
|
break
|
||||||
m = END_STRING.search(s, 0)
|
m = END_STRING.search(s, 0)
|
||||||
if not m:
|
if not m:
|
||||||
curtoken += s
|
curtoken += s
|
||||||
@@ -36,7 +37,7 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
|
|||||||
curtoken += s[:j]
|
curtoken += s[:j]
|
||||||
c = s[j : j + 1]
|
c = s[j : j + 1]
|
||||||
if c == b"\\":
|
if c == b"\\":
|
||||||
self._current_parse_func = self._parse_string_1
|
_parse_string_1
|
||||||
return j + 1
|
return j + 1
|
||||||
elif c == b"(":
|
elif c == b"(":
|
||||||
paren += 1
|
paren += 1
|
||||||
@@ -44,7 +45,7 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
|
|||||||
return j + 1
|
return j + 1
|
||||||
elif c == b")":
|
elif c == b")":
|
||||||
paren -= 1
|
paren -= 1
|
||||||
if paren: # WTF, they said balanced parens need no special treatment.
|
if paren:
|
||||||
curtoken += c
|
curtoken += c
|
||||||
return j + 1
|
return j + 1
|
||||||
return TokenString(
|
return TokenString(
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||||
from pdfmajor.tokenizer.token import Token
|
from pdfmajor.tokenizer.token import Token
|
||||||
from typing import Callable, Iterator, NamedTuple
|
from typing import Callable, Iterator, NamedTuple
|
||||||
|
|
||||||
@@ -22,3 +23,19 @@ def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
|
|||||||
int: length of index
|
int: length of index
|
||||||
"""
|
"""
|
||||||
return curpos - initialpos + tokenend_idx
|
return curpos - initialpos + tokenend_idx
|
||||||
|
|
||||||
|
|
||||||
|
class SafeBufferIt:
|
||||||
|
__slots__ = ["skip", "buffer"]
|
||||||
|
|
||||||
|
def __init__(self, buffer: bytes, skip: int = 0) -> None:
|
||||||
|
self.skip = skip
|
||||||
|
self.buffer = buffer
|
||||||
|
|
||||||
|
def into_iter(self) -> Iterator[bytes]:
|
||||||
|
for i in range(len(self.buffer) + 1):
|
||||||
|
if i > len(self.buffer):
|
||||||
|
raise TokenizerEOF("Max Iteration Reached!")
|
||||||
|
if self.skip >= len(self.buffer):
|
||||||
|
break
|
||||||
|
yield self.buffer[self.skip :]
|
||||||
Reference in new issue
Block a user