moved num parsing
This commit is contained in:
1 parent
0eb5e4b8fc
commit
c752346638
12 files changed
+168
-64
No files matched your search
@@ -14,3 +14,10 @@ class InvalidHexToken(LexerError):
|
||||
super().__init__(
|
||||
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
|
||||
)
|
||||
|
||||
|
||||
class InvalidToken(LexerError):
|
||||
def __init__(self, pos: int, token: bytes) -> None:
|
||||
super().__init__(
|
||||
"Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token))
|
||||
)
|
||||
@@ -0,0 +1,54 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError
|
||||
from pdfmajor.lexer.token import (
|
||||
TokenDecimal,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_NUMBER,
|
||||
)
|
||||
|
||||
|
||||
def parse_number(buffer: SafeBufferIt, sign: bytes = b"+") -> TokenNumber:
|
||||
"""Parses input stream into a number
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
sign (str): either a + or -
|
||||
|
||||
Returns:
|
||||
TokenNumber
|
||||
"""
|
||||
if sign not in [b"+", b"-"]:
|
||||
raise LexerError("Number was provided an invalid sign {!r}".format(sign))
|
||||
curtoken = b"" + sign
|
||||
is_decimal = False
|
||||
initialpos = buffer.tell() - 1
|
||||
for pos, buf in buffer:
|
||||
m = END_NUMBER.search(buf, 0)
|
||||
if not m:
|
||||
curtoken += buf
|
||||
else:
|
||||
j = m.start(0)
|
||||
c = buf[j : j + 1]
|
||||
if (c == b".") and not is_decimal:
|
||||
is_decimal = True
|
||||
buffer.seek(pos + j + 1)
|
||||
curtoken += buf[: j + 1]
|
||||
else:
|
||||
curtoken += buf[:j]
|
||||
buffer.seek(pos + j)
|
||||
try:
|
||||
if is_decimal:
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
buffer.tell(),
|
||||
Decimal(curtoken.decode()),
|
||||
)
|
||||
else:
|
||||
return TokenInteger(initialpos, buffer.tell(), int(curtoken))
|
||||
except ValueError:
|
||||
raise InvalidToken(initialpos, curtoken)
|
||||
raise LexerEOF
|
||||
@@ -74,7 +74,7 @@ class PSTokenizer:
|
||||
elif c == b"/":
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_literal(bufpos + j + 1, self.iter_buffer())
|
||||
cur, parse_literal(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c in b"-+":
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
@@ -116,5 +116,5 @@ class PSTokenizer:
|
||||
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
|
||||
|
||||
def _check_token(self, cur: PInput, token: Token) -> Token:
|
||||
self.inc_buf_skipstep(token.size)
|
||||
self.inc_buf_skipstep(token.end_loc - cur.pos)
|
||||
return token
|
||||
@@ -8,22 +8,25 @@ T = TypeVar("T")
|
||||
class Token(Generic[T]):
|
||||
__slots__ = ["pos", "size", "value"]
|
||||
|
||||
def __init__(self, pos: int, size: int, value: T) -> None:
|
||||
self.size = size
|
||||
self.pos = pos
|
||||
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
|
||||
self.start_loc = start_loc
|
||||
self.end_loc = end_loc
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "{cls}(pos={pos}, size={size}, value={value})".format(
|
||||
cls=self.__class__.__name__, pos=self.pos, value=self.value, size=self.size
|
||||
return "{cls}(start={start_loc}, end={end_loc}, value='{value}')".format(
|
||||
cls=self.__class__.__name__,
|
||||
start_loc=self.start_loc,
|
||||
value=self.value,
|
||||
end_loc=self.end_loc,
|
||||
)
|
||||
|
||||
def __eq__(self, other: Any):
|
||||
if isinstance(other, self.__class__):
|
||||
return (
|
||||
(other.value == self.value)
|
||||
and (other.pos == self.pos)
|
||||
and (other.size == self.size)
|
||||
and (other.start_loc == self.start_loc)
|
||||
and (other.end_loc == self.end_loc)
|
||||
)
|
||||
else:
|
||||
return False
|
||||
|
||||
@@ -27,5 +27,5 @@ def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
return TokenComment(initialpos, cmp_tsize(curpos, initialpos, j), curtoken)
|
||||
return TokenComment(initialpos, curpos + j, curtoken)
|
||||
raise TokenizerEOF
|
||||
@@ -1,6 +1,7 @@
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerError
|
||||
from pdfmajor.tokenizer.constants import END_HEX_STRING, HEX_PAIR, SPC
|
||||
from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenHexString
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.utils import int2byte
|
||||
from typing import Iterator, Union
|
||||
|
||||
@@ -27,6 +28,7 @@ def parse_double_angled_bracket(initialpos: int, inp: Iterator[PInput]) -> Token
|
||||
curtoken = parse_hexstring(initialpos, step, curtoken)
|
||||
if isinstance(curtoken, TokenHexString):
|
||||
return curtoken
|
||||
raise TokenizerError("parse_double_angled_bracket ended")
|
||||
|
||||
|
||||
def parse_hexstring(
|
||||
@@ -41,4 +43,4 @@ def parse_hexstring(
|
||||
token = HEX_PAIR.sub(
|
||||
lambda x: int2byte(int(x.group(0), 16)), SPC.sub(b"", curtoken)
|
||||
)
|
||||
return TokenHexString(initialpos, cmp_tsize(inp.pos, initialpos, j + 1), token)
|
||||
return TokenHexString(initialpos, inp.pos + j + 1, token)
|
||||
@@ -2,7 +2,7 @@ from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword
|
||||
from pdfmajor.tokenizer.constants import END_KEYWORD
|
||||
from typing import Iterator, Union
|
||||
from .util import cmp_tsize, PInput
|
||||
from .util import PInput
|
||||
|
||||
|
||||
def parse_keyword(
|
||||
@@ -26,11 +26,9 @@ def parse_keyword(
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
if curtoken == b"true":
|
||||
return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), True)
|
||||
return TokenBoolean(initialpos, curpos + j, True)
|
||||
elif curtoken == b"false":
|
||||
return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), False)
|
||||
return TokenBoolean(initialpos, curpos + j, False)
|
||||
else:
|
||||
return TokenKeyword(
|
||||
initialpos, cmp_tsize(curpos, initialpos, j), curtoken
|
||||
)
|
||||
return TokenKeyword(initialpos, curpos + j, curtoken)
|
||||
raise TokenizerEOF
|
||||
@@ -1,12 +1,10 @@
|
||||
from dataclasses import dataclass
|
||||
from pdfmajor.tokenizer.exceptions import (
|
||||
InvalidHexToken,
|
||||
InvalidToken,
|
||||
TokenizerEOF,
|
||||
TokenizerError,
|
||||
)
|
||||
|
||||
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize
|
||||
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token import TokenLiteral
|
||||
from pdfmajor.utils import int2byte
|
||||
@@ -48,14 +46,14 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
else:
|
||||
raise InvalidHexToken(curpos + ci, c)
|
||||
else:
|
||||
m = END_LITERAL.search(buf[it.skip :], 0)
|
||||
m = END_LITERAL.search(subbuf, 0)
|
||||
if not m:
|
||||
state.curtoken += buf[it.skip :]
|
||||
state.curtoken += subbuf
|
||||
it.skip = len(buf)
|
||||
else:
|
||||
j: int = m.start(0) + it.skip
|
||||
state.curtoken += buf[it.skip : j]
|
||||
c = buf[j : j + 1]
|
||||
j: int = m.start(0)
|
||||
state.curtoken += subbuf[:j]
|
||||
c = subbuf[j : j + 1]
|
||||
if c == b"#":
|
||||
state.hex_value = b""
|
||||
it.skip = j + 1
|
||||
@@ -63,7 +61,7 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
else:
|
||||
return TokenLiteral(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curpos + it.skip + j,
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
raise TokenizerEOF
|
||||
@@ -1,7 +1,5 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
|
||||
from pdfmajor.utils import int2byte
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF, TokenizerError
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenDecimal,
|
||||
TokenNumber,
|
||||
@@ -10,8 +8,8 @@ from pdfmajor.tokenizer.token import (
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_NUMBER,
|
||||
)
|
||||
from typing import Iterator, Literal, Union
|
||||
from .util import cmp_tsize, PInput
|
||||
from typing import Iterator
|
||||
from .util import PInput
|
||||
|
||||
|
||||
def parse_number(
|
||||
@@ -28,7 +26,7 @@ def parse_number(
|
||||
TokenNumber
|
||||
"""
|
||||
if sign not in [b"+", b"-"]:
|
||||
raise InvalidToken("Number was provided an invalid sign {!r}".format(sign))
|
||||
raise TokenizerError("Number was provided an invalid sign {!r}".format(sign))
|
||||
curtoken = b"" + sign
|
||||
is_dec = False
|
||||
for curpos, s in inp:
|
||||
@@ -53,20 +51,18 @@ def parse_number(
|
||||
j += nj
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j + 1),
|
||||
curpos + j + 1,
|
||||
Decimal(curtoken.decode()),
|
||||
)
|
||||
try:
|
||||
if is_dec:
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curpos + j,
|
||||
Decimal(curtoken.decode()),
|
||||
)
|
||||
else:
|
||||
return TokenInteger(
|
||||
initialpos, cmp_tsize(curpos, initialpos, j), int(curtoken)
|
||||
)
|
||||
return TokenInteger(initialpos, curpos + j, int(curtoken))
|
||||
except ValueError:
|
||||
raise InvalidToken(initialpos, curtoken)
|
||||
raise TokenizerEOF
|
||||
@@ -1,15 +1,18 @@
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerError
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenString,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_STRING,
|
||||
ESC_STRING,
|
||||
OCT_STRING,
|
||||
)
|
||||
from typing import Iterator, Optional
|
||||
|
||||
OCT_STRING = re.compile(br"^[0-7]{1,3}$")
|
||||
|
||||
|
||||
@dataclass
|
||||
class StringParseState:
|
||||
@@ -33,12 +36,12 @@ def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString:
|
||||
it = SafeBufferIt(buf)
|
||||
for s in it.into_iter():
|
||||
if state.escaped_value is not None:
|
||||
it.skip += parse_string_escape(s, state)
|
||||
parse_string_escape(it, state)
|
||||
else:
|
||||
m = END_STRING.search(s, 0)
|
||||
if not m:
|
||||
state.curtoken += s
|
||||
it.skip = len(s)
|
||||
it.skip += len(s)
|
||||
else:
|
||||
j = m.start(0)
|
||||
state.curtoken += s[:j]
|
||||
@@ -55,25 +58,35 @@ def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString:
|
||||
if state.paren:
|
||||
state.curtoken += next_char
|
||||
it.skip += j + 1
|
||||
return TokenString(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
else:
|
||||
return TokenString(
|
||||
initialpos,
|
||||
curpos + it.skip + j + 1,
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
raise TokenizerError("end of parse_string")
|
||||
|
||||
|
||||
def parse_string_escape(s: bytes, state: StringParseState) -> int:
|
||||
for ci in range(len(s)):
|
||||
c = s[ci : ci + 1]
|
||||
if (ci < 1) and (c in ESC_STRING):
|
||||
state.curtoken += bytes([ESC_STRING[c]])
|
||||
state.escaped_value = None
|
||||
return 1
|
||||
if OCT_STRING.match(c) and len(state.escaped_value) < 3:
|
||||
state.escaped_value += c
|
||||
elif len(state.escaped_value) == 3:
|
||||
state.curtoken += bytes([int(state.escaped_value, 8)])
|
||||
return 3
|
||||
|
||||
state.escaped_value = None
|
||||
return len(s) + 1
|
||||
def parse_string_escape(it: SafeBufferIt, state: StringParseState):
|
||||
if state.escaped_value is None:
|
||||
raise TokenizerError("state.escaped_value is None")
|
||||
for s in it.into_iter():
|
||||
for ci in range(len(s)):
|
||||
c = s[ci : ci + 1]
|
||||
if len(state.escaped_value) < 3:
|
||||
it.skip += 1
|
||||
state.escaped_value += c
|
||||
if len(state.escaped_value) == 1:
|
||||
if (ci < 1) and (c in ESC_STRING):
|
||||
state.curtoken += bytes([ESC_STRING[c]])
|
||||
state.escaped_value = None
|
||||
return
|
||||
elif OCT_STRING.match(state.escaped_value):
|
||||
if len(state.escaped_value) == 3:
|
||||
state.curtoken += bytes([int(state.escaped_value, 8)])
|
||||
state.escaped_value = None
|
||||
return
|
||||
else:
|
||||
it.skip -= 1
|
||||
state.escaped_value = None
|
||||
return
|
||||
@@ -26,6 +26,13 @@ def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
|
||||
|
||||
|
||||
class SafeBufferIt:
|
||||
"""A safe while-loop to use on a buffer
|
||||
this class will make sure that there is no risk of an infinite loop and that the max number of loops is reasonable
|
||||
|
||||
Raises:
|
||||
TokenizerEOF: when we reach a maximum iteration we raise an 'end of file' error
|
||||
"""
|
||||
|
||||
__slots__ = ["skip", "buffer"]
|
||||
|
||||
def __init__(self, buffer: bytes, skip: int = 0) -> None:
|
||||
@@ -33,8 +40,8 @@ class SafeBufferIt:
|
||||
self.buffer = buffer
|
||||
|
||||
def into_iter(self) -> Iterator[bytes]:
|
||||
for i in range(len(self.buffer) + 1):
|
||||
if i > len(self.buffer):
|
||||
for i in range(2 * len(self.buffer) + 1):
|
||||
if i > 2 * len(self.buffer):
|
||||
raise TokenizerEOF("Max Iteration Reached!")
|
||||
if self.skip >= len(self.buffer):
|
||||
break
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
from pdfmajor.lexer.number import parse_number
|
||||
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
|
||||
from pdfmajor.lexer.string import parse_string
|
||||
from unittest import TestCase
|
||||
@@ -10,8 +12,10 @@ from pdfmajor.lexer.token import (
|
||||
TDictVaue,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenDictionary,
|
||||
TokenHexString,
|
||||
TokenInteger,
|
||||
TokenName,
|
||||
TokenString,
|
||||
)
|
||||
@@ -164,3 +168,25 @@ class Basics(TestCase):
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, b"\x10 1\xa1*")
|
||||
self.assertEqual(token.end_loc, 14)
|
||||
|
||||
def test_parse_float(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(b"0.120 + some invalid text", buf_size)
|
||||
token = parse_number(buffer)
|
||||
self.assertIsInstance(token, TokenDecimal)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, Decimal("0.12"))
|
||||
self.assertEqual(token.end_loc, 5)
|
||||
|
||||
def test_parse_int(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(
|
||||
b"130 oh oh 0.12 + some invalid text", buf_size
|
||||
)
|
||||
token = parse_number(buffer)
|
||||
self.assertIsInstance(token, TokenInteger)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, 130)
|
||||
self.assertEqual(token.end_loc, 3)
|
||||
Reference in new issue
Block a user