moved num parsing

This commit is contained in:
ari committed 2020-10-16 17:07:40 -04:00
1 parent 0eb5e4b8fc
commit c752346638
12 files changed
+168 -64

No files matched your search

+7
View File
@@ -14,3 +14,10 @@ class InvalidHexToken(LexerError):
super().__init__(
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
)
class InvalidToken(LexerError):
def __init__(self, pos: int, token: bytes) -> None:
super().__init__(
"Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token))
)
+54
View File
@@ -0,0 +1,54 @@
from decimal import Decimal
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError
from pdfmajor.lexer.token import (
TokenDecimal,
TokenNumber,
TokenInteger,
)
from pdfmajor.tokenizer.constants import (
END_NUMBER,
)
def parse_number(buffer: SafeBufferIt, sign: bytes = b"+") -> TokenNumber:
"""Parses input stream into a number
Args:
buffer (SafeBufferIt)
sign (str): either a + or -
Returns:
TokenNumber
"""
if sign not in [b"+", b"-"]:
raise LexerError("Number was provided an invalid sign {!r}".format(sign))
curtoken = b"" + sign
is_decimal = False
initialpos = buffer.tell() - 1
for pos, buf in buffer:
m = END_NUMBER.search(buf, 0)
if not m:
curtoken += buf
else:
j = m.start(0)
c = buf[j : j + 1]
if (c == b".") and not is_decimal:
is_decimal = True
buffer.seek(pos + j + 1)
curtoken += buf[: j + 1]
else:
curtoken += buf[:j]
buffer.seek(pos + j)
try:
if is_decimal:
return TokenDecimal(
initialpos,
buffer.tell(),
Decimal(curtoken.decode()),
)
else:
return TokenInteger(initialpos, buffer.tell(), int(curtoken))
except ValueError:
raise InvalidToken(initialpos, curtoken)
raise LexerEOF
+2 -2
View File
@@ -74,7 +74,7 @@ class PSTokenizer:
elif c == b"/":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_literal(bufpos + j + 1, self.iter_buffer())
cur, parse_literal(bufpos + j, self.iter_buffer())
)
elif c in b"-+":
self.inc_buf_skipstep(j + 1)
@@ -116,5 +116,5 @@ class PSTokenizer:
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
def _check_token(self, cur: PInput, token: Token) -> Token:
self.inc_buf_skipstep(token.size)
self.inc_buf_skipstep(token.end_loc - cur.pos)
return token
+10 -7
View File
@@ -8,22 +8,25 @@ T = TypeVar("T")
class Token(Generic[T]):
__slots__ = ["pos", "size", "value"]
def __init__(self, pos: int, size: int, value: T) -> None:
self.size = size
self.pos = pos
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
self.start_loc = start_loc
self.end_loc = end_loc
self.value = value
def __repr__(self) -> str:
return "{cls}(pos={pos}, size={size}, value={value})".format(
cls=self.__class__.__name__, pos=self.pos, value=self.value, size=self.size
return "{cls}(start={start_loc}, end={end_loc}, value='{value}')".format(
cls=self.__class__.__name__,
start_loc=self.start_loc,
value=self.value,
end_loc=self.end_loc,
)
def __eq__(self, other: Any):
if isinstance(other, self.__class__):
return (
(other.value == self.value)
and (other.pos == self.pos)
and (other.size == self.size)
and (other.start_loc == self.start_loc)
and (other.end_loc == self.end_loc)
)
else:
return False
+1 -1
View File
@@ -27,5 +27,5 @@ def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
else:
j = m.start(0)
curtoken += s[:j]
return TokenComment(initialpos, cmp_tsize(curpos, initialpos, j), curtoken)
return TokenComment(initialpos, curpos + j, curtoken)
raise TokenizerEOF
@@ -1,6 +1,7 @@
from pdfmajor.tokenizer.exceptions import TokenizerError
from pdfmajor.tokenizer.constants import END_HEX_STRING, HEX_PAIR, SPC
from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenHexString
from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.utils import int2byte
from typing import Iterator, Union
@@ -27,6 +28,7 @@ def parse_double_angled_bracket(initialpos: int, inp: Iterator[PInput]) -> Token
curtoken = parse_hexstring(initialpos, step, curtoken)
if isinstance(curtoken, TokenHexString):
return curtoken
raise TokenizerError("parse_double_angled_bracket ended")
def parse_hexstring(
@@ -41,4 +43,4 @@ def parse_hexstring(
token = HEX_PAIR.sub(
lambda x: int2byte(int(x.group(0), 16)), SPC.sub(b"", curtoken)
)
return TokenHexString(initialpos, cmp_tsize(inp.pos, initialpos, j + 1), token)
return TokenHexString(initialpos, inp.pos + j + 1, token)
+4 -6
View File
@@ -2,7 +2,7 @@ from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword
from pdfmajor.tokenizer.constants import END_KEYWORD
from typing import Iterator, Union
from .util import cmp_tsize, PInput
from .util import PInput
def parse_keyword(
@@ -26,11 +26,9 @@ def parse_keyword(
j = m.start(0)
curtoken += s[:j]
if curtoken == b"true":
return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), True)
return TokenBoolean(initialpos, curpos + j, True)
elif curtoken == b"false":
return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), False)
return TokenBoolean(initialpos, curpos + j, False)
else:
return TokenKeyword(
initialpos, cmp_tsize(curpos, initialpos, j), curtoken
)
return TokenKeyword(initialpos, curpos + j, curtoken)
raise TokenizerEOF
+7 -9
View File
@@ -1,12 +1,10 @@
from dataclasses import dataclass
from pdfmajor.tokenizer.exceptions import (
InvalidHexToken,
InvalidToken,
TokenizerEOF,
TokenizerError,
)
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token import TokenLiteral
from pdfmajor.utils import int2byte
@@ -48,14 +46,14 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
else:
raise InvalidHexToken(curpos + ci, c)
else:
m = END_LITERAL.search(buf[it.skip :], 0)
m = END_LITERAL.search(subbuf, 0)
if not m:
state.curtoken += buf[it.skip :]
state.curtoken += subbuf
it.skip = len(buf)
else:
j: int = m.start(0) + it.skip
state.curtoken += buf[it.skip : j]
c = buf[j : j + 1]
j: int = m.start(0)
state.curtoken += subbuf[:j]
c = subbuf[j : j + 1]
if c == b"#":
state.hex_value = b""
it.skip = j + 1
@@ -63,7 +61,7 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
else:
return TokenLiteral(
initialpos,
cmp_tsize(curpos, initialpos, j),
curpos + it.skip + j,
state.curtoken.decode(),
)
raise TokenizerEOF
+7 -11
View File
@@ -1,7 +1,5 @@
from decimal import Decimal
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
from pdfmajor.utils import int2byte
from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF
from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF, TokenizerError
from pdfmajor.tokenizer.token import (
TokenDecimal,
TokenNumber,
@@ -10,8 +8,8 @@ from pdfmajor.tokenizer.token import (
from pdfmajor.tokenizer.constants import (
END_NUMBER,
)
from typing import Iterator, Literal, Union
from .util import cmp_tsize, PInput
from typing import Iterator
from .util import PInput
def parse_number(
@@ -28,7 +26,7 @@ def parse_number(
TokenNumber
"""
if sign not in [b"+", b"-"]:
raise InvalidToken("Number was provided an invalid sign {!r}".format(sign))
raise TokenizerError("Number was provided an invalid sign {!r}".format(sign))
curtoken = b"" + sign
is_dec = False
for curpos, s in inp:
@@ -53,20 +51,18 @@ def parse_number(
j += nj
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j + 1),
curpos + j + 1,
Decimal(curtoken.decode()),
)
try:
if is_dec:
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
curpos + j,
Decimal(curtoken.decode()),
)
else:
return TokenInteger(
initialpos, cmp_tsize(curpos, initialpos, j), int(curtoken)
)
return TokenInteger(initialpos, curpos + j, int(curtoken))
except ValueError:
raise InvalidToken(initialpos, curtoken)
raise TokenizerEOF
+37 -24
View File
@@ -1,15 +1,18 @@
from pdfmajor.tokenizer.exceptions import TokenizerError
import re
from dataclasses import dataclass
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
from pdfmajor.tokenizer.token import (
TokenString,
)
from pdfmajor.tokenizer.constants import (
END_STRING,
ESC_STRING,
OCT_STRING,
)
from typing import Iterator, Optional
OCT_STRING = re.compile(br"^[0-7]{1,3}$")
@dataclass
class StringParseState:
@@ -33,12 +36,12 @@ def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString:
it = SafeBufferIt(buf)
for s in it.into_iter():
if state.escaped_value is not None:
it.skip += parse_string_escape(s, state)
parse_string_escape(it, state)
else:
m = END_STRING.search(s, 0)
if not m:
state.curtoken += s
it.skip = len(s)
it.skip += len(s)
else:
j = m.start(0)
state.curtoken += s[:j]
@@ -55,25 +58,35 @@ def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString:
if state.paren:
state.curtoken += next_char
it.skip += j + 1
return TokenString(
initialpos,
cmp_tsize(curpos, initialpos, j),
state.curtoken.decode(),
)
else:
return TokenString(
initialpos,
curpos + it.skip + j + 1,
state.curtoken.decode(),
)
raise TokenizerError("end of parse_string")
def parse_string_escape(s: bytes, state: StringParseState) -> int:
for ci in range(len(s)):
c = s[ci : ci + 1]
if (ci < 1) and (c in ESC_STRING):
state.curtoken += bytes([ESC_STRING[c]])
state.escaped_value = None
return 1
if OCT_STRING.match(c) and len(state.escaped_value) < 3:
state.escaped_value += c
elif len(state.escaped_value) == 3:
state.curtoken += bytes([int(state.escaped_value, 8)])
return 3
state.escaped_value = None
return len(s) + 1
def parse_string_escape(it: SafeBufferIt, state: StringParseState):
if state.escaped_value is None:
raise TokenizerError("state.escaped_value is None")
for s in it.into_iter():
for ci in range(len(s)):
c = s[ci : ci + 1]
if len(state.escaped_value) < 3:
it.skip += 1
state.escaped_value += c
if len(state.escaped_value) == 1:
if (ci < 1) and (c in ESC_STRING):
state.curtoken += bytes([ESC_STRING[c]])
state.escaped_value = None
return
elif OCT_STRING.match(state.escaped_value):
if len(state.escaped_value) == 3:
state.curtoken += bytes([int(state.escaped_value, 8)])
state.escaped_value = None
return
else:
it.skip -= 1
state.escaped_value = None
return
+9 -2
View File
@@ -26,6 +26,13 @@ def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
class SafeBufferIt:
"""A safe while-loop to use on a buffer
this class will make sure that there is no risk of an infinite loop and that the max number of loops is reasonable
Raises:
TokenizerEOF: when we reach a maximum iteration we raise an 'end of file' error
"""
__slots__ = ["skip", "buffer"]
def __init__(self, buffer: bytes, skip: int = 0) -> None:
@@ -33,8 +40,8 @@ class SafeBufferIt:
self.buffer = buffer
def into_iter(self) -> Iterator[bytes]:
for i in range(len(self.buffer) + 1):
if i > len(self.buffer):
for i in range(2 * len(self.buffer) + 1):
if i > 2 * len(self.buffer):
raise TokenizerEOF("Max Iteration Reached!")
if self.skip >= len(self.buffer):
break
+26
View File
@@ -1,4 +1,6 @@
from decimal import Decimal
import io
from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
from pdfmajor.lexer.string import parse_string
from unittest import TestCase
@@ -10,8 +12,10 @@ from pdfmajor.lexer.token import (
TDictVaue,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenDictionary,
TokenHexString,
TokenInteger,
TokenName,
TokenString,
)
@@ -164,3 +168,25 @@ class Basics(TestCase):
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, b"\x10 1\xa1*")
self.assertEqual(token.end_loc, 14)
def test_parse_float(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(b"0.120 + some invalid text", buf_size)
token = parse_number(buffer)
self.assertIsInstance(token, TokenDecimal)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, Decimal("0.12"))
self.assertEqual(token.end_loc, 5)
def test_parse_int(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(
b"130 oh oh 0.12 + some invalid text", buf_size
)
token = parse_number(buffer)
self.assertIsInstance(token, TokenInteger)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, 130)
self.assertEqual(token.end_loc, 3)