From c75234663886cb73fc13287ac292aba09b3ec922 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Fri, 16 Oct 2020 17:07:40 -0400 Subject: [PATCH] moved num parsing --- pdfmajor/lexer/exceptions.py | 7 +++ pdfmajor/lexer/number.py | 54 ++++++++++++++++ pdfmajor/tokenizer/__init__.py | 4 +- pdfmajor/tokenizer/token.py | 17 +++--- pdfmajor/tokenizer/token_parsers/comment.py | 2 +- .../token_parsers/dictionary_hexstring.py | 6 +- pdfmajor/tokenizer/token_parsers/keyword.py | 10 ++- pdfmajor/tokenizer/token_parsers/literal.py | 16 +++-- pdfmajor/tokenizer/token_parsers/number.py | 18 +++--- pdfmajor/tokenizer/token_parsers/string.py | 61 +++++++++++-------- pdfmajor/tokenizer/token_parsers/util.py | 11 +++- tests/test_lexer.py | 26 ++++++++ 12 files changed, 168 insertions(+), 64 deletions(-) create mode 100644 pdfmajor/lexer/number.py diff --git a/pdfmajor/lexer/exceptions.py b/pdfmajor/lexer/exceptions.py index 274e1ed..13db39a 100644 --- a/pdfmajor/lexer/exceptions.py +++ b/pdfmajor/lexer/exceptions.py @@ -14,3 +14,10 @@ class InvalidHexToken(LexerError): super().__init__( "Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token)) ) + + +class InvalidToken(LexerError): + def __init__(self, pos: int, token: bytes) -> None: + super().__init__( + "Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token)) + ) diff --git a/pdfmajor/lexer/number.py b/pdfmajor/lexer/number.py new file mode 100644 index 0000000..341c30b --- /dev/null +++ b/pdfmajor/lexer/number.py @@ -0,0 +1,54 @@ +from decimal import Decimal +from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError +from pdfmajor.lexer.token import ( + TokenDecimal, + TokenNumber, + TokenInteger, +) +from pdfmajor.tokenizer.constants import ( + END_NUMBER, +) + + +def parse_number(buffer: SafeBufferIt, sign: bytes = b"+") -> TokenNumber: + """Parses input stream into a number + + Args: + buffer (SafeBufferIt) + sign (str): either a + or - + + Returns: + TokenNumber + """ + if sign not in [b"+", b"-"]: + raise LexerError("Number was provided an invalid sign {!r}".format(sign)) + curtoken = b"" + sign + is_decimal = False + initialpos = buffer.tell() - 1 + for pos, buf in buffer: + m = END_NUMBER.search(buf, 0) + if not m: + curtoken += buf + else: + j = m.start(0) + c = buf[j : j + 1] + if (c == b".") and not is_decimal: + is_decimal = True + buffer.seek(pos + j + 1) + curtoken += buf[: j + 1] + else: + curtoken += buf[:j] + buffer.seek(pos + j) + try: + if is_decimal: + return TokenDecimal( + initialpos, + buffer.tell(), + Decimal(curtoken.decode()), + ) + else: + return TokenInteger(initialpos, buffer.tell(), int(curtoken)) + except ValueError: + raise InvalidToken(initialpos, curtoken) + raise LexerEOF diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py index f5e001e..5c6fa22 100644 --- a/pdfmajor/tokenizer/__init__.py +++ b/pdfmajor/tokenizer/__init__.py @@ -74,7 +74,7 @@ class PSTokenizer: elif c == b"/": self.inc_buf_skipstep(j + 1) yield self._check_token( - cur, parse_literal(bufpos + j + 1, self.iter_buffer()) + cur, parse_literal(bufpos + j, self.iter_buffer()) ) elif c in b"-+": self.inc_buf_skipstep(j + 1) @@ -116,5 +116,5 @@ class PSTokenizer: yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c)) def _check_token(self, cur: PInput, token: Token) -> Token: - self.inc_buf_skipstep(token.size) + self.inc_buf_skipstep(token.end_loc - cur.pos) return token diff --git a/pdfmajor/tokenizer/token.py b/pdfmajor/tokenizer/token.py index 98e4d26..d311017 100644 --- a/pdfmajor/tokenizer/token.py +++ b/pdfmajor/tokenizer/token.py @@ -8,22 +8,25 @@ T = TypeVar("T") class Token(Generic[T]): __slots__ = ["pos", "size", "value"] - def __init__(self, pos: int, size: int, value: T) -> None: - self.size = size - self.pos = pos + def __init__(self, start_loc: int, end_loc: int, value: T) -> None: + self.start_loc = start_loc + self.end_loc = end_loc self.value = value def __repr__(self) -> str: - return "{cls}(pos={pos}, size={size}, value={value})".format( - cls=self.__class__.__name__, pos=self.pos, value=self.value, size=self.size + return "{cls}(start={start_loc}, end={end_loc}, value='{value}')".format( + cls=self.__class__.__name__, + start_loc=self.start_loc, + value=self.value, + end_loc=self.end_loc, ) def __eq__(self, other: Any): if isinstance(other, self.__class__): return ( (other.value == self.value) - and (other.pos == self.pos) - and (other.size == self.size) + and (other.start_loc == self.start_loc) + and (other.end_loc == self.end_loc) ) else: return False diff --git a/pdfmajor/tokenizer/token_parsers/comment.py b/pdfmajor/tokenizer/token_parsers/comment.py index cad6789..172e0e1 100644 --- a/pdfmajor/tokenizer/token_parsers/comment.py +++ b/pdfmajor/tokenizer/token_parsers/comment.py @@ -27,5 +27,5 @@ def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment: else: j = m.start(0) curtoken += s[:j] - return TokenComment(initialpos, cmp_tsize(curpos, initialpos, j), curtoken) + return TokenComment(initialpos, curpos + j, curtoken) raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/dictionary_hexstring.py b/pdfmajor/tokenizer/token_parsers/dictionary_hexstring.py index 80f6dc8..3373f52 100644 --- a/pdfmajor/tokenizer/token_parsers/dictionary_hexstring.py +++ b/pdfmajor/tokenizer/token_parsers/dictionary_hexstring.py @@ -1,6 +1,7 @@ +from pdfmajor.tokenizer.exceptions import TokenizerError from pdfmajor.tokenizer.constants import END_HEX_STRING, HEX_PAIR, SPC from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenHexString -from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize +from pdfmajor.tokenizer.token_parsers.util import PInput from pdfmajor.utils import int2byte from typing import Iterator, Union @@ -27,6 +28,7 @@ def parse_double_angled_bracket(initialpos: int, inp: Iterator[PInput]) -> Token curtoken = parse_hexstring(initialpos, step, curtoken) if isinstance(curtoken, TokenHexString): return curtoken + raise TokenizerError("parse_double_angled_bracket ended") def parse_hexstring( @@ -41,4 +43,4 @@ def parse_hexstring( token = HEX_PAIR.sub( lambda x: int2byte(int(x.group(0), 16)), SPC.sub(b"", curtoken) ) - return TokenHexString(initialpos, cmp_tsize(inp.pos, initialpos, j + 1), token) + return TokenHexString(initialpos, inp.pos + j + 1, token) diff --git a/pdfmajor/tokenizer/token_parsers/keyword.py b/pdfmajor/tokenizer/token_parsers/keyword.py index 9e41ee8..e79bf9f 100644 --- a/pdfmajor/tokenizer/token_parsers/keyword.py +++ b/pdfmajor/tokenizer/token_parsers/keyword.py @@ -2,7 +2,7 @@ from pdfmajor.tokenizer.exceptions import TokenizerEOF from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword from pdfmajor.tokenizer.constants import END_KEYWORD from typing import Iterator, Union -from .util import cmp_tsize, PInput +from .util import PInput def parse_keyword( @@ -26,11 +26,9 @@ def parse_keyword( j = m.start(0) curtoken += s[:j] if curtoken == b"true": - return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), True) + return TokenBoolean(initialpos, curpos + j, True) elif curtoken == b"false": - return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), False) + return TokenBoolean(initialpos, curpos + j, False) else: - return TokenKeyword( - initialpos, cmp_tsize(curpos, initialpos, j), curtoken - ) + return TokenKeyword(initialpos, curpos + j, curtoken) raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py index 7cc9115..83ef0ac 100644 --- a/pdfmajor/tokenizer/token_parsers/literal.py +++ b/pdfmajor/tokenizer/token_parsers/literal.py @@ -1,12 +1,10 @@ from dataclasses import dataclass from pdfmajor.tokenizer.exceptions import ( InvalidHexToken, - InvalidToken, TokenizerEOF, - TokenizerError, ) -from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize +from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt from pdfmajor.tokenizer.token_parsers.util import PInput from pdfmajor.tokenizer.token import TokenLiteral from pdfmajor.utils import int2byte @@ -48,14 +46,14 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: else: raise InvalidHexToken(curpos + ci, c) else: - m = END_LITERAL.search(buf[it.skip :], 0) + m = END_LITERAL.search(subbuf, 0) if not m: - state.curtoken += buf[it.skip :] + state.curtoken += subbuf it.skip = len(buf) else: - j: int = m.start(0) + it.skip - state.curtoken += buf[it.skip : j] - c = buf[j : j + 1] + j: int = m.start(0) + state.curtoken += subbuf[:j] + c = subbuf[j : j + 1] if c == b"#": state.hex_value = b"" it.skip = j + 1 @@ -63,7 +61,7 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: else: return TokenLiteral( initialpos, - cmp_tsize(curpos, initialpos, j), + curpos + it.skip + j, state.curtoken.decode(), ) raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/number.py b/pdfmajor/tokenizer/token_parsers/number.py index 0a65f27..f390b09 100644 --- a/pdfmajor/tokenizer/token_parsers/number.py +++ b/pdfmajor/tokenizer/token_parsers/number.py @@ -1,7 +1,5 @@ from decimal import Decimal -from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX -from pdfmajor.utils import int2byte -from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF +from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF, TokenizerError from pdfmajor.tokenizer.token import ( TokenDecimal, TokenNumber, @@ -10,8 +8,8 @@ from pdfmajor.tokenizer.token import ( from pdfmajor.tokenizer.constants import ( END_NUMBER, ) -from typing import Iterator, Literal, Union -from .util import cmp_tsize, PInput +from typing import Iterator +from .util import PInput def parse_number( @@ -28,7 +26,7 @@ def parse_number( TokenNumber """ if sign not in [b"+", b"-"]: - raise InvalidToken("Number was provided an invalid sign {!r}".format(sign)) + raise TokenizerError("Number was provided an invalid sign {!r}".format(sign)) curtoken = b"" + sign is_dec = False for curpos, s in inp: @@ -53,20 +51,18 @@ def parse_number( j += nj return TokenDecimal( initialpos, - cmp_tsize(curpos, initialpos, j + 1), + curpos + j + 1, Decimal(curtoken.decode()), ) try: if is_dec: return TokenDecimal( initialpos, - cmp_tsize(curpos, initialpos, j), + curpos + j, Decimal(curtoken.decode()), ) else: - return TokenInteger( - initialpos, cmp_tsize(curpos, initialpos, j), int(curtoken) - ) + return TokenInteger(initialpos, curpos + j, int(curtoken)) except ValueError: raise InvalidToken(initialpos, curtoken) raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/string.py b/pdfmajor/tokenizer/token_parsers/string.py index c3878e5..f8f50ed 100644 --- a/pdfmajor/tokenizer/token_parsers/string.py +++ b/pdfmajor/tokenizer/token_parsers/string.py @@ -1,15 +1,18 @@ +from pdfmajor.tokenizer.exceptions import TokenizerError +import re from dataclasses import dataclass -from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize +from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt from pdfmajor.tokenizer.token import ( TokenString, ) from pdfmajor.tokenizer.constants import ( END_STRING, ESC_STRING, - OCT_STRING, ) from typing import Iterator, Optional +OCT_STRING = re.compile(br"^[0-7]{1,3}$") + @dataclass class StringParseState: @@ -33,12 +36,12 @@ def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString: it = SafeBufferIt(buf) for s in it.into_iter(): if state.escaped_value is not None: - it.skip += parse_string_escape(s, state) + parse_string_escape(it, state) else: m = END_STRING.search(s, 0) if not m: state.curtoken += s - it.skip = len(s) + it.skip += len(s) else: j = m.start(0) state.curtoken += s[:j] @@ -55,25 +58,35 @@ def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString: if state.paren: state.curtoken += next_char it.skip += j + 1 - return TokenString( - initialpos, - cmp_tsize(curpos, initialpos, j), - state.curtoken.decode(), - ) + else: + return TokenString( + initialpos, + curpos + it.skip + j + 1, + state.curtoken.decode(), + ) + raise TokenizerError("end of parse_string") -def parse_string_escape(s: bytes, state: StringParseState) -> int: - for ci in range(len(s)): - c = s[ci : ci + 1] - if (ci < 1) and (c in ESC_STRING): - state.curtoken += bytes([ESC_STRING[c]]) - state.escaped_value = None - return 1 - if OCT_STRING.match(c) and len(state.escaped_value) < 3: - state.escaped_value += c - elif len(state.escaped_value) == 3: - state.curtoken += bytes([int(state.escaped_value, 8)]) - return 3 - - state.escaped_value = None - return len(s) + 1 +def parse_string_escape(it: SafeBufferIt, state: StringParseState): + if state.escaped_value is None: + raise TokenizerError("state.escaped_value is None") + for s in it.into_iter(): + for ci in range(len(s)): + c = s[ci : ci + 1] + if len(state.escaped_value) < 3: + it.skip += 1 + state.escaped_value += c + if len(state.escaped_value) == 1: + if (ci < 1) and (c in ESC_STRING): + state.curtoken += bytes([ESC_STRING[c]]) + state.escaped_value = None + return + elif OCT_STRING.match(state.escaped_value): + if len(state.escaped_value) == 3: + state.curtoken += bytes([int(state.escaped_value, 8)]) + state.escaped_value = None + return + else: + it.skip -= 1 + state.escaped_value = None + return diff --git a/pdfmajor/tokenizer/token_parsers/util.py b/pdfmajor/tokenizer/token_parsers/util.py index 77b057c..400ab67 100644 --- a/pdfmajor/tokenizer/token_parsers/util.py +++ b/pdfmajor/tokenizer/token_parsers/util.py @@ -26,6 +26,13 @@ def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int: class SafeBufferIt: + """A safe while-loop to use on a buffer + this class will make sure that there is no risk of an infinite loop and that the max number of loops is reasonable + + Raises: + TokenizerEOF: when we reach a maximum iteration we raise an 'end of file' error + """ + __slots__ = ["skip", "buffer"] def __init__(self, buffer: bytes, skip: int = 0) -> None: @@ -33,8 +40,8 @@ class SafeBufferIt: self.buffer = buffer def into_iter(self) -> Iterator[bytes]: - for i in range(len(self.buffer) + 1): - if i > len(self.buffer): + for i in range(2 * len(self.buffer) + 1): + if i > 2 * len(self.buffer): raise TokenizerEOF("Max Iteration Reached!") if self.skip >= len(self.buffer): break diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 1b16390..e36589d 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -1,4 +1,6 @@ +from decimal import Decimal import io +from pdfmajor.lexer.number import parse_number from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring from pdfmajor.lexer.string import parse_string from unittest import TestCase @@ -10,8 +12,10 @@ from pdfmajor.lexer.token import ( TDictVaue, TokenBoolean, TokenComment, + TokenDecimal, TokenDictionary, TokenHexString, + TokenInteger, TokenName, TokenString, ) @@ -164,3 +168,25 @@ class Basics(TestCase): self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, b"\x10 1\xa1*") self.assertEqual(token.end_loc, 14) + + def test_parse_float(self): + for buf_size in [2, 3, 50]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(b"0.120 + some invalid text", buf_size) + token = parse_number(buffer) + self.assertIsInstance(token, TokenDecimal) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, Decimal("0.12")) + self.assertEqual(token.end_loc, 5) + + def test_parse_int(self): + for buf_size in [2, 3, 50]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter( + b"130 oh oh 0.12 + some invalid text", buf_size + ) + token = parse_number(buffer) + self.assertIsInstance(token, TokenInteger) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, 130) + self.assertEqual(token.end_loc, 3)