diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py index 038e7d2..4485079 100644 --- a/pdfmajor/tokenizer/__init__.py +++ b/pdfmajor/tokenizer/__init__.py @@ -1,4 +1,3 @@ - import io from typing import Iterator @@ -14,85 +13,99 @@ from pdfmajor.tokenizer.token import Token, TokenKeyword class PSTokenizer: - def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None: - self.fp = fp - self.bufsize = bufsize - self.bufpos: int = 0 - self.buf: bytes = b'' - self.buf_skipstep: int = 0 + def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None: + self.fp = fp + self.bufsize = bufsize + self.bufpos: int = 0 + self.buf: bytes = b"" + self.buf_skipstep: int = 0 - def iter_buffer(self) -> Iterator[PInput]: - while True: - if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)): - yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:]) - else: - self.buf_skipstep = 0 - self.bufpos = self.fp.tell() - self.buf = self.fp.read(self.bufsize) - if not self.buf: - raise TokenizerEOF("Unexpected EOF") - else: - yield PInput(self.bufpos, self.buf) + def iter_buffer(self) -> Iterator[PInput]: + while True: + if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)): + yield PInput( + self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :] + ) + else: + self.buf_skipstep = 0 + self.bufpos = self.fp.tell() + self.buf = self.fp.read(self.bufsize) + if not self.buf: + raise TokenizerEOF("Unexpected EOF") + else: + yield PInput(self.bufpos, self.buf) - def seek(self, pos: int): - self.fp.seek(pos) - - def step_back(self, steps: int = 1): - """Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize) + def seek(self, pos: int): + self.fp.seek(pos) - Args: - steps (int, optional): numbers of steps. Defaults to 1. - """ - self.fp.seek(self.fp.tell() - steps * self.bufsize) + def step_back(self, steps: int = 1): + """Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize) - def iter_tokens(self) -> Iterator[Token]: - for cur in self.iter_buffer(): - bufpos, buf = cur - m = NONSPC.search(buf, 0) - if not m: - self.buf_skipstep = 0 - continue - else: - j = m.start(0) - c = buf[j : j + 1] - if c == b"%": - self.buf_skipstep += j + 1 - yield self._check_token(cur, parse_comment(bufpos + j, self.iter_buffer())) - elif c == b"/": - self.buf_skipstep += j + 1 - yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer())) - elif c in b"-+": - self.buf_skipstep += j + 1 - yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c)) - elif c.isdigit(): - self.buf_skipstep += j - yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer())) - elif c == b".": - self.buf_skipstep += j - yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer())) - elif c.isalpha(): - self.buf_skipstep += j - yield self._check_token(cur, parse_keyword(bufpos + j, self.iter_buffer())) - elif c == b"(": - raise "WIP" - # self._curtoken = b"" - # self.paren = 1 - # self._current_parse_func = self._parse_string - # return j + 1 - elif c == b"<": - raise "WIP" - # self._curtoken = b"" - # self._current_parse_func = self._parse_wopen - # return j + 1 - elif c == b">": - raise "WIP" - # self._curtoken = b"" - # self._current_parse_func = self._parse_wclose - # return j + 1 - else: - self.buf_skipstep += j + 1 - yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c)) + Args: + steps (int, optional): numbers of steps. Defaults to 1. + """ + self.fp.seek(self.fp.tell() - steps * self.bufsize) - def _check_token(self, cur: PInput, token: Token) -> Token: - self.buf_skipstep += token.size - return token + def iter_tokens(self) -> Iterator[Token]: + for cur in self.iter_buffer(): + bufpos, buf = cur + m = NONSPC.search(buf, 0) + if not m: + self.buf_skipstep = 0 + continue + else: + j = m.start(0) + c = buf[j : j + 1] + if c == b"%": + self.buf_skipstep += j + 1 + yield self._check_token( + cur, parse_comment(bufpos + j, self.iter_buffer()) + ) + elif c == b"/": + self.buf_skipstep += j + 1 + yield self._check_token( + cur, parse_literal(bufpos + j, self.iter_buffer()) + ) + elif c in b"-+": + self.buf_skipstep += j + 1 + yield self._check_token( + cur, parse_number(bufpos + j, self.iter_buffer(), c) + ) + elif c.isdigit(): + self.buf_skipstep += j + yield self._check_token( + cur, parse_number(bufpos + j, self.iter_buffer()) + ) + elif c == b".": + self.buf_skipstep += j + yield self._check_token( + cur, parse_number(bufpos + j, self.iter_buffer()) + ) + elif c.isalpha(): + self.buf_skipstep += j + yield self._check_token( + cur, parse_keyword(bufpos + j, self.iter_buffer()) + ) + elif c == b"(": + raise "WIP" + # self._curtoken = b"" + # self.paren = 1 + # self._current_parse_func = self._parse_string + # return j + 1 + elif c == b"<": + raise "WIP" + # self._curtoken = b"" + # self._current_parse_func = self._parse_wopen + # return j + 1 + elif c == b">": + raise "WIP" + # self._curtoken = b"" + # self._current_parse_func = self._parse_wclose + # return j + 1 + else: + self.buf_skipstep += j + 1 + yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c)) + + def _check_token(self, cur: PInput, token: Token) -> Token: + self.buf_skipstep += token.size + return token diff --git a/pdfmajor/tokenizer/exceptions.py b/pdfmajor/tokenizer/exceptions.py index 261b7ff..f9323f7 100644 --- a/pdfmajor/tokenizer/exceptions.py +++ b/pdfmajor/tokenizer/exceptions.py @@ -8,9 +8,9 @@ class TokenizerError(PDFMajorException): class TokenizerEOF(TokenizerError): pass + class InvalidToken(TokenizerError): - def __init__(self, pos: int, token: bytes) -> None: - super().__init__("Invalid Token Found at {pos}, of value={v}".format( - pos=pos, - v=token - )) + def __init__(self, pos: int, token: bytes) -> None: + super().__init__( + "Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=token) + ) diff --git a/pdfmajor/tokenizer/token.py b/pdfmajor/tokenizer/token.py index ed9c16e..15c582e 100644 --- a/pdfmajor/tokenizer/token.py +++ b/pdfmajor/tokenizer/token.py @@ -5,45 +5,49 @@ T = TypeVar("T") class Token(Generic[T]): - __slots__ = ["pos", "size", "value"] + __slots__ = ["pos", "size", "value"] - def __init__(self, pos: int, size: int, value: T) -> None: - self.size = size - self.pos = pos - self.value = value + def __init__(self, pos: int, size: int, value: T) -> None: + self.size = size + self.pos = pos + self.value = value + + def __repr__(self) -> str: + return "<{cls} pos={pos}>{value}".format( + cls=self.__class__.__name__, pos=self.pos, value=self.value + ) - def __repr__(self) -> str: - return "<{cls} pos={pos}>{value}".format( - cls=self.__class__.__name__, pos=self.pos, value=self.value - ) - class TokenComment(Token[bytes]): - pass + pass class TokenHex(Token[bytes]): - pass + pass class TokenKeyword(Token[bytes]): - pass + pass + class TokenLiteral(Token[str]): - pass + pass + class TokenBoolean(Token[bool]): - pass + pass + class TokenString(Token[str]): - pass + pass class TokenInteger(Token[int]): - pass + pass class TokenDecimal(Token[Decimal]): - pass + pass + TokenNumber = Union[TokenInteger, TokenDecimal] diff --git a/pdfmajor/tokenizer/token_parsers/comment.py b/pdfmajor/tokenizer/token_parsers/comment.py index 343e928..c85744a 100644 --- a/pdfmajor/tokenizer/token_parsers/comment.py +++ b/pdfmajor/tokenizer/token_parsers/comment.py @@ -1,32 +1,29 @@ from pdfmajor.tokenizer.token import ( - TokenComment, + TokenComment, ) from pdfmajor.tokenizer.constants import ( - EOL, + EOL, ) from typing import Iterator from .util import cmp_tsize, PInput + def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment: - """Parses input stream into a comment + """Parses input stream into a comment - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) - Returns: - TokenComment - """ - curtoken = b"" - for curpos, s in inp: - m = EOL.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - return TokenComment( - initialpos, - cmp_tsize(curpos, initialpos, j), - curtoken - ) + Returns: + TokenComment + """ + curtoken = b"" + for curpos, s in inp: + m = EOL.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + return TokenComment(initialpos, cmp_tsize(curpos, initialpos, j), curtoken) diff --git a/pdfmajor/tokenizer/token_parsers/keyword.py b/pdfmajor/tokenizer/token_parsers/keyword.py index 95bf3c3..1ce4696 100644 --- a/pdfmajor/tokenizer/token_parsers/keyword.py +++ b/pdfmajor/tokenizer/token_parsers/keyword.py @@ -1,50 +1,34 @@ -from pdfmajor.tokenizer.token import ( - TokenBoolean, - TokenKeyword -) -from pdfmajor.tokenizer.constants import ( - END_KEYWORD -) +from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword +from pdfmajor.tokenizer.constants import END_KEYWORD from typing import Iterator, Union from .util import cmp_tsize, PInput + def parse_keyword( - initialpos: int, - inp: Iterator[PInput] + initialpos: int, inp: Iterator[PInput] ) -> Union[TokenKeyword, TokenBoolean]: - """Parses input stream into a keyword or bool + """Parses input stream into a keyword or bool - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - - Returns: - Union[TokenKeyword, TokenBoolean] - """ - curtoken = b"" - for curpos, s in inp: - m = END_KEYWORD.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - if curtoken == b"true": - return TokenBoolean( - initialpos, - cmp_tsize(curpos, initialpos, j), - True - ) - elif curtoken == b"false": - return TokenBoolean( - initialpos, - cmp_tsize(curpos, initialpos, j), - False - ) - else: - return TokenKeyword( - initialpos, - cmp_tsize(curpos, initialpos, j), - curtoken - ) + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) + Returns: + Union[TokenKeyword, TokenBoolean] + """ + curtoken = b"" + for curpos, s in inp: + m = END_KEYWORD.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + if curtoken == b"true": + return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), True) + elif curtoken == b"false": + return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), False) + else: + return TokenKeyword( + initialpos, cmp_tsize(curpos, initialpos, j), curtoken + ) diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py index a11d048..e6139d5 100644 --- a/pdfmajor/tokenizer/token_parsers/literal.py +++ b/pdfmajor/tokenizer/token_parsers/literal.py @@ -8,55 +8,57 @@ from ..constants import HEX, END_LITERAL from typing import Iterator, NamedTuple, Optional, Union + @dataclass class LiteralParseState: - curtoken: bytes - hex_value: Optional[bytes] = None + curtoken: bytes + hex_value: Optional[bytes] = None + def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: - """Parses input stream into a literal + """Parses input stream into a literal - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) - Returns: - TokenLiteral - """ - state = LiteralParseState(b"") - for curpos, buf in inp: - skip: int = 0 - for i in range(len(buf) + 1): - assert i < len(buf), "MAX_LOOP_REACHED" - if skip >= len(buf): - break - if state.hex_value is not None: - for ci in range(len(buf[skip:])): - c = buf[skip+ci:skip+ci+1] - if HEX.match(c) and len(state.hex_value) < 2: - state.hex_value += c - else: - if state.hex_value: - state.curtoken += int2byte(int(state.hex_value, 16)) - state.hex_value = None - skip += ci - break - else: - m = END_LITERAL.search(buf[skip:], 0) - if not m: - state.curtoken += buf[skip:] - skip = len(buf) - else: - j: int = m.start(0) + skip - state.curtoken += buf[skip:j] - c = buf[j : j + 1] - if c == b"#": - state.hex_value = b"" - skip = j+1 - continue - else: - return TokenLiteral( - initialpos, - cmp_tsize(curpos, initialpos, j), - state.curtoken.decode() - ) + Returns: + TokenLiteral + """ + state = LiteralParseState(b"") + for curpos, buf in inp: + skip: int = 0 + for i in range(len(buf) + 1): + assert i < len(buf), "MAX_LOOP_REACHED" + if skip >= len(buf): + break + if state.hex_value is not None: + for ci in range(len(buf[skip:])): + c = buf[skip + ci : skip + ci + 1] + if HEX.match(c) and len(state.hex_value) < 2: + state.hex_value += c + else: + if state.hex_value: + state.curtoken += int2byte(int(state.hex_value, 16)) + state.hex_value = None + skip += ci + break + else: + m = END_LITERAL.search(buf[skip:], 0) + if not m: + state.curtoken += buf[skip:] + skip = len(buf) + else: + j: int = m.start(0) + skip + state.curtoken += buf[skip:j] + c = buf[j : j + 1] + if c == b"#": + state.hex_value = b"" + skip = j + 1 + continue + else: + return TokenLiteral( + initialpos, + cmp_tsize(curpos, initialpos, j), + state.curtoken.decode(), + ) diff --git a/pdfmajor/tokenizer/token_parsers/number.py b/pdfmajor/tokenizer/token_parsers/number.py index 3e6528e..6e044a1 100644 --- a/pdfmajor/tokenizer/token_parsers/number.py +++ b/pdfmajor/tokenizer/token_parsers/number.py @@ -3,73 +3,70 @@ from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX from pdfmajor.utils import int2byte from pdfmajor.tokenizer.exceptions import InvalidToken from pdfmajor.tokenizer.token import ( - TokenDecimal, - TokenNumber, - TokenInteger, + TokenDecimal, + TokenNumber, + TokenInteger, ) from pdfmajor.tokenizer.constants import ( - END_NUMBER, + END_NUMBER, ) from typing import Iterator, Literal, Union from .util import cmp_tsize, PInput -VALID_SIGNS = Union[Literal[b'+'], Literal[b'-']] +VALID_SIGNS = Union[Literal[b"+"], Literal[b"-"]] + def parse_number( - initialpos: int, - inp: Iterator[PInput], - sign: VALID_SIGNS = b'+' + initialpos: int, inp: Iterator[PInput], sign: VALID_SIGNS = b"+" ) -> TokenNumber: - """Parses input stream into a number + """Parses input stream into a number - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - sign (str): either a + or - + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) + sign (str): either a + or - - Returns: - TokenNumber - """ - assert sign in [b'+', b'-'], f"Number was provided an invalid sign {sign}" - curtoken = b"" + sign - is_dec = False - for curpos, s in inp: - m = END_NUMBER.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - c = s[j : j + 1] - if (c == b".") and not is_dec: - is_dec = True - curtoken += c - s = s[j+1:] - m = END_NUMBER.search(s, 0) - if not m: - curtoken += s - continue - else: - nj = m.start(0) - curtoken += s[:nj] - j += nj - return TokenDecimal( - initialpos, - cmp_tsize(curpos, initialpos, j), - Decimal(curtoken.decode()) - ) - try: - if is_dec: - return TokenDecimal( - initialpos, - cmp_tsize(curpos, initialpos, j), - Decimal(curtoken.decode()) - ) - else: - return TokenInteger( - initialpos, - cmp_tsize(curpos, initialpos, j), - int(curtoken) - ) - except ValueError: - raise InvalidToken(initialpos, curtoken) + Returns: + TokenNumber + """ + assert sign in [b"+", b"-"], f"Number was provided an invalid sign {sign}" + curtoken = b"" + sign + is_dec = False + for curpos, s in inp: + m = END_NUMBER.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + c = s[j : j + 1] + if (c == b".") and not is_dec: + is_dec = True + curtoken += c + s = s[j + 1 :] + m = END_NUMBER.search(s, 0) + if not m: + curtoken += s + continue + else: + nj = m.start(0) + curtoken += s[:nj] + j += nj + return TokenDecimal( + initialpos, + cmp_tsize(curpos, initialpos, j), + Decimal(curtoken.decode()), + ) + try: + if is_dec: + return TokenDecimal( + initialpos, + cmp_tsize(curpos, initialpos, j), + Decimal(curtoken.decode()), + ) + else: + return TokenInteger( + initialpos, cmp_tsize(curpos, initialpos, j), int(curtoken) + ) + except ValueError: + raise InvalidToken(initialpos, curtoken) diff --git a/pdfmajor/tokenizer/token_parsers/string.py b/pdfmajor/tokenizer/token_parsers/string.py index d1e182a..b137503 100644 --- a/pdfmajor/tokenizer/token_parsers/string.py +++ b/pdfmajor/tokenizer/token_parsers/string.py @@ -3,66 +3,66 @@ from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX from pdfmajor.utils import int2byte from pdfmajor.tokenizer.exceptions import InvalidToken from pdfmajor.tokenizer.token import ( - Token, - TokenBoolean, - TokenComment, - TokenDecimal, - TokenKeyword, TokenLiteral, - TokenNumber, - TokenInteger, - TokenString + Token, + TokenBoolean, + TokenComment, + TokenDecimal, + TokenKeyword, + TokenLiteral, + TokenNumber, + TokenInteger, + TokenString, ) from pdfmajor.tokenizer.constants import ( - END_KEYWORD, - END_NUMBER, - END_STRING, - EOL, - ESC_STRING, - OCT_STRING + END_KEYWORD, + END_NUMBER, + END_STRING, + EOL, + ESC_STRING, + OCT_STRING, ) from typing import Callable, Iterator, Literal, NamedTuple, Union def parse_string(initialpos: int, inp: Iterator[PInput]): - curtoken = b"" - paren = 1 - for curpos, s in inp: - m = END_STRING.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - c = s[j : j + 1] - if c == b"\\": - self._current_parse_func = self._parse_string_1 - return j + 1 - elif c == b"(": - paren += 1 - curtoken += c - return j + 1 - elif c == b")": - paren -= 1 - if paren: # WTF, they said balanced parens need no special treatment. - curtoken += c - return j + 1 - return TokenString( - initialpos, - cmp_tsize(curpos, initialpos, j), - curtoken.decode() - ) + curtoken = b"" + paren = 1 + for curpos, s in inp: + m = END_STRING.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + c = s[j : j + 1] + if c == b"\\": + self._current_parse_func = self._parse_string_1 + return j + 1 + elif c == b"(": + paren += 1 + curtoken += c + return j + 1 + elif c == b")": + paren -= 1 + if paren: # WTF, they said balanced parens need no special treatment. + curtoken += c + return j + 1 + return TokenString( + initialpos, cmp_tsize(curpos, initialpos, j), curtoken.decode() + ) + def parse_string_oct(initialpos: int, inp: Iterator[PInput]): - curtoken: bytes = b"" - oct_val: bytes = b"" - for curpos, s in inp: - for ci in range(len(s)): - c = s[ci:ci+1] - if OCT_STRING.match(c) and len(oct_val) < 3: - oct_val += c - elif oct_val: - curtoken += int2byte(int(oct_val, 8)) - return (curtoken, curpos+ci) - elif c in ESC_STRING: - curtoken += int2byte(ESC_STRING[c]) - return (curtoken, curpos+ci+1) + curtoken: bytes = b"" + oct_val: bytes = b"" + for curpos, s in inp: + for ci in range(len(s)): + c = s[ci : ci + 1] + if OCT_STRING.match(c) and len(oct_val) < 3: + oct_val += c + elif oct_val: + curtoken += int2byte(int(oct_val, 8)) + return (curtoken, curpos + ci) + elif c in ESC_STRING: + curtoken += int2byte(ESC_STRING[c]) + return (curtoken, curpos + ci + 1) diff --git a/pdfmajor/tokenizer/token_parsers/util.py b/pdfmajor/tokenizer/token_parsers/util.py index f23974c..ce6d0be 100644 --- a/pdfmajor/tokenizer/token_parsers/util.py +++ b/pdfmajor/tokenizer/token_parsers/util.py @@ -1,25 +1,24 @@ -from pdfmajor.tokenizer.token import ( - Token -) +from pdfmajor.tokenizer.token import Token from typing import Callable, Iterator, NamedTuple class PInput(NamedTuple): - pos: int - buf: bytes + pos: int + buf: bytes ParserFunc = Callable[[int, Iterator[PInput]], Token] + def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int: - """Computes the size of the token based on its positional data + """Computes the size of the token based on its positional data - Args: - curpos (int): current position of the buffer - initialpos (int): initial position where the parser started working - tokenend_idx (int): last relevant character index within the last buffered value + Args: + curpos (int): current position of the buffer + initialpos (int): initial position where the parser started working + tokenend_idx (int): last relevant character index within the last buffered value - Returns: - int: length of index - """ - return curpos-initialpos+tokenend_idx + Returns: + int: length of index + """ + return curpos - initialpos + tokenend_idx