From ab657deda450ff67c79f7776824b90effff3655a Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Tue, 13 Oct 2020 20:52:23 -0400 Subject: [PATCH] working literal parser first mock of main tokenizer --- pdfmajor/tokenizer/__init__.py | 123 ++++++++++++++------ pdfmajor/tokenizer/token.py | 2 +- pdfmajor/tokenizer/token_parsers/literal.py | 104 +++++++---------- pdfmajor/tokenizer/tokenizer.py | 88 -------------- tests/test_tokenizer.py | 47 +++++++- 5 files changed, 173 insertions(+), 191 deletions(-) delete mode 100644 pdfmajor/tokenizer/tokenizer.py diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py index cc39a76..a80f38c 100644 --- a/pdfmajor/tokenizer/__init__.py +++ b/pdfmajor/tokenizer/__init__.py @@ -1,46 +1,95 @@ + import io -from typing import List -from pdfmajor.tokenizer.token import Token + +from typing import Iterator + +from pdfmajor.tokenizer.constants import NONSPC from pdfmajor.tokenizer.exceptions import TokenizerEOF -from pdfmajor.tokenizer.parse_funcs import ParseFunc +from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword +from pdfmajor.tokenizer.token_parsers.literal import parse_literal +from pdfmajor.tokenizer.token_parsers.number import parse_number +from pdfmajor.tokenizer.token_parsers.comment import parse_comment +from pdfmajor.tokenizer.token_parsers.util import PInput +from pdfmajor.tokenizer.token import Token, TokenKeyword class PSTokenizer: - def __init__(self, fp: io.BufferedIOBase) -> None: - self.fp = fp - self.seek(0) + def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None: + self.fp = fp + self.bufsize = bufsize + self.bufpos: int = 0 + self.buf: bytes = b'' + self.buf_skipstep: int = 0 - def seek(self, pos: int): - """Seeks tokenizer to the given position.""" - self.fp.seek(pos) - # reset the status for nextline() - self.bufpos: int = pos - self.buf: bytes = b"" - self.charpos: int = 0 - # reset the status for nexttoken() - self._current_parse_func: ParseFunc = self._parse_main - self._curtoken: bytes = b"" - self._curtokenpos: int = 0 - self._tokens: List[Token] = [] - return + def iter_buffer(self) -> Iterator[PInput]: + while True: + if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)): + yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:]) + else: + self.buf_skipstep = 0 + self.bufpos = self.fp.tell() + self.buf = self.fp.read(self.bufsize) + if not self.buf: + raise TokenizerEOF("Unexpected EOF") + else: + yield PInput(self.bufpos, self.buf) - def __repr__(self): - return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos) + def seek(self, pos: int): + self.fp.seek(pos) + + def step_back(self, steps: int = 1): + """Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize) - def nexttoken(self): - while not self._tokens: - self.fillbuf() - self.charpos = self._current_parse_func(self.buf, self.charpos) - token = self._tokens.pop(0) - return token + Args: + steps (int, optional): numbers of steps. Defaults to 1. + """ + self.fp.seek(self.fp.tell() - steps * self.bufsize) - def fillbuf(self): - if self.charpos < len(self.buf): - return - # fetch next chunk. - self.bufpos = self.fp.tell() - self.buf = self.fp.read(self.BUFSIZ) - if not self.buf: - raise TokenizerEOF("Unexpected EOF") - self.charpos = 0 - return + def iter_tokens(self) -> Iterator[Token]: + for cur in self.iter_buffer(): + bufpos, buf = cur + m = NONSPC.search(buf, 0) + if not m: + self.buf_skipstep = 0 + continue + else: + j = m.start(0) + c = buf[j : j + 1] + if c == b"%": + self.buf_skipstep += j + 1 + yield self._check_token(cur, parse_comment(bufpos + j, self.iter_buffer())) + elif c == b"/": + self.buf_skipstep += j + 1 + yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer())) + elif c in b"-+" or c.isdigit(): + self.buf_skipstep += j + 1 + yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c)) + elif c == b".": + self.buf_skipstep += j + yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer())) + elif c.isalpha(): + self.buf_skipstep += j + yield self._check_token(cur, parse_keyword(bufpos + j, self.iter_buffer())) + elif c == b"(": + raise "WIP" + # self._curtoken = b"" + # self.paren = 1 + # self._current_parse_func = self._parse_string + # return j + 1 + elif c == b"<": + raise "WIP" + # self._curtoken = b"" + # self._current_parse_func = self._parse_wopen + # return j + 1 + elif c == b">": + raise "WIP" + # self._curtoken = b"" + # self._current_parse_func = self._parse_wclose + # return j + 1 + else: + self.buf_skipstep += j + 1 + yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c)) + + def _check_token(self, cur: PInput, token: Token) -> Token: + self.buf_skipstep += token.size + return token diff --git a/pdfmajor/tokenizer/token.py b/pdfmajor/tokenizer/token.py index 08fbf11..ed9c16e 100644 --- a/pdfmajor/tokenizer/token.py +++ b/pdfmajor/tokenizer/token.py @@ -29,7 +29,7 @@ class TokenHex(Token[bytes]): class TokenKeyword(Token[bytes]): pass -class TokenLiteral(Token[bytes]): +class TokenLiteral(Token[str]): pass class TokenBoolean(Token[bool]): diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py index f5efaa1..a11d048 100644 --- a/pdfmajor/tokenizer/token_parsers/literal.py +++ b/pdfmajor/tokenizer/token_parsers/literal.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from pdfmajor.tokenizer.token_parsers.util import cmp_tsize from pdfmajor.tokenizer.token_parsers.util import PInput @@ -7,6 +8,11 @@ from ..constants import HEX, END_LITERAL from typing import Iterator, NamedTuple, Optional, Union +@dataclass +class LiteralParseState: + curtoken: bytes + hex_value: Optional[bytes] = None + def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: """Parses input stream into a literal @@ -17,66 +23,40 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: Returns: TokenLiteral """ - curtoken = b"" - parsing_hex_mode = False - hex_value = b"" - for curpos, s in inp: - if parsing_hex_mode: - hex_value, curtoken, skip_idx = _parse_literal_hex(s, hex_value, curtoken) - if skip_idx is not None: - curpos, s = curpos + skip_idx, s[skip_idx:] + state = LiteralParseState(b"") + for curpos, buf in inp: + skip: int = 0 + for i in range(len(buf) + 1): + assert i < len(buf), "MAX_LOOP_REACHED" + if skip >= len(buf): + break + if state.hex_value is not None: + for ci in range(len(buf[skip:])): + c = buf[skip+ci:skip+ci+1] + if HEX.match(c) and len(state.hex_value) < 2: + state.hex_value += c + else: + if state.hex_value: + state.curtoken += int2byte(int(state.hex_value, 16)) + state.hex_value = None + skip += ci + break else: - continue - out = _parse_literal_raw(initialpos, curpos, s, curtoken) - if isinstance(out, TokenLiteral): - return out - else: - curtoken, parsing_hex_mode = out - if parsing_hex_mode: - hex_value = b"" - -class LiteralHexParseOut(NamedTuple): - hex_value: bytes - curtoken: bytes - skip_idx: Optional[int] = None - -def _parse_literal_hex( - s: bytes, - hex_value: bytes, - curtoken: bytes -) -> LiteralHexParseOut: - for ci in range(len(s)): - c = s[ci:ci+1] - if HEX.match(c) and len(hex_value) < 2: - hex_value += c - elif hex_value: - curtoken += int2byte(int(hex_value, 16)) - hex_value = b"" - return LiteralHexParseOut(hex_value, curtoken, ci) - return LiteralHexParseOut(hex_value, curtoken) - -class LiteralParseOut(NamedTuple): - curtoken: bytes - change_to_hex_mode: bool = False - -def _parse_literal_raw( - initialpos: int, - curpos: int, - s: bytes, - curtoken: bytes -) -> Union[LiteralParseOut, TokenLiteral]: - m = END_LITERAL.search(s, 0) - if not m: - return LiteralParseOut(curtoken+s) - else: - j = m.start(0) - curtoken += s[:j] - c = s[j : j + 1] - if c == b"#": - return LiteralParseOut(curtoken, True) - curtoken = curtoken.decode() - return TokenLiteral( - initialpos, - cmp_tsize(curpos, initialpos, j), - curtoken - ) + m = END_LITERAL.search(buf[skip:], 0) + if not m: + state.curtoken += buf[skip:] + skip = len(buf) + else: + j: int = m.start(0) + skip + state.curtoken += buf[skip:j] + c = buf[j : j + 1] + if c == b"#": + state.hex_value = b"" + skip = j+1 + continue + else: + return TokenLiteral( + initialpos, + cmp_tsize(curpos, initialpos, j), + state.curtoken.decode() + ) diff --git a/pdfmajor/tokenizer/tokenizer.py b/pdfmajor/tokenizer/tokenizer.py deleted file mode 100644 index e902839..0000000 --- a/pdfmajor/tokenizer/tokenizer.py +++ /dev/null @@ -1,88 +0,0 @@ - -import io -from pdfmajor.tokenizer.token import TokenKeyword -from pdfmajor.tokenizer.constants import NONSPC -from typing import Iterator -from pdfmajor.tokenizer.exceptions import TokenizerEOF -from .token_parser import PInput, parse_comment, parse_number - - -class PSTokenizer: - def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None: - self.fp = fp - self.bufsize = bufsize - self.bufpos: int = 0 - self.buf: bytes = b'' - self.buf_skipstep: int = 0 - - def iter_buffer(self) -> Iterator[PInput]: - while True: - if self.buf_skipstep > 0: - yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:]) - self.buf_skipstep = 0 - self.bufpos = self.fp.tell() - self.buf = self.fp.read(self.bufsize) - if not self.buf: - raise TokenizerEOF("Unexpected EOF") - else: - yield PInput(self.bufpos, self.buf) - - def seek(self, pos: int): - self.fp.seek(pos) - - def step_back(self, steps: int = 1): - """Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize) - - Args: - steps (int, optional): numbers of steps. Defaults to 1. - """ - self.fp.seek(self.fp.tell() - steps * self.bufsize) - - def iter_tokens(self): - for bufpos, buf in self.iter_buffer(): - m = NONSPC.search(buf, 0) - if not m: - self.buf_skipstep = 0 - continue - else: - j = m.start(0) - c = buf[j : j + 1] - if c == b"%": - self.buf_skipstep = j + 1 - yield parse_comment(bufpos + j, self.iter_buffer()) - elif c == b"/": - raise "WIP" - # self._curtoken = b"" - # self._current_parse_func = self._parse_literal - # return j + 1 - elif c in b"-+" or c.isdigit(): - self.buf_skipstep = j + 1 - yield parse_number(bufpos + j, self.iter_buffer(), c) - elif c == b".": - self.buf_skipstep = j - yield parse_number(bufpos + j, self.iter_buffer()) - elif c.isalpha(): - raise "WIP" - # self._curtoken = c - # self._current_parse_func = self._parse_keyword - # return j + 1 - elif c == b"(": - raise "WIP" - # self._curtoken = b"" - # self.paren = 1 - # self._current_parse_func = self._parse_string - # return j + 1 - elif c == b"<": - raise "WIP" - # self._curtoken = b"" - # self._current_parse_func = self._parse_wopen - # return j + 1 - elif c == b">": - raise "WIP" - # self._curtoken = b"" - # self._current_parse_func = self._parse_wclose - # return j + 1 - else: - self.buf_skipstep = j + 1 - yield TokenKeyword(bufpos + j, 1, c) - diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 0c08aa2..bdf683e 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -1,6 +1,13 @@ from decimal import Decimal import io -from pdfmajor.tokenizer.token import TokenBoolean, TokenComment, TokenDecimal, TokenInteger, TokenLiteral +from pdfmajor.tokenizer import PSTokenizer +from pdfmajor.tokenizer.token import ( + TokenBoolean, + TokenComment, + TokenDecimal, + TokenInteger, + TokenLiteral +) from pdfmajor.tokenizer.token_parsers.util import PInput from pdfmajor.tokenizer.token_parsers.comment import parse_comment from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword @@ -21,7 +28,7 @@ def make_stream_iter(data: bytes, initpos: int = 0): raise "EOF" return iter_stream -class Basics(TestCase): +class Individual(TestCase): def test_comment_parser_ln(self): iter_stream = make_stream_iter( b"this is a lengthy comment that ends here\nso this is not reachable" @@ -90,7 +97,7 @@ class Basics(TestCase): def test_parse_literals(self): iter_stream = make_stream_iter( - b"Some_Name /foo#5f#xbaa", + br"Some_Name /foo#5f#xbaa", 71 ) token = parse_literal(71, iter_stream(1)) @@ -98,3 +105,37 @@ class Basics(TestCase): self.assertEqual(token.pos, 71) self.assertEqual(token.value, 'Some_Name') self.assertEqual(token.size, 9) + + def test_parse_literals_hex(self): + iter_stream = make_stream_iter( + br"foo#5f#xbaa ", + 71 + ) + token = parse_literal(71, iter_stream(100)) + self.assertIsInstance(token, TokenLiteral) + self.assertEqual(token.pos, 71) + self.assertEqual(token.value, 'foo_xbaa') + self.assertEqual(token.size, 11) + +class Tokenizer(TestCase): + def test_case1(self): + tokenizer = PSTokenizer(io.BytesIO(br"""%!PS +begin end + " @ # +/a/BCD /Some_Name /foo#5f#xbaa +0 +1 -2 .5 1.234 +(abc) () (abc ( def ) ghi) +(def\040\0\0404ghi) (bach\\slask) (foo\nbaa) +(this % is not a comment.) +(foo +baa) +(foo\ +baa) +<> <20> < 40 4020 > + +func/a/b{(c)do*}def +[ 1 (z) ! ] +<< /foo (bar) >>""")) + for token in tokenizer.iter_tokens(): + print(token)