diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py index 97c6b72..f8c558b 100644 --- a/pdfmajor/tokenizer/__init__.py +++ b/pdfmajor/tokenizer/__init__.py @@ -1,6 +1,7 @@ import io +from pdfmajor.tokenizer.token_parsers.string import parse_string -from typing import Iterator +from typing import Iterator, Optional from pdfmajor.tokenizer.constants import NONSPC from pdfmajor.tokenizer.exceptions import TokenizerEOF @@ -18,16 +19,22 @@ class PSTokenizer: self.bufsize = bufsize self.bufpos: int = 0 self.buf: bytes = b"" - self.buf_skipstep: int = 0 + self.buf_skipstep: Optional[int] = None + + def inc_buf_skipstep(self, value: int): + if self.buf_skipstep is None: + self.buf_skipstep = value + else: + self.buf_skipstep += value def iter_buffer(self) -> Iterator[PInput]: while True: - if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)): + if (self.buf_skipstep is not None) and (self.buf_skipstep < len(self.buf)): yield PInput( self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :] ) else: - self.buf_skipstep = 0 + self.buf_skipstep = None self.bufpos = self.fp.tell() self.buf = self.fp.read(self.bufsize) if not self.buf: @@ -51,47 +58,46 @@ class PSTokenizer: bufpos, buf = cur m = NONSPC.search(buf, 0) if not m: - self.buf_skipstep = 0 + self.buf_skipstep = None continue else: j = m.start(0) c = buf[j : j + 1] if c == b"%": - self.buf_skipstep += j + 1 + self.inc_buf_skipstep(j + 1) yield self._check_token( cur, parse_comment(bufpos + j, self.iter_buffer()) ) elif c == b"/": - self.buf_skipstep += j + 1 + self.inc_buf_skipstep(j + 1) yield self._check_token( cur, parse_literal(bufpos + j, self.iter_buffer()) ) elif c in b"-+": - self.buf_skipstep += j + 1 + self.inc_buf_skipstep(j + 1) yield self._check_token( cur, parse_number(bufpos + j, self.iter_buffer(), c) ) elif c.isdigit(): - self.buf_skipstep += j + self.inc_buf_skipstep(j) yield self._check_token( cur, parse_number(bufpos + j, self.iter_buffer()) ) elif c == b".": - self.buf_skipstep += j + self.inc_buf_skipstep(j) yield self._check_token( cur, parse_number(bufpos + j, self.iter_buffer()) ) elif c.isalpha(): - self.buf_skipstep += j + self.inc_buf_skipstep(j) yield self._check_token( cur, parse_keyword(bufpos + j, self.iter_buffer()) ) elif c == b"(": - raise NotImplementedError - # self._curtoken = b"" - # self.paren = 1 - # self._current_parse_func = self._parse_string - # return j + 1 + self.inc_buf_skipstep(j + 1) + yield self._check_token( + cur, parse_string(bufpos + j, self.iter_buffer()) + ) elif c == b"<": raise NotImplementedError # self._curtoken = b"" @@ -103,9 +109,9 @@ class PSTokenizer: # self._current_parse_func = self._parse_wclose # return j + 1 else: - self.buf_skipstep += j + 1 + self.inc_buf_skipstep(j + 1) yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c)) def _check_token(self, cur: PInput, token: Token) -> Token: - self.buf_skipstep += token.size + self.inc_buf_skipstep(token.size) return token diff --git a/pdfmajor/tokenizer/token.py b/pdfmajor/tokenizer/token.py index 15c582e..21903dd 100644 --- a/pdfmajor/tokenizer/token.py +++ b/pdfmajor/tokenizer/token.py @@ -1,4 +1,4 @@ -from typing import Generic, TypeVar, Union +from typing import Any, Generic, TypeVar, Union from decimal import Decimal T = TypeVar("T") @@ -13,10 +13,20 @@ class Token(Generic[T]): self.value = value def __repr__(self) -> str: - return "<{cls} pos={pos}>{value}".format( - cls=self.__class__.__name__, pos=self.pos, value=self.value + return "{cls}(pos={pos}, size={size}, value={value})".format( + cls=self.__class__.__name__, pos=self.pos, value=self.value, size=self.size ) + def __eq__(self, other: Any): + if isinstance(other, self.__class__): + return ( + (other.value == self.value) + and (other.pos == self.pos) + and (other.size == self.size) + ) + else: + return False + class TokenComment(Token[bytes]): pass diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py index c682a40..28fe579 100644 --- a/pdfmajor/tokenizer/token_parsers/literal.py +++ b/pdfmajor/tokenizer/token_parsers/literal.py @@ -57,7 +57,7 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: else: return TokenLiteral( initialpos, - cmp_tsize(curpos, initialpos, j), + cmp_tsize(curpos, initialpos, j) - 1, state.curtoken.decode(), ) raise TokenizerEOF diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 246467f..3491d88 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -1,12 +1,15 @@ from decimal import Decimal import io +from typing import List from pdfmajor.tokenizer.token_parsers.string import parse_string from pdfmajor.tokenizer import PSTokenizer from pdfmajor.tokenizer.token import ( + Token, TokenBoolean, TokenComment, TokenDecimal, TokenInteger, + TokenKeyword, TokenLiteral, TokenString, ) @@ -137,3 +140,49 @@ func/a/b{(c)do*}def ) for token in tokenizer.iter_tokens(): print(token) + + def test_case2(self): + tokenizer = PSTokenizer( + io.BytesIO( + br"""10.32 +0 +1 -2 .5 1.234 +-1.234 990""" + ) + ) + for token in tokenizer.iter_tokens(): + print(token) + + def test_case_words1(self): + tokenizer = PSTokenizer( + io.BytesIO( + br"""%!PS +% This is some secret sauce +begin end + " @ # +/a/BCD /Some_Name /foo#5f#xbaa /bar#6a +/need /oh1oh +func/a/b +""" + ) + ) + expected: List[Token] = [ + TokenComment(pos=0, size=4, value=b"!PS"), + TokenComment(pos=5, size=27, value=b" This is some secret sauce"), + TokenKeyword(pos=33, size=5, value=b"begin"), + TokenKeyword(pos=39, size=3, value=b"end"), + TokenKeyword(pos=44, size=1, value=b'"'), + TokenKeyword(pos=47, size=1, value=b"@"), + TokenKeyword(pos=49, size=1, value=b"#"), + TokenLiteral(pos=51, size=1, value="a"), + TokenLiteral(pos=53, size=3, value="BCD"), + TokenLiteral(pos=58, size=9, value="Some_Name"), + TokenLiteral(pos=69, size=11, value="foo_xbaa"), + TokenLiteral(pos=82, size=6, value="barj"), + TokenLiteral(pos=90, size=4, value="need"), + TokenLiteral(pos=96, size=5, value="oh1oh"), + TokenKeyword(pos=103, size=4, value=b"func"), + TokenLiteral(pos=107, size=1, value="a"), + TokenLiteral(pos=109, size=1, value="b"), + ] + for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())): + self.assertEqual(token, etoken, f"Failed at example {i}")