fixed up literal parsing in global tokenizer

This commit is contained in:
ari committed 2020-10-14 11:03:01 -04:00
1 parent 96d68cdf68
commit 5f19f71c4d
4 files changed
+87 -22

No files matched your search

+24 -18
View File
@@ -1,6 +1,7 @@
import io
from pdfmajor.tokenizer.token_parsers.string import parse_string
from typing import Iterator
from typing import Iterator, Optional
from pdfmajor.tokenizer.constants import NONSPC
from pdfmajor.tokenizer.exceptions import TokenizerEOF
@@ -18,16 +19,22 @@ class PSTokenizer:
self.bufsize = bufsize
self.bufpos: int = 0
self.buf: bytes = b""
self.buf_skipstep: int = 0
self.buf_skipstep: Optional[int] = None
def inc_buf_skipstep(self, value: int):
if self.buf_skipstep is None:
self.buf_skipstep = value
else:
self.buf_skipstep += value
def iter_buffer(self) -> Iterator[PInput]:
while True:
if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)):
if (self.buf_skipstep is not None) and (self.buf_skipstep < len(self.buf)):
yield PInput(
self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :]
)
else:
self.buf_skipstep = 0
self.buf_skipstep = None
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.bufsize)
if not self.buf:
@@ -51,47 +58,46 @@ class PSTokenizer:
bufpos, buf = cur
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = 0
self.buf_skipstep = None
continue
else:
j = m.start(0)
c = buf[j : j + 1]
if c == b"%":
self.buf_skipstep += j + 1
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_comment(bufpos + j, self.iter_buffer())
)
elif c == b"/":
self.buf_skipstep += j + 1
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_literal(bufpos + j, self.iter_buffer())
)
elif c in b"-+":
self.buf_skipstep += j + 1
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer(), c)
)
elif c.isdigit():
self.buf_skipstep += j
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer())
)
elif c == b".":
self.buf_skipstep += j
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer())
)
elif c.isalpha():
self.buf_skipstep += j
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_keyword(bufpos + j, self.iter_buffer())
)
elif c == b"(":
raise NotImplementedError
# self._curtoken = b""
# self.paren = 1
# self._current_parse_func = self._parse_string
# return j + 1
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_string(bufpos + j, self.iter_buffer())
)
elif c == b"<":
raise NotImplementedError
# self._curtoken = b""
@@ -103,9 +109,9 @@ class PSTokenizer:
# self._current_parse_func = self._parse_wclose
# return j + 1
else:
self.buf_skipstep += j + 1
self.inc_buf_skipstep(j + 1)
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
def _check_token(self, cur: PInput, token: Token) -> Token:
self.buf_skipstep += token.size
self.inc_buf_skipstep(token.size)
return token
+13 -3
View File
@@ -1,4 +1,4 @@
from typing import Generic, TypeVar, Union
from typing import Any, Generic, TypeVar, Union
from decimal import Decimal
T = TypeVar("T")
@@ -13,10 +13,20 @@ class Token(Generic[T]):
self.value = value
def __repr__(self) -> str:
return "<{cls} pos={pos}>{value}</{cls}>".format(
cls=self.__class__.__name__, pos=self.pos, value=self.value
return "{cls}(pos={pos}, size={size}, value={value})".format(
cls=self.__class__.__name__, pos=self.pos, value=self.value, size=self.size
)
def __eq__(self, other: Any):
if isinstance(other, self.__class__):
return (
(other.value == self.value)
and (other.pos == self.pos)
and (other.size == self.size)
)
else:
return False
class TokenComment(Token[bytes]):
pass
+1 -1
View File
@@ -57,7 +57,7 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
else:
return TokenLiteral(
initialpos,
cmp_tsize(curpos, initialpos, j),
cmp_tsize(curpos, initialpos, j) - 1,
state.curtoken.decode(),
)
raise TokenizerEOF
+49
View File
@@ -1,12 +1,15 @@
from decimal import Decimal
import io
from typing import List
from pdfmajor.tokenizer.token_parsers.string import parse_string
from pdfmajor.tokenizer import PSTokenizer
from pdfmajor.tokenizer.token import (
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenInteger,
TokenKeyword,
TokenLiteral,
TokenString,
)
@@ -137,3 +140,49 @@ func/a/b{(c)do*}def
)
for token in tokenizer.iter_tokens():
print(token)
def test_case2(self):
tokenizer = PSTokenizer(
io.BytesIO(
br"""10.32
0 +1 -2 .5 1.234
-1.234 990"""
)
)
for token in tokenizer.iter_tokens():
print(token)
def test_case_words1(self):
tokenizer = PSTokenizer(
io.BytesIO(
br"""%!PS
% This is some secret sauce
begin end
" @ #
/a/BCD /Some_Name /foo#5f#xbaa /bar#6a
/need /oh1oh
func/a/b
"""
)
)
expected: List[Token] = [
TokenComment(pos=0, size=4, value=b"!PS"),
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
TokenKeyword(pos=33, size=5, value=b"begin"),
TokenKeyword(pos=39, size=3, value=b"end"),
TokenKeyword(pos=44, size=1, value=b'"'),
TokenKeyword(pos=47, size=1, value=b"@"),
TokenKeyword(pos=49, size=1, value=b"#"),
TokenLiteral(pos=51, size=1, value="a"),
TokenLiteral(pos=53, size=3, value="BCD"),
TokenLiteral(pos=58, size=9, value="Some_Name"),
TokenLiteral(pos=69, size=11, value="foo_xbaa"),
TokenLiteral(pos=82, size=6, value="barj"),
TokenLiteral(pos=90, size=4, value="need"),
TokenLiteral(pos=96, size=5, value="oh1oh"),
TokenKeyword(pos=103, size=4, value=b"func"),
TokenLiteral(pos=107, size=1, value="a"),
TokenLiteral(pos=109, size=1, value="b"),
]
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
self.assertEqual(token, etoken, f"Failed at example {i}")