fixed up literal parsing in global tokenizer
This commit is contained in:
1 parent
96d68cdf68
commit
5f19f71c4d
4 files changed
+87
-22
No files matched your search
@@ -1,6 +1,7 @@
|
||||
import io
|
||||
from pdfmajor.tokenizer.token_parsers.string import parse_string
|
||||
|
||||
from typing import Iterator
|
||||
from typing import Iterator, Optional
|
||||
|
||||
from pdfmajor.tokenizer.constants import NONSPC
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
@@ -18,16 +19,22 @@ class PSTokenizer:
|
||||
self.bufsize = bufsize
|
||||
self.bufpos: int = 0
|
||||
self.buf: bytes = b""
|
||||
self.buf_skipstep: int = 0
|
||||
self.buf_skipstep: Optional[int] = None
|
||||
|
||||
def inc_buf_skipstep(self, value: int):
|
||||
if self.buf_skipstep is None:
|
||||
self.buf_skipstep = value
|
||||
else:
|
||||
self.buf_skipstep += value
|
||||
|
||||
def iter_buffer(self) -> Iterator[PInput]:
|
||||
while True:
|
||||
if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)):
|
||||
if (self.buf_skipstep is not None) and (self.buf_skipstep < len(self.buf)):
|
||||
yield PInput(
|
||||
self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :]
|
||||
)
|
||||
else:
|
||||
self.buf_skipstep = 0
|
||||
self.buf_skipstep = None
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.bufsize)
|
||||
if not self.buf:
|
||||
@@ -51,47 +58,46 @@ class PSTokenizer:
|
||||
bufpos, buf = cur
|
||||
m = NONSPC.search(buf, 0)
|
||||
if not m:
|
||||
self.buf_skipstep = 0
|
||||
self.buf_skipstep = None
|
||||
continue
|
||||
else:
|
||||
j = m.start(0)
|
||||
c = buf[j : j + 1]
|
||||
if c == b"%":
|
||||
self.buf_skipstep += j + 1
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_comment(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b"/":
|
||||
self.buf_skipstep += j + 1
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_literal(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c in b"-+":
|
||||
self.buf_skipstep += j + 1
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer(), c)
|
||||
)
|
||||
elif c.isdigit():
|
||||
self.buf_skipstep += j
|
||||
self.inc_buf_skipstep(j)
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b".":
|
||||
self.buf_skipstep += j
|
||||
self.inc_buf_skipstep(j)
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c.isalpha():
|
||||
self.buf_skipstep += j
|
||||
self.inc_buf_skipstep(j)
|
||||
yield self._check_token(
|
||||
cur, parse_keyword(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b"(":
|
||||
raise NotImplementedError
|
||||
# self._curtoken = b""
|
||||
# self.paren = 1
|
||||
# self._current_parse_func = self._parse_string
|
||||
# return j + 1
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_string(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b"<":
|
||||
raise NotImplementedError
|
||||
# self._curtoken = b""
|
||||
@@ -103,9 +109,9 @@ class PSTokenizer:
|
||||
# self._current_parse_func = self._parse_wclose
|
||||
# return j + 1
|
||||
else:
|
||||
self.buf_skipstep += j + 1
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
|
||||
|
||||
def _check_token(self, cur: PInput, token: Token) -> Token:
|
||||
self.buf_skipstep += token.size
|
||||
self.inc_buf_skipstep(token.size)
|
||||
return token
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Generic, TypeVar, Union
|
||||
from typing import Any, Generic, TypeVar, Union
|
||||
from decimal import Decimal
|
||||
|
||||
T = TypeVar("T")
|
||||
@@ -13,10 +13,20 @@ class Token(Generic[T]):
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "<{cls} pos={pos}>{value}</{cls}>".format(
|
||||
cls=self.__class__.__name__, pos=self.pos, value=self.value
|
||||
return "{cls}(pos={pos}, size={size}, value={value})".format(
|
||||
cls=self.__class__.__name__, pos=self.pos, value=self.value, size=self.size
|
||||
)
|
||||
|
||||
def __eq__(self, other: Any):
|
||||
if isinstance(other, self.__class__):
|
||||
return (
|
||||
(other.value == self.value)
|
||||
and (other.pos == self.pos)
|
||||
and (other.size == self.size)
|
||||
)
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
class TokenComment(Token[bytes]):
|
||||
pass
|
||||
|
||||
@@ -57,7 +57,7 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
else:
|
||||
return TokenLiteral(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
cmp_tsize(curpos, initialpos, j) - 1,
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
raise TokenizerEOF
|
||||
@@ -1,12 +1,15 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
from typing import List
|
||||
from pdfmajor.tokenizer.token_parsers.string import parse_string
|
||||
from pdfmajor.tokenizer import PSTokenizer
|
||||
from pdfmajor.tokenizer.token import (
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenInteger,
|
||||
TokenKeyword,
|
||||
TokenLiteral,
|
||||
TokenString,
|
||||
)
|
||||
@@ -137,3 +140,49 @@ func/a/b{(c)do*}def
|
||||
)
|
||||
for token in tokenizer.iter_tokens():
|
||||
print(token)
|
||||
|
||||
def test_case2(self):
|
||||
tokenizer = PSTokenizer(
|
||||
io.BytesIO(
|
||||
br"""10.32
|
||||
0 +1 -2 .5 1.234
|
||||
-1.234 990"""
|
||||
)
|
||||
)
|
||||
for token in tokenizer.iter_tokens():
|
||||
print(token)
|
||||
|
||||
def test_case_words1(self):
|
||||
tokenizer = PSTokenizer(
|
||||
io.BytesIO(
|
||||
br"""%!PS
|
||||
% This is some secret sauce
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#xbaa /bar#6a
|
||||
/need /oh1oh
|
||||
func/a/b
|
||||
"""
|
||||
)
|
||||
)
|
||||
expected: List[Token] = [
|
||||
TokenComment(pos=0, size=4, value=b"!PS"),
|
||||
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
|
||||
TokenKeyword(pos=33, size=5, value=b"begin"),
|
||||
TokenKeyword(pos=39, size=3, value=b"end"),
|
||||
TokenKeyword(pos=44, size=1, value=b'"'),
|
||||
TokenKeyword(pos=47, size=1, value=b"@"),
|
||||
TokenKeyword(pos=49, size=1, value=b"#"),
|
||||
TokenLiteral(pos=51, size=1, value="a"),
|
||||
TokenLiteral(pos=53, size=3, value="BCD"),
|
||||
TokenLiteral(pos=58, size=9, value="Some_Name"),
|
||||
TokenLiteral(pos=69, size=11, value="foo_xbaa"),
|
||||
TokenLiteral(pos=82, size=6, value="barj"),
|
||||
TokenLiteral(pos=90, size=4, value="need"),
|
||||
TokenLiteral(pos=96, size=5, value="oh1oh"),
|
||||
TokenKeyword(pos=103, size=4, value=b"func"),
|
||||
TokenLiteral(pos=107, size=1, value="a"),
|
||||
TokenLiteral(pos=109, size=1, value="b"),
|
||||
]
|
||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
|
||||
self.assertEqual(token, etoken, f"Failed at example {i}")
|
||||
Reference in new issue
Block a user