working literal parser
first mock of main tokenizer
This commit is contained in:
1 parent
a0cc862804
commit
ab657deda4
5 files changed
+173
-191
No files matched your search
@@ -1,46 +1,95 @@
|
||||
|
||||
import io
|
||||
from typing import List
|
||||
from pdfmajor.tokenizer.token import Token
|
||||
|
||||
from typing import Iterator
|
||||
|
||||
from pdfmajor.tokenizer.constants import NONSPC
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.parse_funcs import ParseFunc
|
||||
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
||||
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
|
||||
from pdfmajor.tokenizer.token_parsers.number import parse_number
|
||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token import Token, TokenKeyword
|
||||
|
||||
|
||||
class PSTokenizer:
|
||||
def __init__(self, fp: io.BufferedIOBase) -> None:
|
||||
self.fp = fp
|
||||
self.seek(0)
|
||||
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
|
||||
self.fp = fp
|
||||
self.bufsize = bufsize
|
||||
self.bufpos: int = 0
|
||||
self.buf: bytes = b''
|
||||
self.buf_skipstep: int = 0
|
||||
|
||||
def seek(self, pos: int):
|
||||
"""Seeks tokenizer to the given position."""
|
||||
self.fp.seek(pos)
|
||||
# reset the status for nextline()
|
||||
self.bufpos: int = pos
|
||||
self.buf: bytes = b""
|
||||
self.charpos: int = 0
|
||||
# reset the status for nexttoken()
|
||||
self._current_parse_func: ParseFunc = self._parse_main
|
||||
self._curtoken: bytes = b""
|
||||
self._curtokenpos: int = 0
|
||||
self._tokens: List[Token] = []
|
||||
return
|
||||
def iter_buffer(self) -> Iterator[PInput]:
|
||||
while True:
|
||||
if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)):
|
||||
yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:])
|
||||
else:
|
||||
self.buf_skipstep = 0
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.bufsize)
|
||||
if not self.buf:
|
||||
raise TokenizerEOF("Unexpected EOF")
|
||||
else:
|
||||
yield PInput(self.bufpos, self.buf)
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos)
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
|
||||
def step_back(self, steps: int = 1):
|
||||
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
|
||||
|
||||
def nexttoken(self):
|
||||
while not self._tokens:
|
||||
self.fillbuf()
|
||||
self.charpos = self._current_parse_func(self.buf, self.charpos)
|
||||
token = self._tokens.pop(0)
|
||||
return token
|
||||
Args:
|
||||
steps (int, optional): numbers of steps. Defaults to 1.
|
||||
"""
|
||||
self.fp.seek(self.fp.tell() - steps * self.bufsize)
|
||||
|
||||
def fillbuf(self):
|
||||
if self.charpos < len(self.buf):
|
||||
return
|
||||
# fetch next chunk.
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.BUFSIZ)
|
||||
if not self.buf:
|
||||
raise TokenizerEOF("Unexpected EOF")
|
||||
self.charpos = 0
|
||||
return
|
||||
def iter_tokens(self) -> Iterator[Token]:
|
||||
for cur in self.iter_buffer():
|
||||
bufpos, buf = cur
|
||||
m = NONSPC.search(buf, 0)
|
||||
if not m:
|
||||
self.buf_skipstep = 0
|
||||
continue
|
||||
else:
|
||||
j = m.start(0)
|
||||
c = buf[j : j + 1]
|
||||
if c == b"%":
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, parse_comment(bufpos + j, self.iter_buffer()))
|
||||
elif c == b"/":
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer()))
|
||||
elif c in b"-+" or c.isdigit():
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c))
|
||||
elif c == b".":
|
||||
self.buf_skipstep += j
|
||||
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
|
||||
elif c.isalpha():
|
||||
self.buf_skipstep += j
|
||||
yield self._check_token(cur, parse_keyword(bufpos + j, self.iter_buffer()))
|
||||
elif c == b"(":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self.paren = 1
|
||||
# self._current_parse_func = self._parse_string
|
||||
# return j + 1
|
||||
elif c == b"<":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wopen
|
||||
# return j + 1
|
||||
elif c == b">":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wclose
|
||||
# return j + 1
|
||||
else:
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
|
||||
|
||||
def _check_token(self, cur: PInput, token: Token) -> Token:
|
||||
self.buf_skipstep += token.size
|
||||
return token
|
||||
@@ -29,7 +29,7 @@ class TokenHex(Token[bytes]):
|
||||
class TokenKeyword(Token[bytes]):
|
||||
pass
|
||||
|
||||
class TokenLiteral(Token[bytes]):
|
||||
class TokenLiteral(Token[str]):
|
||||
pass
|
||||
|
||||
class TokenBoolean(Token[bool]):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
from pdfmajor.tokenizer.token_parsers.util import cmp_tsize
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
@@ -7,6 +8,11 @@ from ..constants import HEX, END_LITERAL
|
||||
|
||||
from typing import Iterator, NamedTuple, Optional, Union
|
||||
|
||||
@dataclass
|
||||
class LiteralParseState:
|
||||
curtoken: bytes
|
||||
hex_value: Optional[bytes] = None
|
||||
|
||||
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
"""Parses input stream into a literal
|
||||
|
||||
@@ -17,66 +23,40 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
Returns:
|
||||
TokenLiteral
|
||||
"""
|
||||
curtoken = b""
|
||||
parsing_hex_mode = False
|
||||
hex_value = b""
|
||||
for curpos, s in inp:
|
||||
if parsing_hex_mode:
|
||||
hex_value, curtoken, skip_idx = _parse_literal_hex(s, hex_value, curtoken)
|
||||
if skip_idx is not None:
|
||||
curpos, s = curpos + skip_idx, s[skip_idx:]
|
||||
state = LiteralParseState(b"")
|
||||
for curpos, buf in inp:
|
||||
skip: int = 0
|
||||
for i in range(len(buf) + 1):
|
||||
assert i < len(buf), "MAX_LOOP_REACHED"
|
||||
if skip >= len(buf):
|
||||
break
|
||||
if state.hex_value is not None:
|
||||
for ci in range(len(buf[skip:])):
|
||||
c = buf[skip+ci:skip+ci+1]
|
||||
if HEX.match(c) and len(state.hex_value) < 2:
|
||||
state.hex_value += c
|
||||
else:
|
||||
if state.hex_value:
|
||||
state.curtoken += int2byte(int(state.hex_value, 16))
|
||||
state.hex_value = None
|
||||
skip += ci
|
||||
break
|
||||
else:
|
||||
continue
|
||||
out = _parse_literal_raw(initialpos, curpos, s, curtoken)
|
||||
if isinstance(out, TokenLiteral):
|
||||
return out
|
||||
else:
|
||||
curtoken, parsing_hex_mode = out
|
||||
if parsing_hex_mode:
|
||||
hex_value = b""
|
||||
|
||||
class LiteralHexParseOut(NamedTuple):
|
||||
hex_value: bytes
|
||||
curtoken: bytes
|
||||
skip_idx: Optional[int] = None
|
||||
|
||||
def _parse_literal_hex(
|
||||
s: bytes,
|
||||
hex_value: bytes,
|
||||
curtoken: bytes
|
||||
) -> LiteralHexParseOut:
|
||||
for ci in range(len(s)):
|
||||
c = s[ci:ci+1]
|
||||
if HEX.match(c) and len(hex_value) < 2:
|
||||
hex_value += c
|
||||
elif hex_value:
|
||||
curtoken += int2byte(int(hex_value, 16))
|
||||
hex_value = b""
|
||||
return LiteralHexParseOut(hex_value, curtoken, ci)
|
||||
return LiteralHexParseOut(hex_value, curtoken)
|
||||
|
||||
class LiteralParseOut(NamedTuple):
|
||||
curtoken: bytes
|
||||
change_to_hex_mode: bool = False
|
||||
|
||||
def _parse_literal_raw(
|
||||
initialpos: int,
|
||||
curpos: int,
|
||||
s: bytes,
|
||||
curtoken: bytes
|
||||
) -> Union[LiteralParseOut, TokenLiteral]:
|
||||
m = END_LITERAL.search(s, 0)
|
||||
if not m:
|
||||
return LiteralParseOut(curtoken+s)
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"#":
|
||||
return LiteralParseOut(curtoken, True)
|
||||
curtoken = curtoken.decode()
|
||||
return TokenLiteral(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken
|
||||
)
|
||||
m = END_LITERAL.search(buf[skip:], 0)
|
||||
if not m:
|
||||
state.curtoken += buf[skip:]
|
||||
skip = len(buf)
|
||||
else:
|
||||
j: int = m.start(0) + skip
|
||||
state.curtoken += buf[skip:j]
|
||||
c = buf[j : j + 1]
|
||||
if c == b"#":
|
||||
state.hex_value = b""
|
||||
skip = j+1
|
||||
continue
|
||||
else:
|
||||
return TokenLiteral(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
state.curtoken.decode()
|
||||
)
|
||||
@@ -1,88 +0,0 @@
|
||||
|
||||
import io
|
||||
from pdfmajor.tokenizer.token import TokenKeyword
|
||||
from pdfmajor.tokenizer.constants import NONSPC
|
||||
from typing import Iterator
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from .token_parser import PInput, parse_comment, parse_number
|
||||
|
||||
|
||||
class PSTokenizer:
|
||||
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
|
||||
self.fp = fp
|
||||
self.bufsize = bufsize
|
||||
self.bufpos: int = 0
|
||||
self.buf: bytes = b''
|
||||
self.buf_skipstep: int = 0
|
||||
|
||||
def iter_buffer(self) -> Iterator[PInput]:
|
||||
while True:
|
||||
if self.buf_skipstep > 0:
|
||||
yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:])
|
||||
self.buf_skipstep = 0
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.bufsize)
|
||||
if not self.buf:
|
||||
raise TokenizerEOF("Unexpected EOF")
|
||||
else:
|
||||
yield PInput(self.bufpos, self.buf)
|
||||
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
|
||||
def step_back(self, steps: int = 1):
|
||||
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
|
||||
|
||||
Args:
|
||||
steps (int, optional): numbers of steps. Defaults to 1.
|
||||
"""
|
||||
self.fp.seek(self.fp.tell() - steps * self.bufsize)
|
||||
|
||||
def iter_tokens(self):
|
||||
for bufpos, buf in self.iter_buffer():
|
||||
m = NONSPC.search(buf, 0)
|
||||
if not m:
|
||||
self.buf_skipstep = 0
|
||||
continue
|
||||
else:
|
||||
j = m.start(0)
|
||||
c = buf[j : j + 1]
|
||||
if c == b"%":
|
||||
self.buf_skipstep = j + 1
|
||||
yield parse_comment(bufpos + j, self.iter_buffer())
|
||||
elif c == b"/":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_literal
|
||||
# return j + 1
|
||||
elif c in b"-+" or c.isdigit():
|
||||
self.buf_skipstep = j + 1
|
||||
yield parse_number(bufpos + j, self.iter_buffer(), c)
|
||||
elif c == b".":
|
||||
self.buf_skipstep = j
|
||||
yield parse_number(bufpos + j, self.iter_buffer())
|
||||
elif c.isalpha():
|
||||
raise "WIP"
|
||||
# self._curtoken = c
|
||||
# self._current_parse_func = self._parse_keyword
|
||||
# return j + 1
|
||||
elif c == b"(":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self.paren = 1
|
||||
# self._current_parse_func = self._parse_string
|
||||
# return j + 1
|
||||
elif c == b"<":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wopen
|
||||
# return j + 1
|
||||
elif c == b">":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wclose
|
||||
# return j + 1
|
||||
else:
|
||||
self.buf_skipstep = j + 1
|
||||
yield TokenKeyword(bufpos + j, 1, c)
|
||||
|
||||
+44
-3
@@ -1,6 +1,13 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
from pdfmajor.tokenizer.token import TokenBoolean, TokenComment, TokenDecimal, TokenInteger, TokenLiteral
|
||||
from pdfmajor.tokenizer import PSTokenizer
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenInteger,
|
||||
TokenLiteral
|
||||
)
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
||||
@@ -21,7 +28,7 @@ def make_stream_iter(data: bytes, initpos: int = 0):
|
||||
raise "EOF"
|
||||
return iter_stream
|
||||
|
||||
class Basics(TestCase):
|
||||
class Individual(TestCase):
|
||||
def test_comment_parser_ln(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
@@ -90,7 +97,7 @@ class Basics(TestCase):
|
||||
|
||||
def test_parse_literals(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"Some_Name /foo#5f#xbaa",
|
||||
br"Some_Name /foo#5f#xbaa",
|
||||
71
|
||||
)
|
||||
token = parse_literal(71, iter_stream(1))
|
||||
@@ -98,3 +105,37 @@ class Basics(TestCase):
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, 'Some_Name')
|
||||
self.assertEqual(token.size, 9)
|
||||
|
||||
def test_parse_literals_hex(self):
|
||||
iter_stream = make_stream_iter(
|
||||
br"foo#5f#xbaa ",
|
||||
71
|
||||
)
|
||||
token = parse_literal(71, iter_stream(100))
|
||||
self.assertIsInstance(token, TokenLiteral)
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, 'foo_xbaa')
|
||||
self.assertEqual(token.size, 11)
|
||||
|
||||
class Tokenizer(TestCase):
|
||||
def test_case1(self):
|
||||
tokenizer = PSTokenizer(io.BytesIO(br"""%!PS
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#xbaa
|
||||
0 +1 -2 .5 1.234
|
||||
(abc) () (abc ( def ) ghi)
|
||||
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
||||
(this % is not a comment.)
|
||||
(foo
|
||||
baa)
|
||||
(foo\
|
||||
baa)
|
||||
<> <20> < 40 4020 >
|
||||
<abcd00
|
||||
12345>
|
||||
func/a/b{(c)do*}def
|
||||
[ 1 (z) ! ]
|
||||
<< /foo (bar) >>"""))
|
||||
for token in tokenizer.iter_tokens():
|
||||
print(token)
|
||||
Reference in new issue
Block a user