working literal parser

first mock of main tokenizer
This commit is contained in:
ari committed 2020-10-13 20:52:23 -04:00
1 parent a0cc862804
commit ab657deda4
5 files changed
+173 -191

No files matched your search

+86 -37
View File
@@ -1,46 +1,95 @@
import io
from typing import List
from pdfmajor.tokenizer.token import Token
from typing import Iterator
from pdfmajor.tokenizer.constants import NONSPC
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.parse_funcs import ParseFunc
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
from pdfmajor.tokenizer.token_parsers.number import parse_number
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token import Token, TokenKeyword
class PSTokenizer:
def __init__(self, fp: io.BufferedIOBase) -> None:
self.fp = fp
self.seek(0)
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
self.fp = fp
self.bufsize = bufsize
self.bufpos: int = 0
self.buf: bytes = b''
self.buf_skipstep: int = 0
def seek(self, pos: int):
"""Seeks tokenizer to the given position."""
self.fp.seek(pos)
# reset the status for nextline()
self.bufpos: int = pos
self.buf: bytes = b""
self.charpos: int = 0
# reset the status for nexttoken()
self._current_parse_func: ParseFunc = self._parse_main
self._curtoken: bytes = b""
self._curtokenpos: int = 0
self._tokens: List[Token] = []
return
def iter_buffer(self) -> Iterator[PInput]:
while True:
if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)):
yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:])
else:
self.buf_skipstep = 0
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.bufsize)
if not self.buf:
raise TokenizerEOF("Unexpected EOF")
else:
yield PInput(self.bufpos, self.buf)
def __repr__(self):
return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos)
def seek(self, pos: int):
self.fp.seek(pos)
def step_back(self, steps: int = 1):
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
def nexttoken(self):
while not self._tokens:
self.fillbuf()
self.charpos = self._current_parse_func(self.buf, self.charpos)
token = self._tokens.pop(0)
return token
Args:
steps (int, optional): numbers of steps. Defaults to 1.
"""
self.fp.seek(self.fp.tell() - steps * self.bufsize)
def fillbuf(self):
if self.charpos < len(self.buf):
return
# fetch next chunk.
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.BUFSIZ)
if not self.buf:
raise TokenizerEOF("Unexpected EOF")
self.charpos = 0
return
def iter_tokens(self) -> Iterator[Token]:
for cur in self.iter_buffer():
bufpos, buf = cur
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = 0
continue
else:
j = m.start(0)
c = buf[j : j + 1]
if c == b"%":
self.buf_skipstep += j + 1
yield self._check_token(cur, parse_comment(bufpos + j, self.iter_buffer()))
elif c == b"/":
self.buf_skipstep += j + 1
yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer()))
elif c in b"-+" or c.isdigit():
self.buf_skipstep += j + 1
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c))
elif c == b".":
self.buf_skipstep += j
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
elif c.isalpha():
self.buf_skipstep += j
yield self._check_token(cur, parse_keyword(bufpos + j, self.iter_buffer()))
elif c == b"(":
raise "WIP"
# self._curtoken = b""
# self.paren = 1
# self._current_parse_func = self._parse_string
# return j + 1
elif c == b"<":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wopen
# return j + 1
elif c == b">":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wclose
# return j + 1
else:
self.buf_skipstep += j + 1
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
def _check_token(self, cur: PInput, token: Token) -> Token:
self.buf_skipstep += token.size
return token
+1 -1
View File
@@ -29,7 +29,7 @@ class TokenHex(Token[bytes]):
class TokenKeyword(Token[bytes]):
pass
class TokenLiteral(Token[bytes]):
class TokenLiteral(Token[str]):
pass
class TokenBoolean(Token[bool]):
+42 -62
View File
@@ -1,3 +1,4 @@
from dataclasses import dataclass
from pdfmajor.tokenizer.token_parsers.util import cmp_tsize
from pdfmajor.tokenizer.token_parsers.util import PInput
@@ -7,6 +8,11 @@ from ..constants import HEX, END_LITERAL
from typing import Iterator, NamedTuple, Optional, Union
@dataclass
class LiteralParseState:
curtoken: bytes
hex_value: Optional[bytes] = None
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
"""Parses input stream into a literal
@@ -17,66 +23,40 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
Returns:
TokenLiteral
"""
curtoken = b""
parsing_hex_mode = False
hex_value = b""
for curpos, s in inp:
if parsing_hex_mode:
hex_value, curtoken, skip_idx = _parse_literal_hex(s, hex_value, curtoken)
if skip_idx is not None:
curpos, s = curpos + skip_idx, s[skip_idx:]
state = LiteralParseState(b"")
for curpos, buf in inp:
skip: int = 0
for i in range(len(buf) + 1):
assert i < len(buf), "MAX_LOOP_REACHED"
if skip >= len(buf):
break
if state.hex_value is not None:
for ci in range(len(buf[skip:])):
c = buf[skip+ci:skip+ci+1]
if HEX.match(c) and len(state.hex_value) < 2:
state.hex_value += c
else:
if state.hex_value:
state.curtoken += int2byte(int(state.hex_value, 16))
state.hex_value = None
skip += ci
break
else:
continue
out = _parse_literal_raw(initialpos, curpos, s, curtoken)
if isinstance(out, TokenLiteral):
return out
else:
curtoken, parsing_hex_mode = out
if parsing_hex_mode:
hex_value = b""
class LiteralHexParseOut(NamedTuple):
hex_value: bytes
curtoken: bytes
skip_idx: Optional[int] = None
def _parse_literal_hex(
s: bytes,
hex_value: bytes,
curtoken: bytes
) -> LiteralHexParseOut:
for ci in range(len(s)):
c = s[ci:ci+1]
if HEX.match(c) and len(hex_value) < 2:
hex_value += c
elif hex_value:
curtoken += int2byte(int(hex_value, 16))
hex_value = b""
return LiteralHexParseOut(hex_value, curtoken, ci)
return LiteralHexParseOut(hex_value, curtoken)
class LiteralParseOut(NamedTuple):
curtoken: bytes
change_to_hex_mode: bool = False
def _parse_literal_raw(
initialpos: int,
curpos: int,
s: bytes,
curtoken: bytes
) -> Union[LiteralParseOut, TokenLiteral]:
m = END_LITERAL.search(s, 0)
if not m:
return LiteralParseOut(curtoken+s)
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if c == b"#":
return LiteralParseOut(curtoken, True)
curtoken = curtoken.decode()
return TokenLiteral(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken
)
m = END_LITERAL.search(buf[skip:], 0)
if not m:
state.curtoken += buf[skip:]
skip = len(buf)
else:
j: int = m.start(0) + skip
state.curtoken += buf[skip:j]
c = buf[j : j + 1]
if c == b"#":
state.hex_value = b""
skip = j+1
continue
else:
return TokenLiteral(
initialpos,
cmp_tsize(curpos, initialpos, j),
state.curtoken.decode()
)
-88
View File
@@ -1,88 +0,0 @@
import io
from pdfmajor.tokenizer.token import TokenKeyword
from pdfmajor.tokenizer.constants import NONSPC
from typing import Iterator
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from .token_parser import PInput, parse_comment, parse_number
class PSTokenizer:
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
self.fp = fp
self.bufsize = bufsize
self.bufpos: int = 0
self.buf: bytes = b''
self.buf_skipstep: int = 0
def iter_buffer(self) -> Iterator[PInput]:
while True:
if self.buf_skipstep > 0:
yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:])
self.buf_skipstep = 0
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.bufsize)
if not self.buf:
raise TokenizerEOF("Unexpected EOF")
else:
yield PInput(self.bufpos, self.buf)
def seek(self, pos: int):
self.fp.seek(pos)
def step_back(self, steps: int = 1):
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
Args:
steps (int, optional): numbers of steps. Defaults to 1.
"""
self.fp.seek(self.fp.tell() - steps * self.bufsize)
def iter_tokens(self):
for bufpos, buf in self.iter_buffer():
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = 0
continue
else:
j = m.start(0)
c = buf[j : j + 1]
if c == b"%":
self.buf_skipstep = j + 1
yield parse_comment(bufpos + j, self.iter_buffer())
elif c == b"/":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_literal
# return j + 1
elif c in b"-+" or c.isdigit():
self.buf_skipstep = j + 1
yield parse_number(bufpos + j, self.iter_buffer(), c)
elif c == b".":
self.buf_skipstep = j
yield parse_number(bufpos + j, self.iter_buffer())
elif c.isalpha():
raise "WIP"
# self._curtoken = c
# self._current_parse_func = self._parse_keyword
# return j + 1
elif c == b"(":
raise "WIP"
# self._curtoken = b""
# self.paren = 1
# self._current_parse_func = self._parse_string
# return j + 1
elif c == b"<":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wopen
# return j + 1
elif c == b">":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wclose
# return j + 1
else:
self.buf_skipstep = j + 1
yield TokenKeyword(bufpos + j, 1, c)
+44 -3
View File
@@ -1,6 +1,13 @@
from decimal import Decimal
import io
from pdfmajor.tokenizer.token import TokenBoolean, TokenComment, TokenDecimal, TokenInteger, TokenLiteral
from pdfmajor.tokenizer import PSTokenizer
from pdfmajor.tokenizer.token import (
TokenBoolean,
TokenComment,
TokenDecimal,
TokenInteger,
TokenLiteral
)
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
@@ -21,7 +28,7 @@ def make_stream_iter(data: bytes, initpos: int = 0):
raise "EOF"
return iter_stream
class Basics(TestCase):
class Individual(TestCase):
def test_comment_parser_ln(self):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\nso this is not reachable"
@@ -90,7 +97,7 @@ class Basics(TestCase):
def test_parse_literals(self):
iter_stream = make_stream_iter(
b"Some_Name /foo#5f#xbaa",
br"Some_Name /foo#5f#xbaa",
71
)
token = parse_literal(71, iter_stream(1))
@@ -98,3 +105,37 @@ class Basics(TestCase):
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, 'Some_Name')
self.assertEqual(token.size, 9)
def test_parse_literals_hex(self):
iter_stream = make_stream_iter(
br"foo#5f#xbaa ",
71
)
token = parse_literal(71, iter_stream(100))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, 'foo_xbaa')
self.assertEqual(token.size, 11)
class Tokenizer(TestCase):
def test_case1(self):
tokenizer = PSTokenizer(io.BytesIO(br"""%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#xbaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>"""))
for token in tokenizer.iter_tokens():
print(token)