using a more functional approach for lexer

This commit is contained in:
ari committed 2020-10-26 17:55:55 -04:00
1 parent e2a92517cd
commit 087c1c2fe9
2 files changed
+69 -84

No files matched your search

+47 -75
View File
@@ -1,4 +1,3 @@
import io
from typing import Iterator
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket
from pdfmajor.lexer.string import parse_string
@@ -18,80 +17,53 @@ from pdfmajor.lexer.token import (
from pdfmajor.streambuffer import BufferStream
class PDFLexer:
def __init__(self, buffer: BufferStream) -> None:
self.buffer = buffer
def iter_tokens(buffer: BufferStream) -> Iterator[Token]:
"""Iterates over the tokens in the stream
@classmethod
def from_iobuffer(cls, fp: io.BufferedIOBase, buffer_size: int = 4096):
return cls(BufferStream(fp, buffer_size=buffer_size))
@classmethod
def from_bytes(cls, data: bytes, buffer_size: int = 4096):
return cls(BufferStream(io.BytesIO(data), buffer_size=buffer_size))
def tell(self) -> int:
return self.buffer.tell()
def seek(self, pos: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.buffer.seek(pos)
def iter_tokens(self) -> Iterator[Token]:
"""Iterates over the tokens in the stream
Yields:
Iterator[Token]
"""
for bufpos, buf in self.buffer:
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = None
Yields:
Iterator[Token]
"""
for bufpos, buf in buffer:
m = NONSPC.search(buf, 0)
if not m:
continue
else:
j = m.start(0)
next_char = buf[j : j + 1]
if next_char == b"~":
continue
elif next_char == b"%":
buffer.seek(bufpos + j + 1)
yield parse_comment(buffer)
elif next_char == b"/":
buffer.seek(bufpos + j + 1)
yield parse_name(buffer)
elif next_char in b"-+":
buffer.seek(bufpos + j + 1)
yield parse_number(buffer, next_char)
elif next_char.isdigit() or next_char == b".":
buffer.seek(bufpos + j)
yield parse_number(buffer, initialpos=buffer.tell())
elif next_char.isalpha():
buffer.seek(bufpos + j)
yield parse_keyword(buffer, buffer.tell())
elif next_char == b"(":
buffer.seek(bufpos + j + 1)
yield parse_string(buffer)
elif next_char == b"<":
buffer.seek(bufpos + j + 1)
yield parse_double_angled_bracket(buffer)
elif next_char == b">":
buffer.seek(bufpos + j + 1)
if buf[j : j + 2] == b">>":
buffer.seekd(1)
yield TokenDictionary(bufpos + j, bufpos + j + 2, TDictValue.CLOSE)
elif next_char == b"[":
buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
elif next_char == b"]":
buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE)
else:
j = m.start(0)
next_char = buf[j : j + 1]
if next_char == b"%":
self.buffer.seek(bufpos + j + 1)
yield parse_comment(self.buffer)
elif next_char == b"/":
self.buffer.seek(bufpos + j + 1)
yield parse_name(self.buffer)
elif next_char in b"-+":
self.buffer.seek(bufpos + j + 1)
yield parse_number(self.buffer, next_char)
elif next_char.isdigit() or next_char == b".":
self.buffer.seek(bufpos + j)
yield parse_number(self.buffer, initialpos=self.buffer.tell())
elif next_char.isalpha():
self.buffer.seek(bufpos + j)
yield parse_keyword(self.buffer, self.buffer.tell())
elif next_char == b"(":
self.buffer.seek(bufpos + j + 1)
yield parse_string(self.buffer)
elif next_char == b"<":
self.buffer.seek(bufpos + j + 1)
yield parse_double_angled_bracket(self.buffer)
elif next_char == b">":
self.buffer.seek(bufpos + j + 1)
if buf[j : j + 2] == b">>":
self.buffer.seekd(1)
yield TokenDictionary(
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
)
elif next_char == b"[":
self.buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
elif next_char == b"]":
self.buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE)
else:
self.buffer.seek(bufpos + j + 1)
yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char)
buffer.seek(bufpos + j + 1)
yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char)
+22 -9
View File
@@ -4,7 +4,7 @@ from decimal import Decimal
from typing import List
from unittest import TestCase
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer import iter_tokens
from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
from pdfmajor.lexer.string import parse_string
@@ -222,14 +222,27 @@ class Basics(TestCase):
class Lexer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PDFLexer.from_bytes(raw)
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1):
self.assertEqual(token, etoken, f"Failed at example #{i}")
try:
next_token = next(tokenizer.iter_tokens())
self.assertIsNone(next_token)
except (EOFError, StopIteration) as e:
self.assertIsNotNone(e)
size = len(raw)
for buf_size in [
2,
3,
size // 4,
size // 3,
size // 2,
3 * size // 4,
size,
2 * size,
4 * size,
]:
with self.subTest(raw=raw, buf_size=buf_size):
it = iter_tokens(make_stream_iter(raw, buf_size))
for i, (etoken, token) in enumerate(zip(expected, it)):
self.assertEqual(token, etoken, f"Failed at example #{i}")
try:
next_token = next(it)
self.assertIsNone(next_token)
except (EOFError, StopIteration) as e:
self.assertIsNotNone(e)
def test_scase1(self):
self.run_test(