using a more functional approach for lexer
This commit is contained in:
1 parent
e2a92517cd
commit
087c1c2fe9
2 files changed
+69
-84
No files matched your search
+47
-75
@@ -1,4 +1,3 @@
|
|||||||
import io
|
|
||||||
from typing import Iterator
|
from typing import Iterator
|
||||||
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket
|
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket
|
||||||
from pdfmajor.lexer.string import parse_string
|
from pdfmajor.lexer.string import parse_string
|
||||||
@@ -18,80 +17,53 @@ from pdfmajor.lexer.token import (
|
|||||||
from pdfmajor.streambuffer import BufferStream
|
from pdfmajor.streambuffer import BufferStream
|
||||||
|
|
||||||
|
|
||||||
class PDFLexer:
|
def iter_tokens(buffer: BufferStream) -> Iterator[Token]:
|
||||||
def __init__(self, buffer: BufferStream) -> None:
|
"""Iterates over the tokens in the stream
|
||||||
self.buffer = buffer
|
|
||||||
|
|
||||||
@classmethod
|
Yields:
|
||||||
def from_iobuffer(cls, fp: io.BufferedIOBase, buffer_size: int = 4096):
|
Iterator[Token]
|
||||||
return cls(BufferStream(fp, buffer_size=buffer_size))
|
"""
|
||||||
|
for bufpos, buf in buffer:
|
||||||
@classmethod
|
m = NONSPC.search(buf, 0)
|
||||||
def from_bytes(cls, data: bytes, buffer_size: int = 4096):
|
if not m:
|
||||||
return cls(BufferStream(io.BytesIO(data), buffer_size=buffer_size))
|
continue
|
||||||
|
else:
|
||||||
def tell(self) -> int:
|
j = m.start(0)
|
||||||
return self.buffer.tell()
|
next_char = buf[j : j + 1]
|
||||||
|
if next_char == b"~":
|
||||||
def seek(self, pos: int) -> int:
|
|
||||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
|
||||||
|
|
||||||
Args:
|
|
||||||
offset (int)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
int: new offset
|
|
||||||
"""
|
|
||||||
return self.buffer.seek(pos)
|
|
||||||
|
|
||||||
def iter_tokens(self) -> Iterator[Token]:
|
|
||||||
"""Iterates over the tokens in the stream
|
|
||||||
|
|
||||||
Yields:
|
|
||||||
Iterator[Token]
|
|
||||||
"""
|
|
||||||
for bufpos, buf in self.buffer:
|
|
||||||
m = NONSPC.search(buf, 0)
|
|
||||||
if not m:
|
|
||||||
self.buf_skipstep = None
|
|
||||||
continue
|
continue
|
||||||
|
elif next_char == b"%":
|
||||||
|
buffer.seek(bufpos + j + 1)
|
||||||
|
yield parse_comment(buffer)
|
||||||
|
elif next_char == b"/":
|
||||||
|
buffer.seek(bufpos + j + 1)
|
||||||
|
yield parse_name(buffer)
|
||||||
|
elif next_char in b"-+":
|
||||||
|
buffer.seek(bufpos + j + 1)
|
||||||
|
yield parse_number(buffer, next_char)
|
||||||
|
elif next_char.isdigit() or next_char == b".":
|
||||||
|
buffer.seek(bufpos + j)
|
||||||
|
yield parse_number(buffer, initialpos=buffer.tell())
|
||||||
|
elif next_char.isalpha():
|
||||||
|
buffer.seek(bufpos + j)
|
||||||
|
yield parse_keyword(buffer, buffer.tell())
|
||||||
|
elif next_char == b"(":
|
||||||
|
buffer.seek(bufpos + j + 1)
|
||||||
|
yield parse_string(buffer)
|
||||||
|
elif next_char == b"<":
|
||||||
|
buffer.seek(bufpos + j + 1)
|
||||||
|
yield parse_double_angled_bracket(buffer)
|
||||||
|
elif next_char == b">":
|
||||||
|
buffer.seek(bufpos + j + 1)
|
||||||
|
if buf[j : j + 2] == b">>":
|
||||||
|
buffer.seekd(1)
|
||||||
|
yield TokenDictionary(bufpos + j, bufpos + j + 2, TDictValue.CLOSE)
|
||||||
|
elif next_char == b"[":
|
||||||
|
buffer.seek(bufpos + j + 1)
|
||||||
|
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
|
||||||
|
elif next_char == b"]":
|
||||||
|
buffer.seek(bufpos + j + 1)
|
||||||
|
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE)
|
||||||
else:
|
else:
|
||||||
j = m.start(0)
|
buffer.seek(bufpos + j + 1)
|
||||||
next_char = buf[j : j + 1]
|
yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char)
|
||||||
if next_char == b"%":
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
yield parse_comment(self.buffer)
|
|
||||||
elif next_char == b"/":
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
yield parse_name(self.buffer)
|
|
||||||
elif next_char in b"-+":
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
yield parse_number(self.buffer, next_char)
|
|
||||||
elif next_char.isdigit() or next_char == b".":
|
|
||||||
self.buffer.seek(bufpos + j)
|
|
||||||
yield parse_number(self.buffer, initialpos=self.buffer.tell())
|
|
||||||
elif next_char.isalpha():
|
|
||||||
self.buffer.seek(bufpos + j)
|
|
||||||
yield parse_keyword(self.buffer, self.buffer.tell())
|
|
||||||
elif next_char == b"(":
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
yield parse_string(self.buffer)
|
|
||||||
elif next_char == b"<":
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
yield parse_double_angled_bracket(self.buffer)
|
|
||||||
elif next_char == b">":
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
if buf[j : j + 2] == b">>":
|
|
||||||
self.buffer.seekd(1)
|
|
||||||
yield TokenDictionary(
|
|
||||||
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
|
|
||||||
)
|
|
||||||
elif next_char == b"[":
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
|
|
||||||
elif next_char == b"]":
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE)
|
|
||||||
else:
|
|
||||||
self.buffer.seek(bufpos + j + 1)
|
|
||||||
yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char)
|
|
||||||
+22
-9
@@ -4,7 +4,7 @@ from decimal import Decimal
|
|||||||
from typing import List
|
from typing import List
|
||||||
from unittest import TestCase
|
from unittest import TestCase
|
||||||
|
|
||||||
from pdfmajor.lexer import PDFLexer
|
from pdfmajor.lexer import iter_tokens
|
||||||
from pdfmajor.lexer.number import parse_number
|
from pdfmajor.lexer.number import parse_number
|
||||||
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
|
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
|
||||||
from pdfmajor.lexer.string import parse_string
|
from pdfmajor.lexer.string import parse_string
|
||||||
@@ -222,14 +222,27 @@ class Basics(TestCase):
|
|||||||
|
|
||||||
class Lexer(TestCase):
|
class Lexer(TestCase):
|
||||||
def run_test(self, raw: bytes, expected: List[Token]):
|
def run_test(self, raw: bytes, expected: List[Token]):
|
||||||
tokenizer = PDFLexer.from_bytes(raw)
|
size = len(raw)
|
||||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1):
|
for buf_size in [
|
||||||
self.assertEqual(token, etoken, f"Failed at example #{i}")
|
2,
|
||||||
try:
|
3,
|
||||||
next_token = next(tokenizer.iter_tokens())
|
size // 4,
|
||||||
self.assertIsNone(next_token)
|
size // 3,
|
||||||
except (EOFError, StopIteration) as e:
|
size // 2,
|
||||||
self.assertIsNotNone(e)
|
3 * size // 4,
|
||||||
|
size,
|
||||||
|
2 * size,
|
||||||
|
4 * size,
|
||||||
|
]:
|
||||||
|
with self.subTest(raw=raw, buf_size=buf_size):
|
||||||
|
it = iter_tokens(make_stream_iter(raw, buf_size))
|
||||||
|
for i, (etoken, token) in enumerate(zip(expected, it)):
|
||||||
|
self.assertEqual(token, etoken, f"Failed at example #{i}")
|
||||||
|
try:
|
||||||
|
next_token = next(it)
|
||||||
|
self.assertIsNone(next_token)
|
||||||
|
except (EOFError, StopIteration) as e:
|
||||||
|
self.assertIsNotNone(e)
|
||||||
|
|
||||||
def test_scase1(self):
|
def test_scase1(self):
|
||||||
self.run_test(
|
self.run_test(
|
||||||
|
|||||||
Reference in new issue
Block a user