diff --git a/pdfmajor/lexer/__init__.py b/pdfmajor/lexer/__init__.py index f3808cb..3b28586 100644 --- a/pdfmajor/lexer/__init__.py +++ b/pdfmajor/lexer/__init__.py @@ -1,4 +1,3 @@ -import io from typing import Iterator from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket from pdfmajor.lexer.string import parse_string @@ -18,80 +17,53 @@ from pdfmajor.lexer.token import ( from pdfmajor.streambuffer import BufferStream -class PDFLexer: - def __init__(self, buffer: BufferStream) -> None: - self.buffer = buffer +def iter_tokens(buffer: BufferStream) -> Iterator[Token]: + """Iterates over the tokens in the stream - @classmethod - def from_iobuffer(cls, fp: io.BufferedIOBase, buffer_size: int = 4096): - return cls(BufferStream(fp, buffer_size=buffer_size)) - - @classmethod - def from_bytes(cls, data: bytes, buffer_size: int = 4096): - return cls(BufferStream(io.BytesIO(data), buffer_size=buffer_size)) - - def tell(self) -> int: - return self.buffer.tell() - - def seek(self, pos: int) -> int: - """moves the offset of the reader to a certain spot, and returns the new offset - - Args: - offset (int) - - Returns: - int: new offset - """ - return self.buffer.seek(pos) - - def iter_tokens(self) -> Iterator[Token]: - """Iterates over the tokens in the stream - - Yields: - Iterator[Token] - """ - for bufpos, buf in self.buffer: - m = NONSPC.search(buf, 0) - if not m: - self.buf_skipstep = None + Yields: + Iterator[Token] + """ + for bufpos, buf in buffer: + m = NONSPC.search(buf, 0) + if not m: + continue + else: + j = m.start(0) + next_char = buf[j : j + 1] + if next_char == b"~": continue + elif next_char == b"%": + buffer.seek(bufpos + j + 1) + yield parse_comment(buffer) + elif next_char == b"/": + buffer.seek(bufpos + j + 1) + yield parse_name(buffer) + elif next_char in b"-+": + buffer.seek(bufpos + j + 1) + yield parse_number(buffer, next_char) + elif next_char.isdigit() or next_char == b".": + buffer.seek(bufpos + j) + yield parse_number(buffer, initialpos=buffer.tell()) + elif next_char.isalpha(): + buffer.seek(bufpos + j) + yield parse_keyword(buffer, buffer.tell()) + elif next_char == b"(": + buffer.seek(bufpos + j + 1) + yield parse_string(buffer) + elif next_char == b"<": + buffer.seek(bufpos + j + 1) + yield parse_double_angled_bracket(buffer) + elif next_char == b">": + buffer.seek(bufpos + j + 1) + if buf[j : j + 2] == b">>": + buffer.seekd(1) + yield TokenDictionary(bufpos + j, bufpos + j + 2, TDictValue.CLOSE) + elif next_char == b"[": + buffer.seek(bufpos + j + 1) + yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN) + elif next_char == b"]": + buffer.seek(bufpos + j + 1) + yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE) else: - j = m.start(0) - next_char = buf[j : j + 1] - if next_char == b"%": - self.buffer.seek(bufpos + j + 1) - yield parse_comment(self.buffer) - elif next_char == b"/": - self.buffer.seek(bufpos + j + 1) - yield parse_name(self.buffer) - elif next_char in b"-+": - self.buffer.seek(bufpos + j + 1) - yield parse_number(self.buffer, next_char) - elif next_char.isdigit() or next_char == b".": - self.buffer.seek(bufpos + j) - yield parse_number(self.buffer, initialpos=self.buffer.tell()) - elif next_char.isalpha(): - self.buffer.seek(bufpos + j) - yield parse_keyword(self.buffer, self.buffer.tell()) - elif next_char == b"(": - self.buffer.seek(bufpos + j + 1) - yield parse_string(self.buffer) - elif next_char == b"<": - self.buffer.seek(bufpos + j + 1) - yield parse_double_angled_bracket(self.buffer) - elif next_char == b">": - self.buffer.seek(bufpos + j + 1) - if buf[j : j + 2] == b">>": - self.buffer.seekd(1) - yield TokenDictionary( - bufpos + j, bufpos + j + 2, TDictValue.CLOSE - ) - elif next_char == b"[": - self.buffer.seek(bufpos + j + 1) - yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN) - elif next_char == b"]": - self.buffer.seek(bufpos + j + 1) - yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE) - else: - self.buffer.seek(bufpos + j + 1) - yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char) + buffer.seek(bufpos + j + 1) + yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char) diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 44ee271..5f39664 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -4,7 +4,7 @@ from decimal import Decimal from typing import List from unittest import TestCase -from pdfmajor.lexer import PDFLexer +from pdfmajor.lexer import iter_tokens from pdfmajor.lexer.number import parse_number from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring from pdfmajor.lexer.string import parse_string @@ -222,14 +222,27 @@ class Basics(TestCase): class Lexer(TestCase): def run_test(self, raw: bytes, expected: List[Token]): - tokenizer = PDFLexer.from_bytes(raw) - for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1): - self.assertEqual(token, etoken, f"Failed at example #{i}") - try: - next_token = next(tokenizer.iter_tokens()) - self.assertIsNone(next_token) - except (EOFError, StopIteration) as e: - self.assertIsNotNone(e) + size = len(raw) + for buf_size in [ + 2, + 3, + size // 4, + size // 3, + size // 2, + 3 * size // 4, + size, + 2 * size, + 4 * size, + ]: + with self.subTest(raw=raw, buf_size=buf_size): + it = iter_tokens(make_stream_iter(raw, buf_size)) + for i, (etoken, token) in enumerate(zip(expected, it)): + self.assertEqual(token, etoken, f"Failed at example #{i}") + try: + next_token = next(it) + self.assertIsNone(next_token) + except (EOFError, StopIteration) as e: + self.assertIsNotNone(e) def test_scase1(self): self.run_test(