From faaf7925bffb1caf1fea63f147aee3c629eb5a33 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Fri, 16 Oct 2020 16:10:10 -0400 Subject: [PATCH] using new approach for lexing --- pdfmajor/lexer/__init__.py | 9 +++ pdfmajor/lexer/comment.py | 29 +++++++++ pdfmajor/lexer/exceptions.py | 16 +++++ pdfmajor/lexer/keyword.py | 33 ++++++++++ pdfmajor/lexer/name.py | 67 ++++++++++++++++++++ pdfmajor/lexer/regex.py | 6 ++ pdfmajor/lexer/token.py | 116 ++++++++++++++++++++++++++++++++++ pdfmajor/safebufiterator.py | 73 +++++++++++++++++++++ tests/test_lexer.py | 94 +++++++++++++++++++++++++++ tests/test_safebufiterator.py | 39 ++++++++++++ 10 files changed, 482 insertions(+) create mode 100644 pdfmajor/lexer/__init__.py create mode 100644 pdfmajor/lexer/comment.py create mode 100644 pdfmajor/lexer/exceptions.py create mode 100644 pdfmajor/lexer/keyword.py create mode 100644 pdfmajor/lexer/name.py create mode 100644 pdfmajor/lexer/regex.py create mode 100644 pdfmajor/lexer/token.py create mode 100644 pdfmajor/safebufiterator.py create mode 100644 tests/test_lexer.py create mode 100644 tests/test_safebufiterator.py diff --git a/pdfmajor/lexer/__init__.py b/pdfmajor/lexer/__init__.py new file mode 100644 index 0000000..5c27d2a --- /dev/null +++ b/pdfmajor/lexer/__init__.py @@ -0,0 +1,9 @@ +import io + + +class PDFLexer: + def __init__(self, fp: io.BufferedIOBase) -> None: + self.fp = fp + + def seek(self, pos: int): + self.fp.seek(pos) diff --git a/pdfmajor/lexer/comment.py b/pdfmajor/lexer/comment.py new file mode 100644 index 0000000..03ea99d --- /dev/null +++ b/pdfmajor/lexer/comment.py @@ -0,0 +1,29 @@ +from pdfmajor.lexer.exceptions import LexerEOF +from pdfmajor.lexer.regex import EOL +from pdfmajor.lexer.token import ( + TokenComment, +) +from pdfmajor.safebufiterator import SafeBufferIt + + +def parse_comment(buffer: SafeBufferIt) -> TokenComment: + """Parses input stream into a comment + + Args: + buffer (SafeBufferIt) + + Returns: + TokenComment + """ + curtoken = b"" + initialpos = buffer.tell() - 1 + for pos, buf in buffer: + m = EOL.search(buf, 0) + if not m: + curtoken += buf + else: + j = m.start(0) + curtoken += buf[:j] + buffer.seek(pos + j) + return TokenComment(initialpos, buffer.tell(), curtoken) + raise LexerEOF diff --git a/pdfmajor/lexer/exceptions.py b/pdfmajor/lexer/exceptions.py new file mode 100644 index 0000000..274e1ed --- /dev/null +++ b/pdfmajor/lexer/exceptions.py @@ -0,0 +1,16 @@ +from pdfmajor.execptions import PDFMajorException + + +class LexerError(PDFMajorException): + pass + + +class LexerEOF(LexerError): + pass + + +class InvalidHexToken(LexerError): + def __init__(self, pos: int, token: bytes) -> None: + super().__init__( + "Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token)) + ) diff --git a/pdfmajor/lexer/keyword.py b/pdfmajor/lexer/keyword.py new file mode 100644 index 0000000..514524e --- /dev/null +++ b/pdfmajor/lexer/keyword.py @@ -0,0 +1,33 @@ +from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.lexer.exceptions import LexerEOF +from pdfmajor.lexer.token import TokenBoolean, TokenKeyword +from pdfmajor.lexer.regex import END_KEYWORD +from typing import Union + + +def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]: + """Parses input stream into a keyword or bool + + Args: + buffer (SafeBufferIt) + + Returns: + Union[TokenKeyword, TokenBoolean] + """ + curtoken = b"" + initialpos = buffer.tell() - 1 + for pos, buf in buffer: + m = END_KEYWORD.search(buf, 0) + if not m: + curtoken += buf + else: + j = m.start(0) + curtoken += buf[:j] + buffer.seek(pos + j) + if curtoken == b"true": + return TokenBoolean(initialpos, buffer.tell(), True) + elif curtoken == b"false": + return TokenBoolean(initialpos, buffer.tell(), False) + else: + return TokenKeyword(initialpos, buffer.tell(), curtoken) + raise LexerEOF diff --git a/pdfmajor/lexer/name.py b/pdfmajor/lexer/name.py new file mode 100644 index 0000000..f9afb7a --- /dev/null +++ b/pdfmajor/lexer/name.py @@ -0,0 +1,67 @@ +from dataclasses import dataclass +from pdfmajor.lexer.exceptions import ( + InvalidHexToken, + LexerEOF, + LexerError, +) + +from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.lexer.token import TokenName +from pdfmajor.lexer.regex import HEX, END_LITERAL + + +from typing import Optional + + +@dataclass +class LiteralParseState: + curtoken: bytes + hex_value: Optional[bytes] = None + + +def parse_name(buffer: SafeBufferIt) -> TokenName: + """Parses input stream into a literal name + + Args: + buffer (SafeBufferIt) + + Returns: + TokenName + """ + initialpos = buffer.tell() - 1 + state = LiteralParseState(b"") + for pos, buf in buffer: + if state.hex_value is not None: + for ci in range(len(buf)): + c = buf[ci : ci + 1] + if state.hex_value is None: + raise LexerError("state.hex_value became None") + if HEX.match(c) and len(state.hex_value) < 2: + state.hex_value += c + buffer.seek(pos + ci + 1) + if len(state.hex_value) >= 2: # type: ignore + state.curtoken += bytes([int(state.hex_value, 16)]) # type: ignore + state.hex_value = None + break + else: + raise InvalidHexToken(buffer.tell() - buffer.buffer_size, c) + else: + m = END_LITERAL.search(buf, 0) + if not m: + state.curtoken += buf + else: + j: int = m.start(0) + state.curtoken += buf[:j] + next_char = buf[j : j + 1] + if next_char == b"#": + state.hex_value = b"" + buffer.seek(pos + j + 1) + continue + else: + buffer.seek(pos + j) + return TokenName( + initialpos, + buffer.tell(), + state.curtoken.decode(), + ) + raise LexerEOF diff --git a/pdfmajor/lexer/regex.py b/pdfmajor/lexer/regex.py new file mode 100644 index 0000000..6a59169 --- /dev/null +++ b/pdfmajor/lexer/regex.py @@ -0,0 +1,6 @@ +import re + +EOL = re.compile(br"[\r\n]") +END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]") +HEX = re.compile(br"[0-9a-fA-F]") +END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]") diff --git a/pdfmajor/lexer/token.py b/pdfmajor/lexer/token.py new file mode 100644 index 0000000..aa88a71 --- /dev/null +++ b/pdfmajor/lexer/token.py @@ -0,0 +1,116 @@ +from enum import Enum +from typing import Any, Generic, TypeVar, Union +from decimal import Decimal + +T = TypeVar("T") + + +class Token(Generic[T]): + """A generic class representing PDF Token""" + + __slots__ = ["pos", "size", "value"] + + def __init__(self, start_loc: int, end_loc: int, value: T) -> None: + self.start_loc = start_loc + self.end_loc = end_loc + self.value = value + + def __repr__(self) -> str: + return ( + "{cls}(start_loc={start_loc}, end_loc={end_loc}, value='{value}')".format( + cls=self.__class__.__name__, + start_loc=self.start_loc, + value=self.value, + end_loc=self.end_loc, + ) + ) + + def __eq__(self, other: Any): + if isinstance(other, self.__class__): + return ( + (other.value == self.value) + and (other.start_loc == self.start_loc) + and (other.end_loc == self.end_loc) + ) + else: + return False + + +class TokenComment(Token[bytes]): + """Token representing PDF comments + Detection of this type of token can be found in PDF 1.7 spec section 7.2.3 + """ + + pass + + +class TokenKeyword(Token[bytes]): + """Token representing PDF comments + Detection of this type of token can be found in PDF 1.7 spec + """ + + pass + + +class TokenName(Token[str]): + """Token representing literal name tokens + Detection of this type of token can be found in PDF 1.7 spec section 7.3.5 + """ + + pass + + +class TokenBoolean(Token[bool]): + """Token representing PDF booleans + Detection of this type of token can be found in PDF 1.7 spec section 7.3.2 + """ + + pass + + +class TokenString(Token[str]): + """Token representing PDF literal string + Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2 + """ + + pass + + +class TokenHexString(Token[bytes]): + """Token representing PDF hex strings + Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3 + """ + + pass + + +class TokenInteger(Token[int]): + """Token representing PDF integers + Detection of this type of token can be found in PDF 1.7 + """ + + pass + + +class TokenDecimal(Token[Decimal]): + """Token representing PDF decimals + Detection of this type of token can be found in PDF 1.7 + """ + + pass + + +class TDictVaue(Enum): + OPEN = "<<" + CLOSE = ">>" + + +class TokenDictionary(Token[TDictVaue]): + """Token representing the brackets of dictionary + Detection of this type of token can be found in PDF 1.7 spec section 7.3.7 + """ + + pass + + +TokenNumber = Union[TokenInteger, TokenDecimal] diff --git a/pdfmajor/safebufiterator.py b/pdfmajor/safebufiterator.py new file mode 100644 index 0000000..a6d1686 --- /dev/null +++ b/pdfmajor/safebufiterator.py @@ -0,0 +1,73 @@ +import io +from typing import NamedTuple +from pdfmajor.execptions import PDFMajorException + + +class EOF(PDFMajorException): + def __init__(self) -> None: + super().__init__("reached the end of the file or buffer stream!") + + +class BufferedBytes(NamedTuple): + pos: int + data: bytes + + +class SafeBufferIt: + """A safe reader for bytes that has a few nice additional utilities""" + + __slots__ = ["fp", "buffer_size"] + + def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None: + self.fp = fp + self.buffer_size = buffer_size + + def seekd(self, step: int) -> int: + """similar to self.fp.seek but you move the offset relative to where you are now + + Args: + step (int): number of steps to move + """ + cur_pos = self.tell() + return self.seek(max(cur_pos + step, 0)) + + def seek(self, offset: int) -> int: + """moves the offset of the reader to a certain spot, and returns the new offset + + Args: + offset (int) + + Returns: + int: new offset + """ + return self.fp.seek(offset) + + def tell(self) -> int: + """returns the current offset + + Returns: + int: current offset + """ + return self.fp.tell() + + def __iter__(self): + if self.fp.read(1) == b"": + raise EOF + self.seekd(-1) + return self + + def __next__(self): + pos = self.tell() + buf = self.fp.read(self.buffer_size) + if len(buf) == 0: + raise StopIteration + return BufferedBytes(pos, buf) + + def close(self): + self.fp.close() + + def __del__(self): + self.close() + + def __exit__(self): + self.close() diff --git a/tests/test_lexer.py b/tests/test_lexer.py new file mode 100644 index 0000000..3f0d248 --- /dev/null +++ b/tests/test_lexer.py @@ -0,0 +1,94 @@ +import io +from unittest import TestCase + +from pdfmajor.lexer.name import parse_name +from pdfmajor.lexer.keyword import parse_keyword +from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName +from pdfmajor.lexer.comment import parse_comment + + +def make_stream_iter(data: bytes, buf_size: int): + return SafeBufferIt(io.BytesIO(data), buf_size) + + +class Basics(TestCase): + def test_comment_parser_ln(self): + for buf_size in [2, 7, 11, 150]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter( + b"this is a lengthy comment that ends here\nso this is not reachable", + buf_size, + ) + token = parse_comment(buffer) + self.assertIsInstance(token, TokenComment) + self.assertEqual(token.start_loc, -1) + self.assertEqual( + token.value, b"this is a lengthy comment that ends here" + ) + self.assertEqual(token.end_loc, 40) + + def test_comment_parser_lr(self): + for buf_size in [2, 7, 11, 150]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter( + b"this is a lengthy comment that ends here\rso this is not reachable", + buf_size, + ) + token = parse_comment(buffer) + self.assertIsInstance(token, TokenComment) + self.assertEqual(token.start_loc, -1) + self.assertEqual( + token.value, b"this is a lengthy comment that ends here" + ) + self.assertEqual(token.end_loc, 40) + + def test_parse_bool_true(self): + for buf_size in [2, 3, 50]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(b"true and the american way", buf_size) + token = parse_keyword(buffer) + self.assertIsInstance(token, TokenBoolean) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, True) + self.assertEqual(token.end_loc, 4) + + def test_parse_bool_false(self): + for buf_size in [2, 3, 50]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(b"false mannn", buf_size) + token = parse_keyword(buffer) + self.assertIsInstance(token, TokenBoolean) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, False) + self.assertEqual(token.end_loc, 5) + + def test_parse_literal_names(self): + for buf_size in [2, 3, 50, 444]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(br"Some_Name /foo#5f#xbaa", buf_size) + token = parse_name(buffer) + self.assertIsInstance(token, TokenName) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, "Some_Name") + self.assertEqual(token.end_loc, 9) + + def test_parse_literal_names_hex(self): + for buf_size in [50, 1000]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(br"foo#5faa ", buf_size) + token = parse_name(buffer) + self.assertIsInstance(token, TokenName) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, "foo_aa") + self.assertEqual(token.end_loc, 8) + + # def test_parse_string(self): + # for buf_size in [2, 3, 50, 1000]: + # with self.subTest(buf_end_loc=buf_size): + # iter_stream = make_stream_iter(br"this % is not a comment.) ", 88) + # token = parse_string(88, iter_stream(buf_size)) + # self.assertIsInstance(token, TokenString) + # self.assertEqual(token.start_loc, 88) + # self.assertEqual(token.value, "this % is not a comment.") + # self.assertEqual(token.end_loc, 88 + 25) diff --git a/tests/test_safebufiterator.py b/tests/test_safebufiterator.py new file mode 100644 index 0000000..a75459e --- /dev/null +++ b/tests/test_safebufiterator.py @@ -0,0 +1,39 @@ +import io +from unittest import TestCase +from pdfmajor.safebufiterator import EOF, SafeBufferIt + + +class UseCase(TestCase): + def test_safebuff(self): + for bufsize in [2, 33, 65, 66, 100, 200]: + it = SafeBufferIt( + io.BytesIO( + b"this is a lengthy comment that ends here\nso this is not reachable" + ), + buffer_size=bufsize, + ) + for subbuf in it: + self.assertLessEqual(len(subbuf), it.buffer_size) + + def test_itraises(self): + it = SafeBufferIt(io.BytesIO(b"this")) + list(it) # read everything + with self.assertRaises(EOF): + for _ in it: + pass + + def test_seekd(self): + for bufsize in [2, 33, 65, 66, 100, 200]: + it = SafeBufferIt( + io.BytesIO( + b"this is a lengthy comment that ends here\nso this is not reachable" + ), + buffer_size=bufsize, + ) + last_buf = next(it) + it.seekd(-it.buffer_size) + for i, buf in enumerate(it): + self.assertEqual(last_buf, buf) + it.seekd(-it.buffer_size) + if i > 4: + break