using new approach for lexing

This commit is contained in:
ari committed 2020-10-16 16:10:10 -04:00
1 parent a2d2b7808d
commit faaf7925bf
10 files changed
+482

No files matched your search

+9
View File
@@ -0,0 +1,9 @@
import io
class PDFLexer:
def __init__(self, fp: io.BufferedIOBase) -> None:
self.fp = fp
def seek(self, pos: int):
self.fp.seek(pos)
+29
View File
@@ -0,0 +1,29 @@
from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.regex import EOL
from pdfmajor.lexer.token import (
TokenComment,
)
from pdfmajor.safebufiterator import SafeBufferIt
def parse_comment(buffer: SafeBufferIt) -> TokenComment:
"""Parses input stream into a comment
Args:
buffer (SafeBufferIt)
Returns:
TokenComment
"""
curtoken = b""
initialpos = buffer.tell() - 1
for pos, buf in buffer:
m = EOL.search(buf, 0)
if not m:
curtoken += buf
else:
j = m.start(0)
curtoken += buf[:j]
buffer.seek(pos + j)
return TokenComment(initialpos, buffer.tell(), curtoken)
raise LexerEOF
+16
View File
@@ -0,0 +1,16 @@
from pdfmajor.execptions import PDFMajorException
class LexerError(PDFMajorException):
pass
class LexerEOF(LexerError):
pass
class InvalidHexToken(LexerError):
def __init__(self, pos: int, token: bytes) -> None:
super().__init__(
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
)
+33
View File
@@ -0,0 +1,33 @@
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword
from pdfmajor.lexer.regex import END_KEYWORD
from typing import Union
def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]:
"""Parses input stream into a keyword or bool
Args:
buffer (SafeBufferIt)
Returns:
Union[TokenKeyword, TokenBoolean]
"""
curtoken = b""
initialpos = buffer.tell() - 1
for pos, buf in buffer:
m = END_KEYWORD.search(buf, 0)
if not m:
curtoken += buf
else:
j = m.start(0)
curtoken += buf[:j]
buffer.seek(pos + j)
if curtoken == b"true":
return TokenBoolean(initialpos, buffer.tell(), True)
elif curtoken == b"false":
return TokenBoolean(initialpos, buffer.tell(), False)
else:
return TokenKeyword(initialpos, buffer.tell(), curtoken)
raise LexerEOF
+67
View File
@@ -0,0 +1,67 @@
from dataclasses import dataclass
from pdfmajor.lexer.exceptions import (
InvalidHexToken,
LexerEOF,
LexerError,
)
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.lexer.token import TokenName
from pdfmajor.lexer.regex import HEX, END_LITERAL
from typing import Optional
@dataclass
class LiteralParseState:
curtoken: bytes
hex_value: Optional[bytes] = None
def parse_name(buffer: SafeBufferIt) -> TokenName:
"""Parses input stream into a literal name
Args:
buffer (SafeBufferIt)
Returns:
TokenName
"""
initialpos = buffer.tell() - 1
state = LiteralParseState(b"")
for pos, buf in buffer:
if state.hex_value is not None:
for ci in range(len(buf)):
c = buf[ci : ci + 1]
if state.hex_value is None:
raise LexerError("state.hex_value became None")
if HEX.match(c) and len(state.hex_value) < 2:
state.hex_value += c
buffer.seek(pos + ci + 1)
if len(state.hex_value) >= 2: # type: ignore
state.curtoken += bytes([int(state.hex_value, 16)]) # type: ignore
state.hex_value = None
break
else:
raise InvalidHexToken(buffer.tell() - buffer.buffer_size, c)
else:
m = END_LITERAL.search(buf, 0)
if not m:
state.curtoken += buf
else:
j: int = m.start(0)
state.curtoken += buf[:j]
next_char = buf[j : j + 1]
if next_char == b"#":
state.hex_value = b""
buffer.seek(pos + j + 1)
continue
else:
buffer.seek(pos + j)
return TokenName(
initialpos,
buffer.tell(),
state.curtoken.decode(),
)
raise LexerEOF
+6
View File
@@ -0,0 +1,6 @@
import re
EOL = re.compile(br"[\r\n]")
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
HEX = re.compile(br"[0-9a-fA-F]")
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
+116
View File
@@ -0,0 +1,116 @@
from enum import Enum
from typing import Any, Generic, TypeVar, Union
from decimal import Decimal
T = TypeVar("T")
class Token(Generic[T]):
"""A generic class representing PDF Token"""
__slots__ = ["pos", "size", "value"]
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
self.start_loc = start_loc
self.end_loc = end_loc
self.value = value
def __repr__(self) -> str:
return (
"{cls}(start_loc={start_loc}, end_loc={end_loc}, value='{value}')".format(
cls=self.__class__.__name__,
start_loc=self.start_loc,
value=self.value,
end_loc=self.end_loc,
)
)
def __eq__(self, other: Any):
if isinstance(other, self.__class__):
return (
(other.value == self.value)
and (other.start_loc == self.start_loc)
and (other.end_loc == self.end_loc)
)
else:
return False
class TokenComment(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
"""
pass
class TokenKeyword(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec
"""
pass
class TokenName(Token[str]):
"""Token representing literal name tokens
Detection of this type of token can be found in PDF 1.7 spec section 7.3.5
"""
pass
class TokenBoolean(Token[bool]):
"""Token representing PDF booleans
Detection of this type of token can be found in PDF 1.7 spec section 7.3.2
"""
pass
class TokenString(Token[str]):
"""Token representing PDF literal string
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2
"""
pass
class TokenHexString(Token[bytes]):
"""Token representing PDF hex strings
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3
"""
pass
class TokenInteger(Token[int]):
"""Token representing PDF integers
Detection of this type of token can be found in PDF 1.7
"""
pass
class TokenDecimal(Token[Decimal]):
"""Token representing PDF decimals
Detection of this type of token can be found in PDF 1.7
"""
pass
class TDictVaue(Enum):
OPEN = "<<"
CLOSE = ">>"
class TokenDictionary(Token[TDictVaue]):
"""Token representing the brackets of dictionary
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
"""
pass
TokenNumber = Union[TokenInteger, TokenDecimal]
+73
View File
@@ -0,0 +1,73 @@
import io
from typing import NamedTuple
from pdfmajor.execptions import PDFMajorException
class EOF(PDFMajorException):
def __init__(self) -> None:
super().__init__("reached the end of the file or buffer stream!")
class BufferedBytes(NamedTuple):
pos: int
data: bytes
class SafeBufferIt:
"""A safe reader for bytes that has a few nice additional utilities"""
__slots__ = ["fp", "buffer_size"]
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
self.fp = fp
self.buffer_size = buffer_size
def seekd(self, step: int) -> int:
"""similar to self.fp.seek but you move the offset relative to where you are now
Args:
step (int): number of steps to move
"""
cur_pos = self.tell()
return self.seek(max(cur_pos + step, 0))
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.fp.seek(offset)
def tell(self) -> int:
"""returns the current offset
Returns:
int: current offset
"""
return self.fp.tell()
def __iter__(self):
if self.fp.read(1) == b"":
raise EOF
self.seekd(-1)
return self
def __next__(self):
pos = self.tell()
buf = self.fp.read(self.buffer_size)
if len(buf) == 0:
raise StopIteration
return BufferedBytes(pos, buf)
def close(self):
self.fp.close()
def __del__(self):
self.close()
def __exit__(self):
self.close()
+94
View File
@@ -0,0 +1,94 @@
import io
from unittest import TestCase
from pdfmajor.lexer.name import parse_name
from pdfmajor.lexer.keyword import parse_keyword
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName
from pdfmajor.lexer.comment import parse_comment
def make_stream_iter(data: bytes, buf_size: int):
return SafeBufferIt(io.BytesIO(data), buf_size)
class Basics(TestCase):
def test_comment_parser_ln(self):
for buf_size in [2, 7, 11, 150]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(
b"this is a lengthy comment that ends here\nso this is not reachable",
buf_size,
)
token = parse_comment(buffer)
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.start_loc, -1)
self.assertEqual(
token.value, b"this is a lengthy comment that ends here"
)
self.assertEqual(token.end_loc, 40)
def test_comment_parser_lr(self):
for buf_size in [2, 7, 11, 150]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(
b"this is a lengthy comment that ends here\rso this is not reachable",
buf_size,
)
token = parse_comment(buffer)
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.start_loc, -1)
self.assertEqual(
token.value, b"this is a lengthy comment that ends here"
)
self.assertEqual(token.end_loc, 40)
def test_parse_bool_true(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(b"true and the american way", buf_size)
token = parse_keyword(buffer)
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, True)
self.assertEqual(token.end_loc, 4)
def test_parse_bool_false(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(b"false mannn", buf_size)
token = parse_keyword(buffer)
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, False)
self.assertEqual(token.end_loc, 5)
def test_parse_literal_names(self):
for buf_size in [2, 3, 50, 444]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(br"Some_Name /foo#5f#xbaa", buf_size)
token = parse_name(buffer)
self.assertIsInstance(token, TokenName)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, "Some_Name")
self.assertEqual(token.end_loc, 9)
def test_parse_literal_names_hex(self):
for buf_size in [50, 1000]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(br"foo#5faa ", buf_size)
token = parse_name(buffer)
self.assertIsInstance(token, TokenName)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, "foo_aa")
self.assertEqual(token.end_loc, 8)
# def test_parse_string(self):
# for buf_size in [2, 3, 50, 1000]:
# with self.subTest(buf_end_loc=buf_size):
# iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
# token = parse_string(88, iter_stream(buf_size))
# self.assertIsInstance(token, TokenString)
# self.assertEqual(token.start_loc, 88)
# self.assertEqual(token.value, "this % is not a comment.")
# self.assertEqual(token.end_loc, 88 + 25)
+39
View File
@@ -0,0 +1,39 @@
import io
from unittest import TestCase
from pdfmajor.safebufiterator import EOF, SafeBufferIt
class UseCase(TestCase):
def test_safebuff(self):
for bufsize in [2, 33, 65, 66, 100, 200]:
it = SafeBufferIt(
io.BytesIO(
b"this is a lengthy comment that ends here\nso this is not reachable"
),
buffer_size=bufsize,
)
for subbuf in it:
self.assertLessEqual(len(subbuf), it.buffer_size)
def test_itraises(self):
it = SafeBufferIt(io.BytesIO(b"this"))
list(it) # read everything
with self.assertRaises(EOF):
for _ in it:
pass
def test_seekd(self):
for bufsize in [2, 33, 65, 66, 100, 200]:
it = SafeBufferIt(
io.BytesIO(
b"this is a lengthy comment that ends here\nso this is not reachable"
),
buffer_size=bufsize,
)
last_buf = next(it)
it.seekd(-it.buffer_size)
for i, buf in enumerate(it):
self.assertEqual(last_buf, buf)
it.seekd(-it.buffer_size)
if i > 4:
break