using new approach for lexing
This commit is contained in:
1 parent
a2d2b7808d
commit
faaf7925bf
10 files changed
+482
No files matched your search
@@ -0,0 +1,9 @@
|
||||
import io
|
||||
|
||||
|
||||
class PDFLexer:
|
||||
def __init__(self, fp: io.BufferedIOBase) -> None:
|
||||
self.fp = fp
|
||||
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
@@ -0,0 +1,29 @@
|
||||
from pdfmajor.lexer.exceptions import LexerEOF
|
||||
from pdfmajor.lexer.regex import EOL
|
||||
from pdfmajor.lexer.token import (
|
||||
TokenComment,
|
||||
)
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
|
||||
|
||||
def parse_comment(buffer: SafeBufferIt) -> TokenComment:
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
initialpos = buffer.tell() - 1
|
||||
for pos, buf in buffer:
|
||||
m = EOL.search(buf, 0)
|
||||
if not m:
|
||||
curtoken += buf
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += buf[:j]
|
||||
buffer.seek(pos + j)
|
||||
return TokenComment(initialpos, buffer.tell(), curtoken)
|
||||
raise LexerEOF
|
||||
@@ -0,0 +1,16 @@
|
||||
from pdfmajor.execptions import PDFMajorException
|
||||
|
||||
|
||||
class LexerError(PDFMajorException):
|
||||
pass
|
||||
|
||||
|
||||
class LexerEOF(LexerError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidHexToken(LexerError):
|
||||
def __init__(self, pos: int, token: bytes) -> None:
|
||||
super().__init__(
|
||||
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.lexer.exceptions import LexerEOF
|
||||
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword
|
||||
from pdfmajor.lexer.regex import END_KEYWORD
|
||||
from typing import Union
|
||||
|
||||
|
||||
def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]:
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
"""
|
||||
curtoken = b""
|
||||
initialpos = buffer.tell() - 1
|
||||
for pos, buf in buffer:
|
||||
m = END_KEYWORD.search(buf, 0)
|
||||
if not m:
|
||||
curtoken += buf
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += buf[:j]
|
||||
buffer.seek(pos + j)
|
||||
if curtoken == b"true":
|
||||
return TokenBoolean(initialpos, buffer.tell(), True)
|
||||
elif curtoken == b"false":
|
||||
return TokenBoolean(initialpos, buffer.tell(), False)
|
||||
else:
|
||||
return TokenKeyword(initialpos, buffer.tell(), curtoken)
|
||||
raise LexerEOF
|
||||
@@ -0,0 +1,67 @@
|
||||
from dataclasses import dataclass
|
||||
from pdfmajor.lexer.exceptions import (
|
||||
InvalidHexToken,
|
||||
LexerEOF,
|
||||
LexerError,
|
||||
)
|
||||
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.lexer.token import TokenName
|
||||
from pdfmajor.lexer.regex import HEX, END_LITERAL
|
||||
|
||||
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class LiteralParseState:
|
||||
curtoken: bytes
|
||||
hex_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_name(buffer: SafeBufferIt) -> TokenName:
|
||||
"""Parses input stream into a literal name
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
|
||||
Returns:
|
||||
TokenName
|
||||
"""
|
||||
initialpos = buffer.tell() - 1
|
||||
state = LiteralParseState(b"")
|
||||
for pos, buf in buffer:
|
||||
if state.hex_value is not None:
|
||||
for ci in range(len(buf)):
|
||||
c = buf[ci : ci + 1]
|
||||
if state.hex_value is None:
|
||||
raise LexerError("state.hex_value became None")
|
||||
if HEX.match(c) and len(state.hex_value) < 2:
|
||||
state.hex_value += c
|
||||
buffer.seek(pos + ci + 1)
|
||||
if len(state.hex_value) >= 2: # type: ignore
|
||||
state.curtoken += bytes([int(state.hex_value, 16)]) # type: ignore
|
||||
state.hex_value = None
|
||||
break
|
||||
else:
|
||||
raise InvalidHexToken(buffer.tell() - buffer.buffer_size, c)
|
||||
else:
|
||||
m = END_LITERAL.search(buf, 0)
|
||||
if not m:
|
||||
state.curtoken += buf
|
||||
else:
|
||||
j: int = m.start(0)
|
||||
state.curtoken += buf[:j]
|
||||
next_char = buf[j : j + 1]
|
||||
if next_char == b"#":
|
||||
state.hex_value = b""
|
||||
buffer.seek(pos + j + 1)
|
||||
continue
|
||||
else:
|
||||
buffer.seek(pos + j)
|
||||
return TokenName(
|
||||
initialpos,
|
||||
buffer.tell(),
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
raise LexerEOF
|
||||
@@ -0,0 +1,6 @@
|
||||
import re
|
||||
|
||||
EOL = re.compile(br"[\r\n]")
|
||||
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
HEX = re.compile(br"[0-9a-fA-F]")
|
||||
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
@@ -0,0 +1,116 @@
|
||||
from enum import Enum
|
||||
from typing import Any, Generic, TypeVar, Union
|
||||
from decimal import Decimal
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Token(Generic[T]):
|
||||
"""A generic class representing PDF Token"""
|
||||
|
||||
__slots__ = ["pos", "size", "value"]
|
||||
|
||||
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
|
||||
self.start_loc = start_loc
|
||||
self.end_loc = end_loc
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
"{cls}(start_loc={start_loc}, end_loc={end_loc}, value='{value}')".format(
|
||||
cls=self.__class__.__name__,
|
||||
start_loc=self.start_loc,
|
||||
value=self.value,
|
||||
end_loc=self.end_loc,
|
||||
)
|
||||
)
|
||||
|
||||
def __eq__(self, other: Any):
|
||||
if isinstance(other, self.__class__):
|
||||
return (
|
||||
(other.value == self.value)
|
||||
and (other.start_loc == self.start_loc)
|
||||
and (other.end_loc == self.end_loc)
|
||||
)
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
class TokenComment(Token[bytes]):
|
||||
"""Token representing PDF comments
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenKeyword(Token[bytes]):
|
||||
"""Token representing PDF comments
|
||||
Detection of this type of token can be found in PDF 1.7 spec
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenName(Token[str]):
|
||||
"""Token representing literal name tokens
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.5
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenBoolean(Token[bool]):
|
||||
"""Token representing PDF booleans
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.2
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenString(Token[str]):
|
||||
"""Token representing PDF literal string
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenHexString(Token[bytes]):
|
||||
"""Token representing PDF hex strings
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenInteger(Token[int]):
|
||||
"""Token representing PDF integers
|
||||
Detection of this type of token can be found in PDF 1.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenDecimal(Token[Decimal]):
|
||||
"""Token representing PDF decimals
|
||||
Detection of this type of token can be found in PDF 1.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TDictVaue(Enum):
|
||||
OPEN = "<<"
|
||||
CLOSE = ">>"
|
||||
|
||||
|
||||
class TokenDictionary(Token[TDictVaue]):
|
||||
"""Token representing the brackets of dictionary
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
TokenNumber = Union[TokenInteger, TokenDecimal]
|
||||
@@ -0,0 +1,73 @@
|
||||
import io
|
||||
from typing import NamedTuple
|
||||
from pdfmajor.execptions import PDFMajorException
|
||||
|
||||
|
||||
class EOF(PDFMajorException):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("reached the end of the file or buffer stream!")
|
||||
|
||||
|
||||
class BufferedBytes(NamedTuple):
|
||||
pos: int
|
||||
data: bytes
|
||||
|
||||
|
||||
class SafeBufferIt:
|
||||
"""A safe reader for bytes that has a few nice additional utilities"""
|
||||
|
||||
__slots__ = ["fp", "buffer_size"]
|
||||
|
||||
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
|
||||
self.fp = fp
|
||||
self.buffer_size = buffer_size
|
||||
|
||||
def seekd(self, step: int) -> int:
|
||||
"""similar to self.fp.seek but you move the offset relative to where you are now
|
||||
|
||||
Args:
|
||||
step (int): number of steps to move
|
||||
"""
|
||||
cur_pos = self.tell()
|
||||
return self.seek(max(cur_pos + step, 0))
|
||||
|
||||
def seek(self, offset: int) -> int:
|
||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||
|
||||
Args:
|
||||
offset (int)
|
||||
|
||||
Returns:
|
||||
int: new offset
|
||||
"""
|
||||
return self.fp.seek(offset)
|
||||
|
||||
def tell(self) -> int:
|
||||
"""returns the current offset
|
||||
|
||||
Returns:
|
||||
int: current offset
|
||||
"""
|
||||
return self.fp.tell()
|
||||
|
||||
def __iter__(self):
|
||||
if self.fp.read(1) == b"":
|
||||
raise EOF
|
||||
self.seekd(-1)
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
pos = self.tell()
|
||||
buf = self.fp.read(self.buffer_size)
|
||||
if len(buf) == 0:
|
||||
raise StopIteration
|
||||
return BufferedBytes(pos, buf)
|
||||
|
||||
def close(self):
|
||||
self.fp.close()
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def __exit__(self):
|
||||
self.close()
|
||||
@@ -0,0 +1,94 @@
|
||||
import io
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.lexer.name import parse_name
|
||||
from pdfmajor.lexer.keyword import parse_keyword
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName
|
||||
from pdfmajor.lexer.comment import parse_comment
|
||||
|
||||
|
||||
def make_stream_iter(data: bytes, buf_size: int):
|
||||
return SafeBufferIt(io.BytesIO(data), buf_size)
|
||||
|
||||
|
||||
class Basics(TestCase):
|
||||
def test_comment_parser_ln(self):
|
||||
for buf_size in [2, 7, 11, 150]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable",
|
||||
buf_size,
|
||||
)
|
||||
token = parse_comment(buffer)
|
||||
self.assertIsInstance(token, TokenComment)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(
|
||||
token.value, b"this is a lengthy comment that ends here"
|
||||
)
|
||||
self.assertEqual(token.end_loc, 40)
|
||||
|
||||
def test_comment_parser_lr(self):
|
||||
for buf_size in [2, 7, 11, 150]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\rso this is not reachable",
|
||||
buf_size,
|
||||
)
|
||||
token = parse_comment(buffer)
|
||||
self.assertIsInstance(token, TokenComment)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(
|
||||
token.value, b"this is a lengthy comment that ends here"
|
||||
)
|
||||
self.assertEqual(token.end_loc, 40)
|
||||
|
||||
def test_parse_bool_true(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(b"true and the american way", buf_size)
|
||||
token = parse_keyword(buffer)
|
||||
self.assertIsInstance(token, TokenBoolean)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, True)
|
||||
self.assertEqual(token.end_loc, 4)
|
||||
|
||||
def test_parse_bool_false(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(b"false mannn", buf_size)
|
||||
token = parse_keyword(buffer)
|
||||
self.assertIsInstance(token, TokenBoolean)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, False)
|
||||
self.assertEqual(token.end_loc, 5)
|
||||
|
||||
def test_parse_literal_names(self):
|
||||
for buf_size in [2, 3, 50, 444]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(br"Some_Name /foo#5f#xbaa", buf_size)
|
||||
token = parse_name(buffer)
|
||||
self.assertIsInstance(token, TokenName)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, "Some_Name")
|
||||
self.assertEqual(token.end_loc, 9)
|
||||
|
||||
def test_parse_literal_names_hex(self):
|
||||
for buf_size in [50, 1000]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(br"foo#5faa ", buf_size)
|
||||
token = parse_name(buffer)
|
||||
self.assertIsInstance(token, TokenName)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, "foo_aa")
|
||||
self.assertEqual(token.end_loc, 8)
|
||||
|
||||
# def test_parse_string(self):
|
||||
# for buf_size in [2, 3, 50, 1000]:
|
||||
# with self.subTest(buf_end_loc=buf_size):
|
||||
# iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
|
||||
# token = parse_string(88, iter_stream(buf_size))
|
||||
# self.assertIsInstance(token, TokenString)
|
||||
# self.assertEqual(token.start_loc, 88)
|
||||
# self.assertEqual(token.value, "this % is not a comment.")
|
||||
# self.assertEqual(token.end_loc, 88 + 25)
|
||||
@@ -0,0 +1,39 @@
|
||||
import io
|
||||
from unittest import TestCase
|
||||
from pdfmajor.safebufiterator import EOF, SafeBufferIt
|
||||
|
||||
|
||||
class UseCase(TestCase):
|
||||
def test_safebuff(self):
|
||||
for bufsize in [2, 33, 65, 66, 100, 200]:
|
||||
it = SafeBufferIt(
|
||||
io.BytesIO(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
),
|
||||
buffer_size=bufsize,
|
||||
)
|
||||
for subbuf in it:
|
||||
self.assertLessEqual(len(subbuf), it.buffer_size)
|
||||
|
||||
def test_itraises(self):
|
||||
it = SafeBufferIt(io.BytesIO(b"this"))
|
||||
list(it) # read everything
|
||||
with self.assertRaises(EOF):
|
||||
for _ in it:
|
||||
pass
|
||||
|
||||
def test_seekd(self):
|
||||
for bufsize in [2, 33, 65, 66, 100, 200]:
|
||||
it = SafeBufferIt(
|
||||
io.BytesIO(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
),
|
||||
buffer_size=bufsize,
|
||||
)
|
||||
last_buf = next(it)
|
||||
it.seekd(-it.buffer_size)
|
||||
for i, buf in enumerate(it):
|
||||
self.assertEqual(last_buf, buf)
|
||||
it.seekd(-it.buffer_size)
|
||||
if i > 4:
|
||||
break
|
||||
Reference in new issue
Block a user