using new approach for lexing

This commit is contained in:
ari committed 2020-10-16 16:10:10 -04:00
1 parent a2d2b7808d
commit faaf7925bf
10 files changed
+482

No files matched your search

+9
View File
@@ -0,0 +1,9 @@
import io
class PDFLexer:
def __init__(self, fp: io.BufferedIOBase) -> None:
self.fp = fp
def seek(self, pos: int):
self.fp.seek(pos)
+29
View File
@@ -0,0 +1,29 @@
from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.regex import EOL
from pdfmajor.lexer.token import (
TokenComment,
)
from pdfmajor.safebufiterator import SafeBufferIt
def parse_comment(buffer: SafeBufferIt) -> TokenComment:
"""Parses input stream into a comment
Args:
buffer (SafeBufferIt)
Returns:
TokenComment
"""
curtoken = b""
initialpos = buffer.tell() - 1
for pos, buf in buffer:
m = EOL.search(buf, 0)
if not m:
curtoken += buf
else:
j = m.start(0)
curtoken += buf[:j]
buffer.seek(pos + j)
return TokenComment(initialpos, buffer.tell(), curtoken)
raise LexerEOF
+16
View File
@@ -0,0 +1,16 @@
from pdfmajor.execptions import PDFMajorException
class LexerError(PDFMajorException):
pass
class LexerEOF(LexerError):
pass
class InvalidHexToken(LexerError):
def __init__(self, pos: int, token: bytes) -> None:
super().__init__(
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
)
+33
View File
@@ -0,0 +1,33 @@
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword
from pdfmajor.lexer.regex import END_KEYWORD
from typing import Union
def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]:
"""Parses input stream into a keyword or bool
Args:
buffer (SafeBufferIt)
Returns:
Union[TokenKeyword, TokenBoolean]
"""
curtoken = b""
initialpos = buffer.tell() - 1
for pos, buf in buffer:
m = END_KEYWORD.search(buf, 0)
if not m:
curtoken += buf
else:
j = m.start(0)
curtoken += buf[:j]
buffer.seek(pos + j)
if curtoken == b"true":
return TokenBoolean(initialpos, buffer.tell(), True)
elif curtoken == b"false":
return TokenBoolean(initialpos, buffer.tell(), False)
else:
return TokenKeyword(initialpos, buffer.tell(), curtoken)
raise LexerEOF
+67
View File
@@ -0,0 +1,67 @@
from dataclasses import dataclass
from pdfmajor.lexer.exceptions import (
InvalidHexToken,
LexerEOF,
LexerError,
)
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.lexer.token import TokenName
from pdfmajor.lexer.regex import HEX, END_LITERAL
from typing import Optional
@dataclass
class LiteralParseState:
curtoken: bytes
hex_value: Optional[bytes] = None
def parse_name(buffer: SafeBufferIt) -> TokenName:
"""Parses input stream into a literal name
Args:
buffer (SafeBufferIt)
Returns:
TokenName
"""
initialpos = buffer.tell() - 1
state = LiteralParseState(b"")
for pos, buf in buffer:
if state.hex_value is not None:
for ci in range(len(buf)):
c = buf[ci : ci + 1]
if state.hex_value is None:
raise LexerError("state.hex_value became None")
if HEX.match(c) and len(state.hex_value) < 2:
state.hex_value += c
buffer.seek(pos + ci + 1)
if len(state.hex_value) >= 2: # type: ignore
state.curtoken += bytes([int(state.hex_value, 16)]) # type: ignore
state.hex_value = None
break
else:
raise InvalidHexToken(buffer.tell() - buffer.buffer_size, c)
else:
m = END_LITERAL.search(buf, 0)
if not m:
state.curtoken += buf
else:
j: int = m.start(0)
state.curtoken += buf[:j]
next_char = buf[j : j + 1]
if next_char == b"#":
state.hex_value = b""
buffer.seek(pos + j + 1)
continue
else:
buffer.seek(pos + j)
return TokenName(
initialpos,
buffer.tell(),
state.curtoken.decode(),
)
raise LexerEOF
+6
View File
@@ -0,0 +1,6 @@
import re
EOL = re.compile(br"[\r\n]")
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
HEX = re.compile(br"[0-9a-fA-F]")
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
+116
View File
@@ -0,0 +1,116 @@
from enum import Enum
from typing import Any, Generic, TypeVar, Union
from decimal import Decimal
T = TypeVar("T")
class Token(Generic[T]):
"""A generic class representing PDF Token"""
__slots__ = ["pos", "size", "value"]
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
self.start_loc = start_loc
self.end_loc = end_loc
self.value = value
def __repr__(self) -> str:
return (
"{cls}(start_loc={start_loc}, end_loc={end_loc}, value='{value}')".format(
cls=self.__class__.__name__,
start_loc=self.start_loc,
value=self.value,
end_loc=self.end_loc,
)
)
def __eq__(self, other: Any):
if isinstance(other, self.__class__):
return (
(other.value == self.value)
and (other.start_loc == self.start_loc)
and (other.end_loc == self.end_loc)
)
else:
return False
class TokenComment(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
"""
pass
class TokenKeyword(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec
"""
pass
class TokenName(Token[str]):
"""Token representing literal name tokens
Detection of this type of token can be found in PDF 1.7 spec section 7.3.5
"""
pass
class TokenBoolean(Token[bool]):
"""Token representing PDF booleans
Detection of this type of token can be found in PDF 1.7 spec section 7.3.2
"""
pass
class TokenString(Token[str]):
"""Token representing PDF literal string
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2
"""
pass
class TokenHexString(Token[bytes]):
"""Token representing PDF hex strings
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3
"""
pass
class TokenInteger(Token[int]):
"""Token representing PDF integers
Detection of this type of token can be found in PDF 1.7
"""
pass
class TokenDecimal(Token[Decimal]):
"""Token representing PDF decimals
Detection of this type of token can be found in PDF 1.7
"""
pass
class TDictVaue(Enum):
OPEN = "<<"
CLOSE = ">>"
class TokenDictionary(Token[TDictVaue]):
"""Token representing the brackets of dictionary
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
"""
pass
TokenNumber = Union[TokenInteger, TokenDecimal]
+73
View File
@@ -0,0 +1,73 @@
import io
from typing import NamedTuple
from pdfmajor.execptions import PDFMajorException
class EOF(PDFMajorException):
def __init__(self) -> None:
super().__init__("reached the end of the file or buffer stream!")
class BufferedBytes(NamedTuple):
pos: int
data: bytes
class SafeBufferIt:
"""A safe reader for bytes that has a few nice additional utilities"""
__slots__ = ["fp", "buffer_size"]
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
self.fp = fp
self.buffer_size = buffer_size
def seekd(self, step: int) -> int:
"""similar to self.fp.seek but you move the offset relative to where you are now
Args:
step (int): number of steps to move
"""
cur_pos = self.tell()
return self.seek(max(cur_pos + step, 0))
def seek(self, offset: int) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
return self.fp.seek(offset)
def tell(self) -> int:
"""returns the current offset
Returns:
int: current offset
"""
return self.fp.tell()
def __iter__(self):
if self.fp.read(1) == b"":
raise EOF
self.seekd(-1)
return self
def __next__(self):
pos = self.tell()
buf = self.fp.read(self.buffer_size)
if len(buf) == 0:
raise StopIteration
return BufferedBytes(pos, buf)
def close(self):
self.fp.close()
def __del__(self):
self.close()
def __exit__(self):
self.close()