using new approach for lexing
This commit is contained in:
1 parent
a2d2b7808d
commit
faaf7925bf
10 files changed
+482
No files matched your search
@@ -0,0 +1,9 @@
|
||||
import io
|
||||
|
||||
|
||||
class PDFLexer:
|
||||
def __init__(self, fp: io.BufferedIOBase) -> None:
|
||||
self.fp = fp
|
||||
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
@@ -0,0 +1,29 @@
|
||||
from pdfmajor.lexer.exceptions import LexerEOF
|
||||
from pdfmajor.lexer.regex import EOL
|
||||
from pdfmajor.lexer.token import (
|
||||
TokenComment,
|
||||
)
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
|
||||
|
||||
def parse_comment(buffer: SafeBufferIt) -> TokenComment:
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
initialpos = buffer.tell() - 1
|
||||
for pos, buf in buffer:
|
||||
m = EOL.search(buf, 0)
|
||||
if not m:
|
||||
curtoken += buf
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += buf[:j]
|
||||
buffer.seek(pos + j)
|
||||
return TokenComment(initialpos, buffer.tell(), curtoken)
|
||||
raise LexerEOF
|
||||
@@ -0,0 +1,16 @@
|
||||
from pdfmajor.execptions import PDFMajorException
|
||||
|
||||
|
||||
class LexerError(PDFMajorException):
|
||||
pass
|
||||
|
||||
|
||||
class LexerEOF(LexerError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidHexToken(LexerError):
|
||||
def __init__(self, pos: int, token: bytes) -> None:
|
||||
super().__init__(
|
||||
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.lexer.exceptions import LexerEOF
|
||||
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword
|
||||
from pdfmajor.lexer.regex import END_KEYWORD
|
||||
from typing import Union
|
||||
|
||||
|
||||
def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]:
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
"""
|
||||
curtoken = b""
|
||||
initialpos = buffer.tell() - 1
|
||||
for pos, buf in buffer:
|
||||
m = END_KEYWORD.search(buf, 0)
|
||||
if not m:
|
||||
curtoken += buf
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += buf[:j]
|
||||
buffer.seek(pos + j)
|
||||
if curtoken == b"true":
|
||||
return TokenBoolean(initialpos, buffer.tell(), True)
|
||||
elif curtoken == b"false":
|
||||
return TokenBoolean(initialpos, buffer.tell(), False)
|
||||
else:
|
||||
return TokenKeyword(initialpos, buffer.tell(), curtoken)
|
||||
raise LexerEOF
|
||||
@@ -0,0 +1,67 @@
|
||||
from dataclasses import dataclass
|
||||
from pdfmajor.lexer.exceptions import (
|
||||
InvalidHexToken,
|
||||
LexerEOF,
|
||||
LexerError,
|
||||
)
|
||||
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.lexer.token import TokenName
|
||||
from pdfmajor.lexer.regex import HEX, END_LITERAL
|
||||
|
||||
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class LiteralParseState:
|
||||
curtoken: bytes
|
||||
hex_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_name(buffer: SafeBufferIt) -> TokenName:
|
||||
"""Parses input stream into a literal name
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
|
||||
Returns:
|
||||
TokenName
|
||||
"""
|
||||
initialpos = buffer.tell() - 1
|
||||
state = LiteralParseState(b"")
|
||||
for pos, buf in buffer:
|
||||
if state.hex_value is not None:
|
||||
for ci in range(len(buf)):
|
||||
c = buf[ci : ci + 1]
|
||||
if state.hex_value is None:
|
||||
raise LexerError("state.hex_value became None")
|
||||
if HEX.match(c) and len(state.hex_value) < 2:
|
||||
state.hex_value += c
|
||||
buffer.seek(pos + ci + 1)
|
||||
if len(state.hex_value) >= 2: # type: ignore
|
||||
state.curtoken += bytes([int(state.hex_value, 16)]) # type: ignore
|
||||
state.hex_value = None
|
||||
break
|
||||
else:
|
||||
raise InvalidHexToken(buffer.tell() - buffer.buffer_size, c)
|
||||
else:
|
||||
m = END_LITERAL.search(buf, 0)
|
||||
if not m:
|
||||
state.curtoken += buf
|
||||
else:
|
||||
j: int = m.start(0)
|
||||
state.curtoken += buf[:j]
|
||||
next_char = buf[j : j + 1]
|
||||
if next_char == b"#":
|
||||
state.hex_value = b""
|
||||
buffer.seek(pos + j + 1)
|
||||
continue
|
||||
else:
|
||||
buffer.seek(pos + j)
|
||||
return TokenName(
|
||||
initialpos,
|
||||
buffer.tell(),
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
raise LexerEOF
|
||||
@@ -0,0 +1,6 @@
|
||||
import re
|
||||
|
||||
EOL = re.compile(br"[\r\n]")
|
||||
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
HEX = re.compile(br"[0-9a-fA-F]")
|
||||
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
@@ -0,0 +1,116 @@
|
||||
from enum import Enum
|
||||
from typing import Any, Generic, TypeVar, Union
|
||||
from decimal import Decimal
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Token(Generic[T]):
|
||||
"""A generic class representing PDF Token"""
|
||||
|
||||
__slots__ = ["pos", "size", "value"]
|
||||
|
||||
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
|
||||
self.start_loc = start_loc
|
||||
self.end_loc = end_loc
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
"{cls}(start_loc={start_loc}, end_loc={end_loc}, value='{value}')".format(
|
||||
cls=self.__class__.__name__,
|
||||
start_loc=self.start_loc,
|
||||
value=self.value,
|
||||
end_loc=self.end_loc,
|
||||
)
|
||||
)
|
||||
|
||||
def __eq__(self, other: Any):
|
||||
if isinstance(other, self.__class__):
|
||||
return (
|
||||
(other.value == self.value)
|
||||
and (other.start_loc == self.start_loc)
|
||||
and (other.end_loc == self.end_loc)
|
||||
)
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
class TokenComment(Token[bytes]):
|
||||
"""Token representing PDF comments
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenKeyword(Token[bytes]):
|
||||
"""Token representing PDF comments
|
||||
Detection of this type of token can be found in PDF 1.7 spec
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenName(Token[str]):
|
||||
"""Token representing literal name tokens
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.5
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenBoolean(Token[bool]):
|
||||
"""Token representing PDF booleans
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.2
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenString(Token[str]):
|
||||
"""Token representing PDF literal string
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenHexString(Token[bytes]):
|
||||
"""Token representing PDF hex strings
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenInteger(Token[int]):
|
||||
"""Token representing PDF integers
|
||||
Detection of this type of token can be found in PDF 1.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenDecimal(Token[Decimal]):
|
||||
"""Token representing PDF decimals
|
||||
Detection of this type of token can be found in PDF 1.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TDictVaue(Enum):
|
||||
OPEN = "<<"
|
||||
CLOSE = ">>"
|
||||
|
||||
|
||||
class TokenDictionary(Token[TDictVaue]):
|
||||
"""Token representing the brackets of dictionary
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
TokenNumber = Union[TokenInteger, TokenDecimal]
|
||||
@@ -0,0 +1,73 @@
|
||||
import io
|
||||
from typing import NamedTuple
|
||||
from pdfmajor.execptions import PDFMajorException
|
||||
|
||||
|
||||
class EOF(PDFMajorException):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("reached the end of the file or buffer stream!")
|
||||
|
||||
|
||||
class BufferedBytes(NamedTuple):
|
||||
pos: int
|
||||
data: bytes
|
||||
|
||||
|
||||
class SafeBufferIt:
|
||||
"""A safe reader for bytes that has a few nice additional utilities"""
|
||||
|
||||
__slots__ = ["fp", "buffer_size"]
|
||||
|
||||
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
|
||||
self.fp = fp
|
||||
self.buffer_size = buffer_size
|
||||
|
||||
def seekd(self, step: int) -> int:
|
||||
"""similar to self.fp.seek but you move the offset relative to where you are now
|
||||
|
||||
Args:
|
||||
step (int): number of steps to move
|
||||
"""
|
||||
cur_pos = self.tell()
|
||||
return self.seek(max(cur_pos + step, 0))
|
||||
|
||||
def seek(self, offset: int) -> int:
|
||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||
|
||||
Args:
|
||||
offset (int)
|
||||
|
||||
Returns:
|
||||
int: new offset
|
||||
"""
|
||||
return self.fp.seek(offset)
|
||||
|
||||
def tell(self) -> int:
|
||||
"""returns the current offset
|
||||
|
||||
Returns:
|
||||
int: current offset
|
||||
"""
|
||||
return self.fp.tell()
|
||||
|
||||
def __iter__(self):
|
||||
if self.fp.read(1) == b"":
|
||||
raise EOF
|
||||
self.seekd(-1)
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
pos = self.tell()
|
||||
buf = self.fp.read(self.buffer_size)
|
||||
if len(buf) == 0:
|
||||
raise StopIteration
|
||||
return BufferedBytes(pos, buf)
|
||||
|
||||
def close(self):
|
||||
self.fp.close()
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def __exit__(self):
|
||||
self.close()
|
||||
Reference in new issue
Block a user