- clean up
- renamed safebuf to streambuf - added arrays
This commit is contained in:
1 parent
c752346638
commit
a03857d0f7
29 files changed
+280
-1381
No files matched your search
+2
-2
@@ -1,9 +1,9 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = tab
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = false
|
||||
insert_final_newline = true
|
||||
insert_final_newline = true
|
||||
@@ -1,9 +1,73 @@
|
||||
import io
|
||||
from typing import Iterator
|
||||
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket
|
||||
from pdfmajor.lexer.string import parse_string
|
||||
from pdfmajor.lexer.keyword import parse_keyword
|
||||
from pdfmajor.lexer.number import parse_number
|
||||
from pdfmajor.lexer.name import parse_name
|
||||
from pdfmajor.lexer.comment import parse_comment
|
||||
from pdfmajor.lexer.regex import NONSPC
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
Token,
|
||||
TokenDictionary,
|
||||
TokenKeyword,
|
||||
TokenArray,
|
||||
)
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
|
||||
|
||||
class PDFLexer:
|
||||
def __init__(self, fp: io.BufferedIOBase) -> None:
|
||||
self.fp = fp
|
||||
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
|
||||
self.buffer = BufferStream(fp, buffer_size=buffer_size)
|
||||
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
self.buffer.seek(pos)
|
||||
|
||||
def iter_tokens(self) -> Iterator[Token]:
|
||||
for bufpos, buf in self.buffer:
|
||||
m = NONSPC.search(buf, 0)
|
||||
if not m:
|
||||
self.buf_skipstep = None
|
||||
continue
|
||||
else:
|
||||
j = m.start(0)
|
||||
next_char = buf[j : j + 1]
|
||||
if next_char == b"%":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield parse_comment(self.buffer)
|
||||
elif next_char == b"/":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield parse_name(self.buffer)
|
||||
elif next_char in b"-+":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield parse_number(self.buffer, next_char)
|
||||
elif next_char.isdigit() or next_char == b".":
|
||||
self.buffer.seek(bufpos + j)
|
||||
yield parse_number(self.buffer)
|
||||
elif next_char.isalpha():
|
||||
self.buffer.seek(bufpos + j)
|
||||
yield parse_keyword(self.buffer)
|
||||
elif next_char == b"(":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield parse_string(self.buffer)
|
||||
elif next_char == b"<":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield parse_double_angled_bracket(self.buffer)
|
||||
elif next_char == b">":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
if buf[j : j + 2] == ">>":
|
||||
self.buffer.seekd(1)
|
||||
yield TokenDictionary(
|
||||
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
|
||||
)
|
||||
elif next_char == b"[":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
|
||||
elif next_char == b"]":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE)
|
||||
else:
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char)
|
||||
@@ -3,14 +3,14 @@ from pdfmajor.lexer.regex import EOL
|
||||
from pdfmajor.lexer.token import (
|
||||
TokenComment,
|
||||
)
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
|
||||
|
||||
def parse_comment(buffer: SafeBufferIt) -> TokenComment:
|
||||
def parse_comment(buffer: BufferStream) -> TokenComment:
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
buffer (BufferStream)
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
from pdfmajor.lexer.exceptions import LexerEOF
|
||||
from pdfmajor.lexer.regex import END_HEX_STRING, HEX_PAIR, SPC
|
||||
from pdfmajor.lexer.token import TDictVaue, TokenDictionary, TokenHexString
|
||||
from pdfmajor.lexer.token import TDictValue, TokenDictionary, TokenHexString
|
||||
from typing import Union
|
||||
|
||||
|
||||
def parse_double_angled_bracket(
|
||||
buffer: SafeBufferIt,
|
||||
buffer: BufferStream,
|
||||
) -> Union[TokenHexString, TokenDictionary]:
|
||||
"""Parses input stream into a hex string object or match a simple dictionary token
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
buffer (BufferStream)
|
||||
|
||||
Returns:
|
||||
Union[TokenHexString, TokenDictionary]
|
||||
@@ -22,16 +21,16 @@ def parse_double_angled_bracket(
|
||||
first_char = step.data[0:1]
|
||||
if first_char == b"<":
|
||||
buffer.seek(step.pos + 1)
|
||||
return TokenDictionary(initialpos, 2, TDictVaue.OPEN)
|
||||
return TokenDictionary(initialpos, 2, TDictValue.OPEN)
|
||||
buffer.seek(step.pos)
|
||||
return parse_hexstring(buffer)
|
||||
|
||||
|
||||
def parse_hexstring(buffer: SafeBufferIt) -> TokenHexString:
|
||||
def parse_hexstring(buffer: BufferStream) -> TokenHexString:
|
||||
"""Parses the input string into hex string
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
buffer (BufferStream)
|
||||
|
||||
Returns:
|
||||
TokenHexString
|
||||
|
||||
@@ -5,7 +5,7 @@ class LexerError(PDFMajorException):
|
||||
pass
|
||||
|
||||
|
||||
class LexerEOF(LexerError):
|
||||
class LexerEOF(LexerError, EOFError):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
from pdfmajor.lexer.exceptions import LexerEOF
|
||||
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword
|
||||
from pdfmajor.lexer.regex import END_KEYWORD
|
||||
from typing import Union
|
||||
|
||||
|
||||
def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]:
|
||||
def parse_keyword(buffer: BufferStream) -> Union[TokenKeyword, TokenBoolean]:
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
buffer (BufferStream)
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
|
||||
@@ -5,7 +5,7 @@ from pdfmajor.lexer.exceptions import (
|
||||
LexerError,
|
||||
)
|
||||
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
from pdfmajor.lexer.token import TokenName
|
||||
from pdfmajor.lexer.regex import HEX, END_LITERAL
|
||||
|
||||
@@ -19,11 +19,11 @@ class LiteralParseState:
|
||||
hex_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_name(buffer: SafeBufferIt) -> TokenName:
|
||||
def parse_name(buffer: BufferStream) -> TokenName:
|
||||
"""Parses input stream into a literal name
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
buffer (BufferStream)
|
||||
|
||||
Returns:
|
||||
TokenName
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError
|
||||
from pdfmajor.lexer.token import (
|
||||
TokenDecimal,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
from pdfmajor.lexer.regex import (
|
||||
END_NUMBER,
|
||||
)
|
||||
|
||||
|
||||
def parse_number(buffer: SafeBufferIt, sign: bytes = b"+") -> TokenNumber:
|
||||
def parse_number(buffer: BufferStream, sign: bytes = b"+") -> TokenNumber:
|
||||
"""Parses input stream into a number
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
buffer (BufferStream)
|
||||
sign (str): either a + or -
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -19,3 +19,5 @@ ESC_STRING = {
|
||||
END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]")
|
||||
HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.")
|
||||
SPC = re.compile(br"\s")
|
||||
NONSPC = re.compile(br"\S")
|
||||
END_NUMBER = re.compile(br"[^0-9]")
|
||||
@@ -3,7 +3,7 @@ from typing import Optional
|
||||
from pdfmajor.lexer.regex import END_STRING, ESC_STRING, OCT_STRING
|
||||
from pdfmajor.lexer.token import TokenString
|
||||
from pdfmajor.lexer.exceptions import LexerEOF, LexerError
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -13,11 +13,11 @@ class StringParseState:
|
||||
escaped_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_string(buffer: SafeBufferIt) -> TokenString:
|
||||
def parse_string(buffer: BufferStream) -> TokenString:
|
||||
"""Parses input stream into a simple string object
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
buffer (BufferStream)
|
||||
|
||||
Returns:
|
||||
TokenString
|
||||
@@ -55,7 +55,7 @@ def parse_string(buffer: SafeBufferIt) -> TokenString:
|
||||
raise LexerEOF
|
||||
|
||||
|
||||
def parse_string_escape(buffer: SafeBufferIt, state: StringParseState):
|
||||
def parse_string_escape(buffer: BufferStream, state: StringParseState):
|
||||
if state.escaped_value is None:
|
||||
raise LexerError("state.escaped_value is None")
|
||||
for pos, buf in buffer:
|
||||
|
||||
+16
-3
@@ -100,12 +100,15 @@ class TokenDecimal(Token[Decimal]):
|
||||
pass
|
||||
|
||||
|
||||
class TDictVaue(Enum):
|
||||
TokenNumber = Union[TokenInteger, TokenDecimal]
|
||||
|
||||
|
||||
class TDictValue(Enum):
|
||||
OPEN = "<<"
|
||||
CLOSE = ">>"
|
||||
|
||||
|
||||
class TokenDictionary(Token[TDictVaue]):
|
||||
class TokenDictionary(Token[TDictValue]):
|
||||
"""Token representing the brackets of dictionary
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
|
||||
"""
|
||||
@@ -113,4 +116,14 @@ class TokenDictionary(Token[TDictVaue]):
|
||||
pass
|
||||
|
||||
|
||||
TokenNumber = Union[TokenInteger, TokenDecimal]
|
||||
class TArrayValue(Enum):
|
||||
OPEN = "["
|
||||
CLOSE = "]"
|
||||
|
||||
|
||||
class TokenArray(Token[TArrayValue]):
|
||||
"""Token representing the brackets of an array
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.6
|
||||
"""
|
||||
|
||||
pass
|
||||
@@ -3,7 +3,7 @@ from typing import NamedTuple
|
||||
from pdfmajor.execptions import PDFMajorException
|
||||
|
||||
|
||||
class EOF(PDFMajorException):
|
||||
class StreamEOF(PDFMajorException, EOFError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("reached the end of the file or buffer stream!")
|
||||
|
||||
@@ -13,7 +13,7 @@ class BufferedBytes(NamedTuple):
|
||||
data: bytes
|
||||
|
||||
|
||||
class SafeBufferIt:
|
||||
class BufferStream:
|
||||
"""A safe reader for bytes that has a few nice additional utilities"""
|
||||
|
||||
__slots__ = ["fp", "buffer_size"]
|
||||
@@ -50,18 +50,30 @@ class SafeBufferIt:
|
||||
"""
|
||||
return self.fp.tell()
|
||||
|
||||
def read(self, size: int) -> BufferedBytes:
|
||||
"""Reads the amount of bytes specified
|
||||
|
||||
Args:
|
||||
size (int): number of bytes to read
|
||||
|
||||
Returns:
|
||||
BufferedBytes
|
||||
"""
|
||||
pos = self.tell()
|
||||
buf = self.fp.read(size)
|
||||
return BufferedBytes(pos, buf)
|
||||
|
||||
def __iter__(self):
|
||||
if self.fp.read(1) == b"":
|
||||
raise EOF
|
||||
raise StreamEOF
|
||||
self.seekd(-1)
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
pos = self.tell()
|
||||
buf = self.fp.read(self.buffer_size)
|
||||
if len(buf) == 0:
|
||||
bbyte = self.read(self.buffer_size)
|
||||
if len(bbyte.data) == 0:
|
||||
raise StopIteration
|
||||
return BufferedBytes(pos, buf)
|
||||
return bbyte
|
||||
|
||||
def close(self):
|
||||
self.fp.close()
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
import io
|
||||
from pdfmajor.tokenizer.token_parsers.dictionary_hexstring import (
|
||||
parse_double_angled_bracket,
|
||||
)
|
||||
from pdfmajor.tokenizer.token_parsers.string import parse_string
|
||||
|
||||
from typing import Iterator, Optional
|
||||
|
||||
from pdfmajor.tokenizer.constants import NONSPC
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
||||
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
|
||||
from pdfmajor.tokenizer.token_parsers.number import parse_number
|
||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenKeyword
|
||||
|
||||
|
||||
class PSTokenizer:
|
||||
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
|
||||
self.fp = fp
|
||||
self.bufsize = bufsize
|
||||
self.bufpos: int = 0
|
||||
self.buf: bytes = b""
|
||||
self.buf_skipstep: Optional[int] = None
|
||||
|
||||
def inc_buf_skipstep(self, value: int):
|
||||
if self.buf_skipstep is None:
|
||||
self.buf_skipstep = value
|
||||
else:
|
||||
self.buf_skipstep += value
|
||||
|
||||
def iter_buffer(self) -> Iterator[PInput]:
|
||||
while True:
|
||||
if (self.buf_skipstep is not None) and (self.buf_skipstep < len(self.buf)):
|
||||
yield PInput(
|
||||
self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :]
|
||||
)
|
||||
else:
|
||||
self.buf_skipstep = None
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.bufsize)
|
||||
if not self.buf:
|
||||
raise TokenizerEOF("Unexpected EOF")
|
||||
else:
|
||||
yield PInput(self.bufpos, self.buf)
|
||||
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
|
||||
def step_back(self, steps: int = 1):
|
||||
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
|
||||
|
||||
Args:
|
||||
steps (int, optional): numbers of steps. Defaults to 1.
|
||||
"""
|
||||
self.fp.seek(self.fp.tell() - steps * self.bufsize)
|
||||
|
||||
def iter_tokens(self) -> Iterator[Token]:
|
||||
for cur in self.iter_buffer():
|
||||
bufpos, buf = cur
|
||||
m = NONSPC.search(buf, 0)
|
||||
if not m:
|
||||
self.buf_skipstep = None
|
||||
continue
|
||||
else:
|
||||
j = m.start(0)
|
||||
c = buf[j : j + 1]
|
||||
if c == b"%":
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_comment(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b"/":
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_literal(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c in b"-+":
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer(), c)
|
||||
)
|
||||
elif c.isdigit():
|
||||
self.inc_buf_skipstep(j)
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b".":
|
||||
self.inc_buf_skipstep(j)
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c.isalpha():
|
||||
self.inc_buf_skipstep(j)
|
||||
yield self._check_token(
|
||||
cur, parse_keyword(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b"(":
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(
|
||||
cur, parse_string(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b"<":
|
||||
self.inc_buf_skipstep(j)
|
||||
yield self._check_token(
|
||||
cur, parse_double_angled_bracket(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b">":
|
||||
self.inc_buf_skipstep(j)
|
||||
if buf[j : j + 2] == ">>":
|
||||
self.inc_buf_skipstep(1)
|
||||
yield TokenDictionary(bufpos + j, 2, TDictVaue.CLOSE)
|
||||
else:
|
||||
self.inc_buf_skipstep(j + 1)
|
||||
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
|
||||
|
||||
def _check_token(self, cur: PInput, token: Token) -> Token:
|
||||
self.inc_buf_skipstep(token.end_loc - cur.pos)
|
||||
return token
|
||||
@@ -1,37 +0,0 @@
|
||||
import re
|
||||
from .types import PSLiteral, PSSymbolTable, PSKeyword
|
||||
|
||||
PSLiteralTable = PSSymbolTable(PSLiteral)
|
||||
PSKeywordTable = PSSymbolTable(PSKeyword)
|
||||
LIT = PSLiteralTable.intern
|
||||
KWD = PSKeywordTable.intern
|
||||
KEYWORD_PROC_BEGIN = KWD(b"{")
|
||||
KEYWORD_PROC_END = KWD(b"}")
|
||||
KEYWORD_ARRAY_BEGIN = KWD(b"[")
|
||||
KEYWORD_ARRAY_END = KWD(b"]")
|
||||
KEYWORD_DICT_BEGIN = KWD(b"<<")
|
||||
KEYWORD_DICT_END = KWD(b">>")
|
||||
|
||||
## PSBaseParser
|
||||
##
|
||||
EOL = re.compile(br"[\r\n]")
|
||||
SPC = re.compile(br"\s")
|
||||
NONSPC = re.compile(br"\S")
|
||||
HEX = re.compile(br"[0-9a-fA-F]")
|
||||
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]")
|
||||
HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.")
|
||||
END_NUMBER = re.compile(br"[^0-9]")
|
||||
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
END_STRING = re.compile(br"[()\134]")
|
||||
OCT_STRING = re.compile(br"[0-7]")
|
||||
ESC_STRING = {
|
||||
b"b": 8,
|
||||
b"t": 9,
|
||||
b"n": 10,
|
||||
b"f": 12,
|
||||
b"r": 13,
|
||||
b"(": 40,
|
||||
b")": 41,
|
||||
b"\\": 92,
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
from pdfmajor.execptions import PDFMajorException
|
||||
|
||||
|
||||
class TokenizerError(PDFMajorException):
|
||||
pass
|
||||
|
||||
|
||||
class TokenizerEOF(TokenizerError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidToken(TokenizerError):
|
||||
def __init__(self, pos: int, token: bytes) -> None:
|
||||
super().__init__(
|
||||
"Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token))
|
||||
)
|
||||
|
||||
|
||||
class InvalidHexToken(TokenizerError):
|
||||
def __init__(self, pos: int, token: bytes) -> None:
|
||||
super().__init__(
|
||||
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
|
||||
)
|
||||
@@ -1,341 +0,0 @@
|
||||
from typing import List, Tuple, Union
|
||||
from io import TextIOWrapper
|
||||
|
||||
from pdfmajor.utils import int2byte
|
||||
from pdfmajor.execptions import PSEOF
|
||||
|
||||
from .types import PSLiteral
|
||||
from .constants import OCT_STRING, ESC_STRING, KEYWORD_DICT_BEGIN, KEYWORD_DICT_END
|
||||
from .constants import END_NUMBER, END_STRING, END_KEYWORD, END_HEX_STRING
|
||||
from .constants import SPC, HEX_PAIR
|
||||
from .constants import EOL, NONSPC, KWD, END_LITERAL, LIT, HEX
|
||||
|
||||
|
||||
class PSTokenizer(object):
|
||||
"""Tokenizatizes bytes stream to PS-Tokens"""
|
||||
|
||||
BUFSIZ = 4096
|
||||
|
||||
def __init__(self, fp: TextIOWrapper):
|
||||
self.fp = fp
|
||||
self.seek(0)
|
||||
return
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos)
|
||||
|
||||
def flush(self):
|
||||
return
|
||||
|
||||
def close(self):
|
||||
self.flush()
|
||||
return
|
||||
|
||||
def tell(self):
|
||||
return self.bufpos + self.charpos
|
||||
|
||||
def poll(self, pos=None, n=80):
|
||||
pos0 = self.fp.tell()
|
||||
if not pos:
|
||||
pos = self.bufpos + self.charpos
|
||||
self.fp.seek(pos)
|
||||
self.fp.read(n)
|
||||
self.fp.seek(pos0)
|
||||
return
|
||||
|
||||
def seek(self, pos: int):
|
||||
"""Seeks the parser to the given position."""
|
||||
self.fp.seek(pos)
|
||||
# reset the status for nextline()
|
||||
self.bufpos: int = pos
|
||||
self.buf: bytes = b""
|
||||
self.charpos: int = 0
|
||||
# reset the status for nexttoken()
|
||||
self._current_parse_func = self._parse_main
|
||||
self._curtoken: bytes = b""
|
||||
self._curtokenpos: int = 0
|
||||
self._tokens: List[
|
||||
Tuple[int, Union[int, float, bool, str, bytes, PSLiteral]]
|
||||
] = []
|
||||
return
|
||||
|
||||
def fillbuf(self):
|
||||
if self.charpos < len(self.buf):
|
||||
return
|
||||
# fetch next chunk.
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.BUFSIZ)
|
||||
if not self.buf:
|
||||
raise PSEOF("Unexpected EOF")
|
||||
self.charpos = 0
|
||||
return
|
||||
|
||||
def nextline(self):
|
||||
"""Fetches a next line that ends either with \\r or \\n."""
|
||||
linebuf = b""
|
||||
linepos = self.bufpos + self.charpos
|
||||
eol = False
|
||||
while 1:
|
||||
self.fillbuf()
|
||||
if eol:
|
||||
c = self.buf[self.charpos : self.charpos + 1]
|
||||
# handle b'\r\n'
|
||||
if c == b"\n":
|
||||
linebuf += c
|
||||
self.charpos += 1
|
||||
break
|
||||
m = EOL.search(self.buf, self.charpos)
|
||||
if m:
|
||||
linebuf += self.buf[self.charpos : m.end(0)]
|
||||
self.charpos = m.end(0)
|
||||
if linebuf[-1:] == b"\r":
|
||||
eol = True
|
||||
else:
|
||||
break
|
||||
else:
|
||||
linebuf += self.buf[self.charpos :]
|
||||
self.charpos = len(self.buf)
|
||||
return (linepos, linebuf)
|
||||
|
||||
def revreadlines(self):
|
||||
"""Fetches a next line backword.
|
||||
|
||||
This is used to locate the trailers at the end of a file.
|
||||
"""
|
||||
self.fp.seek(0, 2)
|
||||
pos = self.fp.tell()
|
||||
buf = b""
|
||||
while 0 < pos:
|
||||
prevpos = pos
|
||||
pos = max(0, pos - self.BUFSIZ)
|
||||
self.fp.seek(pos)
|
||||
s = self.fp.read(prevpos - pos)
|
||||
if not s:
|
||||
break
|
||||
while 1:
|
||||
n = max(s.rfind(b"\r"), s.rfind(b"\n"))
|
||||
if n == -1:
|
||||
buf = s + buf
|
||||
break
|
||||
yield s[n:] + buf
|
||||
s = s[:n]
|
||||
buf = b""
|
||||
return
|
||||
|
||||
def _parse_main(self, s, i):
|
||||
m = NONSPC.search(s, i)
|
||||
if not m:
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
c = s[j : j + 1]
|
||||
self._curtokenpos = self.bufpos + j
|
||||
if c == b"%":
|
||||
self._curtoken = b"%"
|
||||
self._current_parse_func = self._parse_comment
|
||||
return j + 1
|
||||
elif c == b"/":
|
||||
self._curtoken = b""
|
||||
self._current_parse_func = self._parse_literal
|
||||
return j + 1
|
||||
elif c in b"-+" or c.isdigit():
|
||||
self._curtoken = c
|
||||
self._current_parse_func = self._parse_number
|
||||
return j + 1
|
||||
elif c == b".":
|
||||
self._curtoken = c
|
||||
self._current_parse_func = self._parse_float
|
||||
return j + 1
|
||||
elif c.isalpha():
|
||||
self._curtoken = c
|
||||
self._current_parse_func = self._parse_keyword
|
||||
return j + 1
|
||||
elif c == b"(":
|
||||
self._curtoken = b""
|
||||
self.paren = 1
|
||||
self._current_parse_func = self._parse_string
|
||||
return j + 1
|
||||
elif c == b"<":
|
||||
self._curtoken = b""
|
||||
self._current_parse_func = self._parse_wopen
|
||||
return j + 1
|
||||
elif c == b">":
|
||||
self._curtoken = b""
|
||||
self._current_parse_func = self._parse_wclose
|
||||
return j + 1
|
||||
else:
|
||||
self._add_token(KWD(c))
|
||||
return j + 1
|
||||
|
||||
def _add_token(self, obj):
|
||||
self._tokens.append((self._curtokenpos, obj))
|
||||
return
|
||||
|
||||
def _parse_comment(self, s, i):
|
||||
m = EOL.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
self._current_parse_func = self._parse_main
|
||||
# We ignore comments.
|
||||
# self._tokens.append(self._curtoken)
|
||||
return j
|
||||
|
||||
def _parse_literal(self, s, i):
|
||||
m = END_LITERAL.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"#":
|
||||
self.hex = b""
|
||||
self._current_parse_func = self._parse_literal_hex
|
||||
return j + 1
|
||||
try:
|
||||
self._curtoken = str(self._curtoken, "utf-8")
|
||||
except:
|
||||
pass
|
||||
self._add_token(LIT(self._curtoken))
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def _parse_literal_hex(self, s, i):
|
||||
c = s[i : i + 1]
|
||||
if HEX.match(c) and len(self.hex) < 2:
|
||||
self.hex += c
|
||||
return i + 1
|
||||
if self.hex:
|
||||
self._curtoken += int2byte(int(self.hex, 16))
|
||||
self._current_parse_func = self._parse_literal
|
||||
return i
|
||||
|
||||
def _parse_number(self, s, i):
|
||||
m = END_NUMBER.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b".":
|
||||
self._curtoken += c
|
||||
self._current_parse_func = self._parse_float
|
||||
return j + 1
|
||||
try:
|
||||
self._add_token(int(self._curtoken))
|
||||
except ValueError:
|
||||
pass
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def _parse_float(self, s, i):
|
||||
m = END_NUMBER.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
try:
|
||||
self._add_token(float(self._curtoken))
|
||||
except ValueError:
|
||||
pass
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def _parse_keyword(self, s, i):
|
||||
m = END_KEYWORD.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
if self._curtoken == b"true":
|
||||
token = True
|
||||
elif self._curtoken == b"false":
|
||||
token = False
|
||||
else:
|
||||
token = KWD(self._curtoken)
|
||||
self._add_token(token)
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def _parse_string(self, s, i):
|
||||
m = END_STRING.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"\\":
|
||||
self.oct = b""
|
||||
self._current_parse_func = self._parse_string_1
|
||||
return j + 1
|
||||
if c == b"(":
|
||||
self.paren += 1
|
||||
self._curtoken += c
|
||||
return j + 1
|
||||
if c == b")":
|
||||
self.paren -= 1
|
||||
if self.paren: # WTF, they said balanced parens need no special treatment.
|
||||
self._curtoken += c
|
||||
return j + 1
|
||||
self._add_token(self._curtoken)
|
||||
self._current_parse_func = self._parse_main
|
||||
return j + 1
|
||||
|
||||
def _parse_string_1(self, s, i):
|
||||
c = s[i : i + 1]
|
||||
if OCT_STRING.match(c) and len(self.oct) < 3:
|
||||
self.oct += c
|
||||
return i + 1
|
||||
if self.oct:
|
||||
self._curtoken += int2byte(int(self.oct, 8))
|
||||
self._current_parse_func = self._parse_string
|
||||
return i
|
||||
if c in ESC_STRING:
|
||||
self._curtoken += int2byte(ESC_STRING[c])
|
||||
self._current_parse_func = self._parse_string
|
||||
return i + 1
|
||||
|
||||
def _parse_wopen(self, s, i):
|
||||
c = s[i : i + 1]
|
||||
if c == b"<":
|
||||
self._add_token(KEYWORD_DICT_BEGIN)
|
||||
self._current_parse_func = self._parse_main
|
||||
i += 1
|
||||
else:
|
||||
self._current_parse_func = self._parse_hexstring
|
||||
return i
|
||||
|
||||
def _parse_wclose(self, s, i):
|
||||
c = s[i : i + 1]
|
||||
if c == b">":
|
||||
self._add_token(KEYWORD_DICT_END)
|
||||
i += 1
|
||||
self._current_parse_func = self._parse_main
|
||||
return i
|
||||
|
||||
def _parse_hexstring(self, s, i):
|
||||
m = END_HEX_STRING.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
token = HEX_PAIR.sub(
|
||||
lambda m: int2byte(int(m.group(0), 16)), SPC.sub(b"", self._curtoken)
|
||||
)
|
||||
self._add_token(token)
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def nexttoken(self):
|
||||
while not self._tokens:
|
||||
self.fillbuf()
|
||||
self.charpos = self._current_parse_func(self.buf, self.charpos)
|
||||
token = self._tokens.pop(0)
|
||||
return token
|
||||
@@ -1,112 +0,0 @@
|
||||
from enum import Enum
|
||||
from typing import Any, Generic, TypeVar, Union
|
||||
from decimal import Decimal
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Token(Generic[T]):
|
||||
__slots__ = ["pos", "size", "value"]
|
||||
|
||||
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
|
||||
self.start_loc = start_loc
|
||||
self.end_loc = end_loc
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "{cls}(start={start_loc}, end={end_loc}, value='{value}')".format(
|
||||
cls=self.__class__.__name__,
|
||||
start_loc=self.start_loc,
|
||||
value=self.value,
|
||||
end_loc=self.end_loc,
|
||||
)
|
||||
|
||||
def __eq__(self, other: Any):
|
||||
if isinstance(other, self.__class__):
|
||||
return (
|
||||
(other.value == self.value)
|
||||
and (other.start_loc == self.start_loc)
|
||||
and (other.end_loc == self.end_loc)
|
||||
)
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
class TokenComment(Token[bytes]):
|
||||
"""Token representing PDF comments
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenKeyword(Token[bytes]):
|
||||
"""Token representing PDF comments
|
||||
Detection of this type of token can be found in PDF 1.7 spec
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenLiteral(Token[str]):
|
||||
"""Token representing literal name tokens
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.5
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenBoolean(Token[bool]):
|
||||
"""Token representing PDF booleans
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.2
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenString(Token[str]):
|
||||
"""Token representing PDF literal string
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenHexString(Token[bytes]):
|
||||
"""Token representing PDF hex strings
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenInteger(Token[int]):
|
||||
"""Token representing PDF integers
|
||||
Detection of this type of token can be found in PDF 1.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TokenDecimal(Token[Decimal]):
|
||||
"""Token representing PDF decimals
|
||||
Detection of this type of token can be found in PDF 1.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TDictVaue(Enum):
|
||||
OPEN = "<<"
|
||||
CLOSE = ">>"
|
||||
|
||||
|
||||
class TokenDictionary(Token[TDictVaue]):
|
||||
"""Token representing the brackets of dictionary
|
||||
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
TokenNumber = Union[TokenInteger, TokenDecimal]
|
||||
Whitespace-only changes.
@@ -1,31 +0,0 @@
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenComment,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
EOL,
|
||||
)
|
||||
from typing import Iterator
|
||||
from .util import cmp_tsize, PInput
|
||||
|
||||
|
||||
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = EOL.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
return TokenComment(initialpos, curpos + j, curtoken)
|
||||
raise TokenizerEOF
|
||||
@@ -1,46 +0,0 @@
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerError
|
||||
from pdfmajor.tokenizer.constants import END_HEX_STRING, HEX_PAIR, SPC
|
||||
from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenHexString
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.utils import int2byte
|
||||
from typing import Iterator, Union
|
||||
|
||||
|
||||
def parse_double_angled_bracket(initialpos: int, inp: Iterator[PInput]) -> Token:
|
||||
"""Parses input stream into a simple string object
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
Union[TokenHexString, TokenDictionary]
|
||||
"""
|
||||
step = next(inp)
|
||||
first_char = step.buf[0:2]
|
||||
if first_char == b"<<":
|
||||
return TokenDictionary(initialpos, 2, TDictVaue.OPEN)
|
||||
curtoken = parse_hexstring(initialpos, PInput(step.pos + 1, step.buf[1:]), b"")
|
||||
if isinstance(curtoken, TokenHexString):
|
||||
return curtoken
|
||||
else:
|
||||
for step in inp:
|
||||
curtoken = parse_hexstring(initialpos, step, curtoken)
|
||||
if isinstance(curtoken, TokenHexString):
|
||||
return curtoken
|
||||
raise TokenizerError("parse_double_angled_bracket ended")
|
||||
|
||||
|
||||
def parse_hexstring(
|
||||
initialpos: int, inp: PInput, curtoken: bytes
|
||||
) -> Union[bytes, TokenHexString]:
|
||||
m = END_HEX_STRING.search(inp.buf, 0)
|
||||
if not m:
|
||||
curtoken += inp.buf
|
||||
return curtoken
|
||||
j = m.start(0)
|
||||
curtoken += inp.buf[:j]
|
||||
token = HEX_PAIR.sub(
|
||||
lambda x: int2byte(int(x.group(0), 16)), SPC.sub(b"", curtoken)
|
||||
)
|
||||
return TokenHexString(initialpos, inp.pos + j + 1, token)
|
||||
@@ -1,34 +0,0 @@
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword
|
||||
from pdfmajor.tokenizer.constants import END_KEYWORD
|
||||
from typing import Iterator, Union
|
||||
from .util import PInput
|
||||
|
||||
|
||||
def parse_keyword(
|
||||
initialpos: int, inp: Iterator[PInput]
|
||||
) -> Union[TokenKeyword, TokenBoolean]:
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = END_KEYWORD.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
if curtoken == b"true":
|
||||
return TokenBoolean(initialpos, curpos + j, True)
|
||||
elif curtoken == b"false":
|
||||
return TokenBoolean(initialpos, curpos + j, False)
|
||||
else:
|
||||
return TokenKeyword(initialpos, curpos + j, curtoken)
|
||||
raise TokenizerEOF
|
||||
@@ -1,67 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from pdfmajor.tokenizer.exceptions import (
|
||||
InvalidHexToken,
|
||||
TokenizerEOF,
|
||||
)
|
||||
|
||||
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token import TokenLiteral
|
||||
from pdfmajor.utils import int2byte
|
||||
from ..constants import HEX, END_LITERAL
|
||||
|
||||
from typing import Iterator, NamedTuple, Optional, Union
|
||||
|
||||
|
||||
@dataclass
|
||||
class LiteralParseState:
|
||||
curtoken: bytes
|
||||
hex_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
"""Parses input stream into a literal
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenLiteral
|
||||
"""
|
||||
state = LiteralParseState(b"")
|
||||
for curpos, buf in inp:
|
||||
it = SafeBufferIt(buf)
|
||||
for subbuf in it.into_iter():
|
||||
if state.hex_value is not None:
|
||||
for ci in range(len(subbuf)):
|
||||
c = subbuf[ci : ci + 1]
|
||||
if HEX.match(c) and len(state.hex_value) < 2:
|
||||
state.hex_value += c
|
||||
it.skip += 1
|
||||
if len(state.hex_value) >= 2:
|
||||
state.curtoken += int2byte(int(state.hex_value, 16))
|
||||
state.hex_value = None
|
||||
break
|
||||
else:
|
||||
raise InvalidHexToken(curpos + ci, c)
|
||||
else:
|
||||
m = END_LITERAL.search(subbuf, 0)
|
||||
if not m:
|
||||
state.curtoken += subbuf
|
||||
it.skip = len(buf)
|
||||
else:
|
||||
j: int = m.start(0)
|
||||
state.curtoken += subbuf[:j]
|
||||
c = subbuf[j : j + 1]
|
||||
if c == b"#":
|
||||
state.hex_value = b""
|
||||
it.skip = j + 1
|
||||
continue
|
||||
else:
|
||||
return TokenLiteral(
|
||||
initialpos,
|
||||
curpos + it.skip + j,
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
raise TokenizerEOF
|
||||
@@ -1,68 +0,0 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF, TokenizerError
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenDecimal,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_NUMBER,
|
||||
)
|
||||
from typing import Iterator
|
||||
from .util import PInput
|
||||
|
||||
|
||||
def parse_number(
|
||||
initialpos: int, inp: Iterator[PInput], sign: bytes = b"+"
|
||||
) -> TokenNumber:
|
||||
"""Parses input stream into a number
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
sign (str): either a + or -
|
||||
|
||||
Returns:
|
||||
TokenNumber
|
||||
"""
|
||||
if sign not in [b"+", b"-"]:
|
||||
raise TokenizerError("Number was provided an invalid sign {!r}".format(sign))
|
||||
curtoken = b"" + sign
|
||||
is_dec = False
|
||||
for curpos, s in inp:
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if (c == b".") and not is_dec:
|
||||
is_dec = True
|
||||
curtoken += c
|
||||
s = s[j + 1 :]
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
continue
|
||||
else:
|
||||
nj = m.start(0)
|
||||
curtoken += s[:nj]
|
||||
j += nj
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
curpos + j + 1,
|
||||
Decimal(curtoken.decode()),
|
||||
)
|
||||
try:
|
||||
if is_dec:
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
curpos + j,
|
||||
Decimal(curtoken.decode()),
|
||||
)
|
||||
else:
|
||||
return TokenInteger(initialpos, curpos + j, int(curtoken))
|
||||
except ValueError:
|
||||
raise InvalidToken(initialpos, curtoken)
|
||||
raise TokenizerEOF
|
||||
@@ -1,92 +0,0 @@
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerError
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenString,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_STRING,
|
||||
ESC_STRING,
|
||||
)
|
||||
from typing import Iterator, Optional
|
||||
|
||||
OCT_STRING = re.compile(br"^[0-7]{1,3}$")
|
||||
|
||||
|
||||
@dataclass
|
||||
class StringParseState:
|
||||
curtoken: bytes
|
||||
paren: int
|
||||
escaped_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString:
|
||||
"""Parses input stream into a simple string object
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenString
|
||||
"""
|
||||
state = StringParseState(b"", 1)
|
||||
for curpos, buf in inp:
|
||||
it = SafeBufferIt(buf)
|
||||
for s in it.into_iter():
|
||||
if state.escaped_value is not None:
|
||||
parse_string_escape(it, state)
|
||||
else:
|
||||
m = END_STRING.search(s, 0)
|
||||
if not m:
|
||||
state.curtoken += s
|
||||
it.skip += len(s)
|
||||
else:
|
||||
j = m.start(0)
|
||||
state.curtoken += s[:j]
|
||||
next_char = s[j : j + 1]
|
||||
if next_char == b"\\":
|
||||
state.escaped_value = b""
|
||||
it.skip += j + 1
|
||||
elif next_char == b"(":
|
||||
state.paren += 1
|
||||
state.curtoken += next_char
|
||||
it.skip += j + 1
|
||||
elif next_char == b")":
|
||||
state.paren -= 1
|
||||
if state.paren:
|
||||
state.curtoken += next_char
|
||||
it.skip += j + 1
|
||||
else:
|
||||
return TokenString(
|
||||
initialpos,
|
||||
curpos + it.skip + j + 1,
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
raise TokenizerError("end of parse_string")
|
||||
|
||||
|
||||
def parse_string_escape(it: SafeBufferIt, state: StringParseState):
|
||||
if state.escaped_value is None:
|
||||
raise TokenizerError("state.escaped_value is None")
|
||||
for s in it.into_iter():
|
||||
for ci in range(len(s)):
|
||||
c = s[ci : ci + 1]
|
||||
if len(state.escaped_value) < 3:
|
||||
it.skip += 1
|
||||
state.escaped_value += c
|
||||
if len(state.escaped_value) == 1:
|
||||
if (ci < 1) and (c in ESC_STRING):
|
||||
state.curtoken += bytes([ESC_STRING[c]])
|
||||
state.escaped_value = None
|
||||
return
|
||||
elif OCT_STRING.match(state.escaped_value):
|
||||
if len(state.escaped_value) == 3:
|
||||
state.curtoken += bytes([int(state.escaped_value, 8)])
|
||||
state.escaped_value = None
|
||||
return
|
||||
else:
|
||||
it.skip -= 1
|
||||
state.escaped_value = None
|
||||
return
|
||||
@@ -1,48 +0,0 @@
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.token import Token
|
||||
from typing import Callable, Iterator, NamedTuple
|
||||
|
||||
|
||||
class PInput(NamedTuple):
|
||||
pos: int
|
||||
buf: bytes
|
||||
|
||||
|
||||
ParserFunc = Callable[[int, Iterator[PInput]], Token]
|
||||
|
||||
|
||||
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
|
||||
"""Computes the size of the token based on its positional data
|
||||
|
||||
Args:
|
||||
curpos (int): current position of the buffer
|
||||
initialpos (int): initial position where the parser started working
|
||||
tokenend_idx (int): last relevant character index within the last buffered value
|
||||
|
||||
Returns:
|
||||
int: length of index
|
||||
"""
|
||||
return curpos - initialpos + tokenend_idx
|
||||
|
||||
|
||||
class SafeBufferIt:
|
||||
"""A safe while-loop to use on a buffer
|
||||
this class will make sure that there is no risk of an infinite loop and that the max number of loops is reasonable
|
||||
|
||||
Raises:
|
||||
TokenizerEOF: when we reach a maximum iteration we raise an 'end of file' error
|
||||
"""
|
||||
|
||||
__slots__ = ["skip", "buffer"]
|
||||
|
||||
def __init__(self, buffer: bytes, skip: int = 0) -> None:
|
||||
self.skip = skip
|
||||
self.buffer = buffer
|
||||
|
||||
def into_iter(self) -> Iterator[bytes]:
|
||||
for i in range(2 * len(self.buffer) + 1):
|
||||
if i > 2 * len(self.buffer):
|
||||
raise TokenizerEOF("Max Iteration Reached!")
|
||||
if self.skip >= len(self.buffer):
|
||||
break
|
||||
yield self.buffer[self.skip :]
|
||||
@@ -1,80 +0,0 @@
|
||||
## Basic PostScript Types
|
||||
##
|
||||
from typing import Union
|
||||
|
||||
## PSObject
|
||||
##
|
||||
class PSObject(object):
|
||||
|
||||
"""Base class for all PS or PDF-related data types."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
## PSLiteral
|
||||
##
|
||||
class PSLiteral(PSObject):
|
||||
|
||||
"""A class that represents a PostScript literal.
|
||||
|
||||
Postscript literals are used as identifiers, such as
|
||||
variable names, property names and dictionary keys.
|
||||
Literals are case sensitive and denoted by a preceding
|
||||
slash sign (e.g. "/Name")
|
||||
|
||||
Note: Do not create an instance of PSLiteral directly.
|
||||
Always use PSLiteralTable.intern().
|
||||
"""
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __repr__(self):
|
||||
name = self.name
|
||||
return "/%r" % name
|
||||
|
||||
|
||||
## PSKeyword
|
||||
##
|
||||
class PSKeyword(PSObject):
|
||||
|
||||
"""A class that represents a PostScript keyword.
|
||||
|
||||
PostScript keywords are a dozen of predefined words.
|
||||
Commands and directives in PostScript are expressed by keywords.
|
||||
They are also used to denote the content boundaries.
|
||||
|
||||
Note: Do not create an instance of PSKeyword directly.
|
||||
Always use PSKeywordTable.intern().
|
||||
"""
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
return
|
||||
|
||||
def __repr__(self):
|
||||
name = self.name
|
||||
return "/%r" % name
|
||||
|
||||
|
||||
## PSSymbolTable
|
||||
##
|
||||
class PSSymbolTable(object):
|
||||
|
||||
"""A utility class for storing PSLiteral/PSKeyword objects.
|
||||
|
||||
Interned objects can be checked its identity with "is" operator.
|
||||
"""
|
||||
|
||||
def __init__(self, klass):
|
||||
self.dict = {}
|
||||
self.klass = klass
|
||||
return
|
||||
|
||||
def intern(self, name):
|
||||
if name in self.dict:
|
||||
lit = self.dict[name]
|
||||
else:
|
||||
lit = self.klass(name)
|
||||
self.dict[name] = lit
|
||||
return lit
|
||||
+145
-7
@@ -1,21 +1,27 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
|
||||
from decimal import Decimal
|
||||
from typing import List
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.lexer.exceptions import LexerError
|
||||
from pdfmajor.lexer.number import parse_number
|
||||
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
|
||||
from pdfmajor.lexer.string import parse_string
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.lexer.name import parse_name
|
||||
from pdfmajor.lexer.keyword import parse_keyword
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
from pdfmajor.lexer.token import (
|
||||
TDictVaue,
|
||||
TDictValue,
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenDictionary,
|
||||
TokenHexString,
|
||||
TokenInteger,
|
||||
TokenKeyword,
|
||||
TokenName,
|
||||
TokenString,
|
||||
)
|
||||
@@ -23,7 +29,7 @@ from pdfmajor.lexer.comment import parse_comment
|
||||
|
||||
|
||||
def make_stream_iter(data: bytes, buf_size: int):
|
||||
return SafeBufferIt(io.BytesIO(data), buf_size)
|
||||
return BufferStream(io.BytesIO(data), buf_size)
|
||||
|
||||
|
||||
class Basics(TestCase):
|
||||
@@ -156,7 +162,7 @@ class Basics(TestCase):
|
||||
token = parse_double_angled_bracket(buffer)
|
||||
self.assertIsInstance(token, TokenDictionary)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, TDictVaue.OPEN)
|
||||
self.assertEqual(token.value, TDictValue.OPEN)
|
||||
self.assertEqual(token.end_loc, 2)
|
||||
|
||||
def test_parse_dd_to_hex(self):
|
||||
@@ -190,3 +196,135 @@ class Basics(TestCase):
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, 130)
|
||||
self.assertEqual(token.end_loc, 3)
|
||||
|
||||
|
||||
class Lexer(TestCase):
|
||||
def run_test(self, raw: bytes, expected: List[Token]):
|
||||
tokenizer = PDFLexer(io.BytesIO(raw))
|
||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
|
||||
self.assertEqual(token, etoken, f"Failed at example {i}")
|
||||
try:
|
||||
next_token = next(tokenizer.iter_tokens())
|
||||
self.assertIsNone(next_token)
|
||||
except LexerError as e:
|
||||
self.assertIsNotNone(e)
|
||||
|
||||
def test_case1(self):
|
||||
self.run_test(
|
||||
raw=br"""%!PS
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#6eaa
|
||||
0 +1 -2 .5 1.234
|
||||
(abc) () (abc ( def ) ghi)
|
||||
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
||||
(this % is not a comment.)
|
||||
(foo
|
||||
baa)
|
||||
(foo\
|
||||
baa)
|
||||
<> <20> < 40 4020 >
|
||||
<abcd00
|
||||
12345>
|
||||
func/a/b{(c)do*}def
|
||||
[ 1 (z) ! ]
|
||||
<< /foo (bar) >>""",
|
||||
expected=[
|
||||
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
|
||||
TokenKeyword(start_loc=5, end_loc=5, value=b"begin"),
|
||||
TokenKeyword(start_loc=11, end_loc=3, value=b"end"),
|
||||
TokenKeyword(start_loc=16, end_loc=1, value=b'"'),
|
||||
TokenKeyword(start_loc=19, end_loc=1, value=b"@"),
|
||||
TokenKeyword(start_loc=21, end_loc=1, value=b"#"),
|
||||
TokenName(start_loc=24, end_loc=1, value="a"),
|
||||
TokenName(start_loc=26, end_loc=3, value="BCD"),
|
||||
TokenName(start_loc=31, end_loc=9, value="Some_Name"),
|
||||
TokenName(start_loc=42, end_loc=11, value="foo_naa"),
|
||||
TokenInteger(start_loc=54, end_loc=1, value=0),
|
||||
TokenInteger(start_loc=56, end_loc=2, value=1),
|
||||
TokenInteger(start_loc=59, end_loc=2, value=-2),
|
||||
TokenDecimal(start_loc=62, end_loc=2, value=Decimal("0.5")),
|
||||
TokenDecimal(start_loc=65, end_loc=5, value=Decimal("1.234")),
|
||||
TokenString(start_loc=71, end_loc=5, value="abc"),
|
||||
TokenString(start_loc=77, end_loc=2, value=""),
|
||||
TokenString(start_loc=80, end_loc=17, value="abc ( def ) ghi"),
|
||||
TokenString(start_loc=98, end_loc=19, value="def 4ghi"),
|
||||
TokenString(start_loc=118, end_loc=13, value=r"bach\slask"),
|
||||
TokenString(start_loc=132, end_loc=10, value="foo\nbaa"),
|
||||
TokenString(
|
||||
start_loc=143, end_loc=26, value="this % is not a comment."
|
||||
),
|
||||
TokenString(start_loc=170, end_loc=9, value="foo\nbaa"),
|
||||
TokenString(start_loc=180, end_loc=10, value="foobaa"),
|
||||
TokenHexString(start_loc=191, end_loc=2, value=b""),
|
||||
TokenHexString(start_loc=194, end_loc=4, value=b" "),
|
||||
TokenHexString(start_loc=199, end_loc=11, value=b"@@ "),
|
||||
TokenHexString(
|
||||
start_loc=211, end_loc=14, value=b"\xab\xcd\x00\x124\x05"
|
||||
),
|
||||
TokenKeyword(start_loc=226, end_loc=4, value=b"func"),
|
||||
TokenName(start_loc=231, end_loc=1, value="a"),
|
||||
TokenName(start_loc=233, end_loc=1, value="b"),
|
||||
TokenKeyword(start_loc=234, end_loc=1, value=b"{"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case2(self):
|
||||
self.run_test(
|
||||
raw=br"""10.32
|
||||
0 +1 -2 .5 1.234
|
||||
-1.234 990 """,
|
||||
expected=[
|
||||
TokenDecimal(start_loc=0, end_loc=5, value=Decimal("10.32")),
|
||||
TokenInteger(start_loc=6, end_loc=1, value=0),
|
||||
TokenInteger(start_loc=8, end_loc=2, value=1),
|
||||
TokenInteger(start_loc=11, end_loc=2, value=-2),
|
||||
TokenDecimal(start_loc=14, end_loc=2, value=Decimal("0.5")),
|
||||
TokenDecimal(start_loc=17, end_loc=5, value=Decimal("1.234")),
|
||||
TokenDecimal(start_loc=23, end_loc=6, value=Decimal("-1.234")),
|
||||
TokenInteger(start_loc=30, end_loc=3, value=990),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case3(self):
|
||||
self.run_test(
|
||||
raw=br"""/b{(c)do*}def""",
|
||||
expected=[
|
||||
TokenName(start_loc=0, end_loc=2, value="b"),
|
||||
TokenKeyword(start_loc=2, end_loc=3, value=b"{"),
|
||||
TokenString(start_loc=4, end_loc=5, value="c"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case4(self):
|
||||
self.run_test(
|
||||
raw=br"""%!PS
|
||||
% This is some secret sauce
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
|
||||
/need /oh1oh
|
||||
func/a/b
|
||||
""",
|
||||
expected=[
|
||||
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
|
||||
TokenComment(
|
||||
start_loc=5, end_loc=27, value=b" This is some secret sauce"
|
||||
),
|
||||
TokenKeyword(start_loc=33, end_loc=5, value=b"begin"),
|
||||
TokenKeyword(start_loc=39, end_loc=3, value=b"end"),
|
||||
TokenKeyword(start_loc=44, end_loc=1, value=b'"'),
|
||||
TokenKeyword(start_loc=47, end_loc=1, value=b"@"),
|
||||
TokenKeyword(start_loc=49, end_loc=1, value=b"#"),
|
||||
TokenName(start_loc=51, end_loc=2, value="a"),
|
||||
TokenName(start_loc=53, end_loc=4, value="BCD"),
|
||||
TokenName(start_loc=58, end_loc=10, value="Some_Name"),
|
||||
TokenName(start_loc=69, end_loc=12, value="foo_naa"),
|
||||
TokenName(start_loc=83, end_loc=7, value="barj"),
|
||||
TokenName(start_loc=90, end_loc=5, value="need"),
|
||||
TokenName(start_loc=96, end_loc=6, value="oh1oh"),
|
||||
TokenKeyword(start_loc=103, end_loc=4, value=b"func"),
|
||||
TokenName(start_loc=107, end_loc=3, value="a"),
|
||||
TokenName(start_loc=109, end_loc=3, value="b"),
|
||||
],
|
||||
)
|
||||
@@ -1,12 +1,12 @@
|
||||
import io
|
||||
from unittest import TestCase
|
||||
from pdfmajor.safebufiterator import EOF, SafeBufferIt
|
||||
from pdfmajor.safebufiterator import EOF, BufferStream
|
||||
|
||||
|
||||
class UseCase(TestCase):
|
||||
def test_safebuff(self):
|
||||
for bufsize in [2, 33, 65, 66, 100, 200]:
|
||||
it = SafeBufferIt(
|
||||
it = BufferStream(
|
||||
io.BytesIO(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
),
|
||||
@@ -16,7 +16,7 @@ class UseCase(TestCase):
|
||||
self.assertLessEqual(len(subbuf), it.buffer_size)
|
||||
|
||||
def test_itraises(self):
|
||||
it = SafeBufferIt(io.BytesIO(b"this"))
|
||||
it = BufferStream(io.BytesIO(b"this"))
|
||||
list(it) # read everything
|
||||
with self.assertRaises(EOF):
|
||||
for _ in it:
|
||||
@@ -24,7 +24,7 @@ class UseCase(TestCase):
|
||||
|
||||
def test_seekd(self):
|
||||
for bufsize in [2, 33, 65, 66, 100, 200]:
|
||||
it = SafeBufferIt(
|
||||
it = BufferStream(
|
||||
io.BytesIO(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
),
|
||||
|
||||
@@ -1,230 +0,0 @@
|
||||
import io
|
||||
from random import randint
|
||||
from decimal import Decimal
|
||||
from typing import List
|
||||
from pdfmajor.tokenizer.token_parsers.string import parse_string
|
||||
from pdfmajor.tokenizer import PSTokenizer
|
||||
from pdfmajor.tokenizer.token import (
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenInteger,
|
||||
TokenKeyword,
|
||||
TokenLiteral,
|
||||
TokenString,
|
||||
)
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
|
||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
||||
from pdfmajor.tokenizer.token_parsers.number import parse_number
|
||||
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
|
||||
from unittest import TestCase
|
||||
|
||||
|
||||
def make_stream_iter(data: bytes, initpos: int = 0):
|
||||
stream = io.BytesIO(data)
|
||||
|
||||
def iter_stream(buf_size: int):
|
||||
while True:
|
||||
bufpos = stream.tell()
|
||||
buf = stream.read(buf_size)
|
||||
if buf:
|
||||
yield PInput(initpos + bufpos, buf)
|
||||
else:
|
||||
raise "EOF"
|
||||
|
||||
return iter_stream
|
||||
|
||||
|
||||
class Util(TestCase):
|
||||
def test_safebuff(self):
|
||||
it = SafeBufferIt(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
)
|
||||
for subbuf in it.into_iter():
|
||||
self.assertEqual(len(subbuf), len(it.buffer) - it.skip)
|
||||
it.skip += randint(1, max(len(subbuf) - 1, 1))
|
||||
|
||||
|
||||
class Individual(TestCase):
|
||||
def test_comment_parser_ln(self):
|
||||
for buf_size in [2, 7, 11, 150]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
)
|
||||
token = parse_comment(0, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenComment)
|
||||
self.assertEqual(token.pos, 0)
|
||||
self.assertEqual(
|
||||
token.value, b"this is a lengthy comment that ends here"
|
||||
)
|
||||
self.assertEqual(token.size, 40)
|
||||
|
||||
def test_comment_parser_lr(self):
|
||||
for buf_size in [2, 7, 11, 150]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\rso this is not reachable"
|
||||
)
|
||||
token = parse_comment(0, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenComment)
|
||||
self.assertEqual(token.pos, 0)
|
||||
self.assertEqual(
|
||||
token.value, b"this is a lengthy comment that ends here"
|
||||
)
|
||||
self.assertEqual(token.size, 40)
|
||||
|
||||
def test_parse_float(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
|
||||
token = parse_number(5, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenDecimal)
|
||||
self.assertEqual(token.pos, 5)
|
||||
self.assertEqual(token.value, Decimal("0.12"))
|
||||
self.assertEqual(token.size, 5)
|
||||
|
||||
def test_parse_int(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(
|
||||
b"130 oh oh 0.12 + some invalid text", 11
|
||||
)
|
||||
token = parse_number(11, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenInteger)
|
||||
self.assertEqual(token.pos, 11)
|
||||
self.assertEqual(token.value, 130)
|
||||
self.assertEqual(token.size, 3)
|
||||
|
||||
def test_parse_bool_true(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(b"true and the american way", 91)
|
||||
token = parse_keyword(91, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenBoolean)
|
||||
self.assertEqual(token.pos, 91)
|
||||
self.assertEqual(token.value, True)
|
||||
self.assertEqual(token.size, 4)
|
||||
|
||||
def test_parse_bool_false(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(b"false mannn", 33)
|
||||
token = parse_keyword(33, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenBoolean)
|
||||
self.assertEqual(token.pos, 33)
|
||||
self.assertEqual(token.value, False)
|
||||
self.assertEqual(token.size, 5)
|
||||
|
||||
def test_parse_literals(self):
|
||||
for buf_size in [2, 3, 50, 444]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
|
||||
token = parse_literal(71, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenLiteral)
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, "Some_Name")
|
||||
self.assertEqual(token.size, 9)
|
||||
|
||||
def test_parse_literals_hex(self):
|
||||
for buf_size in [2]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(br"foo#5faa ", 71)
|
||||
token = parse_literal(71, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenLiteral)
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, "foo_aa")
|
||||
self.assertEqual(token.size, 8)
|
||||
|
||||
def test_parse_string(self):
|
||||
for buf_size in [2]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
|
||||
token = parse_string(88, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenString)
|
||||
self.assertEqual(token.pos, 88)
|
||||
self.assertEqual(token.value, "this % is not a comment.")
|
||||
self.assertEqual(token.size, 24)
|
||||
|
||||
|
||||
class Tokenizer(TestCase):
|
||||
def run_test(self, raw: bytes, expected: List[Token]):
|
||||
tokenizer = PSTokenizer(io.BytesIO(raw))
|
||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
|
||||
self.assertEqual(token, etoken, f"Failed at example {i}")
|
||||
|
||||
def test_case1(self):
|
||||
tokenizer = PSTokenizer(
|
||||
io.BytesIO(
|
||||
br"""%!PS
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#6eaa
|
||||
0 +1 -2 .5 1.234
|
||||
(abc) () (abc ( def ) ghi)
|
||||
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
||||
(this % is not a comment.)
|
||||
(foo
|
||||
baa)
|
||||
(foo\
|
||||
baa)
|
||||
<> <20> < 40 4020 >
|
||||
<abcd00
|
||||
12345>
|
||||
func/a/b{(c)do*}def
|
||||
[ 1 (z) ! ]
|
||||
<< /foo (bar) >>"""
|
||||
)
|
||||
)
|
||||
for token in tokenizer.iter_tokens():
|
||||
print(token)
|
||||
|
||||
def test_case2(self):
|
||||
self.run_test(
|
||||
raw=br"""10.32
|
||||
0 +1 -2 .5 1.234
|
||||
-1.234 990 """,
|
||||
expected=[
|
||||
TokenDecimal(pos=0, size=5, value=Decimal("10.32")),
|
||||
TokenInteger(pos=6, size=1, value=0),
|
||||
TokenInteger(pos=8, size=2, value=1),
|
||||
TokenInteger(pos=11, size=2, value=-2),
|
||||
TokenDecimal(pos=14, size=2, value=Decimal("0.5")),
|
||||
TokenDecimal(pos=17, size=5, value=Decimal("1.234")),
|
||||
TokenDecimal(pos=23, size=6, value=Decimal("-1.234")),
|
||||
TokenInteger(pos=30, size=3, value=990),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case_words1(self):
|
||||
self.run_test(
|
||||
raw=br"""%!PS
|
||||
% This is some secret sauce
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
|
||||
/need /oh1oh
|
||||
func/a/b
|
||||
""",
|
||||
expected=[
|
||||
TokenComment(pos=0, size=4, value=b"!PS"),
|
||||
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
|
||||
TokenKeyword(pos=33, size=5, value=b"begin"),
|
||||
TokenKeyword(pos=39, size=3, value=b"end"),
|
||||
TokenKeyword(pos=44, size=1, value=b'"'),
|
||||
TokenKeyword(pos=47, size=1, value=b"@"),
|
||||
TokenKeyword(pos=49, size=1, value=b"#"),
|
||||
TokenLiteral(pos=52, size=1, value="a"),
|
||||
TokenLiteral(pos=54, size=3, value="BCD"),
|
||||
TokenLiteral(pos=59, size=9, value="Some_Name"),
|
||||
TokenLiteral(pos=70, size=11, value="foo_naa"),
|
||||
TokenLiteral(pos=83, size=6, value="barj"),
|
||||
TokenLiteral(pos=91, size=4, value="need"),
|
||||
TokenLiteral(pos=97, size=5, value="oh1oh"),
|
||||
TokenKeyword(pos=103, size=4, value=b"func"),
|
||||
TokenLiteral(pos=108, size=1, value="a"),
|
||||
TokenLiteral(pos=110, size=1, value="b"),
|
||||
],
|
||||
)
|
||||
Reference in new issue
Block a user