- clean up

- renamed safebuf to streambuf
- added arrays
This commit is contained in:
ari committed 2020-10-17 15:54:39 -04:00
1 parent c752346638
commit a03857d0f7
29 files changed
+280 -1381

No files matched your search

+2 -2
View File
@@ -1,9 +1,9 @@
root = true
[*]
indent_style = tab
indent_style = space
indent_size = 4
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = false
insert_final_newline = true
insert_final_newline = true
+67 -3
View File
@@ -1,9 +1,73 @@
import io
from typing import Iterator
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket
from pdfmajor.lexer.string import parse_string
from pdfmajor.lexer.keyword import parse_keyword
from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.name import parse_name
from pdfmajor.lexer.comment import parse_comment
from pdfmajor.lexer.regex import NONSPC
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
Token,
TokenDictionary,
TokenKeyword,
TokenArray,
)
from pdfmajor.safebufiterator import BufferStream
class PDFLexer:
def __init__(self, fp: io.BufferedIOBase) -> None:
self.fp = fp
def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
self.buffer = BufferStream(fp, buffer_size=buffer_size)
def seek(self, pos: int):
self.fp.seek(pos)
self.buffer.seek(pos)
def iter_tokens(self) -> Iterator[Token]:
for bufpos, buf in self.buffer:
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = None
continue
else:
j = m.start(0)
next_char = buf[j : j + 1]
if next_char == b"%":
self.buffer.seek(bufpos + j + 1)
yield parse_comment(self.buffer)
elif next_char == b"/":
self.buffer.seek(bufpos + j + 1)
yield parse_name(self.buffer)
elif next_char in b"-+":
self.buffer.seek(bufpos + j + 1)
yield parse_number(self.buffer, next_char)
elif next_char.isdigit() or next_char == b".":
self.buffer.seek(bufpos + j)
yield parse_number(self.buffer)
elif next_char.isalpha():
self.buffer.seek(bufpos + j)
yield parse_keyword(self.buffer)
elif next_char == b"(":
self.buffer.seek(bufpos + j + 1)
yield parse_string(self.buffer)
elif next_char == b"<":
self.buffer.seek(bufpos + j + 1)
yield parse_double_angled_bracket(self.buffer)
elif next_char == b">":
self.buffer.seek(bufpos + j + 1)
if buf[j : j + 2] == ">>":
self.buffer.seekd(1)
yield TokenDictionary(
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
)
elif next_char == b"[":
self.buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
elif next_char == b"]":
self.buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE)
else:
self.buffer.seek(bufpos + j + 1)
yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char)
+3 -3
View File
@@ -3,14 +3,14 @@ from pdfmajor.lexer.regex import EOL
from pdfmajor.lexer.token import (
TokenComment,
)
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.safebufiterator import BufferStream
def parse_comment(buffer: SafeBufferIt) -> TokenComment:
def parse_comment(buffer: BufferStream) -> TokenComment:
"""Parses input stream into a comment
Args:
buffer (SafeBufferIt)
buffer (BufferStream)
Returns:
TokenComment
+7 -8
View File
@@ -1,18 +1,17 @@
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.regex import END_HEX_STRING, HEX_PAIR, SPC
from pdfmajor.lexer.token import TDictVaue, TokenDictionary, TokenHexString
from pdfmajor.lexer.token import TDictValue, TokenDictionary, TokenHexString
from typing import Union
def parse_double_angled_bracket(
buffer: SafeBufferIt,
buffer: BufferStream,
) -> Union[TokenHexString, TokenDictionary]:
"""Parses input stream into a hex string object or match a simple dictionary token
Args:
buffer (SafeBufferIt)
buffer (BufferStream)
Returns:
Union[TokenHexString, TokenDictionary]
@@ -22,16 +21,16 @@ def parse_double_angled_bracket(
first_char = step.data[0:1]
if first_char == b"<":
buffer.seek(step.pos + 1)
return TokenDictionary(initialpos, 2, TDictVaue.OPEN)
return TokenDictionary(initialpos, 2, TDictValue.OPEN)
buffer.seek(step.pos)
return parse_hexstring(buffer)
def parse_hexstring(buffer: SafeBufferIt) -> TokenHexString:
def parse_hexstring(buffer: BufferStream) -> TokenHexString:
"""Parses the input string into hex string
Args:
buffer (SafeBufferIt)
buffer (BufferStream)
Returns:
TokenHexString
+1 -1
View File
@@ -5,7 +5,7 @@ class LexerError(PDFMajorException):
pass
class LexerEOF(LexerError):
class LexerEOF(LexerError, EOFError):
pass
+3 -3
View File
@@ -1,15 +1,15 @@
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword
from pdfmajor.lexer.regex import END_KEYWORD
from typing import Union
def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]:
def parse_keyword(buffer: BufferStream) -> Union[TokenKeyword, TokenBoolean]:
"""Parses input stream into a keyword or bool
Args:
buffer (SafeBufferIt)
buffer (BufferStream)
Returns:
Union[TokenKeyword, TokenBoolean]
+3 -3
View File
@@ -5,7 +5,7 @@ from pdfmajor.lexer.exceptions import (
LexerError,
)
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.token import TokenName
from pdfmajor.lexer.regex import HEX, END_LITERAL
@@ -19,11 +19,11 @@ class LiteralParseState:
hex_value: Optional[bytes] = None
def parse_name(buffer: SafeBufferIt) -> TokenName:
def parse_name(buffer: BufferStream) -> TokenName:
"""Parses input stream into a literal name
Args:
buffer (SafeBufferIt)
buffer (BufferStream)
Returns:
TokenName
+4 -4
View File
@@ -1,21 +1,21 @@
from decimal import Decimal
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError
from pdfmajor.lexer.token import (
TokenDecimal,
TokenNumber,
TokenInteger,
)
from pdfmajor.tokenizer.constants import (
from pdfmajor.lexer.regex import (
END_NUMBER,
)
def parse_number(buffer: SafeBufferIt, sign: bytes = b"+") -> TokenNumber:
def parse_number(buffer: BufferStream, sign: bytes = b"+") -> TokenNumber:
"""Parses input stream into a number
Args:
buffer (SafeBufferIt)
buffer (BufferStream)
sign (str): either a + or -
Returns:
+2
View File
@@ -19,3 +19,5 @@ ESC_STRING = {
END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]")
HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.")
SPC = re.compile(br"\s")
NONSPC = re.compile(br"\S")
END_NUMBER = re.compile(br"[^0-9]")
+4 -4
View File
@@ -3,7 +3,7 @@ from typing import Optional
from pdfmajor.lexer.regex import END_STRING, ESC_STRING, OCT_STRING
from pdfmajor.lexer.token import TokenString
from pdfmajor.lexer.exceptions import LexerEOF, LexerError
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.safebufiterator import BufferStream
@dataclass
@@ -13,11 +13,11 @@ class StringParseState:
escaped_value: Optional[bytes] = None
def parse_string(buffer: SafeBufferIt) -> TokenString:
def parse_string(buffer: BufferStream) -> TokenString:
"""Parses input stream into a simple string object
Args:
buffer (SafeBufferIt)
buffer (BufferStream)
Returns:
TokenString
@@ -55,7 +55,7 @@ def parse_string(buffer: SafeBufferIt) -> TokenString:
raise LexerEOF
def parse_string_escape(buffer: SafeBufferIt, state: StringParseState):
def parse_string_escape(buffer: BufferStream, state: StringParseState):
if state.escaped_value is None:
raise LexerError("state.escaped_value is None")
for pos, buf in buffer:
+16 -3
View File
@@ -100,12 +100,15 @@ class TokenDecimal(Token[Decimal]):
pass
class TDictVaue(Enum):
TokenNumber = Union[TokenInteger, TokenDecimal]
class TDictValue(Enum):
OPEN = "<<"
CLOSE = ">>"
class TokenDictionary(Token[TDictVaue]):
class TokenDictionary(Token[TDictValue]):
"""Token representing the brackets of dictionary
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
"""
@@ -113,4 +116,14 @@ class TokenDictionary(Token[TDictVaue]):
pass
TokenNumber = Union[TokenInteger, TokenDecimal]
class TArrayValue(Enum):
OPEN = "["
CLOSE = "]"
class TokenArray(Token[TArrayValue]):
"""Token representing the brackets of an array
Detection of this type of token can be found in PDF 1.7 spec section 7.3.6
"""
pass
+19 -7
View File
@@ -3,7 +3,7 @@ from typing import NamedTuple
from pdfmajor.execptions import PDFMajorException
class EOF(PDFMajorException):
class StreamEOF(PDFMajorException, EOFError):
def __init__(self) -> None:
super().__init__("reached the end of the file or buffer stream!")
@@ -13,7 +13,7 @@ class BufferedBytes(NamedTuple):
data: bytes
class SafeBufferIt:
class BufferStream:
"""A safe reader for bytes that has a few nice additional utilities"""
__slots__ = ["fp", "buffer_size"]
@@ -50,18 +50,30 @@ class SafeBufferIt:
"""
return self.fp.tell()
def read(self, size: int) -> BufferedBytes:
"""Reads the amount of bytes specified
Args:
size (int): number of bytes to read
Returns:
BufferedBytes
"""
pos = self.tell()
buf = self.fp.read(size)
return BufferedBytes(pos, buf)
def __iter__(self):
if self.fp.read(1) == b"":
raise EOF
raise StreamEOF
self.seekd(-1)
return self
def __next__(self):
pos = self.tell()
buf = self.fp.read(self.buffer_size)
if len(buf) == 0:
bbyte = self.read(self.buffer_size)
if len(bbyte.data) == 0:
raise StopIteration
return BufferedBytes(pos, buf)
return bbyte
def close(self):
self.fp.close()
-120
View File
@@ -1,120 +0,0 @@
import io
from pdfmajor.tokenizer.token_parsers.dictionary_hexstring import (
parse_double_angled_bracket,
)
from pdfmajor.tokenizer.token_parsers.string import parse_string
from typing import Iterator, Optional
from pdfmajor.tokenizer.constants import NONSPC
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
from pdfmajor.tokenizer.token_parsers.number import parse_number
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenKeyword
class PSTokenizer:
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
self.fp = fp
self.bufsize = bufsize
self.bufpos: int = 0
self.buf: bytes = b""
self.buf_skipstep: Optional[int] = None
def inc_buf_skipstep(self, value: int):
if self.buf_skipstep is None:
self.buf_skipstep = value
else:
self.buf_skipstep += value
def iter_buffer(self) -> Iterator[PInput]:
while True:
if (self.buf_skipstep is not None) and (self.buf_skipstep < len(self.buf)):
yield PInput(
self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :]
)
else:
self.buf_skipstep = None
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.bufsize)
if not self.buf:
raise TokenizerEOF("Unexpected EOF")
else:
yield PInput(self.bufpos, self.buf)
def seek(self, pos: int):
self.fp.seek(pos)
def step_back(self, steps: int = 1):
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
Args:
steps (int, optional): numbers of steps. Defaults to 1.
"""
self.fp.seek(self.fp.tell() - steps * self.bufsize)
def iter_tokens(self) -> Iterator[Token]:
for cur in self.iter_buffer():
bufpos, buf = cur
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = None
continue
else:
j = m.start(0)
c = buf[j : j + 1]
if c == b"%":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_comment(bufpos + j, self.iter_buffer())
)
elif c == b"/":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_literal(bufpos + j, self.iter_buffer())
)
elif c in b"-+":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer(), c)
)
elif c.isdigit():
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer())
)
elif c == b".":
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer())
)
elif c.isalpha():
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_keyword(bufpos + j, self.iter_buffer())
)
elif c == b"(":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_string(bufpos + j, self.iter_buffer())
)
elif c == b"<":
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_double_angled_bracket(bufpos + j, self.iter_buffer())
)
elif c == b">":
self.inc_buf_skipstep(j)
if buf[j : j + 2] == ">>":
self.inc_buf_skipstep(1)
yield TokenDictionary(bufpos + j, 2, TDictVaue.CLOSE)
else:
self.inc_buf_skipstep(j + 1)
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
def _check_token(self, cur: PInput, token: Token) -> Token:
self.inc_buf_skipstep(token.end_loc - cur.pos)
return token
-37
View File
@@ -1,37 +0,0 @@
import re
from .types import PSLiteral, PSSymbolTable, PSKeyword
PSLiteralTable = PSSymbolTable(PSLiteral)
PSKeywordTable = PSSymbolTable(PSKeyword)
LIT = PSLiteralTable.intern
KWD = PSKeywordTable.intern
KEYWORD_PROC_BEGIN = KWD(b"{")
KEYWORD_PROC_END = KWD(b"}")
KEYWORD_ARRAY_BEGIN = KWD(b"[")
KEYWORD_ARRAY_END = KWD(b"]")
KEYWORD_DICT_BEGIN = KWD(b"<<")
KEYWORD_DICT_END = KWD(b">>")
## PSBaseParser
##
EOL = re.compile(br"[\r\n]")
SPC = re.compile(br"\s")
NONSPC = re.compile(br"\S")
HEX = re.compile(br"[0-9a-fA-F]")
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]")
HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.")
END_NUMBER = re.compile(br"[^0-9]")
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
END_STRING = re.compile(br"[()\134]")
OCT_STRING = re.compile(br"[0-7]")
ESC_STRING = {
b"b": 8,
b"t": 9,
b"n": 10,
b"f": 12,
b"r": 13,
b"(": 40,
b")": 41,
b"\\": 92,
}
-23
View File
@@ -1,23 +0,0 @@
from pdfmajor.execptions import PDFMajorException
class TokenizerError(PDFMajorException):
pass
class TokenizerEOF(TokenizerError):
pass
class InvalidToken(TokenizerError):
def __init__(self, pos: int, token: bytes) -> None:
super().__init__(
"Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token))
)
class InvalidHexToken(TokenizerError):
def __init__(self, pos: int, token: bytes) -> None:
super().__init__(
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
)
-341
View File
@@ -1,341 +0,0 @@
from typing import List, Tuple, Union
from io import TextIOWrapper
from pdfmajor.utils import int2byte
from pdfmajor.execptions import PSEOF
from .types import PSLiteral
from .constants import OCT_STRING, ESC_STRING, KEYWORD_DICT_BEGIN, KEYWORD_DICT_END
from .constants import END_NUMBER, END_STRING, END_KEYWORD, END_HEX_STRING
from .constants import SPC, HEX_PAIR
from .constants import EOL, NONSPC, KWD, END_LITERAL, LIT, HEX
class PSTokenizer(object):
"""Tokenizatizes bytes stream to PS-Tokens"""
BUFSIZ = 4096
def __init__(self, fp: TextIOWrapper):
self.fp = fp
self.seek(0)
return
def __repr__(self):
return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos)
def flush(self):
return
def close(self):
self.flush()
return
def tell(self):
return self.bufpos + self.charpos
def poll(self, pos=None, n=80):
pos0 = self.fp.tell()
if not pos:
pos = self.bufpos + self.charpos
self.fp.seek(pos)
self.fp.read(n)
self.fp.seek(pos0)
return
def seek(self, pos: int):
"""Seeks the parser to the given position."""
self.fp.seek(pos)
# reset the status for nextline()
self.bufpos: int = pos
self.buf: bytes = b""
self.charpos: int = 0
# reset the status for nexttoken()
self._current_parse_func = self._parse_main
self._curtoken: bytes = b""
self._curtokenpos: int = 0
self._tokens: List[
Tuple[int, Union[int, float, bool, str, bytes, PSLiteral]]
] = []
return
def fillbuf(self):
if self.charpos < len(self.buf):
return
# fetch next chunk.
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.BUFSIZ)
if not self.buf:
raise PSEOF("Unexpected EOF")
self.charpos = 0
return
def nextline(self):
"""Fetches a next line that ends either with \\r or \\n."""
linebuf = b""
linepos = self.bufpos + self.charpos
eol = False
while 1:
self.fillbuf()
if eol:
c = self.buf[self.charpos : self.charpos + 1]
# handle b'\r\n'
if c == b"\n":
linebuf += c
self.charpos += 1
break
m = EOL.search(self.buf, self.charpos)
if m:
linebuf += self.buf[self.charpos : m.end(0)]
self.charpos = m.end(0)
if linebuf[-1:] == b"\r":
eol = True
else:
break
else:
linebuf += self.buf[self.charpos :]
self.charpos = len(self.buf)
return (linepos, linebuf)
def revreadlines(self):
"""Fetches a next line backword.
This is used to locate the trailers at the end of a file.
"""
self.fp.seek(0, 2)
pos = self.fp.tell()
buf = b""
while 0 < pos:
prevpos = pos
pos = max(0, pos - self.BUFSIZ)
self.fp.seek(pos)
s = self.fp.read(prevpos - pos)
if not s:
break
while 1:
n = max(s.rfind(b"\r"), s.rfind(b"\n"))
if n == -1:
buf = s + buf
break
yield s[n:] + buf
s = s[:n]
buf = b""
return
def _parse_main(self, s, i):
m = NONSPC.search(s, i)
if not m:
return len(s)
j = m.start(0)
c = s[j : j + 1]
self._curtokenpos = self.bufpos + j
if c == b"%":
self._curtoken = b"%"
self._current_parse_func = self._parse_comment
return j + 1
elif c == b"/":
self._curtoken = b""
self._current_parse_func = self._parse_literal
return j + 1
elif c in b"-+" or c.isdigit():
self._curtoken = c
self._current_parse_func = self._parse_number
return j + 1
elif c == b".":
self._curtoken = c
self._current_parse_func = self._parse_float
return j + 1
elif c.isalpha():
self._curtoken = c
self._current_parse_func = self._parse_keyword
return j + 1
elif c == b"(":
self._curtoken = b""
self.paren = 1
self._current_parse_func = self._parse_string
return j + 1
elif c == b"<":
self._curtoken = b""
self._current_parse_func = self._parse_wopen
return j + 1
elif c == b">":
self._curtoken = b""
self._current_parse_func = self._parse_wclose
return j + 1
else:
self._add_token(KWD(c))
return j + 1
def _add_token(self, obj):
self._tokens.append((self._curtokenpos, obj))
return
def _parse_comment(self, s, i):
m = EOL.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
self._current_parse_func = self._parse_main
# We ignore comments.
# self._tokens.append(self._curtoken)
return j
def _parse_literal(self, s, i):
m = END_LITERAL.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
c = s[j : j + 1]
if c == b"#":
self.hex = b""
self._current_parse_func = self._parse_literal_hex
return j + 1
try:
self._curtoken = str(self._curtoken, "utf-8")
except:
pass
self._add_token(LIT(self._curtoken))
self._current_parse_func = self._parse_main
return j
def _parse_literal_hex(self, s, i):
c = s[i : i + 1]
if HEX.match(c) and len(self.hex) < 2:
self.hex += c
return i + 1
if self.hex:
self._curtoken += int2byte(int(self.hex, 16))
self._current_parse_func = self._parse_literal
return i
def _parse_number(self, s, i):
m = END_NUMBER.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
c = s[j : j + 1]
if c == b".":
self._curtoken += c
self._current_parse_func = self._parse_float
return j + 1
try:
self._add_token(int(self._curtoken))
except ValueError:
pass
self._current_parse_func = self._parse_main
return j
def _parse_float(self, s, i):
m = END_NUMBER.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
try:
self._add_token(float(self._curtoken))
except ValueError:
pass
self._current_parse_func = self._parse_main
return j
def _parse_keyword(self, s, i):
m = END_KEYWORD.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
if self._curtoken == b"true":
token = True
elif self._curtoken == b"false":
token = False
else:
token = KWD(self._curtoken)
self._add_token(token)
self._current_parse_func = self._parse_main
return j
def _parse_string(self, s, i):
m = END_STRING.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
c = s[j : j + 1]
if c == b"\\":
self.oct = b""
self._current_parse_func = self._parse_string_1
return j + 1
if c == b"(":
self.paren += 1
self._curtoken += c
return j + 1
if c == b")":
self.paren -= 1
if self.paren: # WTF, they said balanced parens need no special treatment.
self._curtoken += c
return j + 1
self._add_token(self._curtoken)
self._current_parse_func = self._parse_main
return j + 1
def _parse_string_1(self, s, i):
c = s[i : i + 1]
if OCT_STRING.match(c) and len(self.oct) < 3:
self.oct += c
return i + 1
if self.oct:
self._curtoken += int2byte(int(self.oct, 8))
self._current_parse_func = self._parse_string
return i
if c in ESC_STRING:
self._curtoken += int2byte(ESC_STRING[c])
self._current_parse_func = self._parse_string
return i + 1
def _parse_wopen(self, s, i):
c = s[i : i + 1]
if c == b"<":
self._add_token(KEYWORD_DICT_BEGIN)
self._current_parse_func = self._parse_main
i += 1
else:
self._current_parse_func = self._parse_hexstring
return i
def _parse_wclose(self, s, i):
c = s[i : i + 1]
if c == b">":
self._add_token(KEYWORD_DICT_END)
i += 1
self._current_parse_func = self._parse_main
return i
def _parse_hexstring(self, s, i):
m = END_HEX_STRING.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
token = HEX_PAIR.sub(
lambda m: int2byte(int(m.group(0), 16)), SPC.sub(b"", self._curtoken)
)
self._add_token(token)
self._current_parse_func = self._parse_main
return j
def nexttoken(self):
while not self._tokens:
self.fillbuf()
self.charpos = self._current_parse_func(self.buf, self.charpos)
token = self._tokens.pop(0)
return token
-112
View File
@@ -1,112 +0,0 @@
from enum import Enum
from typing import Any, Generic, TypeVar, Union
from decimal import Decimal
T = TypeVar("T")
class Token(Generic[T]):
__slots__ = ["pos", "size", "value"]
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
self.start_loc = start_loc
self.end_loc = end_loc
self.value = value
def __repr__(self) -> str:
return "{cls}(start={start_loc}, end={end_loc}, value='{value}')".format(
cls=self.__class__.__name__,
start_loc=self.start_loc,
value=self.value,
end_loc=self.end_loc,
)
def __eq__(self, other: Any):
if isinstance(other, self.__class__):
return (
(other.value == self.value)
and (other.start_loc == self.start_loc)
and (other.end_loc == self.end_loc)
)
else:
return False
class TokenComment(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
"""
pass
class TokenKeyword(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec
"""
pass
class TokenLiteral(Token[str]):
"""Token representing literal name tokens
Detection of this type of token can be found in PDF 1.7 spec section 7.3.5
"""
pass
class TokenBoolean(Token[bool]):
"""Token representing PDF booleans
Detection of this type of token can be found in PDF 1.7 spec section 7.3.2
"""
pass
class TokenString(Token[str]):
"""Token representing PDF literal string
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2
"""
pass
class TokenHexString(Token[bytes]):
"""Token representing PDF hex strings
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3
"""
pass
class TokenInteger(Token[int]):
"""Token representing PDF integers
Detection of this type of token can be found in PDF 1.7
"""
pass
class TokenDecimal(Token[Decimal]):
"""Token representing PDF decimals
Detection of this type of token can be found in PDF 1.7
"""
pass
class TDictVaue(Enum):
OPEN = "<<"
CLOSE = ">>"
class TokenDictionary(Token[TDictVaue]):
"""Token representing the brackets of dictionary
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
"""
pass
TokenNumber = Union[TokenInteger, TokenDecimal]
Whitespace-only changes.
@@ -1,31 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token import (
TokenComment,
)
from pdfmajor.tokenizer.constants import (
EOL,
)
from typing import Iterator
from .util import cmp_tsize, PInput
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
"""Parses input stream into a comment
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenComment
"""
curtoken = b""
for curpos, s in inp:
m = EOL.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
return TokenComment(initialpos, curpos + j, curtoken)
raise TokenizerEOF
@@ -1,46 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerError
from pdfmajor.tokenizer.constants import END_HEX_STRING, HEX_PAIR, SPC
from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenHexString
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.utils import int2byte
from typing import Iterator, Union
def parse_double_angled_bracket(initialpos: int, inp: Iterator[PInput]) -> Token:
"""Parses input stream into a simple string object
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
Union[TokenHexString, TokenDictionary]
"""
step = next(inp)
first_char = step.buf[0:2]
if first_char == b"<<":
return TokenDictionary(initialpos, 2, TDictVaue.OPEN)
curtoken = parse_hexstring(initialpos, PInput(step.pos + 1, step.buf[1:]), b"")
if isinstance(curtoken, TokenHexString):
return curtoken
else:
for step in inp:
curtoken = parse_hexstring(initialpos, step, curtoken)
if isinstance(curtoken, TokenHexString):
return curtoken
raise TokenizerError("parse_double_angled_bracket ended")
def parse_hexstring(
initialpos: int, inp: PInput, curtoken: bytes
) -> Union[bytes, TokenHexString]:
m = END_HEX_STRING.search(inp.buf, 0)
if not m:
curtoken += inp.buf
return curtoken
j = m.start(0)
curtoken += inp.buf[:j]
token = HEX_PAIR.sub(
lambda x: int2byte(int(x.group(0), 16)), SPC.sub(b"", curtoken)
)
return TokenHexString(initialpos, inp.pos + j + 1, token)
@@ -1,34 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword
from pdfmajor.tokenizer.constants import END_KEYWORD
from typing import Iterator, Union
from .util import PInput
def parse_keyword(
initialpos: int, inp: Iterator[PInput]
) -> Union[TokenKeyword, TokenBoolean]:
"""Parses input stream into a keyword or bool
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
Union[TokenKeyword, TokenBoolean]
"""
curtoken = b""
for curpos, s in inp:
m = END_KEYWORD.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
if curtoken == b"true":
return TokenBoolean(initialpos, curpos + j, True)
elif curtoken == b"false":
return TokenBoolean(initialpos, curpos + j, False)
else:
return TokenKeyword(initialpos, curpos + j, curtoken)
raise TokenizerEOF
@@ -1,67 +0,0 @@
from dataclasses import dataclass
from pdfmajor.tokenizer.exceptions import (
InvalidHexToken,
TokenizerEOF,
)
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token import TokenLiteral
from pdfmajor.utils import int2byte
from ..constants import HEX, END_LITERAL
from typing import Iterator, NamedTuple, Optional, Union
@dataclass
class LiteralParseState:
curtoken: bytes
hex_value: Optional[bytes] = None
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
"""Parses input stream into a literal
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenLiteral
"""
state = LiteralParseState(b"")
for curpos, buf in inp:
it = SafeBufferIt(buf)
for subbuf in it.into_iter():
if state.hex_value is not None:
for ci in range(len(subbuf)):
c = subbuf[ci : ci + 1]
if HEX.match(c) and len(state.hex_value) < 2:
state.hex_value += c
it.skip += 1
if len(state.hex_value) >= 2:
state.curtoken += int2byte(int(state.hex_value, 16))
state.hex_value = None
break
else:
raise InvalidHexToken(curpos + ci, c)
else:
m = END_LITERAL.search(subbuf, 0)
if not m:
state.curtoken += subbuf
it.skip = len(buf)
else:
j: int = m.start(0)
state.curtoken += subbuf[:j]
c = subbuf[j : j + 1]
if c == b"#":
state.hex_value = b""
it.skip = j + 1
continue
else:
return TokenLiteral(
initialpos,
curpos + it.skip + j,
state.curtoken.decode(),
)
raise TokenizerEOF
@@ -1,68 +0,0 @@
from decimal import Decimal
from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF, TokenizerError
from pdfmajor.tokenizer.token import (
TokenDecimal,
TokenNumber,
TokenInteger,
)
from pdfmajor.tokenizer.constants import (
END_NUMBER,
)
from typing import Iterator
from .util import PInput
def parse_number(
initialpos: int, inp: Iterator[PInput], sign: bytes = b"+"
) -> TokenNumber:
"""Parses input stream into a number
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
sign (str): either a + or -
Returns:
TokenNumber
"""
if sign not in [b"+", b"-"]:
raise TokenizerError("Number was provided an invalid sign {!r}".format(sign))
curtoken = b"" + sign
is_dec = False
for curpos, s in inp:
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if (c == b".") and not is_dec:
is_dec = True
curtoken += c
s = s[j + 1 :]
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
continue
else:
nj = m.start(0)
curtoken += s[:nj]
j += nj
return TokenDecimal(
initialpos,
curpos + j + 1,
Decimal(curtoken.decode()),
)
try:
if is_dec:
return TokenDecimal(
initialpos,
curpos + j,
Decimal(curtoken.decode()),
)
else:
return TokenInteger(initialpos, curpos + j, int(curtoken))
except ValueError:
raise InvalidToken(initialpos, curtoken)
raise TokenizerEOF
@@ -1,92 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerError
import re
from dataclasses import dataclass
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
from pdfmajor.tokenizer.token import (
TokenString,
)
from pdfmajor.tokenizer.constants import (
END_STRING,
ESC_STRING,
)
from typing import Iterator, Optional
OCT_STRING = re.compile(br"^[0-7]{1,3}$")
@dataclass
class StringParseState:
curtoken: bytes
paren: int
escaped_value: Optional[bytes] = None
def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString:
"""Parses input stream into a simple string object
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenString
"""
state = StringParseState(b"", 1)
for curpos, buf in inp:
it = SafeBufferIt(buf)
for s in it.into_iter():
if state.escaped_value is not None:
parse_string_escape(it, state)
else:
m = END_STRING.search(s, 0)
if not m:
state.curtoken += s
it.skip += len(s)
else:
j = m.start(0)
state.curtoken += s[:j]
next_char = s[j : j + 1]
if next_char == b"\\":
state.escaped_value = b""
it.skip += j + 1
elif next_char == b"(":
state.paren += 1
state.curtoken += next_char
it.skip += j + 1
elif next_char == b")":
state.paren -= 1
if state.paren:
state.curtoken += next_char
it.skip += j + 1
else:
return TokenString(
initialpos,
curpos + it.skip + j + 1,
state.curtoken.decode(),
)
raise TokenizerError("end of parse_string")
def parse_string_escape(it: SafeBufferIt, state: StringParseState):
if state.escaped_value is None:
raise TokenizerError("state.escaped_value is None")
for s in it.into_iter():
for ci in range(len(s)):
c = s[ci : ci + 1]
if len(state.escaped_value) < 3:
it.skip += 1
state.escaped_value += c
if len(state.escaped_value) == 1:
if (ci < 1) and (c in ESC_STRING):
state.curtoken += bytes([ESC_STRING[c]])
state.escaped_value = None
return
elif OCT_STRING.match(state.escaped_value):
if len(state.escaped_value) == 3:
state.curtoken += bytes([int(state.escaped_value, 8)])
state.escaped_value = None
return
else:
it.skip -= 1
state.escaped_value = None
return
-48
View File
@@ -1,48 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token import Token
from typing import Callable, Iterator, NamedTuple
class PInput(NamedTuple):
pos: int
buf: bytes
ParserFunc = Callable[[int, Iterator[PInput]], Token]
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
"""Computes the size of the token based on its positional data
Args:
curpos (int): current position of the buffer
initialpos (int): initial position where the parser started working
tokenend_idx (int): last relevant character index within the last buffered value
Returns:
int: length of index
"""
return curpos - initialpos + tokenend_idx
class SafeBufferIt:
"""A safe while-loop to use on a buffer
this class will make sure that there is no risk of an infinite loop and that the max number of loops is reasonable
Raises:
TokenizerEOF: when we reach a maximum iteration we raise an 'end of file' error
"""
__slots__ = ["skip", "buffer"]
def __init__(self, buffer: bytes, skip: int = 0) -> None:
self.skip = skip
self.buffer = buffer
def into_iter(self) -> Iterator[bytes]:
for i in range(2 * len(self.buffer) + 1):
if i > 2 * len(self.buffer):
raise TokenizerEOF("Max Iteration Reached!")
if self.skip >= len(self.buffer):
break
yield self.buffer[self.skip :]
-80
View File
@@ -1,80 +0,0 @@
## Basic PostScript Types
##
from typing import Union
## PSObject
##
class PSObject(object):
"""Base class for all PS or PDF-related data types."""
pass
## PSLiteral
##
class PSLiteral(PSObject):
"""A class that represents a PostScript literal.
Postscript literals are used as identifiers, such as
variable names, property names and dictionary keys.
Literals are case sensitive and denoted by a preceding
slash sign (e.g. "/Name")
Note: Do not create an instance of PSLiteral directly.
Always use PSLiteralTable.intern().
"""
def __init__(self, name):
self.name = name
def __repr__(self):
name = self.name
return "/%r" % name
## PSKeyword
##
class PSKeyword(PSObject):
"""A class that represents a PostScript keyword.
PostScript keywords are a dozen of predefined words.
Commands and directives in PostScript are expressed by keywords.
They are also used to denote the content boundaries.
Note: Do not create an instance of PSKeyword directly.
Always use PSKeywordTable.intern().
"""
def __init__(self, name):
self.name = name
return
def __repr__(self):
name = self.name
return "/%r" % name
## PSSymbolTable
##
class PSSymbolTable(object):
"""A utility class for storing PSLiteral/PSKeyword objects.
Interned objects can be checked its identity with "is" operator.
"""
def __init__(self, klass):
self.dict = {}
self.klass = klass
return
def intern(self, name):
if name in self.dict:
lit = self.dict[name]
else:
lit = self.klass(name)
self.dict[name] = lit
return lit
+145 -7
View File
@@ -1,21 +1,27 @@
from decimal import Decimal
import io
from decimal import Decimal
from typing import List
from unittest import TestCase
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.exceptions import LexerError
from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
from pdfmajor.lexer.string import parse_string
from unittest import TestCase
from pdfmajor.lexer.name import parse_name
from pdfmajor.lexer.keyword import parse_keyword
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.token import (
TDictVaue,
TDictValue,
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenDictionary,
TokenHexString,
TokenInteger,
TokenKeyword,
TokenName,
TokenString,
)
@@ -23,7 +29,7 @@ from pdfmajor.lexer.comment import parse_comment
def make_stream_iter(data: bytes, buf_size: int):
return SafeBufferIt(io.BytesIO(data), buf_size)
return BufferStream(io.BytesIO(data), buf_size)
class Basics(TestCase):
@@ -156,7 +162,7 @@ class Basics(TestCase):
token = parse_double_angled_bracket(buffer)
self.assertIsInstance(token, TokenDictionary)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, TDictVaue.OPEN)
self.assertEqual(token.value, TDictValue.OPEN)
self.assertEqual(token.end_loc, 2)
def test_parse_dd_to_hex(self):
@@ -190,3 +196,135 @@ class Basics(TestCase):
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, 130)
self.assertEqual(token.end_loc, 3)
class Lexer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PDFLexer(io.BytesIO(raw))
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
self.assertEqual(token, etoken, f"Failed at example {i}")
try:
next_token = next(tokenizer.iter_tokens())
self.assertIsNone(next_token)
except LexerError as e:
self.assertIsNotNone(e)
def test_case1(self):
self.run_test(
raw=br"""%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>""",
expected=[
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
TokenKeyword(start_loc=5, end_loc=5, value=b"begin"),
TokenKeyword(start_loc=11, end_loc=3, value=b"end"),
TokenKeyword(start_loc=16, end_loc=1, value=b'"'),
TokenKeyword(start_loc=19, end_loc=1, value=b"@"),
TokenKeyword(start_loc=21, end_loc=1, value=b"#"),
TokenName(start_loc=24, end_loc=1, value="a"),
TokenName(start_loc=26, end_loc=3, value="BCD"),
TokenName(start_loc=31, end_loc=9, value="Some_Name"),
TokenName(start_loc=42, end_loc=11, value="foo_naa"),
TokenInteger(start_loc=54, end_loc=1, value=0),
TokenInteger(start_loc=56, end_loc=2, value=1),
TokenInteger(start_loc=59, end_loc=2, value=-2),
TokenDecimal(start_loc=62, end_loc=2, value=Decimal("0.5")),
TokenDecimal(start_loc=65, end_loc=5, value=Decimal("1.234")),
TokenString(start_loc=71, end_loc=5, value="abc"),
TokenString(start_loc=77, end_loc=2, value=""),
TokenString(start_loc=80, end_loc=17, value="abc ( def ) ghi"),
TokenString(start_loc=98, end_loc=19, value="def 4ghi"),
TokenString(start_loc=118, end_loc=13, value=r"bach\slask"),
TokenString(start_loc=132, end_loc=10, value="foo\nbaa"),
TokenString(
start_loc=143, end_loc=26, value="this % is not a comment."
),
TokenString(start_loc=170, end_loc=9, value="foo\nbaa"),
TokenString(start_loc=180, end_loc=10, value="foobaa"),
TokenHexString(start_loc=191, end_loc=2, value=b""),
TokenHexString(start_loc=194, end_loc=4, value=b" "),
TokenHexString(start_loc=199, end_loc=11, value=b"@@ "),
TokenHexString(
start_loc=211, end_loc=14, value=b"\xab\xcd\x00\x124\x05"
),
TokenKeyword(start_loc=226, end_loc=4, value=b"func"),
TokenName(start_loc=231, end_loc=1, value="a"),
TokenName(start_loc=233, end_loc=1, value="b"),
TokenKeyword(start_loc=234, end_loc=1, value=b"{"),
],
)
def test_case2(self):
self.run_test(
raw=br"""10.32
0 +1 -2 .5 1.234
-1.234 990 """,
expected=[
TokenDecimal(start_loc=0, end_loc=5, value=Decimal("10.32")),
TokenInteger(start_loc=6, end_loc=1, value=0),
TokenInteger(start_loc=8, end_loc=2, value=1),
TokenInteger(start_loc=11, end_loc=2, value=-2),
TokenDecimal(start_loc=14, end_loc=2, value=Decimal("0.5")),
TokenDecimal(start_loc=17, end_loc=5, value=Decimal("1.234")),
TokenDecimal(start_loc=23, end_loc=6, value=Decimal("-1.234")),
TokenInteger(start_loc=30, end_loc=3, value=990),
],
)
def test_case3(self):
self.run_test(
raw=br"""/b{(c)do*}def""",
expected=[
TokenName(start_loc=0, end_loc=2, value="b"),
TokenKeyword(start_loc=2, end_loc=3, value=b"{"),
TokenString(start_loc=4, end_loc=5, value="c"),
],
)
def test_case4(self):
self.run_test(
raw=br"""%!PS
% This is some secret sauce
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
/need /oh1oh
func/a/b
""",
expected=[
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
TokenComment(
start_loc=5, end_loc=27, value=b" This is some secret sauce"
),
TokenKeyword(start_loc=33, end_loc=5, value=b"begin"),
TokenKeyword(start_loc=39, end_loc=3, value=b"end"),
TokenKeyword(start_loc=44, end_loc=1, value=b'"'),
TokenKeyword(start_loc=47, end_loc=1, value=b"@"),
TokenKeyword(start_loc=49, end_loc=1, value=b"#"),
TokenName(start_loc=51, end_loc=2, value="a"),
TokenName(start_loc=53, end_loc=4, value="BCD"),
TokenName(start_loc=58, end_loc=10, value="Some_Name"),
TokenName(start_loc=69, end_loc=12, value="foo_naa"),
TokenName(start_loc=83, end_loc=7, value="barj"),
TokenName(start_loc=90, end_loc=5, value="need"),
TokenName(start_loc=96, end_loc=6, value="oh1oh"),
TokenKeyword(start_loc=103, end_loc=4, value=b"func"),
TokenName(start_loc=107, end_loc=3, value="a"),
TokenName(start_loc=109, end_loc=3, value="b"),
],
)
+4 -4
View File
@@ -1,12 +1,12 @@
import io
from unittest import TestCase
from pdfmajor.safebufiterator import EOF, SafeBufferIt
from pdfmajor.safebufiterator import EOF, BufferStream
class UseCase(TestCase):
def test_safebuff(self):
for bufsize in [2, 33, 65, 66, 100, 200]:
it = SafeBufferIt(
it = BufferStream(
io.BytesIO(
b"this is a lengthy comment that ends here\nso this is not reachable"
),
@@ -16,7 +16,7 @@ class UseCase(TestCase):
self.assertLessEqual(len(subbuf), it.buffer_size)
def test_itraises(self):
it = SafeBufferIt(io.BytesIO(b"this"))
it = BufferStream(io.BytesIO(b"this"))
list(it) # read everything
with self.assertRaises(EOF):
for _ in it:
@@ -24,7 +24,7 @@ class UseCase(TestCase):
def test_seekd(self):
for bufsize in [2, 33, 65, 66, 100, 200]:
it = SafeBufferIt(
it = BufferStream(
io.BytesIO(
b"this is a lengthy comment that ends here\nso this is not reachable"
),
-230
View File
@@ -1,230 +0,0 @@
import io
from random import randint
from decimal import Decimal
from typing import List
from pdfmajor.tokenizer.token_parsers.string import parse_string
from pdfmajor.tokenizer import PSTokenizer
from pdfmajor.tokenizer.token import (
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenInteger,
TokenKeyword,
TokenLiteral,
TokenString,
)
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
from pdfmajor.tokenizer.token_parsers.number import parse_number
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
from unittest import TestCase
def make_stream_iter(data: bytes, initpos: int = 0):
stream = io.BytesIO(data)
def iter_stream(buf_size: int):
while True:
bufpos = stream.tell()
buf = stream.read(buf_size)
if buf:
yield PInput(initpos + bufpos, buf)
else:
raise "EOF"
return iter_stream
class Util(TestCase):
def test_safebuff(self):
it = SafeBufferIt(
b"this is a lengthy comment that ends here\nso this is not reachable"
)
for subbuf in it.into_iter():
self.assertEqual(len(subbuf), len(it.buffer) - it.skip)
it.skip += randint(1, max(len(subbuf) - 1, 1))
class Individual(TestCase):
def test_comment_parser_ln(self):
for buf_size in [2, 7, 11, 150]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\nso this is not reachable"
)
token = parse_comment(0, iter_stream(buf_size))
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.pos, 0)
self.assertEqual(
token.value, b"this is a lengthy comment that ends here"
)
self.assertEqual(token.size, 40)
def test_comment_parser_lr(self):
for buf_size in [2, 7, 11, 150]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\rso this is not reachable"
)
token = parse_comment(0, iter_stream(buf_size))
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.pos, 0)
self.assertEqual(
token.value, b"this is a lengthy comment that ends here"
)
self.assertEqual(token.size, 40)
def test_parse_float(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
token = parse_number(5, iter_stream(buf_size))
self.assertIsInstance(token, TokenDecimal)
self.assertEqual(token.pos, 5)
self.assertEqual(token.value, Decimal("0.12"))
self.assertEqual(token.size, 5)
def test_parse_int(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"130 oh oh 0.12 + some invalid text", 11
)
token = parse_number(11, iter_stream(buf_size))
self.assertIsInstance(token, TokenInteger)
self.assertEqual(token.pos, 11)
self.assertEqual(token.value, 130)
self.assertEqual(token.size, 3)
def test_parse_bool_true(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"true and the american way", 91)
token = parse_keyword(91, iter_stream(buf_size))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 91)
self.assertEqual(token.value, True)
self.assertEqual(token.size, 4)
def test_parse_bool_false(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"false mannn", 33)
token = parse_keyword(33, iter_stream(buf_size))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 33)
self.assertEqual(token.value, False)
self.assertEqual(token.size, 5)
def test_parse_literals(self):
for buf_size in [2, 3, 50, 444]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
token = parse_literal(71, iter_stream(buf_size))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, "Some_Name")
self.assertEqual(token.size, 9)
def test_parse_literals_hex(self):
for buf_size in [2]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"foo#5faa ", 71)
token = parse_literal(71, iter_stream(buf_size))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, "foo_aa")
self.assertEqual(token.size, 8)
def test_parse_string(self):
for buf_size in [2]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
token = parse_string(88, iter_stream(buf_size))
self.assertIsInstance(token, TokenString)
self.assertEqual(token.pos, 88)
self.assertEqual(token.value, "this % is not a comment.")
self.assertEqual(token.size, 24)
class Tokenizer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PSTokenizer(io.BytesIO(raw))
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
self.assertEqual(token, etoken, f"Failed at example {i}")
def test_case1(self):
tokenizer = PSTokenizer(
io.BytesIO(
br"""%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>"""
)
)
for token in tokenizer.iter_tokens():
print(token)
def test_case2(self):
self.run_test(
raw=br"""10.32
0 +1 -2 .5 1.234
-1.234 990 """,
expected=[
TokenDecimal(pos=0, size=5, value=Decimal("10.32")),
TokenInteger(pos=6, size=1, value=0),
TokenInteger(pos=8, size=2, value=1),
TokenInteger(pos=11, size=2, value=-2),
TokenDecimal(pos=14, size=2, value=Decimal("0.5")),
TokenDecimal(pos=17, size=5, value=Decimal("1.234")),
TokenDecimal(pos=23, size=6, value=Decimal("-1.234")),
TokenInteger(pos=30, size=3, value=990),
],
)
def test_case_words1(self):
self.run_test(
raw=br"""%!PS
% This is some secret sauce
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
/need /oh1oh
func/a/b
""",
expected=[
TokenComment(pos=0, size=4, value=b"!PS"),
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
TokenKeyword(pos=33, size=5, value=b"begin"),
TokenKeyword(pos=39, size=3, value=b"end"),
TokenKeyword(pos=44, size=1, value=b'"'),
TokenKeyword(pos=47, size=1, value=b"@"),
TokenKeyword(pos=49, size=1, value=b"#"),
TokenLiteral(pos=52, size=1, value="a"),
TokenLiteral(pos=54, size=3, value="BCD"),
TokenLiteral(pos=59, size=9, value="Some_Name"),
TokenLiteral(pos=70, size=11, value="foo_naa"),
TokenLiteral(pos=83, size=6, value="barj"),
TokenLiteral(pos=91, size=4, value="need"),
TokenLiteral(pos=97, size=5, value="oh1oh"),
TokenKeyword(pos=103, size=4, value=b"func"),
TokenLiteral(pos=108, size=1, value="a"),
TokenLiteral(pos=110, size=1, value="b"),
],
)