- clean up

- renamed safebuf to streambuf
- added arrays
This commit is contained in:
ari committed 2020-10-17 15:54:39 -04:00
1 parent c752346638
commit a03857d0f7
29 files changed
+280 -1381

No files matched your search

+2 -2
View File
@@ -1,9 +1,9 @@
root = true root = true
[*] [*]
indent_style = tab indent_style = space
indent_size = 4 indent_size = 4
end_of_line = lf end_of_line = lf
charset = utf-8 charset = utf-8
trim_trailing_whitespace = false trim_trailing_whitespace = false
insert_final_newline = true insert_final_newline = true
+67 -3
View File
@@ -1,9 +1,73 @@
import io import io
from typing import Iterator
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket
from pdfmajor.lexer.string import parse_string
from pdfmajor.lexer.keyword import parse_keyword
from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.name import parse_name
from pdfmajor.lexer.comment import parse_comment
from pdfmajor.lexer.regex import NONSPC
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
Token,
TokenDictionary,
TokenKeyword,
TokenArray,
)
from pdfmajor.safebufiterator import BufferStream
class PDFLexer: class PDFLexer:
def __init__(self, fp: io.BufferedIOBase) -> None: def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None:
self.fp = fp self.buffer = BufferStream(fp, buffer_size=buffer_size)
def seek(self, pos: int): def seek(self, pos: int):
self.fp.seek(pos) self.buffer.seek(pos)
def iter_tokens(self) -> Iterator[Token]:
for bufpos, buf in self.buffer:
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = None
continue
else:
j = m.start(0)
next_char = buf[j : j + 1]
if next_char == b"%":
self.buffer.seek(bufpos + j + 1)
yield parse_comment(self.buffer)
elif next_char == b"/":
self.buffer.seek(bufpos + j + 1)
yield parse_name(self.buffer)
elif next_char in b"-+":
self.buffer.seek(bufpos + j + 1)
yield parse_number(self.buffer, next_char)
elif next_char.isdigit() or next_char == b".":
self.buffer.seek(bufpos + j)
yield parse_number(self.buffer)
elif next_char.isalpha():
self.buffer.seek(bufpos + j)
yield parse_keyword(self.buffer)
elif next_char == b"(":
self.buffer.seek(bufpos + j + 1)
yield parse_string(self.buffer)
elif next_char == b"<":
self.buffer.seek(bufpos + j + 1)
yield parse_double_angled_bracket(self.buffer)
elif next_char == b">":
self.buffer.seek(bufpos + j + 1)
if buf[j : j + 2] == ">>":
self.buffer.seekd(1)
yield TokenDictionary(
bufpos + j, bufpos + j + 2, TDictValue.CLOSE
)
elif next_char == b"[":
self.buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN)
elif next_char == b"]":
self.buffer.seek(bufpos + j + 1)
yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE)
else:
self.buffer.seek(bufpos + j + 1)
yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char)
+3 -3
View File
@@ -3,14 +3,14 @@ from pdfmajor.lexer.regex import EOL
from pdfmajor.lexer.token import ( from pdfmajor.lexer.token import (
TokenComment, TokenComment,
) )
from pdfmajor.safebufiterator import SafeBufferIt from pdfmajor.safebufiterator import BufferStream
def parse_comment(buffer: SafeBufferIt) -> TokenComment: def parse_comment(buffer: BufferStream) -> TokenComment:
"""Parses input stream into a comment """Parses input stream into a comment
Args: Args:
buffer (SafeBufferIt) buffer (BufferStream)
Returns: Returns:
TokenComment TokenComment
+7 -8
View File
@@ -1,18 +1,17 @@
from pdfmajor.tokenizer.token_parsers.util import PInput from pdfmajor.safebufiterator import BufferStream
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.lexer.exceptions import LexerEOF from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.regex import END_HEX_STRING, HEX_PAIR, SPC from pdfmajor.lexer.regex import END_HEX_STRING, HEX_PAIR, SPC
from pdfmajor.lexer.token import TDictVaue, TokenDictionary, TokenHexString from pdfmajor.lexer.token import TDictValue, TokenDictionary, TokenHexString
from typing import Union from typing import Union
def parse_double_angled_bracket( def parse_double_angled_bracket(
buffer: SafeBufferIt, buffer: BufferStream,
) -> Union[TokenHexString, TokenDictionary]: ) -> Union[TokenHexString, TokenDictionary]:
"""Parses input stream into a hex string object or match a simple dictionary token """Parses input stream into a hex string object or match a simple dictionary token
Args: Args:
buffer (SafeBufferIt) buffer (BufferStream)
Returns: Returns:
Union[TokenHexString, TokenDictionary] Union[TokenHexString, TokenDictionary]
@@ -22,16 +21,16 @@ def parse_double_angled_bracket(
first_char = step.data[0:1] first_char = step.data[0:1]
if first_char == b"<": if first_char == b"<":
buffer.seek(step.pos + 1) buffer.seek(step.pos + 1)
return TokenDictionary(initialpos, 2, TDictVaue.OPEN) return TokenDictionary(initialpos, 2, TDictValue.OPEN)
buffer.seek(step.pos) buffer.seek(step.pos)
return parse_hexstring(buffer) return parse_hexstring(buffer)
def parse_hexstring(buffer: SafeBufferIt) -> TokenHexString: def parse_hexstring(buffer: BufferStream) -> TokenHexString:
"""Parses the input string into hex string """Parses the input string into hex string
Args: Args:
buffer (SafeBufferIt) buffer (BufferStream)
Returns: Returns:
TokenHexString TokenHexString
+1 -1
View File
@@ -5,7 +5,7 @@ class LexerError(PDFMajorException):
pass pass
class LexerEOF(LexerError): class LexerEOF(LexerError, EOFError):
pass pass
+3 -3
View File
@@ -1,15 +1,15 @@
from pdfmajor.safebufiterator import SafeBufferIt from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.exceptions import LexerEOF from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword from pdfmajor.lexer.token import TokenBoolean, TokenKeyword
from pdfmajor.lexer.regex import END_KEYWORD from pdfmajor.lexer.regex import END_KEYWORD
from typing import Union from typing import Union
def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]: def parse_keyword(buffer: BufferStream) -> Union[TokenKeyword, TokenBoolean]:
"""Parses input stream into a keyword or bool """Parses input stream into a keyword or bool
Args: Args:
buffer (SafeBufferIt) buffer (BufferStream)
Returns: Returns:
Union[TokenKeyword, TokenBoolean] Union[TokenKeyword, TokenBoolean]
+3 -3
View File
@@ -5,7 +5,7 @@ from pdfmajor.lexer.exceptions import (
LexerError, LexerError,
) )
from pdfmajor.safebufiterator import SafeBufferIt from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.token import TokenName from pdfmajor.lexer.token import TokenName
from pdfmajor.lexer.regex import HEX, END_LITERAL from pdfmajor.lexer.regex import HEX, END_LITERAL
@@ -19,11 +19,11 @@ class LiteralParseState:
hex_value: Optional[bytes] = None hex_value: Optional[bytes] = None
def parse_name(buffer: SafeBufferIt) -> TokenName: def parse_name(buffer: BufferStream) -> TokenName:
"""Parses input stream into a literal name """Parses input stream into a literal name
Args: Args:
buffer (SafeBufferIt) buffer (BufferStream)
Returns: Returns:
TokenName TokenName
+4 -4
View File
@@ -1,21 +1,21 @@
from decimal import Decimal from decimal import Decimal
from pdfmajor.safebufiterator import SafeBufferIt from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError
from pdfmajor.lexer.token import ( from pdfmajor.lexer.token import (
TokenDecimal, TokenDecimal,
TokenNumber, TokenNumber,
TokenInteger, TokenInteger,
) )
from pdfmajor.tokenizer.constants import ( from pdfmajor.lexer.regex import (
END_NUMBER, END_NUMBER,
) )
def parse_number(buffer: SafeBufferIt, sign: bytes = b"+") -> TokenNumber: def parse_number(buffer: BufferStream, sign: bytes = b"+") -> TokenNumber:
"""Parses input stream into a number """Parses input stream into a number
Args: Args:
buffer (SafeBufferIt) buffer (BufferStream)
sign (str): either a + or - sign (str): either a + or -
Returns: Returns:
+2
View File
@@ -19,3 +19,5 @@ ESC_STRING = {
END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]") END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]")
HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.") HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.")
SPC = re.compile(br"\s") SPC = re.compile(br"\s")
NONSPC = re.compile(br"\S")
END_NUMBER = re.compile(br"[^0-9]")
+4 -4
View File
@@ -3,7 +3,7 @@ from typing import Optional
from pdfmajor.lexer.regex import END_STRING, ESC_STRING, OCT_STRING from pdfmajor.lexer.regex import END_STRING, ESC_STRING, OCT_STRING
from pdfmajor.lexer.token import TokenString from pdfmajor.lexer.token import TokenString
from pdfmajor.lexer.exceptions import LexerEOF, LexerError from pdfmajor.lexer.exceptions import LexerEOF, LexerError
from pdfmajor.safebufiterator import SafeBufferIt from pdfmajor.safebufiterator import BufferStream
@dataclass @dataclass
@@ -13,11 +13,11 @@ class StringParseState:
escaped_value: Optional[bytes] = None escaped_value: Optional[bytes] = None
def parse_string(buffer: SafeBufferIt) -> TokenString: def parse_string(buffer: BufferStream) -> TokenString:
"""Parses input stream into a simple string object """Parses input stream into a simple string object
Args: Args:
buffer (SafeBufferIt) buffer (BufferStream)
Returns: Returns:
TokenString TokenString
@@ -55,7 +55,7 @@ def parse_string(buffer: SafeBufferIt) -> TokenString:
raise LexerEOF raise LexerEOF
def parse_string_escape(buffer: SafeBufferIt, state: StringParseState): def parse_string_escape(buffer: BufferStream, state: StringParseState):
if state.escaped_value is None: if state.escaped_value is None:
raise LexerError("state.escaped_value is None") raise LexerError("state.escaped_value is None")
for pos, buf in buffer: for pos, buf in buffer:
+16 -3
View File
@@ -100,12 +100,15 @@ class TokenDecimal(Token[Decimal]):
pass pass
class TDictVaue(Enum): TokenNumber = Union[TokenInteger, TokenDecimal]
class TDictValue(Enum):
OPEN = "<<" OPEN = "<<"
CLOSE = ">>" CLOSE = ">>"
class TokenDictionary(Token[TDictVaue]): class TokenDictionary(Token[TDictValue]):
"""Token representing the brackets of dictionary """Token representing the brackets of dictionary
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7 Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
""" """
@@ -113,4 +116,14 @@ class TokenDictionary(Token[TDictVaue]):
pass pass
TokenNumber = Union[TokenInteger, TokenDecimal] class TArrayValue(Enum):
OPEN = "["
CLOSE = "]"
class TokenArray(Token[TArrayValue]):
"""Token representing the brackets of an array
Detection of this type of token can be found in PDF 1.7 spec section 7.3.6
"""
pass
+19 -7
View File
@@ -3,7 +3,7 @@ from typing import NamedTuple
from pdfmajor.execptions import PDFMajorException from pdfmajor.execptions import PDFMajorException
class EOF(PDFMajorException): class StreamEOF(PDFMajorException, EOFError):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__("reached the end of the file or buffer stream!") super().__init__("reached the end of the file or buffer stream!")
@@ -13,7 +13,7 @@ class BufferedBytes(NamedTuple):
data: bytes data: bytes
class SafeBufferIt: class BufferStream:
"""A safe reader for bytes that has a few nice additional utilities""" """A safe reader for bytes that has a few nice additional utilities"""
__slots__ = ["fp", "buffer_size"] __slots__ = ["fp", "buffer_size"]
@@ -50,18 +50,30 @@ class SafeBufferIt:
""" """
return self.fp.tell() return self.fp.tell()
def read(self, size: int) -> BufferedBytes:
"""Reads the amount of bytes specified
Args:
size (int): number of bytes to read
Returns:
BufferedBytes
"""
pos = self.tell()
buf = self.fp.read(size)
return BufferedBytes(pos, buf)
def __iter__(self): def __iter__(self):
if self.fp.read(1) == b"": if self.fp.read(1) == b"":
raise EOF raise StreamEOF
self.seekd(-1) self.seekd(-1)
return self return self
def __next__(self): def __next__(self):
pos = self.tell() bbyte = self.read(self.buffer_size)
buf = self.fp.read(self.buffer_size) if len(bbyte.data) == 0:
if len(buf) == 0:
raise StopIteration raise StopIteration
return BufferedBytes(pos, buf) return bbyte
def close(self): def close(self):
self.fp.close() self.fp.close()
-120
View File
@@ -1,120 +0,0 @@
import io
from pdfmajor.tokenizer.token_parsers.dictionary_hexstring import (
parse_double_angled_bracket,
)
from pdfmajor.tokenizer.token_parsers.string import parse_string
from typing import Iterator, Optional
from pdfmajor.tokenizer.constants import NONSPC
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
from pdfmajor.tokenizer.token_parsers.number import parse_number
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenKeyword
class PSTokenizer:
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
self.fp = fp
self.bufsize = bufsize
self.bufpos: int = 0
self.buf: bytes = b""
self.buf_skipstep: Optional[int] = None
def inc_buf_skipstep(self, value: int):
if self.buf_skipstep is None:
self.buf_skipstep = value
else:
self.buf_skipstep += value
def iter_buffer(self) -> Iterator[PInput]:
while True:
if (self.buf_skipstep is not None) and (self.buf_skipstep < len(self.buf)):
yield PInput(
self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :]
)
else:
self.buf_skipstep = None
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.bufsize)
if not self.buf:
raise TokenizerEOF("Unexpected EOF")
else:
yield PInput(self.bufpos, self.buf)
def seek(self, pos: int):
self.fp.seek(pos)
def step_back(self, steps: int = 1):
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
Args:
steps (int, optional): numbers of steps. Defaults to 1.
"""
self.fp.seek(self.fp.tell() - steps * self.bufsize)
def iter_tokens(self) -> Iterator[Token]:
for cur in self.iter_buffer():
bufpos, buf = cur
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = None
continue
else:
j = m.start(0)
c = buf[j : j + 1]
if c == b"%":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_comment(bufpos + j, self.iter_buffer())
)
elif c == b"/":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_literal(bufpos + j, self.iter_buffer())
)
elif c in b"-+":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer(), c)
)
elif c.isdigit():
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer())
)
elif c == b".":
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer())
)
elif c.isalpha():
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_keyword(bufpos + j, self.iter_buffer())
)
elif c == b"(":
self.inc_buf_skipstep(j + 1)
yield self._check_token(
cur, parse_string(bufpos + j, self.iter_buffer())
)
elif c == b"<":
self.inc_buf_skipstep(j)
yield self._check_token(
cur, parse_double_angled_bracket(bufpos + j, self.iter_buffer())
)
elif c == b">":
self.inc_buf_skipstep(j)
if buf[j : j + 2] == ">>":
self.inc_buf_skipstep(1)
yield TokenDictionary(bufpos + j, 2, TDictVaue.CLOSE)
else:
self.inc_buf_skipstep(j + 1)
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
def _check_token(self, cur: PInput, token: Token) -> Token:
self.inc_buf_skipstep(token.end_loc - cur.pos)
return token
-37
View File
@@ -1,37 +0,0 @@
import re
from .types import PSLiteral, PSSymbolTable, PSKeyword
PSLiteralTable = PSSymbolTable(PSLiteral)
PSKeywordTable = PSSymbolTable(PSKeyword)
LIT = PSLiteralTable.intern
KWD = PSKeywordTable.intern
KEYWORD_PROC_BEGIN = KWD(b"{")
KEYWORD_PROC_END = KWD(b"}")
KEYWORD_ARRAY_BEGIN = KWD(b"[")
KEYWORD_ARRAY_END = KWD(b"]")
KEYWORD_DICT_BEGIN = KWD(b"<<")
KEYWORD_DICT_END = KWD(b">>")
## PSBaseParser
##
EOL = re.compile(br"[\r\n]")
SPC = re.compile(br"\s")
NONSPC = re.compile(br"\S")
HEX = re.compile(br"[0-9a-fA-F]")
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]")
HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.")
END_NUMBER = re.compile(br"[^0-9]")
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
END_STRING = re.compile(br"[()\134]")
OCT_STRING = re.compile(br"[0-7]")
ESC_STRING = {
b"b": 8,
b"t": 9,
b"n": 10,
b"f": 12,
b"r": 13,
b"(": 40,
b")": 41,
b"\\": 92,
}
-23
View File
@@ -1,23 +0,0 @@
from pdfmajor.execptions import PDFMajorException
class TokenizerError(PDFMajorException):
pass
class TokenizerEOF(TokenizerError):
pass
class InvalidToken(TokenizerError):
def __init__(self, pos: int, token: bytes) -> None:
super().__init__(
"Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token))
)
class InvalidHexToken(TokenizerError):
def __init__(self, pos: int, token: bytes) -> None:
super().__init__(
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
)
-341
View File
@@ -1,341 +0,0 @@
from typing import List, Tuple, Union
from io import TextIOWrapper
from pdfmajor.utils import int2byte
from pdfmajor.execptions import PSEOF
from .types import PSLiteral
from .constants import OCT_STRING, ESC_STRING, KEYWORD_DICT_BEGIN, KEYWORD_DICT_END
from .constants import END_NUMBER, END_STRING, END_KEYWORD, END_HEX_STRING
from .constants import SPC, HEX_PAIR
from .constants import EOL, NONSPC, KWD, END_LITERAL, LIT, HEX
class PSTokenizer(object):
"""Tokenizatizes bytes stream to PS-Tokens"""
BUFSIZ = 4096
def __init__(self, fp: TextIOWrapper):
self.fp = fp
self.seek(0)
return
def __repr__(self):
return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos)
def flush(self):
return
def close(self):
self.flush()
return
def tell(self):
return self.bufpos + self.charpos
def poll(self, pos=None, n=80):
pos0 = self.fp.tell()
if not pos:
pos = self.bufpos + self.charpos
self.fp.seek(pos)
self.fp.read(n)
self.fp.seek(pos0)
return
def seek(self, pos: int):
"""Seeks the parser to the given position."""
self.fp.seek(pos)
# reset the status for nextline()
self.bufpos: int = pos
self.buf: bytes = b""
self.charpos: int = 0
# reset the status for nexttoken()
self._current_parse_func = self._parse_main
self._curtoken: bytes = b""
self._curtokenpos: int = 0
self._tokens: List[
Tuple[int, Union[int, float, bool, str, bytes, PSLiteral]]
] = []
return
def fillbuf(self):
if self.charpos < len(self.buf):
return
# fetch next chunk.
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.BUFSIZ)
if not self.buf:
raise PSEOF("Unexpected EOF")
self.charpos = 0
return
def nextline(self):
"""Fetches a next line that ends either with \\r or \\n."""
linebuf = b""
linepos = self.bufpos + self.charpos
eol = False
while 1:
self.fillbuf()
if eol:
c = self.buf[self.charpos : self.charpos + 1]
# handle b'\r\n'
if c == b"\n":
linebuf += c
self.charpos += 1
break
m = EOL.search(self.buf, self.charpos)
if m:
linebuf += self.buf[self.charpos : m.end(0)]
self.charpos = m.end(0)
if linebuf[-1:] == b"\r":
eol = True
else:
break
else:
linebuf += self.buf[self.charpos :]
self.charpos = len(self.buf)
return (linepos, linebuf)
def revreadlines(self):
"""Fetches a next line backword.
This is used to locate the trailers at the end of a file.
"""
self.fp.seek(0, 2)
pos = self.fp.tell()
buf = b""
while 0 < pos:
prevpos = pos
pos = max(0, pos - self.BUFSIZ)
self.fp.seek(pos)
s = self.fp.read(prevpos - pos)
if not s:
break
while 1:
n = max(s.rfind(b"\r"), s.rfind(b"\n"))
if n == -1:
buf = s + buf
break
yield s[n:] + buf
s = s[:n]
buf = b""
return
def _parse_main(self, s, i):
m = NONSPC.search(s, i)
if not m:
return len(s)
j = m.start(0)
c = s[j : j + 1]
self._curtokenpos = self.bufpos + j
if c == b"%":
self._curtoken = b"%"
self._current_parse_func = self._parse_comment
return j + 1
elif c == b"/":
self._curtoken = b""
self._current_parse_func = self._parse_literal
return j + 1
elif c in b"-+" or c.isdigit():
self._curtoken = c
self._current_parse_func = self._parse_number
return j + 1
elif c == b".":
self._curtoken = c
self._current_parse_func = self._parse_float
return j + 1
elif c.isalpha():
self._curtoken = c
self._current_parse_func = self._parse_keyword
return j + 1
elif c == b"(":
self._curtoken = b""
self.paren = 1
self._current_parse_func = self._parse_string
return j + 1
elif c == b"<":
self._curtoken = b""
self._current_parse_func = self._parse_wopen
return j + 1
elif c == b">":
self._curtoken = b""
self._current_parse_func = self._parse_wclose
return j + 1
else:
self._add_token(KWD(c))
return j + 1
def _add_token(self, obj):
self._tokens.append((self._curtokenpos, obj))
return
def _parse_comment(self, s, i):
m = EOL.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
self._current_parse_func = self._parse_main
# We ignore comments.
# self._tokens.append(self._curtoken)
return j
def _parse_literal(self, s, i):
m = END_LITERAL.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
c = s[j : j + 1]
if c == b"#":
self.hex = b""
self._current_parse_func = self._parse_literal_hex
return j + 1
try:
self._curtoken = str(self._curtoken, "utf-8")
except:
pass
self._add_token(LIT(self._curtoken))
self._current_parse_func = self._parse_main
return j
def _parse_literal_hex(self, s, i):
c = s[i : i + 1]
if HEX.match(c) and len(self.hex) < 2:
self.hex += c
return i + 1
if self.hex:
self._curtoken += int2byte(int(self.hex, 16))
self._current_parse_func = self._parse_literal
return i
def _parse_number(self, s, i):
m = END_NUMBER.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
c = s[j : j + 1]
if c == b".":
self._curtoken += c
self._current_parse_func = self._parse_float
return j + 1
try:
self._add_token(int(self._curtoken))
except ValueError:
pass
self._current_parse_func = self._parse_main
return j
def _parse_float(self, s, i):
m = END_NUMBER.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
try:
self._add_token(float(self._curtoken))
except ValueError:
pass
self._current_parse_func = self._parse_main
return j
def _parse_keyword(self, s, i):
m = END_KEYWORD.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
if self._curtoken == b"true":
token = True
elif self._curtoken == b"false":
token = False
else:
token = KWD(self._curtoken)
self._add_token(token)
self._current_parse_func = self._parse_main
return j
def _parse_string(self, s, i):
m = END_STRING.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
c = s[j : j + 1]
if c == b"\\":
self.oct = b""
self._current_parse_func = self._parse_string_1
return j + 1
if c == b"(":
self.paren += 1
self._curtoken += c
return j + 1
if c == b")":
self.paren -= 1
if self.paren: # WTF, they said balanced parens need no special treatment.
self._curtoken += c
return j + 1
self._add_token(self._curtoken)
self._current_parse_func = self._parse_main
return j + 1
def _parse_string_1(self, s, i):
c = s[i : i + 1]
if OCT_STRING.match(c) and len(self.oct) < 3:
self.oct += c
return i + 1
if self.oct:
self._curtoken += int2byte(int(self.oct, 8))
self._current_parse_func = self._parse_string
return i
if c in ESC_STRING:
self._curtoken += int2byte(ESC_STRING[c])
self._current_parse_func = self._parse_string
return i + 1
def _parse_wopen(self, s, i):
c = s[i : i + 1]
if c == b"<":
self._add_token(KEYWORD_DICT_BEGIN)
self._current_parse_func = self._parse_main
i += 1
else:
self._current_parse_func = self._parse_hexstring
return i
def _parse_wclose(self, s, i):
c = s[i : i + 1]
if c == b">":
self._add_token(KEYWORD_DICT_END)
i += 1
self._current_parse_func = self._parse_main
return i
def _parse_hexstring(self, s, i):
m = END_HEX_STRING.search(s, i)
if not m:
self._curtoken += s[i:]
return len(s)
j = m.start(0)
self._curtoken += s[i:j]
token = HEX_PAIR.sub(
lambda m: int2byte(int(m.group(0), 16)), SPC.sub(b"", self._curtoken)
)
self._add_token(token)
self._current_parse_func = self._parse_main
return j
def nexttoken(self):
while not self._tokens:
self.fillbuf()
self.charpos = self._current_parse_func(self.buf, self.charpos)
token = self._tokens.pop(0)
return token
-112
View File
@@ -1,112 +0,0 @@
from enum import Enum
from typing import Any, Generic, TypeVar, Union
from decimal import Decimal
T = TypeVar("T")
class Token(Generic[T]):
__slots__ = ["pos", "size", "value"]
def __init__(self, start_loc: int, end_loc: int, value: T) -> None:
self.start_loc = start_loc
self.end_loc = end_loc
self.value = value
def __repr__(self) -> str:
return "{cls}(start={start_loc}, end={end_loc}, value='{value}')".format(
cls=self.__class__.__name__,
start_loc=self.start_loc,
value=self.value,
end_loc=self.end_loc,
)
def __eq__(self, other: Any):
if isinstance(other, self.__class__):
return (
(other.value == self.value)
and (other.start_loc == self.start_loc)
and (other.end_loc == self.end_loc)
)
else:
return False
class TokenComment(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec section 7.2.3
"""
pass
class TokenKeyword(Token[bytes]):
"""Token representing PDF comments
Detection of this type of token can be found in PDF 1.7 spec
"""
pass
class TokenLiteral(Token[str]):
"""Token representing literal name tokens
Detection of this type of token can be found in PDF 1.7 spec section 7.3.5
"""
pass
class TokenBoolean(Token[bool]):
"""Token representing PDF booleans
Detection of this type of token can be found in PDF 1.7 spec section 7.3.2
"""
pass
class TokenString(Token[str]):
"""Token representing PDF literal string
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2
"""
pass
class TokenHexString(Token[bytes]):
"""Token representing PDF hex strings
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3
"""
pass
class TokenInteger(Token[int]):
"""Token representing PDF integers
Detection of this type of token can be found in PDF 1.7
"""
pass
class TokenDecimal(Token[Decimal]):
"""Token representing PDF decimals
Detection of this type of token can be found in PDF 1.7
"""
pass
class TDictVaue(Enum):
OPEN = "<<"
CLOSE = ">>"
class TokenDictionary(Token[TDictVaue]):
"""Token representing the brackets of dictionary
Detection of this type of token can be found in PDF 1.7 spec section 7.3.7
"""
pass
TokenNumber = Union[TokenInteger, TokenDecimal]
Whitespace-only changes.
@@ -1,31 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token import (
TokenComment,
)
from pdfmajor.tokenizer.constants import (
EOL,
)
from typing import Iterator
from .util import cmp_tsize, PInput
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
"""Parses input stream into a comment
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenComment
"""
curtoken = b""
for curpos, s in inp:
m = EOL.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
return TokenComment(initialpos, curpos + j, curtoken)
raise TokenizerEOF
@@ -1,46 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerError
from pdfmajor.tokenizer.constants import END_HEX_STRING, HEX_PAIR, SPC
from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenHexString
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.utils import int2byte
from typing import Iterator, Union
def parse_double_angled_bracket(initialpos: int, inp: Iterator[PInput]) -> Token:
"""Parses input stream into a simple string object
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
Union[TokenHexString, TokenDictionary]
"""
step = next(inp)
first_char = step.buf[0:2]
if first_char == b"<<":
return TokenDictionary(initialpos, 2, TDictVaue.OPEN)
curtoken = parse_hexstring(initialpos, PInput(step.pos + 1, step.buf[1:]), b"")
if isinstance(curtoken, TokenHexString):
return curtoken
else:
for step in inp:
curtoken = parse_hexstring(initialpos, step, curtoken)
if isinstance(curtoken, TokenHexString):
return curtoken
raise TokenizerError("parse_double_angled_bracket ended")
def parse_hexstring(
initialpos: int, inp: PInput, curtoken: bytes
) -> Union[bytes, TokenHexString]:
m = END_HEX_STRING.search(inp.buf, 0)
if not m:
curtoken += inp.buf
return curtoken
j = m.start(0)
curtoken += inp.buf[:j]
token = HEX_PAIR.sub(
lambda x: int2byte(int(x.group(0), 16)), SPC.sub(b"", curtoken)
)
return TokenHexString(initialpos, inp.pos + j + 1, token)
@@ -1,34 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword
from pdfmajor.tokenizer.constants import END_KEYWORD
from typing import Iterator, Union
from .util import PInput
def parse_keyword(
initialpos: int, inp: Iterator[PInput]
) -> Union[TokenKeyword, TokenBoolean]:
"""Parses input stream into a keyword or bool
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
Union[TokenKeyword, TokenBoolean]
"""
curtoken = b""
for curpos, s in inp:
m = END_KEYWORD.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
if curtoken == b"true":
return TokenBoolean(initialpos, curpos + j, True)
elif curtoken == b"false":
return TokenBoolean(initialpos, curpos + j, False)
else:
return TokenKeyword(initialpos, curpos + j, curtoken)
raise TokenizerEOF
@@ -1,67 +0,0 @@
from dataclasses import dataclass
from pdfmajor.tokenizer.exceptions import (
InvalidHexToken,
TokenizerEOF,
)
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token import TokenLiteral
from pdfmajor.utils import int2byte
from ..constants import HEX, END_LITERAL
from typing import Iterator, NamedTuple, Optional, Union
@dataclass
class LiteralParseState:
curtoken: bytes
hex_value: Optional[bytes] = None
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
"""Parses input stream into a literal
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenLiteral
"""
state = LiteralParseState(b"")
for curpos, buf in inp:
it = SafeBufferIt(buf)
for subbuf in it.into_iter():
if state.hex_value is not None:
for ci in range(len(subbuf)):
c = subbuf[ci : ci + 1]
if HEX.match(c) and len(state.hex_value) < 2:
state.hex_value += c
it.skip += 1
if len(state.hex_value) >= 2:
state.curtoken += int2byte(int(state.hex_value, 16))
state.hex_value = None
break
else:
raise InvalidHexToken(curpos + ci, c)
else:
m = END_LITERAL.search(subbuf, 0)
if not m:
state.curtoken += subbuf
it.skip = len(buf)
else:
j: int = m.start(0)
state.curtoken += subbuf[:j]
c = subbuf[j : j + 1]
if c == b"#":
state.hex_value = b""
it.skip = j + 1
continue
else:
return TokenLiteral(
initialpos,
curpos + it.skip + j,
state.curtoken.decode(),
)
raise TokenizerEOF
@@ -1,68 +0,0 @@
from decimal import Decimal
from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF, TokenizerError
from pdfmajor.tokenizer.token import (
TokenDecimal,
TokenNumber,
TokenInteger,
)
from pdfmajor.tokenizer.constants import (
END_NUMBER,
)
from typing import Iterator
from .util import PInput
def parse_number(
initialpos: int, inp: Iterator[PInput], sign: bytes = b"+"
) -> TokenNumber:
"""Parses input stream into a number
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
sign (str): either a + or -
Returns:
TokenNumber
"""
if sign not in [b"+", b"-"]:
raise TokenizerError("Number was provided an invalid sign {!r}".format(sign))
curtoken = b"" + sign
is_dec = False
for curpos, s in inp:
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if (c == b".") and not is_dec:
is_dec = True
curtoken += c
s = s[j + 1 :]
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
continue
else:
nj = m.start(0)
curtoken += s[:nj]
j += nj
return TokenDecimal(
initialpos,
curpos + j + 1,
Decimal(curtoken.decode()),
)
try:
if is_dec:
return TokenDecimal(
initialpos,
curpos + j,
Decimal(curtoken.decode()),
)
else:
return TokenInteger(initialpos, curpos + j, int(curtoken))
except ValueError:
raise InvalidToken(initialpos, curtoken)
raise TokenizerEOF
@@ -1,92 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerError
import re
from dataclasses import dataclass
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
from pdfmajor.tokenizer.token import (
TokenString,
)
from pdfmajor.tokenizer.constants import (
END_STRING,
ESC_STRING,
)
from typing import Iterator, Optional
OCT_STRING = re.compile(br"^[0-7]{1,3}$")
@dataclass
class StringParseState:
curtoken: bytes
paren: int
escaped_value: Optional[bytes] = None
def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString:
"""Parses input stream into a simple string object
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenString
"""
state = StringParseState(b"", 1)
for curpos, buf in inp:
it = SafeBufferIt(buf)
for s in it.into_iter():
if state.escaped_value is not None:
parse_string_escape(it, state)
else:
m = END_STRING.search(s, 0)
if not m:
state.curtoken += s
it.skip += len(s)
else:
j = m.start(0)
state.curtoken += s[:j]
next_char = s[j : j + 1]
if next_char == b"\\":
state.escaped_value = b""
it.skip += j + 1
elif next_char == b"(":
state.paren += 1
state.curtoken += next_char
it.skip += j + 1
elif next_char == b")":
state.paren -= 1
if state.paren:
state.curtoken += next_char
it.skip += j + 1
else:
return TokenString(
initialpos,
curpos + it.skip + j + 1,
state.curtoken.decode(),
)
raise TokenizerError("end of parse_string")
def parse_string_escape(it: SafeBufferIt, state: StringParseState):
if state.escaped_value is None:
raise TokenizerError("state.escaped_value is None")
for s in it.into_iter():
for ci in range(len(s)):
c = s[ci : ci + 1]
if len(state.escaped_value) < 3:
it.skip += 1
state.escaped_value += c
if len(state.escaped_value) == 1:
if (ci < 1) and (c in ESC_STRING):
state.curtoken += bytes([ESC_STRING[c]])
state.escaped_value = None
return
elif OCT_STRING.match(state.escaped_value):
if len(state.escaped_value) == 3:
state.curtoken += bytes([int(state.escaped_value, 8)])
state.escaped_value = None
return
else:
it.skip -= 1
state.escaped_value = None
return
-48
View File
@@ -1,48 +0,0 @@
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token import Token
from typing import Callable, Iterator, NamedTuple
class PInput(NamedTuple):
pos: int
buf: bytes
ParserFunc = Callable[[int, Iterator[PInput]], Token]
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
"""Computes the size of the token based on its positional data
Args:
curpos (int): current position of the buffer
initialpos (int): initial position where the parser started working
tokenend_idx (int): last relevant character index within the last buffered value
Returns:
int: length of index
"""
return curpos - initialpos + tokenend_idx
class SafeBufferIt:
"""A safe while-loop to use on a buffer
this class will make sure that there is no risk of an infinite loop and that the max number of loops is reasonable
Raises:
TokenizerEOF: when we reach a maximum iteration we raise an 'end of file' error
"""
__slots__ = ["skip", "buffer"]
def __init__(self, buffer: bytes, skip: int = 0) -> None:
self.skip = skip
self.buffer = buffer
def into_iter(self) -> Iterator[bytes]:
for i in range(2 * len(self.buffer) + 1):
if i > 2 * len(self.buffer):
raise TokenizerEOF("Max Iteration Reached!")
if self.skip >= len(self.buffer):
break
yield self.buffer[self.skip :]
-80
View File
@@ -1,80 +0,0 @@
## Basic PostScript Types
##
from typing import Union
## PSObject
##
class PSObject(object):
"""Base class for all PS or PDF-related data types."""
pass
## PSLiteral
##
class PSLiteral(PSObject):
"""A class that represents a PostScript literal.
Postscript literals are used as identifiers, such as
variable names, property names and dictionary keys.
Literals are case sensitive and denoted by a preceding
slash sign (e.g. "/Name")
Note: Do not create an instance of PSLiteral directly.
Always use PSLiteralTable.intern().
"""
def __init__(self, name):
self.name = name
def __repr__(self):
name = self.name
return "/%r" % name
## PSKeyword
##
class PSKeyword(PSObject):
"""A class that represents a PostScript keyword.
PostScript keywords are a dozen of predefined words.
Commands and directives in PostScript are expressed by keywords.
They are also used to denote the content boundaries.
Note: Do not create an instance of PSKeyword directly.
Always use PSKeywordTable.intern().
"""
def __init__(self, name):
self.name = name
return
def __repr__(self):
name = self.name
return "/%r" % name
## PSSymbolTable
##
class PSSymbolTable(object):
"""A utility class for storing PSLiteral/PSKeyword objects.
Interned objects can be checked its identity with "is" operator.
"""
def __init__(self, klass):
self.dict = {}
self.klass = klass
return
def intern(self, name):
if name in self.dict:
lit = self.dict[name]
else:
lit = self.klass(name)
self.dict[name] = lit
return lit
+145 -7
View File
@@ -1,21 +1,27 @@
from decimal import Decimal
import io import io
from decimal import Decimal
from typing import List
from unittest import TestCase
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.exceptions import LexerError
from pdfmajor.lexer.number import parse_number from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
from pdfmajor.lexer.string import parse_string from pdfmajor.lexer.string import parse_string
from unittest import TestCase
from pdfmajor.lexer.name import parse_name from pdfmajor.lexer.name import parse_name
from pdfmajor.lexer.keyword import parse_keyword from pdfmajor.lexer.keyword import parse_keyword
from pdfmajor.safebufiterator import SafeBufferIt from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.token import ( from pdfmajor.lexer.token import (
TDictVaue, TDictValue,
Token,
TokenBoolean, TokenBoolean,
TokenComment, TokenComment,
TokenDecimal, TokenDecimal,
TokenDictionary, TokenDictionary,
TokenHexString, TokenHexString,
TokenInteger, TokenInteger,
TokenKeyword,
TokenName, TokenName,
TokenString, TokenString,
) )
@@ -23,7 +29,7 @@ from pdfmajor.lexer.comment import parse_comment
def make_stream_iter(data: bytes, buf_size: int): def make_stream_iter(data: bytes, buf_size: int):
return SafeBufferIt(io.BytesIO(data), buf_size) return BufferStream(io.BytesIO(data), buf_size)
class Basics(TestCase): class Basics(TestCase):
@@ -156,7 +162,7 @@ class Basics(TestCase):
token = parse_double_angled_bracket(buffer) token = parse_double_angled_bracket(buffer)
self.assertIsInstance(token, TokenDictionary) self.assertIsInstance(token, TokenDictionary)
self.assertEqual(token.start_loc, -1) self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, TDictVaue.OPEN) self.assertEqual(token.value, TDictValue.OPEN)
self.assertEqual(token.end_loc, 2) self.assertEqual(token.end_loc, 2)
def test_parse_dd_to_hex(self): def test_parse_dd_to_hex(self):
@@ -190,3 +196,135 @@ class Basics(TestCase):
self.assertEqual(token.start_loc, -1) self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, 130) self.assertEqual(token.value, 130)
self.assertEqual(token.end_loc, 3) self.assertEqual(token.end_loc, 3)
class Lexer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PDFLexer(io.BytesIO(raw))
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
self.assertEqual(token, etoken, f"Failed at example {i}")
try:
next_token = next(tokenizer.iter_tokens())
self.assertIsNone(next_token)
except LexerError as e:
self.assertIsNotNone(e)
def test_case1(self):
self.run_test(
raw=br"""%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>""",
expected=[
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
TokenKeyword(start_loc=5, end_loc=5, value=b"begin"),
TokenKeyword(start_loc=11, end_loc=3, value=b"end"),
TokenKeyword(start_loc=16, end_loc=1, value=b'"'),
TokenKeyword(start_loc=19, end_loc=1, value=b"@"),
TokenKeyword(start_loc=21, end_loc=1, value=b"#"),
TokenName(start_loc=24, end_loc=1, value="a"),
TokenName(start_loc=26, end_loc=3, value="BCD"),
TokenName(start_loc=31, end_loc=9, value="Some_Name"),
TokenName(start_loc=42, end_loc=11, value="foo_naa"),
TokenInteger(start_loc=54, end_loc=1, value=0),
TokenInteger(start_loc=56, end_loc=2, value=1),
TokenInteger(start_loc=59, end_loc=2, value=-2),
TokenDecimal(start_loc=62, end_loc=2, value=Decimal("0.5")),
TokenDecimal(start_loc=65, end_loc=5, value=Decimal("1.234")),
TokenString(start_loc=71, end_loc=5, value="abc"),
TokenString(start_loc=77, end_loc=2, value=""),
TokenString(start_loc=80, end_loc=17, value="abc ( def ) ghi"),
TokenString(start_loc=98, end_loc=19, value="def 4ghi"),
TokenString(start_loc=118, end_loc=13, value=r"bach\slask"),
TokenString(start_loc=132, end_loc=10, value="foo\nbaa"),
TokenString(
start_loc=143, end_loc=26, value="this % is not a comment."
),
TokenString(start_loc=170, end_loc=9, value="foo\nbaa"),
TokenString(start_loc=180, end_loc=10, value="foobaa"),
TokenHexString(start_loc=191, end_loc=2, value=b""),
TokenHexString(start_loc=194, end_loc=4, value=b" "),
TokenHexString(start_loc=199, end_loc=11, value=b"@@ "),
TokenHexString(
start_loc=211, end_loc=14, value=b"\xab\xcd\x00\x124\x05"
),
TokenKeyword(start_loc=226, end_loc=4, value=b"func"),
TokenName(start_loc=231, end_loc=1, value="a"),
TokenName(start_loc=233, end_loc=1, value="b"),
TokenKeyword(start_loc=234, end_loc=1, value=b"{"),
],
)
def test_case2(self):
self.run_test(
raw=br"""10.32
0 +1 -2 .5 1.234
-1.234 990 """,
expected=[
TokenDecimal(start_loc=0, end_loc=5, value=Decimal("10.32")),
TokenInteger(start_loc=6, end_loc=1, value=0),
TokenInteger(start_loc=8, end_loc=2, value=1),
TokenInteger(start_loc=11, end_loc=2, value=-2),
TokenDecimal(start_loc=14, end_loc=2, value=Decimal("0.5")),
TokenDecimal(start_loc=17, end_loc=5, value=Decimal("1.234")),
TokenDecimal(start_loc=23, end_loc=6, value=Decimal("-1.234")),
TokenInteger(start_loc=30, end_loc=3, value=990),
],
)
def test_case3(self):
self.run_test(
raw=br"""/b{(c)do*}def""",
expected=[
TokenName(start_loc=0, end_loc=2, value="b"),
TokenKeyword(start_loc=2, end_loc=3, value=b"{"),
TokenString(start_loc=4, end_loc=5, value="c"),
],
)
def test_case4(self):
self.run_test(
raw=br"""%!PS
% This is some secret sauce
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
/need /oh1oh
func/a/b
""",
expected=[
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
TokenComment(
start_loc=5, end_loc=27, value=b" This is some secret sauce"
),
TokenKeyword(start_loc=33, end_loc=5, value=b"begin"),
TokenKeyword(start_loc=39, end_loc=3, value=b"end"),
TokenKeyword(start_loc=44, end_loc=1, value=b'"'),
TokenKeyword(start_loc=47, end_loc=1, value=b"@"),
TokenKeyword(start_loc=49, end_loc=1, value=b"#"),
TokenName(start_loc=51, end_loc=2, value="a"),
TokenName(start_loc=53, end_loc=4, value="BCD"),
TokenName(start_loc=58, end_loc=10, value="Some_Name"),
TokenName(start_loc=69, end_loc=12, value="foo_naa"),
TokenName(start_loc=83, end_loc=7, value="barj"),
TokenName(start_loc=90, end_loc=5, value="need"),
TokenName(start_loc=96, end_loc=6, value="oh1oh"),
TokenKeyword(start_loc=103, end_loc=4, value=b"func"),
TokenName(start_loc=107, end_loc=3, value="a"),
TokenName(start_loc=109, end_loc=3, value="b"),
],
)
+4 -4
View File
@@ -1,12 +1,12 @@
import io import io
from unittest import TestCase from unittest import TestCase
from pdfmajor.safebufiterator import EOF, SafeBufferIt from pdfmajor.safebufiterator import EOF, BufferStream
class UseCase(TestCase): class UseCase(TestCase):
def test_safebuff(self): def test_safebuff(self):
for bufsize in [2, 33, 65, 66, 100, 200]: for bufsize in [2, 33, 65, 66, 100, 200]:
it = SafeBufferIt( it = BufferStream(
io.BytesIO( io.BytesIO(
b"this is a lengthy comment that ends here\nso this is not reachable" b"this is a lengthy comment that ends here\nso this is not reachable"
), ),
@@ -16,7 +16,7 @@ class UseCase(TestCase):
self.assertLessEqual(len(subbuf), it.buffer_size) self.assertLessEqual(len(subbuf), it.buffer_size)
def test_itraises(self): def test_itraises(self):
it = SafeBufferIt(io.BytesIO(b"this")) it = BufferStream(io.BytesIO(b"this"))
list(it) # read everything list(it) # read everything
with self.assertRaises(EOF): with self.assertRaises(EOF):
for _ in it: for _ in it:
@@ -24,7 +24,7 @@ class UseCase(TestCase):
def test_seekd(self): def test_seekd(self):
for bufsize in [2, 33, 65, 66, 100, 200]: for bufsize in [2, 33, 65, 66, 100, 200]:
it = SafeBufferIt( it = BufferStream(
io.BytesIO( io.BytesIO(
b"this is a lengthy comment that ends here\nso this is not reachable" b"this is a lengthy comment that ends here\nso this is not reachable"
), ),
-230
View File
@@ -1,230 +0,0 @@
import io
from random import randint
from decimal import Decimal
from typing import List
from pdfmajor.tokenizer.token_parsers.string import parse_string
from pdfmajor.tokenizer import PSTokenizer
from pdfmajor.tokenizer.token import (
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenInteger,
TokenKeyword,
TokenLiteral,
TokenString,
)
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
from pdfmajor.tokenizer.token_parsers.number import parse_number
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
from unittest import TestCase
def make_stream_iter(data: bytes, initpos: int = 0):
stream = io.BytesIO(data)
def iter_stream(buf_size: int):
while True:
bufpos = stream.tell()
buf = stream.read(buf_size)
if buf:
yield PInput(initpos + bufpos, buf)
else:
raise "EOF"
return iter_stream
class Util(TestCase):
def test_safebuff(self):
it = SafeBufferIt(
b"this is a lengthy comment that ends here\nso this is not reachable"
)
for subbuf in it.into_iter():
self.assertEqual(len(subbuf), len(it.buffer) - it.skip)
it.skip += randint(1, max(len(subbuf) - 1, 1))
class Individual(TestCase):
def test_comment_parser_ln(self):
for buf_size in [2, 7, 11, 150]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\nso this is not reachable"
)
token = parse_comment(0, iter_stream(buf_size))
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.pos, 0)
self.assertEqual(
token.value, b"this is a lengthy comment that ends here"
)
self.assertEqual(token.size, 40)
def test_comment_parser_lr(self):
for buf_size in [2, 7, 11, 150]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\rso this is not reachable"
)
token = parse_comment(0, iter_stream(buf_size))
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.pos, 0)
self.assertEqual(
token.value, b"this is a lengthy comment that ends here"
)
self.assertEqual(token.size, 40)
def test_parse_float(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
token = parse_number(5, iter_stream(buf_size))
self.assertIsInstance(token, TokenDecimal)
self.assertEqual(token.pos, 5)
self.assertEqual(token.value, Decimal("0.12"))
self.assertEqual(token.size, 5)
def test_parse_int(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"130 oh oh 0.12 + some invalid text", 11
)
token = parse_number(11, iter_stream(buf_size))
self.assertIsInstance(token, TokenInteger)
self.assertEqual(token.pos, 11)
self.assertEqual(token.value, 130)
self.assertEqual(token.size, 3)
def test_parse_bool_true(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"true and the american way", 91)
token = parse_keyword(91, iter_stream(buf_size))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 91)
self.assertEqual(token.value, True)
self.assertEqual(token.size, 4)
def test_parse_bool_false(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"false mannn", 33)
token = parse_keyword(33, iter_stream(buf_size))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 33)
self.assertEqual(token.value, False)
self.assertEqual(token.size, 5)
def test_parse_literals(self):
for buf_size in [2, 3, 50, 444]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
token = parse_literal(71, iter_stream(buf_size))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, "Some_Name")
self.assertEqual(token.size, 9)
def test_parse_literals_hex(self):
for buf_size in [2]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"foo#5faa ", 71)
token = parse_literal(71, iter_stream(buf_size))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, "foo_aa")
self.assertEqual(token.size, 8)
def test_parse_string(self):
for buf_size in [2]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
token = parse_string(88, iter_stream(buf_size))
self.assertIsInstance(token, TokenString)
self.assertEqual(token.pos, 88)
self.assertEqual(token.value, "this % is not a comment.")
self.assertEqual(token.size, 24)
class Tokenizer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PSTokenizer(io.BytesIO(raw))
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
self.assertEqual(token, etoken, f"Failed at example {i}")
def test_case1(self):
tokenizer = PSTokenizer(
io.BytesIO(
br"""%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>"""
)
)
for token in tokenizer.iter_tokens():
print(token)
def test_case2(self):
self.run_test(
raw=br"""10.32
0 +1 -2 .5 1.234
-1.234 990 """,
expected=[
TokenDecimal(pos=0, size=5, value=Decimal("10.32")),
TokenInteger(pos=6, size=1, value=0),
TokenInteger(pos=8, size=2, value=1),
TokenInteger(pos=11, size=2, value=-2),
TokenDecimal(pos=14, size=2, value=Decimal("0.5")),
TokenDecimal(pos=17, size=5, value=Decimal("1.234")),
TokenDecimal(pos=23, size=6, value=Decimal("-1.234")),
TokenInteger(pos=30, size=3, value=990),
],
)
def test_case_words1(self):
self.run_test(
raw=br"""%!PS
% This is some secret sauce
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
/need /oh1oh
func/a/b
""",
expected=[
TokenComment(pos=0, size=4, value=b"!PS"),
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
TokenKeyword(pos=33, size=5, value=b"begin"),
TokenKeyword(pos=39, size=3, value=b"end"),
TokenKeyword(pos=44, size=1, value=b'"'),
TokenKeyword(pos=47, size=1, value=b"@"),
TokenKeyword(pos=49, size=1, value=b"#"),
TokenLiteral(pos=52, size=1, value="a"),
TokenLiteral(pos=54, size=3, value="BCD"),
TokenLiteral(pos=59, size=9, value="Some_Name"),
TokenLiteral(pos=70, size=11, value="foo_naa"),
TokenLiteral(pos=83, size=6, value="barj"),
TokenLiteral(pos=91, size=4, value="need"),
TokenLiteral(pos=97, size=5, value="oh1oh"),
TokenKeyword(pos=103, size=4, value=b"func"),
TokenLiteral(pos=108, size=1, value="a"),
TokenLiteral(pos=110, size=1, value="b"),
],
)