initial tokenizer
This commit is contained in:
1 parent
fe9fd304ab
commit
460d213700
10 files changed
+868
-1
No files matched your search
+1
-1
@@ -6,4 +6,4 @@ indent_size = 4
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = false
|
||||
insert_final_newline = false
|
||||
insert_final_newline = true
|
||||
@@ -0,0 +1,46 @@
|
||||
import io
|
||||
from typing import List
|
||||
from pdfmajor.tokenizer.token import Token
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from pdfmajor.tokenizer.parse_funcs import ParseFunc
|
||||
|
||||
|
||||
class PSTokenizer:
|
||||
def __init__(self, fp: io.BufferedIOBase) -> None:
|
||||
self.fp = fp
|
||||
self.seek(0)
|
||||
|
||||
def seek(self, pos: int):
|
||||
"""Seeks tokenizer to the given position."""
|
||||
self.fp.seek(pos)
|
||||
# reset the status for nextline()
|
||||
self.bufpos: int = pos
|
||||
self.buf: bytes = b""
|
||||
self.charpos: int = 0
|
||||
# reset the status for nexttoken()
|
||||
self._current_parse_func: ParseFunc = self._parse_main
|
||||
self._curtoken: bytes = b""
|
||||
self._curtokenpos: int = 0
|
||||
self._tokens: List[Token] = []
|
||||
return
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos)
|
||||
|
||||
def nexttoken(self):
|
||||
while not self._tokens:
|
||||
self.fillbuf()
|
||||
self.charpos = self._current_parse_func(self.buf, self.charpos)
|
||||
token = self._tokens.pop(0)
|
||||
return token
|
||||
|
||||
def fillbuf(self):
|
||||
if self.charpos < len(self.buf):
|
||||
return
|
||||
# fetch next chunk.
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.BUFSIZ)
|
||||
if not self.buf:
|
||||
raise TokenizerEOF("Unexpected EOF")
|
||||
self.charpos = 0
|
||||
return
|
||||
@@ -0,0 +1,37 @@
|
||||
import re
|
||||
from .types import PSLiteral, PSSymbolTable, PSKeyword
|
||||
|
||||
PSLiteralTable = PSSymbolTable(PSLiteral)
|
||||
PSKeywordTable = PSSymbolTable(PSKeyword)
|
||||
LIT = PSLiteralTable.intern
|
||||
KWD = PSKeywordTable.intern
|
||||
KEYWORD_PROC_BEGIN = KWD(b"{")
|
||||
KEYWORD_PROC_END = KWD(b"}")
|
||||
KEYWORD_ARRAY_BEGIN = KWD(b"[")
|
||||
KEYWORD_ARRAY_END = KWD(b"]")
|
||||
KEYWORD_DICT_BEGIN = KWD(b"<<")
|
||||
KEYWORD_DICT_END = KWD(b">>")
|
||||
|
||||
## PSBaseParser
|
||||
##
|
||||
EOL = re.compile(br"[\r\n]")
|
||||
SPC = re.compile(br"\s")
|
||||
NONSPC = re.compile(br"\S")
|
||||
HEX = re.compile(br"[0-9a-fA-F]")
|
||||
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]")
|
||||
HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.")
|
||||
END_NUMBER = re.compile(br"[^0-9]")
|
||||
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
END_STRING = re.compile(br"[()\134]")
|
||||
OCT_STRING = re.compile(br"[0-7]")
|
||||
ESC_STRING = {
|
||||
b"b": 8,
|
||||
b"t": 9,
|
||||
b"n": 10,
|
||||
b"f": 12,
|
||||
b"r": 13,
|
||||
b"(": 40,
|
||||
b")": 41,
|
||||
b"\\": 92,
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
from pdfmajor.execptions import PDFMajorException
|
||||
|
||||
|
||||
class TokenizerError(PDFMajorException):
|
||||
pass
|
||||
|
||||
|
||||
class TokenizerEOF(TokenizerError):
|
||||
pass
|
||||
|
||||
class InvalidToken(TokenizerError):
|
||||
def __init__(self, pos: int, token: bytes) -> None:
|
||||
super().__init__("Invalid Token Found at {pos}, of value={v}".format(
|
||||
pos=pos,
|
||||
v=token
|
||||
))
|
||||
@@ -0,0 +1,341 @@
|
||||
from typing import List, Tuple, Union
|
||||
from io import TextIOWrapper
|
||||
|
||||
from pdfmajor.utils import int2byte
|
||||
from pdfmajor.execptions import PSEOF
|
||||
|
||||
from .types import PSLiteral
|
||||
from .constants import OCT_STRING, ESC_STRING, KEYWORD_DICT_BEGIN, KEYWORD_DICT_END
|
||||
from .constants import END_NUMBER, END_STRING, END_KEYWORD, END_HEX_STRING
|
||||
from .constants import SPC, HEX_PAIR
|
||||
from .constants import EOL, NONSPC, KWD, END_LITERAL, LIT, HEX
|
||||
|
||||
|
||||
class PSTokenizer(object):
|
||||
"""Tokenizatizes bytes stream to PS-Tokens"""
|
||||
|
||||
BUFSIZ = 4096
|
||||
|
||||
def __init__(self, fp: TextIOWrapper):
|
||||
self.fp = fp
|
||||
self.seek(0)
|
||||
return
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos)
|
||||
|
||||
def flush(self):
|
||||
return
|
||||
|
||||
def close(self):
|
||||
self.flush()
|
||||
return
|
||||
|
||||
def tell(self):
|
||||
return self.bufpos + self.charpos
|
||||
|
||||
def poll(self, pos=None, n=80):
|
||||
pos0 = self.fp.tell()
|
||||
if not pos:
|
||||
pos = self.bufpos + self.charpos
|
||||
self.fp.seek(pos)
|
||||
self.fp.read(n)
|
||||
self.fp.seek(pos0)
|
||||
return
|
||||
|
||||
def seek(self, pos: int):
|
||||
"""Seeks the parser to the given position."""
|
||||
self.fp.seek(pos)
|
||||
# reset the status for nextline()
|
||||
self.bufpos: int = pos
|
||||
self.buf: bytes = b""
|
||||
self.charpos: int = 0
|
||||
# reset the status for nexttoken()
|
||||
self._current_parse_func = self._parse_main
|
||||
self._curtoken: bytes = b""
|
||||
self._curtokenpos: int = 0
|
||||
self._tokens: List[
|
||||
Tuple[int, Union[int, float, bool, str, bytes, PSLiteral]]
|
||||
] = []
|
||||
return
|
||||
|
||||
def fillbuf(self):
|
||||
if self.charpos < len(self.buf):
|
||||
return
|
||||
# fetch next chunk.
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.BUFSIZ)
|
||||
if not self.buf:
|
||||
raise PSEOF("Unexpected EOF")
|
||||
self.charpos = 0
|
||||
return
|
||||
|
||||
def nextline(self):
|
||||
"""Fetches a next line that ends either with \\r or \\n."""
|
||||
linebuf = b""
|
||||
linepos = self.bufpos + self.charpos
|
||||
eol = False
|
||||
while 1:
|
||||
self.fillbuf()
|
||||
if eol:
|
||||
c = self.buf[self.charpos : self.charpos + 1]
|
||||
# handle b'\r\n'
|
||||
if c == b"\n":
|
||||
linebuf += c
|
||||
self.charpos += 1
|
||||
break
|
||||
m = EOL.search(self.buf, self.charpos)
|
||||
if m:
|
||||
linebuf += self.buf[self.charpos : m.end(0)]
|
||||
self.charpos = m.end(0)
|
||||
if linebuf[-1:] == b"\r":
|
||||
eol = True
|
||||
else:
|
||||
break
|
||||
else:
|
||||
linebuf += self.buf[self.charpos :]
|
||||
self.charpos = len(self.buf)
|
||||
return (linepos, linebuf)
|
||||
|
||||
def revreadlines(self):
|
||||
"""Fetches a next line backword.
|
||||
|
||||
This is used to locate the trailers at the end of a file.
|
||||
"""
|
||||
self.fp.seek(0, 2)
|
||||
pos = self.fp.tell()
|
||||
buf = b""
|
||||
while 0 < pos:
|
||||
prevpos = pos
|
||||
pos = max(0, pos - self.BUFSIZ)
|
||||
self.fp.seek(pos)
|
||||
s = self.fp.read(prevpos - pos)
|
||||
if not s:
|
||||
break
|
||||
while 1:
|
||||
n = max(s.rfind(b"\r"), s.rfind(b"\n"))
|
||||
if n == -1:
|
||||
buf = s + buf
|
||||
break
|
||||
yield s[n:] + buf
|
||||
s = s[:n]
|
||||
buf = b""
|
||||
return
|
||||
|
||||
def _parse_main(self, s, i):
|
||||
m = NONSPC.search(s, i)
|
||||
if not m:
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
c = s[j : j + 1]
|
||||
self._curtokenpos = self.bufpos + j
|
||||
if c == b"%":
|
||||
self._curtoken = b"%"
|
||||
self._current_parse_func = self._parse_comment
|
||||
return j + 1
|
||||
elif c == b"/":
|
||||
self._curtoken = b""
|
||||
self._current_parse_func = self._parse_literal
|
||||
return j + 1
|
||||
elif c in b"-+" or c.isdigit():
|
||||
self._curtoken = c
|
||||
self._current_parse_func = self._parse_number
|
||||
return j + 1
|
||||
elif c == b".":
|
||||
self._curtoken = c
|
||||
self._current_parse_func = self._parse_float
|
||||
return j + 1
|
||||
elif c.isalpha():
|
||||
self._curtoken = c
|
||||
self._current_parse_func = self._parse_keyword
|
||||
return j + 1
|
||||
elif c == b"(":
|
||||
self._curtoken = b""
|
||||
self.paren = 1
|
||||
self._current_parse_func = self._parse_string
|
||||
return j + 1
|
||||
elif c == b"<":
|
||||
self._curtoken = b""
|
||||
self._current_parse_func = self._parse_wopen
|
||||
return j + 1
|
||||
elif c == b">":
|
||||
self._curtoken = b""
|
||||
self._current_parse_func = self._parse_wclose
|
||||
return j + 1
|
||||
else:
|
||||
self._add_token(KWD(c))
|
||||
return j + 1
|
||||
|
||||
def _add_token(self, obj):
|
||||
self._tokens.append((self._curtokenpos, obj))
|
||||
return
|
||||
|
||||
def _parse_comment(self, s, i):
|
||||
m = EOL.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
self._current_parse_func = self._parse_main
|
||||
# We ignore comments.
|
||||
# self._tokens.append(self._curtoken)
|
||||
return j
|
||||
|
||||
def _parse_literal(self, s, i):
|
||||
m = END_LITERAL.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"#":
|
||||
self.hex = b""
|
||||
self._current_parse_func = self._parse_literal_hex
|
||||
return j + 1
|
||||
try:
|
||||
self._curtoken = str(self._curtoken, "utf-8")
|
||||
except:
|
||||
pass
|
||||
self._add_token(LIT(self._curtoken))
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def _parse_literal_hex(self, s, i):
|
||||
c = s[i : i + 1]
|
||||
if HEX.match(c) and len(self.hex) < 2:
|
||||
self.hex += c
|
||||
return i + 1
|
||||
if self.hex:
|
||||
self._curtoken += int2byte(int(self.hex, 16))
|
||||
self._current_parse_func = self._parse_literal
|
||||
return i
|
||||
|
||||
def _parse_number(self, s, i):
|
||||
m = END_NUMBER.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b".":
|
||||
self._curtoken += c
|
||||
self._current_parse_func = self._parse_float
|
||||
return j + 1
|
||||
try:
|
||||
self._add_token(int(self._curtoken))
|
||||
except ValueError:
|
||||
pass
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def _parse_float(self, s, i):
|
||||
m = END_NUMBER.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
try:
|
||||
self._add_token(float(self._curtoken))
|
||||
except ValueError:
|
||||
pass
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def _parse_keyword(self, s, i):
|
||||
m = END_KEYWORD.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
if self._curtoken == b"true":
|
||||
token = True
|
||||
elif self._curtoken == b"false":
|
||||
token = False
|
||||
else:
|
||||
token = KWD(self._curtoken)
|
||||
self._add_token(token)
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def _parse_string(self, s, i):
|
||||
m = END_STRING.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"\\":
|
||||
self.oct = b""
|
||||
self._current_parse_func = self._parse_string_1
|
||||
return j + 1
|
||||
if c == b"(":
|
||||
self.paren += 1
|
||||
self._curtoken += c
|
||||
return j + 1
|
||||
if c == b")":
|
||||
self.paren -= 1
|
||||
if self.paren: # WTF, they said balanced parens need no special treatment.
|
||||
self._curtoken += c
|
||||
return j + 1
|
||||
self._add_token(self._curtoken)
|
||||
self._current_parse_func = self._parse_main
|
||||
return j + 1
|
||||
|
||||
def _parse_string_1(self, s, i):
|
||||
c = s[i : i + 1]
|
||||
if OCT_STRING.match(c) and len(self.oct) < 3:
|
||||
self.oct += c
|
||||
return i + 1
|
||||
if self.oct:
|
||||
self._curtoken += int2byte(int(self.oct, 8))
|
||||
self._current_parse_func = self._parse_string
|
||||
return i
|
||||
if c in ESC_STRING:
|
||||
self._curtoken += int2byte(ESC_STRING[c])
|
||||
self._current_parse_func = self._parse_string
|
||||
return i + 1
|
||||
|
||||
def _parse_wopen(self, s, i):
|
||||
c = s[i : i + 1]
|
||||
if c == b"<":
|
||||
self._add_token(KEYWORD_DICT_BEGIN)
|
||||
self._current_parse_func = self._parse_main
|
||||
i += 1
|
||||
else:
|
||||
self._current_parse_func = self._parse_hexstring
|
||||
return i
|
||||
|
||||
def _parse_wclose(self, s, i):
|
||||
c = s[i : i + 1]
|
||||
if c == b">":
|
||||
self._add_token(KEYWORD_DICT_END)
|
||||
i += 1
|
||||
self._current_parse_func = self._parse_main
|
||||
return i
|
||||
|
||||
def _parse_hexstring(self, s, i):
|
||||
m = END_HEX_STRING.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
token = HEX_PAIR.sub(
|
||||
lambda m: int2byte(int(m.group(0), 16)), SPC.sub(b"", self._curtoken)
|
||||
)
|
||||
self._add_token(token)
|
||||
self._current_parse_func = self._parse_main
|
||||
return j
|
||||
|
||||
def nexttoken(self):
|
||||
while not self._tokens:
|
||||
self.fillbuf()
|
||||
self.charpos = self._current_parse_func(self.buf, self.charpos)
|
||||
token = self._tokens.pop(0)
|
||||
return token
|
||||
@@ -0,0 +1,100 @@
|
||||
from pdfmajor.tokenizer.token import Token, TokenKeyword
|
||||
from typing import Callable, NamedTuple, Optional, Union
|
||||
from .constants import EOL, NONSPC
|
||||
|
||||
ParseFunc = Callable[["ParseInput"], "NextParseAction"]
|
||||
|
||||
|
||||
class NextParseAction(NamedTuple):
|
||||
next_parse_action: ParseFunc
|
||||
charpos: int
|
||||
curtokenpos: int
|
||||
curtoken: Union[Token, bytes] = b""
|
||||
paren: Optional[int] = None
|
||||
|
||||
|
||||
class ParseInput(NamedTuple):
|
||||
bufvalue: bytes
|
||||
charpos: int
|
||||
bufpos: int
|
||||
|
||||
|
||||
def parse_main(inp: ParseInput) -> NextParseAction:
|
||||
m = NONSPC.search(inp.bufvalue, inp.charpos)
|
||||
if not m:
|
||||
return len(inp.bufvalue)
|
||||
j = m.start(0)
|
||||
c = inp.bufvalue[j : j + 1]
|
||||
if c == b"%":
|
||||
return NextParseAction(
|
||||
curtoken=b"%",
|
||||
next_parse_action=__parse_comment,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
)
|
||||
elif c == b"/":
|
||||
return NextParseAction(
|
||||
next_parse_action=__parse_literal,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
)
|
||||
elif c in b"-+" or c.isdigit():
|
||||
return NextParseAction(
|
||||
curtoken=c,
|
||||
next_parse_action=__parse_number,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
)
|
||||
elif c == b".":
|
||||
return NextParseAction(
|
||||
curtoken=c,
|
||||
next_parse_action=__parse_float,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
)
|
||||
elif c.isalpha():
|
||||
return NextParseAction(
|
||||
curtoken=c,
|
||||
next_parse_action=__parse_keyword,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
)
|
||||
elif c == b"(":
|
||||
return NextParseAction(
|
||||
next_parse_action=__parse_string,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
paren=1,
|
||||
)
|
||||
elif c == b"<":
|
||||
return NextParseAction(
|
||||
next_parse_action=__parse_wopen,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
)
|
||||
elif c == b">":
|
||||
return NextParseAction(
|
||||
next_parse_action=__parse_wclose,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
)
|
||||
else:
|
||||
return NextParseAction(
|
||||
next_parse_action=parse_main,
|
||||
curtokenpos=inp.bufpos + j,
|
||||
charpos=j + 1,
|
||||
curtoken=TokenKeyword(inp.bufpos + j, c),
|
||||
)
|
||||
|
||||
|
||||
def __parse_comment(s, i):
|
||||
m = EOL.search(s, i)
|
||||
if not m:
|
||||
self._curtoken += s[i:]
|
||||
return len(s)
|
||||
j = m.start(0)
|
||||
self._curtoken += s[i:j]
|
||||
self._current_parse_func = self._parse_main
|
||||
# We ignore comments.
|
||||
# self._tokens.append(self._curtoken)
|
||||
return j
|
||||
@@ -0,0 +1,44 @@
|
||||
from typing import Generic, TypeVar, Union
|
||||
from decimal import Decimal
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Token(Generic[T]):
|
||||
__slots__ = ["pos", "size", "value"]
|
||||
|
||||
def __init__(self, pos: int, size: int, value: T) -> None:
|
||||
self.size = size
|
||||
self.pos = pos
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "<{cls} pos={pos}>{value}</{cls}>".format(
|
||||
cls=self.__class__.__name__, pos=self.pos, value=self.value
|
||||
)
|
||||
|
||||
|
||||
class TokenComment(Token[bytes]):
|
||||
pass
|
||||
|
||||
|
||||
class TokenHex(Token[bytes]):
|
||||
pass
|
||||
|
||||
|
||||
class TokenKeyword(Token[bytes]):
|
||||
pass
|
||||
|
||||
|
||||
class TokenString(Token[str]):
|
||||
pass
|
||||
|
||||
|
||||
class TokenInteger(Token[int]):
|
||||
pass
|
||||
|
||||
|
||||
class TokenDecimal(Token[Decimal]):
|
||||
pass
|
||||
|
||||
TokenNumber = Union[TokenInteger, TokenDecimal]
|
||||
@@ -0,0 +1,139 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken
|
||||
from pdfmajor.tokenizer.token import Token, TokenComment, TokenDecimal, TokenNumber, TokenInteger
|
||||
from pdfmajor.tokenizer.constants import END_NUMBER, EOL
|
||||
from typing import Callable, Iterator, NamedTuple
|
||||
|
||||
|
||||
class PInput(NamedTuple):
|
||||
pos: int
|
||||
buf: bytes
|
||||
|
||||
|
||||
ParserFunc = Callable[[int, Iterator[PInput]], Token]
|
||||
|
||||
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
|
||||
"""Computes the size of the token based on its positional data
|
||||
|
||||
Args:
|
||||
curpos (int): current position of the buffer
|
||||
initialpos (int): initial position where the parser started working
|
||||
tokenend_idx (int): last relevant character index within the last buffered value
|
||||
|
||||
Returns:
|
||||
int: length of index
|
||||
"""
|
||||
return curpos-initialpos+tokenend_idx
|
||||
|
||||
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = EOL.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
return TokenComment(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken
|
||||
)
|
||||
|
||||
def parse_number(initialpos: int, inp: Iterator[PInput]) -> TokenNumber:
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
is_dec = False
|
||||
for curpos, s in inp:
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if (c == b".") and not is_dec:
|
||||
is_dec = True
|
||||
curtoken += c
|
||||
s = s[j+1:]
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
continue
|
||||
else:
|
||||
nj = m.start(0)
|
||||
curtoken += s[:nj]
|
||||
j += nj
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode())
|
||||
)
|
||||
try:
|
||||
if is_dec:
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode())
|
||||
)
|
||||
else:
|
||||
return TokenInteger(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
int(curtoken)
|
||||
)
|
||||
except ValueError:
|
||||
raise InvalidToken(initialpos, curtoken)
|
||||
|
||||
# def parse_float(
|
||||
# initialpos: int,
|
||||
# inp: Iterator[PInput],
|
||||
# initialtokenval = b""
|
||||
# ) -> TokenFloat:
|
||||
# curtoken = b"" + initialtokenval
|
||||
# for curpos, s in inp:
|
||||
# print('->',curpos, s)
|
||||
# m = END_NUMBER.search(s, 0)
|
||||
# if not m:
|
||||
# curtoken += s
|
||||
# else:
|
||||
# j = m.start(0)
|
||||
# curtoken += s[:j]
|
||||
# next_t = s[j:j+1]
|
||||
# if next_t == b'.':
|
||||
# curtoken += b'.'
|
||||
# s = s[j+1:]
|
||||
# m = END_NUMBER.search(s, 0)
|
||||
# if not m:
|
||||
# curtoken += s
|
||||
# continue
|
||||
# else:
|
||||
# j = m.start(0)
|
||||
# curtoken += s[:j]
|
||||
# next_t = s[j+1]
|
||||
# try:
|
||||
# print(curpos,j, initialpos)
|
||||
# return TokenFloat(
|
||||
# initialpos,
|
||||
# cmp_tsize(curpos, initialpos, j),
|
||||
# Decimal(curtoken.decode())
|
||||
# )
|
||||
# except ValueError:
|
||||
# raise InvalidToken(initialpos, curtoken)
|
||||
@@ -0,0 +1,80 @@
|
||||
## Basic PostScript Types
|
||||
##
|
||||
from typing import Union
|
||||
|
||||
## PSObject
|
||||
##
|
||||
class PSObject(object):
|
||||
|
||||
"""Base class for all PS or PDF-related data types."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
## PSLiteral
|
||||
##
|
||||
class PSLiteral(PSObject):
|
||||
|
||||
"""A class that represents a PostScript literal.
|
||||
|
||||
Postscript literals are used as identifiers, such as
|
||||
variable names, property names and dictionary keys.
|
||||
Literals are case sensitive and denoted by a preceding
|
||||
slash sign (e.g. "/Name")
|
||||
|
||||
Note: Do not create an instance of PSLiteral directly.
|
||||
Always use PSLiteralTable.intern().
|
||||
"""
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __repr__(self):
|
||||
name = self.name
|
||||
return "/%r" % name
|
||||
|
||||
|
||||
## PSKeyword
|
||||
##
|
||||
class PSKeyword(PSObject):
|
||||
|
||||
"""A class that represents a PostScript keyword.
|
||||
|
||||
PostScript keywords are a dozen of predefined words.
|
||||
Commands and directives in PostScript are expressed by keywords.
|
||||
They are also used to denote the content boundaries.
|
||||
|
||||
Note: Do not create an instance of PSKeyword directly.
|
||||
Always use PSKeywordTable.intern().
|
||||
"""
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
return
|
||||
|
||||
def __repr__(self):
|
||||
name = self.name
|
||||
return "/%r" % name
|
||||
|
||||
|
||||
## PSSymbolTable
|
||||
##
|
||||
class PSSymbolTable(object):
|
||||
|
||||
"""A utility class for storing PSLiteral/PSKeyword objects.
|
||||
|
||||
Interned objects can be checked its identity with "is" operator.
|
||||
"""
|
||||
|
||||
def __init__(self, klass):
|
||||
self.dict = {}
|
||||
self.klass = klass
|
||||
return
|
||||
|
||||
def intern(self, name):
|
||||
if name in self.dict:
|
||||
lit = self.dict[name]
|
||||
else:
|
||||
lit = self.klass(name)
|
||||
self.dict[name] = lit
|
||||
return lit
|
||||
@@ -0,0 +1,64 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
from pdfmajor.tokenizer.token import TokenComment, TokenDecimal, TokenInteger
|
||||
from pdfmajor.tokenizer.token_parser import PInput, parse_comment, parse_number
|
||||
from unittest import TestCase
|
||||
|
||||
def make_stream_iter(data: bytes, initpos: int = 0):
|
||||
stream = io.BytesIO(data)
|
||||
|
||||
def iter_stream(buf_size: int):
|
||||
while True:
|
||||
bufpos = stream.tell()
|
||||
buf = stream.read(buf_size)
|
||||
if buf:
|
||||
yield PInput(initpos+bufpos, buf)
|
||||
else:
|
||||
raise "EOF"
|
||||
return iter_stream
|
||||
|
||||
class Basics(TestCase):
|
||||
def test_comment_parser_ln(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
)
|
||||
token = parse_comment(0, iter_stream(3))
|
||||
self.assertIsInstance(token, TokenComment)
|
||||
self.assertEqual(token.pos, 0)
|
||||
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
|
||||
self.assertEqual(token.size, 40)
|
||||
|
||||
def test_comment_parser_lr(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\rso this is not reachable"
|
||||
)
|
||||
token = parse_comment(0, iter_stream(7))
|
||||
self.assertIsInstance(token, TokenComment)
|
||||
self.assertEqual(token.pos, 0)
|
||||
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
|
||||
self.assertEqual(token.size, 40)
|
||||
|
||||
def test_parse_float(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"0.120 + some invalid text",
|
||||
5
|
||||
)
|
||||
token = parse_number(5, iter_stream(3))
|
||||
self.assertIsInstance(token, TokenDecimal)
|
||||
self.assertEqual(token.pos, 5)
|
||||
self.assertEqual(token.value, Decimal('0.12'))
|
||||
self.assertEqual(token.size, 5)
|
||||
|
||||
|
||||
def test_parse_int(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"130 oh oh 0.12 + some invalid text",
|
||||
11
|
||||
)
|
||||
token = parse_number(11, iter_stream(10))
|
||||
self.assertIsInstance(token, TokenInteger)
|
||||
self.assertEqual(token.pos, 11)
|
||||
self.assertEqual(token.value, 130)
|
||||
self.assertEqual(token.size, 3)
|
||||
|
||||
|
||||
Reference in new issue
Block a user