ran linter

This commit is contained in:
ari committed 2020-10-13 21:05:25 -04:00
1 parent c430c57fe0
commit e1e644cc38
9 files changed
+335 -339

No files matched your search

+91 -78
View File
@@ -1,4 +1,3 @@
import io
from typing import Iterator
@@ -14,85 +13,99 @@ from pdfmajor.tokenizer.token import Token, TokenKeyword
class PSTokenizer:
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
self.fp = fp
self.bufsize = bufsize
self.bufpos: int = 0
self.buf: bytes = b''
self.buf_skipstep: int = 0
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
self.fp = fp
self.bufsize = bufsize
self.bufpos: int = 0
self.buf: bytes = b""
self.buf_skipstep: int = 0
def iter_buffer(self) -> Iterator[PInput]:
while True:
if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)):
yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:])
else:
self.buf_skipstep = 0
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.bufsize)
if not self.buf:
raise TokenizerEOF("Unexpected EOF")
else:
yield PInput(self.bufpos, self.buf)
def iter_buffer(self) -> Iterator[PInput]:
while True:
if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)):
yield PInput(
self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :]
)
else:
self.buf_skipstep = 0
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.bufsize)
if not self.buf:
raise TokenizerEOF("Unexpected EOF")
else:
yield PInput(self.bufpos, self.buf)
def seek(self, pos: int):
self.fp.seek(pos)
def step_back(self, steps: int = 1):
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
def seek(self, pos: int):
self.fp.seek(pos)
Args:
steps (int, optional): numbers of steps. Defaults to 1.
"""
self.fp.seek(self.fp.tell() - steps * self.bufsize)
def step_back(self, steps: int = 1):
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
def iter_tokens(self) -> Iterator[Token]:
for cur in self.iter_buffer():
bufpos, buf = cur
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = 0
continue
else:
j = m.start(0)
c = buf[j : j + 1]
if c == b"%":
self.buf_skipstep += j + 1
yield self._check_token(cur, parse_comment(bufpos + j, self.iter_buffer()))
elif c == b"/":
self.buf_skipstep += j + 1
yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer()))
elif c in b"-+":
self.buf_skipstep += j + 1
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c))
elif c.isdigit():
self.buf_skipstep += j
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
elif c == b".":
self.buf_skipstep += j
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
elif c.isalpha():
self.buf_skipstep += j
yield self._check_token(cur, parse_keyword(bufpos + j, self.iter_buffer()))
elif c == b"(":
raise "WIP"
# self._curtoken = b""
# self.paren = 1
# self._current_parse_func = self._parse_string
# return j + 1
elif c == b"<":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wopen
# return j + 1
elif c == b">":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wclose
# return j + 1
else:
self.buf_skipstep += j + 1
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
Args:
steps (int, optional): numbers of steps. Defaults to 1.
"""
self.fp.seek(self.fp.tell() - steps * self.bufsize)
def _check_token(self, cur: PInput, token: Token) -> Token:
self.buf_skipstep += token.size
return token
def iter_tokens(self) -> Iterator[Token]:
for cur in self.iter_buffer():
bufpos, buf = cur
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = 0
continue
else:
j = m.start(0)
c = buf[j : j + 1]
if c == b"%":
self.buf_skipstep += j + 1
yield self._check_token(
cur, parse_comment(bufpos + j, self.iter_buffer())
)
elif c == b"/":
self.buf_skipstep += j + 1
yield self._check_token(
cur, parse_literal(bufpos + j, self.iter_buffer())
)
elif c in b"-+":
self.buf_skipstep += j + 1
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer(), c)
)
elif c.isdigit():
self.buf_skipstep += j
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer())
)
elif c == b".":
self.buf_skipstep += j
yield self._check_token(
cur, parse_number(bufpos + j, self.iter_buffer())
)
elif c.isalpha():
self.buf_skipstep += j
yield self._check_token(
cur, parse_keyword(bufpos + j, self.iter_buffer())
)
elif c == b"(":
raise "WIP"
# self._curtoken = b""
# self.paren = 1
# self._current_parse_func = self._parse_string
# return j + 1
elif c == b"<":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wopen
# return j + 1
elif c == b">":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wclose
# return j + 1
else:
self.buf_skipstep += j + 1
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
def _check_token(self, cur: PInput, token: Token) -> Token:
self.buf_skipstep += token.size
return token
+5 -5
View File
@@ -8,9 +8,9 @@ class TokenizerError(PDFMajorException):
class TokenizerEOF(TokenizerError):
pass
class InvalidToken(TokenizerError):
def __init__(self, pos: int, token: bytes) -> None:
super().__init__("Invalid Token Found at {pos}, of value={v}".format(
pos=pos,
v=token
))
def __init__(self, pos: int, token: bytes) -> None:
super().__init__(
"Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=token)
)
+22 -18
View File
@@ -5,45 +5,49 @@ T = TypeVar("T")
class Token(Generic[T]):
__slots__ = ["pos", "size", "value"]
__slots__ = ["pos", "size", "value"]
def __init__(self, pos: int, size: int, value: T) -> None:
self.size = size
self.pos = pos
self.value = value
def __init__(self, pos: int, size: int, value: T) -> None:
self.size = size
self.pos = pos
self.value = value
def __repr__(self) -> str:
return "<{cls} pos={pos}>{value}</{cls}>".format(
cls=self.__class__.__name__, pos=self.pos, value=self.value
)
def __repr__(self) -> str:
return "<{cls} pos={pos}>{value}</{cls}>".format(
cls=self.__class__.__name__, pos=self.pos, value=self.value
)
class TokenComment(Token[bytes]):
pass
pass
class TokenHex(Token[bytes]):
pass
pass
class TokenKeyword(Token[bytes]):
pass
pass
class TokenLiteral(Token[str]):
pass
pass
class TokenBoolean(Token[bool]):
pass
pass
class TokenString(Token[str]):
pass
pass
class TokenInteger(Token[int]):
pass
pass
class TokenDecimal(Token[Decimal]):
pass
pass
TokenNumber = Union[TokenInteger, TokenDecimal]
+19 -22
View File
@@ -1,32 +1,29 @@
from pdfmajor.tokenizer.token import (
TokenComment,
TokenComment,
)
from pdfmajor.tokenizer.constants import (
EOL,
EOL,
)
from typing import Iterator
from .util import cmp_tsize, PInput
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
"""Parses input stream into a comment
"""Parses input stream into a comment
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenComment
"""
curtoken = b""
for curpos, s in inp:
m = EOL.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
return TokenComment(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken
)
Returns:
TokenComment
"""
curtoken = b""
for curpos, s in inp:
m = EOL.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
return TokenComment(initialpos, cmp_tsize(curpos, initialpos, j), curtoken)
+27 -43
View File
@@ -1,50 +1,34 @@
from pdfmajor.tokenizer.token import (
TokenBoolean,
TokenKeyword
)
from pdfmajor.tokenizer.constants import (
END_KEYWORD
)
from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword
from pdfmajor.tokenizer.constants import END_KEYWORD
from typing import Iterator, Union
from .util import cmp_tsize, PInput
def parse_keyword(
initialpos: int,
inp: Iterator[PInput]
initialpos: int, inp: Iterator[PInput]
) -> Union[TokenKeyword, TokenBoolean]:
"""Parses input stream into a keyword or bool
"""Parses input stream into a keyword or bool
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
Union[TokenKeyword, TokenBoolean]
"""
curtoken = b""
for curpos, s in inp:
m = END_KEYWORD.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
if curtoken == b"true":
return TokenBoolean(
initialpos,
cmp_tsize(curpos, initialpos, j),
True
)
elif curtoken == b"false":
return TokenBoolean(
initialpos,
cmp_tsize(curpos, initialpos, j),
False
)
else:
return TokenKeyword(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken
)
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
Union[TokenKeyword, TokenBoolean]
"""
curtoken = b""
for curpos, s in inp:
m = END_KEYWORD.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
if curtoken == b"true":
return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), True)
elif curtoken == b"false":
return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), False)
else:
return TokenKeyword(
initialpos, cmp_tsize(curpos, initialpos, j), curtoken
)
+48 -46
View File
@@ -8,55 +8,57 @@ from ..constants import HEX, END_LITERAL
from typing import Iterator, NamedTuple, Optional, Union
@dataclass
class LiteralParseState:
curtoken: bytes
hex_value: Optional[bytes] = None
curtoken: bytes
hex_value: Optional[bytes] = None
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
"""Parses input stream into a literal
"""Parses input stream into a literal
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenLiteral
"""
state = LiteralParseState(b"")
for curpos, buf in inp:
skip: int = 0
for i in range(len(buf) + 1):
assert i < len(buf), "MAX_LOOP_REACHED"
if skip >= len(buf):
break
if state.hex_value is not None:
for ci in range(len(buf[skip:])):
c = buf[skip+ci:skip+ci+1]
if HEX.match(c) and len(state.hex_value) < 2:
state.hex_value += c
else:
if state.hex_value:
state.curtoken += int2byte(int(state.hex_value, 16))
state.hex_value = None
skip += ci
break
else:
m = END_LITERAL.search(buf[skip:], 0)
if not m:
state.curtoken += buf[skip:]
skip = len(buf)
else:
j: int = m.start(0) + skip
state.curtoken += buf[skip:j]
c = buf[j : j + 1]
if c == b"#":
state.hex_value = b""
skip = j+1
continue
else:
return TokenLiteral(
initialpos,
cmp_tsize(curpos, initialpos, j),
state.curtoken.decode()
)
Returns:
TokenLiteral
"""
state = LiteralParseState(b"")
for curpos, buf in inp:
skip: int = 0
for i in range(len(buf) + 1):
assert i < len(buf), "MAX_LOOP_REACHED"
if skip >= len(buf):
break
if state.hex_value is not None:
for ci in range(len(buf[skip:])):
c = buf[skip + ci : skip + ci + 1]
if HEX.match(c) and len(state.hex_value) < 2:
state.hex_value += c
else:
if state.hex_value:
state.curtoken += int2byte(int(state.hex_value, 16))
state.hex_value = None
skip += ci
break
else:
m = END_LITERAL.search(buf[skip:], 0)
if not m:
state.curtoken += buf[skip:]
skip = len(buf)
else:
j: int = m.start(0) + skip
state.curtoken += buf[skip:j]
c = buf[j : j + 1]
if c == b"#":
state.hex_value = b""
skip = j + 1
continue
else:
return TokenLiteral(
initialpos,
cmp_tsize(curpos, initialpos, j),
state.curtoken.decode(),
)
+56 -59
View File
@@ -3,73 +3,70 @@ from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
from pdfmajor.utils import int2byte
from pdfmajor.tokenizer.exceptions import InvalidToken
from pdfmajor.tokenizer.token import (
TokenDecimal,
TokenNumber,
TokenInteger,
TokenDecimal,
TokenNumber,
TokenInteger,
)
from pdfmajor.tokenizer.constants import (
END_NUMBER,
END_NUMBER,
)
from typing import Iterator, Literal, Union
from .util import cmp_tsize, PInput
VALID_SIGNS = Union[Literal[b'+'], Literal[b'-']]
VALID_SIGNS = Union[Literal[b"+"], Literal[b"-"]]
def parse_number(
initialpos: int,
inp: Iterator[PInput],
sign: VALID_SIGNS = b'+'
initialpos: int, inp: Iterator[PInput], sign: VALID_SIGNS = b"+"
) -> TokenNumber:
"""Parses input stream into a number
"""Parses input stream into a number
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
sign (str): either a + or -
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
sign (str): either a + or -
Returns:
TokenNumber
"""
assert sign in [b'+', b'-'], f"Number was provided an invalid sign {sign}"
curtoken = b"" + sign
is_dec = False
for curpos, s in inp:
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if (c == b".") and not is_dec:
is_dec = True
curtoken += c
s = s[j+1:]
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
continue
else:
nj = m.start(0)
curtoken += s[:nj]
j += nj
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
Decimal(curtoken.decode())
)
try:
if is_dec:
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
Decimal(curtoken.decode())
)
else:
return TokenInteger(
initialpos,
cmp_tsize(curpos, initialpos, j),
int(curtoken)
)
except ValueError:
raise InvalidToken(initialpos, curtoken)
Returns:
TokenNumber
"""
assert sign in [b"+", b"-"], f"Number was provided an invalid sign {sign}"
curtoken = b"" + sign
is_dec = False
for curpos, s in inp:
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if (c == b".") and not is_dec:
is_dec = True
curtoken += c
s = s[j + 1 :]
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
continue
else:
nj = m.start(0)
curtoken += s[:nj]
j += nj
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
Decimal(curtoken.decode()),
)
try:
if is_dec:
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
Decimal(curtoken.decode()),
)
else:
return TokenInteger(
initialpos, cmp_tsize(curpos, initialpos, j), int(curtoken)
)
except ValueError:
raise InvalidToken(initialpos, curtoken)
+54 -54
View File
@@ -3,66 +3,66 @@ from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
from pdfmajor.utils import int2byte
from pdfmajor.tokenizer.exceptions import InvalidToken
from pdfmajor.tokenizer.token import (
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenKeyword, TokenLiteral,
TokenNumber,
TokenInteger,
TokenString
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenKeyword,
TokenLiteral,
TokenNumber,
TokenInteger,
TokenString,
)
from pdfmajor.tokenizer.constants import (
END_KEYWORD,
END_NUMBER,
END_STRING,
EOL,
ESC_STRING,
OCT_STRING
END_KEYWORD,
END_NUMBER,
END_STRING,
EOL,
ESC_STRING,
OCT_STRING,
)
from typing import Callable, Iterator, Literal, NamedTuple, Union
def parse_string(initialpos: int, inp: Iterator[PInput]):
curtoken = b""
paren = 1
for curpos, s in inp:
m = END_STRING.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if c == b"\\":
self._current_parse_func = self._parse_string_1
return j + 1
elif c == b"(":
paren += 1
curtoken += c
return j + 1
elif c == b")":
paren -= 1
if paren: # WTF, they said balanced parens need no special treatment.
curtoken += c
return j + 1
return TokenString(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken.decode()
)
curtoken = b""
paren = 1
for curpos, s in inp:
m = END_STRING.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if c == b"\\":
self._current_parse_func = self._parse_string_1
return j + 1
elif c == b"(":
paren += 1
curtoken += c
return j + 1
elif c == b")":
paren -= 1
if paren: # WTF, they said balanced parens need no special treatment.
curtoken += c
return j + 1
return TokenString(
initialpos, cmp_tsize(curpos, initialpos, j), curtoken.decode()
)
def parse_string_oct(initialpos: int, inp: Iterator[PInput]):
curtoken: bytes = b""
oct_val: bytes = b""
for curpos, s in inp:
for ci in range(len(s)):
c = s[ci:ci+1]
if OCT_STRING.match(c) and len(oct_val) < 3:
oct_val += c
elif oct_val:
curtoken += int2byte(int(oct_val, 8))
return (curtoken, curpos+ci)
elif c in ESC_STRING:
curtoken += int2byte(ESC_STRING[c])
return (curtoken, curpos+ci+1)
curtoken: bytes = b""
oct_val: bytes = b""
for curpos, s in inp:
for ci in range(len(s)):
c = s[ci : ci + 1]
if OCT_STRING.match(c) and len(oct_val) < 3:
oct_val += c
elif oct_val:
curtoken += int2byte(int(oct_val, 8))
return (curtoken, curpos + ci)
elif c in ESC_STRING:
curtoken += int2byte(ESC_STRING[c])
return (curtoken, curpos + ci + 1)
+13 -14
View File
@@ -1,25 +1,24 @@
from pdfmajor.tokenizer.token import (
Token
)
from pdfmajor.tokenizer.token import Token
from typing import Callable, Iterator, NamedTuple
class PInput(NamedTuple):
pos: int
buf: bytes
pos: int
buf: bytes
ParserFunc = Callable[[int, Iterator[PInput]], Token]
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
"""Computes the size of the token based on its positional data
"""Computes the size of the token based on its positional data
Args:
curpos (int): current position of the buffer
initialpos (int): initial position where the parser started working
tokenend_idx (int): last relevant character index within the last buffered value
Args:
curpos (int): current position of the buffer
initialpos (int): initial position where the parser started working
tokenend_idx (int): last relevant character index within the last buffered value
Returns:
int: length of index
"""
return curpos-initialpos+tokenend_idx
Returns:
int: length of index
"""
return curpos - initialpos + tokenend_idx