ran linter
This commit is contained in:
1 parent
c430c57fe0
commit
e1e644cc38
9 files changed
+335
-339
No files matched your search
@@ -1,4 +1,3 @@
|
||||
|
||||
import io
|
||||
|
||||
from typing import Iterator
|
||||
@@ -14,85 +13,99 @@ from pdfmajor.tokenizer.token import Token, TokenKeyword
|
||||
|
||||
|
||||
class PSTokenizer:
|
||||
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
|
||||
self.fp = fp
|
||||
self.bufsize = bufsize
|
||||
self.bufpos: int = 0
|
||||
self.buf: bytes = b''
|
||||
self.buf_skipstep: int = 0
|
||||
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
|
||||
self.fp = fp
|
||||
self.bufsize = bufsize
|
||||
self.bufpos: int = 0
|
||||
self.buf: bytes = b""
|
||||
self.buf_skipstep: int = 0
|
||||
|
||||
def iter_buffer(self) -> Iterator[PInput]:
|
||||
while True:
|
||||
if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)):
|
||||
yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:])
|
||||
else:
|
||||
self.buf_skipstep = 0
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.bufsize)
|
||||
if not self.buf:
|
||||
raise TokenizerEOF("Unexpected EOF")
|
||||
else:
|
||||
yield PInput(self.bufpos, self.buf)
|
||||
def iter_buffer(self) -> Iterator[PInput]:
|
||||
while True:
|
||||
if (self.buf_skipstep > 0) and (self.buf_skipstep < len(self.buf)):
|
||||
yield PInput(
|
||||
self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :]
|
||||
)
|
||||
else:
|
||||
self.buf_skipstep = 0
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.bufsize)
|
||||
if not self.buf:
|
||||
raise TokenizerEOF("Unexpected EOF")
|
||||
else:
|
||||
yield PInput(self.bufpos, self.buf)
|
||||
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
|
||||
def step_back(self, steps: int = 1):
|
||||
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
|
||||
Args:
|
||||
steps (int, optional): numbers of steps. Defaults to 1.
|
||||
"""
|
||||
self.fp.seek(self.fp.tell() - steps * self.bufsize)
|
||||
def step_back(self, steps: int = 1):
|
||||
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
|
||||
|
||||
def iter_tokens(self) -> Iterator[Token]:
|
||||
for cur in self.iter_buffer():
|
||||
bufpos, buf = cur
|
||||
m = NONSPC.search(buf, 0)
|
||||
if not m:
|
||||
self.buf_skipstep = 0
|
||||
continue
|
||||
else:
|
||||
j = m.start(0)
|
||||
c = buf[j : j + 1]
|
||||
if c == b"%":
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, parse_comment(bufpos + j, self.iter_buffer()))
|
||||
elif c == b"/":
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer()))
|
||||
elif c in b"-+":
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c))
|
||||
elif c.isdigit():
|
||||
self.buf_skipstep += j
|
||||
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
|
||||
elif c == b".":
|
||||
self.buf_skipstep += j
|
||||
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
|
||||
elif c.isalpha():
|
||||
self.buf_skipstep += j
|
||||
yield self._check_token(cur, parse_keyword(bufpos + j, self.iter_buffer()))
|
||||
elif c == b"(":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self.paren = 1
|
||||
# self._current_parse_func = self._parse_string
|
||||
# return j + 1
|
||||
elif c == b"<":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wopen
|
||||
# return j + 1
|
||||
elif c == b">":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wclose
|
||||
# return j + 1
|
||||
else:
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
|
||||
Args:
|
||||
steps (int, optional): numbers of steps. Defaults to 1.
|
||||
"""
|
||||
self.fp.seek(self.fp.tell() - steps * self.bufsize)
|
||||
|
||||
def _check_token(self, cur: PInput, token: Token) -> Token:
|
||||
self.buf_skipstep += token.size
|
||||
return token
|
||||
def iter_tokens(self) -> Iterator[Token]:
|
||||
for cur in self.iter_buffer():
|
||||
bufpos, buf = cur
|
||||
m = NONSPC.search(buf, 0)
|
||||
if not m:
|
||||
self.buf_skipstep = 0
|
||||
continue
|
||||
else:
|
||||
j = m.start(0)
|
||||
c = buf[j : j + 1]
|
||||
if c == b"%":
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(
|
||||
cur, parse_comment(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b"/":
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(
|
||||
cur, parse_literal(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c in b"-+":
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer(), c)
|
||||
)
|
||||
elif c.isdigit():
|
||||
self.buf_skipstep += j
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b".":
|
||||
self.buf_skipstep += j
|
||||
yield self._check_token(
|
||||
cur, parse_number(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c.isalpha():
|
||||
self.buf_skipstep += j
|
||||
yield self._check_token(
|
||||
cur, parse_keyword(bufpos + j, self.iter_buffer())
|
||||
)
|
||||
elif c == b"(":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self.paren = 1
|
||||
# self._current_parse_func = self._parse_string
|
||||
# return j + 1
|
||||
elif c == b"<":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wopen
|
||||
# return j + 1
|
||||
elif c == b">":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wclose
|
||||
# return j + 1
|
||||
else:
|
||||
self.buf_skipstep += j + 1
|
||||
yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))
|
||||
|
||||
def _check_token(self, cur: PInput, token: Token) -> Token:
|
||||
self.buf_skipstep += token.size
|
||||
return token
|
||||
@@ -8,9 +8,9 @@ class TokenizerError(PDFMajorException):
|
||||
class TokenizerEOF(TokenizerError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidToken(TokenizerError):
|
||||
def __init__(self, pos: int, token: bytes) -> None:
|
||||
super().__init__("Invalid Token Found at {pos}, of value={v}".format(
|
||||
pos=pos,
|
||||
v=token
|
||||
))
|
||||
def __init__(self, pos: int, token: bytes) -> None:
|
||||
super().__init__(
|
||||
"Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=token)
|
||||
)
|
||||
+22
-18
@@ -5,45 +5,49 @@ T = TypeVar("T")
|
||||
|
||||
|
||||
class Token(Generic[T]):
|
||||
__slots__ = ["pos", "size", "value"]
|
||||
__slots__ = ["pos", "size", "value"]
|
||||
|
||||
def __init__(self, pos: int, size: int, value: T) -> None:
|
||||
self.size = size
|
||||
self.pos = pos
|
||||
self.value = value
|
||||
def __init__(self, pos: int, size: int, value: T) -> None:
|
||||
self.size = size
|
||||
self.pos = pos
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "<{cls} pos={pos}>{value}</{cls}>".format(
|
||||
cls=self.__class__.__name__, pos=self.pos, value=self.value
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "<{cls} pos={pos}>{value}</{cls}>".format(
|
||||
cls=self.__class__.__name__, pos=self.pos, value=self.value
|
||||
)
|
||||
|
||||
|
||||
class TokenComment(Token[bytes]):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class TokenHex(Token[bytes]):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class TokenKeyword(Token[bytes]):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class TokenLiteral(Token[str]):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class TokenBoolean(Token[bool]):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class TokenString(Token[str]):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class TokenInteger(Token[int]):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class TokenDecimal(Token[Decimal]):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
TokenNumber = Union[TokenInteger, TokenDecimal]
|
||||
@@ -1,32 +1,29 @@
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenComment,
|
||||
TokenComment,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
EOL,
|
||||
EOL,
|
||||
)
|
||||
from typing import Iterator
|
||||
from .util import cmp_tsize, PInput
|
||||
|
||||
|
||||
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
|
||||
"""Parses input stream into a comment
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = EOL.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
return TokenComment(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken
|
||||
)
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = EOL.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
return TokenComment(initialpos, cmp_tsize(curpos, initialpos, j), curtoken)
|
||||
@@ -1,50 +1,34 @@
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenBoolean,
|
||||
TokenKeyword
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_KEYWORD
|
||||
)
|
||||
from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword
|
||||
from pdfmajor.tokenizer.constants import END_KEYWORD
|
||||
from typing import Iterator, Union
|
||||
from .util import cmp_tsize, PInput
|
||||
|
||||
|
||||
def parse_keyword(
|
||||
initialpos: int,
|
||||
inp: Iterator[PInput]
|
||||
initialpos: int, inp: Iterator[PInput]
|
||||
) -> Union[TokenKeyword, TokenBoolean]:
|
||||
"""Parses input stream into a keyword or bool
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = END_KEYWORD.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
if curtoken == b"true":
|
||||
return TokenBoolean(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
True
|
||||
)
|
||||
elif curtoken == b"false":
|
||||
return TokenBoolean(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
False
|
||||
)
|
||||
else:
|
||||
return TokenKeyword(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken
|
||||
)
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = END_KEYWORD.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
if curtoken == b"true":
|
||||
return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), True)
|
||||
elif curtoken == b"false":
|
||||
return TokenBoolean(initialpos, cmp_tsize(curpos, initialpos, j), False)
|
||||
else:
|
||||
return TokenKeyword(
|
||||
initialpos, cmp_tsize(curpos, initialpos, j), curtoken
|
||||
)
|
||||
@@ -8,55 +8,57 @@ from ..constants import HEX, END_LITERAL
|
||||
|
||||
from typing import Iterator, NamedTuple, Optional, Union
|
||||
|
||||
|
||||
@dataclass
|
||||
class LiteralParseState:
|
||||
curtoken: bytes
|
||||
hex_value: Optional[bytes] = None
|
||||
curtoken: bytes
|
||||
hex_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
"""Parses input stream into a literal
|
||||
"""Parses input stream into a literal
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenLiteral
|
||||
"""
|
||||
state = LiteralParseState(b"")
|
||||
for curpos, buf in inp:
|
||||
skip: int = 0
|
||||
for i in range(len(buf) + 1):
|
||||
assert i < len(buf), "MAX_LOOP_REACHED"
|
||||
if skip >= len(buf):
|
||||
break
|
||||
if state.hex_value is not None:
|
||||
for ci in range(len(buf[skip:])):
|
||||
c = buf[skip+ci:skip+ci+1]
|
||||
if HEX.match(c) and len(state.hex_value) < 2:
|
||||
state.hex_value += c
|
||||
else:
|
||||
if state.hex_value:
|
||||
state.curtoken += int2byte(int(state.hex_value, 16))
|
||||
state.hex_value = None
|
||||
skip += ci
|
||||
break
|
||||
else:
|
||||
m = END_LITERAL.search(buf[skip:], 0)
|
||||
if not m:
|
||||
state.curtoken += buf[skip:]
|
||||
skip = len(buf)
|
||||
else:
|
||||
j: int = m.start(0) + skip
|
||||
state.curtoken += buf[skip:j]
|
||||
c = buf[j : j + 1]
|
||||
if c == b"#":
|
||||
state.hex_value = b""
|
||||
skip = j+1
|
||||
continue
|
||||
else:
|
||||
return TokenLiteral(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
state.curtoken.decode()
|
||||
)
|
||||
Returns:
|
||||
TokenLiteral
|
||||
"""
|
||||
state = LiteralParseState(b"")
|
||||
for curpos, buf in inp:
|
||||
skip: int = 0
|
||||
for i in range(len(buf) + 1):
|
||||
assert i < len(buf), "MAX_LOOP_REACHED"
|
||||
if skip >= len(buf):
|
||||
break
|
||||
if state.hex_value is not None:
|
||||
for ci in range(len(buf[skip:])):
|
||||
c = buf[skip + ci : skip + ci + 1]
|
||||
if HEX.match(c) and len(state.hex_value) < 2:
|
||||
state.hex_value += c
|
||||
else:
|
||||
if state.hex_value:
|
||||
state.curtoken += int2byte(int(state.hex_value, 16))
|
||||
state.hex_value = None
|
||||
skip += ci
|
||||
break
|
||||
else:
|
||||
m = END_LITERAL.search(buf[skip:], 0)
|
||||
if not m:
|
||||
state.curtoken += buf[skip:]
|
||||
skip = len(buf)
|
||||
else:
|
||||
j: int = m.start(0) + skip
|
||||
state.curtoken += buf[skip:j]
|
||||
c = buf[j : j + 1]
|
||||
if c == b"#":
|
||||
state.hex_value = b""
|
||||
skip = j + 1
|
||||
continue
|
||||
else:
|
||||
return TokenLiteral(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
@@ -3,73 +3,70 @@ from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
|
||||
from pdfmajor.utils import int2byte
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenDecimal,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
TokenDecimal,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_NUMBER,
|
||||
END_NUMBER,
|
||||
)
|
||||
from typing import Iterator, Literal, Union
|
||||
from .util import cmp_tsize, PInput
|
||||
|
||||
VALID_SIGNS = Union[Literal[b'+'], Literal[b'-']]
|
||||
VALID_SIGNS = Union[Literal[b"+"], Literal[b"-"]]
|
||||
|
||||
|
||||
def parse_number(
|
||||
initialpos: int,
|
||||
inp: Iterator[PInput],
|
||||
sign: VALID_SIGNS = b'+'
|
||||
initialpos: int, inp: Iterator[PInput], sign: VALID_SIGNS = b"+"
|
||||
) -> TokenNumber:
|
||||
"""Parses input stream into a number
|
||||
"""Parses input stream into a number
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
sign (str): either a + or -
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
sign (str): either a + or -
|
||||
|
||||
Returns:
|
||||
TokenNumber
|
||||
"""
|
||||
assert sign in [b'+', b'-'], f"Number was provided an invalid sign {sign}"
|
||||
curtoken = b"" + sign
|
||||
is_dec = False
|
||||
for curpos, s in inp:
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if (c == b".") and not is_dec:
|
||||
is_dec = True
|
||||
curtoken += c
|
||||
s = s[j+1:]
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
continue
|
||||
else:
|
||||
nj = m.start(0)
|
||||
curtoken += s[:nj]
|
||||
j += nj
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode())
|
||||
)
|
||||
try:
|
||||
if is_dec:
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode())
|
||||
)
|
||||
else:
|
||||
return TokenInteger(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
int(curtoken)
|
||||
)
|
||||
except ValueError:
|
||||
raise InvalidToken(initialpos, curtoken)
|
||||
Returns:
|
||||
TokenNumber
|
||||
"""
|
||||
assert sign in [b"+", b"-"], f"Number was provided an invalid sign {sign}"
|
||||
curtoken = b"" + sign
|
||||
is_dec = False
|
||||
for curpos, s in inp:
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if (c == b".") and not is_dec:
|
||||
is_dec = True
|
||||
curtoken += c
|
||||
s = s[j + 1 :]
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
continue
|
||||
else:
|
||||
nj = m.start(0)
|
||||
curtoken += s[:nj]
|
||||
j += nj
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode()),
|
||||
)
|
||||
try:
|
||||
if is_dec:
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode()),
|
||||
)
|
||||
else:
|
||||
return TokenInteger(
|
||||
initialpos, cmp_tsize(curpos, initialpos, j), int(curtoken)
|
||||
)
|
||||
except ValueError:
|
||||
raise InvalidToken(initialpos, curtoken)
|
||||
@@ -3,66 +3,66 @@ from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
|
||||
from pdfmajor.utils import int2byte
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken
|
||||
from pdfmajor.tokenizer.token import (
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenKeyword, TokenLiteral,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
TokenString
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenKeyword,
|
||||
TokenLiteral,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
TokenString,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_KEYWORD,
|
||||
END_NUMBER,
|
||||
END_STRING,
|
||||
EOL,
|
||||
ESC_STRING,
|
||||
OCT_STRING
|
||||
END_KEYWORD,
|
||||
END_NUMBER,
|
||||
END_STRING,
|
||||
EOL,
|
||||
ESC_STRING,
|
||||
OCT_STRING,
|
||||
)
|
||||
from typing import Callable, Iterator, Literal, NamedTuple, Union
|
||||
|
||||
|
||||
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
||||
curtoken = b""
|
||||
paren = 1
|
||||
for curpos, s in inp:
|
||||
m = END_STRING.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"\\":
|
||||
self._current_parse_func = self._parse_string_1
|
||||
return j + 1
|
||||
elif c == b"(":
|
||||
paren += 1
|
||||
curtoken += c
|
||||
return j + 1
|
||||
elif c == b")":
|
||||
paren -= 1
|
||||
if paren: # WTF, they said balanced parens need no special treatment.
|
||||
curtoken += c
|
||||
return j + 1
|
||||
return TokenString(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken.decode()
|
||||
)
|
||||
curtoken = b""
|
||||
paren = 1
|
||||
for curpos, s in inp:
|
||||
m = END_STRING.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"\\":
|
||||
self._current_parse_func = self._parse_string_1
|
||||
return j + 1
|
||||
elif c == b"(":
|
||||
paren += 1
|
||||
curtoken += c
|
||||
return j + 1
|
||||
elif c == b")":
|
||||
paren -= 1
|
||||
if paren: # WTF, they said balanced parens need no special treatment.
|
||||
curtoken += c
|
||||
return j + 1
|
||||
return TokenString(
|
||||
initialpos, cmp_tsize(curpos, initialpos, j), curtoken.decode()
|
||||
)
|
||||
|
||||
|
||||
def parse_string_oct(initialpos: int, inp: Iterator[PInput]):
|
||||
curtoken: bytes = b""
|
||||
oct_val: bytes = b""
|
||||
for curpos, s in inp:
|
||||
for ci in range(len(s)):
|
||||
c = s[ci:ci+1]
|
||||
if OCT_STRING.match(c) and len(oct_val) < 3:
|
||||
oct_val += c
|
||||
elif oct_val:
|
||||
curtoken += int2byte(int(oct_val, 8))
|
||||
return (curtoken, curpos+ci)
|
||||
elif c in ESC_STRING:
|
||||
curtoken += int2byte(ESC_STRING[c])
|
||||
return (curtoken, curpos+ci+1)
|
||||
curtoken: bytes = b""
|
||||
oct_val: bytes = b""
|
||||
for curpos, s in inp:
|
||||
for ci in range(len(s)):
|
||||
c = s[ci : ci + 1]
|
||||
if OCT_STRING.match(c) and len(oct_val) < 3:
|
||||
oct_val += c
|
||||
elif oct_val:
|
||||
curtoken += int2byte(int(oct_val, 8))
|
||||
return (curtoken, curpos + ci)
|
||||
elif c in ESC_STRING:
|
||||
curtoken += int2byte(ESC_STRING[c])
|
||||
return (curtoken, curpos + ci + 1)
|
||||
@@ -1,25 +1,24 @@
|
||||
from pdfmajor.tokenizer.token import (
|
||||
Token
|
||||
)
|
||||
from pdfmajor.tokenizer.token import Token
|
||||
from typing import Callable, Iterator, NamedTuple
|
||||
|
||||
|
||||
class PInput(NamedTuple):
|
||||
pos: int
|
||||
buf: bytes
|
||||
pos: int
|
||||
buf: bytes
|
||||
|
||||
|
||||
ParserFunc = Callable[[int, Iterator[PInput]], Token]
|
||||
|
||||
|
||||
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
|
||||
"""Computes the size of the token based on its positional data
|
||||
"""Computes the size of the token based on its positional data
|
||||
|
||||
Args:
|
||||
curpos (int): current position of the buffer
|
||||
initialpos (int): initial position where the parser started working
|
||||
tokenend_idx (int): last relevant character index within the last buffered value
|
||||
Args:
|
||||
curpos (int): current position of the buffer
|
||||
initialpos (int): initial position where the parser started working
|
||||
tokenend_idx (int): last relevant character index within the last buffered value
|
||||
|
||||
Returns:
|
||||
int: length of index
|
||||
"""
|
||||
return curpos-initialpos+tokenend_idx
|
||||
Returns:
|
||||
int: length of index
|
||||
"""
|
||||
return curpos - initialpos + tokenend_idx
|
||||
Reference in new issue
Block a user