changed layout + literal tokens
This commit is contained in:
1 parent
da066edaf5
commit
a0cc862804
12 files changed
+487
-106
No files matched your search
@@ -0,0 +1,24 @@
|
||||
import io
|
||||
from pdfmajor.parser.PSStackParser.PSBaseParser import PSBaseParser
|
||||
|
||||
TESTDATA = br'''%!PS
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#xbaa
|
||||
0 +1 -2 .5 1.234
|
||||
(abc) () (abc ( def ) ghi)
|
||||
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
||||
(this % is not a comment.)
|
||||
(foo
|
||||
baa)
|
||||
(foo\
|
||||
baa)
|
||||
<> <20> < 40 4020 >
|
||||
<abcd00
|
||||
12345>
|
||||
func/a/b{(c)do*}def
|
||||
[ 1 (z) ! ]
|
||||
<< /foo (bar) >>
|
||||
'''
|
||||
|
||||
parser = PSBaseParser(io.BytesIO(TESTDATA))
|
||||
@@ -29,6 +29,11 @@ class TokenHex(Token[bytes]):
|
||||
class TokenKeyword(Token[bytes]):
|
||||
pass
|
||||
|
||||
class TokenLiteral(Token[bytes]):
|
||||
pass
|
||||
|
||||
class TokenBoolean(Token[bool]):
|
||||
pass
|
||||
|
||||
class TokenString(Token[str]):
|
||||
pass
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken
|
||||
from pdfmajor.tokenizer.token import Token, TokenComment, TokenDecimal, TokenNumber, TokenInteger
|
||||
from pdfmajor.tokenizer.constants import END_NUMBER, EOL
|
||||
from typing import Callable, Iterator, NamedTuple
|
||||
|
||||
|
||||
class PInput(NamedTuple):
|
||||
pos: int
|
||||
buf: bytes
|
||||
|
||||
|
||||
ParserFunc = Callable[[int, Iterator[PInput]], Token]
|
||||
|
||||
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
|
||||
"""Computes the size of the token based on its positional data
|
||||
|
||||
Args:
|
||||
curpos (int): current position of the buffer
|
||||
initialpos (int): initial position where the parser started working
|
||||
tokenend_idx (int): last relevant character index within the last buffered value
|
||||
|
||||
Returns:
|
||||
int: length of index
|
||||
"""
|
||||
return curpos-initialpos+tokenend_idx
|
||||
|
||||
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = EOL.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
return TokenComment(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken
|
||||
)
|
||||
|
||||
def parse_number(initialpos: int, inp: Iterator[PInput]) -> TokenNumber:
|
||||
"""Parses input stream into a number
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenNumber
|
||||
"""
|
||||
curtoken = b""
|
||||
is_dec = False
|
||||
for curpos, s in inp:
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if (c == b".") and not is_dec:
|
||||
is_dec = True
|
||||
curtoken += c
|
||||
s = s[j+1:]
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
continue
|
||||
else:
|
||||
nj = m.start(0)
|
||||
curtoken += s[:nj]
|
||||
j += nj
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode())
|
||||
)
|
||||
try:
|
||||
if is_dec:
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode())
|
||||
)
|
||||
else:
|
||||
return TokenInteger(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
int(curtoken)
|
||||
)
|
||||
except ValueError:
|
||||
raise InvalidToken(initialpos, curtoken)
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,32 @@
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenComment,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
EOL,
|
||||
)
|
||||
from typing import Iterator
|
||||
from .util import cmp_tsize, PInput
|
||||
|
||||
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
|
||||
"""Parses input stream into a comment
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenComment
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = EOL.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
return TokenComment(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken
|
||||
)
|
||||
@@ -0,0 +1,50 @@
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenBoolean,
|
||||
TokenKeyword
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_KEYWORD
|
||||
)
|
||||
from typing import Iterator, Union
|
||||
from .util import cmp_tsize, PInput
|
||||
|
||||
def parse_keyword(
|
||||
initialpos: int,
|
||||
inp: Iterator[PInput]
|
||||
) -> Union[TokenKeyword, TokenBoolean]:
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
"""
|
||||
curtoken = b""
|
||||
for curpos, s in inp:
|
||||
m = END_KEYWORD.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
if curtoken == b"true":
|
||||
return TokenBoolean(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
True
|
||||
)
|
||||
elif curtoken == b"false":
|
||||
return TokenBoolean(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
False
|
||||
)
|
||||
else:
|
||||
return TokenKeyword(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken
|
||||
)
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
|
||||
from pdfmajor.tokenizer.token_parsers.util import cmp_tsize
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token import TokenLiteral
|
||||
from pdfmajor.utils import int2byte
|
||||
from ..constants import HEX, END_LITERAL
|
||||
|
||||
from typing import Iterator, NamedTuple, Optional, Union
|
||||
|
||||
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
||||
"""Parses input stream into a literal
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
|
||||
Returns:
|
||||
TokenLiteral
|
||||
"""
|
||||
curtoken = b""
|
||||
parsing_hex_mode = False
|
||||
hex_value = b""
|
||||
for curpos, s in inp:
|
||||
if parsing_hex_mode:
|
||||
hex_value, curtoken, skip_idx = _parse_literal_hex(s, hex_value, curtoken)
|
||||
if skip_idx is not None:
|
||||
curpos, s = curpos + skip_idx, s[skip_idx:]
|
||||
else:
|
||||
continue
|
||||
out = _parse_literal_raw(initialpos, curpos, s, curtoken)
|
||||
if isinstance(out, TokenLiteral):
|
||||
return out
|
||||
else:
|
||||
curtoken, parsing_hex_mode = out
|
||||
if parsing_hex_mode:
|
||||
hex_value = b""
|
||||
|
||||
class LiteralHexParseOut(NamedTuple):
|
||||
hex_value: bytes
|
||||
curtoken: bytes
|
||||
skip_idx: Optional[int] = None
|
||||
|
||||
def _parse_literal_hex(
|
||||
s: bytes,
|
||||
hex_value: bytes,
|
||||
curtoken: bytes
|
||||
) -> LiteralHexParseOut:
|
||||
for ci in range(len(s)):
|
||||
c = s[ci:ci+1]
|
||||
if HEX.match(c) and len(hex_value) < 2:
|
||||
hex_value += c
|
||||
elif hex_value:
|
||||
curtoken += int2byte(int(hex_value, 16))
|
||||
hex_value = b""
|
||||
return LiteralHexParseOut(hex_value, curtoken, ci)
|
||||
return LiteralHexParseOut(hex_value, curtoken)
|
||||
|
||||
class LiteralParseOut(NamedTuple):
|
||||
curtoken: bytes
|
||||
change_to_hex_mode: bool = False
|
||||
|
||||
def _parse_literal_raw(
|
||||
initialpos: int,
|
||||
curpos: int,
|
||||
s: bytes,
|
||||
curtoken: bytes
|
||||
) -> Union[LiteralParseOut, TokenLiteral]:
|
||||
m = END_LITERAL.search(s, 0)
|
||||
if not m:
|
||||
return LiteralParseOut(curtoken+s)
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"#":
|
||||
return LiteralParseOut(curtoken, True)
|
||||
curtoken = curtoken.decode()
|
||||
return TokenLiteral(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
|
||||
from pdfmajor.utils import int2byte
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenDecimal,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_NUMBER,
|
||||
)
|
||||
from typing import Iterator, Literal, Union
|
||||
from .util import cmp_tsize, PInput
|
||||
|
||||
VALID_SIGNS = Union[Literal[b'+'], Literal[b'-']]
|
||||
|
||||
def parse_number(
|
||||
initialpos: int,
|
||||
inp: Iterator[PInput],
|
||||
sign: VALID_SIGNS = b'+'
|
||||
) -> TokenNumber:
|
||||
"""Parses input stream into a number
|
||||
|
||||
Args:
|
||||
initialpos (int): initial position where we started
|
||||
inp (Iterator[PInput])
|
||||
sign (str): either a + or -
|
||||
|
||||
Returns:
|
||||
TokenNumber
|
||||
"""
|
||||
assert sign in [b'+', b'-'], f"Number was provided an invalid sign {sign}"
|
||||
curtoken = b"" + sign
|
||||
is_dec = False
|
||||
for curpos, s in inp:
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if (c == b".") and not is_dec:
|
||||
is_dec = True
|
||||
curtoken += c
|
||||
s = s[j+1:]
|
||||
m = END_NUMBER.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
continue
|
||||
else:
|
||||
nj = m.start(0)
|
||||
curtoken += s[:nj]
|
||||
j += nj
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode())
|
||||
)
|
||||
try:
|
||||
if is_dec:
|
||||
return TokenDecimal(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
Decimal(curtoken.decode())
|
||||
)
|
||||
else:
|
||||
return TokenInteger(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
int(curtoken)
|
||||
)
|
||||
except ValueError:
|
||||
raise InvalidToken(initialpos, curtoken)
|
||||
@@ -0,0 +1,68 @@
|
||||
from decimal import Decimal
|
||||
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
|
||||
from pdfmajor.utils import int2byte
|
||||
from pdfmajor.tokenizer.exceptions import InvalidToken
|
||||
from pdfmajor.tokenizer.token import (
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenKeyword, TokenLiteral,
|
||||
TokenNumber,
|
||||
TokenInteger,
|
||||
TokenString
|
||||
)
|
||||
from pdfmajor.tokenizer.constants import (
|
||||
END_KEYWORD,
|
||||
END_NUMBER,
|
||||
END_STRING,
|
||||
EOL,
|
||||
ESC_STRING,
|
||||
OCT_STRING
|
||||
)
|
||||
from typing import Callable, Iterator, Literal, NamedTuple, Union
|
||||
|
||||
|
||||
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
||||
curtoken = b""
|
||||
paren = 1
|
||||
for curpos, s in inp:
|
||||
m = END_STRING.search(s, 0)
|
||||
if not m:
|
||||
curtoken += s
|
||||
else:
|
||||
j = m.start(0)
|
||||
curtoken += s[:j]
|
||||
c = s[j : j + 1]
|
||||
if c == b"\\":
|
||||
self._current_parse_func = self._parse_string_1
|
||||
return j + 1
|
||||
elif c == b"(":
|
||||
paren += 1
|
||||
curtoken += c
|
||||
return j + 1
|
||||
elif c == b")":
|
||||
paren -= 1
|
||||
if paren: # WTF, they said balanced parens need no special treatment.
|
||||
curtoken += c
|
||||
return j + 1
|
||||
return TokenString(
|
||||
initialpos,
|
||||
cmp_tsize(curpos, initialpos, j),
|
||||
curtoken.decode()
|
||||
)
|
||||
|
||||
def parse_string_oct(initialpos: int, inp: Iterator[PInput]):
|
||||
curtoken: bytes = b""
|
||||
oct_val: bytes = b""
|
||||
for curpos, s in inp:
|
||||
for ci in range(len(s)):
|
||||
c = s[ci:ci+1]
|
||||
if OCT_STRING.match(c) and len(oct_val) < 3:
|
||||
oct_val += c
|
||||
elif oct_val:
|
||||
curtoken += int2byte(int(oct_val, 8))
|
||||
return (curtoken, curpos+ci)
|
||||
elif c in ESC_STRING:
|
||||
curtoken += int2byte(ESC_STRING[c])
|
||||
return (curtoken, curpos+ci+1)
|
||||
@@ -0,0 +1,25 @@
|
||||
from pdfmajor.tokenizer.token import (
|
||||
Token
|
||||
)
|
||||
from typing import Callable, Iterator, NamedTuple
|
||||
|
||||
|
||||
class PInput(NamedTuple):
|
||||
pos: int
|
||||
buf: bytes
|
||||
|
||||
|
||||
ParserFunc = Callable[[int, Iterator[PInput]], Token]
|
||||
|
||||
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
|
||||
"""Computes the size of the token based on its positional data
|
||||
|
||||
Args:
|
||||
curpos (int): current position of the buffer
|
||||
initialpos (int): initial position where the parser started working
|
||||
tokenend_idx (int): last relevant character index within the last buffered value
|
||||
|
||||
Returns:
|
||||
int: length of index
|
||||
"""
|
||||
return curpos-initialpos+tokenend_idx
|
||||
@@ -0,0 +1,88 @@
|
||||
|
||||
import io
|
||||
from pdfmajor.tokenizer.token import TokenKeyword
|
||||
from pdfmajor.tokenizer.constants import NONSPC
|
||||
from typing import Iterator
|
||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||
from .token_parser import PInput, parse_comment, parse_number
|
||||
|
||||
|
||||
class PSTokenizer:
|
||||
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
|
||||
self.fp = fp
|
||||
self.bufsize = bufsize
|
||||
self.bufpos: int = 0
|
||||
self.buf: bytes = b''
|
||||
self.buf_skipstep: int = 0
|
||||
|
||||
def iter_buffer(self) -> Iterator[PInput]:
|
||||
while True:
|
||||
if self.buf_skipstep > 0:
|
||||
yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:])
|
||||
self.buf_skipstep = 0
|
||||
self.bufpos = self.fp.tell()
|
||||
self.buf = self.fp.read(self.bufsize)
|
||||
if not self.buf:
|
||||
raise TokenizerEOF("Unexpected EOF")
|
||||
else:
|
||||
yield PInput(self.bufpos, self.buf)
|
||||
|
||||
def seek(self, pos: int):
|
||||
self.fp.seek(pos)
|
||||
|
||||
def step_back(self, steps: int = 1):
|
||||
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
|
||||
|
||||
Args:
|
||||
steps (int, optional): numbers of steps. Defaults to 1.
|
||||
"""
|
||||
self.fp.seek(self.fp.tell() - steps * self.bufsize)
|
||||
|
||||
def iter_tokens(self):
|
||||
for bufpos, buf in self.iter_buffer():
|
||||
m = NONSPC.search(buf, 0)
|
||||
if not m:
|
||||
self.buf_skipstep = 0
|
||||
continue
|
||||
else:
|
||||
j = m.start(0)
|
||||
c = buf[j : j + 1]
|
||||
if c == b"%":
|
||||
self.buf_skipstep = j + 1
|
||||
yield parse_comment(bufpos + j, self.iter_buffer())
|
||||
elif c == b"/":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_literal
|
||||
# return j + 1
|
||||
elif c in b"-+" or c.isdigit():
|
||||
self.buf_skipstep = j + 1
|
||||
yield parse_number(bufpos + j, self.iter_buffer(), c)
|
||||
elif c == b".":
|
||||
self.buf_skipstep = j
|
||||
yield parse_number(bufpos + j, self.iter_buffer())
|
||||
elif c.isalpha():
|
||||
raise "WIP"
|
||||
# self._curtoken = c
|
||||
# self._current_parse_func = self._parse_keyword
|
||||
# return j + 1
|
||||
elif c == b"(":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self.paren = 1
|
||||
# self._current_parse_func = self._parse_string
|
||||
# return j + 1
|
||||
elif c == b"<":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wopen
|
||||
# return j + 1
|
||||
elif c == b">":
|
||||
raise "WIP"
|
||||
# self._curtoken = b""
|
||||
# self._current_parse_func = self._parse_wclose
|
||||
# return j + 1
|
||||
else:
|
||||
self.buf_skipstep = j + 1
|
||||
yield TokenKeyword(bufpos + j, 1, c)
|
||||
|
||||
+38
-2
@@ -1,7 +1,11 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
from pdfmajor.tokenizer.token import TokenComment, TokenDecimal, TokenInteger
|
||||
from pdfmajor.tokenizer.token_parser import PInput, parse_comment, parse_number
|
||||
from pdfmajor.tokenizer.token import TokenBoolean, TokenComment, TokenDecimal, TokenInteger, TokenLiteral
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
||||
from pdfmajor.tokenizer.token_parsers.number import parse_number
|
||||
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
|
||||
from unittest import TestCase
|
||||
|
||||
def make_stream_iter(data: bytes, initpos: int = 0):
|
||||
@@ -62,3 +66,35 @@ class Basics(TestCase):
|
||||
self.assertEqual(token.size, 3)
|
||||
|
||||
|
||||
def test_parse_bool_true(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"true and the american way",
|
||||
91
|
||||
)
|
||||
token = parse_keyword(91, iter_stream(33))
|
||||
self.assertIsInstance(token, TokenBoolean)
|
||||
self.assertEqual(token.pos, 91)
|
||||
self.assertEqual(token.value, True)
|
||||
self.assertEqual(token.size, 4)
|
||||
|
||||
def test_parse_bool_false(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"false mannn",
|
||||
33
|
||||
)
|
||||
token = parse_keyword(33, iter_stream(100))
|
||||
self.assertIsInstance(token, TokenBoolean)
|
||||
self.assertEqual(token.pos, 33)
|
||||
self.assertEqual(token.value, False)
|
||||
self.assertEqual(token.size, 5)
|
||||
|
||||
def test_parse_literals(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"Some_Name /foo#5f#xbaa",
|
||||
71
|
||||
)
|
||||
token = parse_literal(71, iter_stream(1))
|
||||
self.assertIsInstance(token, TokenLiteral)
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, 'Some_Name')
|
||||
self.assertEqual(token.size, 9)
|
||||
Reference in new issue
Block a user