changed layout + literal tokens

This commit is contained in:
ari committed 2020-10-13 16:41:41 -04:00
1 parent da066edaf5
commit a0cc862804
12 files changed
+487 -106

No files matched your search

+24
View File
@@ -0,0 +1,24 @@
import io
from pdfmajor.parser.PSStackParser.PSBaseParser import PSBaseParser
TESTDATA = br'''%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#xbaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>
'''
parser = PSBaseParser(io.BytesIO(TESTDATA))
+5
View File
@@ -29,6 +29,11 @@ class TokenHex(Token[bytes]):
class TokenKeyword(Token[bytes]):
pass
class TokenLiteral(Token[bytes]):
pass
class TokenBoolean(Token[bool]):
pass
class TokenString(Token[str]):
pass
-104
View File
@@ -1,104 +0,0 @@
from decimal import Decimal
from pdfmajor.tokenizer.exceptions import InvalidToken
from pdfmajor.tokenizer.token import Token, TokenComment, TokenDecimal, TokenNumber, TokenInteger
from pdfmajor.tokenizer.constants import END_NUMBER, EOL
from typing import Callable, Iterator, NamedTuple
class PInput(NamedTuple):
pos: int
buf: bytes
ParserFunc = Callable[[int, Iterator[PInput]], Token]
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
"""Computes the size of the token based on its positional data
Args:
curpos (int): current position of the buffer
initialpos (int): initial position where the parser started working
tokenend_idx (int): last relevant character index within the last buffered value
Returns:
int: length of index
"""
return curpos-initialpos+tokenend_idx
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
"""Parses input stream into a comment
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenComment
"""
curtoken = b""
for curpos, s in inp:
m = EOL.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
return TokenComment(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken
)
def parse_number(initialpos: int, inp: Iterator[PInput]) -> TokenNumber:
"""Parses input stream into a number
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenNumber
"""
curtoken = b""
is_dec = False
for curpos, s in inp:
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if (c == b".") and not is_dec:
is_dec = True
curtoken += c
s = s[j+1:]
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
continue
else:
nj = m.start(0)
curtoken += s[:nj]
j += nj
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
Decimal(curtoken.decode())
)
try:
if is_dec:
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
Decimal(curtoken.decode())
)
else:
return TokenInteger(
initialpos,
cmp_tsize(curpos, initialpos, j),
int(curtoken)
)
except ValueError:
raise InvalidToken(initialpos, curtoken)
Whitespace-only changes.
@@ -0,0 +1,32 @@
from pdfmajor.tokenizer.token import (
TokenComment,
)
from pdfmajor.tokenizer.constants import (
EOL,
)
from typing import Iterator
from .util import cmp_tsize, PInput
def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment:
"""Parses input stream into a comment
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenComment
"""
curtoken = b""
for curpos, s in inp:
m = EOL.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
return TokenComment(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken
)
@@ -0,0 +1,50 @@
from pdfmajor.tokenizer.token import (
TokenBoolean,
TokenKeyword
)
from pdfmajor.tokenizer.constants import (
END_KEYWORD
)
from typing import Iterator, Union
from .util import cmp_tsize, PInput
def parse_keyword(
initialpos: int,
inp: Iterator[PInput]
) -> Union[TokenKeyword, TokenBoolean]:
"""Parses input stream into a keyword or bool
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
Union[TokenKeyword, TokenBoolean]
"""
curtoken = b""
for curpos, s in inp:
m = END_KEYWORD.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
if curtoken == b"true":
return TokenBoolean(
initialpos,
cmp_tsize(curpos, initialpos, j),
True
)
elif curtoken == b"false":
return TokenBoolean(
initialpos,
cmp_tsize(curpos, initialpos, j),
False
)
else:
return TokenKeyword(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken
)
@@ -0,0 +1,82 @@
from pdfmajor.tokenizer.token_parsers.util import cmp_tsize
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token import TokenLiteral
from pdfmajor.utils import int2byte
from ..constants import HEX, END_LITERAL
from typing import Iterator, NamedTuple, Optional, Union
def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
"""Parses input stream into a literal
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
Returns:
TokenLiteral
"""
curtoken = b""
parsing_hex_mode = False
hex_value = b""
for curpos, s in inp:
if parsing_hex_mode:
hex_value, curtoken, skip_idx = _parse_literal_hex(s, hex_value, curtoken)
if skip_idx is not None:
curpos, s = curpos + skip_idx, s[skip_idx:]
else:
continue
out = _parse_literal_raw(initialpos, curpos, s, curtoken)
if isinstance(out, TokenLiteral):
return out
else:
curtoken, parsing_hex_mode = out
if parsing_hex_mode:
hex_value = b""
class LiteralHexParseOut(NamedTuple):
hex_value: bytes
curtoken: bytes
skip_idx: Optional[int] = None
def _parse_literal_hex(
s: bytes,
hex_value: bytes,
curtoken: bytes
) -> LiteralHexParseOut:
for ci in range(len(s)):
c = s[ci:ci+1]
if HEX.match(c) and len(hex_value) < 2:
hex_value += c
elif hex_value:
curtoken += int2byte(int(hex_value, 16))
hex_value = b""
return LiteralHexParseOut(hex_value, curtoken, ci)
return LiteralHexParseOut(hex_value, curtoken)
class LiteralParseOut(NamedTuple):
curtoken: bytes
change_to_hex_mode: bool = False
def _parse_literal_raw(
initialpos: int,
curpos: int,
s: bytes,
curtoken: bytes
) -> Union[LiteralParseOut, TokenLiteral]:
m = END_LITERAL.search(s, 0)
if not m:
return LiteralParseOut(curtoken+s)
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if c == b"#":
return LiteralParseOut(curtoken, True)
curtoken = curtoken.decode()
return TokenLiteral(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken
)
@@ -0,0 +1,75 @@
from decimal import Decimal
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
from pdfmajor.utils import int2byte
from pdfmajor.tokenizer.exceptions import InvalidToken
from pdfmajor.tokenizer.token import (
TokenDecimal,
TokenNumber,
TokenInteger,
)
from pdfmajor.tokenizer.constants import (
END_NUMBER,
)
from typing import Iterator, Literal, Union
from .util import cmp_tsize, PInput
VALID_SIGNS = Union[Literal[b'+'], Literal[b'-']]
def parse_number(
initialpos: int,
inp: Iterator[PInput],
sign: VALID_SIGNS = b'+'
) -> TokenNumber:
"""Parses input stream into a number
Args:
initialpos (int): initial position where we started
inp (Iterator[PInput])
sign (str): either a + or -
Returns:
TokenNumber
"""
assert sign in [b'+', b'-'], f"Number was provided an invalid sign {sign}"
curtoken = b"" + sign
is_dec = False
for curpos, s in inp:
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if (c == b".") and not is_dec:
is_dec = True
curtoken += c
s = s[j+1:]
m = END_NUMBER.search(s, 0)
if not m:
curtoken += s
continue
else:
nj = m.start(0)
curtoken += s[:nj]
j += nj
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
Decimal(curtoken.decode())
)
try:
if is_dec:
return TokenDecimal(
initialpos,
cmp_tsize(curpos, initialpos, j),
Decimal(curtoken.decode())
)
else:
return TokenInteger(
initialpos,
cmp_tsize(curpos, initialpos, j),
int(curtoken)
)
except ValueError:
raise InvalidToken(initialpos, curtoken)
@@ -0,0 +1,68 @@
from decimal import Decimal
from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX
from pdfmajor.utils import int2byte
from pdfmajor.tokenizer.exceptions import InvalidToken
from pdfmajor.tokenizer.token import (
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenKeyword, TokenLiteral,
TokenNumber,
TokenInteger,
TokenString
)
from pdfmajor.tokenizer.constants import (
END_KEYWORD,
END_NUMBER,
END_STRING,
EOL,
ESC_STRING,
OCT_STRING
)
from typing import Callable, Iterator, Literal, NamedTuple, Union
def parse_string(initialpos: int, inp: Iterator[PInput]):
curtoken = b""
paren = 1
for curpos, s in inp:
m = END_STRING.search(s, 0)
if not m:
curtoken += s
else:
j = m.start(0)
curtoken += s[:j]
c = s[j : j + 1]
if c == b"\\":
self._current_parse_func = self._parse_string_1
return j + 1
elif c == b"(":
paren += 1
curtoken += c
return j + 1
elif c == b")":
paren -= 1
if paren: # WTF, they said balanced parens need no special treatment.
curtoken += c
return j + 1
return TokenString(
initialpos,
cmp_tsize(curpos, initialpos, j),
curtoken.decode()
)
def parse_string_oct(initialpos: int, inp: Iterator[PInput]):
curtoken: bytes = b""
oct_val: bytes = b""
for curpos, s in inp:
for ci in range(len(s)):
c = s[ci:ci+1]
if OCT_STRING.match(c) and len(oct_val) < 3:
oct_val += c
elif oct_val:
curtoken += int2byte(int(oct_val, 8))
return (curtoken, curpos+ci)
elif c in ESC_STRING:
curtoken += int2byte(ESC_STRING[c])
return (curtoken, curpos+ci+1)
+25
View File
@@ -0,0 +1,25 @@
from pdfmajor.tokenizer.token import (
Token
)
from typing import Callable, Iterator, NamedTuple
class PInput(NamedTuple):
pos: int
buf: bytes
ParserFunc = Callable[[int, Iterator[PInput]], Token]
def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int:
"""Computes the size of the token based on its positional data
Args:
curpos (int): current position of the buffer
initialpos (int): initial position where the parser started working
tokenend_idx (int): last relevant character index within the last buffered value
Returns:
int: length of index
"""
return curpos-initialpos+tokenend_idx
+88
View File
@@ -0,0 +1,88 @@
import io
from pdfmajor.tokenizer.token import TokenKeyword
from pdfmajor.tokenizer.constants import NONSPC
from typing import Iterator
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from .token_parser import PInput, parse_comment, parse_number
class PSTokenizer:
def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None:
self.fp = fp
self.bufsize = bufsize
self.bufpos: int = 0
self.buf: bytes = b''
self.buf_skipstep: int = 0
def iter_buffer(self) -> Iterator[PInput]:
while True:
if self.buf_skipstep > 0:
yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:])
self.buf_skipstep = 0
self.bufpos = self.fp.tell()
self.buf = self.fp.read(self.bufsize)
if not self.buf:
raise TokenizerEOF("Unexpected EOF")
else:
yield PInput(self.bufpos, self.buf)
def seek(self, pos: int):
self.fp.seek(pos)
def step_back(self, steps: int = 1):
"""Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize)
Args:
steps (int, optional): numbers of steps. Defaults to 1.
"""
self.fp.seek(self.fp.tell() - steps * self.bufsize)
def iter_tokens(self):
for bufpos, buf in self.iter_buffer():
m = NONSPC.search(buf, 0)
if not m:
self.buf_skipstep = 0
continue
else:
j = m.start(0)
c = buf[j : j + 1]
if c == b"%":
self.buf_skipstep = j + 1
yield parse_comment(bufpos + j, self.iter_buffer())
elif c == b"/":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_literal
# return j + 1
elif c in b"-+" or c.isdigit():
self.buf_skipstep = j + 1
yield parse_number(bufpos + j, self.iter_buffer(), c)
elif c == b".":
self.buf_skipstep = j
yield parse_number(bufpos + j, self.iter_buffer())
elif c.isalpha():
raise "WIP"
# self._curtoken = c
# self._current_parse_func = self._parse_keyword
# return j + 1
elif c == b"(":
raise "WIP"
# self._curtoken = b""
# self.paren = 1
# self._current_parse_func = self._parse_string
# return j + 1
elif c == b"<":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wopen
# return j + 1
elif c == b">":
raise "WIP"
# self._curtoken = b""
# self._current_parse_func = self._parse_wclose
# return j + 1
else:
self.buf_skipstep = j + 1
yield TokenKeyword(bufpos + j, 1, c)
+38 -2
View File
@@ -1,7 +1,11 @@
from decimal import Decimal
import io
from pdfmajor.tokenizer.token import TokenComment, TokenDecimal, TokenInteger
from pdfmajor.tokenizer.token_parser import PInput, parse_comment, parse_number
from pdfmajor.tokenizer.token import TokenBoolean, TokenComment, TokenDecimal, TokenInteger, TokenLiteral
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
from pdfmajor.tokenizer.token_parsers.number import parse_number
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
from unittest import TestCase
def make_stream_iter(data: bytes, initpos: int = 0):
@@ -62,3 +66,35 @@ class Basics(TestCase):
self.assertEqual(token.size, 3)
def test_parse_bool_true(self):
iter_stream = make_stream_iter(
b"true and the american way",
91
)
token = parse_keyword(91, iter_stream(33))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 91)
self.assertEqual(token.value, True)
self.assertEqual(token.size, 4)
def test_parse_bool_false(self):
iter_stream = make_stream_iter(
b"false mannn",
33
)
token = parse_keyword(33, iter_stream(100))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 33)
self.assertEqual(token.value, False)
self.assertEqual(token.size, 5)
def test_parse_literals(self):
iter_stream = make_stream_iter(
b"Some_Name /foo#5f#xbaa",
71
)
token = parse_literal(71, iter_stream(1))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, 'Some_Name')
self.assertEqual(token.size, 9)