strict hex format
This commit is contained in:
1 parent
5f19f71c4d
commit
4a6b0608a0
5 files changed
+150
-106
No files matched your search
@@ -71,7 +71,7 @@ class PSTokenizer:
|
|||||||
elif c == b"/":
|
elif c == b"/":
|
||||||
self.inc_buf_skipstep(j + 1)
|
self.inc_buf_skipstep(j + 1)
|
||||||
yield self._check_token(
|
yield self._check_token(
|
||||||
cur, parse_literal(bufpos + j, self.iter_buffer())
|
cur, parse_literal(bufpos + j + 1, self.iter_buffer())
|
||||||
)
|
)
|
||||||
elif c in b"-+":
|
elif c in b"-+":
|
||||||
self.inc_buf_skipstep(j + 1)
|
self.inc_buf_skipstep(j + 1)
|
||||||
|
|||||||
@@ -12,5 +12,12 @@ class TokenizerEOF(TokenizerError):
|
|||||||
class InvalidToken(TokenizerError):
|
class InvalidToken(TokenizerError):
|
||||||
def __init__(self, pos: int, token: bytes) -> None:
|
def __init__(self, pos: int, token: bytes) -> None:
|
||||||
super().__init__(
|
super().__init__(
|
||||||
"Invalid Token Found at {pos}, of value={v:r}".format(pos=pos, v=token)
|
"Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidHexToken(TokenizerError):
|
||||||
|
def __init__(self, pos: int, token: bytes) -> None:
|
||||||
|
super().__init__(
|
||||||
|
"Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token))
|
||||||
)
|
)
|
||||||
@@ -1,5 +1,10 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF, TokenizerError
|
from pdfmajor.tokenizer.exceptions import (
|
||||||
|
InvalidHexToken,
|
||||||
|
InvalidToken,
|
||||||
|
TokenizerEOF,
|
||||||
|
TokenizerError,
|
||||||
|
)
|
||||||
|
|
||||||
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize
|
from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt, cmp_tsize
|
||||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||||
@@ -29,18 +34,19 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
|||||||
state = LiteralParseState(b"")
|
state = LiteralParseState(b"")
|
||||||
for curpos, buf in inp:
|
for curpos, buf in inp:
|
||||||
it = SafeBufferIt(buf)
|
it = SafeBufferIt(buf)
|
||||||
for _ in it.into_iter():
|
for subbuf in it.into_iter():
|
||||||
if state.hex_value is not None:
|
if state.hex_value is not None:
|
||||||
for ci in range(len(buf[it.skip :])):
|
for ci in range(len(subbuf)):
|
||||||
c = buf[it.skip + ci : it.skip + ci + 1]
|
c = subbuf[ci : ci + 1]
|
||||||
if HEX.match(c) and len(state.hex_value) < 2:
|
if HEX.match(c) and len(state.hex_value) < 2:
|
||||||
state.hex_value += c
|
state.hex_value += c
|
||||||
else:
|
it.skip += 1
|
||||||
if state.hex_value:
|
if len(state.hex_value) >= 2:
|
||||||
state.curtoken += int2byte(int(state.hex_value, 16))
|
state.curtoken += int2byte(int(state.hex_value, 16))
|
||||||
state.hex_value = None
|
state.hex_value = None
|
||||||
it.skip += ci
|
break
|
||||||
break
|
else:
|
||||||
|
raise InvalidHexToken(curpos + ci, c)
|
||||||
else:
|
else:
|
||||||
m = END_LITERAL.search(buf[it.skip :], 0)
|
m = END_LITERAL.search(buf[it.skip :], 0)
|
||||||
if not m:
|
if not m:
|
||||||
@@ -57,7 +63,7 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral:
|
|||||||
else:
|
else:
|
||||||
return TokenLiteral(
|
return TokenLiteral(
|
||||||
initialpos,
|
initialpos,
|
||||||
cmp_tsize(curpos, initialpos, j) - 1,
|
cmp_tsize(curpos, initialpos, j),
|
||||||
state.curtoken.decode(),
|
state.curtoken.decode(),
|
||||||
)
|
)
|
||||||
raise TokenizerEOF
|
raise TokenizerEOF
|
||||||
@@ -53,7 +53,7 @@ def parse_number(
|
|||||||
j += nj
|
j += nj
|
||||||
return TokenDecimal(
|
return TokenDecimal(
|
||||||
initialpos,
|
initialpos,
|
||||||
cmp_tsize(curpos, initialpos, j),
|
cmp_tsize(curpos, initialpos, j + 1),
|
||||||
Decimal(curtoken.decode()),
|
Decimal(curtoken.decode()),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
|
|||||||
+124
-93
@@ -38,90 +38,119 @@ def make_stream_iter(data: bytes, initpos: int = 0):
|
|||||||
|
|
||||||
class Individual(TestCase):
|
class Individual(TestCase):
|
||||||
def test_comment_parser_ln(self):
|
def test_comment_parser_ln(self):
|
||||||
iter_stream = make_stream_iter(
|
for buf_size in [2, 7, 11, 150]:
|
||||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
with self.subTest(buf_size=buf_size):
|
||||||
)
|
iter_stream = make_stream_iter(
|
||||||
token = parse_comment(0, iter_stream(3))
|
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||||
self.assertIsInstance(token, TokenComment)
|
)
|
||||||
self.assertEqual(token.pos, 0)
|
token = parse_comment(0, iter_stream(buf_size))
|
||||||
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
|
self.assertIsInstance(token, TokenComment)
|
||||||
self.assertEqual(token.size, 40)
|
self.assertEqual(token.pos, 0)
|
||||||
|
self.assertEqual(
|
||||||
|
token.value, b"this is a lengthy comment that ends here"
|
||||||
|
)
|
||||||
|
self.assertEqual(token.size, 40)
|
||||||
|
|
||||||
def test_comment_parser_lr(self):
|
def test_comment_parser_lr(self):
|
||||||
iter_stream = make_stream_iter(
|
for buf_size in [2, 7, 11, 150]:
|
||||||
b"this is a lengthy comment that ends here\rso this is not reachable"
|
with self.subTest(buf_size=buf_size):
|
||||||
)
|
iter_stream = make_stream_iter(
|
||||||
token = parse_comment(0, iter_stream(7))
|
b"this is a lengthy comment that ends here\rso this is not reachable"
|
||||||
self.assertIsInstance(token, TokenComment)
|
)
|
||||||
self.assertEqual(token.pos, 0)
|
token = parse_comment(0, iter_stream(buf_size))
|
||||||
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
|
self.assertIsInstance(token, TokenComment)
|
||||||
self.assertEqual(token.size, 40)
|
self.assertEqual(token.pos, 0)
|
||||||
|
self.assertEqual(
|
||||||
|
token.value, b"this is a lengthy comment that ends here"
|
||||||
|
)
|
||||||
|
self.assertEqual(token.size, 40)
|
||||||
|
|
||||||
def test_parse_float(self):
|
def test_parse_float(self):
|
||||||
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
|
for buf_size in [2, 3, 50]:
|
||||||
token = parse_number(5, iter_stream(3))
|
with self.subTest(buf_size=buf_size):
|
||||||
self.assertIsInstance(token, TokenDecimal)
|
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
|
||||||
self.assertEqual(token.pos, 5)
|
token = parse_number(5, iter_stream(buf_size))
|
||||||
self.assertEqual(token.value, Decimal("0.12"))
|
self.assertIsInstance(token, TokenDecimal)
|
||||||
self.assertEqual(token.size, 5)
|
self.assertEqual(token.pos, 5)
|
||||||
|
self.assertEqual(token.value, Decimal("0.12"))
|
||||||
|
self.assertEqual(token.size, 5)
|
||||||
|
|
||||||
def test_parse_int(self):
|
def test_parse_int(self):
|
||||||
iter_stream = make_stream_iter(b"130 oh oh 0.12 + some invalid text", 11)
|
for buf_size in [2, 3, 50]:
|
||||||
token = parse_number(11, iter_stream(10))
|
with self.subTest(buf_size=buf_size):
|
||||||
self.assertIsInstance(token, TokenInteger)
|
iter_stream = make_stream_iter(
|
||||||
self.assertEqual(token.pos, 11)
|
b"130 oh oh 0.12 + some invalid text", 11
|
||||||
self.assertEqual(token.value, 130)
|
)
|
||||||
self.assertEqual(token.size, 3)
|
token = parse_number(11, iter_stream(buf_size))
|
||||||
|
self.assertIsInstance(token, TokenInteger)
|
||||||
|
self.assertEqual(token.pos, 11)
|
||||||
|
self.assertEqual(token.value, 130)
|
||||||
|
self.assertEqual(token.size, 3)
|
||||||
|
|
||||||
def test_parse_bool_true(self):
|
def test_parse_bool_true(self):
|
||||||
iter_stream = make_stream_iter(b"true and the american way", 91)
|
for buf_size in [2, 3, 50]:
|
||||||
token = parse_keyword(91, iter_stream(33))
|
with self.subTest(buf_size=buf_size):
|
||||||
self.assertIsInstance(token, TokenBoolean)
|
iter_stream = make_stream_iter(b"true and the american way", 91)
|
||||||
self.assertEqual(token.pos, 91)
|
token = parse_keyword(91, iter_stream(buf_size))
|
||||||
self.assertEqual(token.value, True)
|
self.assertIsInstance(token, TokenBoolean)
|
||||||
self.assertEqual(token.size, 4)
|
self.assertEqual(token.pos, 91)
|
||||||
|
self.assertEqual(token.value, True)
|
||||||
|
self.assertEqual(token.size, 4)
|
||||||
|
|
||||||
def test_parse_bool_false(self):
|
def test_parse_bool_false(self):
|
||||||
iter_stream = make_stream_iter(b"false mannn", 33)
|
for buf_size in [2, 3, 50]:
|
||||||
token = parse_keyword(33, iter_stream(100))
|
with self.subTest(buf_size=buf_size):
|
||||||
self.assertIsInstance(token, TokenBoolean)
|
iter_stream = make_stream_iter(b"false mannn", 33)
|
||||||
self.assertEqual(token.pos, 33)
|
token = parse_keyword(33, iter_stream(buf_size))
|
||||||
self.assertEqual(token.value, False)
|
self.assertIsInstance(token, TokenBoolean)
|
||||||
self.assertEqual(token.size, 5)
|
self.assertEqual(token.pos, 33)
|
||||||
|
self.assertEqual(token.value, False)
|
||||||
|
self.assertEqual(token.size, 5)
|
||||||
|
|
||||||
def test_parse_literals(self):
|
def test_parse_literals(self):
|
||||||
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
|
for buf_size in [2, 3, 50, 444]:
|
||||||
token = parse_literal(71, iter_stream(1))
|
with self.subTest(buf_size=buf_size):
|
||||||
self.assertIsInstance(token, TokenLiteral)
|
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
|
||||||
self.assertEqual(token.pos, 71)
|
token = parse_literal(71, iter_stream(buf_size))
|
||||||
self.assertEqual(token.value, "Some_Name")
|
self.assertIsInstance(token, TokenLiteral)
|
||||||
self.assertEqual(token.size, 9)
|
self.assertEqual(token.pos, 71)
|
||||||
|
self.assertEqual(token.value, "Some_Name")
|
||||||
|
self.assertEqual(token.size, 9)
|
||||||
|
|
||||||
def test_parse_literals_hex(self):
|
def test_parse_literals_hex(self):
|
||||||
iter_stream = make_stream_iter(br"foo#5f#xbaa ", 71)
|
for buf_size in [2]:
|
||||||
token = parse_literal(71, iter_stream(100))
|
with self.subTest(buf_size=buf_size):
|
||||||
self.assertIsInstance(token, TokenLiteral)
|
iter_stream = make_stream_iter(br"foo#5faa ", 71)
|
||||||
self.assertEqual(token.pos, 71)
|
token = parse_literal(71, iter_stream(buf_size))
|
||||||
self.assertEqual(token.value, "foo_xbaa")
|
self.assertIsInstance(token, TokenLiteral)
|
||||||
self.assertEqual(token.size, 11)
|
self.assertEqual(token.pos, 71)
|
||||||
|
self.assertEqual(token.value, "foo_aa")
|
||||||
|
self.assertEqual(token.size, 8)
|
||||||
|
|
||||||
def test_parse_string(self):
|
def test_parse_string(self):
|
||||||
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
|
for buf_size in [2]:
|
||||||
token = parse_string(88, iter_stream(452))
|
with self.subTest(buf_size=buf_size):
|
||||||
self.assertIsInstance(token, TokenString)
|
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
|
||||||
self.assertEqual(token.pos, 88)
|
token = parse_string(88, iter_stream(buf_size))
|
||||||
self.assertEqual(token.value, "this % is not a comment.")
|
self.assertIsInstance(token, TokenString)
|
||||||
self.assertEqual(token.size, 24)
|
self.assertEqual(token.pos, 88)
|
||||||
|
self.assertEqual(token.value, "this % is not a comment.")
|
||||||
|
self.assertEqual(token.size, 24)
|
||||||
|
|
||||||
|
|
||||||
class Tokenizer(TestCase):
|
class Tokenizer(TestCase):
|
||||||
|
def run_test(self, raw: bytes, expected: List[Token]):
|
||||||
|
tokenizer = PSTokenizer(io.BytesIO(raw))
|
||||||
|
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
|
||||||
|
self.assertEqual(token, etoken, f"Failed at example {i}")
|
||||||
|
|
||||||
def test_case1(self):
|
def test_case1(self):
|
||||||
tokenizer = PSTokenizer(
|
tokenizer = PSTokenizer(
|
||||||
io.BytesIO(
|
io.BytesIO(
|
||||||
br"""%!PS
|
br"""%!PS
|
||||||
begin end
|
begin end
|
||||||
" @ #
|
" @ #
|
||||||
/a/BCD /Some_Name /foo#5f#xbaa
|
/a/BCD /Some_Name /foo#5f#6eaa
|
||||||
0 +1 -2 .5 1.234
|
0 +1 -2 .5 1.234
|
||||||
(abc) () (abc ( def ) ghi)
|
(abc) () (abc ( def ) ghi)
|
||||||
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
||||||
@@ -142,47 +171,49 @@ func/a/b{(c)do*}def
|
|||||||
print(token)
|
print(token)
|
||||||
|
|
||||||
def test_case2(self):
|
def test_case2(self):
|
||||||
tokenizer = PSTokenizer(
|
self.run_test(
|
||||||
io.BytesIO(
|
raw=br"""10.32
|
||||||
br"""10.32
|
|
||||||
0 +1 -2 .5 1.234
|
0 +1 -2 .5 1.234
|
||||||
-1.234 990"""
|
-1.234 990""",
|
||||||
)
|
expected=[
|
||||||
|
TokenDecimal(pos=0, size=5, value=Decimal("10.32")),
|
||||||
|
TokenInteger(pos=6, size=1, value=0),
|
||||||
|
TokenInteger(pos=8, size=2, value=1),
|
||||||
|
TokenInteger(pos=11, size=2, value=-2),
|
||||||
|
TokenDecimal(pos=14, size=2, value=Decimal("0.5")),
|
||||||
|
TokenDecimal(pos=17, size=5, value=Decimal("1.234")),
|
||||||
|
TokenDecimal(pos=23, size=6, value=Decimal("-1.234")),
|
||||||
|
TokenInteger(pos=30, size=3, value=990),
|
||||||
|
],
|
||||||
)
|
)
|
||||||
for token in tokenizer.iter_tokens():
|
|
||||||
print(token)
|
|
||||||
|
|
||||||
def test_case_words1(self):
|
def test_case_words1(self):
|
||||||
tokenizer = PSTokenizer(
|
self.run_test(
|
||||||
io.BytesIO(
|
raw=br"""%!PS
|
||||||
br"""%!PS
|
|
||||||
% This is some secret sauce
|
% This is some secret sauce
|
||||||
begin end
|
begin end
|
||||||
" @ #
|
" @ #
|
||||||
/a/BCD /Some_Name /foo#5f#xbaa /bar#6a
|
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
|
||||||
/need /oh1oh
|
/need /oh1oh
|
||||||
func/a/b
|
func/a/b
|
||||||
"""
|
""",
|
||||||
)
|
expected=[
|
||||||
|
TokenComment(pos=0, size=4, value=b"!PS"),
|
||||||
|
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
|
||||||
|
TokenKeyword(pos=33, size=5, value=b"begin"),
|
||||||
|
TokenKeyword(pos=39, size=3, value=b"end"),
|
||||||
|
TokenKeyword(pos=44, size=1, value=b'"'),
|
||||||
|
TokenKeyword(pos=47, size=1, value=b"@"),
|
||||||
|
TokenKeyword(pos=49, size=1, value=b"#"),
|
||||||
|
TokenLiteral(pos=52, size=1, value="a"),
|
||||||
|
TokenLiteral(pos=54, size=3, value="BCD"),
|
||||||
|
TokenLiteral(pos=59, size=9, value="Some_Name"),
|
||||||
|
TokenLiteral(pos=70, size=11, value="foo_naa"),
|
||||||
|
TokenLiteral(pos=83, size=6, value="barj"),
|
||||||
|
TokenLiteral(pos=91, size=4, value="need"),
|
||||||
|
TokenLiteral(pos=97, size=5, value="oh1oh"),
|
||||||
|
TokenKeyword(pos=103, size=4, value=b"func"),
|
||||||
|
TokenLiteral(pos=108, size=1, value="a"),
|
||||||
|
TokenLiteral(pos=110, size=1, value="b"),
|
||||||
|
],
|
||||||
)
|
)
|
||||||
expected: List[Token] = [
|
|
||||||
TokenComment(pos=0, size=4, value=b"!PS"),
|
|
||||||
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
|
|
||||||
TokenKeyword(pos=33, size=5, value=b"begin"),
|
|
||||||
TokenKeyword(pos=39, size=3, value=b"end"),
|
|
||||||
TokenKeyword(pos=44, size=1, value=b'"'),
|
|
||||||
TokenKeyword(pos=47, size=1, value=b"@"),
|
|
||||||
TokenKeyword(pos=49, size=1, value=b"#"),
|
|
||||||
TokenLiteral(pos=51, size=1, value="a"),
|
|
||||||
TokenLiteral(pos=53, size=3, value="BCD"),
|
|
||||||
TokenLiteral(pos=58, size=9, value="Some_Name"),
|
|
||||||
TokenLiteral(pos=69, size=11, value="foo_xbaa"),
|
|
||||||
TokenLiteral(pos=82, size=6, value="barj"),
|
|
||||||
TokenLiteral(pos=90, size=4, value="need"),
|
|
||||||
TokenLiteral(pos=96, size=5, value="oh1oh"),
|
|
||||||
TokenKeyword(pos=103, size=4, value=b"func"),
|
|
||||||
TokenLiteral(pos=107, size=1, value="a"),
|
|
||||||
TokenLiteral(pos=109, size=1, value="b"),
|
|
||||||
]
|
|
||||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
|
|
||||||
self.assertEqual(token, etoken, f"Failed at example {i}")
|
|
||||||
Reference in new issue
Block a user