strict hex format

This commit is contained in:
ari committed 2020-10-14 21:29:46 -04:00
1 parent 5f19f71c4d
commit 4a6b0608a0
5 files changed
+150 -106

No files matched your search

+124 -93
View File
@@ -38,90 +38,119 @@ def make_stream_iter(data: bytes, initpos: int = 0):
class Individual(TestCase):
def test_comment_parser_ln(self):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\nso this is not reachable"
)
token = parse_comment(0, iter_stream(3))
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.pos, 0)
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
self.assertEqual(token.size, 40)
for buf_size in [2, 7, 11, 150]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\nso this is not reachable"
)
token = parse_comment(0, iter_stream(buf_size))
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.pos, 0)
self.assertEqual(
token.value, b"this is a lengthy comment that ends here"
)
self.assertEqual(token.size, 40)
def test_comment_parser_lr(self):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\rso this is not reachable"
)
token = parse_comment(0, iter_stream(7))
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.pos, 0)
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
self.assertEqual(token.size, 40)
for buf_size in [2, 7, 11, 150]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\rso this is not reachable"
)
token = parse_comment(0, iter_stream(buf_size))
self.assertIsInstance(token, TokenComment)
self.assertEqual(token.pos, 0)
self.assertEqual(
token.value, b"this is a lengthy comment that ends here"
)
self.assertEqual(token.size, 40)
def test_parse_float(self):
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
token = parse_number(5, iter_stream(3))
self.assertIsInstance(token, TokenDecimal)
self.assertEqual(token.pos, 5)
self.assertEqual(token.value, Decimal("0.12"))
self.assertEqual(token.size, 5)
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
token = parse_number(5, iter_stream(buf_size))
self.assertIsInstance(token, TokenDecimal)
self.assertEqual(token.pos, 5)
self.assertEqual(token.value, Decimal("0.12"))
self.assertEqual(token.size, 5)
def test_parse_int(self):
iter_stream = make_stream_iter(b"130 oh oh 0.12 + some invalid text", 11)
token = parse_number(11, iter_stream(10))
self.assertIsInstance(token, TokenInteger)
self.assertEqual(token.pos, 11)
self.assertEqual(token.value, 130)
self.assertEqual(token.size, 3)
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(
b"130 oh oh 0.12 + some invalid text", 11
)
token = parse_number(11, iter_stream(buf_size))
self.assertIsInstance(token, TokenInteger)
self.assertEqual(token.pos, 11)
self.assertEqual(token.value, 130)
self.assertEqual(token.size, 3)
def test_parse_bool_true(self):
iter_stream = make_stream_iter(b"true and the american way", 91)
token = parse_keyword(91, iter_stream(33))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 91)
self.assertEqual(token.value, True)
self.assertEqual(token.size, 4)
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"true and the american way", 91)
token = parse_keyword(91, iter_stream(buf_size))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 91)
self.assertEqual(token.value, True)
self.assertEqual(token.size, 4)
def test_parse_bool_false(self):
iter_stream = make_stream_iter(b"false mannn", 33)
token = parse_keyword(33, iter_stream(100))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 33)
self.assertEqual(token.value, False)
self.assertEqual(token.size, 5)
for buf_size in [2, 3, 50]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(b"false mannn", 33)
token = parse_keyword(33, iter_stream(buf_size))
self.assertIsInstance(token, TokenBoolean)
self.assertEqual(token.pos, 33)
self.assertEqual(token.value, False)
self.assertEqual(token.size, 5)
def test_parse_literals(self):
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
token = parse_literal(71, iter_stream(1))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, "Some_Name")
self.assertEqual(token.size, 9)
for buf_size in [2, 3, 50, 444]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
token = parse_literal(71, iter_stream(buf_size))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, "Some_Name")
self.assertEqual(token.size, 9)
def test_parse_literals_hex(self):
iter_stream = make_stream_iter(br"foo#5f#xbaa ", 71)
token = parse_literal(71, iter_stream(100))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, "foo_xbaa")
self.assertEqual(token.size, 11)
for buf_size in [2]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"foo#5faa ", 71)
token = parse_literal(71, iter_stream(buf_size))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, "foo_aa")
self.assertEqual(token.size, 8)
def test_parse_string(self):
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
token = parse_string(88, iter_stream(452))
self.assertIsInstance(token, TokenString)
self.assertEqual(token.pos, 88)
self.assertEqual(token.value, "this % is not a comment.")
self.assertEqual(token.size, 24)
for buf_size in [2]:
with self.subTest(buf_size=buf_size):
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
token = parse_string(88, iter_stream(buf_size))
self.assertIsInstance(token, TokenString)
self.assertEqual(token.pos, 88)
self.assertEqual(token.value, "this % is not a comment.")
self.assertEqual(token.size, 24)
class Tokenizer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PSTokenizer(io.BytesIO(raw))
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
self.assertEqual(token, etoken, f"Failed at example {i}")
def test_case1(self):
tokenizer = PSTokenizer(
io.BytesIO(
br"""%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#xbaa
/a/BCD /Some_Name /foo#5f#6eaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
@@ -142,47 +171,49 @@ func/a/b{(c)do*}def
print(token)
def test_case2(self):
tokenizer = PSTokenizer(
io.BytesIO(
br"""10.32
self.run_test(
raw=br"""10.32
0 +1 -2 .5 1.234
-1.234 990"""
)
-1.234 990""",
expected=[
TokenDecimal(pos=0, size=5, value=Decimal("10.32")),
TokenInteger(pos=6, size=1, value=0),
TokenInteger(pos=8, size=2, value=1),
TokenInteger(pos=11, size=2, value=-2),
TokenDecimal(pos=14, size=2, value=Decimal("0.5")),
TokenDecimal(pos=17, size=5, value=Decimal("1.234")),
TokenDecimal(pos=23, size=6, value=Decimal("-1.234")),
TokenInteger(pos=30, size=3, value=990),
],
)
for token in tokenizer.iter_tokens():
print(token)
def test_case_words1(self):
tokenizer = PSTokenizer(
io.BytesIO(
br"""%!PS
self.run_test(
raw=br"""%!PS
% This is some secret sauce
begin end
" @ #
/a/BCD /Some_Name /foo#5f#xbaa /bar#6a
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
/need /oh1oh
func/a/b
"""
)
""",
expected=[
TokenComment(pos=0, size=4, value=b"!PS"),
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
TokenKeyword(pos=33, size=5, value=b"begin"),
TokenKeyword(pos=39, size=3, value=b"end"),
TokenKeyword(pos=44, size=1, value=b'"'),
TokenKeyword(pos=47, size=1, value=b"@"),
TokenKeyword(pos=49, size=1, value=b"#"),
TokenLiteral(pos=52, size=1, value="a"),
TokenLiteral(pos=54, size=3, value="BCD"),
TokenLiteral(pos=59, size=9, value="Some_Name"),
TokenLiteral(pos=70, size=11, value="foo_naa"),
TokenLiteral(pos=83, size=6, value="barj"),
TokenLiteral(pos=91, size=4, value="need"),
TokenLiteral(pos=97, size=5, value="oh1oh"),
TokenKeyword(pos=103, size=4, value=b"func"),
TokenLiteral(pos=108, size=1, value="a"),
TokenLiteral(pos=110, size=1, value="b"),
],
)
expected: List[Token] = [
TokenComment(pos=0, size=4, value=b"!PS"),
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
TokenKeyword(pos=33, size=5, value=b"begin"),
TokenKeyword(pos=39, size=3, value=b"end"),
TokenKeyword(pos=44, size=1, value=b'"'),
TokenKeyword(pos=47, size=1, value=b"@"),
TokenKeyword(pos=49, size=1, value=b"#"),
TokenLiteral(pos=51, size=1, value="a"),
TokenLiteral(pos=53, size=3, value="BCD"),
TokenLiteral(pos=58, size=9, value="Some_Name"),
TokenLiteral(pos=69, size=11, value="foo_xbaa"),
TokenLiteral(pos=82, size=6, value="barj"),
TokenLiteral(pos=90, size=4, value="need"),
TokenLiteral(pos=96, size=5, value="oh1oh"),
TokenKeyword(pos=103, size=4, value=b"func"),
TokenLiteral(pos=107, size=1, value="a"),
TokenLiteral(pos=109, size=1, value="b"),
]
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
self.assertEqual(token, etoken, f"Failed at example {i}")