working literal parser

first mock of main tokenizer
This commit is contained in:
ari committed 2020-10-13 20:52:23 -04:00
1 parent a0cc862804
commit ab657deda4
5 files changed
+173 -191

No files matched your search

+44 -3
View File
@@ -1,6 +1,13 @@
from decimal import Decimal
import io
from pdfmajor.tokenizer.token import TokenBoolean, TokenComment, TokenDecimal, TokenInteger, TokenLiteral
from pdfmajor.tokenizer import PSTokenizer
from pdfmajor.tokenizer.token import (
TokenBoolean,
TokenComment,
TokenDecimal,
TokenInteger,
TokenLiteral
)
from pdfmajor.tokenizer.token_parsers.util import PInput
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
@@ -21,7 +28,7 @@ def make_stream_iter(data: bytes, initpos: int = 0):
raise "EOF"
return iter_stream
class Basics(TestCase):
class Individual(TestCase):
def test_comment_parser_ln(self):
iter_stream = make_stream_iter(
b"this is a lengthy comment that ends here\nso this is not reachable"
@@ -90,7 +97,7 @@ class Basics(TestCase):
def test_parse_literals(self):
iter_stream = make_stream_iter(
b"Some_Name /foo#5f#xbaa",
br"Some_Name /foo#5f#xbaa",
71
)
token = parse_literal(71, iter_stream(1))
@@ -98,3 +105,37 @@ class Basics(TestCase):
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, 'Some_Name')
self.assertEqual(token.size, 9)
def test_parse_literals_hex(self):
iter_stream = make_stream_iter(
br"foo#5f#xbaa ",
71
)
token = parse_literal(71, iter_stream(100))
self.assertIsInstance(token, TokenLiteral)
self.assertEqual(token.pos, 71)
self.assertEqual(token.value, 'foo_xbaa')
self.assertEqual(token.size, 11)
class Tokenizer(TestCase):
def test_case1(self):
tokenizer = PSTokenizer(io.BytesIO(br"""%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#xbaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>"""))
for token in tokenizer.iter_tokens():
print(token)