working literal parser
first mock of main tokenizer
This commit is contained in:
1 parent
a0cc862804
commit
ab657deda4
5 files changed
+173
-191
No files matched your search
+44
-3
@@ -1,6 +1,13 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
from pdfmajor.tokenizer.token import TokenBoolean, TokenComment, TokenDecimal, TokenInteger, TokenLiteral
|
||||
from pdfmajor.tokenizer import PSTokenizer
|
||||
from pdfmajor.tokenizer.token import (
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenInteger,
|
||||
TokenLiteral
|
||||
)
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
||||
@@ -21,7 +28,7 @@ def make_stream_iter(data: bytes, initpos: int = 0):
|
||||
raise "EOF"
|
||||
return iter_stream
|
||||
|
||||
class Basics(TestCase):
|
||||
class Individual(TestCase):
|
||||
def test_comment_parser_ln(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
@@ -90,7 +97,7 @@ class Basics(TestCase):
|
||||
|
||||
def test_parse_literals(self):
|
||||
iter_stream = make_stream_iter(
|
||||
b"Some_Name /foo#5f#xbaa",
|
||||
br"Some_Name /foo#5f#xbaa",
|
||||
71
|
||||
)
|
||||
token = parse_literal(71, iter_stream(1))
|
||||
@@ -98,3 +105,37 @@ class Basics(TestCase):
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, 'Some_Name')
|
||||
self.assertEqual(token.size, 9)
|
||||
|
||||
def test_parse_literals_hex(self):
|
||||
iter_stream = make_stream_iter(
|
||||
br"foo#5f#xbaa ",
|
||||
71
|
||||
)
|
||||
token = parse_literal(71, iter_stream(100))
|
||||
self.assertIsInstance(token, TokenLiteral)
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, 'foo_xbaa')
|
||||
self.assertEqual(token.size, 11)
|
||||
|
||||
class Tokenizer(TestCase):
|
||||
def test_case1(self):
|
||||
tokenizer = PSTokenizer(io.BytesIO(br"""%!PS
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#xbaa
|
||||
0 +1 -2 .5 1.234
|
||||
(abc) () (abc ( def ) ghi)
|
||||
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
||||
(this % is not a comment.)
|
||||
(foo
|
||||
baa)
|
||||
(foo\
|
||||
baa)
|
||||
<> <20> < 40 4020 >
|
||||
<abcd00
|
||||
12345>
|
||||
func/a/b{(c)do*}def
|
||||
[ 1 (z) ! ]
|
||||
<< /foo (bar) >>"""))
|
||||
for token in tokenizer.iter_tokens():
|
||||
print(token)
|
||||
Reference in new issue
Block a user