added string tokenizer
This commit is contained in:
1 parent
d8be2885b8
commit
96d68cdf68
2 files changed
+141
-142
No files matched your search
@@ -1,6 +1,6 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
||||||
from pdfmajor.tokenizer.token_parsers.util import PInput, cmp_tsize
|
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize
|
||||||
from pdfmajor.utils import int2byte
|
from pdfmajor.utils import int2byte
|
||||||
from pdfmajor.tokenizer.token import (
|
from pdfmajor.tokenizer.token import (
|
||||||
TokenString,
|
TokenString,
|
||||||
@@ -23,47 +23,48 @@ class StringParseState:
|
|||||||
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
||||||
state = StringParseState(b"", 1)
|
state = StringParseState(b"", 1)
|
||||||
for curpos, buf in inp:
|
for curpos, buf in inp:
|
||||||
skip: int = 0
|
it = SafeBufferIt(buf)
|
||||||
for i in range(len(buf) + 1):
|
for s in it.into_iter():
|
||||||
if i < len(buf):
|
if state.oct_value is not None:
|
||||||
raise TokenizerEOF("Max Iteration Reached!")
|
it.skip += parse_string_oct(s, state)
|
||||||
if skip >= len(buf):
|
else:
|
||||||
break
|
m = END_STRING.search(s, 0)
|
||||||
m = END_STRING.search(s, 0)
|
if not m:
|
||||||
if not m:
|
state.curtoken += s
|
||||||
curtoken += s
|
else:
|
||||||
else:
|
j = m.start(0)
|
||||||
j = m.start(0)
|
state.curtoken += s[:j]
|
||||||
curtoken += s[:j]
|
c = s[j : j + 1]
|
||||||
c = s[j : j + 1]
|
if c == b"\\":
|
||||||
if c == b"\\":
|
state.oct_value = b""
|
||||||
_parse_string_1
|
it.skip += j + 1
|
||||||
return j + 1
|
elif c == b"(":
|
||||||
elif c == b"(":
|
state.paren += 1
|
||||||
paren += 1
|
state.curtoken += c
|
||||||
curtoken += c
|
it.skip += j + 1
|
||||||
return j + 1
|
elif c == b")":
|
||||||
elif c == b")":
|
state.paren -= 1
|
||||||
paren -= 1
|
if state.paren:
|
||||||
if paren:
|
state.curtoken += c
|
||||||
curtoken += c
|
it.skip += j + 1
|
||||||
return j + 1
|
return TokenString(
|
||||||
return TokenString(
|
initialpos,
|
||||||
initialpos, cmp_tsize(curpos, initialpos, j), curtoken.decode()
|
cmp_tsize(curpos, initialpos, j),
|
||||||
)
|
state.curtoken.decode(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_string_oct(initialpos: int, inp: Iterator[PInput]):
|
def parse_string_oct(s: bytes, state: StringParseState) -> int:
|
||||||
curtoken: bytes = b""
|
for ci in range(len(s)):
|
||||||
oct_val: bytes = b""
|
c = s[ci : ci + 1]
|
||||||
for curpos, s in inp:
|
if OCT_STRING.match(c) and len(state.oct_value) < 3:
|
||||||
for ci in range(len(s)):
|
state.oct_value += c
|
||||||
c = s[ci : ci + 1]
|
elif state.oct_value:
|
||||||
if OCT_STRING.match(c) and len(oct_val) < 3:
|
state.curtoken += int2byte(int(state.oct_value, 8))
|
||||||
oct_val += c
|
return ci
|
||||||
elif oct_val:
|
elif c in ESC_STRING:
|
||||||
curtoken += int2byte(int(oct_val, 8))
|
state.curtoken += int2byte(ESC_STRING[c])
|
||||||
return (curtoken, curpos + ci)
|
state.oct_value = None
|
||||||
elif c in ESC_STRING:
|
return ci + 1
|
||||||
curtoken += int2byte(ESC_STRING[c])
|
state.oct_value = None
|
||||||
return (curtoken, curpos + ci + 1)
|
return len(s) + 1
|
||||||
+97
-99
@@ -1,12 +1,14 @@
|
|||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
import io
|
import io
|
||||||
|
from pdfmajor.tokenizer.token_parsers.string import parse_string
|
||||||
from pdfmajor.tokenizer import PSTokenizer
|
from pdfmajor.tokenizer import PSTokenizer
|
||||||
from pdfmajor.tokenizer.token import (
|
from pdfmajor.tokenizer.token import (
|
||||||
TokenBoolean,
|
TokenBoolean,
|
||||||
TokenComment,
|
TokenComment,
|
||||||
TokenDecimal,
|
TokenDecimal,
|
||||||
TokenInteger,
|
TokenInteger,
|
||||||
TokenLiteral
|
TokenLiteral,
|
||||||
|
TokenString,
|
||||||
)
|
)
|
||||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||||
@@ -15,111 +17,105 @@ from pdfmajor.tokenizer.token_parsers.number import parse_number
|
|||||||
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
|
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
|
||||||
from unittest import TestCase
|
from unittest import TestCase
|
||||||
|
|
||||||
def make_stream_iter(data: bytes, initpos: int = 0):
|
|
||||||
stream = io.BytesIO(data)
|
|
||||||
|
|
||||||
def iter_stream(buf_size: int):
|
def make_stream_iter(data: bytes, initpos: int = 0):
|
||||||
while True:
|
stream = io.BytesIO(data)
|
||||||
bufpos = stream.tell()
|
|
||||||
buf = stream.read(buf_size)
|
def iter_stream(buf_size: int):
|
||||||
if buf:
|
while True:
|
||||||
yield PInput(initpos+bufpos, buf)
|
bufpos = stream.tell()
|
||||||
else:
|
buf = stream.read(buf_size)
|
||||||
raise "EOF"
|
if buf:
|
||||||
return iter_stream
|
yield PInput(initpos + bufpos, buf)
|
||||||
|
else:
|
||||||
|
raise "EOF"
|
||||||
|
|
||||||
|
return iter_stream
|
||||||
|
|
||||||
|
|
||||||
class Individual(TestCase):
|
class Individual(TestCase):
|
||||||
def test_comment_parser_ln(self):
|
def test_comment_parser_ln(self):
|
||||||
iter_stream = make_stream_iter(
|
iter_stream = make_stream_iter(
|
||||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||||
)
|
)
|
||||||
token = parse_comment(0, iter_stream(3))
|
token = parse_comment(0, iter_stream(3))
|
||||||
self.assertIsInstance(token, TokenComment)
|
self.assertIsInstance(token, TokenComment)
|
||||||
self.assertEqual(token.pos, 0)
|
self.assertEqual(token.pos, 0)
|
||||||
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
|
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
|
||||||
self.assertEqual(token.size, 40)
|
self.assertEqual(token.size, 40)
|
||||||
|
|
||||||
def test_comment_parser_lr(self):
|
def test_comment_parser_lr(self):
|
||||||
iter_stream = make_stream_iter(
|
iter_stream = make_stream_iter(
|
||||||
b"this is a lengthy comment that ends here\rso this is not reachable"
|
b"this is a lengthy comment that ends here\rso this is not reachable"
|
||||||
)
|
)
|
||||||
token = parse_comment(0, iter_stream(7))
|
token = parse_comment(0, iter_stream(7))
|
||||||
self.assertIsInstance(token, TokenComment)
|
self.assertIsInstance(token, TokenComment)
|
||||||
self.assertEqual(token.pos, 0)
|
self.assertEqual(token.pos, 0)
|
||||||
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
|
self.assertEqual(token.value, b"this is a lengthy comment that ends here")
|
||||||
self.assertEqual(token.size, 40)
|
self.assertEqual(token.size, 40)
|
||||||
|
|
||||||
def test_parse_float(self):
|
def test_parse_float(self):
|
||||||
iter_stream = make_stream_iter(
|
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
|
||||||
b"0.120 + some invalid text",
|
token = parse_number(5, iter_stream(3))
|
||||||
5
|
self.assertIsInstance(token, TokenDecimal)
|
||||||
)
|
self.assertEqual(token.pos, 5)
|
||||||
token = parse_number(5, iter_stream(3))
|
self.assertEqual(token.value, Decimal("0.12"))
|
||||||
self.assertIsInstance(token, TokenDecimal)
|
self.assertEqual(token.size, 5)
|
||||||
self.assertEqual(token.pos, 5)
|
|
||||||
self.assertEqual(token.value, Decimal('0.12'))
|
|
||||||
self.assertEqual(token.size, 5)
|
|
||||||
|
|
||||||
|
def test_parse_int(self):
|
||||||
|
iter_stream = make_stream_iter(b"130 oh oh 0.12 + some invalid text", 11)
|
||||||
|
token = parse_number(11, iter_stream(10))
|
||||||
|
self.assertIsInstance(token, TokenInteger)
|
||||||
|
self.assertEqual(token.pos, 11)
|
||||||
|
self.assertEqual(token.value, 130)
|
||||||
|
self.assertEqual(token.size, 3)
|
||||||
|
|
||||||
def test_parse_int(self):
|
def test_parse_bool_true(self):
|
||||||
iter_stream = make_stream_iter(
|
iter_stream = make_stream_iter(b"true and the american way", 91)
|
||||||
b"130 oh oh 0.12 + some invalid text",
|
token = parse_keyword(91, iter_stream(33))
|
||||||
11
|
self.assertIsInstance(token, TokenBoolean)
|
||||||
)
|
self.assertEqual(token.pos, 91)
|
||||||
token = parse_number(11, iter_stream(10))
|
self.assertEqual(token.value, True)
|
||||||
self.assertIsInstance(token, TokenInteger)
|
self.assertEqual(token.size, 4)
|
||||||
self.assertEqual(token.pos, 11)
|
|
||||||
self.assertEqual(token.value, 130)
|
|
||||||
self.assertEqual(token.size, 3)
|
|
||||||
|
|
||||||
|
def test_parse_bool_false(self):
|
||||||
|
iter_stream = make_stream_iter(b"false mannn", 33)
|
||||||
|
token = parse_keyword(33, iter_stream(100))
|
||||||
|
self.assertIsInstance(token, TokenBoolean)
|
||||||
|
self.assertEqual(token.pos, 33)
|
||||||
|
self.assertEqual(token.value, False)
|
||||||
|
self.assertEqual(token.size, 5)
|
||||||
|
|
||||||
def test_parse_bool_true(self):
|
def test_parse_literals(self):
|
||||||
iter_stream = make_stream_iter(
|
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
|
||||||
b"true and the american way",
|
token = parse_literal(71, iter_stream(1))
|
||||||
91
|
self.assertIsInstance(token, TokenLiteral)
|
||||||
)
|
self.assertEqual(token.pos, 71)
|
||||||
token = parse_keyword(91, iter_stream(33))
|
self.assertEqual(token.value, "Some_Name")
|
||||||
self.assertIsInstance(token, TokenBoolean)
|
self.assertEqual(token.size, 9)
|
||||||
self.assertEqual(token.pos, 91)
|
|
||||||
self.assertEqual(token.value, True)
|
|
||||||
self.assertEqual(token.size, 4)
|
|
||||||
|
|
||||||
def test_parse_bool_false(self):
|
def test_parse_literals_hex(self):
|
||||||
iter_stream = make_stream_iter(
|
iter_stream = make_stream_iter(br"foo#5f#xbaa ", 71)
|
||||||
b"false mannn",
|
token = parse_literal(71, iter_stream(100))
|
||||||
33
|
self.assertIsInstance(token, TokenLiteral)
|
||||||
)
|
self.assertEqual(token.pos, 71)
|
||||||
token = parse_keyword(33, iter_stream(100))
|
self.assertEqual(token.value, "foo_xbaa")
|
||||||
self.assertIsInstance(token, TokenBoolean)
|
self.assertEqual(token.size, 11)
|
||||||
self.assertEqual(token.pos, 33)
|
|
||||||
self.assertEqual(token.value, False)
|
|
||||||
self.assertEqual(token.size, 5)
|
|
||||||
|
|
||||||
def test_parse_literals(self):
|
def test_parse_string(self):
|
||||||
iter_stream = make_stream_iter(
|
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
|
||||||
br"Some_Name /foo#5f#xbaa",
|
token = parse_string(88, iter_stream(452))
|
||||||
71
|
self.assertIsInstance(token, TokenString)
|
||||||
)
|
self.assertEqual(token.pos, 88)
|
||||||
token = parse_literal(71, iter_stream(1))
|
self.assertEqual(token.value, "this % is not a comment.")
|
||||||
self.assertIsInstance(token, TokenLiteral)
|
self.assertEqual(token.size, 24)
|
||||||
self.assertEqual(token.pos, 71)
|
|
||||||
self.assertEqual(token.value, 'Some_Name')
|
|
||||||
self.assertEqual(token.size, 9)
|
|
||||||
|
|
||||||
def test_parse_literals_hex(self):
|
|
||||||
iter_stream = make_stream_iter(
|
|
||||||
br"foo#5f#xbaa ",
|
|
||||||
71
|
|
||||||
)
|
|
||||||
token = parse_literal(71, iter_stream(100))
|
|
||||||
self.assertIsInstance(token, TokenLiteral)
|
|
||||||
self.assertEqual(token.pos, 71)
|
|
||||||
self.assertEqual(token.value, 'foo_xbaa')
|
|
||||||
self.assertEqual(token.size, 11)
|
|
||||||
|
|
||||||
class Tokenizer(TestCase):
|
class Tokenizer(TestCase):
|
||||||
def test_case1(self):
|
def test_case1(self):
|
||||||
tokenizer = PSTokenizer(io.BytesIO(br"""%!PS
|
tokenizer = PSTokenizer(
|
||||||
|
io.BytesIO(
|
||||||
|
br"""%!PS
|
||||||
begin end
|
begin end
|
||||||
" @ #
|
" @ #
|
||||||
/a/BCD /Some_Name /foo#5f#xbaa
|
/a/BCD /Some_Name /foo#5f#xbaa
|
||||||
@@ -136,6 +132,8 @@ baa)
|
|||||||
12345>
|
12345>
|
||||||
func/a/b{(c)do*}def
|
func/a/b{(c)do*}def
|
||||||
[ 1 (z) ! ]
|
[ 1 (z) ! ]
|
||||||
<< /foo (bar) >>"""))
|
<< /foo (bar) >>"""
|
||||||
for token in tokenizer.iter_tokens():
|
)
|
||||||
print(token)
|
)
|
||||||
|
for token in tokenizer.iter_tokens():
|
||||||
|
print(token)
|
||||||
Reference in new issue
Block a user