moved string parsing to new method
This commit is contained in:
1 parent
faaf7925bf
commit
370a1b40a1
3 files changed
+133
-10
No files matched your search
@@ -4,3 +4,15 @@ EOL = re.compile(br"[\r\n]")
|
||||
END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
HEX = re.compile(br"[0-9a-fA-F]")
|
||||
END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]")
|
||||
END_STRING = re.compile(br"[()\134]")
|
||||
OCT_STRING = re.compile(br"^[0-7]{1,3}$")
|
||||
ESC_STRING = {
|
||||
b"b": 8,
|
||||
b"t": 9,
|
||||
b"n": 10,
|
||||
b"f": 12,
|
||||
b"r": 13,
|
||||
b"(": 40,
|
||||
b")": 41,
|
||||
b"\\": 92,
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from pdfmajor.lexer.regex import END_STRING, ESC_STRING, OCT_STRING
|
||||
from pdfmajor.lexer.token import TokenString
|
||||
from pdfmajor.lexer.exceptions import LexerEOF, LexerError
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
|
||||
|
||||
@dataclass
|
||||
class StringParseState:
|
||||
curtoken: bytes
|
||||
paren: int
|
||||
escaped_value: Optional[bytes] = None
|
||||
|
||||
|
||||
def parse_string(buffer: SafeBufferIt) -> TokenString:
|
||||
"""Parses input stream into a simple string object
|
||||
|
||||
Args:
|
||||
buffer (SafeBufferIt)
|
||||
|
||||
Returns:
|
||||
TokenString
|
||||
"""
|
||||
initialpos = buffer.tell() - 1
|
||||
state = StringParseState(b"", 1)
|
||||
for curpos, buf in buffer:
|
||||
if state.escaped_value is not None:
|
||||
buffer.seek(curpos)
|
||||
parse_string_escape(buffer, state)
|
||||
else:
|
||||
m = END_STRING.search(buf, 0)
|
||||
if not m:
|
||||
state.curtoken += buf
|
||||
else:
|
||||
j = m.start(0)
|
||||
state.curtoken += buf[:j]
|
||||
buffer.seek(curpos + j + 1)
|
||||
next_char = buf[j : j + 1]
|
||||
if next_char == b"\\":
|
||||
state.escaped_value = b""
|
||||
elif next_char == b"(":
|
||||
state.paren += 1
|
||||
state.curtoken += next_char
|
||||
elif next_char == b")":
|
||||
state.paren -= 1
|
||||
if state.paren:
|
||||
state.curtoken += next_char
|
||||
else:
|
||||
return TokenString(
|
||||
initialpos,
|
||||
buffer.tell(),
|
||||
state.curtoken.decode(),
|
||||
)
|
||||
raise LexerEOF
|
||||
|
||||
|
||||
def parse_string_escape(buffer: SafeBufferIt, state: StringParseState):
|
||||
if state.escaped_value is None:
|
||||
raise LexerError("state.escaped_value is None")
|
||||
for pos, buf in buffer:
|
||||
for ci in range(len(buf)):
|
||||
c = buf[ci : ci + 1]
|
||||
if len(state.escaped_value) < 3:
|
||||
state.escaped_value += c
|
||||
buffer.seek(pos + ci + 1)
|
||||
if len(state.escaped_value) == 1:
|
||||
if (ci < 1) and (c in ESC_STRING):
|
||||
state.curtoken += bytes([ESC_STRING[c]])
|
||||
state.escaped_value = None
|
||||
return
|
||||
elif OCT_STRING.match(state.escaped_value):
|
||||
if len(state.escaped_value) == 3:
|
||||
state.curtoken += bytes([int(state.escaped_value, 8)])
|
||||
state.escaped_value = None
|
||||
return
|
||||
else:
|
||||
state.escaped_value = None
|
||||
buffer.seekd(-1)
|
||||
return
|
||||
+41
-10
@@ -1,10 +1,11 @@
|
||||
import io
|
||||
from pdfmajor.lexer.string import parse_string
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.lexer.name import parse_name
|
||||
from pdfmajor.lexer.keyword import parse_keyword
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName
|
||||
from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName, TokenString
|
||||
from pdfmajor.lexer.comment import parse_comment
|
||||
|
||||
|
||||
@@ -83,12 +84,42 @@ class Basics(TestCase):
|
||||
self.assertEqual(token.value, "foo_aa")
|
||||
self.assertEqual(token.end_loc, 8)
|
||||
|
||||
# def test_parse_string(self):
|
||||
# for buf_size in [2, 3, 50, 1000]:
|
||||
# with self.subTest(buf_end_loc=buf_size):
|
||||
# iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
|
||||
# token = parse_string(88, iter_stream(buf_size))
|
||||
# self.assertIsInstance(token, TokenString)
|
||||
# self.assertEqual(token.start_loc, 88)
|
||||
# self.assertEqual(token.value, "this % is not a comment.")
|
||||
# self.assertEqual(token.end_loc, 88 + 25)
|
||||
def test_parse_string(self):
|
||||
for buf_size in [2, 3, 50, 1000]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(br"this % is not a comment.) ", buf_size)
|
||||
token = parse_string(buffer)
|
||||
self.assertIsInstance(token, TokenString)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, "this % is not a comment.")
|
||||
self.assertEqual(token.end_loc, 25)
|
||||
|
||||
def test_parse_string_nest_brackets(self):
|
||||
for buf_size in [1, 2, 3, 50, 100]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(br"x - (y*2-z-(2+3))) ", 88)
|
||||
token = parse_string(buffer)
|
||||
self.assertIsInstance(token, TokenString)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, "x - (y*2-z-(2+3))")
|
||||
self.assertEqual(token.end_loc, 18)
|
||||
|
||||
def test_parse_string_hex_chars(self):
|
||||
for buf_size in [1, 2, 3, 50, 100]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(br"def\040\0\0404ghi) (bach\\slask) ", 39)
|
||||
token = parse_string(buffer)
|
||||
self.assertIsInstance(token, TokenString)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, "def 4ghi")
|
||||
self.assertEqual(token.end_loc, 18)
|
||||
|
||||
def test_parse_string_escape_char(self):
|
||||
for buf_size in [1, 2, 3, 50, 100]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(br"bach\\slask) ", 39)
|
||||
token = parse_string(buffer)
|
||||
self.assertIsInstance(token, TokenString)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, r"bach\slask")
|
||||
self.assertEqual(token.end_loc, 12)
|
||||
Reference in new issue
Block a user