string tokenizer works

This commit is contained in:
ari committed 2020-10-14 21:53:27 -04:00
1 parent 4a6b0608a0
commit af927e86ec
2 files changed
+36 -25

No files matched your search

+23 -23
View File
@@ -1,7 +1,5 @@
from dataclasses import dataclass from dataclasses import dataclass
from pdfmajor.tokenizer.exceptions import TokenizerEOF
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize
from pdfmajor.utils import int2byte
from pdfmajor.tokenizer.token import ( from pdfmajor.tokenizer.token import (
TokenString, TokenString,
) )
@@ -17,7 +15,7 @@ from typing import Iterator, Optional
class StringParseState: class StringParseState:
curtoken: bytes curtoken: bytes
paren: int paren: int
oct_value: Optional[bytes] = None escaped_value: Optional[bytes] = None
def parse_string(initialpos: int, inp: Iterator[PInput]): def parse_string(initialpos: int, inp: Iterator[PInput]):
@@ -25,27 +23,28 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
for curpos, buf in inp: for curpos, buf in inp:
it = SafeBufferIt(buf) it = SafeBufferIt(buf)
for s in it.into_iter(): for s in it.into_iter():
if state.oct_value is not None: if state.escaped_value is not None:
it.skip += parse_string_oct(s, state) it.skip += parse_string_escape(s, state)
else: else:
m = END_STRING.search(s, 0) m = END_STRING.search(s, 0)
if not m: if not m:
state.curtoken += s state.curtoken += s
it.skip = len(s)
else: else:
j = m.start(0) j = m.start(0)
state.curtoken += s[:j] state.curtoken += s[:j]
c = s[j : j + 1] next_char = s[j : j + 1]
if c == b"\\": if next_char == b"\\":
state.oct_value = b"" state.escaped_value = b""
it.skip += j + 1 it.skip += j + 1
elif c == b"(": elif next_char == b"(":
state.paren += 1 state.paren += 1
state.curtoken += c state.curtoken += next_char
it.skip += j + 1 it.skip += j + 1
elif c == b")": elif next_char == b")":
state.paren -= 1 state.paren -= 1
if state.paren: if state.paren:
state.curtoken += c state.curtoken += next_char
it.skip += j + 1 it.skip += j + 1
return TokenString( return TokenString(
initialpos, initialpos,
@@ -54,17 +53,18 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
) )
def parse_string_oct(s: bytes, state: StringParseState) -> int: def parse_string_escape(s: bytes, state: StringParseState) -> int:
for ci in range(len(s)): for ci in range(len(s)):
c = s[ci : ci + 1] c = s[ci : ci + 1]
if OCT_STRING.match(c) and len(state.oct_value) < 3: if (ci < 1) and (c in ESC_STRING):
state.oct_value += c state.curtoken += bytes([ESC_STRING[c]])
elif state.oct_value: state.escaped_value = None
state.curtoken += int2byte(int(state.oct_value, 8)) return 1
return ci if OCT_STRING.match(c) and len(state.escaped_value) < 3:
elif c in ESC_STRING: state.escaped_value += c
state.curtoken += int2byte(ESC_STRING[c]) elif len(state.escaped_value) == 3:
state.oct_value = None state.curtoken += bytes([int(state.escaped_value, 8)])
return ci + 1 return 3
state.oct_value = None
state.escaped_value = None
return len(s) + 1 return len(s) + 1
+13 -2
View File
@@ -1,5 +1,6 @@
from decimal import Decimal
import io import io
from random import randint
from decimal import Decimal
from typing import List from typing import List
from pdfmajor.tokenizer.token_parsers.string import parse_string from pdfmajor.tokenizer.token_parsers.string import parse_string
from pdfmajor.tokenizer import PSTokenizer from pdfmajor.tokenizer import PSTokenizer
@@ -13,7 +14,7 @@ from pdfmajor.tokenizer.token import (
TokenLiteral, TokenLiteral,
TokenString, TokenString,
) )
from pdfmajor.tokenizer.token_parsers.util import PInput from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
from pdfmajor.tokenizer.token_parsers.comment import parse_comment from pdfmajor.tokenizer.token_parsers.comment import parse_comment
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
from pdfmajor.tokenizer.token_parsers.number import parse_number from pdfmajor.tokenizer.token_parsers.number import parse_number
@@ -36,6 +37,16 @@ def make_stream_iter(data: bytes, initpos: int = 0):
return iter_stream return iter_stream
class Util(TestCase):
def test_safebuff(self):
it = SafeBufferIt(
b"this is a lengthy comment that ends here\nso this is not reachable"
)
for subbuf in it.into_iter():
self.assertEqual(len(subbuf), len(it.buffer) - it.skip)
it.skip += randint(1, max(len(subbuf) - 1, 1))
class Individual(TestCase): class Individual(TestCase):
def test_comment_parser_ln(self): def test_comment_parser_ln(self):
for buf_size in [2, 7, 11, 150]: for buf_size in [2, 7, 11, 150]: