string tokenizer works
This commit is contained in:
1 parent
4a6b0608a0
commit
af927e86ec
2 files changed
+36
-25
No files matched your search
@@ -1,7 +1,5 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pdfmajor.tokenizer.exceptions import TokenizerEOF
|
|
||||||
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize
|
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt, cmp_tsize
|
||||||
from pdfmajor.utils import int2byte
|
|
||||||
from pdfmajor.tokenizer.token import (
|
from pdfmajor.tokenizer.token import (
|
||||||
TokenString,
|
TokenString,
|
||||||
)
|
)
|
||||||
@@ -17,7 +15,7 @@ from typing import Iterator, Optional
|
|||||||
class StringParseState:
|
class StringParseState:
|
||||||
curtoken: bytes
|
curtoken: bytes
|
||||||
paren: int
|
paren: int
|
||||||
oct_value: Optional[bytes] = None
|
escaped_value: Optional[bytes] = None
|
||||||
|
|
||||||
|
|
||||||
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
def parse_string(initialpos: int, inp: Iterator[PInput]):
|
||||||
@@ -25,27 +23,28 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
|
|||||||
for curpos, buf in inp:
|
for curpos, buf in inp:
|
||||||
it = SafeBufferIt(buf)
|
it = SafeBufferIt(buf)
|
||||||
for s in it.into_iter():
|
for s in it.into_iter():
|
||||||
if state.oct_value is not None:
|
if state.escaped_value is not None:
|
||||||
it.skip += parse_string_oct(s, state)
|
it.skip += parse_string_escape(s, state)
|
||||||
else:
|
else:
|
||||||
m = END_STRING.search(s, 0)
|
m = END_STRING.search(s, 0)
|
||||||
if not m:
|
if not m:
|
||||||
state.curtoken += s
|
state.curtoken += s
|
||||||
|
it.skip = len(s)
|
||||||
else:
|
else:
|
||||||
j = m.start(0)
|
j = m.start(0)
|
||||||
state.curtoken += s[:j]
|
state.curtoken += s[:j]
|
||||||
c = s[j : j + 1]
|
next_char = s[j : j + 1]
|
||||||
if c == b"\\":
|
if next_char == b"\\":
|
||||||
state.oct_value = b""
|
state.escaped_value = b""
|
||||||
it.skip += j + 1
|
it.skip += j + 1
|
||||||
elif c == b"(":
|
elif next_char == b"(":
|
||||||
state.paren += 1
|
state.paren += 1
|
||||||
state.curtoken += c
|
state.curtoken += next_char
|
||||||
it.skip += j + 1
|
it.skip += j + 1
|
||||||
elif c == b")":
|
elif next_char == b")":
|
||||||
state.paren -= 1
|
state.paren -= 1
|
||||||
if state.paren:
|
if state.paren:
|
||||||
state.curtoken += c
|
state.curtoken += next_char
|
||||||
it.skip += j + 1
|
it.skip += j + 1
|
||||||
return TokenString(
|
return TokenString(
|
||||||
initialpos,
|
initialpos,
|
||||||
@@ -54,17 +53,18 @@ def parse_string(initialpos: int, inp: Iterator[PInput]):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_string_oct(s: bytes, state: StringParseState) -> int:
|
def parse_string_escape(s: bytes, state: StringParseState) -> int:
|
||||||
for ci in range(len(s)):
|
for ci in range(len(s)):
|
||||||
c = s[ci : ci + 1]
|
c = s[ci : ci + 1]
|
||||||
if OCT_STRING.match(c) and len(state.oct_value) < 3:
|
if (ci < 1) and (c in ESC_STRING):
|
||||||
state.oct_value += c
|
state.curtoken += bytes([ESC_STRING[c]])
|
||||||
elif state.oct_value:
|
state.escaped_value = None
|
||||||
state.curtoken += int2byte(int(state.oct_value, 8))
|
return 1
|
||||||
return ci
|
if OCT_STRING.match(c) and len(state.escaped_value) < 3:
|
||||||
elif c in ESC_STRING:
|
state.escaped_value += c
|
||||||
state.curtoken += int2byte(ESC_STRING[c])
|
elif len(state.escaped_value) == 3:
|
||||||
state.oct_value = None
|
state.curtoken += bytes([int(state.escaped_value, 8)])
|
||||||
return ci + 1
|
return 3
|
||||||
state.oct_value = None
|
|
||||||
|
state.escaped_value = None
|
||||||
return len(s) + 1
|
return len(s) + 1
|
||||||
+13
-2
@@ -1,5 +1,6 @@
|
|||||||
from decimal import Decimal
|
|
||||||
import io
|
import io
|
||||||
|
from random import randint
|
||||||
|
from decimal import Decimal
|
||||||
from typing import List
|
from typing import List
|
||||||
from pdfmajor.tokenizer.token_parsers.string import parse_string
|
from pdfmajor.tokenizer.token_parsers.string import parse_string
|
||||||
from pdfmajor.tokenizer import PSTokenizer
|
from pdfmajor.tokenizer import PSTokenizer
|
||||||
@@ -13,7 +14,7 @@ from pdfmajor.tokenizer.token import (
|
|||||||
TokenLiteral,
|
TokenLiteral,
|
||||||
TokenString,
|
TokenString,
|
||||||
)
|
)
|
||||||
from pdfmajor.tokenizer.token_parsers.util import PInput
|
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
|
||||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||||
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
||||||
from pdfmajor.tokenizer.token_parsers.number import parse_number
|
from pdfmajor.tokenizer.token_parsers.number import parse_number
|
||||||
@@ -36,6 +37,16 @@ def make_stream_iter(data: bytes, initpos: int = 0):
|
|||||||
return iter_stream
|
return iter_stream
|
||||||
|
|
||||||
|
|
||||||
|
class Util(TestCase):
|
||||||
|
def test_safebuff(self):
|
||||||
|
it = SafeBufferIt(
|
||||||
|
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||||
|
)
|
||||||
|
for subbuf in it.into_iter():
|
||||||
|
self.assertEqual(len(subbuf), len(it.buffer) - it.skip)
|
||||||
|
it.skip += randint(1, max(len(subbuf) - 1, 1))
|
||||||
|
|
||||||
|
|
||||||
class Individual(TestCase):
|
class Individual(TestCase):
|
||||||
def test_comment_parser_ln(self):
|
def test_comment_parser_ln(self):
|
||||||
for buf_size in [2, 7, 11, 150]:
|
for buf_size in [2, 7, 11, 150]:
|
||||||
|
|||||||
Reference in new issue
Block a user