implement dd bracket
This commit is contained in:
1 parent
370a1b40a1
commit
0eb5e4b8fc
4 files changed
+106
-9
No files matched your search
@@ -0,0 +1,53 @@
|
|||||||
|
from pdfmajor.tokenizer.token_parsers.util import PInput
|
||||||
|
from pdfmajor.safebufiterator import SafeBufferIt
|
||||||
|
from pdfmajor.lexer.exceptions import LexerEOF
|
||||||
|
from pdfmajor.lexer.regex import END_HEX_STRING, HEX_PAIR, SPC
|
||||||
|
from pdfmajor.lexer.token import TDictVaue, TokenDictionary, TokenHexString
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
|
||||||
|
def parse_double_angled_bracket(
|
||||||
|
buffer: SafeBufferIt,
|
||||||
|
) -> Union[TokenHexString, TokenDictionary]:
|
||||||
|
"""Parses input stream into a hex string object or match a simple dictionary token
|
||||||
|
|
||||||
|
Args:
|
||||||
|
buffer (SafeBufferIt)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Union[TokenHexString, TokenDictionary]
|
||||||
|
"""
|
||||||
|
initialpos = buffer.tell() - 1
|
||||||
|
step = next(buffer)
|
||||||
|
first_char = step.data[0:1]
|
||||||
|
if first_char == b"<":
|
||||||
|
buffer.seek(step.pos + 1)
|
||||||
|
return TokenDictionary(initialpos, 2, TDictVaue.OPEN)
|
||||||
|
buffer.seek(step.pos)
|
||||||
|
return parse_hexstring(buffer)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_hexstring(buffer: SafeBufferIt) -> TokenHexString:
|
||||||
|
"""Parses the input string into hex string
|
||||||
|
|
||||||
|
Args:
|
||||||
|
buffer (SafeBufferIt)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
TokenHexString
|
||||||
|
"""
|
||||||
|
curtoken = b""
|
||||||
|
initialpos = buffer.tell() - 1
|
||||||
|
for pos, buf in buffer:
|
||||||
|
m = END_HEX_STRING.search(buf, 0)
|
||||||
|
if not m:
|
||||||
|
curtoken += buf
|
||||||
|
else:
|
||||||
|
j = m.start(0)
|
||||||
|
curtoken += buf[:j]
|
||||||
|
buffer.seek(pos + j + 1)
|
||||||
|
token = HEX_PAIR.sub(
|
||||||
|
lambda x: bytes([int(x.group(0), 16)]), SPC.sub(b"", curtoken)
|
||||||
|
)
|
||||||
|
return TokenHexString(initialpos, buffer.tell(), token)
|
||||||
|
raise LexerEOF
|
||||||
@@ -16,3 +16,6 @@ ESC_STRING = {
|
|||||||
b")": 41,
|
b")": 41,
|
||||||
b"\\": 92,
|
b"\\": 92,
|
||||||
}
|
}
|
||||||
|
END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]")
|
||||||
|
HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.")
|
||||||
|
SPC = re.compile(br"\s")
|
||||||
@@ -61,17 +61,17 @@ def parse_string_escape(buffer: SafeBufferIt, state: StringParseState):
|
|||||||
for pos, buf in buffer:
|
for pos, buf in buffer:
|
||||||
for ci in range(len(buf)):
|
for ci in range(len(buf)):
|
||||||
c = buf[ci : ci + 1]
|
c = buf[ci : ci + 1]
|
||||||
if len(state.escaped_value) < 3:
|
if len(state.escaped_value) < 3: # type: ignore
|
||||||
state.escaped_value += c
|
state.escaped_value += c
|
||||||
buffer.seek(pos + ci + 1)
|
buffer.seek(pos + ci + 1)
|
||||||
if len(state.escaped_value) == 1:
|
if len(state.escaped_value) == 1: # type: ignore
|
||||||
if (ci < 1) and (c in ESC_STRING):
|
if (ci < 1) and (c in ESC_STRING):
|
||||||
state.curtoken += bytes([ESC_STRING[c]])
|
state.curtoken += bytes([ESC_STRING[c]])
|
||||||
state.escaped_value = None
|
state.escaped_value = None
|
||||||
return
|
return
|
||||||
elif OCT_STRING.match(state.escaped_value):
|
elif OCT_STRING.match(state.escaped_value): # type: ignore
|
||||||
if len(state.escaped_value) == 3:
|
if len(state.escaped_value) == 3: # type: ignore
|
||||||
state.curtoken += bytes([int(state.escaped_value, 8)])
|
state.curtoken += bytes([int(state.escaped_value, 8)]) # type: ignore
|
||||||
state.escaped_value = None
|
state.escaped_value = None
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|||||||
+45
-4
@@ -1,11 +1,20 @@
|
|||||||
import io
|
import io
|
||||||
|
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
|
||||||
from pdfmajor.lexer.string import parse_string
|
from pdfmajor.lexer.string import parse_string
|
||||||
from unittest import TestCase
|
from unittest import TestCase
|
||||||
|
|
||||||
from pdfmajor.lexer.name import parse_name
|
from pdfmajor.lexer.name import parse_name
|
||||||
from pdfmajor.lexer.keyword import parse_keyword
|
from pdfmajor.lexer.keyword import parse_keyword
|
||||||
from pdfmajor.safebufiterator import SafeBufferIt
|
from pdfmajor.safebufiterator import SafeBufferIt
|
||||||
from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName, TokenString
|
from pdfmajor.lexer.token import (
|
||||||
|
TDictVaue,
|
||||||
|
TokenBoolean,
|
||||||
|
TokenComment,
|
||||||
|
TokenDictionary,
|
||||||
|
TokenHexString,
|
||||||
|
TokenName,
|
||||||
|
TokenString,
|
||||||
|
)
|
||||||
from pdfmajor.lexer.comment import parse_comment
|
from pdfmajor.lexer.comment import parse_comment
|
||||||
|
|
||||||
|
|
||||||
@@ -97,7 +106,7 @@ class Basics(TestCase):
|
|||||||
def test_parse_string_nest_brackets(self):
|
def test_parse_string_nest_brackets(self):
|
||||||
for buf_size in [1, 2, 3, 50, 100]:
|
for buf_size in [1, 2, 3, 50, 100]:
|
||||||
with self.subTest(buf_end_loc=buf_size):
|
with self.subTest(buf_end_loc=buf_size):
|
||||||
buffer = make_stream_iter(br"x - (y*2-z-(2+3))) ", 88)
|
buffer = make_stream_iter(br"x - (y*2-z-(2+3))) ", buf_size)
|
||||||
token = parse_string(buffer)
|
token = parse_string(buffer)
|
||||||
self.assertIsInstance(token, TokenString)
|
self.assertIsInstance(token, TokenString)
|
||||||
self.assertEqual(token.start_loc, -1)
|
self.assertEqual(token.start_loc, -1)
|
||||||
@@ -107,7 +116,9 @@ class Basics(TestCase):
|
|||||||
def test_parse_string_hex_chars(self):
|
def test_parse_string_hex_chars(self):
|
||||||
for buf_size in [1, 2, 3, 50, 100]:
|
for buf_size in [1, 2, 3, 50, 100]:
|
||||||
with self.subTest(buf_end_loc=buf_size):
|
with self.subTest(buf_end_loc=buf_size):
|
||||||
buffer = make_stream_iter(br"def\040\0\0404ghi) (bach\\slask) ", 39)
|
buffer = make_stream_iter(
|
||||||
|
br"def\040\0\0404ghi) (bach\\slask) ", buf_size
|
||||||
|
)
|
||||||
token = parse_string(buffer)
|
token = parse_string(buffer)
|
||||||
self.assertIsInstance(token, TokenString)
|
self.assertIsInstance(token, TokenString)
|
||||||
self.assertEqual(token.start_loc, -1)
|
self.assertEqual(token.start_loc, -1)
|
||||||
@@ -117,9 +128,39 @@ class Basics(TestCase):
|
|||||||
def test_parse_string_escape_char(self):
|
def test_parse_string_escape_char(self):
|
||||||
for buf_size in [1, 2, 3, 50, 100]:
|
for buf_size in [1, 2, 3, 50, 100]:
|
||||||
with self.subTest(buf_end_loc=buf_size):
|
with self.subTest(buf_end_loc=buf_size):
|
||||||
buffer = make_stream_iter(br"bach\\slask) ", 39)
|
buffer = make_stream_iter(br"bach\\slask) ", buf_size)
|
||||||
token = parse_string(buffer)
|
token = parse_string(buffer)
|
||||||
self.assertIsInstance(token, TokenString)
|
self.assertIsInstance(token, TokenString)
|
||||||
self.assertEqual(token.start_loc, -1)
|
self.assertEqual(token.start_loc, -1)
|
||||||
self.assertEqual(token.value, r"bach\slask")
|
self.assertEqual(token.value, r"bach\slask")
|
||||||
self.assertEqual(token.end_loc, 12)
|
self.assertEqual(token.end_loc, 12)
|
||||||
|
|
||||||
|
def test_parse_hex_string(self):
|
||||||
|
for buf_size in [1, 2, 3, 50, 100]:
|
||||||
|
with self.subTest(buf_end_loc=buf_size):
|
||||||
|
buffer = make_stream_iter(br"10 20 31a 12a>", buf_size)
|
||||||
|
token = parse_hexstring(buffer)
|
||||||
|
self.assertIsInstance(token, TokenHexString)
|
||||||
|
self.assertEqual(token.start_loc, -1)
|
||||||
|
self.assertEqual(token.value, b"\x10 1\xa1*")
|
||||||
|
self.assertEqual(token.end_loc, 14)
|
||||||
|
|
||||||
|
def test_parse_dict_keyword(self):
|
||||||
|
for buf_size in [1, 2, 3, 50, 100]:
|
||||||
|
with self.subTest(buf_end_loc=buf_size):
|
||||||
|
buffer = make_stream_iter(br"<wow wee", buf_size)
|
||||||
|
token = parse_double_angled_bracket(buffer)
|
||||||
|
self.assertIsInstance(token, TokenDictionary)
|
||||||
|
self.assertEqual(token.start_loc, -1)
|
||||||
|
self.assertEqual(token.value, TDictVaue.OPEN)
|
||||||
|
self.assertEqual(token.end_loc, 2)
|
||||||
|
|
||||||
|
def test_parse_dd_to_hex(self):
|
||||||
|
for buf_size in [1, 2, 3, 50, 100]:
|
||||||
|
with self.subTest(buf_end_loc=buf_size):
|
||||||
|
buffer = make_stream_iter(br"10 20 31a 12a>", buf_size)
|
||||||
|
token = parse_double_angled_bracket(buffer)
|
||||||
|
self.assertIsInstance(token, TokenHexString)
|
||||||
|
self.assertEqual(token.start_loc, -1)
|
||||||
|
self.assertEqual(token.value, b"\x10 1\xa1*")
|
||||||
|
self.assertEqual(token.end_loc, 14)
|
||||||
Reference in new issue
Block a user