From 0eb5e4b8fcda50feccdb7df12bf7012e8e0c3bc5 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Fri, 16 Oct 2020 16:55:54 -0400 Subject: [PATCH] implement dd bracket --- pdfmajor/lexer/dict_and_hex.py | 53 ++++++++++++++++++++++++++++++++++ pdfmajor/lexer/regex.py | 3 ++ pdfmajor/lexer/string.py | 10 +++---- tests/test_lexer.py | 49 ++++++++++++++++++++++++++++--- 4 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 pdfmajor/lexer/dict_and_hex.py diff --git a/pdfmajor/lexer/dict_and_hex.py b/pdfmajor/lexer/dict_and_hex.py new file mode 100644 index 0000000..807ebc3 --- /dev/null +++ b/pdfmajor/lexer/dict_and_hex.py @@ -0,0 +1,53 @@ +from pdfmajor.tokenizer.token_parsers.util import PInput +from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.lexer.exceptions import LexerEOF +from pdfmajor.lexer.regex import END_HEX_STRING, HEX_PAIR, SPC +from pdfmajor.lexer.token import TDictVaue, TokenDictionary, TokenHexString +from typing import Union + + +def parse_double_angled_bracket( + buffer: SafeBufferIt, +) -> Union[TokenHexString, TokenDictionary]: + """Parses input stream into a hex string object or match a simple dictionary token + + Args: + buffer (SafeBufferIt) + + Returns: + Union[TokenHexString, TokenDictionary] + """ + initialpos = buffer.tell() - 1 + step = next(buffer) + first_char = step.data[0:1] + if first_char == b"<": + buffer.seek(step.pos + 1) + return TokenDictionary(initialpos, 2, TDictVaue.OPEN) + buffer.seek(step.pos) + return parse_hexstring(buffer) + + +def parse_hexstring(buffer: SafeBufferIt) -> TokenHexString: + """Parses the input string into hex string + + Args: + buffer (SafeBufferIt) + + Returns: + TokenHexString + """ + curtoken = b"" + initialpos = buffer.tell() - 1 + for pos, buf in buffer: + m = END_HEX_STRING.search(buf, 0) + if not m: + curtoken += buf + else: + j = m.start(0) + curtoken += buf[:j] + buffer.seek(pos + j + 1) + token = HEX_PAIR.sub( + lambda x: bytes([int(x.group(0), 16)]), SPC.sub(b"", curtoken) + ) + return TokenHexString(initialpos, buffer.tell(), token) + raise LexerEOF diff --git a/pdfmajor/lexer/regex.py b/pdfmajor/lexer/regex.py index 1354ddb..f00d4f5 100644 --- a/pdfmajor/lexer/regex.py +++ b/pdfmajor/lexer/regex.py @@ -16,3 +16,6 @@ ESC_STRING = { b")": 41, b"\\": 92, } +END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]") +HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.") +SPC = re.compile(br"\s") diff --git a/pdfmajor/lexer/string.py b/pdfmajor/lexer/string.py index bab5683..4215ea1 100644 --- a/pdfmajor/lexer/string.py +++ b/pdfmajor/lexer/string.py @@ -61,17 +61,17 @@ def parse_string_escape(buffer: SafeBufferIt, state: StringParseState): for pos, buf in buffer: for ci in range(len(buf)): c = buf[ci : ci + 1] - if len(state.escaped_value) < 3: + if len(state.escaped_value) < 3: # type: ignore state.escaped_value += c buffer.seek(pos + ci + 1) - if len(state.escaped_value) == 1: + if len(state.escaped_value) == 1: # type: ignore if (ci < 1) and (c in ESC_STRING): state.curtoken += bytes([ESC_STRING[c]]) state.escaped_value = None return - elif OCT_STRING.match(state.escaped_value): - if len(state.escaped_value) == 3: - state.curtoken += bytes([int(state.escaped_value, 8)]) + elif OCT_STRING.match(state.escaped_value): # type: ignore + if len(state.escaped_value) == 3: # type: ignore + state.curtoken += bytes([int(state.escaped_value, 8)]) # type: ignore state.escaped_value = None return else: diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 503a5bb..1b16390 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -1,11 +1,20 @@ import io +from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring from pdfmajor.lexer.string import parse_string from unittest import TestCase from pdfmajor.lexer.name import parse_name from pdfmajor.lexer.keyword import parse_keyword from pdfmajor.safebufiterator import SafeBufferIt -from pdfmajor.lexer.token import TokenBoolean, TokenComment, TokenName, TokenString +from pdfmajor.lexer.token import ( + TDictVaue, + TokenBoolean, + TokenComment, + TokenDictionary, + TokenHexString, + TokenName, + TokenString, +) from pdfmajor.lexer.comment import parse_comment @@ -97,7 +106,7 @@ class Basics(TestCase): def test_parse_string_nest_brackets(self): for buf_size in [1, 2, 3, 50, 100]: with self.subTest(buf_end_loc=buf_size): - buffer = make_stream_iter(br"x - (y*2-z-(2+3))) ", 88) + buffer = make_stream_iter(br"x - (y*2-z-(2+3))) ", buf_size) token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) @@ -107,7 +116,9 @@ class Basics(TestCase): def test_parse_string_hex_chars(self): for buf_size in [1, 2, 3, 50, 100]: with self.subTest(buf_end_loc=buf_size): - buffer = make_stream_iter(br"def\040\0\0404ghi) (bach\\slask) ", 39) + buffer = make_stream_iter( + br"def\040\0\0404ghi) (bach\\slask) ", buf_size + ) token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) @@ -117,9 +128,39 @@ class Basics(TestCase): def test_parse_string_escape_char(self): for buf_size in [1, 2, 3, 50, 100]: with self.subTest(buf_end_loc=buf_size): - buffer = make_stream_iter(br"bach\\slask) ", 39) + buffer = make_stream_iter(br"bach\\slask) ", buf_size) token = parse_string(buffer) self.assertIsInstance(token, TokenString) self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, r"bach\slask") self.assertEqual(token.end_loc, 12) + + def test_parse_hex_string(self): + for buf_size in [1, 2, 3, 50, 100]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(br"10 20 31a 12a>", buf_size) + token = parse_hexstring(buffer) + self.assertIsInstance(token, TokenHexString) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, b"\x10 1\xa1*") + self.assertEqual(token.end_loc, 14) + + def test_parse_dict_keyword(self): + for buf_size in [1, 2, 3, 50, 100]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(br"", buf_size) + token = parse_double_angled_bracket(buffer) + self.assertIsInstance(token, TokenHexString) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, b"\x10 1\xa1*") + self.assertEqual(token.end_loc, 14)