From a2d2b7808d8af922a7a069d1dcbbdbca96651e23 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Thu, 15 Oct 2020 09:37:30 -0400 Subject: [PATCH] added literal parsing to main tokenizer --- pdfmajor/tokenizer/__init__.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py index 97994c0..f5e001e 100644 --- a/pdfmajor/tokenizer/__init__.py +++ b/pdfmajor/tokenizer/__init__.py @@ -1,4 +1,7 @@ import io +from pdfmajor.tokenizer.token_parsers.dictionary_hexstring import ( + parse_double_angled_bracket, +) from pdfmajor.tokenizer.token_parsers.string import parse_string from typing import Iterator, Optional @@ -10,7 +13,7 @@ from pdfmajor.tokenizer.token_parsers.literal import parse_literal from pdfmajor.tokenizer.token_parsers.number import parse_number from pdfmajor.tokenizer.token_parsers.comment import parse_comment from pdfmajor.tokenizer.token_parsers.util import PInput -from pdfmajor.tokenizer.token import Token, TokenKeyword +from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenKeyword class PSTokenizer: @@ -99,15 +102,15 @@ class PSTokenizer: cur, parse_string(bufpos + j, self.iter_buffer()) ) elif c == b"<": - raise NotImplementedError - # self._curtoken = b"" - # self._current_parse_func = self._parse_wopen - # return j + 1 + self.inc_buf_skipstep(j) + yield self._check_token( + cur, parse_double_angled_bracket(bufpos + j, self.iter_buffer()) + ) elif c == b">": - raise NotImplementedError - # self._curtoken = b"" - # self._current_parse_func = self._parse_wclose - # return j + 1 + self.inc_buf_skipstep(j) + if buf[j : j + 2] == ">>": + self.inc_buf_skipstep(1) + yield TokenDictionary(bufpos + j, 2, TDictVaue.CLOSE) else: self.inc_buf_skipstep(j + 1) yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c))