From c430c57fe0cff845fb94548a34793b3fa284125e Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Tue, 13 Oct 2020 20:54:59 -0400 Subject: [PATCH] split digit tokenizer --- pdfmajor/tokenizer/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py index a80f38c..038e7d2 100644 --- a/pdfmajor/tokenizer/__init__.py +++ b/pdfmajor/tokenizer/__init__.py @@ -61,9 +61,12 @@ class PSTokenizer: elif c == b"/": self.buf_skipstep += j + 1 yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer())) - elif c in b"-+" or c.isdigit(): + elif c in b"-+": self.buf_skipstep += j + 1 yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c)) + elif c.isdigit(): + self.buf_skipstep += j + yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer())) elif c == b".": self.buf_skipstep += j yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))