split digit tokenizer

This commit is contained in:
ari committed 2020-10-13 20:54:59 -04:00
1 parent ab657deda4
commit c430c57fe0
1 file changed
+4 -1
+4 -1
View File
@@ -61,9 +61,12 @@ class PSTokenizer:
elif c == b"/":
self.buf_skipstep += j + 1
yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer()))
elif c in b"-+" or c.isdigit():
elif c in b"-+":
self.buf_skipstep += j + 1
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c))
elif c.isdigit():
self.buf_skipstep += j
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
elif c == b".":
self.buf_skipstep += j
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))