split digit tokenizer
This commit is contained in:
1 parent
ab657deda4
commit
c430c57fe0
1 file changed
+4
-1
@@ -61,9 +61,12 @@ class PSTokenizer:
|
|||||||
elif c == b"/":
|
elif c == b"/":
|
||||||
self.buf_skipstep += j + 1
|
self.buf_skipstep += j + 1
|
||||||
yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer()))
|
yield self._check_token(cur, parse_literal(bufpos + j, self.iter_buffer()))
|
||||||
elif c in b"-+" or c.isdigit():
|
elif c in b"-+":
|
||||||
self.buf_skipstep += j + 1
|
self.buf_skipstep += j + 1
|
||||||
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c))
|
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer(), c))
|
||||||
|
elif c.isdigit():
|
||||||
|
self.buf_skipstep += j
|
||||||
|
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
|
||||||
elif c == b".":
|
elif c == b".":
|
||||||
self.buf_skipstep += j
|
self.buf_skipstep += j
|
||||||
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
|
yield self._check_token(cur, parse_number(bufpos + j, self.iter_buffer()))
|
||||||
|
|||||||
Reference in new issue
Block a user