added Null token

This commit is contained in:
ari committed 2020-10-17 20:14:03 -04:00
1 parent da936e6a48
commit 4e1f45e205
3 files changed
+22 -1

No files matched your search

+3 -1
View File
@@ -1,6 +1,6 @@
from pdfmajor.streambuffer import BufferStream from pdfmajor.streambuffer import BufferStream
from pdfmajor.lexer.exceptions import LexerEOF from pdfmajor.lexer.exceptions import LexerEOF
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword from pdfmajor.lexer.token import TokenBoolean, TokenKeyword, TokenNull
from pdfmajor.lexer.regex import END_KEYWORD from pdfmajor.lexer.regex import END_KEYWORD
from typing import Optional, Union from typing import Optional, Union
@@ -31,6 +31,8 @@ def parse_keyword(
return TokenBoolean(initialpos, buffer.tell(), True) return TokenBoolean(initialpos, buffer.tell(), True)
elif curtoken == b"false": elif curtoken == b"false":
return TokenBoolean(initialpos, buffer.tell(), False) return TokenBoolean(initialpos, buffer.tell(), False)
elif curtoken == b"null":
return TokenNull(initialpos, buffer.tell(), None)
else: else:
return TokenKeyword(initialpos, buffer.tell(), curtoken) return TokenKeyword(initialpos, buffer.tell(), curtoken)
raise LexerEOF raise LexerEOF
+8
View File
@@ -68,6 +68,14 @@ class TokenBoolean(Token[bool]):
pass pass
class TokenNull(Token[None]):
"""Token representing the PDF null object
Detection of this type of token can be found in PDF 1.7 spec section 7.3.9
"""
pass
class TokenString(Token[str]): class TokenString(Token[str]):
"""Token representing PDF literal string """Token representing PDF literal string
Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2 Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2
+11
View File
@@ -19,6 +19,7 @@ from pdfmajor.lexer.token import (
TokenArray, TokenArray,
TokenBoolean, TokenBoolean,
TokenComment, TokenComment,
TokenNull,
TokenReal, TokenReal,
TokenDictionary, TokenDictionary,
TokenHexString, TokenHexString,
@@ -85,6 +86,16 @@ class Basics(TestCase):
self.assertEqual(token.value, False) self.assertEqual(token.value, False)
self.assertEqual(token.end_loc, 5) self.assertEqual(token.end_loc, 5)
def test_parse_null(self):
for buf_size in [2, 3, 50]:
with self.subTest(buf_end_loc=buf_size):
buffer = make_stream_iter(b"null Nothing here", buf_size)
token = parse_keyword(buffer)
self.assertIsInstance(token, TokenNull)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, None)
self.assertEqual(token.end_loc, 4)
def test_parse_word(self): def test_parse_word(self):
for buf_size in [2, 3, 50]: for buf_size in [2, 3, 50]:
with self.subTest(buf_end_loc=buf_size): with self.subTest(buf_end_loc=buf_size):