From 4e1f45e205921d19a290ceaf949e17e8733f3bbd Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sat, 17 Oct 2020 20:14:03 -0400 Subject: [PATCH] added Null token --- pdfmajor/lexer/keyword.py | 4 +++- pdfmajor/lexer/token.py | 8 ++++++++ tests/test_lexer.py | 11 +++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pdfmajor/lexer/keyword.py b/pdfmajor/lexer/keyword.py index a3a30c2..5c1ce3a 100644 --- a/pdfmajor/lexer/keyword.py +++ b/pdfmajor/lexer/keyword.py @@ -1,6 +1,6 @@ from pdfmajor.streambuffer import BufferStream from pdfmajor.lexer.exceptions import LexerEOF -from pdfmajor.lexer.token import TokenBoolean, TokenKeyword +from pdfmajor.lexer.token import TokenBoolean, TokenKeyword, TokenNull from pdfmajor.lexer.regex import END_KEYWORD from typing import Optional, Union @@ -31,6 +31,8 @@ def parse_keyword( return TokenBoolean(initialpos, buffer.tell(), True) elif curtoken == b"false": return TokenBoolean(initialpos, buffer.tell(), False) + elif curtoken == b"null": + return TokenNull(initialpos, buffer.tell(), None) else: return TokenKeyword(initialpos, buffer.tell(), curtoken) raise LexerEOF diff --git a/pdfmajor/lexer/token.py b/pdfmajor/lexer/token.py index b3f616f..78a0b2f 100644 --- a/pdfmajor/lexer/token.py +++ b/pdfmajor/lexer/token.py @@ -68,6 +68,14 @@ class TokenBoolean(Token[bool]): pass +class TokenNull(Token[None]): + """Token representing the PDF null object + Detection of this type of token can be found in PDF 1.7 spec section 7.3.9 + """ + + pass + + class TokenString(Token[str]): """Token representing PDF literal string Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2 diff --git a/tests/test_lexer.py b/tests/test_lexer.py index dd85a15..b9ac168 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -19,6 +19,7 @@ from pdfmajor.lexer.token import ( TokenArray, TokenBoolean, TokenComment, + TokenNull, TokenReal, TokenDictionary, TokenHexString, @@ -85,6 +86,16 @@ class Basics(TestCase): self.assertEqual(token.value, False) self.assertEqual(token.end_loc, 5) + def test_parse_null(self): + for buf_size in [2, 3, 50]: + with self.subTest(buf_end_loc=buf_size): + buffer = make_stream_iter(b"null Nothing here", buf_size) + token = parse_keyword(buffer) + self.assertIsInstance(token, TokenNull) + self.assertEqual(token.start_loc, -1) + self.assertEqual(token.value, None) + self.assertEqual(token.end_loc, 4) + def test_parse_word(self): for buf_size in [2, 3, 50]: with self.subTest(buf_end_loc=buf_size):