diff --git a/pdfmajor/tokenizer/token.py b/pdfmajor/tokenizer/token.py index 21903dd..98e4d26 100644 --- a/pdfmajor/tokenizer/token.py +++ b/pdfmajor/tokenizer/token.py @@ -1,3 +1,4 @@ +from enum import Enum from typing import Any, Generic, TypeVar, Union from decimal import Decimal @@ -29,34 +30,79 @@ class Token(Generic[T]): class TokenComment(Token[bytes]): - pass + """Token representing PDF comments + Detection of this type of token can be found in PDF 1.7 spec section 7.2.3 + """ - -class TokenHex(Token[bytes]): pass class TokenKeyword(Token[bytes]): + """Token representing PDF comments + Detection of this type of token can be found in PDF 1.7 spec + """ + pass class TokenLiteral(Token[str]): + """Token representing literal name tokens + Detection of this type of token can be found in PDF 1.7 spec section 7.3.5 + """ + pass class TokenBoolean(Token[bool]): + """Token representing PDF booleans + Detection of this type of token can be found in PDF 1.7 spec section 7.3.2 + """ + pass class TokenString(Token[str]): + """Token representing PDF literal string + Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2 + """ + + pass + + +class TokenHexString(Token[bytes]): + """Token representing PDF hex strings + Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3 + """ + pass class TokenInteger(Token[int]): + """Token representing PDF integers + Detection of this type of token can be found in PDF 1.7 + """ + pass class TokenDecimal(Token[Decimal]): + """Token representing PDF decimals + Detection of this type of token can be found in PDF 1.7 + """ + + pass + + +class TDictVaue(Enum): + OPEN = "<<" + CLOSE = ">>" + + +class TokenDictionary(Token[TDictVaue]): + """Token representing the brackets of dictionary + Detection of this type of token can be found in PDF 1.7 spec section 7.3.7 + """ + pass