diff --git a/Makefile b/Makefile index 3a2bae0..4293f35 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ clean: rm -rf dist rm -rf *.egg-info +lint: + black pdfmajor + test: ./venv/bin/python -m unittest discover tests @@ -19,4 +22,4 @@ deploy: python setup.py sdist bdist_wheel git tag $(cat .version) git push -u origin $(cat .version) - twine upload dist/* \ No newline at end of file + twine upload dist/* diff --git a/pdfmajor/execptions.py b/pdfmajor/execptions.py index 3215e0b..b65b35c 100644 --- a/pdfmajor/execptions.py +++ b/pdfmajor/execptions.py @@ -80,10 +80,6 @@ class PDFSyntaxError(PDFException): pass -class PDFTextExtractionNotAllowed(PDFException): - pass - - class PSEOF(PSException): pass diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py index 4485079..97c6b72 100644 --- a/pdfmajor/tokenizer/__init__.py +++ b/pdfmajor/tokenizer/__init__.py @@ -87,18 +87,18 @@ class PSTokenizer: cur, parse_keyword(bufpos + j, self.iter_buffer()) ) elif c == b"(": - raise "WIP" + raise NotImplementedError # self._curtoken = b"" # self.paren = 1 # self._current_parse_func = self._parse_string # return j + 1 elif c == b"<": - raise "WIP" + raise NotImplementedError # self._curtoken = b"" # self._current_parse_func = self._parse_wopen # return j + 1 elif c == b">": - raise "WIP" + raise NotImplementedError # self._curtoken = b"" # self._current_parse_func = self._parse_wclose # return j + 1 diff --git a/pdfmajor/tokenizer/exceptions.py b/pdfmajor/tokenizer/exceptions.py index f9323f7..ce904ed 100644 --- a/pdfmajor/tokenizer/exceptions.py +++ b/pdfmajor/tokenizer/exceptions.py @@ -12,5 +12,5 @@ class TokenizerEOF(TokenizerError): class InvalidToken(TokenizerError): def __init__(self, pos: int, token: bytes) -> None: super().__init__( - "Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=token) + "Invalid Token Found at {pos}, of value={v:r}".format(pos=pos, v=token) ) diff --git a/pdfmajor/tokenizer/parse_funcs.py b/pdfmajor/tokenizer/parse_funcs.py deleted file mode 100644 index db0c73b..0000000 --- a/pdfmajor/tokenizer/parse_funcs.py +++ /dev/null @@ -1,100 +0,0 @@ -from pdfmajor.tokenizer.token import Token, TokenKeyword -from typing import Callable, NamedTuple, Optional, Union -from .constants import EOL, NONSPC - -ParseFunc = Callable[["ParseInput"], "NextParseAction"] - - -class NextParseAction(NamedTuple): - next_parse_action: ParseFunc - charpos: int - curtokenpos: int - curtoken: Union[Token, bytes] = b"" - paren: Optional[int] = None - - -class ParseInput(NamedTuple): - bufvalue: bytes - charpos: int - bufpos: int - - -def parse_main(inp: ParseInput) -> NextParseAction: - m = NONSPC.search(inp.bufvalue, inp.charpos) - if not m: - return len(inp.bufvalue) - j = m.start(0) - c = inp.bufvalue[j : j + 1] - if c == b"%": - return NextParseAction( - curtoken=b"%", - next_parse_action=__parse_comment, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - ) - elif c == b"/": - return NextParseAction( - next_parse_action=__parse_literal, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - ) - elif c in b"-+" or c.isdigit(): - return NextParseAction( - curtoken=c, - next_parse_action=__parse_number, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - ) - elif c == b".": - return NextParseAction( - curtoken=c, - next_parse_action=__parse_float, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - ) - elif c.isalpha(): - return NextParseAction( - curtoken=c, - next_parse_action=__parse_keyword, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - ) - elif c == b"(": - return NextParseAction( - next_parse_action=__parse_string, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - paren=1, - ) - elif c == b"<": - return NextParseAction( - next_parse_action=__parse_wopen, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - ) - elif c == b">": - return NextParseAction( - next_parse_action=__parse_wclose, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - ) - else: - return NextParseAction( - next_parse_action=parse_main, - curtokenpos=inp.bufpos + j, - charpos=j + 1, - curtoken=TokenKeyword(inp.bufpos + j, c), - ) - - -def __parse_comment(s, i): - m = EOL.search(s, i) - if not m: - self._curtoken += s[i:] - return len(s) - j = m.start(0) - self._curtoken += s[i:j] - self._current_parse_func = self._parse_main - # We ignore comments. - # self._tokens.append(self._curtoken) - return j diff --git a/pdfmajor/tokenizer/token_parsers/comment.py b/pdfmajor/tokenizer/token_parsers/comment.py index c85744a..cad6789 100644 --- a/pdfmajor/tokenizer/token_parsers/comment.py +++ b/pdfmajor/tokenizer/token_parsers/comment.py @@ -1,3 +1,4 @@ +from pdfmajor.tokenizer.exceptions import TokenizerEOF from pdfmajor.tokenizer.token import ( TokenComment, ) @@ -12,11 +13,11 @@ def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment: """Parses input stream into a comment Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) + initialpos (int): initial position where we started + inp (Iterator[PInput]) Returns: - TokenComment + TokenComment """ curtoken = b"" for curpos, s in inp: @@ -27,3 +28,4 @@ def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment: j = m.start(0) curtoken += s[:j] return TokenComment(initialpos, cmp_tsize(curpos, initialpos, j), curtoken) + raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/keyword.py b/pdfmajor/tokenizer/token_parsers/keyword.py index 1ce4696..ed2d6ca 100644 --- a/pdfmajor/tokenizer/token_parsers/keyword.py +++ b/pdfmajor/tokenizer/token_parsers/keyword.py @@ -1,3 +1,4 @@ +from pdfmajor.tokenizer.exceptions import TokenizerEOF from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword from pdfmajor.tokenizer.constants import END_KEYWORD from typing import Iterator, Union @@ -10,11 +11,11 @@ def parse_keyword( """Parses input stream into a keyword or bool Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) + initialpos (int): initial position where we started + inp (Iterator[PInput]) Returns: - Union[TokenKeyword, TokenBoolean] + Union[TokenKeyword, TokenBoolean] """ curtoken = b"" for curpos, s in inp: @@ -32,3 +33,4 @@ def parse_keyword( return TokenKeyword( initialpos, cmp_tsize(curpos, initialpos, j), curtoken ) + raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py index e6139d5..9ddaea1 100644 --- a/pdfmajor/tokenizer/token_parsers/literal.py +++ b/pdfmajor/tokenizer/token_parsers/literal.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from pdfmajor.tokenizer.exceptions import TokenizerEOF from pdfmajor.tokenizer.token_parsers.util import cmp_tsize from pdfmajor.tokenizer.token_parsers.util import PInput @@ -19,11 +20,11 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: """Parses input stream into a literal Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) + initialpos (int): initial position where we started + inp (Iterator[PInput]) Returns: - TokenLiteral + TokenLiteral """ state = LiteralParseState(b"") for curpos, buf in inp: @@ -62,3 +63,4 @@ def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: cmp_tsize(curpos, initialpos, j), state.curtoken.decode(), ) + raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/number.py b/pdfmajor/tokenizer/token_parsers/number.py index 6e044a1..c00bc28 100644 --- a/pdfmajor/tokenizer/token_parsers/number.py +++ b/pdfmajor/tokenizer/token_parsers/number.py @@ -1,7 +1,7 @@ from decimal import Decimal from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX from pdfmajor.utils import int2byte -from pdfmajor.tokenizer.exceptions import InvalidToken +from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF from pdfmajor.tokenizer.token import ( TokenDecimal, TokenNumber, @@ -13,23 +13,21 @@ from pdfmajor.tokenizer.constants import ( from typing import Iterator, Literal, Union from .util import cmp_tsize, PInput -VALID_SIGNS = Union[Literal[b"+"], Literal[b"-"]] - def parse_number( - initialpos: int, inp: Iterator[PInput], sign: VALID_SIGNS = b"+" + initialpos: int, inp: Iterator[PInput], sign: bytes = b"+" ) -> TokenNumber: """Parses input stream into a number Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - sign (str): either a + or - + initialpos (int): initial position where we started + inp (Iterator[PInput]) + sign (str): either a + or - Returns: - TokenNumber + TokenNumber """ - assert sign in [b"+", b"-"], f"Number was provided an invalid sign {sign}" + assert sign in [b"+", b"-"], "Number was provided an invalid sign {!r}".format(sign) curtoken = b"" + sign is_dec = False for curpos, s in inp: @@ -70,3 +68,4 @@ def parse_number( ) except ValueError: raise InvalidToken(initialpos, curtoken) + raise TokenizerEOF diff --git a/pdfmajor/utils/log.py b/pdfmajor/utils/log.py index 143d318..d799860 100644 --- a/pdfmajor/utils/log.py +++ b/pdfmajor/utils/log.py @@ -1,8 +1,10 @@ import logging +from logging import Logger +from typing import Dict logging.basicConfig(level=logging.WARNING) -LOGGERS = {} +LOGGERS: Dict[str, Logger] = {} def set_log_level(debug_level: int): diff --git a/pdfmajor/utils/types.py b/pdfmajor/utils/types.py index 0db2dd5..f36091a 100644 --- a/pdfmajor/utils/types.py +++ b/pdfmajor/utils/types.py @@ -1,7 +1,7 @@ from typing import List, Tuple -INF = (1 << 31) - 1 -Point = Tuple[float] +INF = float((1 << 31) - 1) +Point = Tuple[float, float] unicode = str