diff --git a/_test.py b/_test.py new file mode 100644 index 0000000..a830f61 --- /dev/null +++ b/_test.py @@ -0,0 +1,24 @@ +import io +from pdfmajor.parser.PSStackParser.PSBaseParser import PSBaseParser + +TESTDATA = br'''%!PS +begin end + " @ # +/a/BCD /Some_Name /foo#5f#xbaa +0 +1 -2 .5 1.234 +(abc) () (abc ( def ) ghi) +(def\040\0\0404ghi) (bach\\slask) (foo\nbaa) +(this % is not a comment.) +(foo +baa) +(foo\ +baa) +<> <20> < 40 4020 > + +func/a/b{(c)do*}def +[ 1 (z) ! ] +<< /foo (bar) >> +''' + +parser = PSBaseParser(io.BytesIO(TESTDATA)) diff --git a/pdfmajor/tokenizer/token.py b/pdfmajor/tokenizer/token.py index 57670a6..08fbf11 100644 --- a/pdfmajor/tokenizer/token.py +++ b/pdfmajor/tokenizer/token.py @@ -29,6 +29,11 @@ class TokenHex(Token[bytes]): class TokenKeyword(Token[bytes]): pass +class TokenLiteral(Token[bytes]): + pass + +class TokenBoolean(Token[bool]): + pass class TokenString(Token[str]): pass diff --git a/pdfmajor/tokenizer/token_parser.py b/pdfmajor/tokenizer/token_parser.py deleted file mode 100644 index 77bbb18..0000000 --- a/pdfmajor/tokenizer/token_parser.py +++ /dev/null @@ -1,104 +0,0 @@ -from decimal import Decimal -from pdfmajor.tokenizer.exceptions import InvalidToken -from pdfmajor.tokenizer.token import Token, TokenComment, TokenDecimal, TokenNumber, TokenInteger -from pdfmajor.tokenizer.constants import END_NUMBER, EOL -from typing import Callable, Iterator, NamedTuple - - -class PInput(NamedTuple): - pos: int - buf: bytes - - -ParserFunc = Callable[[int, Iterator[PInput]], Token] - -def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int: - """Computes the size of the token based on its positional data - - Args: - curpos (int): current position of the buffer - initialpos (int): initial position where the parser started working - tokenend_idx (int): last relevant character index within the last buffered value - - Returns: - int: length of index - """ - return curpos-initialpos+tokenend_idx - -def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment: - """Parses input stream into a comment - - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - - Returns: - TokenComment - """ - curtoken = b"" - for curpos, s in inp: - m = EOL.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - return TokenComment( - initialpos, - cmp_tsize(curpos, initialpos, j), - curtoken - ) - -def parse_number(initialpos: int, inp: Iterator[PInput]) -> TokenNumber: - """Parses input stream into a number - - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - - Returns: - TokenNumber - """ - curtoken = b"" - is_dec = False - for curpos, s in inp: - m = END_NUMBER.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - c = s[j : j + 1] - if (c == b".") and not is_dec: - is_dec = True - curtoken += c - s = s[j+1:] - m = END_NUMBER.search(s, 0) - if not m: - curtoken += s - continue - else: - nj = m.start(0) - curtoken += s[:nj] - j += nj - return TokenDecimal( - initialpos, - cmp_tsize(curpos, initialpos, j), - Decimal(curtoken.decode()) - ) - try: - if is_dec: - return TokenDecimal( - initialpos, - cmp_tsize(curpos, initialpos, j), - Decimal(curtoken.decode()) - ) - else: - return TokenInteger( - initialpos, - cmp_tsize(curpos, initialpos, j), - int(curtoken) - ) - except ValueError: - raise InvalidToken(initialpos, curtoken) - diff --git a/pdfmajor/tokenizer/token_parsers/__init__.py b/pdfmajor/tokenizer/token_parsers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pdfmajor/tokenizer/token_parsers/comment.py b/pdfmajor/tokenizer/token_parsers/comment.py new file mode 100644 index 0000000..343e928 --- /dev/null +++ b/pdfmajor/tokenizer/token_parsers/comment.py @@ -0,0 +1,32 @@ +from pdfmajor.tokenizer.token import ( + TokenComment, +) +from pdfmajor.tokenizer.constants import ( + EOL, +) +from typing import Iterator +from .util import cmp_tsize, PInput + +def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment: + """Parses input stream into a comment + + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) + + Returns: + TokenComment + """ + curtoken = b"" + for curpos, s in inp: + m = EOL.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + return TokenComment( + initialpos, + cmp_tsize(curpos, initialpos, j), + curtoken + ) diff --git a/pdfmajor/tokenizer/token_parsers/keyword.py b/pdfmajor/tokenizer/token_parsers/keyword.py new file mode 100644 index 0000000..95bf3c3 --- /dev/null +++ b/pdfmajor/tokenizer/token_parsers/keyword.py @@ -0,0 +1,50 @@ +from pdfmajor.tokenizer.token import ( + TokenBoolean, + TokenKeyword +) +from pdfmajor.tokenizer.constants import ( + END_KEYWORD +) +from typing import Iterator, Union +from .util import cmp_tsize, PInput + +def parse_keyword( + initialpos: int, + inp: Iterator[PInput] +) -> Union[TokenKeyword, TokenBoolean]: + """Parses input stream into a keyword or bool + + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) + + Returns: + Union[TokenKeyword, TokenBoolean] + """ + curtoken = b"" + for curpos, s in inp: + m = END_KEYWORD.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + if curtoken == b"true": + return TokenBoolean( + initialpos, + cmp_tsize(curpos, initialpos, j), + True + ) + elif curtoken == b"false": + return TokenBoolean( + initialpos, + cmp_tsize(curpos, initialpos, j), + False + ) + else: + return TokenKeyword( + initialpos, + cmp_tsize(curpos, initialpos, j), + curtoken + ) + diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py new file mode 100644 index 0000000..f5efaa1 --- /dev/null +++ b/pdfmajor/tokenizer/token_parsers/literal.py @@ -0,0 +1,82 @@ + +from pdfmajor.tokenizer.token_parsers.util import cmp_tsize +from pdfmajor.tokenizer.token_parsers.util import PInput +from pdfmajor.tokenizer.token import TokenLiteral +from pdfmajor.utils import int2byte +from ..constants import HEX, END_LITERAL + +from typing import Iterator, NamedTuple, Optional, Union + +def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: + """Parses input stream into a literal + + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) + + Returns: + TokenLiteral + """ + curtoken = b"" + parsing_hex_mode = False + hex_value = b"" + for curpos, s in inp: + if parsing_hex_mode: + hex_value, curtoken, skip_idx = _parse_literal_hex(s, hex_value, curtoken) + if skip_idx is not None: + curpos, s = curpos + skip_idx, s[skip_idx:] + else: + continue + out = _parse_literal_raw(initialpos, curpos, s, curtoken) + if isinstance(out, TokenLiteral): + return out + else: + curtoken, parsing_hex_mode = out + if parsing_hex_mode: + hex_value = b"" + +class LiteralHexParseOut(NamedTuple): + hex_value: bytes + curtoken: bytes + skip_idx: Optional[int] = None + +def _parse_literal_hex( + s: bytes, + hex_value: bytes, + curtoken: bytes +) -> LiteralHexParseOut: + for ci in range(len(s)): + c = s[ci:ci+1] + if HEX.match(c) and len(hex_value) < 2: + hex_value += c + elif hex_value: + curtoken += int2byte(int(hex_value, 16)) + hex_value = b"" + return LiteralHexParseOut(hex_value, curtoken, ci) + return LiteralHexParseOut(hex_value, curtoken) + +class LiteralParseOut(NamedTuple): + curtoken: bytes + change_to_hex_mode: bool = False + +def _parse_literal_raw( + initialpos: int, + curpos: int, + s: bytes, + curtoken: bytes +) -> Union[LiteralParseOut, TokenLiteral]: + m = END_LITERAL.search(s, 0) + if not m: + return LiteralParseOut(curtoken+s) + else: + j = m.start(0) + curtoken += s[:j] + c = s[j : j + 1] + if c == b"#": + return LiteralParseOut(curtoken, True) + curtoken = curtoken.decode() + return TokenLiteral( + initialpos, + cmp_tsize(curpos, initialpos, j), + curtoken + ) diff --git a/pdfmajor/tokenizer/token_parsers/number.py b/pdfmajor/tokenizer/token_parsers/number.py new file mode 100644 index 0000000..3e6528e --- /dev/null +++ b/pdfmajor/tokenizer/token_parsers/number.py @@ -0,0 +1,75 @@ +from decimal import Decimal +from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX +from pdfmajor.utils import int2byte +from pdfmajor.tokenizer.exceptions import InvalidToken +from pdfmajor.tokenizer.token import ( + TokenDecimal, + TokenNumber, + TokenInteger, +) +from pdfmajor.tokenizer.constants import ( + END_NUMBER, +) +from typing import Iterator, Literal, Union +from .util import cmp_tsize, PInput + +VALID_SIGNS = Union[Literal[b'+'], Literal[b'-']] + +def parse_number( + initialpos: int, + inp: Iterator[PInput], + sign: VALID_SIGNS = b'+' +) -> TokenNumber: + """Parses input stream into a number + + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) + sign (str): either a + or - + + Returns: + TokenNumber + """ + assert sign in [b'+', b'-'], f"Number was provided an invalid sign {sign}" + curtoken = b"" + sign + is_dec = False + for curpos, s in inp: + m = END_NUMBER.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + c = s[j : j + 1] + if (c == b".") and not is_dec: + is_dec = True + curtoken += c + s = s[j+1:] + m = END_NUMBER.search(s, 0) + if not m: + curtoken += s + continue + else: + nj = m.start(0) + curtoken += s[:nj] + j += nj + return TokenDecimal( + initialpos, + cmp_tsize(curpos, initialpos, j), + Decimal(curtoken.decode()) + ) + try: + if is_dec: + return TokenDecimal( + initialpos, + cmp_tsize(curpos, initialpos, j), + Decimal(curtoken.decode()) + ) + else: + return TokenInteger( + initialpos, + cmp_tsize(curpos, initialpos, j), + int(curtoken) + ) + except ValueError: + raise InvalidToken(initialpos, curtoken) diff --git a/pdfmajor/tokenizer/token_parsers/string.py b/pdfmajor/tokenizer/token_parsers/string.py new file mode 100644 index 0000000..d1e182a --- /dev/null +++ b/pdfmajor/tokenizer/token_parsers/string.py @@ -0,0 +1,68 @@ +from decimal import Decimal +from pdfmajor.parser.PSStackParser.constants import END_LITERAL, HEX +from pdfmajor.utils import int2byte +from pdfmajor.tokenizer.exceptions import InvalidToken +from pdfmajor.tokenizer.token import ( + Token, + TokenBoolean, + TokenComment, + TokenDecimal, + TokenKeyword, TokenLiteral, + TokenNumber, + TokenInteger, + TokenString +) +from pdfmajor.tokenizer.constants import ( + END_KEYWORD, + END_NUMBER, + END_STRING, + EOL, + ESC_STRING, + OCT_STRING +) +from typing import Callable, Iterator, Literal, NamedTuple, Union + + +def parse_string(initialpos: int, inp: Iterator[PInput]): + curtoken = b"" + paren = 1 + for curpos, s in inp: + m = END_STRING.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + c = s[j : j + 1] + if c == b"\\": + self._current_parse_func = self._parse_string_1 + return j + 1 + elif c == b"(": + paren += 1 + curtoken += c + return j + 1 + elif c == b")": + paren -= 1 + if paren: # WTF, they said balanced parens need no special treatment. + curtoken += c + return j + 1 + return TokenString( + initialpos, + cmp_tsize(curpos, initialpos, j), + curtoken.decode() + ) + +def parse_string_oct(initialpos: int, inp: Iterator[PInput]): + curtoken: bytes = b"" + oct_val: bytes = b"" + for curpos, s in inp: + for ci in range(len(s)): + c = s[ci:ci+1] + if OCT_STRING.match(c) and len(oct_val) < 3: + oct_val += c + elif oct_val: + curtoken += int2byte(int(oct_val, 8)) + return (curtoken, curpos+ci) + elif c in ESC_STRING: + curtoken += int2byte(ESC_STRING[c]) + return (curtoken, curpos+ci+1) diff --git a/pdfmajor/tokenizer/token_parsers/util.py b/pdfmajor/tokenizer/token_parsers/util.py new file mode 100644 index 0000000..f23974c --- /dev/null +++ b/pdfmajor/tokenizer/token_parsers/util.py @@ -0,0 +1,25 @@ +from pdfmajor.tokenizer.token import ( + Token +) +from typing import Callable, Iterator, NamedTuple + + +class PInput(NamedTuple): + pos: int + buf: bytes + + +ParserFunc = Callable[[int, Iterator[PInput]], Token] + +def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int: + """Computes the size of the token based on its positional data + + Args: + curpos (int): current position of the buffer + initialpos (int): initial position where the parser started working + tokenend_idx (int): last relevant character index within the last buffered value + + Returns: + int: length of index + """ + return curpos-initialpos+tokenend_idx diff --git a/pdfmajor/tokenizer/tokenizer.py b/pdfmajor/tokenizer/tokenizer.py new file mode 100644 index 0000000..e902839 --- /dev/null +++ b/pdfmajor/tokenizer/tokenizer.py @@ -0,0 +1,88 @@ + +import io +from pdfmajor.tokenizer.token import TokenKeyword +from pdfmajor.tokenizer.constants import NONSPC +from typing import Iterator +from pdfmajor.tokenizer.exceptions import TokenizerEOF +from .token_parser import PInput, parse_comment, parse_number + + +class PSTokenizer: + def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None: + self.fp = fp + self.bufsize = bufsize + self.bufpos: int = 0 + self.buf: bytes = b'' + self.buf_skipstep: int = 0 + + def iter_buffer(self) -> Iterator[PInput]: + while True: + if self.buf_skipstep > 0: + yield PInput(self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep:]) + self.buf_skipstep = 0 + self.bufpos = self.fp.tell() + self.buf = self.fp.read(self.bufsize) + if not self.buf: + raise TokenizerEOF("Unexpected EOF") + else: + yield PInput(self.bufpos, self.buf) + + def seek(self, pos: int): + self.fp.seek(pos) + + def step_back(self, steps: int = 1): + """Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize) + + Args: + steps (int, optional): numbers of steps. Defaults to 1. + """ + self.fp.seek(self.fp.tell() - steps * self.bufsize) + + def iter_tokens(self): + for bufpos, buf in self.iter_buffer(): + m = NONSPC.search(buf, 0) + if not m: + self.buf_skipstep = 0 + continue + else: + j = m.start(0) + c = buf[j : j + 1] + if c == b"%": + self.buf_skipstep = j + 1 + yield parse_comment(bufpos + j, self.iter_buffer()) + elif c == b"/": + raise "WIP" + # self._curtoken = b"" + # self._current_parse_func = self._parse_literal + # return j + 1 + elif c in b"-+" or c.isdigit(): + self.buf_skipstep = j + 1 + yield parse_number(bufpos + j, self.iter_buffer(), c) + elif c == b".": + self.buf_skipstep = j + yield parse_number(bufpos + j, self.iter_buffer()) + elif c.isalpha(): + raise "WIP" + # self._curtoken = c + # self._current_parse_func = self._parse_keyword + # return j + 1 + elif c == b"(": + raise "WIP" + # self._curtoken = b"" + # self.paren = 1 + # self._current_parse_func = self._parse_string + # return j + 1 + elif c == b"<": + raise "WIP" + # self._curtoken = b"" + # self._current_parse_func = self._parse_wopen + # return j + 1 + elif c == b">": + raise "WIP" + # self._curtoken = b"" + # self._current_parse_func = self._parse_wclose + # return j + 1 + else: + self.buf_skipstep = j + 1 + yield TokenKeyword(bufpos + j, 1, c) + diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index c7db855..0c08aa2 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -1,7 +1,11 @@ from decimal import Decimal import io -from pdfmajor.tokenizer.token import TokenComment, TokenDecimal, TokenInteger -from pdfmajor.tokenizer.token_parser import PInput, parse_comment, parse_number +from pdfmajor.tokenizer.token import TokenBoolean, TokenComment, TokenDecimal, TokenInteger, TokenLiteral +from pdfmajor.tokenizer.token_parsers.util import PInput +from pdfmajor.tokenizer.token_parsers.comment import parse_comment +from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword +from pdfmajor.tokenizer.token_parsers.number import parse_number +from pdfmajor.tokenizer.token_parsers.literal import parse_literal from unittest import TestCase def make_stream_iter(data: bytes, initpos: int = 0): @@ -62,3 +66,35 @@ class Basics(TestCase): self.assertEqual(token.size, 3) + def test_parse_bool_true(self): + iter_stream = make_stream_iter( + b"true and the american way", + 91 + ) + token = parse_keyword(91, iter_stream(33)) + self.assertIsInstance(token, TokenBoolean) + self.assertEqual(token.pos, 91) + self.assertEqual(token.value, True) + self.assertEqual(token.size, 4) + + def test_parse_bool_false(self): + iter_stream = make_stream_iter( + b"false mannn", + 33 + ) + token = parse_keyword(33, iter_stream(100)) + self.assertIsInstance(token, TokenBoolean) + self.assertEqual(token.pos, 33) + self.assertEqual(token.value, False) + self.assertEqual(token.size, 5) + + def test_parse_literals(self): + iter_stream = make_stream_iter( + b"Some_Name /foo#5f#xbaa", + 71 + ) + token = parse_literal(71, iter_stream(1)) + self.assertIsInstance(token, TokenLiteral) + self.assertEqual(token.pos, 71) + self.assertEqual(token.value, 'Some_Name') + self.assertEqual(token.size, 9)