diff --git a/.editorconfig b/.editorconfig index 5341ccb..73390a1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,9 +1,9 @@ root = true [*] -indent_style = tab +indent_style = space indent_size = 4 end_of_line = lf charset = utf-8 trim_trailing_whitespace = false -insert_final_newline = true \ No newline at end of file +insert_final_newline = true diff --git a/pdfmajor/lexer/__init__.py b/pdfmajor/lexer/__init__.py index 5c27d2a..3ccf1ca 100644 --- a/pdfmajor/lexer/__init__.py +++ b/pdfmajor/lexer/__init__.py @@ -1,9 +1,73 @@ import io +from typing import Iterator +from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket +from pdfmajor.lexer.string import parse_string +from pdfmajor.lexer.keyword import parse_keyword +from pdfmajor.lexer.number import parse_number +from pdfmajor.lexer.name import parse_name +from pdfmajor.lexer.comment import parse_comment +from pdfmajor.lexer.regex import NONSPC +from pdfmajor.lexer.token import ( + TArrayValue, + TDictValue, + Token, + TokenDictionary, + TokenKeyword, + TokenArray, +) +from pdfmajor.safebufiterator import BufferStream class PDFLexer: - def __init__(self, fp: io.BufferedIOBase) -> None: - self.fp = fp + def __init__(self, fp: io.BufferedIOBase, buffer_size: int = 4096) -> None: + self.buffer = BufferStream(fp, buffer_size=buffer_size) def seek(self, pos: int): - self.fp.seek(pos) + self.buffer.seek(pos) + + def iter_tokens(self) -> Iterator[Token]: + for bufpos, buf in self.buffer: + m = NONSPC.search(buf, 0) + if not m: + self.buf_skipstep = None + continue + else: + j = m.start(0) + next_char = buf[j : j + 1] + if next_char == b"%": + self.buffer.seek(bufpos + j + 1) + yield parse_comment(self.buffer) + elif next_char == b"/": + self.buffer.seek(bufpos + j + 1) + yield parse_name(self.buffer) + elif next_char in b"-+": + self.buffer.seek(bufpos + j + 1) + yield parse_number(self.buffer, next_char) + elif next_char.isdigit() or next_char == b".": + self.buffer.seek(bufpos + j) + yield parse_number(self.buffer) + elif next_char.isalpha(): + self.buffer.seek(bufpos + j) + yield parse_keyword(self.buffer) + elif next_char == b"(": + self.buffer.seek(bufpos + j + 1) + yield parse_string(self.buffer) + elif next_char == b"<": + self.buffer.seek(bufpos + j + 1) + yield parse_double_angled_bracket(self.buffer) + elif next_char == b">": + self.buffer.seek(bufpos + j + 1) + if buf[j : j + 2] == ">>": + self.buffer.seekd(1) + yield TokenDictionary( + bufpos + j, bufpos + j + 2, TDictValue.CLOSE + ) + elif next_char == b"[": + self.buffer.seek(bufpos + j + 1) + yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.OPEN) + elif next_char == b"]": + self.buffer.seek(bufpos + j + 1) + yield TokenArray(bufpos + j, bufpos + j + 1, TArrayValue.CLOSE) + else: + self.buffer.seek(bufpos + j + 1) + yield TokenKeyword(bufpos + j, bufpos + j + 1, next_char) diff --git a/pdfmajor/lexer/comment.py b/pdfmajor/lexer/comment.py index 03ea99d..9afa772 100644 --- a/pdfmajor/lexer/comment.py +++ b/pdfmajor/lexer/comment.py @@ -3,14 +3,14 @@ from pdfmajor.lexer.regex import EOL from pdfmajor.lexer.token import ( TokenComment, ) -from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.safebufiterator import BufferStream -def parse_comment(buffer: SafeBufferIt) -> TokenComment: +def parse_comment(buffer: BufferStream) -> TokenComment: """Parses input stream into a comment Args: - buffer (SafeBufferIt) + buffer (BufferStream) Returns: TokenComment diff --git a/pdfmajor/lexer/dict_and_hex.py b/pdfmajor/lexer/dict_and_hex.py index 807ebc3..2ab3c14 100644 --- a/pdfmajor/lexer/dict_and_hex.py +++ b/pdfmajor/lexer/dict_and_hex.py @@ -1,18 +1,17 @@ -from pdfmajor.tokenizer.token_parsers.util import PInput -from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.safebufiterator import BufferStream from pdfmajor.lexer.exceptions import LexerEOF from pdfmajor.lexer.regex import END_HEX_STRING, HEX_PAIR, SPC -from pdfmajor.lexer.token import TDictVaue, TokenDictionary, TokenHexString +from pdfmajor.lexer.token import TDictValue, TokenDictionary, TokenHexString from typing import Union def parse_double_angled_bracket( - buffer: SafeBufferIt, + buffer: BufferStream, ) -> Union[TokenHexString, TokenDictionary]: """Parses input stream into a hex string object or match a simple dictionary token Args: - buffer (SafeBufferIt) + buffer (BufferStream) Returns: Union[TokenHexString, TokenDictionary] @@ -22,16 +21,16 @@ def parse_double_angled_bracket( first_char = step.data[0:1] if first_char == b"<": buffer.seek(step.pos + 1) - return TokenDictionary(initialpos, 2, TDictVaue.OPEN) + return TokenDictionary(initialpos, 2, TDictValue.OPEN) buffer.seek(step.pos) return parse_hexstring(buffer) -def parse_hexstring(buffer: SafeBufferIt) -> TokenHexString: +def parse_hexstring(buffer: BufferStream) -> TokenHexString: """Parses the input string into hex string Args: - buffer (SafeBufferIt) + buffer (BufferStream) Returns: TokenHexString diff --git a/pdfmajor/lexer/exceptions.py b/pdfmajor/lexer/exceptions.py index 13db39a..0ea058f 100644 --- a/pdfmajor/lexer/exceptions.py +++ b/pdfmajor/lexer/exceptions.py @@ -5,7 +5,7 @@ class LexerError(PDFMajorException): pass -class LexerEOF(LexerError): +class LexerEOF(LexerError, EOFError): pass diff --git a/pdfmajor/lexer/keyword.py b/pdfmajor/lexer/keyword.py index 514524e..cf8ec60 100644 --- a/pdfmajor/lexer/keyword.py +++ b/pdfmajor/lexer/keyword.py @@ -1,15 +1,15 @@ -from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.safebufiterator import BufferStream from pdfmajor.lexer.exceptions import LexerEOF from pdfmajor.lexer.token import TokenBoolean, TokenKeyword from pdfmajor.lexer.regex import END_KEYWORD from typing import Union -def parse_keyword(buffer: SafeBufferIt) -> Union[TokenKeyword, TokenBoolean]: +def parse_keyword(buffer: BufferStream) -> Union[TokenKeyword, TokenBoolean]: """Parses input stream into a keyword or bool Args: - buffer (SafeBufferIt) + buffer (BufferStream) Returns: Union[TokenKeyword, TokenBoolean] diff --git a/pdfmajor/lexer/name.py b/pdfmajor/lexer/name.py index f9afb7a..eb91d9e 100644 --- a/pdfmajor/lexer/name.py +++ b/pdfmajor/lexer/name.py @@ -5,7 +5,7 @@ from pdfmajor.lexer.exceptions import ( LexerError, ) -from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.safebufiterator import BufferStream from pdfmajor.lexer.token import TokenName from pdfmajor.lexer.regex import HEX, END_LITERAL @@ -19,11 +19,11 @@ class LiteralParseState: hex_value: Optional[bytes] = None -def parse_name(buffer: SafeBufferIt) -> TokenName: +def parse_name(buffer: BufferStream) -> TokenName: """Parses input stream into a literal name Args: - buffer (SafeBufferIt) + buffer (BufferStream) Returns: TokenName diff --git a/pdfmajor/lexer/number.py b/pdfmajor/lexer/number.py index 341c30b..2e823f4 100644 --- a/pdfmajor/lexer/number.py +++ b/pdfmajor/lexer/number.py @@ -1,21 +1,21 @@ from decimal import Decimal -from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.safebufiterator import BufferStream from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError from pdfmajor.lexer.token import ( TokenDecimal, TokenNumber, TokenInteger, ) -from pdfmajor.tokenizer.constants import ( +from pdfmajor.lexer.regex import ( END_NUMBER, ) -def parse_number(buffer: SafeBufferIt, sign: bytes = b"+") -> TokenNumber: +def parse_number(buffer: BufferStream, sign: bytes = b"+") -> TokenNumber: """Parses input stream into a number Args: - buffer (SafeBufferIt) + buffer (BufferStream) sign (str): either a + or - Returns: diff --git a/pdfmajor/lexer/regex.py b/pdfmajor/lexer/regex.py index f00d4f5..ebf4621 100644 --- a/pdfmajor/lexer/regex.py +++ b/pdfmajor/lexer/regex.py @@ -19,3 +19,5 @@ ESC_STRING = { END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]") HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.") SPC = re.compile(br"\s") +NONSPC = re.compile(br"\S") +END_NUMBER = re.compile(br"[^0-9]") diff --git a/pdfmajor/lexer/string.py b/pdfmajor/lexer/string.py index 4215ea1..28eb8a0 100644 --- a/pdfmajor/lexer/string.py +++ b/pdfmajor/lexer/string.py @@ -3,7 +3,7 @@ from typing import Optional from pdfmajor.lexer.regex import END_STRING, ESC_STRING, OCT_STRING from pdfmajor.lexer.token import TokenString from pdfmajor.lexer.exceptions import LexerEOF, LexerError -from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.safebufiterator import BufferStream @dataclass @@ -13,11 +13,11 @@ class StringParseState: escaped_value: Optional[bytes] = None -def parse_string(buffer: SafeBufferIt) -> TokenString: +def parse_string(buffer: BufferStream) -> TokenString: """Parses input stream into a simple string object Args: - buffer (SafeBufferIt) + buffer (BufferStream) Returns: TokenString @@ -55,7 +55,7 @@ def parse_string(buffer: SafeBufferIt) -> TokenString: raise LexerEOF -def parse_string_escape(buffer: SafeBufferIt, state: StringParseState): +def parse_string_escape(buffer: BufferStream, state: StringParseState): if state.escaped_value is None: raise LexerError("state.escaped_value is None") for pos, buf in buffer: diff --git a/pdfmajor/lexer/token.py b/pdfmajor/lexer/token.py index aa88a71..2572a0c 100644 --- a/pdfmajor/lexer/token.py +++ b/pdfmajor/lexer/token.py @@ -100,12 +100,15 @@ class TokenDecimal(Token[Decimal]): pass -class TDictVaue(Enum): +TokenNumber = Union[TokenInteger, TokenDecimal] + + +class TDictValue(Enum): OPEN = "<<" CLOSE = ">>" -class TokenDictionary(Token[TDictVaue]): +class TokenDictionary(Token[TDictValue]): """Token representing the brackets of dictionary Detection of this type of token can be found in PDF 1.7 spec section 7.3.7 """ @@ -113,4 +116,14 @@ class TokenDictionary(Token[TDictVaue]): pass -TokenNumber = Union[TokenInteger, TokenDecimal] +class TArrayValue(Enum): + OPEN = "[" + CLOSE = "]" + + +class TokenArray(Token[TArrayValue]): + """Token representing the brackets of an array + Detection of this type of token can be found in PDF 1.7 spec section 7.3.6 + """ + + pass diff --git a/pdfmajor/safebufiterator.py b/pdfmajor/safebufiterator.py index a6d1686..74fc634 100644 --- a/pdfmajor/safebufiterator.py +++ b/pdfmajor/safebufiterator.py @@ -3,7 +3,7 @@ from typing import NamedTuple from pdfmajor.execptions import PDFMajorException -class EOF(PDFMajorException): +class StreamEOF(PDFMajorException, EOFError): def __init__(self) -> None: super().__init__("reached the end of the file or buffer stream!") @@ -13,7 +13,7 @@ class BufferedBytes(NamedTuple): data: bytes -class SafeBufferIt: +class BufferStream: """A safe reader for bytes that has a few nice additional utilities""" __slots__ = ["fp", "buffer_size"] @@ -50,18 +50,30 @@ class SafeBufferIt: """ return self.fp.tell() + def read(self, size: int) -> BufferedBytes: + """Reads the amount of bytes specified + + Args: + size (int): number of bytes to read + + Returns: + BufferedBytes + """ + pos = self.tell() + buf = self.fp.read(size) + return BufferedBytes(pos, buf) + def __iter__(self): if self.fp.read(1) == b"": - raise EOF + raise StreamEOF self.seekd(-1) return self def __next__(self): - pos = self.tell() - buf = self.fp.read(self.buffer_size) - if len(buf) == 0: + bbyte = self.read(self.buffer_size) + if len(bbyte.data) == 0: raise StopIteration - return BufferedBytes(pos, buf) + return bbyte def close(self): self.fp.close() diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py deleted file mode 100644 index 5c6fa22..0000000 --- a/pdfmajor/tokenizer/__init__.py +++ /dev/null @@ -1,120 +0,0 @@ -import io -from pdfmajor.tokenizer.token_parsers.dictionary_hexstring import ( - parse_double_angled_bracket, -) -from pdfmajor.tokenizer.token_parsers.string import parse_string - -from typing import Iterator, Optional - -from pdfmajor.tokenizer.constants import NONSPC -from pdfmajor.tokenizer.exceptions import TokenizerEOF -from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword -from pdfmajor.tokenizer.token_parsers.literal import parse_literal -from pdfmajor.tokenizer.token_parsers.number import parse_number -from pdfmajor.tokenizer.token_parsers.comment import parse_comment -from pdfmajor.tokenizer.token_parsers.util import PInput -from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenKeyword - - -class PSTokenizer: - def __init__(self, fp: io.BufferedIOBase, bufsize: int = 4096) -> None: - self.fp = fp - self.bufsize = bufsize - self.bufpos: int = 0 - self.buf: bytes = b"" - self.buf_skipstep: Optional[int] = None - - def inc_buf_skipstep(self, value: int): - if self.buf_skipstep is None: - self.buf_skipstep = value - else: - self.buf_skipstep += value - - def iter_buffer(self) -> Iterator[PInput]: - while True: - if (self.buf_skipstep is not None) and (self.buf_skipstep < len(self.buf)): - yield PInput( - self.bufpos + self.buf_skipstep, self.buf[self.buf_skipstep :] - ) - else: - self.buf_skipstep = None - self.bufpos = self.fp.tell() - self.buf = self.fp.read(self.bufsize) - if not self.buf: - raise TokenizerEOF("Unexpected EOF") - else: - yield PInput(self.bufpos, self.buf) - - def seek(self, pos: int): - self.fp.seek(pos) - - def step_back(self, steps: int = 1): - """Changes the offset of the stream some steps back (each step is equal to 1 self.bufsize) - - Args: - steps (int, optional): numbers of steps. Defaults to 1. - """ - self.fp.seek(self.fp.tell() - steps * self.bufsize) - - def iter_tokens(self) -> Iterator[Token]: - for cur in self.iter_buffer(): - bufpos, buf = cur - m = NONSPC.search(buf, 0) - if not m: - self.buf_skipstep = None - continue - else: - j = m.start(0) - c = buf[j : j + 1] - if c == b"%": - self.inc_buf_skipstep(j + 1) - yield self._check_token( - cur, parse_comment(bufpos + j, self.iter_buffer()) - ) - elif c == b"/": - self.inc_buf_skipstep(j + 1) - yield self._check_token( - cur, parse_literal(bufpos + j, self.iter_buffer()) - ) - elif c in b"-+": - self.inc_buf_skipstep(j + 1) - yield self._check_token( - cur, parse_number(bufpos + j, self.iter_buffer(), c) - ) - elif c.isdigit(): - self.inc_buf_skipstep(j) - yield self._check_token( - cur, parse_number(bufpos + j, self.iter_buffer()) - ) - elif c == b".": - self.inc_buf_skipstep(j) - yield self._check_token( - cur, parse_number(bufpos + j, self.iter_buffer()) - ) - elif c.isalpha(): - self.inc_buf_skipstep(j) - yield self._check_token( - cur, parse_keyword(bufpos + j, self.iter_buffer()) - ) - elif c == b"(": - self.inc_buf_skipstep(j + 1) - yield self._check_token( - cur, parse_string(bufpos + j, self.iter_buffer()) - ) - elif c == b"<": - self.inc_buf_skipstep(j) - yield self._check_token( - cur, parse_double_angled_bracket(bufpos + j, self.iter_buffer()) - ) - elif c == b">": - self.inc_buf_skipstep(j) - if buf[j : j + 2] == ">>": - self.inc_buf_skipstep(1) - yield TokenDictionary(bufpos + j, 2, TDictVaue.CLOSE) - else: - self.inc_buf_skipstep(j + 1) - yield self._check_token(cur, TokenKeyword(bufpos + j, 1, c)) - - def _check_token(self, cur: PInput, token: Token) -> Token: - self.inc_buf_skipstep(token.end_loc - cur.pos) - return token diff --git a/pdfmajor/tokenizer/constants.py b/pdfmajor/tokenizer/constants.py deleted file mode 100644 index 1b17b05..0000000 --- a/pdfmajor/tokenizer/constants.py +++ /dev/null @@ -1,37 +0,0 @@ -import re -from .types import PSLiteral, PSSymbolTable, PSKeyword - -PSLiteralTable = PSSymbolTable(PSLiteral) -PSKeywordTable = PSSymbolTable(PSKeyword) -LIT = PSLiteralTable.intern -KWD = PSKeywordTable.intern -KEYWORD_PROC_BEGIN = KWD(b"{") -KEYWORD_PROC_END = KWD(b"}") -KEYWORD_ARRAY_BEGIN = KWD(b"[") -KEYWORD_ARRAY_END = KWD(b"]") -KEYWORD_DICT_BEGIN = KWD(b"<<") -KEYWORD_DICT_END = KWD(b">>") - -## PSBaseParser -## -EOL = re.compile(br"[\r\n]") -SPC = re.compile(br"\s") -NONSPC = re.compile(br"\S") -HEX = re.compile(br"[0-9a-fA-F]") -END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]") -END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]") -HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.") -END_NUMBER = re.compile(br"[^0-9]") -END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]") -END_STRING = re.compile(br"[()\134]") -OCT_STRING = re.compile(br"[0-7]") -ESC_STRING = { - b"b": 8, - b"t": 9, - b"n": 10, - b"f": 12, - b"r": 13, - b"(": 40, - b")": 41, - b"\\": 92, -} diff --git a/pdfmajor/tokenizer/exceptions.py b/pdfmajor/tokenizer/exceptions.py deleted file mode 100644 index 561d281..0000000 --- a/pdfmajor/tokenizer/exceptions.py +++ /dev/null @@ -1,23 +0,0 @@ -from pdfmajor.execptions import PDFMajorException - - -class TokenizerError(PDFMajorException): - pass - - -class TokenizerEOF(TokenizerError): - pass - - -class InvalidToken(TokenizerError): - def __init__(self, pos: int, token: bytes) -> None: - super().__init__( - "Invalid Token Found at {pos}, of value={v}".format(pos=pos, v=str(token)) - ) - - -class InvalidHexToken(TokenizerError): - def __init__(self, pos: int, token: bytes) -> None: - super().__init__( - "Invalid Hex Found at {pos}, of value={v}".format(pos=pos, v=str(token)) - ) diff --git a/pdfmajor/tokenizer/old.py b/pdfmajor/tokenizer/old.py deleted file mode 100644 index 71da1ac..0000000 --- a/pdfmajor/tokenizer/old.py +++ /dev/null @@ -1,341 +0,0 @@ -from typing import List, Tuple, Union -from io import TextIOWrapper - -from pdfmajor.utils import int2byte -from pdfmajor.execptions import PSEOF - -from .types import PSLiteral -from .constants import OCT_STRING, ESC_STRING, KEYWORD_DICT_BEGIN, KEYWORD_DICT_END -from .constants import END_NUMBER, END_STRING, END_KEYWORD, END_HEX_STRING -from .constants import SPC, HEX_PAIR -from .constants import EOL, NONSPC, KWD, END_LITERAL, LIT, HEX - - -class PSTokenizer(object): - """Tokenizatizes bytes stream to PS-Tokens""" - - BUFSIZ = 4096 - - def __init__(self, fp: TextIOWrapper): - self.fp = fp - self.seek(0) - return - - def __repr__(self): - return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos) - - def flush(self): - return - - def close(self): - self.flush() - return - - def tell(self): - return self.bufpos + self.charpos - - def poll(self, pos=None, n=80): - pos0 = self.fp.tell() - if not pos: - pos = self.bufpos + self.charpos - self.fp.seek(pos) - self.fp.read(n) - self.fp.seek(pos0) - return - - def seek(self, pos: int): - """Seeks the parser to the given position.""" - self.fp.seek(pos) - # reset the status for nextline() - self.bufpos: int = pos - self.buf: bytes = b"" - self.charpos: int = 0 - # reset the status for nexttoken() - self._current_parse_func = self._parse_main - self._curtoken: bytes = b"" - self._curtokenpos: int = 0 - self._tokens: List[ - Tuple[int, Union[int, float, bool, str, bytes, PSLiteral]] - ] = [] - return - - def fillbuf(self): - if self.charpos < len(self.buf): - return - # fetch next chunk. - self.bufpos = self.fp.tell() - self.buf = self.fp.read(self.BUFSIZ) - if not self.buf: - raise PSEOF("Unexpected EOF") - self.charpos = 0 - return - - def nextline(self): - """Fetches a next line that ends either with \\r or \\n.""" - linebuf = b"" - linepos = self.bufpos + self.charpos - eol = False - while 1: - self.fillbuf() - if eol: - c = self.buf[self.charpos : self.charpos + 1] - # handle b'\r\n' - if c == b"\n": - linebuf += c - self.charpos += 1 - break - m = EOL.search(self.buf, self.charpos) - if m: - linebuf += self.buf[self.charpos : m.end(0)] - self.charpos = m.end(0) - if linebuf[-1:] == b"\r": - eol = True - else: - break - else: - linebuf += self.buf[self.charpos :] - self.charpos = len(self.buf) - return (linepos, linebuf) - - def revreadlines(self): - """Fetches a next line backword. - - This is used to locate the trailers at the end of a file. - """ - self.fp.seek(0, 2) - pos = self.fp.tell() - buf = b"" - while 0 < pos: - prevpos = pos - pos = max(0, pos - self.BUFSIZ) - self.fp.seek(pos) - s = self.fp.read(prevpos - pos) - if not s: - break - while 1: - n = max(s.rfind(b"\r"), s.rfind(b"\n")) - if n == -1: - buf = s + buf - break - yield s[n:] + buf - s = s[:n] - buf = b"" - return - - def _parse_main(self, s, i): - m = NONSPC.search(s, i) - if not m: - return len(s) - j = m.start(0) - c = s[j : j + 1] - self._curtokenpos = self.bufpos + j - if c == b"%": - self._curtoken = b"%" - self._current_parse_func = self._parse_comment - return j + 1 - elif c == b"/": - self._curtoken = b"" - self._current_parse_func = self._parse_literal - return j + 1 - elif c in b"-+" or c.isdigit(): - self._curtoken = c - self._current_parse_func = self._parse_number - return j + 1 - elif c == b".": - self._curtoken = c - self._current_parse_func = self._parse_float - return j + 1 - elif c.isalpha(): - self._curtoken = c - self._current_parse_func = self._parse_keyword - return j + 1 - elif c == b"(": - self._curtoken = b"" - self.paren = 1 - self._current_parse_func = self._parse_string - return j + 1 - elif c == b"<": - self._curtoken = b"" - self._current_parse_func = self._parse_wopen - return j + 1 - elif c == b">": - self._curtoken = b"" - self._current_parse_func = self._parse_wclose - return j + 1 - else: - self._add_token(KWD(c)) - return j + 1 - - def _add_token(self, obj): - self._tokens.append((self._curtokenpos, obj)) - return - - def _parse_comment(self, s, i): - m = EOL.search(s, i) - if not m: - self._curtoken += s[i:] - return len(s) - j = m.start(0) - self._curtoken += s[i:j] - self._current_parse_func = self._parse_main - # We ignore comments. - # self._tokens.append(self._curtoken) - return j - - def _parse_literal(self, s, i): - m = END_LITERAL.search(s, i) - if not m: - self._curtoken += s[i:] - return len(s) - j = m.start(0) - self._curtoken += s[i:j] - c = s[j : j + 1] - if c == b"#": - self.hex = b"" - self._current_parse_func = self._parse_literal_hex - return j + 1 - try: - self._curtoken = str(self._curtoken, "utf-8") - except: - pass - self._add_token(LIT(self._curtoken)) - self._current_parse_func = self._parse_main - return j - - def _parse_literal_hex(self, s, i): - c = s[i : i + 1] - if HEX.match(c) and len(self.hex) < 2: - self.hex += c - return i + 1 - if self.hex: - self._curtoken += int2byte(int(self.hex, 16)) - self._current_parse_func = self._parse_literal - return i - - def _parse_number(self, s, i): - m = END_NUMBER.search(s, i) - if not m: - self._curtoken += s[i:] - return len(s) - j = m.start(0) - self._curtoken += s[i:j] - c = s[j : j + 1] - if c == b".": - self._curtoken += c - self._current_parse_func = self._parse_float - return j + 1 - try: - self._add_token(int(self._curtoken)) - except ValueError: - pass - self._current_parse_func = self._parse_main - return j - - def _parse_float(self, s, i): - m = END_NUMBER.search(s, i) - if not m: - self._curtoken += s[i:] - return len(s) - j = m.start(0) - self._curtoken += s[i:j] - try: - self._add_token(float(self._curtoken)) - except ValueError: - pass - self._current_parse_func = self._parse_main - return j - - def _parse_keyword(self, s, i): - m = END_KEYWORD.search(s, i) - if not m: - self._curtoken += s[i:] - return len(s) - j = m.start(0) - self._curtoken += s[i:j] - if self._curtoken == b"true": - token = True - elif self._curtoken == b"false": - token = False - else: - token = KWD(self._curtoken) - self._add_token(token) - self._current_parse_func = self._parse_main - return j - - def _parse_string(self, s, i): - m = END_STRING.search(s, i) - if not m: - self._curtoken += s[i:] - return len(s) - j = m.start(0) - self._curtoken += s[i:j] - c = s[j : j + 1] - if c == b"\\": - self.oct = b"" - self._current_parse_func = self._parse_string_1 - return j + 1 - if c == b"(": - self.paren += 1 - self._curtoken += c - return j + 1 - if c == b")": - self.paren -= 1 - if self.paren: # WTF, they said balanced parens need no special treatment. - self._curtoken += c - return j + 1 - self._add_token(self._curtoken) - self._current_parse_func = self._parse_main - return j + 1 - - def _parse_string_1(self, s, i): - c = s[i : i + 1] - if OCT_STRING.match(c) and len(self.oct) < 3: - self.oct += c - return i + 1 - if self.oct: - self._curtoken += int2byte(int(self.oct, 8)) - self._current_parse_func = self._parse_string - return i - if c in ESC_STRING: - self._curtoken += int2byte(ESC_STRING[c]) - self._current_parse_func = self._parse_string - return i + 1 - - def _parse_wopen(self, s, i): - c = s[i : i + 1] - if c == b"<": - self._add_token(KEYWORD_DICT_BEGIN) - self._current_parse_func = self._parse_main - i += 1 - else: - self._current_parse_func = self._parse_hexstring - return i - - def _parse_wclose(self, s, i): - c = s[i : i + 1] - if c == b">": - self._add_token(KEYWORD_DICT_END) - i += 1 - self._current_parse_func = self._parse_main - return i - - def _parse_hexstring(self, s, i): - m = END_HEX_STRING.search(s, i) - if not m: - self._curtoken += s[i:] - return len(s) - j = m.start(0) - self._curtoken += s[i:j] - token = HEX_PAIR.sub( - lambda m: int2byte(int(m.group(0), 16)), SPC.sub(b"", self._curtoken) - ) - self._add_token(token) - self._current_parse_func = self._parse_main - return j - - def nexttoken(self): - while not self._tokens: - self.fillbuf() - self.charpos = self._current_parse_func(self.buf, self.charpos) - token = self._tokens.pop(0) - return token diff --git a/pdfmajor/tokenizer/token.py b/pdfmajor/tokenizer/token.py deleted file mode 100644 index d311017..0000000 --- a/pdfmajor/tokenizer/token.py +++ /dev/null @@ -1,112 +0,0 @@ -from enum import Enum -from typing import Any, Generic, TypeVar, Union -from decimal import Decimal - -T = TypeVar("T") - - -class Token(Generic[T]): - __slots__ = ["pos", "size", "value"] - - def __init__(self, start_loc: int, end_loc: int, value: T) -> None: - self.start_loc = start_loc - self.end_loc = end_loc - self.value = value - - def __repr__(self) -> str: - return "{cls}(start={start_loc}, end={end_loc}, value='{value}')".format( - cls=self.__class__.__name__, - start_loc=self.start_loc, - value=self.value, - end_loc=self.end_loc, - ) - - def __eq__(self, other: Any): - if isinstance(other, self.__class__): - return ( - (other.value == self.value) - and (other.start_loc == self.start_loc) - and (other.end_loc == self.end_loc) - ) - else: - return False - - -class TokenComment(Token[bytes]): - """Token representing PDF comments - Detection of this type of token can be found in PDF 1.7 spec section 7.2.3 - """ - - pass - - -class TokenKeyword(Token[bytes]): - """Token representing PDF comments - Detection of this type of token can be found in PDF 1.7 spec - """ - - pass - - -class TokenLiteral(Token[str]): - """Token representing literal name tokens - Detection of this type of token can be found in PDF 1.7 spec section 7.3.5 - """ - - pass - - -class TokenBoolean(Token[bool]): - """Token representing PDF booleans - Detection of this type of token can be found in PDF 1.7 spec section 7.3.2 - """ - - pass - - -class TokenString(Token[str]): - """Token representing PDF literal string - Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.2 - """ - - pass - - -class TokenHexString(Token[bytes]): - """Token representing PDF hex strings - Detection of this type of token can be found in PDF 1.7 spec section 7.3.4.3 - """ - - pass - - -class TokenInteger(Token[int]): - """Token representing PDF integers - Detection of this type of token can be found in PDF 1.7 - """ - - pass - - -class TokenDecimal(Token[Decimal]): - """Token representing PDF decimals - Detection of this type of token can be found in PDF 1.7 - """ - - pass - - -class TDictVaue(Enum): - OPEN = "<<" - CLOSE = ">>" - - -class TokenDictionary(Token[TDictVaue]): - """Token representing the brackets of dictionary - Detection of this type of token can be found in PDF 1.7 spec section 7.3.7 - """ - - pass - - -TokenNumber = Union[TokenInteger, TokenDecimal] diff --git a/pdfmajor/tokenizer/token_parsers/__init__.py b/pdfmajor/tokenizer/token_parsers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pdfmajor/tokenizer/token_parsers/comment.py b/pdfmajor/tokenizer/token_parsers/comment.py deleted file mode 100644 index 172e0e1..0000000 --- a/pdfmajor/tokenizer/token_parsers/comment.py +++ /dev/null @@ -1,31 +0,0 @@ -from pdfmajor.tokenizer.exceptions import TokenizerEOF -from pdfmajor.tokenizer.token import ( - TokenComment, -) -from pdfmajor.tokenizer.constants import ( - EOL, -) -from typing import Iterator -from .util import cmp_tsize, PInput - - -def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment: - """Parses input stream into a comment - - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - - Returns: - TokenComment - """ - curtoken = b"" - for curpos, s in inp: - m = EOL.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - return TokenComment(initialpos, curpos + j, curtoken) - raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/dictionary_hexstring.py b/pdfmajor/tokenizer/token_parsers/dictionary_hexstring.py deleted file mode 100644 index 3373f52..0000000 --- a/pdfmajor/tokenizer/token_parsers/dictionary_hexstring.py +++ /dev/null @@ -1,46 +0,0 @@ -from pdfmajor.tokenizer.exceptions import TokenizerError -from pdfmajor.tokenizer.constants import END_HEX_STRING, HEX_PAIR, SPC -from pdfmajor.tokenizer.token import TDictVaue, Token, TokenDictionary, TokenHexString -from pdfmajor.tokenizer.token_parsers.util import PInput -from pdfmajor.utils import int2byte -from typing import Iterator, Union - - -def parse_double_angled_bracket(initialpos: int, inp: Iterator[PInput]) -> Token: - """Parses input stream into a simple string object - - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - - Returns: - Union[TokenHexString, TokenDictionary] - """ - step = next(inp) - first_char = step.buf[0:2] - if first_char == b"<<": - return TokenDictionary(initialpos, 2, TDictVaue.OPEN) - curtoken = parse_hexstring(initialpos, PInput(step.pos + 1, step.buf[1:]), b"") - if isinstance(curtoken, TokenHexString): - return curtoken - else: - for step in inp: - curtoken = parse_hexstring(initialpos, step, curtoken) - if isinstance(curtoken, TokenHexString): - return curtoken - raise TokenizerError("parse_double_angled_bracket ended") - - -def parse_hexstring( - initialpos: int, inp: PInput, curtoken: bytes -) -> Union[bytes, TokenHexString]: - m = END_HEX_STRING.search(inp.buf, 0) - if not m: - curtoken += inp.buf - return curtoken - j = m.start(0) - curtoken += inp.buf[:j] - token = HEX_PAIR.sub( - lambda x: int2byte(int(x.group(0), 16)), SPC.sub(b"", curtoken) - ) - return TokenHexString(initialpos, inp.pos + j + 1, token) diff --git a/pdfmajor/tokenizer/token_parsers/keyword.py b/pdfmajor/tokenizer/token_parsers/keyword.py deleted file mode 100644 index e79bf9f..0000000 --- a/pdfmajor/tokenizer/token_parsers/keyword.py +++ /dev/null @@ -1,34 +0,0 @@ -from pdfmajor.tokenizer.exceptions import TokenizerEOF -from pdfmajor.tokenizer.token import TokenBoolean, TokenKeyword -from pdfmajor.tokenizer.constants import END_KEYWORD -from typing import Iterator, Union -from .util import PInput - - -def parse_keyword( - initialpos: int, inp: Iterator[PInput] -) -> Union[TokenKeyword, TokenBoolean]: - """Parses input stream into a keyword or bool - - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - - Returns: - Union[TokenKeyword, TokenBoolean] - """ - curtoken = b"" - for curpos, s in inp: - m = END_KEYWORD.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - if curtoken == b"true": - return TokenBoolean(initialpos, curpos + j, True) - elif curtoken == b"false": - return TokenBoolean(initialpos, curpos + j, False) - else: - return TokenKeyword(initialpos, curpos + j, curtoken) - raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/literal.py b/pdfmajor/tokenizer/token_parsers/literal.py deleted file mode 100644 index 83ef0ac..0000000 --- a/pdfmajor/tokenizer/token_parsers/literal.py +++ /dev/null @@ -1,67 +0,0 @@ -from dataclasses import dataclass -from pdfmajor.tokenizer.exceptions import ( - InvalidHexToken, - TokenizerEOF, -) - -from pdfmajor.tokenizer.token_parsers.util import SafeBufferIt -from pdfmajor.tokenizer.token_parsers.util import PInput -from pdfmajor.tokenizer.token import TokenLiteral -from pdfmajor.utils import int2byte -from ..constants import HEX, END_LITERAL - -from typing import Iterator, NamedTuple, Optional, Union - - -@dataclass -class LiteralParseState: - curtoken: bytes - hex_value: Optional[bytes] = None - - -def parse_literal(initialpos: int, inp: Iterator[PInput]) -> TokenLiteral: - """Parses input stream into a literal - - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - - Returns: - TokenLiteral - """ - state = LiteralParseState(b"") - for curpos, buf in inp: - it = SafeBufferIt(buf) - for subbuf in it.into_iter(): - if state.hex_value is not None: - for ci in range(len(subbuf)): - c = subbuf[ci : ci + 1] - if HEX.match(c) and len(state.hex_value) < 2: - state.hex_value += c - it.skip += 1 - if len(state.hex_value) >= 2: - state.curtoken += int2byte(int(state.hex_value, 16)) - state.hex_value = None - break - else: - raise InvalidHexToken(curpos + ci, c) - else: - m = END_LITERAL.search(subbuf, 0) - if not m: - state.curtoken += subbuf - it.skip = len(buf) - else: - j: int = m.start(0) - state.curtoken += subbuf[:j] - c = subbuf[j : j + 1] - if c == b"#": - state.hex_value = b"" - it.skip = j + 1 - continue - else: - return TokenLiteral( - initialpos, - curpos + it.skip + j, - state.curtoken.decode(), - ) - raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/number.py b/pdfmajor/tokenizer/token_parsers/number.py deleted file mode 100644 index f390b09..0000000 --- a/pdfmajor/tokenizer/token_parsers/number.py +++ /dev/null @@ -1,68 +0,0 @@ -from decimal import Decimal -from pdfmajor.tokenizer.exceptions import InvalidToken, TokenizerEOF, TokenizerError -from pdfmajor.tokenizer.token import ( - TokenDecimal, - TokenNumber, - TokenInteger, -) -from pdfmajor.tokenizer.constants import ( - END_NUMBER, -) -from typing import Iterator -from .util import PInput - - -def parse_number( - initialpos: int, inp: Iterator[PInput], sign: bytes = b"+" -) -> TokenNumber: - """Parses input stream into a number - - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - sign (str): either a + or - - - Returns: - TokenNumber - """ - if sign not in [b"+", b"-"]: - raise TokenizerError("Number was provided an invalid sign {!r}".format(sign)) - curtoken = b"" + sign - is_dec = False - for curpos, s in inp: - m = END_NUMBER.search(s, 0) - if not m: - curtoken += s - else: - j = m.start(0) - curtoken += s[:j] - c = s[j : j + 1] - if (c == b".") and not is_dec: - is_dec = True - curtoken += c - s = s[j + 1 :] - m = END_NUMBER.search(s, 0) - if not m: - curtoken += s - continue - else: - nj = m.start(0) - curtoken += s[:nj] - j += nj - return TokenDecimal( - initialpos, - curpos + j + 1, - Decimal(curtoken.decode()), - ) - try: - if is_dec: - return TokenDecimal( - initialpos, - curpos + j, - Decimal(curtoken.decode()), - ) - else: - return TokenInteger(initialpos, curpos + j, int(curtoken)) - except ValueError: - raise InvalidToken(initialpos, curtoken) - raise TokenizerEOF diff --git a/pdfmajor/tokenizer/token_parsers/string.py b/pdfmajor/tokenizer/token_parsers/string.py deleted file mode 100644 index f8f50ed..0000000 --- a/pdfmajor/tokenizer/token_parsers/string.py +++ /dev/null @@ -1,92 +0,0 @@ -from pdfmajor.tokenizer.exceptions import TokenizerError -import re -from dataclasses import dataclass -from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt -from pdfmajor.tokenizer.token import ( - TokenString, -) -from pdfmajor.tokenizer.constants import ( - END_STRING, - ESC_STRING, -) -from typing import Iterator, Optional - -OCT_STRING = re.compile(br"^[0-7]{1,3}$") - - -@dataclass -class StringParseState: - curtoken: bytes - paren: int - escaped_value: Optional[bytes] = None - - -def parse_string(initialpos: int, inp: Iterator[PInput]) -> TokenString: - """Parses input stream into a simple string object - - Args: - initialpos (int): initial position where we started - inp (Iterator[PInput]) - - Returns: - TokenString - """ - state = StringParseState(b"", 1) - for curpos, buf in inp: - it = SafeBufferIt(buf) - for s in it.into_iter(): - if state.escaped_value is not None: - parse_string_escape(it, state) - else: - m = END_STRING.search(s, 0) - if not m: - state.curtoken += s - it.skip += len(s) - else: - j = m.start(0) - state.curtoken += s[:j] - next_char = s[j : j + 1] - if next_char == b"\\": - state.escaped_value = b"" - it.skip += j + 1 - elif next_char == b"(": - state.paren += 1 - state.curtoken += next_char - it.skip += j + 1 - elif next_char == b")": - state.paren -= 1 - if state.paren: - state.curtoken += next_char - it.skip += j + 1 - else: - return TokenString( - initialpos, - curpos + it.skip + j + 1, - state.curtoken.decode(), - ) - raise TokenizerError("end of parse_string") - - -def parse_string_escape(it: SafeBufferIt, state: StringParseState): - if state.escaped_value is None: - raise TokenizerError("state.escaped_value is None") - for s in it.into_iter(): - for ci in range(len(s)): - c = s[ci : ci + 1] - if len(state.escaped_value) < 3: - it.skip += 1 - state.escaped_value += c - if len(state.escaped_value) == 1: - if (ci < 1) and (c in ESC_STRING): - state.curtoken += bytes([ESC_STRING[c]]) - state.escaped_value = None - return - elif OCT_STRING.match(state.escaped_value): - if len(state.escaped_value) == 3: - state.curtoken += bytes([int(state.escaped_value, 8)]) - state.escaped_value = None - return - else: - it.skip -= 1 - state.escaped_value = None - return diff --git a/pdfmajor/tokenizer/token_parsers/util.py b/pdfmajor/tokenizer/token_parsers/util.py deleted file mode 100644 index 400ab67..0000000 --- a/pdfmajor/tokenizer/token_parsers/util.py +++ /dev/null @@ -1,48 +0,0 @@ -from pdfmajor.tokenizer.exceptions import TokenizerEOF -from pdfmajor.tokenizer.token import Token -from typing import Callable, Iterator, NamedTuple - - -class PInput(NamedTuple): - pos: int - buf: bytes - - -ParserFunc = Callable[[int, Iterator[PInput]], Token] - - -def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int: - """Computes the size of the token based on its positional data - - Args: - curpos (int): current position of the buffer - initialpos (int): initial position where the parser started working - tokenend_idx (int): last relevant character index within the last buffered value - - Returns: - int: length of index - """ - return curpos - initialpos + tokenend_idx - - -class SafeBufferIt: - """A safe while-loop to use on a buffer - this class will make sure that there is no risk of an infinite loop and that the max number of loops is reasonable - - Raises: - TokenizerEOF: when we reach a maximum iteration we raise an 'end of file' error - """ - - __slots__ = ["skip", "buffer"] - - def __init__(self, buffer: bytes, skip: int = 0) -> None: - self.skip = skip - self.buffer = buffer - - def into_iter(self) -> Iterator[bytes]: - for i in range(2 * len(self.buffer) + 1): - if i > 2 * len(self.buffer): - raise TokenizerEOF("Max Iteration Reached!") - if self.skip >= len(self.buffer): - break - yield self.buffer[self.skip :] diff --git a/pdfmajor/tokenizer/types.py b/pdfmajor/tokenizer/types.py deleted file mode 100644 index 296a191..0000000 --- a/pdfmajor/tokenizer/types.py +++ /dev/null @@ -1,80 +0,0 @@ -## Basic PostScript Types -## -from typing import Union - -## PSObject -## -class PSObject(object): - - """Base class for all PS or PDF-related data types.""" - - pass - - -## PSLiteral -## -class PSLiteral(PSObject): - - """A class that represents a PostScript literal. - - Postscript literals are used as identifiers, such as - variable names, property names and dictionary keys. - Literals are case sensitive and denoted by a preceding - slash sign (e.g. "/Name") - - Note: Do not create an instance of PSLiteral directly. - Always use PSLiteralTable.intern(). - """ - - def __init__(self, name): - self.name = name - - def __repr__(self): - name = self.name - return "/%r" % name - - -## PSKeyword -## -class PSKeyword(PSObject): - - """A class that represents a PostScript keyword. - - PostScript keywords are a dozen of predefined words. - Commands and directives in PostScript are expressed by keywords. - They are also used to denote the content boundaries. - - Note: Do not create an instance of PSKeyword directly. - Always use PSKeywordTable.intern(). - """ - - def __init__(self, name): - self.name = name - return - - def __repr__(self): - name = self.name - return "/%r" % name - - -## PSSymbolTable -## -class PSSymbolTable(object): - - """A utility class for storing PSLiteral/PSKeyword objects. - - Interned objects can be checked its identity with "is" operator. - """ - - def __init__(self, klass): - self.dict = {} - self.klass = klass - return - - def intern(self, name): - if name in self.dict: - lit = self.dict[name] - else: - lit = self.klass(name) - self.dict[name] = lit - return lit diff --git a/tests/test_lexer.py b/tests/test_lexer.py index e36589d..a9b9326 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -1,21 +1,27 @@ -from decimal import Decimal import io + +from decimal import Decimal +from typing import List +from unittest import TestCase + +from pdfmajor.lexer import PDFLexer +from pdfmajor.lexer.exceptions import LexerError from pdfmajor.lexer.number import parse_number from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring from pdfmajor.lexer.string import parse_string -from unittest import TestCase - from pdfmajor.lexer.name import parse_name from pdfmajor.lexer.keyword import parse_keyword -from pdfmajor.safebufiterator import SafeBufferIt +from pdfmajor.safebufiterator import BufferStream from pdfmajor.lexer.token import ( - TDictVaue, + TDictValue, + Token, TokenBoolean, TokenComment, TokenDecimal, TokenDictionary, TokenHexString, TokenInteger, + TokenKeyword, TokenName, TokenString, ) @@ -23,7 +29,7 @@ from pdfmajor.lexer.comment import parse_comment def make_stream_iter(data: bytes, buf_size: int): - return SafeBufferIt(io.BytesIO(data), buf_size) + return BufferStream(io.BytesIO(data), buf_size) class Basics(TestCase): @@ -156,7 +162,7 @@ class Basics(TestCase): token = parse_double_angled_bracket(buffer) self.assertIsInstance(token, TokenDictionary) self.assertEqual(token.start_loc, -1) - self.assertEqual(token.value, TDictVaue.OPEN) + self.assertEqual(token.value, TDictValue.OPEN) self.assertEqual(token.end_loc, 2) def test_parse_dd_to_hex(self): @@ -190,3 +196,135 @@ class Basics(TestCase): self.assertEqual(token.start_loc, -1) self.assertEqual(token.value, 130) self.assertEqual(token.end_loc, 3) + + +class Lexer(TestCase): + def run_test(self, raw: bytes, expected: List[Token]): + tokenizer = PDFLexer(io.BytesIO(raw)) + for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())): + self.assertEqual(token, etoken, f"Failed at example {i}") + try: + next_token = next(tokenizer.iter_tokens()) + self.assertIsNone(next_token) + except LexerError as e: + self.assertIsNotNone(e) + + def test_case1(self): + self.run_test( + raw=br"""%!PS +begin end + " @ # +/a/BCD /Some_Name /foo#5f#6eaa +0 +1 -2 .5 1.234 +(abc) () (abc ( def ) ghi) +(def\040\0\0404ghi) (bach\\slask) (foo\nbaa) +(this % is not a comment.) +(foo +baa) +(foo\ +baa) +<> <20> < 40 4020 > + +func/a/b{(c)do*}def +[ 1 (z) ! ] +<< /foo (bar) >>""", + expected=[ + TokenComment(start_loc=0, end_loc=4, value=b"!PS"), + TokenKeyword(start_loc=5, end_loc=5, value=b"begin"), + TokenKeyword(start_loc=11, end_loc=3, value=b"end"), + TokenKeyword(start_loc=16, end_loc=1, value=b'"'), + TokenKeyword(start_loc=19, end_loc=1, value=b"@"), + TokenKeyword(start_loc=21, end_loc=1, value=b"#"), + TokenName(start_loc=24, end_loc=1, value="a"), + TokenName(start_loc=26, end_loc=3, value="BCD"), + TokenName(start_loc=31, end_loc=9, value="Some_Name"), + TokenName(start_loc=42, end_loc=11, value="foo_naa"), + TokenInteger(start_loc=54, end_loc=1, value=0), + TokenInteger(start_loc=56, end_loc=2, value=1), + TokenInteger(start_loc=59, end_loc=2, value=-2), + TokenDecimal(start_loc=62, end_loc=2, value=Decimal("0.5")), + TokenDecimal(start_loc=65, end_loc=5, value=Decimal("1.234")), + TokenString(start_loc=71, end_loc=5, value="abc"), + TokenString(start_loc=77, end_loc=2, value=""), + TokenString(start_loc=80, end_loc=17, value="abc ( def ) ghi"), + TokenString(start_loc=98, end_loc=19, value="def 4ghi"), + TokenString(start_loc=118, end_loc=13, value=r"bach\slask"), + TokenString(start_loc=132, end_loc=10, value="foo\nbaa"), + TokenString( + start_loc=143, end_loc=26, value="this % is not a comment." + ), + TokenString(start_loc=170, end_loc=9, value="foo\nbaa"), + TokenString(start_loc=180, end_loc=10, value="foobaa"), + TokenHexString(start_loc=191, end_loc=2, value=b""), + TokenHexString(start_loc=194, end_loc=4, value=b" "), + TokenHexString(start_loc=199, end_loc=11, value=b"@@ "), + TokenHexString( + start_loc=211, end_loc=14, value=b"\xab\xcd\x00\x124\x05" + ), + TokenKeyword(start_loc=226, end_loc=4, value=b"func"), + TokenName(start_loc=231, end_loc=1, value="a"), + TokenName(start_loc=233, end_loc=1, value="b"), + TokenKeyword(start_loc=234, end_loc=1, value=b"{"), + ], + ) + + def test_case2(self): + self.run_test( + raw=br"""10.32 +0 +1 -2 .5 1.234 +-1.234 990 """, + expected=[ + TokenDecimal(start_loc=0, end_loc=5, value=Decimal("10.32")), + TokenInteger(start_loc=6, end_loc=1, value=0), + TokenInteger(start_loc=8, end_loc=2, value=1), + TokenInteger(start_loc=11, end_loc=2, value=-2), + TokenDecimal(start_loc=14, end_loc=2, value=Decimal("0.5")), + TokenDecimal(start_loc=17, end_loc=5, value=Decimal("1.234")), + TokenDecimal(start_loc=23, end_loc=6, value=Decimal("-1.234")), + TokenInteger(start_loc=30, end_loc=3, value=990), + ], + ) + + def test_case3(self): + self.run_test( + raw=br"""/b{(c)do*}def""", + expected=[ + TokenName(start_loc=0, end_loc=2, value="b"), + TokenKeyword(start_loc=2, end_loc=3, value=b"{"), + TokenString(start_loc=4, end_loc=5, value="c"), + ], + ) + + def test_case4(self): + self.run_test( + raw=br"""%!PS +% This is some secret sauce +begin end + " @ # +/a/BCD /Some_Name /foo#5f#6eaa /bar#6a +/need /oh1oh +func/a/b +""", + expected=[ + TokenComment(start_loc=0, end_loc=4, value=b"!PS"), + TokenComment( + start_loc=5, end_loc=27, value=b" This is some secret sauce" + ), + TokenKeyword(start_loc=33, end_loc=5, value=b"begin"), + TokenKeyword(start_loc=39, end_loc=3, value=b"end"), + TokenKeyword(start_loc=44, end_loc=1, value=b'"'), + TokenKeyword(start_loc=47, end_loc=1, value=b"@"), + TokenKeyword(start_loc=49, end_loc=1, value=b"#"), + TokenName(start_loc=51, end_loc=2, value="a"), + TokenName(start_loc=53, end_loc=4, value="BCD"), + TokenName(start_loc=58, end_loc=10, value="Some_Name"), + TokenName(start_loc=69, end_loc=12, value="foo_naa"), + TokenName(start_loc=83, end_loc=7, value="barj"), + TokenName(start_loc=90, end_loc=5, value="need"), + TokenName(start_loc=96, end_loc=6, value="oh1oh"), + TokenKeyword(start_loc=103, end_loc=4, value=b"func"), + TokenName(start_loc=107, end_loc=3, value="a"), + TokenName(start_loc=109, end_loc=3, value="b"), + ], + ) diff --git a/tests/test_safebufiterator.py b/tests/test_safebufiterator.py index a75459e..2099fe3 100644 --- a/tests/test_safebufiterator.py +++ b/tests/test_safebufiterator.py @@ -1,12 +1,12 @@ import io from unittest import TestCase -from pdfmajor.safebufiterator import EOF, SafeBufferIt +from pdfmajor.safebufiterator import EOF, BufferStream class UseCase(TestCase): def test_safebuff(self): for bufsize in [2, 33, 65, 66, 100, 200]: - it = SafeBufferIt( + it = BufferStream( io.BytesIO( b"this is a lengthy comment that ends here\nso this is not reachable" ), @@ -16,7 +16,7 @@ class UseCase(TestCase): self.assertLessEqual(len(subbuf), it.buffer_size) def test_itraises(self): - it = SafeBufferIt(io.BytesIO(b"this")) + it = BufferStream(io.BytesIO(b"this")) list(it) # read everything with self.assertRaises(EOF): for _ in it: @@ -24,7 +24,7 @@ class UseCase(TestCase): def test_seekd(self): for bufsize in [2, 33, 65, 66, 100, 200]: - it = SafeBufferIt( + it = BufferStream( io.BytesIO( b"this is a lengthy comment that ends here\nso this is not reachable" ), diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py deleted file mode 100644 index f266db8..0000000 --- a/tests/test_tokenizer.py +++ /dev/null @@ -1,230 +0,0 @@ -import io -from random import randint -from decimal import Decimal -from typing import List -from pdfmajor.tokenizer.token_parsers.string import parse_string -from pdfmajor.tokenizer import PSTokenizer -from pdfmajor.tokenizer.token import ( - Token, - TokenBoolean, - TokenComment, - TokenDecimal, - TokenInteger, - TokenKeyword, - TokenLiteral, - TokenString, -) -from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt -from pdfmajor.tokenizer.token_parsers.comment import parse_comment -from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword -from pdfmajor.tokenizer.token_parsers.number import parse_number -from pdfmajor.tokenizer.token_parsers.literal import parse_literal -from unittest import TestCase - - -def make_stream_iter(data: bytes, initpos: int = 0): - stream = io.BytesIO(data) - - def iter_stream(buf_size: int): - while True: - bufpos = stream.tell() - buf = stream.read(buf_size) - if buf: - yield PInput(initpos + bufpos, buf) - else: - raise "EOF" - - return iter_stream - - -class Util(TestCase): - def test_safebuff(self): - it = SafeBufferIt( - b"this is a lengthy comment that ends here\nso this is not reachable" - ) - for subbuf in it.into_iter(): - self.assertEqual(len(subbuf), len(it.buffer) - it.skip) - it.skip += randint(1, max(len(subbuf) - 1, 1)) - - -class Individual(TestCase): - def test_comment_parser_ln(self): - for buf_size in [2, 7, 11, 150]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter( - b"this is a lengthy comment that ends here\nso this is not reachable" - ) - token = parse_comment(0, iter_stream(buf_size)) - self.assertIsInstance(token, TokenComment) - self.assertEqual(token.pos, 0) - self.assertEqual( - token.value, b"this is a lengthy comment that ends here" - ) - self.assertEqual(token.size, 40) - - def test_comment_parser_lr(self): - for buf_size in [2, 7, 11, 150]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter( - b"this is a lengthy comment that ends here\rso this is not reachable" - ) - token = parse_comment(0, iter_stream(buf_size)) - self.assertIsInstance(token, TokenComment) - self.assertEqual(token.pos, 0) - self.assertEqual( - token.value, b"this is a lengthy comment that ends here" - ) - self.assertEqual(token.size, 40) - - def test_parse_float(self): - for buf_size in [2, 3, 50]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter(b"0.120 + some invalid text", 5) - token = parse_number(5, iter_stream(buf_size)) - self.assertIsInstance(token, TokenDecimal) - self.assertEqual(token.pos, 5) - self.assertEqual(token.value, Decimal("0.12")) - self.assertEqual(token.size, 5) - - def test_parse_int(self): - for buf_size in [2, 3, 50]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter( - b"130 oh oh 0.12 + some invalid text", 11 - ) - token = parse_number(11, iter_stream(buf_size)) - self.assertIsInstance(token, TokenInteger) - self.assertEqual(token.pos, 11) - self.assertEqual(token.value, 130) - self.assertEqual(token.size, 3) - - def test_parse_bool_true(self): - for buf_size in [2, 3, 50]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter(b"true and the american way", 91) - token = parse_keyword(91, iter_stream(buf_size)) - self.assertIsInstance(token, TokenBoolean) - self.assertEqual(token.pos, 91) - self.assertEqual(token.value, True) - self.assertEqual(token.size, 4) - - def test_parse_bool_false(self): - for buf_size in [2, 3, 50]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter(b"false mannn", 33) - token = parse_keyword(33, iter_stream(buf_size)) - self.assertIsInstance(token, TokenBoolean) - self.assertEqual(token.pos, 33) - self.assertEqual(token.value, False) - self.assertEqual(token.size, 5) - - def test_parse_literals(self): - for buf_size in [2, 3, 50, 444]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71) - token = parse_literal(71, iter_stream(buf_size)) - self.assertIsInstance(token, TokenLiteral) - self.assertEqual(token.pos, 71) - self.assertEqual(token.value, "Some_Name") - self.assertEqual(token.size, 9) - - def test_parse_literals_hex(self): - for buf_size in [2]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter(br"foo#5faa ", 71) - token = parse_literal(71, iter_stream(buf_size)) - self.assertIsInstance(token, TokenLiteral) - self.assertEqual(token.pos, 71) - self.assertEqual(token.value, "foo_aa") - self.assertEqual(token.size, 8) - - def test_parse_string(self): - for buf_size in [2]: - with self.subTest(buf_size=buf_size): - iter_stream = make_stream_iter(br"this % is not a comment.) ", 88) - token = parse_string(88, iter_stream(buf_size)) - self.assertIsInstance(token, TokenString) - self.assertEqual(token.pos, 88) - self.assertEqual(token.value, "this % is not a comment.") - self.assertEqual(token.size, 24) - - -class Tokenizer(TestCase): - def run_test(self, raw: bytes, expected: List[Token]): - tokenizer = PSTokenizer(io.BytesIO(raw)) - for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())): - self.assertEqual(token, etoken, f"Failed at example {i}") - - def test_case1(self): - tokenizer = PSTokenizer( - io.BytesIO( - br"""%!PS -begin end - " @ # -/a/BCD /Some_Name /foo#5f#6eaa -0 +1 -2 .5 1.234 -(abc) () (abc ( def ) ghi) -(def\040\0\0404ghi) (bach\\slask) (foo\nbaa) -(this % is not a comment.) -(foo -baa) -(foo\ -baa) -<> <20> < 40 4020 > - -func/a/b{(c)do*}def -[ 1 (z) ! ] -<< /foo (bar) >>""" - ) - ) - for token in tokenizer.iter_tokens(): - print(token) - - def test_case2(self): - self.run_test( - raw=br"""10.32 -0 +1 -2 .5 1.234 --1.234 990 """, - expected=[ - TokenDecimal(pos=0, size=5, value=Decimal("10.32")), - TokenInteger(pos=6, size=1, value=0), - TokenInteger(pos=8, size=2, value=1), - TokenInteger(pos=11, size=2, value=-2), - TokenDecimal(pos=14, size=2, value=Decimal("0.5")), - TokenDecimal(pos=17, size=5, value=Decimal("1.234")), - TokenDecimal(pos=23, size=6, value=Decimal("-1.234")), - TokenInteger(pos=30, size=3, value=990), - ], - ) - - def test_case_words1(self): - self.run_test( - raw=br"""%!PS -% This is some secret sauce -begin end - " @ # -/a/BCD /Some_Name /foo#5f#6eaa /bar#6a -/need /oh1oh -func/a/b -""", - expected=[ - TokenComment(pos=0, size=4, value=b"!PS"), - TokenComment(pos=5, size=27, value=b" This is some secret sauce"), - TokenKeyword(pos=33, size=5, value=b"begin"), - TokenKeyword(pos=39, size=3, value=b"end"), - TokenKeyword(pos=44, size=1, value=b'"'), - TokenKeyword(pos=47, size=1, value=b"@"), - TokenKeyword(pos=49, size=1, value=b"#"), - TokenLiteral(pos=52, size=1, value="a"), - TokenLiteral(pos=54, size=3, value="BCD"), - TokenLiteral(pos=59, size=9, value="Some_Name"), - TokenLiteral(pos=70, size=11, value="foo_naa"), - TokenLiteral(pos=83, size=6, value="barj"), - TokenLiteral(pos=91, size=4, value="need"), - TokenLiteral(pos=97, size=5, value="oh1oh"), - TokenKeyword(pos=103, size=4, value=b"func"), - TokenLiteral(pos=108, size=1, value="a"), - TokenLiteral(pos=110, size=1, value="b"), - ], - )