From 460d213700d4439539b07f640a7cad0da6a5975b Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Tue, 13 Oct 2020 10:13:20 -0400 Subject: [PATCH] initial tokenizer --- .editorconfig | 2 +- pdfmajor/tokenizer/__init__.py | 46 ++++ pdfmajor/tokenizer/constants.py | 37 ++++ pdfmajor/tokenizer/exceptions.py | 16 ++ pdfmajor/tokenizer/old.py | 341 +++++++++++++++++++++++++++++ pdfmajor/tokenizer/parse_funcs.py | 100 +++++++++ pdfmajor/tokenizer/token.py | 44 ++++ pdfmajor/tokenizer/token_parser.py | 139 ++++++++++++ pdfmajor/tokenizer/types.py | 80 +++++++ tests/test_tokenizer.py | 64 ++++++ 10 files changed, 868 insertions(+), 1 deletion(-) create mode 100644 pdfmajor/tokenizer/__init__.py create mode 100644 pdfmajor/tokenizer/constants.py create mode 100644 pdfmajor/tokenizer/exceptions.py create mode 100644 pdfmajor/tokenizer/old.py create mode 100644 pdfmajor/tokenizer/parse_funcs.py create mode 100644 pdfmajor/tokenizer/token.py create mode 100644 pdfmajor/tokenizer/token_parser.py create mode 100644 pdfmajor/tokenizer/types.py create mode 100644 tests/test_tokenizer.py diff --git a/.editorconfig b/.editorconfig index 941e639..5341ccb 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,4 +6,4 @@ indent_size = 4 end_of_line = lf charset = utf-8 trim_trailing_whitespace = false -insert_final_newline = false \ No newline at end of file +insert_final_newline = true \ No newline at end of file diff --git a/pdfmajor/tokenizer/__init__.py b/pdfmajor/tokenizer/__init__.py new file mode 100644 index 0000000..cc39a76 --- /dev/null +++ b/pdfmajor/tokenizer/__init__.py @@ -0,0 +1,46 @@ +import io +from typing import List +from pdfmajor.tokenizer.token import Token +from pdfmajor.tokenizer.exceptions import TokenizerEOF +from pdfmajor.tokenizer.parse_funcs import ParseFunc + + +class PSTokenizer: + def __init__(self, fp: io.BufferedIOBase) -> None: + self.fp = fp + self.seek(0) + + def seek(self, pos: int): + """Seeks tokenizer to the given position.""" + self.fp.seek(pos) + # reset the status for nextline() + self.bufpos: int = pos + self.buf: bytes = b"" + self.charpos: int = 0 + # reset the status for nexttoken() + self._current_parse_func: ParseFunc = self._parse_main + self._curtoken: bytes = b"" + self._curtokenpos: int = 0 + self._tokens: List[Token] = [] + return + + def __repr__(self): + return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos) + + def nexttoken(self): + while not self._tokens: + self.fillbuf() + self.charpos = self._current_parse_func(self.buf, self.charpos) + token = self._tokens.pop(0) + return token + + def fillbuf(self): + if self.charpos < len(self.buf): + return + # fetch next chunk. + self.bufpos = self.fp.tell() + self.buf = self.fp.read(self.BUFSIZ) + if not self.buf: + raise TokenizerEOF("Unexpected EOF") + self.charpos = 0 + return diff --git a/pdfmajor/tokenizer/constants.py b/pdfmajor/tokenizer/constants.py new file mode 100644 index 0000000..1b17b05 --- /dev/null +++ b/pdfmajor/tokenizer/constants.py @@ -0,0 +1,37 @@ +import re +from .types import PSLiteral, PSSymbolTable, PSKeyword + +PSLiteralTable = PSSymbolTable(PSLiteral) +PSKeywordTable = PSSymbolTable(PSKeyword) +LIT = PSLiteralTable.intern +KWD = PSKeywordTable.intern +KEYWORD_PROC_BEGIN = KWD(b"{") +KEYWORD_PROC_END = KWD(b"}") +KEYWORD_ARRAY_BEGIN = KWD(b"[") +KEYWORD_ARRAY_END = KWD(b"]") +KEYWORD_DICT_BEGIN = KWD(b"<<") +KEYWORD_DICT_END = KWD(b">>") + +## PSBaseParser +## +EOL = re.compile(br"[\r\n]") +SPC = re.compile(br"\s") +NONSPC = re.compile(br"\S") +HEX = re.compile(br"[0-9a-fA-F]") +END_LITERAL = re.compile(br"[#/%\[\]()<>{}\s]") +END_HEX_STRING = re.compile(br"[^\s0-9a-fA-F]") +HEX_PAIR = re.compile(br"[0-9a-fA-F]{2}|.") +END_NUMBER = re.compile(br"[^0-9]") +END_KEYWORD = re.compile(br"[#/%\[\]()<>{}\s]") +END_STRING = re.compile(br"[()\134]") +OCT_STRING = re.compile(br"[0-7]") +ESC_STRING = { + b"b": 8, + b"t": 9, + b"n": 10, + b"f": 12, + b"r": 13, + b"(": 40, + b")": 41, + b"\\": 92, +} diff --git a/pdfmajor/tokenizer/exceptions.py b/pdfmajor/tokenizer/exceptions.py new file mode 100644 index 0000000..261b7ff --- /dev/null +++ b/pdfmajor/tokenizer/exceptions.py @@ -0,0 +1,16 @@ +from pdfmajor.execptions import PDFMajorException + + +class TokenizerError(PDFMajorException): + pass + + +class TokenizerEOF(TokenizerError): + pass + +class InvalidToken(TokenizerError): + def __init__(self, pos: int, token: bytes) -> None: + super().__init__("Invalid Token Found at {pos}, of value={v}".format( + pos=pos, + v=token + )) diff --git a/pdfmajor/tokenizer/old.py b/pdfmajor/tokenizer/old.py new file mode 100644 index 0000000..71da1ac --- /dev/null +++ b/pdfmajor/tokenizer/old.py @@ -0,0 +1,341 @@ +from typing import List, Tuple, Union +from io import TextIOWrapper + +from pdfmajor.utils import int2byte +from pdfmajor.execptions import PSEOF + +from .types import PSLiteral +from .constants import OCT_STRING, ESC_STRING, KEYWORD_DICT_BEGIN, KEYWORD_DICT_END +from .constants import END_NUMBER, END_STRING, END_KEYWORD, END_HEX_STRING +from .constants import SPC, HEX_PAIR +from .constants import EOL, NONSPC, KWD, END_LITERAL, LIT, HEX + + +class PSTokenizer(object): + """Tokenizatizes bytes stream to PS-Tokens""" + + BUFSIZ = 4096 + + def __init__(self, fp: TextIOWrapper): + self.fp = fp + self.seek(0) + return + + def __repr__(self): + return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos) + + def flush(self): + return + + def close(self): + self.flush() + return + + def tell(self): + return self.bufpos + self.charpos + + def poll(self, pos=None, n=80): + pos0 = self.fp.tell() + if not pos: + pos = self.bufpos + self.charpos + self.fp.seek(pos) + self.fp.read(n) + self.fp.seek(pos0) + return + + def seek(self, pos: int): + """Seeks the parser to the given position.""" + self.fp.seek(pos) + # reset the status for nextline() + self.bufpos: int = pos + self.buf: bytes = b"" + self.charpos: int = 0 + # reset the status for nexttoken() + self._current_parse_func = self._parse_main + self._curtoken: bytes = b"" + self._curtokenpos: int = 0 + self._tokens: List[ + Tuple[int, Union[int, float, bool, str, bytes, PSLiteral]] + ] = [] + return + + def fillbuf(self): + if self.charpos < len(self.buf): + return + # fetch next chunk. + self.bufpos = self.fp.tell() + self.buf = self.fp.read(self.BUFSIZ) + if not self.buf: + raise PSEOF("Unexpected EOF") + self.charpos = 0 + return + + def nextline(self): + """Fetches a next line that ends either with \\r or \\n.""" + linebuf = b"" + linepos = self.bufpos + self.charpos + eol = False + while 1: + self.fillbuf() + if eol: + c = self.buf[self.charpos : self.charpos + 1] + # handle b'\r\n' + if c == b"\n": + linebuf += c + self.charpos += 1 + break + m = EOL.search(self.buf, self.charpos) + if m: + linebuf += self.buf[self.charpos : m.end(0)] + self.charpos = m.end(0) + if linebuf[-1:] == b"\r": + eol = True + else: + break + else: + linebuf += self.buf[self.charpos :] + self.charpos = len(self.buf) + return (linepos, linebuf) + + def revreadlines(self): + """Fetches a next line backword. + + This is used to locate the trailers at the end of a file. + """ + self.fp.seek(0, 2) + pos = self.fp.tell() + buf = b"" + while 0 < pos: + prevpos = pos + pos = max(0, pos - self.BUFSIZ) + self.fp.seek(pos) + s = self.fp.read(prevpos - pos) + if not s: + break + while 1: + n = max(s.rfind(b"\r"), s.rfind(b"\n")) + if n == -1: + buf = s + buf + break + yield s[n:] + buf + s = s[:n] + buf = b"" + return + + def _parse_main(self, s, i): + m = NONSPC.search(s, i) + if not m: + return len(s) + j = m.start(0) + c = s[j : j + 1] + self._curtokenpos = self.bufpos + j + if c == b"%": + self._curtoken = b"%" + self._current_parse_func = self._parse_comment + return j + 1 + elif c == b"/": + self._curtoken = b"" + self._current_parse_func = self._parse_literal + return j + 1 + elif c in b"-+" or c.isdigit(): + self._curtoken = c + self._current_parse_func = self._parse_number + return j + 1 + elif c == b".": + self._curtoken = c + self._current_parse_func = self._parse_float + return j + 1 + elif c.isalpha(): + self._curtoken = c + self._current_parse_func = self._parse_keyword + return j + 1 + elif c == b"(": + self._curtoken = b"" + self.paren = 1 + self._current_parse_func = self._parse_string + return j + 1 + elif c == b"<": + self._curtoken = b"" + self._current_parse_func = self._parse_wopen + return j + 1 + elif c == b">": + self._curtoken = b"" + self._current_parse_func = self._parse_wclose + return j + 1 + else: + self._add_token(KWD(c)) + return j + 1 + + def _add_token(self, obj): + self._tokens.append((self._curtokenpos, obj)) + return + + def _parse_comment(self, s, i): + m = EOL.search(s, i) + if not m: + self._curtoken += s[i:] + return len(s) + j = m.start(0) + self._curtoken += s[i:j] + self._current_parse_func = self._parse_main + # We ignore comments. + # self._tokens.append(self._curtoken) + return j + + def _parse_literal(self, s, i): + m = END_LITERAL.search(s, i) + if not m: + self._curtoken += s[i:] + return len(s) + j = m.start(0) + self._curtoken += s[i:j] + c = s[j : j + 1] + if c == b"#": + self.hex = b"" + self._current_parse_func = self._parse_literal_hex + return j + 1 + try: + self._curtoken = str(self._curtoken, "utf-8") + except: + pass + self._add_token(LIT(self._curtoken)) + self._current_parse_func = self._parse_main + return j + + def _parse_literal_hex(self, s, i): + c = s[i : i + 1] + if HEX.match(c) and len(self.hex) < 2: + self.hex += c + return i + 1 + if self.hex: + self._curtoken += int2byte(int(self.hex, 16)) + self._current_parse_func = self._parse_literal + return i + + def _parse_number(self, s, i): + m = END_NUMBER.search(s, i) + if not m: + self._curtoken += s[i:] + return len(s) + j = m.start(0) + self._curtoken += s[i:j] + c = s[j : j + 1] + if c == b".": + self._curtoken += c + self._current_parse_func = self._parse_float + return j + 1 + try: + self._add_token(int(self._curtoken)) + except ValueError: + pass + self._current_parse_func = self._parse_main + return j + + def _parse_float(self, s, i): + m = END_NUMBER.search(s, i) + if not m: + self._curtoken += s[i:] + return len(s) + j = m.start(0) + self._curtoken += s[i:j] + try: + self._add_token(float(self._curtoken)) + except ValueError: + pass + self._current_parse_func = self._parse_main + return j + + def _parse_keyword(self, s, i): + m = END_KEYWORD.search(s, i) + if not m: + self._curtoken += s[i:] + return len(s) + j = m.start(0) + self._curtoken += s[i:j] + if self._curtoken == b"true": + token = True + elif self._curtoken == b"false": + token = False + else: + token = KWD(self._curtoken) + self._add_token(token) + self._current_parse_func = self._parse_main + return j + + def _parse_string(self, s, i): + m = END_STRING.search(s, i) + if not m: + self._curtoken += s[i:] + return len(s) + j = m.start(0) + self._curtoken += s[i:j] + c = s[j : j + 1] + if c == b"\\": + self.oct = b"" + self._current_parse_func = self._parse_string_1 + return j + 1 + if c == b"(": + self.paren += 1 + self._curtoken += c + return j + 1 + if c == b")": + self.paren -= 1 + if self.paren: # WTF, they said balanced parens need no special treatment. + self._curtoken += c + return j + 1 + self._add_token(self._curtoken) + self._current_parse_func = self._parse_main + return j + 1 + + def _parse_string_1(self, s, i): + c = s[i : i + 1] + if OCT_STRING.match(c) and len(self.oct) < 3: + self.oct += c + return i + 1 + if self.oct: + self._curtoken += int2byte(int(self.oct, 8)) + self._current_parse_func = self._parse_string + return i + if c in ESC_STRING: + self._curtoken += int2byte(ESC_STRING[c]) + self._current_parse_func = self._parse_string + return i + 1 + + def _parse_wopen(self, s, i): + c = s[i : i + 1] + if c == b"<": + self._add_token(KEYWORD_DICT_BEGIN) + self._current_parse_func = self._parse_main + i += 1 + else: + self._current_parse_func = self._parse_hexstring + return i + + def _parse_wclose(self, s, i): + c = s[i : i + 1] + if c == b">": + self._add_token(KEYWORD_DICT_END) + i += 1 + self._current_parse_func = self._parse_main + return i + + def _parse_hexstring(self, s, i): + m = END_HEX_STRING.search(s, i) + if not m: + self._curtoken += s[i:] + return len(s) + j = m.start(0) + self._curtoken += s[i:j] + token = HEX_PAIR.sub( + lambda m: int2byte(int(m.group(0), 16)), SPC.sub(b"", self._curtoken) + ) + self._add_token(token) + self._current_parse_func = self._parse_main + return j + + def nexttoken(self): + while not self._tokens: + self.fillbuf() + self.charpos = self._current_parse_func(self.buf, self.charpos) + token = self._tokens.pop(0) + return token diff --git a/pdfmajor/tokenizer/parse_funcs.py b/pdfmajor/tokenizer/parse_funcs.py new file mode 100644 index 0000000..db0c73b --- /dev/null +++ b/pdfmajor/tokenizer/parse_funcs.py @@ -0,0 +1,100 @@ +from pdfmajor.tokenizer.token import Token, TokenKeyword +from typing import Callable, NamedTuple, Optional, Union +from .constants import EOL, NONSPC + +ParseFunc = Callable[["ParseInput"], "NextParseAction"] + + +class NextParseAction(NamedTuple): + next_parse_action: ParseFunc + charpos: int + curtokenpos: int + curtoken: Union[Token, bytes] = b"" + paren: Optional[int] = None + + +class ParseInput(NamedTuple): + bufvalue: bytes + charpos: int + bufpos: int + + +def parse_main(inp: ParseInput) -> NextParseAction: + m = NONSPC.search(inp.bufvalue, inp.charpos) + if not m: + return len(inp.bufvalue) + j = m.start(0) + c = inp.bufvalue[j : j + 1] + if c == b"%": + return NextParseAction( + curtoken=b"%", + next_parse_action=__parse_comment, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + ) + elif c == b"/": + return NextParseAction( + next_parse_action=__parse_literal, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + ) + elif c in b"-+" or c.isdigit(): + return NextParseAction( + curtoken=c, + next_parse_action=__parse_number, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + ) + elif c == b".": + return NextParseAction( + curtoken=c, + next_parse_action=__parse_float, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + ) + elif c.isalpha(): + return NextParseAction( + curtoken=c, + next_parse_action=__parse_keyword, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + ) + elif c == b"(": + return NextParseAction( + next_parse_action=__parse_string, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + paren=1, + ) + elif c == b"<": + return NextParseAction( + next_parse_action=__parse_wopen, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + ) + elif c == b">": + return NextParseAction( + next_parse_action=__parse_wclose, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + ) + else: + return NextParseAction( + next_parse_action=parse_main, + curtokenpos=inp.bufpos + j, + charpos=j + 1, + curtoken=TokenKeyword(inp.bufpos + j, c), + ) + + +def __parse_comment(s, i): + m = EOL.search(s, i) + if not m: + self._curtoken += s[i:] + return len(s) + j = m.start(0) + self._curtoken += s[i:j] + self._current_parse_func = self._parse_main + # We ignore comments. + # self._tokens.append(self._curtoken) + return j diff --git a/pdfmajor/tokenizer/token.py b/pdfmajor/tokenizer/token.py new file mode 100644 index 0000000..57670a6 --- /dev/null +++ b/pdfmajor/tokenizer/token.py @@ -0,0 +1,44 @@ +from typing import Generic, TypeVar, Union +from decimal import Decimal + +T = TypeVar("T") + + +class Token(Generic[T]): + __slots__ = ["pos", "size", "value"] + + def __init__(self, pos: int, size: int, value: T) -> None: + self.size = size + self.pos = pos + self.value = value + + def __repr__(self) -> str: + return "<{cls} pos={pos}>{value}".format( + cls=self.__class__.__name__, pos=self.pos, value=self.value + ) + + +class TokenComment(Token[bytes]): + pass + + +class TokenHex(Token[bytes]): + pass + + +class TokenKeyword(Token[bytes]): + pass + + +class TokenString(Token[str]): + pass + + +class TokenInteger(Token[int]): + pass + + +class TokenDecimal(Token[Decimal]): + pass + +TokenNumber = Union[TokenInteger, TokenDecimal] diff --git a/pdfmajor/tokenizer/token_parser.py b/pdfmajor/tokenizer/token_parser.py new file mode 100644 index 0000000..3f0aee1 --- /dev/null +++ b/pdfmajor/tokenizer/token_parser.py @@ -0,0 +1,139 @@ +from decimal import Decimal +from pdfmajor.tokenizer.exceptions import InvalidToken +from pdfmajor.tokenizer.token import Token, TokenComment, TokenDecimal, TokenNumber, TokenInteger +from pdfmajor.tokenizer.constants import END_NUMBER, EOL +from typing import Callable, Iterator, NamedTuple + + +class PInput(NamedTuple): + pos: int + buf: bytes + + +ParserFunc = Callable[[int, Iterator[PInput]], Token] + +def cmp_tsize(curpos: int, initialpos: int, tokenend_idx: int) -> int: + """Computes the size of the token based on its positional data + + Args: + curpos (int): current position of the buffer + initialpos (int): initial position where the parser started working + tokenend_idx (int): last relevant character index within the last buffered value + + Returns: + int: length of index + """ + return curpos-initialpos+tokenend_idx + +def parse_comment(initialpos: int, inp: Iterator[PInput]) -> TokenComment: + """Parses input stream into a comment + + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) + + Returns: + TokenComment + """ + curtoken = b"" + for curpos, s in inp: + m = EOL.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + return TokenComment( + initialpos, + cmp_tsize(curpos, initialpos, j), + curtoken + ) + +def parse_number(initialpos: int, inp: Iterator[PInput]) -> TokenNumber: + """Parses input stream into a comment + + Args: + initialpos (int): initial position where we started + inp (Iterator[PInput]) + + Returns: + TokenComment + """ + curtoken = b"" + is_dec = False + for curpos, s in inp: + m = END_NUMBER.search(s, 0) + if not m: + curtoken += s + else: + j = m.start(0) + curtoken += s[:j] + c = s[j : j + 1] + if (c == b".") and not is_dec: + is_dec = True + curtoken += c + s = s[j+1:] + m = END_NUMBER.search(s, 0) + if not m: + curtoken += s + continue + else: + nj = m.start(0) + curtoken += s[:nj] + j += nj + return TokenDecimal( + initialpos, + cmp_tsize(curpos, initialpos, j), + Decimal(curtoken.decode()) + ) + try: + if is_dec: + return TokenDecimal( + initialpos, + cmp_tsize(curpos, initialpos, j), + Decimal(curtoken.decode()) + ) + else: + return TokenInteger( + initialpos, + cmp_tsize(curpos, initialpos, j), + int(curtoken) + ) + except ValueError: + raise InvalidToken(initialpos, curtoken) + +# def parse_float( +# initialpos: int, +# inp: Iterator[PInput], +# initialtokenval = b"" +# ) -> TokenFloat: +# curtoken = b"" + initialtokenval +# for curpos, s in inp: +# print('->',curpos, s) +# m = END_NUMBER.search(s, 0) +# if not m: +# curtoken += s +# else: +# j = m.start(0) +# curtoken += s[:j] +# next_t = s[j:j+1] +# if next_t == b'.': +# curtoken += b'.' +# s = s[j+1:] +# m = END_NUMBER.search(s, 0) +# if not m: +# curtoken += s +# continue +# else: +# j = m.start(0) +# curtoken += s[:j] +# next_t = s[j+1] +# try: +# print(curpos,j, initialpos) +# return TokenFloat( +# initialpos, +# cmp_tsize(curpos, initialpos, j), +# Decimal(curtoken.decode()) +# ) +# except ValueError: +# raise InvalidToken(initialpos, curtoken) diff --git a/pdfmajor/tokenizer/types.py b/pdfmajor/tokenizer/types.py new file mode 100644 index 0000000..296a191 --- /dev/null +++ b/pdfmajor/tokenizer/types.py @@ -0,0 +1,80 @@ +## Basic PostScript Types +## +from typing import Union + +## PSObject +## +class PSObject(object): + + """Base class for all PS or PDF-related data types.""" + + pass + + +## PSLiteral +## +class PSLiteral(PSObject): + + """A class that represents a PostScript literal. + + Postscript literals are used as identifiers, such as + variable names, property names and dictionary keys. + Literals are case sensitive and denoted by a preceding + slash sign (e.g. "/Name") + + Note: Do not create an instance of PSLiteral directly. + Always use PSLiteralTable.intern(). + """ + + def __init__(self, name): + self.name = name + + def __repr__(self): + name = self.name + return "/%r" % name + + +## PSKeyword +## +class PSKeyword(PSObject): + + """A class that represents a PostScript keyword. + + PostScript keywords are a dozen of predefined words. + Commands and directives in PostScript are expressed by keywords. + They are also used to denote the content boundaries. + + Note: Do not create an instance of PSKeyword directly. + Always use PSKeywordTable.intern(). + """ + + def __init__(self, name): + self.name = name + return + + def __repr__(self): + name = self.name + return "/%r" % name + + +## PSSymbolTable +## +class PSSymbolTable(object): + + """A utility class for storing PSLiteral/PSKeyword objects. + + Interned objects can be checked its identity with "is" operator. + """ + + def __init__(self, klass): + self.dict = {} + self.klass = klass + return + + def intern(self, name): + if name in self.dict: + lit = self.dict[name] + else: + lit = self.klass(name) + self.dict[name] = lit + return lit diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py new file mode 100644 index 0000000..c7db855 --- /dev/null +++ b/tests/test_tokenizer.py @@ -0,0 +1,64 @@ +from decimal import Decimal +import io +from pdfmajor.tokenizer.token import TokenComment, TokenDecimal, TokenInteger +from pdfmajor.tokenizer.token_parser import PInput, parse_comment, parse_number +from unittest import TestCase + +def make_stream_iter(data: bytes, initpos: int = 0): + stream = io.BytesIO(data) + + def iter_stream(buf_size: int): + while True: + bufpos = stream.tell() + buf = stream.read(buf_size) + if buf: + yield PInput(initpos+bufpos, buf) + else: + raise "EOF" + return iter_stream + +class Basics(TestCase): + def test_comment_parser_ln(self): + iter_stream = make_stream_iter( + b"this is a lengthy comment that ends here\nso this is not reachable" + ) + token = parse_comment(0, iter_stream(3)) + self.assertIsInstance(token, TokenComment) + self.assertEqual(token.pos, 0) + self.assertEqual(token.value, b"this is a lengthy comment that ends here") + self.assertEqual(token.size, 40) + + def test_comment_parser_lr(self): + iter_stream = make_stream_iter( + b"this is a lengthy comment that ends here\rso this is not reachable" + ) + token = parse_comment(0, iter_stream(7)) + self.assertIsInstance(token, TokenComment) + self.assertEqual(token.pos, 0) + self.assertEqual(token.value, b"this is a lengthy comment that ends here") + self.assertEqual(token.size, 40) + + def test_parse_float(self): + iter_stream = make_stream_iter( + b"0.120 + some invalid text", + 5 + ) + token = parse_number(5, iter_stream(3)) + self.assertIsInstance(token, TokenDecimal) + self.assertEqual(token.pos, 5) + self.assertEqual(token.value, Decimal('0.12')) + self.assertEqual(token.size, 5) + + + def test_parse_int(self): + iter_stream = make_stream_iter( + b"130 oh oh 0.12 + some invalid text", + 11 + ) + token = parse_number(11, iter_stream(10)) + self.assertIsInstance(token, TokenInteger) + self.assertEqual(token.pos, 11) + self.assertEqual(token.value, 130) + self.assertEqual(token.size, 3) + +