- clean up

- renamed safebuf to streambuf
- added arrays
This commit is contained in:
ari committed 2020-10-17 15:54:39 -04:00
1 parent c752346638
commit a03857d0f7
29 files changed
+280 -1381

No files matched your search

+145 -7
View File
@@ -1,21 +1,27 @@
from decimal import Decimal
import io
from decimal import Decimal
from typing import List
from unittest import TestCase
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.exceptions import LexerError
from pdfmajor.lexer.number import parse_number
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
from pdfmajor.lexer.string import parse_string
from unittest import TestCase
from pdfmajor.lexer.name import parse_name
from pdfmajor.lexer.keyword import parse_keyword
from pdfmajor.safebufiterator import SafeBufferIt
from pdfmajor.safebufiterator import BufferStream
from pdfmajor.lexer.token import (
TDictVaue,
TDictValue,
Token,
TokenBoolean,
TokenComment,
TokenDecimal,
TokenDictionary,
TokenHexString,
TokenInteger,
TokenKeyword,
TokenName,
TokenString,
)
@@ -23,7 +29,7 @@ from pdfmajor.lexer.comment import parse_comment
def make_stream_iter(data: bytes, buf_size: int):
return SafeBufferIt(io.BytesIO(data), buf_size)
return BufferStream(io.BytesIO(data), buf_size)
class Basics(TestCase):
@@ -156,7 +162,7 @@ class Basics(TestCase):
token = parse_double_angled_bracket(buffer)
self.assertIsInstance(token, TokenDictionary)
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, TDictVaue.OPEN)
self.assertEqual(token.value, TDictValue.OPEN)
self.assertEqual(token.end_loc, 2)
def test_parse_dd_to_hex(self):
@@ -190,3 +196,135 @@ class Basics(TestCase):
self.assertEqual(token.start_loc, -1)
self.assertEqual(token.value, 130)
self.assertEqual(token.end_loc, 3)
class Lexer(TestCase):
def run_test(self, raw: bytes, expected: List[Token]):
tokenizer = PDFLexer(io.BytesIO(raw))
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
self.assertEqual(token, etoken, f"Failed at example {i}")
try:
next_token = next(tokenizer.iter_tokens())
self.assertIsNone(next_token)
except LexerError as e:
self.assertIsNotNone(e)
def test_case1(self):
self.run_test(
raw=br"""%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>""",
expected=[
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
TokenKeyword(start_loc=5, end_loc=5, value=b"begin"),
TokenKeyword(start_loc=11, end_loc=3, value=b"end"),
TokenKeyword(start_loc=16, end_loc=1, value=b'"'),
TokenKeyword(start_loc=19, end_loc=1, value=b"@"),
TokenKeyword(start_loc=21, end_loc=1, value=b"#"),
TokenName(start_loc=24, end_loc=1, value="a"),
TokenName(start_loc=26, end_loc=3, value="BCD"),
TokenName(start_loc=31, end_loc=9, value="Some_Name"),
TokenName(start_loc=42, end_loc=11, value="foo_naa"),
TokenInteger(start_loc=54, end_loc=1, value=0),
TokenInteger(start_loc=56, end_loc=2, value=1),
TokenInteger(start_loc=59, end_loc=2, value=-2),
TokenDecimal(start_loc=62, end_loc=2, value=Decimal("0.5")),
TokenDecimal(start_loc=65, end_loc=5, value=Decimal("1.234")),
TokenString(start_loc=71, end_loc=5, value="abc"),
TokenString(start_loc=77, end_loc=2, value=""),
TokenString(start_loc=80, end_loc=17, value="abc ( def ) ghi"),
TokenString(start_loc=98, end_loc=19, value="def 4ghi"),
TokenString(start_loc=118, end_loc=13, value=r"bach\slask"),
TokenString(start_loc=132, end_loc=10, value="foo\nbaa"),
TokenString(
start_loc=143, end_loc=26, value="this % is not a comment."
),
TokenString(start_loc=170, end_loc=9, value="foo\nbaa"),
TokenString(start_loc=180, end_loc=10, value="foobaa"),
TokenHexString(start_loc=191, end_loc=2, value=b""),
TokenHexString(start_loc=194, end_loc=4, value=b" "),
TokenHexString(start_loc=199, end_loc=11, value=b"@@ "),
TokenHexString(
start_loc=211, end_loc=14, value=b"\xab\xcd\x00\x124\x05"
),
TokenKeyword(start_loc=226, end_loc=4, value=b"func"),
TokenName(start_loc=231, end_loc=1, value="a"),
TokenName(start_loc=233, end_loc=1, value="b"),
TokenKeyword(start_loc=234, end_loc=1, value=b"{"),
],
)
def test_case2(self):
self.run_test(
raw=br"""10.32
0 +1 -2 .5 1.234
-1.234 990 """,
expected=[
TokenDecimal(start_loc=0, end_loc=5, value=Decimal("10.32")),
TokenInteger(start_loc=6, end_loc=1, value=0),
TokenInteger(start_loc=8, end_loc=2, value=1),
TokenInteger(start_loc=11, end_loc=2, value=-2),
TokenDecimal(start_loc=14, end_loc=2, value=Decimal("0.5")),
TokenDecimal(start_loc=17, end_loc=5, value=Decimal("1.234")),
TokenDecimal(start_loc=23, end_loc=6, value=Decimal("-1.234")),
TokenInteger(start_loc=30, end_loc=3, value=990),
],
)
def test_case3(self):
self.run_test(
raw=br"""/b{(c)do*}def""",
expected=[
TokenName(start_loc=0, end_loc=2, value="b"),
TokenKeyword(start_loc=2, end_loc=3, value=b"{"),
TokenString(start_loc=4, end_loc=5, value="c"),
],
)
def test_case4(self):
self.run_test(
raw=br"""%!PS
% This is some secret sauce
begin end
" @ #
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
/need /oh1oh
func/a/b
""",
expected=[
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
TokenComment(
start_loc=5, end_loc=27, value=b" This is some secret sauce"
),
TokenKeyword(start_loc=33, end_loc=5, value=b"begin"),
TokenKeyword(start_loc=39, end_loc=3, value=b"end"),
TokenKeyword(start_loc=44, end_loc=1, value=b'"'),
TokenKeyword(start_loc=47, end_loc=1, value=b"@"),
TokenKeyword(start_loc=49, end_loc=1, value=b"#"),
TokenName(start_loc=51, end_loc=2, value="a"),
TokenName(start_loc=53, end_loc=4, value="BCD"),
TokenName(start_loc=58, end_loc=10, value="Some_Name"),
TokenName(start_loc=69, end_loc=12, value="foo_naa"),
TokenName(start_loc=83, end_loc=7, value="barj"),
TokenName(start_loc=90, end_loc=5, value="need"),
TokenName(start_loc=96, end_loc=6, value="oh1oh"),
TokenKeyword(start_loc=103, end_loc=4, value=b"func"),
TokenName(start_loc=107, end_loc=3, value="a"),
TokenName(start_loc=109, end_loc=3, value="b"),
],
)