- clean up
- renamed safebuf to streambuf - added arrays
This commit is contained in:
1 parent
c752346638
commit
a03857d0f7
29 files changed
+280
-1381
No files matched your search
+145
-7
@@ -1,21 +1,27 @@
|
||||
from decimal import Decimal
|
||||
import io
|
||||
|
||||
from decimal import Decimal
|
||||
from typing import List
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.lexer.exceptions import LexerError
|
||||
from pdfmajor.lexer.number import parse_number
|
||||
from pdfmajor.lexer.dict_and_hex import parse_double_angled_bracket, parse_hexstring
|
||||
from pdfmajor.lexer.string import parse_string
|
||||
from unittest import TestCase
|
||||
|
||||
from pdfmajor.lexer.name import parse_name
|
||||
from pdfmajor.lexer.keyword import parse_keyword
|
||||
from pdfmajor.safebufiterator import SafeBufferIt
|
||||
from pdfmajor.safebufiterator import BufferStream
|
||||
from pdfmajor.lexer.token import (
|
||||
TDictVaue,
|
||||
TDictValue,
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenDictionary,
|
||||
TokenHexString,
|
||||
TokenInteger,
|
||||
TokenKeyword,
|
||||
TokenName,
|
||||
TokenString,
|
||||
)
|
||||
@@ -23,7 +29,7 @@ from pdfmajor.lexer.comment import parse_comment
|
||||
|
||||
|
||||
def make_stream_iter(data: bytes, buf_size: int):
|
||||
return SafeBufferIt(io.BytesIO(data), buf_size)
|
||||
return BufferStream(io.BytesIO(data), buf_size)
|
||||
|
||||
|
||||
class Basics(TestCase):
|
||||
@@ -156,7 +162,7 @@ class Basics(TestCase):
|
||||
token = parse_double_angled_bracket(buffer)
|
||||
self.assertIsInstance(token, TokenDictionary)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, TDictVaue.OPEN)
|
||||
self.assertEqual(token.value, TDictValue.OPEN)
|
||||
self.assertEqual(token.end_loc, 2)
|
||||
|
||||
def test_parse_dd_to_hex(self):
|
||||
@@ -190,3 +196,135 @@ class Basics(TestCase):
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, 130)
|
||||
self.assertEqual(token.end_loc, 3)
|
||||
|
||||
|
||||
class Lexer(TestCase):
|
||||
def run_test(self, raw: bytes, expected: List[Token]):
|
||||
tokenizer = PDFLexer(io.BytesIO(raw))
|
||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
|
||||
self.assertEqual(token, etoken, f"Failed at example {i}")
|
||||
try:
|
||||
next_token = next(tokenizer.iter_tokens())
|
||||
self.assertIsNone(next_token)
|
||||
except LexerError as e:
|
||||
self.assertIsNotNone(e)
|
||||
|
||||
def test_case1(self):
|
||||
self.run_test(
|
||||
raw=br"""%!PS
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#6eaa
|
||||
0 +1 -2 .5 1.234
|
||||
(abc) () (abc ( def ) ghi)
|
||||
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
||||
(this % is not a comment.)
|
||||
(foo
|
||||
baa)
|
||||
(foo\
|
||||
baa)
|
||||
<> <20> < 40 4020 >
|
||||
<abcd00
|
||||
12345>
|
||||
func/a/b{(c)do*}def
|
||||
[ 1 (z) ! ]
|
||||
<< /foo (bar) >>""",
|
||||
expected=[
|
||||
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
|
||||
TokenKeyword(start_loc=5, end_loc=5, value=b"begin"),
|
||||
TokenKeyword(start_loc=11, end_loc=3, value=b"end"),
|
||||
TokenKeyword(start_loc=16, end_loc=1, value=b'"'),
|
||||
TokenKeyword(start_loc=19, end_loc=1, value=b"@"),
|
||||
TokenKeyword(start_loc=21, end_loc=1, value=b"#"),
|
||||
TokenName(start_loc=24, end_loc=1, value="a"),
|
||||
TokenName(start_loc=26, end_loc=3, value="BCD"),
|
||||
TokenName(start_loc=31, end_loc=9, value="Some_Name"),
|
||||
TokenName(start_loc=42, end_loc=11, value="foo_naa"),
|
||||
TokenInteger(start_loc=54, end_loc=1, value=0),
|
||||
TokenInteger(start_loc=56, end_loc=2, value=1),
|
||||
TokenInteger(start_loc=59, end_loc=2, value=-2),
|
||||
TokenDecimal(start_loc=62, end_loc=2, value=Decimal("0.5")),
|
||||
TokenDecimal(start_loc=65, end_loc=5, value=Decimal("1.234")),
|
||||
TokenString(start_loc=71, end_loc=5, value="abc"),
|
||||
TokenString(start_loc=77, end_loc=2, value=""),
|
||||
TokenString(start_loc=80, end_loc=17, value="abc ( def ) ghi"),
|
||||
TokenString(start_loc=98, end_loc=19, value="def 4ghi"),
|
||||
TokenString(start_loc=118, end_loc=13, value=r"bach\slask"),
|
||||
TokenString(start_loc=132, end_loc=10, value="foo\nbaa"),
|
||||
TokenString(
|
||||
start_loc=143, end_loc=26, value="this % is not a comment."
|
||||
),
|
||||
TokenString(start_loc=170, end_loc=9, value="foo\nbaa"),
|
||||
TokenString(start_loc=180, end_loc=10, value="foobaa"),
|
||||
TokenHexString(start_loc=191, end_loc=2, value=b""),
|
||||
TokenHexString(start_loc=194, end_loc=4, value=b" "),
|
||||
TokenHexString(start_loc=199, end_loc=11, value=b"@@ "),
|
||||
TokenHexString(
|
||||
start_loc=211, end_loc=14, value=b"\xab\xcd\x00\x124\x05"
|
||||
),
|
||||
TokenKeyword(start_loc=226, end_loc=4, value=b"func"),
|
||||
TokenName(start_loc=231, end_loc=1, value="a"),
|
||||
TokenName(start_loc=233, end_loc=1, value="b"),
|
||||
TokenKeyword(start_loc=234, end_loc=1, value=b"{"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case2(self):
|
||||
self.run_test(
|
||||
raw=br"""10.32
|
||||
0 +1 -2 .5 1.234
|
||||
-1.234 990 """,
|
||||
expected=[
|
||||
TokenDecimal(start_loc=0, end_loc=5, value=Decimal("10.32")),
|
||||
TokenInteger(start_loc=6, end_loc=1, value=0),
|
||||
TokenInteger(start_loc=8, end_loc=2, value=1),
|
||||
TokenInteger(start_loc=11, end_loc=2, value=-2),
|
||||
TokenDecimal(start_loc=14, end_loc=2, value=Decimal("0.5")),
|
||||
TokenDecimal(start_loc=17, end_loc=5, value=Decimal("1.234")),
|
||||
TokenDecimal(start_loc=23, end_loc=6, value=Decimal("-1.234")),
|
||||
TokenInteger(start_loc=30, end_loc=3, value=990),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case3(self):
|
||||
self.run_test(
|
||||
raw=br"""/b{(c)do*}def""",
|
||||
expected=[
|
||||
TokenName(start_loc=0, end_loc=2, value="b"),
|
||||
TokenKeyword(start_loc=2, end_loc=3, value=b"{"),
|
||||
TokenString(start_loc=4, end_loc=5, value="c"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case4(self):
|
||||
self.run_test(
|
||||
raw=br"""%!PS
|
||||
% This is some secret sauce
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
|
||||
/need /oh1oh
|
||||
func/a/b
|
||||
""",
|
||||
expected=[
|
||||
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
|
||||
TokenComment(
|
||||
start_loc=5, end_loc=27, value=b" This is some secret sauce"
|
||||
),
|
||||
TokenKeyword(start_loc=33, end_loc=5, value=b"begin"),
|
||||
TokenKeyword(start_loc=39, end_loc=3, value=b"end"),
|
||||
TokenKeyword(start_loc=44, end_loc=1, value=b'"'),
|
||||
TokenKeyword(start_loc=47, end_loc=1, value=b"@"),
|
||||
TokenKeyword(start_loc=49, end_loc=1, value=b"#"),
|
||||
TokenName(start_loc=51, end_loc=2, value="a"),
|
||||
TokenName(start_loc=53, end_loc=4, value="BCD"),
|
||||
TokenName(start_loc=58, end_loc=10, value="Some_Name"),
|
||||
TokenName(start_loc=69, end_loc=12, value="foo_naa"),
|
||||
TokenName(start_loc=83, end_loc=7, value="barj"),
|
||||
TokenName(start_loc=90, end_loc=5, value="need"),
|
||||
TokenName(start_loc=96, end_loc=6, value="oh1oh"),
|
||||
TokenKeyword(start_loc=103, end_loc=4, value=b"func"),
|
||||
TokenName(start_loc=107, end_loc=3, value="a"),
|
||||
TokenName(start_loc=109, end_loc=3, value="b"),
|
||||
],
|
||||
)
|
||||
@@ -1,12 +1,12 @@
|
||||
import io
|
||||
from unittest import TestCase
|
||||
from pdfmajor.safebufiterator import EOF, SafeBufferIt
|
||||
from pdfmajor.safebufiterator import EOF, BufferStream
|
||||
|
||||
|
||||
class UseCase(TestCase):
|
||||
def test_safebuff(self):
|
||||
for bufsize in [2, 33, 65, 66, 100, 200]:
|
||||
it = SafeBufferIt(
|
||||
it = BufferStream(
|
||||
io.BytesIO(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
),
|
||||
@@ -16,7 +16,7 @@ class UseCase(TestCase):
|
||||
self.assertLessEqual(len(subbuf), it.buffer_size)
|
||||
|
||||
def test_itraises(self):
|
||||
it = SafeBufferIt(io.BytesIO(b"this"))
|
||||
it = BufferStream(io.BytesIO(b"this"))
|
||||
list(it) # read everything
|
||||
with self.assertRaises(EOF):
|
||||
for _ in it:
|
||||
@@ -24,7 +24,7 @@ class UseCase(TestCase):
|
||||
|
||||
def test_seekd(self):
|
||||
for bufsize in [2, 33, 65, 66, 100, 200]:
|
||||
it = SafeBufferIt(
|
||||
it = BufferStream(
|
||||
io.BytesIO(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
),
|
||||
|
||||
@@ -1,230 +0,0 @@
|
||||
import io
|
||||
from random import randint
|
||||
from decimal import Decimal
|
||||
from typing import List
|
||||
from pdfmajor.tokenizer.token_parsers.string import parse_string
|
||||
from pdfmajor.tokenizer import PSTokenizer
|
||||
from pdfmajor.tokenizer.token import (
|
||||
Token,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
TokenInteger,
|
||||
TokenKeyword,
|
||||
TokenLiteral,
|
||||
TokenString,
|
||||
)
|
||||
from pdfmajor.tokenizer.token_parsers.util import PInput, SafeBufferIt
|
||||
from pdfmajor.tokenizer.token_parsers.comment import parse_comment
|
||||
from pdfmajor.tokenizer.token_parsers.keyword import parse_keyword
|
||||
from pdfmajor.tokenizer.token_parsers.number import parse_number
|
||||
from pdfmajor.tokenizer.token_parsers.literal import parse_literal
|
||||
from unittest import TestCase
|
||||
|
||||
|
||||
def make_stream_iter(data: bytes, initpos: int = 0):
|
||||
stream = io.BytesIO(data)
|
||||
|
||||
def iter_stream(buf_size: int):
|
||||
while True:
|
||||
bufpos = stream.tell()
|
||||
buf = stream.read(buf_size)
|
||||
if buf:
|
||||
yield PInput(initpos + bufpos, buf)
|
||||
else:
|
||||
raise "EOF"
|
||||
|
||||
return iter_stream
|
||||
|
||||
|
||||
class Util(TestCase):
|
||||
def test_safebuff(self):
|
||||
it = SafeBufferIt(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
)
|
||||
for subbuf in it.into_iter():
|
||||
self.assertEqual(len(subbuf), len(it.buffer) - it.skip)
|
||||
it.skip += randint(1, max(len(subbuf) - 1, 1))
|
||||
|
||||
|
||||
class Individual(TestCase):
|
||||
def test_comment_parser_ln(self):
|
||||
for buf_size in [2, 7, 11, 150]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\nso this is not reachable"
|
||||
)
|
||||
token = parse_comment(0, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenComment)
|
||||
self.assertEqual(token.pos, 0)
|
||||
self.assertEqual(
|
||||
token.value, b"this is a lengthy comment that ends here"
|
||||
)
|
||||
self.assertEqual(token.size, 40)
|
||||
|
||||
def test_comment_parser_lr(self):
|
||||
for buf_size in [2, 7, 11, 150]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(
|
||||
b"this is a lengthy comment that ends here\rso this is not reachable"
|
||||
)
|
||||
token = parse_comment(0, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenComment)
|
||||
self.assertEqual(token.pos, 0)
|
||||
self.assertEqual(
|
||||
token.value, b"this is a lengthy comment that ends here"
|
||||
)
|
||||
self.assertEqual(token.size, 40)
|
||||
|
||||
def test_parse_float(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(b"0.120 + some invalid text", 5)
|
||||
token = parse_number(5, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenDecimal)
|
||||
self.assertEqual(token.pos, 5)
|
||||
self.assertEqual(token.value, Decimal("0.12"))
|
||||
self.assertEqual(token.size, 5)
|
||||
|
||||
def test_parse_int(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(
|
||||
b"130 oh oh 0.12 + some invalid text", 11
|
||||
)
|
||||
token = parse_number(11, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenInteger)
|
||||
self.assertEqual(token.pos, 11)
|
||||
self.assertEqual(token.value, 130)
|
||||
self.assertEqual(token.size, 3)
|
||||
|
||||
def test_parse_bool_true(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(b"true and the american way", 91)
|
||||
token = parse_keyword(91, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenBoolean)
|
||||
self.assertEqual(token.pos, 91)
|
||||
self.assertEqual(token.value, True)
|
||||
self.assertEqual(token.size, 4)
|
||||
|
||||
def test_parse_bool_false(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(b"false mannn", 33)
|
||||
token = parse_keyword(33, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenBoolean)
|
||||
self.assertEqual(token.pos, 33)
|
||||
self.assertEqual(token.value, False)
|
||||
self.assertEqual(token.size, 5)
|
||||
|
||||
def test_parse_literals(self):
|
||||
for buf_size in [2, 3, 50, 444]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(br"Some_Name /foo#5f#xbaa", 71)
|
||||
token = parse_literal(71, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenLiteral)
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, "Some_Name")
|
||||
self.assertEqual(token.size, 9)
|
||||
|
||||
def test_parse_literals_hex(self):
|
||||
for buf_size in [2]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(br"foo#5faa ", 71)
|
||||
token = parse_literal(71, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenLiteral)
|
||||
self.assertEqual(token.pos, 71)
|
||||
self.assertEqual(token.value, "foo_aa")
|
||||
self.assertEqual(token.size, 8)
|
||||
|
||||
def test_parse_string(self):
|
||||
for buf_size in [2]:
|
||||
with self.subTest(buf_size=buf_size):
|
||||
iter_stream = make_stream_iter(br"this % is not a comment.) ", 88)
|
||||
token = parse_string(88, iter_stream(buf_size))
|
||||
self.assertIsInstance(token, TokenString)
|
||||
self.assertEqual(token.pos, 88)
|
||||
self.assertEqual(token.value, "this % is not a comment.")
|
||||
self.assertEqual(token.size, 24)
|
||||
|
||||
|
||||
class Tokenizer(TestCase):
|
||||
def run_test(self, raw: bytes, expected: List[Token]):
|
||||
tokenizer = PSTokenizer(io.BytesIO(raw))
|
||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
|
||||
self.assertEqual(token, etoken, f"Failed at example {i}")
|
||||
|
||||
def test_case1(self):
|
||||
tokenizer = PSTokenizer(
|
||||
io.BytesIO(
|
||||
br"""%!PS
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#6eaa
|
||||
0 +1 -2 .5 1.234
|
||||
(abc) () (abc ( def ) ghi)
|
||||
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
|
||||
(this % is not a comment.)
|
||||
(foo
|
||||
baa)
|
||||
(foo\
|
||||
baa)
|
||||
<> <20> < 40 4020 >
|
||||
<abcd00
|
||||
12345>
|
||||
func/a/b{(c)do*}def
|
||||
[ 1 (z) ! ]
|
||||
<< /foo (bar) >>"""
|
||||
)
|
||||
)
|
||||
for token in tokenizer.iter_tokens():
|
||||
print(token)
|
||||
|
||||
def test_case2(self):
|
||||
self.run_test(
|
||||
raw=br"""10.32
|
||||
0 +1 -2 .5 1.234
|
||||
-1.234 990 """,
|
||||
expected=[
|
||||
TokenDecimal(pos=0, size=5, value=Decimal("10.32")),
|
||||
TokenInteger(pos=6, size=1, value=0),
|
||||
TokenInteger(pos=8, size=2, value=1),
|
||||
TokenInteger(pos=11, size=2, value=-2),
|
||||
TokenDecimal(pos=14, size=2, value=Decimal("0.5")),
|
||||
TokenDecimal(pos=17, size=5, value=Decimal("1.234")),
|
||||
TokenDecimal(pos=23, size=6, value=Decimal("-1.234")),
|
||||
TokenInteger(pos=30, size=3, value=990),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case_words1(self):
|
||||
self.run_test(
|
||||
raw=br"""%!PS
|
||||
% This is some secret sauce
|
||||
begin end
|
||||
" @ #
|
||||
/a/BCD /Some_Name /foo#5f#6eaa /bar#6a
|
||||
/need /oh1oh
|
||||
func/a/b
|
||||
""",
|
||||
expected=[
|
||||
TokenComment(pos=0, size=4, value=b"!PS"),
|
||||
TokenComment(pos=5, size=27, value=b" This is some secret sauce"),
|
||||
TokenKeyword(pos=33, size=5, value=b"begin"),
|
||||
TokenKeyword(pos=39, size=3, value=b"end"),
|
||||
TokenKeyword(pos=44, size=1, value=b'"'),
|
||||
TokenKeyword(pos=47, size=1, value=b"@"),
|
||||
TokenKeyword(pos=49, size=1, value=b"#"),
|
||||
TokenLiteral(pos=52, size=1, value="a"),
|
||||
TokenLiteral(pos=54, size=3, value="BCD"),
|
||||
TokenLiteral(pos=59, size=9, value="Some_Name"),
|
||||
TokenLiteral(pos=70, size=11, value="foo_naa"),
|
||||
TokenLiteral(pos=83, size=6, value="barj"),
|
||||
TokenLiteral(pos=91, size=4, value="need"),
|
||||
TokenLiteral(pos=97, size=5, value="oh1oh"),
|
||||
TokenKeyword(pos=103, size=4, value=b"func"),
|
||||
TokenLiteral(pos=108, size=1, value="a"),
|
||||
TokenLiteral(pos=110, size=1, value="b"),
|
||||
],
|
||||
)
|
||||
Reference in new issue
Block a user