complete lexer
This commit is contained in:
1 parent
24b9c99ea1
commit
472942867b
5 files changed
+137
-76
No files matched your search
@@ -45,10 +45,10 @@ class PDFLexer:
|
||||
yield parse_number(self.buffer, next_char)
|
||||
elif next_char.isdigit() or next_char == b".":
|
||||
self.buffer.seek(bufpos + j)
|
||||
yield parse_number(self.buffer)
|
||||
yield parse_number(self.buffer, initialpos=self.buffer.tell())
|
||||
elif next_char.isalpha():
|
||||
self.buffer.seek(bufpos + j)
|
||||
yield parse_keyword(self.buffer)
|
||||
yield parse_keyword(self.buffer, self.buffer.tell())
|
||||
elif next_char == b"(":
|
||||
self.buffer.seek(bufpos + j + 1)
|
||||
yield parse_string(self.buffer)
|
||||
|
||||
@@ -21,7 +21,7 @@ def parse_double_angled_bracket(
|
||||
first_char = step.data[0:1]
|
||||
if first_char == b"<":
|
||||
buffer.seek(step.pos + 1)
|
||||
return TokenDictionary(initialpos, 2, TDictValue.OPEN)
|
||||
return TokenDictionary(initialpos, buffer.tell(), TDictValue.OPEN)
|
||||
buffer.seek(step.pos)
|
||||
return parse_hexstring(buffer)
|
||||
|
||||
|
||||
@@ -2,20 +2,23 @@ from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.lexer.exceptions import LexerEOF
|
||||
from pdfmajor.lexer.token import TokenBoolean, TokenKeyword
|
||||
from pdfmajor.lexer.regex import END_KEYWORD
|
||||
from typing import Union
|
||||
from typing import Optional, Union
|
||||
|
||||
|
||||
def parse_keyword(buffer: BufferStream) -> Union[TokenKeyword, TokenBoolean]:
|
||||
def parse_keyword(
|
||||
buffer: BufferStream, initialpos: Optional[int] = None
|
||||
) -> Union[TokenKeyword, TokenBoolean]:
|
||||
"""Parses input stream into a keyword or bool
|
||||
|
||||
Args:
|
||||
buffer (BufferStream)
|
||||
initialpos (Optional[int])
|
||||
|
||||
Returns:
|
||||
Union[TokenKeyword, TokenBoolean]
|
||||
"""
|
||||
curtoken = b""
|
||||
initialpos = buffer.tell() - 1
|
||||
initialpos = buffer.tell() - 1 if initialpos is None else initialpos
|
||||
for pos, buf in buffer:
|
||||
m = END_KEYWORD.search(buf, 0)
|
||||
if not m:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from decimal import Decimal
|
||||
from typing import Optional
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.lexer.exceptions import InvalidToken, LexerEOF, LexerError
|
||||
from pdfmajor.lexer.token import (
|
||||
@@ -11,12 +12,15 @@ from pdfmajor.lexer.regex import (
|
||||
)
|
||||
|
||||
|
||||
def parse_number(buffer: BufferStream, sign: bytes = b"+") -> TokenNumber:
|
||||
def parse_number(
|
||||
buffer: BufferStream, sign: bytes = b"+", initialpos: Optional[int] = None
|
||||
) -> TokenNumber:
|
||||
"""Parses input stream into a number
|
||||
|
||||
Args:
|
||||
buffer (BufferStream)
|
||||
sign (str): either a + or -
|
||||
initialpos (Optional[int])
|
||||
|
||||
Returns:
|
||||
TokenNumber
|
||||
@@ -25,7 +29,7 @@ def parse_number(buffer: BufferStream, sign: bytes = b"+") -> TokenNumber:
|
||||
raise LexerError("Number was provided an invalid sign {!r}".format(sign))
|
||||
curtoken = b"" + sign
|
||||
is_decimal = False
|
||||
initialpos = buffer.tell() - 1
|
||||
initialpos = buffer.tell() - 1 if initialpos is None else initialpos
|
||||
for pos, buf in buffer:
|
||||
m = END_NUMBER.search(buf, 0)
|
||||
if not m:
|
||||
|
||||
+122
-68
@@ -13,8 +13,10 @@ from pdfmajor.lexer.name import parse_name
|
||||
from pdfmajor.lexer.keyword import parse_keyword
|
||||
from pdfmajor.streambuffer import BufferStream
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
Token,
|
||||
TokenArray,
|
||||
TokenBoolean,
|
||||
TokenComment,
|
||||
TokenDecimal,
|
||||
@@ -83,6 +85,16 @@ class Basics(TestCase):
|
||||
self.assertEqual(token.value, False)
|
||||
self.assertEqual(token.end_loc, 5)
|
||||
|
||||
def test_parse_word(self):
|
||||
for buf_size in [2, 3, 50]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
buffer = make_stream_iter(b"likeing_turtles ", buf_size)
|
||||
token = parse_keyword(buffer)
|
||||
self.assertIsInstance(token, TokenKeyword)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, b"likeing_turtles")
|
||||
self.assertEqual(token.end_loc, 15)
|
||||
|
||||
def test_parse_literal_names(self):
|
||||
for buf_size in [2, 3, 50, 444]:
|
||||
with self.subTest(buf_end_loc=buf_size):
|
||||
@@ -163,7 +175,7 @@ class Basics(TestCase):
|
||||
self.assertIsInstance(token, TokenDictionary)
|
||||
self.assertEqual(token.start_loc, -1)
|
||||
self.assertEqual(token.value, TDictValue.OPEN)
|
||||
self.assertEqual(token.end_loc, 2)
|
||||
self.assertEqual(token.end_loc, 1)
|
||||
|
||||
def test_parse_dd_to_hex(self):
|
||||
for buf_size in [1, 2, 3, 50, 100]:
|
||||
@@ -201,14 +213,53 @@ class Basics(TestCase):
|
||||
class Lexer(TestCase):
|
||||
def run_test(self, raw: bytes, expected: List[Token]):
|
||||
tokenizer = PDFLexer(io.BytesIO(raw))
|
||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens())):
|
||||
self.assertEqual(token, etoken, f"Failed at example {i}")
|
||||
for i, (etoken, token) in enumerate(zip(expected, tokenizer.iter_tokens()), 1):
|
||||
self.assertEqual(token, etoken, f"Failed at example #{i}")
|
||||
try:
|
||||
next_token = next(tokenizer.iter_tokens())
|
||||
self.assertIsNone(next_token)
|
||||
except LexerError as e:
|
||||
except (EOFError, StopIteration) as e:
|
||||
self.assertIsNotNone(e)
|
||||
|
||||
def test_scase1(self):
|
||||
self.run_test(
|
||||
b"begin end\n",
|
||||
expected=[
|
||||
TokenKeyword(0, 5, b"begin"),
|
||||
TokenKeyword(6, 9, b"end"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_scase2(self):
|
||||
self.run_test(
|
||||
b"/Length 4 0 R\n",
|
||||
expected=[
|
||||
TokenName(0, 7, "Length"),
|
||||
TokenInteger(8, 9, 4),
|
||||
TokenInteger(10, 11, 0),
|
||||
TokenKeyword(12, 13, b"R"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_scase3(self):
|
||||
self.run_test(
|
||||
b"""endstream
|
||||
endobj
|
||||
4 0 obj
|
||||
2618
|
||||
endobj
|
||||
""",
|
||||
expected=[
|
||||
TokenKeyword(0, 9, b"endstream"),
|
||||
TokenKeyword(10, 16, b"endobj"),
|
||||
TokenInteger(17, 18, 4),
|
||||
TokenInteger(19, 20, 0),
|
||||
TokenKeyword(21, 24, b"obj"),
|
||||
TokenInteger(28, 32, 2618),
|
||||
TokenKeyword(33, 39, b"endobj"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case1(self):
|
||||
self.run_test(
|
||||
raw=br"""%!PS
|
||||
@@ -231,41 +282,54 @@ func/a/b{(c)do*}def
|
||||
<< /foo (bar) >>""",
|
||||
expected=[
|
||||
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
|
||||
TokenKeyword(start_loc=5, end_loc=5, value=b"begin"),
|
||||
TokenKeyword(start_loc=11, end_loc=3, value=b"end"),
|
||||
TokenKeyword(start_loc=16, end_loc=1, value=b'"'),
|
||||
TokenKeyword(start_loc=19, end_loc=1, value=b"@"),
|
||||
TokenKeyword(start_loc=21, end_loc=1, value=b"#"),
|
||||
TokenName(start_loc=24, end_loc=1, value="a"),
|
||||
TokenName(start_loc=26, end_loc=3, value="BCD"),
|
||||
TokenName(start_loc=31, end_loc=9, value="Some_Name"),
|
||||
TokenName(start_loc=42, end_loc=11, value="foo_naa"),
|
||||
TokenInteger(start_loc=54, end_loc=1, value=0),
|
||||
TokenInteger(start_loc=56, end_loc=2, value=1),
|
||||
TokenInteger(start_loc=59, end_loc=2, value=-2),
|
||||
TokenDecimal(start_loc=62, end_loc=2, value=Decimal("0.5")),
|
||||
TokenDecimal(start_loc=65, end_loc=5, value=Decimal("1.234")),
|
||||
TokenString(start_loc=71, end_loc=5, value="abc"),
|
||||
TokenString(start_loc=77, end_loc=2, value=""),
|
||||
TokenString(start_loc=80, end_loc=17, value="abc ( def ) ghi"),
|
||||
TokenString(start_loc=98, end_loc=19, value="def 4ghi"),
|
||||
TokenString(start_loc=118, end_loc=13, value=r"bach\slask"),
|
||||
TokenString(start_loc=132, end_loc=10, value="foo\nbaa"),
|
||||
TokenKeyword(start_loc=5, end_loc=10, value=b"begin"),
|
||||
TokenKeyword(start_loc=11, end_loc=14, value=b"end"),
|
||||
TokenKeyword(start_loc=16, end_loc=17, value=b'"'),
|
||||
TokenKeyword(start_loc=19, end_loc=20, value=b"@"),
|
||||
TokenKeyword(start_loc=21, end_loc=22, value=b"#"),
|
||||
TokenName(start_loc=23, end_loc=25, value="a"),
|
||||
TokenName(start_loc=25, end_loc=29, value="BCD"),
|
||||
TokenName(start_loc=30, end_loc=40, value="Some_Name"),
|
||||
TokenName(start_loc=41, end_loc=53, value="foo_naa"),
|
||||
TokenInteger(start_loc=54, end_loc=55, value=0),
|
||||
TokenInteger(start_loc=56, end_loc=58, value=1),
|
||||
TokenInteger(start_loc=59, end_loc=61, value=-2),
|
||||
TokenDecimal(start_loc=62, end_loc=64, value=Decimal("0.5")),
|
||||
TokenDecimal(start_loc=65, end_loc=70, value=Decimal("1.234")),
|
||||
TokenString(start_loc=71, end_loc=76, value="abc"),
|
||||
TokenString(start_loc=77, end_loc=79, value=""),
|
||||
TokenString(start_loc=80, end_loc=97, value="abc ( def ) ghi"),
|
||||
TokenString(start_loc=98, end_loc=117, value="def 4ghi"),
|
||||
TokenString(start_loc=118, end_loc=131, value=r"bach\slask"),
|
||||
TokenString(start_loc=132, end_loc=142, value="foo\nbaa"),
|
||||
TokenString(
|
||||
start_loc=143, end_loc=26, value="this % is not a comment."
|
||||
start_loc=143, end_loc=169, value="this % is not a comment."
|
||||
),
|
||||
TokenString(start_loc=170, end_loc=9, value="foo\nbaa"),
|
||||
TokenString(start_loc=180, end_loc=10, value="foobaa"),
|
||||
TokenHexString(start_loc=191, end_loc=2, value=b""),
|
||||
TokenHexString(start_loc=194, end_loc=4, value=b" "),
|
||||
TokenHexString(start_loc=199, end_loc=11, value=b"@@ "),
|
||||
TokenString(start_loc=170, end_loc=179, value="foo\nbaa"),
|
||||
TokenString(start_loc=180, end_loc=190, value="foobaa"),
|
||||
TokenHexString(start_loc=191, end_loc=193, value=b""),
|
||||
TokenHexString(start_loc=194, end_loc=198, value=b" "),
|
||||
TokenHexString(start_loc=199, end_loc=210, value=b"@@ "),
|
||||
TokenHexString(
|
||||
start_loc=211, end_loc=14, value=b"\xab\xcd\x00\x124\x05"
|
||||
start_loc=211, end_loc=225, value=b"\xab\xcd\x00\x124\x05"
|
||||
),
|
||||
TokenKeyword(start_loc=226, end_loc=4, value=b"func"),
|
||||
TokenName(start_loc=231, end_loc=1, value="a"),
|
||||
TokenName(start_loc=233, end_loc=1, value="b"),
|
||||
TokenKeyword(start_loc=234, end_loc=1, value=b"{"),
|
||||
TokenKeyword(start_loc=226, end_loc=230, value=b"func"),
|
||||
TokenName(start_loc=230, end_loc=232, value="a"),
|
||||
TokenName(start_loc=232, end_loc=234, value="b"),
|
||||
TokenKeyword(start_loc=234, end_loc=235, value=b"{"),
|
||||
TokenString(start_loc=235, end_loc=238, value="c"),
|
||||
TokenKeyword(start_loc=238, end_loc=241, value=b"do*"),
|
||||
TokenKeyword(start_loc=241, end_loc=242, value=b"}"),
|
||||
TokenKeyword(start_loc=242, end_loc=245, value=b"def"),
|
||||
TokenArray(start_loc=246, end_loc=247, value=TArrayValue.OPEN),
|
||||
TokenInteger(start_loc=248, end_loc=249, value=1),
|
||||
TokenString(start_loc=250, end_loc=253, value="z"),
|
||||
TokenKeyword(start_loc=254, end_loc=255, value=b"!"),
|
||||
TokenArray(start_loc=256, end_loc=257, value=TArrayValue.CLOSE),
|
||||
TokenDictionary(start_loc=258, end_loc=260, value=TDictValue.OPEN),
|
||||
TokenName(start_loc=261, end_loc=265, value="foo"),
|
||||
TokenString(start_loc=266, end_loc=271, value="bar"),
|
||||
TokenDictionary(start_loc=273, end_loc=274, value=TDictValue.CLOSE),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -276,27 +340,17 @@ func/a/b{(c)do*}def
|
||||
-1.234 990 """,
|
||||
expected=[
|
||||
TokenDecimal(start_loc=0, end_loc=5, value=Decimal("10.32")),
|
||||
TokenInteger(start_loc=6, end_loc=1, value=0),
|
||||
TokenInteger(start_loc=8, end_loc=2, value=1),
|
||||
TokenInteger(start_loc=11, end_loc=2, value=-2),
|
||||
TokenDecimal(start_loc=14, end_loc=2, value=Decimal("0.5")),
|
||||
TokenDecimal(start_loc=17, end_loc=5, value=Decimal("1.234")),
|
||||
TokenDecimal(start_loc=23, end_loc=6, value=Decimal("-1.234")),
|
||||
TokenInteger(start_loc=30, end_loc=3, value=990),
|
||||
TokenInteger(start_loc=6, end_loc=7, value=0),
|
||||
TokenInteger(start_loc=8, end_loc=10, value=1),
|
||||
TokenInteger(start_loc=11, end_loc=13, value=-2),
|
||||
TokenDecimal(start_loc=14, end_loc=16, value=Decimal("0.5")),
|
||||
TokenDecimal(start_loc=17, end_loc=22, value=Decimal("1.234")),
|
||||
TokenDecimal(start_loc=23, end_loc=29, value=Decimal("-1.234")),
|
||||
TokenInteger(start_loc=30, end_loc=33, value=990),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case3(self):
|
||||
self.run_test(
|
||||
raw=br"""/b{(c)do*}def""",
|
||||
expected=[
|
||||
TokenName(start_loc=0, end_loc=2, value="b"),
|
||||
TokenKeyword(start_loc=2, end_loc=3, value=b"{"),
|
||||
TokenString(start_loc=4, end_loc=5, value="c"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_case4(self):
|
||||
self.run_test(
|
||||
raw=br"""%!PS
|
||||
% This is some secret sauce
|
||||
@@ -309,22 +363,22 @@ func/a/b
|
||||
expected=[
|
||||
TokenComment(start_loc=0, end_loc=4, value=b"!PS"),
|
||||
TokenComment(
|
||||
start_loc=5, end_loc=27, value=b" This is some secret sauce"
|
||||
start_loc=5, end_loc=32, value=b" This is some secret sauce"
|
||||
),
|
||||
TokenKeyword(start_loc=33, end_loc=5, value=b"begin"),
|
||||
TokenKeyword(start_loc=39, end_loc=3, value=b"end"),
|
||||
TokenKeyword(start_loc=44, end_loc=1, value=b'"'),
|
||||
TokenKeyword(start_loc=47, end_loc=1, value=b"@"),
|
||||
TokenKeyword(start_loc=49, end_loc=1, value=b"#"),
|
||||
TokenName(start_loc=51, end_loc=2, value="a"),
|
||||
TokenName(start_loc=53, end_loc=4, value="BCD"),
|
||||
TokenName(start_loc=58, end_loc=10, value="Some_Name"),
|
||||
TokenName(start_loc=69, end_loc=12, value="foo_naa"),
|
||||
TokenName(start_loc=83, end_loc=7, value="barj"),
|
||||
TokenName(start_loc=90, end_loc=5, value="need"),
|
||||
TokenName(start_loc=96, end_loc=6, value="oh1oh"),
|
||||
TokenKeyword(start_loc=103, end_loc=4, value=b"func"),
|
||||
TokenName(start_loc=107, end_loc=3, value="a"),
|
||||
TokenName(start_loc=109, end_loc=3, value="b"),
|
||||
TokenKeyword(start_loc=33, end_loc=38, value=b"begin"),
|
||||
TokenKeyword(start_loc=39, end_loc=42, value=b"end"),
|
||||
TokenKeyword(start_loc=44, end_loc=45, value=b'"'),
|
||||
TokenKeyword(start_loc=47, end_loc=48, value=b"@"),
|
||||
TokenKeyword(start_loc=49, end_loc=50, value=b"#"),
|
||||
TokenName(start_loc=51, end_loc=53, value="a"),
|
||||
TokenName(start_loc=53, end_loc=57, value="BCD"),
|
||||
TokenName(start_loc=58, end_loc=68, value="Some_Name"),
|
||||
TokenName(start_loc=69, end_loc=81, value="foo_naa"),
|
||||
TokenName(start_loc=82, end_loc=89, value="barj"),
|
||||
TokenName(start_loc=90, end_loc=95, value="need"),
|
||||
TokenName(start_loc=96, end_loc=102, value="oh1oh"),
|
||||
TokenKeyword(start_loc=103, end_loc=107, value=b"func"),
|
||||
TokenName(start_loc=107, end_loc=109, value="a"),
|
||||
TokenName(start_loc=109, end_loc=111, value="b"),
|
||||
],
|
||||
)
|
||||
Reference in new issue
Block a user