From 6e7c1f3888df795a4bbb40ad82a0ed6d37ecc425 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sun, 18 Oct 2020 13:17:07 -0400 Subject: [PATCH] ensuring comments do not break parsing --- pdfmajor/parser_v2/l1.py | 2 +- pdfmajor/parser_v2/l2.py | 9 +++++--- pdfmajor/parser_v2/objects.py | 21 +++++++++++++----- tests/test_parser.py | 42 ++++++++++++++++++++++++++++++++--- 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/pdfmajor/parser_v2/l1.py b/pdfmajor/parser_v2/l1.py index 7db50a5..efdb3fa 100644 --- a/pdfmajor/parser_v2/l1.py +++ b/pdfmajor/parser_v2/l1.py @@ -111,7 +111,7 @@ class PDFL1Parser: else: cur_ctx.pass_item(obj) elif isinstance(token, TokenComment): - yield PDFComment(token) + yield PDFComment(token.value, (token.start_loc, token.end_loc)) elif isinstance(token, TokenKeyword): yield token elif isinstance(token, TokenNull): diff --git a/pdfmajor/parser_v2/l2.py b/pdfmajor/parser_v2/l2.py index e8a98d9..883d3ca 100644 --- a/pdfmajor/parser_v2/l2.py +++ b/pdfmajor/parser_v2/l2.py @@ -2,6 +2,7 @@ import io from pdfmajor.parser_v2.l1 import PDFL1Parser from pdfmajor.parser_v2.exceptions import ParserError from pdfmajor.parser_v2.objects import ( + PDFComment, PDFContextualObject, PDFPrimitive, PDFObject, @@ -45,7 +46,7 @@ class PDFL2Parser: ctx_list: List[PDFContextualObject] = [] for obj in self.l1.iter_objects(): if isinstance(obj, TokenKeyword): - if obj.get_value() == b"obj": + if obj.value == b"obj": if len(object_list) < 2: raise ParserError("recieved 'obj' but missing leading tokens") ctx_list.append( @@ -53,14 +54,14 @@ class PDFL2Parser: *_get_indobj_values(object_list) ) ) - elif obj.get_value() == b"endobj": + elif obj.value == b"endobj": if len(ctx_list) > 0: last_ctx_obj = ctx_list.pop() if len(ctx_list) > 0: ctx_list[-1].pass_item(last_ctx_obj) else: yield last_ctx_obj.get_value() - elif obj.get_value() == b"R": + elif obj.value == b"R": cur_indobj = self.inobjects.get_indobject( *_get_indobj_values(object_list) ) @@ -68,6 +69,8 @@ class PDFL2Parser: ctx_list[-1].pass_item(cur_indobj) else: yield cur_indobj.get_value() + elif isinstance(obj, PDFComment): + yield obj elif len(ctx_list) == 0: object_list.append(obj) elif len(ctx_list) > 0: diff --git a/pdfmajor/parser_v2/objects.py b/pdfmajor/parser_v2/objects.py index 3651a3d..10d0dbe 100644 --- a/pdfmajor/parser_v2/objects.py +++ b/pdfmajor/parser_v2/objects.py @@ -10,7 +10,7 @@ from pdfmajor.lexer.token import ( TokenName, TokenPrimitive, ) -from typing import Any, Dict, Generic, List, Optional, TypeVar +from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar TToken = TypeVar("TToken", bound=TokenPrimitive) @@ -33,6 +33,11 @@ class PDFObject(metaclass=ABCMeta): return False return o.to_python() == self.to_python() + def __repr__(self) -> str: + return "{cls}({data})".format( + cls=self.__class__.__name__, data=self.to_python() + ) + class PDFContextualObject(PDFObject): """An abstract class representing objects that require context for construction""" @@ -58,14 +63,20 @@ class PDFPrimitive(Generic[TToken], PDFObject): class PDFComment(PDFObject): """A class representing a PDF comment""" - def __init__(self, token: TokenComment) -> None: - self.token = token + def __init__(self, comment: bytes, loc: Tuple[int, int] = (0, 0)) -> None: + """ + Args: + comment (bytes): the comment data + loc (Tuple[int, int], optional): location of the comment in the byte-stream. Defaults to (0, 0). + """ + self.comment = comment + self.location = loc def get_value(self): - return self.token.value + return self.comment def to_python(self): - return self.get_value() + return (self.get_value(), self.location) class PDFNull(PDFObject): diff --git a/tests/test_parser.py b/tests/test_parser.py index ede4b99..1edc81d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,9 +1,10 @@ from decimal import Decimal import io +from typing import Callable, List from pdfmajor.lexer.token import TokenInteger, TokenKeyword, TokenString -from pdfmajor.parser_v2.indirect_objects import IndirectObject +from pdfmajor.parser_v2.indirect_objects import IndirectObject, IndirectObjectCollection from pdfmajor.parser_v2.l2 import PDFL2Parser -from pdfmajor.parser_v2.objects import PDFName, PDFPrimitive +from pdfmajor.parser_v2.objects import PDFComment, PDFName, PDFObject, PDFPrimitive from pdfmajor.parser_v2.l1 import PDFL1Parser from pdfmajor.streambuffer import StreamEOF from unittest import TestCase @@ -97,6 +98,22 @@ class L1(TestCase): class L2(TestCase): + def run_test( + self, + raw: bytes, + expected: Callable[[IndirectObjectCollection], List[PDFObject]], + ): + parser = PDFL2Parser(io.BytesIO(raw)) + for i, (eobj, obj) in enumerate( + zip(expected(parser.inobjects), parser.iter_objects()), 1 + ): + self.assertEqual(obj, eobj, f"Failed at example #{i}") + try: + next_obj = next(parser.iter_objects()) + self.assertIsNone(next_obj) + except (EOFError, StopIteration) as e: + self.assertIsNotNone(e) + def test_parse_simple_indobj(self): parser = PDFL2Parser( io.BytesIO( @@ -108,7 +125,7 @@ class L2(TestCase): 13 8 obj << /x 1 /y 3>> endobj - """ + """ ) ) expected = [ @@ -122,3 +139,22 @@ class L2(TestCase): self.assertEqual(obj.to_python(), eobj[2]) with self.assertRaises(StreamEOF): next(parser.iter_objects()) + + def test_parse_comments(self): + self.run_test( + br""" + 10 0 % object references + obj % start of object + (testing) % object content + endobj %closing the object + 11 1 obj 2 endobj + """, + expected=lambda indobjc: [ + PDFComment(b" object references", (22, 41)), + PDFComment(b" start of object", (62, 79)), + PDFComment(b" object content", (110, 126)), + indobjc.get_indobject(10, 0), + PDFComment(b"closing the object", (150, 169)), + indobjc.get_indobject(11, 1), + ], + )