added stream parsing

This commit is contained in:
ari committed 2020-10-21 16:32:18 -04:00
1 parent 39501b174b
commit 37e567a352
6 files changed
+294 -177

No files matched your search

+148
View File
@@ -0,0 +1,148 @@
import io
from pdfmajor.parser.objects.stream import PDFStream
from typing import Iterator, Optional
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenComment,
TokenDictionary,
TokenKeyword,
)
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import InvalidKeywordPos, ParserError
from .objects.base import PDFObject
from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment
from .objects.indirect import IndirectObject
class PDFParser:
"""PDF Parser"""
def __init__(
self,
fp: io.BufferedIOBase,
buffer_size: int = 4096,
state: Optional[ParsingState] = None,
strict: bool = True,
retain_obj_on_seek: bool = False,
) -> None:
"""
Args:
fp (io.BufferedIOBase)
buffer_size (int, optional): the size of the reads. Defaults to 4096.
state (Optional[ParsingState], optional): the parsing state. Defaults to None.
strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True.
retain_obj_on_seek (bool, optional): whether we want to ensure we keep the object collection when changing the offset using the self.seek methodm this does not apply if you specify the state when using the seek method!. Defaults to False.
"""
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.state = ParsingState([], []) if state is None else state
self.retain_obj_on_seek = retain_obj_on_seek
def seek(self, offset: int, state: Optional[ParsingState] = None) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
state (Optional[ParsingState]): this will override our current state
Returns:
int: new offset
"""
self.state = (
ParsingState(
[],
[],
indirect_object_collection=self.state.indirect_object_collection
if not self.retain_obj_on_seek
else None,
)
if state is None
else state
)
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
attempt_prim = attempt_parse_prim(token, self.state)
if attempt_prim.success:
cur_ctx = self.state.current_context
for obj in attempt_prim.next_objs:
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
elif isinstance(token, TokenComment):
yield self.state.set_last_obj(
PDFComment(
token.value,
(token.start_loc, token.end_loc),
)
)
elif isinstance(token, TokenKeyword):
if token.value == b"R":
obj = self.state.get_indobjects()
cur_ctx = self.state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
elif token.value == b"obj":
self.state.initialize_indirect_obj()
elif token.value == b"endobj":
last_ctx = self.state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
yield last_ctx.clone()
elif self.strict:
raise InvalidKeywordPos(token)
elif token.value == b"stream":
last_ctx = self.state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
stream = PDFStream(token.end_loc, 0)
stream.pass_item(last_ctx.get_object())
last_ctx.save_object(PDFStream(token.end_loc, 0))
self.state.context_stack.append(stream)
self.lexer.seek(stream.offset + stream.length)
else:
raise ParserError(
f"PDFStream is missing initilization dictionary"
)
elif token.value == b"endstream":
stream = self.state.context_stack.pop()
if not isinstance(stream, PDFStream):
raise InvalidKeywordPos(token)
else:
raise InvalidKeywordPos(token)
elif isinstance(token, TokenArray):
for obj in self.state.flush_int_collections():
yield obj
obj = deal_with_collection_object(
self.state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
for obj in self.state.flush_int_collections():
yield obj
obj = deal_with_collection_object(
self.state,
token=token,
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
strict=self.strict,
)
if obj is not None:
yield obj
+29
View File
@@ -0,0 +1,29 @@
import io
from .objects.stream import PDFStream
from .state import ParsingState
from .PDFParser import PDFParser
from .fliters import process_filters_on_data
class PDFStreamPasrser(PDFParser):
def __init__(
self,
stream: PDFStream,
fp: io.BufferedIOBase,
state: ParsingState,
buffer_size: int = 4096,
strict: bool = True,
) -> None:
fp.seek(stream.offset)
data = fp.read(stream.length)
data = process_filters_on_data(data, stream.filter, stream.decode_parms)
data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms)
self.stream = stream
super().__init__(
fp=io.BytesIO(data),
buffer_size=buffer_size,
state=state,
strict=strict,
retain_obj_on_seek=True,
)
-126
View File
@@ -1,126 +0,0 @@
import io
from pdfmajor.parser.objects.stream import PDFStream
from typing import Iterator, Optional
from pdfmajor.lexer import PDFLexer
from pdfmajor.lexer.token import (
TArrayValue,
TDictValue,
TokenArray,
TokenComment,
TokenDictionary,
TokenKeyword,
)
from .state import ParsingState
from .parsers import attempt_parse_prim, deal_with_collection_object
from .exceptions import InvalidKeywordPos, ParserError
from .objects.base import PDFObject
from .objects.collections import PDFArray, PDFDictionary
from .objects.comment import PDFComment
from .objects.indirect import IndirectObject
class PDFParser:
"""PDF Parser"""
def __init__(
self,
fp: io.BufferedIOBase,
buffer_size: int = 4096,
state: Optional[ParsingState] = None,
strict: bool = True,
) -> None:
self.lexer = PDFLexer(fp, buffer_size)
self.strict = strict
self.state = ParsingState([], []) if state is None else state
def seek(self, offset: int, state: Optional[ParsingState] = None) -> int:
"""moves the offset of the reader to a certain spot, and returns the new offset
Args:
offset (int)
Returns:
int: new offset
"""
self.state = ParsingState([], []) if state is None else state
return self.lexer.seek(offset)
def iter_objects(self) -> Iterator[PDFObject]:
"""Iterates over the objects in the current byte stream using level 1 parsing
Yields:
PDFObject
"""
for token in self.lexer.iter_tokens():
attempt_prim = attempt_parse_prim(token, self.state)
if attempt_prim.success:
cur_ctx = self.state.current_context
for obj in attempt_prim.next_objs:
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
elif isinstance(token, TokenComment):
yield self.state.set_last_obj(
PDFComment(
token.value,
(token.start_loc, token.end_loc),
)
)
elif isinstance(token, TokenKeyword):
if token.value == b"R":
obj = self.state.get_indobjects()
cur_ctx = self.state.current_context
if cur_ctx is not None:
cur_ctx.pass_item(obj)
else:
yield obj
elif token.value == b"obj":
self.state.initialize_indirect_obj()
elif token.value == b"endobj":
last_ctx = self.state.context_stack.pop()
if isinstance(last_ctx, IndirectObject):
yield last_ctx.clone()
elif self.strict:
raise InvalidKeywordPos(token)
elif token.value == b"stream":
last_ctx = self.state.current_context
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
stream = PDFStream(token.end_loc, 0)
stream.pass_item(last_ctx.get_object())
last_ctx.save_object(PDFStream(token.end_loc, 0))
self.state.context_stack.append(stream)
self.lexer.seek(stream.offset + stream.length)
else:
raise ParserError(
f"PDFStream is missing initilization dictionary"
)
elif token.value == b"endstream":
stream = self.state.context_stack.pop()
if not isinstance(stream, PDFStream):
raise InvalidKeywordPos(token)
else:
raise InvalidKeywordPos(token)
elif isinstance(token, TokenArray):
for obj in self.state.flush_int_collections():
yield obj
obj = deal_with_collection_object(
self.state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
)
if obj is not None:
yield obj
elif isinstance(token, TokenDictionary):
for obj in self.state.flush_int_collections():
yield obj
obj = deal_with_collection_object(
self.state,
token=token,
cls_const=PDFDictionary,
open_t=TDictValue.OPEN,
strict=self.strict,
)
if obj is not None:
yield obj
+82 -1
View File
@@ -1,7 +1,9 @@
from typing import Any
import zlib
from typing import Any, Dict, List, Optional
from base64 import a85decode
from binascii import a2b_hex
from ..exceptions import InvalidDecoderOrNotImplemented
from .types import FilterType
from .lzw import lzwdecode
@@ -10,7 +12,86 @@ from .ccit import ccittfaxdecode
from ..exceptions import InvalidDecoderOrNotImplemented
_filter_type_name_mapping: Dict[str, FilterType] = {
"FlateDecode": FilterType.FlateDecode,
"Fl": FilterType.FlateDecode,
"LZWDecode": FilterType.LZWDecode,
"LZW": FilterType.LZWDecode,
"ASCII85Decode": FilterType.ASCII85Decode,
"A85": FilterType.ASCII85Decode,
"ASCIIHexDecode": FilterType.ASCIIHexDecode,
"AHx": FilterType.ASCIIHexDecode,
"RunLengthDecode": FilterType.RunLengthDecode,
"RL": FilterType.RunLengthDecode,
"CCITTFaxDecode": FilterType.CCITTFaxDecode,
"CCF": FilterType.CCITTFaxDecode,
"DCTDecode": FilterType.DCTDecode,
"DCT": FilterType.DCTDecode,
"Crypt": FilterType.Crypt,
"JBIG2Decode": FilterType.JBIG2Decode,
"JPXDecode": FilterType.JPXDecode,
}
def process_filters_on_data(
data: bytes,
filters: Optional[List[str]],
decode_params: Optional[List[Dict[str, Any]]],
) -> bytes:
"""runs the filters provided by the list with the provided params against the data
Args:
data (bytes)
filters (Optional[List[str]])
decode_params (Optional[List[Dict[str, Any]]])
Returns:
bytes
"""
if filters is not None:
decode_parms = []
if decode_params is not None:
decode_parms = decode_params
for f in filters:
params = {}
if len(decode_parms) > 0:
params = decode_parms.pop()
filter_type = detect_filter_type(f.value)
data = decode_bytes(filter_type, data, **params)
return data
def detect_filter_type(name: str) -> FilterType:
"""Check to see if the specified filter exists in our codebase
Args:
name (str)
Raises:
InvalidDecoderOrNotImplemented
Returns:
FilterType
"""
ftype = _filter_type_name_mapping.get(name, None)
if ftype is None:
raise InvalidDecoderOrNotImplemented(name)
return ftype
def decode_bytes(filter_type: FilterType, data: bytes, **kwrgs: Any) -> bytes:
"""Run the decoding function on the filter
Args:
filter_type (FilterType)
data (bytes)
Raises:
InvalidDecoderOrNotImplemented
Returns:
bytes
"""
if filter_type == FilterType.ASCII85Decode:
return a85decode(data, adobe=True)
elif filter_type == FilterType.ASCIIHexDecode:
-30
View File
@@ -1,6 +1,4 @@
import enum
from typing import Dict
from ..exceptions import InvalidDecoderOrNotImplemented
class FilterType(enum.Enum):
@@ -16,31 +14,3 @@ class FilterType(enum.Enum):
DCTDecode = enum.auto()
JPXDecode = enum.auto()
Crypt = enum.auto()
_filter_type_name_mapping: Dict[str, FilterType] = {
"FlateDecode": FilterType.FlateDecode,
"Fl": FilterType.FlateDecode,
"LZWDecode": FilterType.LZWDecode,
"LZW": FilterType.LZWDecode,
"ASCII85Decode": FilterType.ASCII85Decode,
"A85": FilterType.ASCII85Decode,
"ASCIIHexDecode": FilterType.ASCIIHexDecode,
"AHx": FilterType.ASCIIHexDecode,
"RunLengthDecode": FilterType.RunLengthDecode,
"RL": FilterType.RunLengthDecode,
"CCITTFaxDecode": FilterType.CCITTFaxDecode,
"CCF": FilterType.CCITTFaxDecode,
"DCTDecode": FilterType.DCTDecode,
"DCT": FilterType.DCTDecode,
"Crypt": FilterType.Crypt,
"JBIG2Decode": FilterType.JBIG2Decode,
"JPXDecode": FilterType.JPXDecode,
}
def detect_filter_type(name: str) -> FilterType:
ftype = _filter_type_name_mapping.get(name, None)
if ftype is None:
raise InvalidDecoderOrNotImplemented(name)
return ftype
+35 -20
View File
@@ -1,5 +1,5 @@
from pdfmajor.parser.exceptions import ParserError
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Union
from .base import PDFContextualObject, PDFObject
from .primitives import PDFName
from .collections import PDFDictionary
@@ -12,11 +12,11 @@ class PDFStream(PDFContextualObject):
self,
offset: int = 0,
length: int = 0,
filter: Optional[List[PDFName]] = None,
decode_parms: Optional[List[PDFDictionary]] = None,
filter: Optional[List[str]] = None,
decode_parms: Optional[List[Dict[str, Any]]] = None,
f: Optional[Any] = None, # file-specifications
ffilter: Optional[List[PDFName]] = None,
fdecode_parms: Optional[List[PDFDictionary]] = None,
ffilter: Optional[List[str]] = None,
fdecode_parms: Optional[List[Dict[str, Any]]] = None,
dl: Optional[int] = None,
other_metadata: Optional[Dict[str, Any]] = None,
) -> None:
@@ -24,33 +24,39 @@ class PDFStream(PDFContextualObject):
Args:
offset (int): the offset where the stream itself starts
length (int): the length of the stream
filter (Optional[List[PDFName]], optional): The names of a filter that shall be applied in processing the stream data. Defaults to None.
decode_parms (Optional[List[PDFDictionary]], optional): Parameter dictionaries used by the filters specified by filter. Defaults to None.
filter (Optional[List[str]], optional): The names of a filter that shall be applied in processing the stream data. Defaults to None.
decode_parms (Optional[List[Dict[str, Any]]], optional): Parameter dictionaries used by the filters specified by filter. Defaults to None.
f (Optional[Any], optional): The file containing the stream data. If this entry is present, the bytes between stream and endstream shall be ignored. However, the Length entry should still specify the number of those bytes (usually, there are no bytes and Length is 0).. Defaults to None.
ffilter (Optional[List[PDFName]], optional): The names of a filter that shall be applied in processing of the external file. Defaults to None.
fdecode_parms (Optional[List[PDFDictionary]], optional): Parameter dictionaries used by the filters specified by ffilter. Defaults to None.
ffilter (Optional[List[str]], optional): The names of a filter that shall be applied in processing of the external file. Defaults to None.
fdecode_parms (Optional[List[Dict[str, Any]]], optional): Parameter dictionaries used by the filters specified by ffilter. Defaults to None.
dl (Optional[int], optional): A non-negative integer representing the number of bytes in the decoded (defiltered) stream. It can be used to determine, for example, whether enough disk space is available to write a stream to a file.This value shall be considered a hint only; for some stream filters, it may not be possible to determine this value precisely. Defaults to None.
"""
self.offset = offset
self.length = length
self.filter = filter
self.decode_parms = decode_parms
self.f = f
self.ffilter = ffilter
self.fdecode_parms = fdecode_parms
self.dl = dl
self.other_metadata = other_metadata if other_metadata is not None else {}
self.filter: Optional[List[str]] = filter
self.decode_parms: Optional[List[Dict[str, Any]]] = decode_parms
self.f: Optional[Any] = f
self.ffilter: Optional[List[str]] = ffilter
self.fdecode_parms: Optional[List[Dict[str, Any]]] = fdecode_parms
self.dl: Optional[int] = dl
self.other_metadata: Dict[str, Any] = (
other_metadata if other_metadata is not None else {}
)
def pass_item(self, item: PDFObject) -> None:
if not isinstance(item, PDFDictionary):
raise ParserError(f"Cannot initilize a pdfstream with {type(item)}")
data = item.to_python()
self.length = data["Length"]
self.filter = data.get("Filter", self.filter)
self.decode_parms = data.get("DecodeParms", self.decode_parms)
self.filter = get_single_or_list(data.get("Filter", self.filter))
self.decode_parms = get_single_or_list(
data.get("DecodeParms", self.decode_parms)
)
self.f = data.get("F", self.f)
self.ffilter = data.get("FFilter", self.ffilter)
self.fdecode_parms = data.get("FDecodeParms", self.fdecode_parms)
self.ffilter = get_single_or_list(data.get("FFilter", self.ffilter))
self.fdecode_parms = get_single_or_list(
data.get("FDecodeParms", self.fdecode_parms)
)
self.other_metadata = data
def to_python(self):
@@ -68,3 +74,12 @@ class PDFStream(PDFContextualObject):
else [x.to_python() for x in self.fdecode_parms],
"dl": self.dl,
}
def get_single_or_list(obj: Optional[Union[List[Any], Any]]) -> List[Any]:
if obj is None:
return []
elif isinstance(obj, list):
return obj
else:
return [obj]