added stream parsing
This commit is contained in:
1 parent
39501b174b
commit
37e567a352
6 files changed
+294
-177
No files matched your search
@@ -0,0 +1,148 @@
|
||||
import io
|
||||
from pdfmajor.parser.objects.stream import PDFStream
|
||||
from typing import Iterator, Optional
|
||||
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
TokenArray,
|
||||
TokenComment,
|
||||
TokenDictionary,
|
||||
TokenKeyword,
|
||||
)
|
||||
|
||||
from .state import ParsingState
|
||||
from .parsers import attempt_parse_prim, deal_with_collection_object
|
||||
from .exceptions import InvalidKeywordPos, ParserError
|
||||
|
||||
from .objects.base import PDFObject
|
||||
from .objects.collections import PDFArray, PDFDictionary
|
||||
from .objects.comment import PDFComment
|
||||
from .objects.indirect import IndirectObject
|
||||
|
||||
|
||||
class PDFParser:
|
||||
"""PDF Parser"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fp: io.BufferedIOBase,
|
||||
buffer_size: int = 4096,
|
||||
state: Optional[ParsingState] = None,
|
||||
strict: bool = True,
|
||||
retain_obj_on_seek: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
|
||||
Args:
|
||||
fp (io.BufferedIOBase)
|
||||
buffer_size (int, optional): the size of the reads. Defaults to 4096.
|
||||
state (Optional[ParsingState], optional): the parsing state. Defaults to None.
|
||||
strict (bool, optional): whether we want to throw errors over common decoding issues. Defaults to True.
|
||||
retain_obj_on_seek (bool, optional): whether we want to ensure we keep the object collection when changing the offset using the self.seek methodm this does not apply if you specify the state when using the seek method!. Defaults to False.
|
||||
"""
|
||||
self.lexer = PDFLexer(fp, buffer_size)
|
||||
self.strict = strict
|
||||
self.state = ParsingState([], []) if state is None else state
|
||||
self.retain_obj_on_seek = retain_obj_on_seek
|
||||
|
||||
def seek(self, offset: int, state: Optional[ParsingState] = None) -> int:
|
||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||
|
||||
Args:
|
||||
offset (int)
|
||||
state (Optional[ParsingState]): this will override our current state
|
||||
|
||||
Returns:
|
||||
int: new offset
|
||||
"""
|
||||
self.state = (
|
||||
ParsingState(
|
||||
[],
|
||||
[],
|
||||
indirect_object_collection=self.state.indirect_object_collection
|
||||
if not self.retain_obj_on_seek
|
||||
else None,
|
||||
)
|
||||
if state is None
|
||||
else state
|
||||
)
|
||||
return self.lexer.seek(offset)
|
||||
|
||||
def iter_objects(self) -> Iterator[PDFObject]:
|
||||
"""Iterates over the objects in the current byte stream
|
||||
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
for token in self.lexer.iter_tokens():
|
||||
attempt_prim = attempt_parse_prim(token, self.state)
|
||||
if attempt_prim.success:
|
||||
cur_ctx = self.state.current_context
|
||||
for obj in attempt_prim.next_objs:
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
yield obj
|
||||
elif isinstance(token, TokenComment):
|
||||
yield self.state.set_last_obj(
|
||||
PDFComment(
|
||||
token.value,
|
||||
(token.start_loc, token.end_loc),
|
||||
)
|
||||
)
|
||||
elif isinstance(token, TokenKeyword):
|
||||
if token.value == b"R":
|
||||
obj = self.state.get_indobjects()
|
||||
cur_ctx = self.state.current_context
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
yield obj
|
||||
elif token.value == b"obj":
|
||||
self.state.initialize_indirect_obj()
|
||||
elif token.value == b"endobj":
|
||||
last_ctx = self.state.context_stack.pop()
|
||||
if isinstance(last_ctx, IndirectObject):
|
||||
yield last_ctx.clone()
|
||||
elif self.strict:
|
||||
raise InvalidKeywordPos(token)
|
||||
elif token.value == b"stream":
|
||||
last_ctx = self.state.current_context
|
||||
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
|
||||
stream = PDFStream(token.end_loc, 0)
|
||||
stream.pass_item(last_ctx.get_object())
|
||||
last_ctx.save_object(PDFStream(token.end_loc, 0))
|
||||
self.state.context_stack.append(stream)
|
||||
self.lexer.seek(stream.offset + stream.length)
|
||||
else:
|
||||
raise ParserError(
|
||||
f"PDFStream is missing initilization dictionary"
|
||||
)
|
||||
elif token.value == b"endstream":
|
||||
stream = self.state.context_stack.pop()
|
||||
if not isinstance(stream, PDFStream):
|
||||
raise InvalidKeywordPos(token)
|
||||
else:
|
||||
raise InvalidKeywordPos(token)
|
||||
elif isinstance(token, TokenArray):
|
||||
for obj in self.state.flush_int_collections():
|
||||
yield obj
|
||||
obj = deal_with_collection_object(
|
||||
self.state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
|
||||
)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
elif isinstance(token, TokenDictionary):
|
||||
for obj in self.state.flush_int_collections():
|
||||
yield obj
|
||||
obj = deal_with_collection_object(
|
||||
self.state,
|
||||
token=token,
|
||||
cls_const=PDFDictionary,
|
||||
open_t=TDictValue.OPEN,
|
||||
strict=self.strict,
|
||||
)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
@@ -0,0 +1,29 @@
|
||||
import io
|
||||
|
||||
from .objects.stream import PDFStream
|
||||
from .state import ParsingState
|
||||
from .PDFParser import PDFParser
|
||||
from .fliters import process_filters_on_data
|
||||
|
||||
|
||||
class PDFStreamPasrser(PDFParser):
|
||||
def __init__(
|
||||
self,
|
||||
stream: PDFStream,
|
||||
fp: io.BufferedIOBase,
|
||||
state: ParsingState,
|
||||
buffer_size: int = 4096,
|
||||
strict: bool = True,
|
||||
) -> None:
|
||||
fp.seek(stream.offset)
|
||||
data = fp.read(stream.length)
|
||||
data = process_filters_on_data(data, stream.filter, stream.decode_parms)
|
||||
data = process_filters_on_data(data, stream.ffilter, stream.fdecode_parms)
|
||||
self.stream = stream
|
||||
super().__init__(
|
||||
fp=io.BytesIO(data),
|
||||
buffer_size=buffer_size,
|
||||
state=state,
|
||||
strict=strict,
|
||||
retain_obj_on_seek=True,
|
||||
)
|
||||
@@ -1,126 +0,0 @@
|
||||
import io
|
||||
from pdfmajor.parser.objects.stream import PDFStream
|
||||
from typing import Iterator, Optional
|
||||
|
||||
from pdfmajor.lexer import PDFLexer
|
||||
from pdfmajor.lexer.token import (
|
||||
TArrayValue,
|
||||
TDictValue,
|
||||
TokenArray,
|
||||
TokenComment,
|
||||
TokenDictionary,
|
||||
TokenKeyword,
|
||||
)
|
||||
|
||||
from .state import ParsingState
|
||||
from .parsers import attempt_parse_prim, deal_with_collection_object
|
||||
from .exceptions import InvalidKeywordPos, ParserError
|
||||
|
||||
from .objects.base import PDFObject
|
||||
from .objects.collections import PDFArray, PDFDictionary
|
||||
from .objects.comment import PDFComment
|
||||
from .objects.indirect import IndirectObject
|
||||
|
||||
|
||||
class PDFParser:
|
||||
"""PDF Parser"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fp: io.BufferedIOBase,
|
||||
buffer_size: int = 4096,
|
||||
state: Optional[ParsingState] = None,
|
||||
strict: bool = True,
|
||||
) -> None:
|
||||
self.lexer = PDFLexer(fp, buffer_size)
|
||||
self.strict = strict
|
||||
self.state = ParsingState([], []) if state is None else state
|
||||
|
||||
def seek(self, offset: int, state: Optional[ParsingState] = None) -> int:
|
||||
"""moves the offset of the reader to a certain spot, and returns the new offset
|
||||
|
||||
Args:
|
||||
offset (int)
|
||||
|
||||
Returns:
|
||||
int: new offset
|
||||
"""
|
||||
self.state = ParsingState([], []) if state is None else state
|
||||
return self.lexer.seek(offset)
|
||||
|
||||
def iter_objects(self) -> Iterator[PDFObject]:
|
||||
"""Iterates over the objects in the current byte stream using level 1 parsing
|
||||
|
||||
Yields:
|
||||
PDFObject
|
||||
"""
|
||||
for token in self.lexer.iter_tokens():
|
||||
attempt_prim = attempt_parse_prim(token, self.state)
|
||||
if attempt_prim.success:
|
||||
cur_ctx = self.state.current_context
|
||||
for obj in attempt_prim.next_objs:
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
yield obj
|
||||
elif isinstance(token, TokenComment):
|
||||
yield self.state.set_last_obj(
|
||||
PDFComment(
|
||||
token.value,
|
||||
(token.start_loc, token.end_loc),
|
||||
)
|
||||
)
|
||||
elif isinstance(token, TokenKeyword):
|
||||
if token.value == b"R":
|
||||
obj = self.state.get_indobjects()
|
||||
cur_ctx = self.state.current_context
|
||||
if cur_ctx is not None:
|
||||
cur_ctx.pass_item(obj)
|
||||
else:
|
||||
yield obj
|
||||
elif token.value == b"obj":
|
||||
self.state.initialize_indirect_obj()
|
||||
elif token.value == b"endobj":
|
||||
last_ctx = self.state.context_stack.pop()
|
||||
if isinstance(last_ctx, IndirectObject):
|
||||
yield last_ctx.clone()
|
||||
elif self.strict:
|
||||
raise InvalidKeywordPos(token)
|
||||
elif token.value == b"stream":
|
||||
last_ctx = self.state.current_context
|
||||
if last_ctx is not None and isinstance(last_ctx, IndirectObject):
|
||||
stream = PDFStream(token.end_loc, 0)
|
||||
stream.pass_item(last_ctx.get_object())
|
||||
last_ctx.save_object(PDFStream(token.end_loc, 0))
|
||||
self.state.context_stack.append(stream)
|
||||
self.lexer.seek(stream.offset + stream.length)
|
||||
else:
|
||||
raise ParserError(
|
||||
f"PDFStream is missing initilization dictionary"
|
||||
)
|
||||
elif token.value == b"endstream":
|
||||
stream = self.state.context_stack.pop()
|
||||
if not isinstance(stream, PDFStream):
|
||||
raise InvalidKeywordPos(token)
|
||||
else:
|
||||
raise InvalidKeywordPos(token)
|
||||
elif isinstance(token, TokenArray):
|
||||
for obj in self.state.flush_int_collections():
|
||||
yield obj
|
||||
obj = deal_with_collection_object(
|
||||
self.state, token, PDFArray, TArrayValue.OPEN, strict=self.strict
|
||||
)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
elif isinstance(token, TokenDictionary):
|
||||
for obj in self.state.flush_int_collections():
|
||||
yield obj
|
||||
obj = deal_with_collection_object(
|
||||
self.state,
|
||||
token=token,
|
||||
cls_const=PDFDictionary,
|
||||
open_t=TDictValue.OPEN,
|
||||
strict=self.strict,
|
||||
)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
@@ -1,7 +1,9 @@
|
||||
from typing import Any
|
||||
import zlib
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from base64 import a85decode
|
||||
from binascii import a2b_hex
|
||||
from ..exceptions import InvalidDecoderOrNotImplemented
|
||||
|
||||
from .types import FilterType
|
||||
from .lzw import lzwdecode
|
||||
@@ -10,7 +12,86 @@ from .ccit import ccittfaxdecode
|
||||
from ..exceptions import InvalidDecoderOrNotImplemented
|
||||
|
||||
|
||||
_filter_type_name_mapping: Dict[str, FilterType] = {
|
||||
"FlateDecode": FilterType.FlateDecode,
|
||||
"Fl": FilterType.FlateDecode,
|
||||
"LZWDecode": FilterType.LZWDecode,
|
||||
"LZW": FilterType.LZWDecode,
|
||||
"ASCII85Decode": FilterType.ASCII85Decode,
|
||||
"A85": FilterType.ASCII85Decode,
|
||||
"ASCIIHexDecode": FilterType.ASCIIHexDecode,
|
||||
"AHx": FilterType.ASCIIHexDecode,
|
||||
"RunLengthDecode": FilterType.RunLengthDecode,
|
||||
"RL": FilterType.RunLengthDecode,
|
||||
"CCITTFaxDecode": FilterType.CCITTFaxDecode,
|
||||
"CCF": FilterType.CCITTFaxDecode,
|
||||
"DCTDecode": FilterType.DCTDecode,
|
||||
"DCT": FilterType.DCTDecode,
|
||||
"Crypt": FilterType.Crypt,
|
||||
"JBIG2Decode": FilterType.JBIG2Decode,
|
||||
"JPXDecode": FilterType.JPXDecode,
|
||||
}
|
||||
|
||||
|
||||
def process_filters_on_data(
|
||||
data: bytes,
|
||||
filters: Optional[List[str]],
|
||||
decode_params: Optional[List[Dict[str, Any]]],
|
||||
) -> bytes:
|
||||
"""runs the filters provided by the list with the provided params against the data
|
||||
|
||||
Args:
|
||||
data (bytes)
|
||||
filters (Optional[List[str]])
|
||||
decode_params (Optional[List[Dict[str, Any]]])
|
||||
|
||||
Returns:
|
||||
bytes
|
||||
"""
|
||||
if filters is not None:
|
||||
decode_parms = []
|
||||
if decode_params is not None:
|
||||
decode_parms = decode_params
|
||||
for f in filters:
|
||||
params = {}
|
||||
if len(decode_parms) > 0:
|
||||
params = decode_parms.pop()
|
||||
filter_type = detect_filter_type(f.value)
|
||||
data = decode_bytes(filter_type, data, **params)
|
||||
return data
|
||||
|
||||
|
||||
def detect_filter_type(name: str) -> FilterType:
|
||||
"""Check to see if the specified filter exists in our codebase
|
||||
|
||||
Args:
|
||||
name (str)
|
||||
|
||||
Raises:
|
||||
InvalidDecoderOrNotImplemented
|
||||
|
||||
Returns:
|
||||
FilterType
|
||||
"""
|
||||
ftype = _filter_type_name_mapping.get(name, None)
|
||||
if ftype is None:
|
||||
raise InvalidDecoderOrNotImplemented(name)
|
||||
return ftype
|
||||
|
||||
|
||||
def decode_bytes(filter_type: FilterType, data: bytes, **kwrgs: Any) -> bytes:
|
||||
"""Run the decoding function on the filter
|
||||
|
||||
Args:
|
||||
filter_type (FilterType)
|
||||
data (bytes)
|
||||
|
||||
Raises:
|
||||
InvalidDecoderOrNotImplemented
|
||||
|
||||
Returns:
|
||||
bytes
|
||||
"""
|
||||
if filter_type == FilterType.ASCII85Decode:
|
||||
return a85decode(data, adobe=True)
|
||||
elif filter_type == FilterType.ASCIIHexDecode:
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import enum
|
||||
from typing import Dict
|
||||
from ..exceptions import InvalidDecoderOrNotImplemented
|
||||
|
||||
|
||||
class FilterType(enum.Enum):
|
||||
@@ -16,31 +14,3 @@ class FilterType(enum.Enum):
|
||||
DCTDecode = enum.auto()
|
||||
JPXDecode = enum.auto()
|
||||
Crypt = enum.auto()
|
||||
|
||||
|
||||
_filter_type_name_mapping: Dict[str, FilterType] = {
|
||||
"FlateDecode": FilterType.FlateDecode,
|
||||
"Fl": FilterType.FlateDecode,
|
||||
"LZWDecode": FilterType.LZWDecode,
|
||||
"LZW": FilterType.LZWDecode,
|
||||
"ASCII85Decode": FilterType.ASCII85Decode,
|
||||
"A85": FilterType.ASCII85Decode,
|
||||
"ASCIIHexDecode": FilterType.ASCIIHexDecode,
|
||||
"AHx": FilterType.ASCIIHexDecode,
|
||||
"RunLengthDecode": FilterType.RunLengthDecode,
|
||||
"RL": FilterType.RunLengthDecode,
|
||||
"CCITTFaxDecode": FilterType.CCITTFaxDecode,
|
||||
"CCF": FilterType.CCITTFaxDecode,
|
||||
"DCTDecode": FilterType.DCTDecode,
|
||||
"DCT": FilterType.DCTDecode,
|
||||
"Crypt": FilterType.Crypt,
|
||||
"JBIG2Decode": FilterType.JBIG2Decode,
|
||||
"JPXDecode": FilterType.JPXDecode,
|
||||
}
|
||||
|
||||
|
||||
def detect_filter_type(name: str) -> FilterType:
|
||||
ftype = _filter_type_name_mapping.get(name, None)
|
||||
if ftype is None:
|
||||
raise InvalidDecoderOrNotImplemented(name)
|
||||
return ftype
|
||||
@@ -1,5 +1,5 @@
|
||||
from pdfmajor.parser.exceptions import ParserError
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from .base import PDFContextualObject, PDFObject
|
||||
from .primitives import PDFName
|
||||
from .collections import PDFDictionary
|
||||
@@ -12,11 +12,11 @@ class PDFStream(PDFContextualObject):
|
||||
self,
|
||||
offset: int = 0,
|
||||
length: int = 0,
|
||||
filter: Optional[List[PDFName]] = None,
|
||||
decode_parms: Optional[List[PDFDictionary]] = None,
|
||||
filter: Optional[List[str]] = None,
|
||||
decode_parms: Optional[List[Dict[str, Any]]] = None,
|
||||
f: Optional[Any] = None, # file-specifications
|
||||
ffilter: Optional[List[PDFName]] = None,
|
||||
fdecode_parms: Optional[List[PDFDictionary]] = None,
|
||||
ffilter: Optional[List[str]] = None,
|
||||
fdecode_parms: Optional[List[Dict[str, Any]]] = None,
|
||||
dl: Optional[int] = None,
|
||||
other_metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
@@ -24,33 +24,39 @@ class PDFStream(PDFContextualObject):
|
||||
Args:
|
||||
offset (int): the offset where the stream itself starts
|
||||
length (int): the length of the stream
|
||||
filter (Optional[List[PDFName]], optional): The names of a filter that shall be applied in processing the stream data. Defaults to None.
|
||||
decode_parms (Optional[List[PDFDictionary]], optional): Parameter dictionaries used by the filters specified by filter. Defaults to None.
|
||||
filter (Optional[List[str]], optional): The names of a filter that shall be applied in processing the stream data. Defaults to None.
|
||||
decode_parms (Optional[List[Dict[str, Any]]], optional): Parameter dictionaries used by the filters specified by filter. Defaults to None.
|
||||
f (Optional[Any], optional): The file containing the stream data. If this entry is present, the bytes between stream and endstream shall be ignored. However, the Length entry should still specify the number of those bytes (usually, there are no bytes and Length is 0).. Defaults to None.
|
||||
ffilter (Optional[List[PDFName]], optional): The names of a filter that shall be applied in processing of the external file. Defaults to None.
|
||||
fdecode_parms (Optional[List[PDFDictionary]], optional): Parameter dictionaries used by the filters specified by ffilter. Defaults to None.
|
||||
ffilter (Optional[List[str]], optional): The names of a filter that shall be applied in processing of the external file. Defaults to None.
|
||||
fdecode_parms (Optional[List[Dict[str, Any]]], optional): Parameter dictionaries used by the filters specified by ffilter. Defaults to None.
|
||||
dl (Optional[int], optional): A non-negative integer representing the number of bytes in the decoded (defiltered) stream. It can be used to determine, for example, whether enough disk space is available to write a stream to a file.This value shall be considered a hint only; for some stream filters, it may not be possible to determine this value precisely. Defaults to None.
|
||||
"""
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
self.filter = filter
|
||||
self.decode_parms = decode_parms
|
||||
self.f = f
|
||||
self.ffilter = ffilter
|
||||
self.fdecode_parms = fdecode_parms
|
||||
self.dl = dl
|
||||
self.other_metadata = other_metadata if other_metadata is not None else {}
|
||||
self.filter: Optional[List[str]] = filter
|
||||
self.decode_parms: Optional[List[Dict[str, Any]]] = decode_parms
|
||||
self.f: Optional[Any] = f
|
||||
self.ffilter: Optional[List[str]] = ffilter
|
||||
self.fdecode_parms: Optional[List[Dict[str, Any]]] = fdecode_parms
|
||||
self.dl: Optional[int] = dl
|
||||
self.other_metadata: Dict[str, Any] = (
|
||||
other_metadata if other_metadata is not None else {}
|
||||
)
|
||||
|
||||
def pass_item(self, item: PDFObject) -> None:
|
||||
if not isinstance(item, PDFDictionary):
|
||||
raise ParserError(f"Cannot initilize a pdfstream with {type(item)}")
|
||||
data = item.to_python()
|
||||
self.length = data["Length"]
|
||||
self.filter = data.get("Filter", self.filter)
|
||||
self.decode_parms = data.get("DecodeParms", self.decode_parms)
|
||||
self.filter = get_single_or_list(data.get("Filter", self.filter))
|
||||
self.decode_parms = get_single_or_list(
|
||||
data.get("DecodeParms", self.decode_parms)
|
||||
)
|
||||
self.f = data.get("F", self.f)
|
||||
self.ffilter = data.get("FFilter", self.ffilter)
|
||||
self.fdecode_parms = data.get("FDecodeParms", self.fdecode_parms)
|
||||
self.ffilter = get_single_or_list(data.get("FFilter", self.ffilter))
|
||||
self.fdecode_parms = get_single_or_list(
|
||||
data.get("FDecodeParms", self.fdecode_parms)
|
||||
)
|
||||
self.other_metadata = data
|
||||
|
||||
def to_python(self):
|
||||
@@ -68,3 +74,12 @@ class PDFStream(PDFContextualObject):
|
||||
else [x.to_python() for x in self.fdecode_parms],
|
||||
"dl": self.dl,
|
||||
}
|
||||
|
||||
|
||||
def get_single_or_list(obj: Optional[Union[List[Any], Any]]) -> List[Any]:
|
||||
if obj is None:
|
||||
return []
|
||||
elif isinstance(obj, list):
|
||||
return obj
|
||||
else:
|
||||
return [obj]
|
||||
Reference in new issue
Block a user