diff --git a/pdfmajor/parser/PDFStream/ccitt.py b/pdfmajor/parser/PDFStream/ccitt.py index 66ca65c..f0d3f3a 100644 --- a/pdfmajor/parser/PDFStream/ccitt.py +++ b/pdfmajor/parser/PDFStream/ccitt.py @@ -562,46 +562,3 @@ def ccittfaxdecode(data, params): raise ValueError(K) parser.feedbytes(data) return parser.close() - - -# test -def main(argv): - if not argv[1:]: - import unittest - - return unittest.main() - - class Parser(CCITTG4Parser): - def __init__(self, width, bytealign=False): - import pygame - - CCITTG4Parser.__init__(self, width, bytealign=bytealign) - self.img = pygame.Surface((self.width, 1000)) - return - - def output_line(self, y, bits): - for (x, b) in enumerate(bits): - if b: - self.img.set_at((x, y), (255, 255, 255)) - else: - self.img.set_at((x, y), (0, 0, 0)) - return - - def close(self): - import pygame - - pygame.image.save(self.img, "out.bmp") - return - - for path in argv[1:]: - fp = open(path, "rb") - (_, _, _, w, _, _) = path.split(".") - parser = Parser(int(w)) - parser.feedbytes(fp.read()) - parser.close() - fp.close() - return - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/pdfmajor/parser_v3/exceptions.py b/pdfmajor/parser_v3/exceptions.py index 2456cde..082c58b 100644 --- a/pdfmajor/parser_v3/exceptions.py +++ b/pdfmajor/parser_v3/exceptions.py @@ -3,3 +3,11 @@ from pdfmajor.execptions import PDFMajorException class ParserError(PDFMajorException): pass + + +class DecodeFailed(ParserError): + pass + + +class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError): + pass diff --git a/pdfmajor/parser_v3/fliters/__init__.py b/pdfmajor/parser_v3/fliters/__init__.py new file mode 100644 index 0000000..03886bf --- /dev/null +++ b/pdfmajor/parser_v3/fliters/__init__.py @@ -0,0 +1,39 @@ +from typing import Any +import zlib +from base64 import a85decode +from binascii import a2b_hex + +from .types import FilterType +from .lzw import lzwdecode +from .rld import rldecode +from .ccit import ccittfaxdecode +from ..exceptions import InvalidDecoderOrNotImplemented + + +def decode_bytes(filter_type: FilterType, data: bytes, **kwrgs: Any) -> bytes: + if filter_type == FilterType.ASCII85Decode: + return a85decode(data, adobe=True) + elif filter_type == FilterType.ASCIIHexDecode: + clean_data = b"".join(data.split(b" ")) + if len(clean_data) & 2 != 0: + clean_data += b"0" + return a2b_hex(clean_data) + elif filter_type == FilterType.LZWDecode: + return lzwdecode(data) + elif filter_type == FilterType.FlateDecode: + return zlib.decompress(data) + elif filter_type == FilterType.RunLengthDecode: + return rldecode(data) + elif filter_type == FilterType.CCITTFaxDecode: + return ccittfaxdecode(data, **kwrgs) + else: + raise InvalidDecoderOrNotImplemented(filter_type) + # TODO: + # elif filter_type == FilterType.JBIG2Decode: + # pass + # elif filter_type == FilterType.DCTDecode: + # pass + # elif filter_type == FilterType.JPXDecode: + # pass + # elif filter_type == FilterType.Crypt: + # pass diff --git a/pdfmajor/parser_v3/fliters/ccit.py b/pdfmajor/parser_v3/fliters/ccit.py new file mode 100644 index 0000000..45e1f65 --- /dev/null +++ b/pdfmajor/parser_v3/fliters/ccit.py @@ -0,0 +1,567 @@ +# CCITT Fax decoder +# +# Bugs: uncompressed mode untested. +# +# cf. +# ITU-T Recommendation T.4 +# "Standardization of Group 3 facsimile terminals for document transmission" +# ITU-T Recommendation T.6 +# "FACSIMILE CODING SCHEMES AND CODING CONTROL FUNCTIONS FOR GROUP 4 FACSIMILE APPARATUS" + + +import sys +import array +from typing import Optional +from pdfmajor.execptions import EOFB, InvalidData, ByteSkip +from abc import abstractmethod + + +def get_bytes(data): + for byte in data: + yield byte + + +## BitParser +## +class BitParser(object): + def __init__(self): + self._pos = 0 + self._state = None + return + + @classmethod + def add(klass, root, v, bits): + p = root + b = None + for i in range(len(bits)): + if 0 < i: + if p[b] is None: + p[b] = [None, None] + p = p[b] + if bits[i] == "1": + b = 1 + else: + b = 0 + p[b] = v + return + + def feedbytes(self, data): + for byte in get_bytes(data): + for m in (128, 64, 32, 16, 8, 4, 2, 1): + self._parse_bit(byte & m) + return + + def _parse_bit(self, x): + if x: + v = self._state[1] + else: + v = self._state[0] + self._pos += 1 + if isinstance(v, list): + self._state = v + else: + self._state = self._accept(v) + return + + @abstractmethod + def _accept(self, v): + raise NotImplementedError + + +## CCITTG4Parser +## +class CCITTG4Parser(BitParser): + + MODE = [None, None] + BitParser.add(MODE, 0, "1") + BitParser.add(MODE, +1, "011") + BitParser.add(MODE, -1, "010") + BitParser.add(MODE, "h", "001") + BitParser.add(MODE, "p", "0001") + BitParser.add(MODE, +2, "000011") + BitParser.add(MODE, -2, "000010") + BitParser.add(MODE, +3, "0000011") + BitParser.add(MODE, -3, "0000010") + BitParser.add(MODE, "u", "0000001111") + BitParser.add(MODE, "x1", "0000001000") + BitParser.add(MODE, "x2", "0000001001") + BitParser.add(MODE, "x3", "0000001010") + BitParser.add(MODE, "x4", "0000001011") + BitParser.add(MODE, "x5", "0000001100") + BitParser.add(MODE, "x6", "0000001101") + BitParser.add(MODE, "x7", "0000001110") + BitParser.add(MODE, "e", "000000000001000000000001") + + WHITE = [None, None] + BitParser.add(WHITE, 0, "00110101") + BitParser.add(WHITE, 1, "000111") + BitParser.add(WHITE, 2, "0111") + BitParser.add(WHITE, 3, "1000") + BitParser.add(WHITE, 4, "1011") + BitParser.add(WHITE, 5, "1100") + BitParser.add(WHITE, 6, "1110") + BitParser.add(WHITE, 7, "1111") + BitParser.add(WHITE, 8, "10011") + BitParser.add(WHITE, 9, "10100") + BitParser.add(WHITE, 10, "00111") + BitParser.add(WHITE, 11, "01000") + BitParser.add(WHITE, 12, "001000") + BitParser.add(WHITE, 13, "000011") + BitParser.add(WHITE, 14, "110100") + BitParser.add(WHITE, 15, "110101") + BitParser.add(WHITE, 16, "101010") + BitParser.add(WHITE, 17, "101011") + BitParser.add(WHITE, 18, "0100111") + BitParser.add(WHITE, 19, "0001100") + BitParser.add(WHITE, 20, "0001000") + BitParser.add(WHITE, 21, "0010111") + BitParser.add(WHITE, 22, "0000011") + BitParser.add(WHITE, 23, "0000100") + BitParser.add(WHITE, 24, "0101000") + BitParser.add(WHITE, 25, "0101011") + BitParser.add(WHITE, 26, "0010011") + BitParser.add(WHITE, 27, "0100100") + BitParser.add(WHITE, 28, "0011000") + BitParser.add(WHITE, 29, "00000010") + BitParser.add(WHITE, 30, "00000011") + BitParser.add(WHITE, 31, "00011010") + BitParser.add(WHITE, 32, "00011011") + BitParser.add(WHITE, 33, "00010010") + BitParser.add(WHITE, 34, "00010011") + BitParser.add(WHITE, 35, "00010100") + BitParser.add(WHITE, 36, "00010101") + BitParser.add(WHITE, 37, "00010110") + BitParser.add(WHITE, 38, "00010111") + BitParser.add(WHITE, 39, "00101000") + BitParser.add(WHITE, 40, "00101001") + BitParser.add(WHITE, 41, "00101010") + BitParser.add(WHITE, 42, "00101011") + BitParser.add(WHITE, 43, "00101100") + BitParser.add(WHITE, 44, "00101101") + BitParser.add(WHITE, 45, "00000100") + BitParser.add(WHITE, 46, "00000101") + BitParser.add(WHITE, 47, "00001010") + BitParser.add(WHITE, 48, "00001011") + BitParser.add(WHITE, 49, "01010010") + BitParser.add(WHITE, 50, "01010011") + BitParser.add(WHITE, 51, "01010100") + BitParser.add(WHITE, 52, "01010101") + BitParser.add(WHITE, 53, "00100100") + BitParser.add(WHITE, 54, "00100101") + BitParser.add(WHITE, 55, "01011000") + BitParser.add(WHITE, 56, "01011001") + BitParser.add(WHITE, 57, "01011010") + BitParser.add(WHITE, 58, "01011011") + BitParser.add(WHITE, 59, "01001010") + BitParser.add(WHITE, 60, "01001011") + BitParser.add(WHITE, 61, "00110010") + BitParser.add(WHITE, 62, "00110011") + BitParser.add(WHITE, 63, "00110100") + BitParser.add(WHITE, 64, "11011") + BitParser.add(WHITE, 128, "10010") + BitParser.add(WHITE, 192, "010111") + BitParser.add(WHITE, 256, "0110111") + BitParser.add(WHITE, 320, "00110110") + BitParser.add(WHITE, 384, "00110111") + BitParser.add(WHITE, 448, "01100100") + BitParser.add(WHITE, 512, "01100101") + BitParser.add(WHITE, 576, "01101000") + BitParser.add(WHITE, 640, "01100111") + BitParser.add(WHITE, 704, "011001100") + BitParser.add(WHITE, 768, "011001101") + BitParser.add(WHITE, 832, "011010010") + BitParser.add(WHITE, 896, "011010011") + BitParser.add(WHITE, 960, "011010100") + BitParser.add(WHITE, 1024, "011010101") + BitParser.add(WHITE, 1088, "011010110") + BitParser.add(WHITE, 1152, "011010111") + BitParser.add(WHITE, 1216, "011011000") + BitParser.add(WHITE, 1280, "011011001") + BitParser.add(WHITE, 1344, "011011010") + BitParser.add(WHITE, 1408, "011011011") + BitParser.add(WHITE, 1472, "010011000") + BitParser.add(WHITE, 1536, "010011001") + BitParser.add(WHITE, 1600, "010011010") + BitParser.add(WHITE, 1664, "011000") + BitParser.add(WHITE, 1728, "010011011") + BitParser.add(WHITE, 1792, "00000001000") + BitParser.add(WHITE, 1856, "00000001100") + BitParser.add(WHITE, 1920, "00000001101") + BitParser.add(WHITE, 1984, "000000010010") + BitParser.add(WHITE, 2048, "000000010011") + BitParser.add(WHITE, 2112, "000000010100") + BitParser.add(WHITE, 2176, "000000010101") + BitParser.add(WHITE, 2240, "000000010110") + BitParser.add(WHITE, 2304, "000000010111") + BitParser.add(WHITE, 2368, "000000011100") + BitParser.add(WHITE, 2432, "000000011101") + BitParser.add(WHITE, 2496, "000000011110") + BitParser.add(WHITE, 2560, "000000011111") + + BLACK = [None, None] + BitParser.add(BLACK, 0, "0000110111") + BitParser.add(BLACK, 1, "010") + BitParser.add(BLACK, 2, "11") + BitParser.add(BLACK, 3, "10") + BitParser.add(BLACK, 4, "011") + BitParser.add(BLACK, 5, "0011") + BitParser.add(BLACK, 6, "0010") + BitParser.add(BLACK, 7, "00011") + BitParser.add(BLACK, 8, "000101") + BitParser.add(BLACK, 9, "000100") + BitParser.add(BLACK, 10, "0000100") + BitParser.add(BLACK, 11, "0000101") + BitParser.add(BLACK, 12, "0000111") + BitParser.add(BLACK, 13, "00000100") + BitParser.add(BLACK, 14, "00000111") + BitParser.add(BLACK, 15, "000011000") + BitParser.add(BLACK, 16, "0000010111") + BitParser.add(BLACK, 17, "0000011000") + BitParser.add(BLACK, 18, "0000001000") + BitParser.add(BLACK, 19, "00001100111") + BitParser.add(BLACK, 20, "00001101000") + BitParser.add(BLACK, 21, "00001101100") + BitParser.add(BLACK, 22, "00000110111") + BitParser.add(BLACK, 23, "00000101000") + BitParser.add(BLACK, 24, "00000010111") + BitParser.add(BLACK, 25, "00000011000") + BitParser.add(BLACK, 26, "000011001010") + BitParser.add(BLACK, 27, "000011001011") + BitParser.add(BLACK, 28, "000011001100") + BitParser.add(BLACK, 29, "000011001101") + BitParser.add(BLACK, 30, "000001101000") + BitParser.add(BLACK, 31, "000001101001") + BitParser.add(BLACK, 32, "000001101010") + BitParser.add(BLACK, 33, "000001101011") + BitParser.add(BLACK, 34, "000011010010") + BitParser.add(BLACK, 35, "000011010011") + BitParser.add(BLACK, 36, "000011010100") + BitParser.add(BLACK, 37, "000011010101") + BitParser.add(BLACK, 38, "000011010110") + BitParser.add(BLACK, 39, "000011010111") + BitParser.add(BLACK, 40, "000001101100") + BitParser.add(BLACK, 41, "000001101101") + BitParser.add(BLACK, 42, "000011011010") + BitParser.add(BLACK, 43, "000011011011") + BitParser.add(BLACK, 44, "000001010100") + BitParser.add(BLACK, 45, "000001010101") + BitParser.add(BLACK, 46, "000001010110") + BitParser.add(BLACK, 47, "000001010111") + BitParser.add(BLACK, 48, "000001100100") + BitParser.add(BLACK, 49, "000001100101") + BitParser.add(BLACK, 50, "000001010010") + BitParser.add(BLACK, 51, "000001010011") + BitParser.add(BLACK, 52, "000000100100") + BitParser.add(BLACK, 53, "000000110111") + BitParser.add(BLACK, 54, "000000111000") + BitParser.add(BLACK, 55, "000000100111") + BitParser.add(BLACK, 56, "000000101000") + BitParser.add(BLACK, 57, "000001011000") + BitParser.add(BLACK, 58, "000001011001") + BitParser.add(BLACK, 59, "000000101011") + BitParser.add(BLACK, 60, "000000101100") + BitParser.add(BLACK, 61, "000001011010") + BitParser.add(BLACK, 62, "000001100110") + BitParser.add(BLACK, 63, "000001100111") + BitParser.add(BLACK, 64, "0000001111") + BitParser.add(BLACK, 128, "000011001000") + BitParser.add(BLACK, 192, "000011001001") + BitParser.add(BLACK, 256, "000001011011") + BitParser.add(BLACK, 320, "000000110011") + BitParser.add(BLACK, 384, "000000110100") + BitParser.add(BLACK, 448, "000000110101") + BitParser.add(BLACK, 512, "0000001101100") + BitParser.add(BLACK, 576, "0000001101101") + BitParser.add(BLACK, 640, "0000001001010") + BitParser.add(BLACK, 704, "0000001001011") + BitParser.add(BLACK, 768, "0000001001100") + BitParser.add(BLACK, 832, "0000001001101") + BitParser.add(BLACK, 896, "0000001110010") + BitParser.add(BLACK, 960, "0000001110011") + BitParser.add(BLACK, 1024, "0000001110100") + BitParser.add(BLACK, 1088, "0000001110101") + BitParser.add(BLACK, 1152, "0000001110110") + BitParser.add(BLACK, 1216, "0000001110111") + BitParser.add(BLACK, 1280, "0000001010010") + BitParser.add(BLACK, 1344, "0000001010011") + BitParser.add(BLACK, 1408, "0000001010100") + BitParser.add(BLACK, 1472, "0000001010101") + BitParser.add(BLACK, 1536, "0000001011010") + BitParser.add(BLACK, 1600, "0000001011011") + BitParser.add(BLACK, 1664, "0000001100100") + BitParser.add(BLACK, 1728, "0000001100101") + BitParser.add(BLACK, 1792, "00000001000") + BitParser.add(BLACK, 1856, "00000001100") + BitParser.add(BLACK, 1920, "00000001101") + BitParser.add(BLACK, 1984, "000000010010") + BitParser.add(BLACK, 2048, "000000010011") + BitParser.add(BLACK, 2112, "000000010100") + BitParser.add(BLACK, 2176, "000000010101") + BitParser.add(BLACK, 2240, "000000010110") + BitParser.add(BLACK, 2304, "000000010111") + BitParser.add(BLACK, 2368, "000000011100") + BitParser.add(BLACK, 2432, "000000011101") + BitParser.add(BLACK, 2496, "000000011110") + BitParser.add(BLACK, 2560, "000000011111") + + UNCOMPRESSED = [None, None] + BitParser.add(UNCOMPRESSED, "1", "1") + BitParser.add(UNCOMPRESSED, "01", "01") + BitParser.add(UNCOMPRESSED, "001", "001") + BitParser.add(UNCOMPRESSED, "0001", "0001") + BitParser.add(UNCOMPRESSED, "00001", "00001") + BitParser.add(UNCOMPRESSED, "00000", "000001") + BitParser.add(UNCOMPRESSED, "T00", "00000011") + BitParser.add(UNCOMPRESSED, "T10", "00000010") + BitParser.add(UNCOMPRESSED, "T000", "000000011") + BitParser.add(UNCOMPRESSED, "T100", "000000010") + BitParser.add(UNCOMPRESSED, "T0000", "0000000011") + BitParser.add(UNCOMPRESSED, "T1000", "0000000010") + BitParser.add(UNCOMPRESSED, "T00000", "00000000011") + BitParser.add(UNCOMPRESSED, "T10000", "00000000010") + + def __init__(self, width, bytealign=False): + BitParser.__init__(self) + self.width = width + self.bytealign = bytealign + self.reset() + return + + def feedbytes(self, data): + for byte in get_bytes(data): + try: + for m in (128, 64, 32, 16, 8, 4, 2, 1): + self._parse_bit(byte & m) + except ByteSkip: + self._accept = self._parse_mode + self._state = self.MODE + except EOFB: + break + return + + def _parse_mode(self, mode): + if mode == "p": + self._do_pass() + self._flush_line() + return self.MODE + elif mode == "h": + self._n1 = 0 + self._accept = self._parse_horiz1 + if self._color: + return self.WHITE + else: + return self.BLACK + elif mode == "u": + self._accept = self._parse_uncompressed + return self.UNCOMPRESSED + elif mode == "e": + raise EOFB + elif isinstance(mode, int): + self._do_vertical(mode) + self._flush_line() + return self.MODE + else: + raise InvalidData(mode) + + def _parse_horiz1(self, n): + if n is None: + raise InvalidData + self._n1 += n + if n < 64: + self._n2 = 0 + self._color = 1 - self._color + self._accept = self._parse_horiz2 + if self._color: + return self.WHITE + else: + return self.BLACK + + def _parse_horiz2(self, n): + if n is None: + raise InvalidData + self._n2 += n + if n < 64: + self._color = 1 - self._color + self._accept = self._parse_mode + self._do_horizontal(self._n1, self._n2) + self._flush_line() + return self.MODE + elif self._color: + return self.WHITE + else: + return self.BLACK + + def _parse_uncompressed(self, bits): + if not bits: + raise InvalidData + if bits.startswith("T"): + self._accept = self._parse_mode + self._color = int(bits[1]) + self._do_uncompressed(bits[2:]) + return self.MODE + else: + self._do_uncompressed(bits) + return self.UNCOMPRESSED + + def _get_bits(self): + return "".join(str(b) for b in self._curline[: self._curpos]) + + def _get_refline(self, i): + if i < 0: + return "[]" + "".join(str(b) for b in self._refline) + elif len(self._refline) <= i: + return "".join(str(b) for b in self._refline) + "[]" + else: + return ( + "".join(str(b) for b in self._refline[:i]) + + "[" + + str(self._refline[i]) + + "]" + + "".join(str(b) for b in self._refline[i + 1 :]) + ) + + def reset(self): + self._y = 0 + self._curline = array.array("b", [1] * self.width) + self._reset_line() + self._accept = self._parse_mode + self._state = self.MODE + return + + def output_line(self, y, bits): + print(y, "".join(str(b) for b in bits)) + return + + def _reset_line(self): + self._refline = self._curline + self._curline = array.array("b", [1] * self.width) + self._curpos = -1 + self._color = 1 + return + + def _flush_line(self): + if self.width <= self._curpos: + self.output_line(self._y, self._curline) + self._y += 1 + self._reset_line() + if self.bytealign: + raise ByteSkip + return + + def _do_vertical(self, dx): + x1 = self._curpos + 1 + while 1: + if x1 == 0: + if self._color == 1 and self._refline[x1] != self._color: + break + elif x1 == len(self._refline): + break + elif ( + self._refline[x1 - 1] == self._color + and self._refline[x1] != self._color + ): + break + x1 += 1 + x1 += dx + x0 = max(0, self._curpos) + x1 = max(0, min(self.width, x1)) + if x1 < x0: + for x in range(x1, x0): + self._curline[x] = self._color + elif x0 < x1: + for x in range(x0, x1): + self._curline[x] = self._color + self._curpos = x1 + self._color = 1 - self._color + return + + def _do_pass(self): + x1 = self._curpos + 1 + while 1: + if x1 == 0: + if self._color == 1 and self._refline[x1] != self._color: + break + elif x1 == len(self._refline): + break + elif ( + self._refline[x1 - 1] == self._color + and self._refline[x1] != self._color + ): + break + x1 += 1 + while 1: + if x1 == 0: + if self._color == 0 and self._refline[x1] == self._color: + break + elif x1 == len(self._refline): + break + elif ( + self._refline[x1 - 1] != self._color + and self._refline[x1] == self._color + ): + break + x1 += 1 + for x in range(self._curpos, x1): + self._curline[x] = self._color + self._curpos = x1 + return + + def _do_horizontal(self, n1, n2): + if self._curpos < 0: + self._curpos = 0 + x = self._curpos + for _ in range(n1): + if len(self._curline) <= x: + break + self._curline[x] = self._color + x += 1 + for _ in range(n2): + if len(self._curline) <= x: + break + self._curline[x] = 1 - self._color + x += 1 + self._curpos = x + return + + def _do_uncompressed(self, bits): + for c in bits: + self._curline[self._curpos] = int(c) + self._curpos += 1 + self._flush_line() + return + + +class CCITTFaxDecoder(CCITTG4Parser): + def __init__(self, width, bytealign=False, reversed=False): + CCITTG4Parser.__init__(self, width, bytealign=bytealign) + self.reversed = reversed + self._buf = b"" + return + + def close(self): + return self._buf + + def output_line(self, y, bits): + bytes = array.array("B", [0] * ((len(bits) + 7) // 8)) + if self.reversed: + bits = [1 - b for b in bits] + for (i, b) in enumerate(bits): + if b: + bytes[i // 8] += (128, 64, 32, 16, 8, 4, 2, 1)[i % 8] + self._buf += bytes.tostring() + return + + +def ccittfaxdecode( + data: bytes, + Columns: int, + EncodedByteAlign: bool = False, + BlackIs1: bool = False, + K: Optional[int] = None, +) -> bytes: + if K == -1: + parser = CCITTFaxDecoder(Columns, bytealign=EncodedByteAlign, reversed=BlackIs1) + else: + raise ValueError(K) + parser.feedbytes(data) + return parser.close() diff --git a/pdfmajor/parser_v3/fliters/lzw.py b/pdfmajor/parser_v3/fliters/lzw.py new file mode 100644 index 0000000..000eaf1 --- /dev/null +++ b/pdfmajor/parser_v3/fliters/lzw.py @@ -0,0 +1,88 @@ +from io import BytesIO +from typing import List, Optional +from ..exceptions import DecodeFailed + +## LZWDecoder +## +class LZWDecoder(object): + def __init__(self, fp: BytesIO): + self.fp = fp + self.buff: int = 0 + self.bpos: int = 8 + self.nbits: int = 9 + self.table: Optional[List[bytes]] = None + self.prevbuf: Optional[bytes] = None + return + + def readbits(self, bits): + v = 0 + while 1: + # the number of remaining bits we can get from the current buffer. + r = 8 - self.bpos + if bits <= r: + # |-----8-bits-----| + # |-bpos-|-bits-| | + # | |----r----| + v = (v << bits) | ((self.buff >> (r - bits)) & ((1 << bits) - 1)) + self.bpos += bits + break + else: + # |-----8-bits-----| + # |-bpos-|---bits----... + # | |----r----| + v = (v << r) | (self.buff & ((1 << r) - 1)) + bits -= r + x = self.fp.read(1) + if not x: + raise EOFError + self.buff = ord(x) + self.bpos = 0 + return v + + def feed(self, code): + x = b"" + if code == 256: + self.table = [bytes([c]) for c in range(256)] # 0-255 + self.table.append(None) # 256 + self.table.append(None) # 257 + self.prevbuf = b"" + self.nbits = 9 + elif code == 257: + pass + elif not self.prevbuf: + x = self.prevbuf = self.table[code] + else: + if code < len(self.table): + x = self.table[code] + self.table.append(self.prevbuf + x[:1]) + elif code == len(self.table): + self.table.append(self.prevbuf + self.prevbuf[:1]) + x = self.table[code] + else: + raise DecodeFailed("LZW") + l = len(self.table) + if l == 511: + self.nbits = 10 + elif l == 1023: + self.nbits = 11 + elif l == 2047: + self.nbits = 12 + self.prevbuf = x + return x + + def run(self): + while 1: + try: + code = self.readbits(self.nbits) + x = self.feed(code) + except EOFError: + break + yield x + return + + +# lzwdecode +def lzwdecode(data: bytes): + fp = BytesIO(data) + s = LZWDecoder(fp).run() + return b"".join(s) diff --git a/pdfmajor/parser_v3/fliters/rld.py b/pdfmajor/parser_v3/fliters/rld.py new file mode 100644 index 0000000..2ba0b82 --- /dev/null +++ b/pdfmajor/parser_v3/fliters/rld.py @@ -0,0 +1,36 @@ +# +# RunLength decoder (Adobe version) implementation based on PDF Reference +# version 1.4 section 3.3.4. +# +# * public domain * +# +def rldecode(data: bytes) -> bytes: + """ + RunLength decoder (Adobe version) implementation based on PDF Reference + version 1.4 section 3.3.4: + The RunLengthDecode filter decodes data that has been encoded in a + simple byte-oriented format based on run length. The encoded data + is a sequence of runs, where each run consists of a length byte + followed by 1 to 128 bytes of data. If the length byte is in the + range 0 to 127, the following length + 1 (1 to 128) bytes are + copied literally during decompression. If length is in the range + 129 to 255, the following single byte is to be copied 257 - length + (2 to 128) times during decompression. A length value of 128 + denotes EOD. + """ + decoded = b"" + i: int = 0 + while i < len(data): + length = data[i] + if length == 128: + break + if length >= 0 and length < 128: + for j in range(i + 1, (i + 1) + (length + 1)): + decoded += bytes([data[j]]) + + i = (i + 1) + (length + 1) + if length > 128: + run: bytes = bytes([data[i + 1]]) * (257 - length) + decoded += run + i = (i + 1) + 1 + return decoded diff --git a/pdfmajor/parser_v3/fliters/types.py b/pdfmajor/parser_v3/fliters/types.py new file mode 100644 index 0000000..fd270c9 --- /dev/null +++ b/pdfmajor/parser_v3/fliters/types.py @@ -0,0 +1,46 @@ +import enum +from typing import Dict +from ..exceptions import InvalidDecoderOrNotImplemented + + +class FilterType(enum.Enum): + """Standard PDF Filter types see PDF spec 1.7 section 7.4 table 6""" + + ASCIIHexDecode = enum.auto() + ASCII85Decode = enum.auto() + LZWDecode = enum.auto() + FlateDecode = enum.auto() + RunLengthDecode = enum.auto() + CCITTFaxDecode = enum.auto() + JBIG2Decode = enum.auto() + DCTDecode = enum.auto() + JPXDecode = enum.auto() + Crypt = enum.auto() + + +_filter_type_name_mapping: Dict[str, FilterType] = { + "FlateDecode": FilterType.FlateDecode, + "Fl": FilterType.FlateDecode, + "LZWDecode": FilterType.LZWDecode, + "LZW": FilterType.LZWDecode, + "ASCII85Decode": FilterType.ASCII85Decode, + "A85": FilterType.ASCII85Decode, + "ASCIIHexDecode": FilterType.ASCIIHexDecode, + "AHx": FilterType.ASCIIHexDecode, + "RunLengthDecode": FilterType.RunLengthDecode, + "RL": FilterType.RunLengthDecode, + "CCITTFaxDecode": FilterType.CCITTFaxDecode, + "CCF": FilterType.CCITTFaxDecode, + "DCTDecode": FilterType.DCTDecode, + "DCT": FilterType.DCTDecode, + "Crypt": FilterType.Crypt, + "JBIG2Decode": FilterType.JBIG2Decode, + "JPXDecode": FilterType.JPXDecode, +} + + +def detect_filter_type(name: str) -> FilterType: + ftype = _filter_type_name_mapping.get(name, None) + if ftype is None: + raise InvalidDecoderOrNotImplemented(name) + return ftype diff --git a/tests/test_bad_unicode.py b/tests/old/test_bad_unicode.py similarity index 100% rename from tests/test_bad_unicode.py rename to tests/old/test_bad_unicode.py diff --git a/tests/test_caching_err.py b/tests/old/test_caching_err.py similarity index 100% rename from tests/test_caching_err.py rename to tests/old/test_caching_err.py diff --git a/tests/test_cols.py b/tests/old/test_cols.py similarity index 100% rename from tests/test_cols.py rename to tests/old/test_cols.py diff --git a/tests/test_conversions.py b/tests/old/test_conversions.py similarity index 100% rename from tests/test_conversions.py rename to tests/old/test_conversions.py diff --git a/tests/test_html_writer.py b/tests/old/test_html_writer.py similarity index 100% rename from tests/test_html_writer.py rename to tests/old/test_html_writer.py diff --git a/tests/test_image_writer.py b/tests/old/test_image_writer.py similarity index 100% rename from tests/test_image_writer.py rename to tests/old/test_image_writer.py diff --git a/tests/test_interpreter.py b/tests/old/test_interpreter.py similarity index 100% rename from tests/test_interpreter.py rename to tests/old/test_interpreter.py diff --git a/tests/test_json_writer.py b/tests/old/test_json_writer.py similarity index 100% rename from tests/test_json_writer.py rename to tests/old/test_json_writer.py diff --git a/tests/test_yml_writer.py b/tests/old/test_yml_writer.py similarity index 100% rename from tests/test_yml_writer.py rename to tests/old/test_yml_writer.py diff --git a/tests/test_pdfmajor_ccitt.py b/tests/test_parser_filters.py similarity index 60% rename from tests/test_pdfmajor_ccitt.py rename to tests/test_parser_filters.py index 6f63e31..9b689a1 100644 --- a/tests/test_pdfmajor_ccitt.py +++ b/tests/test_parser_filters.py @@ -1,14 +1,24 @@ -#!/usr/bin/env python +from unittest import TestCase -# -*- coding: utf-8 -*- +from pdfmajor.parser_v3.fliters.ccit import CCITTG4Parser +from pdfmajor.parser_v3.fliters.lzw import lzwdecode +from pdfmajor.parser_v3.fliters.rld import rldecode -import unittest -from pdfmajor.parser.PDFStream.ccitt import CCITTG4Parser -## Test cases -## -class TestCCITTG4Parser(unittest.TestCase): +class Filters(TestCase): + def test_lzwdecode(self): + self.assertEqual( + lzwdecode(b"\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01"), + b"\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42", + ) + def test_rldecode(self): + self.assertEqual( + rldecode(b"\x05123456\xfa7\x04abcde\x80junk"), b"1234567777777abcde" + ) + + +class CCIT(TestCase): def get_parser(self, bits): parser = CCITTG4Parser(len(bits)) parser._curline = [int(c) for c in bits] @@ -16,60 +26,60 @@ class TestCCITTG4Parser(unittest.TestCase): return parser def test_b1(self): - parser = self.get_parser('00000') + parser = self.get_parser("00000") parser._do_vertical(0) self.assertEqual(parser._curpos, 0) return def test_b2(self): - parser = self.get_parser('10000') + parser = self.get_parser("10000") parser._do_vertical(-1) self.assertEqual(parser._curpos, 0) return def test_b3(self): - parser = self.get_parser('000111') + parser = self.get_parser("000111") parser._do_pass() self.assertEqual(parser._curpos, 3) - self.assertEqual(parser._get_bits(), '111') + self.assertEqual(parser._get_bits(), "111") return def test_b4(self): - parser = self.get_parser('00000') + parser = self.get_parser("00000") parser._do_vertical(+2) self.assertEqual(parser._curpos, 2) - self.assertEqual(parser._get_bits(), '11') + self.assertEqual(parser._get_bits(), "11") return def test_b5(self): - parser = self.get_parser('11111111100') + parser = self.get_parser("11111111100") parser._do_horizontal(0, 3) self.assertEqual(parser._curpos, 3) parser._do_vertical(1) self.assertEqual(parser._curpos, 10) - self.assertEqual(parser._get_bits(), '0001111111') + self.assertEqual(parser._get_bits(), "0001111111") return def test_e1(self): - parser = self.get_parser('10000') + parser = self.get_parser("10000") parser._do_vertical(0) self.assertEqual(parser._curpos, 1) parser._do_vertical(0) self.assertEqual(parser._curpos, 5) - self.assertEqual(parser._get_bits(), '10000') + self.assertEqual(parser._get_bits(), "10000") return def test_e2(self): - parser = self.get_parser('10011') + parser = self.get_parser("10011") parser._do_vertical(0) self.assertEqual(parser._curpos, 1) parser._do_vertical(2) self.assertEqual(parser._curpos, 5) - self.assertEqual(parser._get_bits(), '10000') + self.assertEqual(parser._get_bits(), "10000") return def test_e3(self): - parser = self.get_parser('011111') + parser = self.get_parser("011111") parser._color = 0 parser._do_vertical(0) self.assertEqual(parser._color, 1) @@ -79,85 +89,82 @@ class TestCCITTG4Parser(unittest.TestCase): self.assertEqual(parser._curpos, 4) parser._do_vertical(0) self.assertEqual(parser._curpos, 6) - self.assertEqual(parser._get_bits(), '011100') + self.assertEqual(parser._get_bits(), "011100") return def test_e4(self): - parser = self.get_parser('10000') + parser = self.get_parser("10000") parser._do_vertical(0) self.assertEqual(parser._curpos, 1) parser._do_vertical(-2) self.assertEqual(parser._curpos, 3) parser._do_vertical(0) self.assertEqual(parser._curpos, 5) - self.assertEqual(parser._get_bits(), '10011') + self.assertEqual(parser._get_bits(), "10011") return def test_e5(self): - parser = self.get_parser('011000') + parser = self.get_parser("011000") parser._color = 0 parser._do_vertical(0) self.assertEqual(parser._curpos, 1) parser._do_vertical(3) self.assertEqual(parser._curpos, 6) - self.assertEqual(parser._get_bits(), '011111') + self.assertEqual(parser._get_bits(), "011111") return def test_e6(self): - parser = self.get_parser('11001') + parser = self.get_parser("11001") parser._do_pass() self.assertEqual(parser._curpos, 4) parser._do_vertical(0) self.assertEqual(parser._curpos, 5) - self.assertEqual(parser._get_bits(), '11111') + self.assertEqual(parser._get_bits(), "11111") return def test_e7(self): - parser = self.get_parser('0000000000') + parser = self.get_parser("0000000000") parser._curpos = 2 parser._color = 1 parser._do_horizontal(2, 6) self.assertEqual(parser._curpos, 10) - self.assertEqual(parser._get_bits(), '1111000000') + self.assertEqual(parser._get_bits(), "1111000000") return def test_e8(self): - parser = self.get_parser('001100000') + parser = self.get_parser("001100000") parser._curpos = 1 parser._color = 0 parser._do_vertical(0) self.assertEqual(parser._curpos, 2) parser._do_horizontal(7, 0) self.assertEqual(parser._curpos, 9) - self.assertEqual(parser._get_bits(), '101111111') + self.assertEqual(parser._get_bits(), "101111111") return def test_m1(self): - parser = self.get_parser('10101') + parser = self.get_parser("10101") parser._do_pass() self.assertEqual(parser._curpos, 2) parser._do_pass() self.assertEqual(parser._curpos, 4) - self.assertEqual(parser._get_bits(), '1111') + self.assertEqual(parser._get_bits(), "1111") return def test_m2(self): - parser = self.get_parser('101011') + parser = self.get_parser("101011") parser._do_vertical(-1) parser._do_vertical(-1) parser._do_vertical(1) parser._do_horizontal(1, 1) - self.assertEqual(parser._get_bits(), '011101') + self.assertEqual(parser._get_bits(), "011101") return def test_m3(self): - parser = self.get_parser('10111011') + parser = self.get_parser("10111011") parser._do_vertical(-1) parser._do_pass() parser._do_vertical(1) parser._do_vertical(1) - self.assertEqual(parser._get_bits(), '00000001') + self.assertEqual(parser._get_bits(), "00000001") return - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/tests/test_pdfmajor_crypto.py b/tests/test_pdfmajor_crypto.py deleted file mode 100644 index 0e4b430..0000000 --- a/tests/test_pdfmajor_crypto.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from unittest import TestCase, main - -#test of various compression/encoding modules (previously in doctests): -from pdfmajor.utils.ascii85 import ascii85decode, asciihexdecode -from pdfmajor.utils.lzw import LZWDecoder, lzwdecode -from pdfmajor.parser.PDFStream import rldecode - -import binascii -def hex(b): return binascii.hexlify(b) #encode('hex') -def dehex(b): return binascii.unhexlify(b) #decode('hex') - -class TestAscii85(TestCase): - def test_ascii85decode(self): - #The sample string is taken from: http://en.wikipedia.org/w/index.php?title=Ascii85 - self.assertEqual(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished') - self.assertEqual(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.') - - def test_asciihexdecode(self): - self.assertEqual(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde') - self.assertEqual(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep') - self.assertEqual(asciihexdecode(b'7>'),b'p') - -class TestLzw(TestCase): - def test_lzwdecode(self): - self.assertEqual(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42') - -class TestRunlength(TestCase): - def test_rldecode(self): - self.assertEqual(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde') - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/tests/test_pdfmajor_psparser.py b/tests/test_pdfmajor_psparser.py deleted file mode 100644 index 76f976f..0000000 --- a/tests/test_pdfmajor_psparser.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- - -import logging -from unittest import TestCase, main -from io import BytesIO -from pdfmajor.parser.PSStackParser import KWD, LIT, PSBaseParser, PSStackParser -from pdfmajor.execptions import PSEOF - -## Simplistic Test cases -## -TESTDATA = br'''%!PS -begin end - " @ # -/a/BCD /Some_Name /foo#5f#xbaa -0 +1 -2 .5 1.234 -(abc) () (abc ( def ) ghi) -(def\040\0\0404ghi) (bach\\slask) (foo\nbaa) -(this % is not a comment.) -(foo -baa) -(foo\ -baa) -<> <20> < 40 4020 > - -func/a/b{(c)do*}def -[ 1 (z) ! ] -<< /foo (bar) >> -''' -TOKENS = [ - (5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')), - (21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), - (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), - (65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'), - (98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'), - (143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'), - (191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'), - (226, KWD(b'func')), (230, LIT('a')), (232, LIT('b')), - (234, KWD(b'{')), (235, b'c'), (238, KWD(b'do*')), (241, KWD(b'}')), - (242, KWD(b'def')), (246, KWD(b'[')), (248, 1), (250, b'z'), (254, KWD(b'!')), - (256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'), - (272, KWD(b'>>')) -] -OBJS = [ - (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')), - (41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5), - (65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'), - (98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'), - (143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'), - (191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'), - (230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']), - (258, {'foo': b'bar'}), -] -class TestPSBaseParser(TestCase): - def get_tokens(self, s): - class MyParser(PSStackParser): - def flush(self): - self.add_results(*self.popall()) - parser = MyParser(BytesIO(s)) - r = [] - try: - while True: - r.append(parser.nexttoken()) - except PSEOF: - pass - return r - - def get_objects(self, s): - class MyParser(PSStackParser): - def flush(self): - self.add_results(*self.popall()) - parser = MyParser(BytesIO(s)) - r = [] - try: - while True: - r.append(parser.nextobject()) - except PSEOF: - pass - return r - - def test_1(self): - tokens = self.get_tokens(TESTDATA) - logging.info(tokens) - self.assertEqual(tokens, TOKENS) - return - - def test_2(self): - objs = self.get_objects(TESTDATA) - logging.info(objs) - self.assertEqual(objs, OBJS) - return - -if __name__ == '__main__': - #import logging,sys,os,six - #logging.basicConfig(level=logging.DEBUG, filename='%s_%d.%d.log'%(os.path.basename(__file__),sys.version_info[0],sys.version_info[1])) - main() \ No newline at end of file