- removed old tests

- ported all custom filters and using python std implemetation for ascii and bin filters
This commit is contained in:
ari committed 2020-10-20 19:24:15 -04:00
1 parent 5fd03d1250
commit 4d1e0f2389
19 files changed
+831 -216

No files matched your search

-43
View File
@@ -562,46 +562,3 @@ def ccittfaxdecode(data, params):
raise ValueError(K)
parser.feedbytes(data)
return parser.close()
# test
def main(argv):
if not argv[1:]:
import unittest
return unittest.main()
class Parser(CCITTG4Parser):
def __init__(self, width, bytealign=False):
import pygame
CCITTG4Parser.__init__(self, width, bytealign=bytealign)
self.img = pygame.Surface((self.width, 1000))
return
def output_line(self, y, bits):
for (x, b) in enumerate(bits):
if b:
self.img.set_at((x, y), (255, 255, 255))
else:
self.img.set_at((x, y), (0, 0, 0))
return
def close(self):
import pygame
pygame.image.save(self.img, "out.bmp")
return
for path in argv[1:]:
fp = open(path, "rb")
(_, _, _, w, _, _) = path.split(".")
parser = Parser(int(w))
parser.feedbytes(fp.read())
parser.close()
fp.close()
return
if __name__ == "__main__":
sys.exit(main(sys.argv))
+8
View File
@@ -3,3 +3,11 @@ from pdfmajor.execptions import PDFMajorException
class ParserError(PDFMajorException):
pass
class DecodeFailed(ParserError):
pass
class InvalidDecoderOrNotImplemented(DecodeFailed, NotImplementedError):
pass
+39
View File
@@ -0,0 +1,39 @@
from typing import Any
import zlib
from base64 import a85decode
from binascii import a2b_hex
from .types import FilterType
from .lzw import lzwdecode
from .rld import rldecode
from .ccit import ccittfaxdecode
from ..exceptions import InvalidDecoderOrNotImplemented
def decode_bytes(filter_type: FilterType, data: bytes, **kwrgs: Any) -> bytes:
if filter_type == FilterType.ASCII85Decode:
return a85decode(data, adobe=True)
elif filter_type == FilterType.ASCIIHexDecode:
clean_data = b"".join(data.split(b" "))
if len(clean_data) & 2 != 0:
clean_data += b"0"
return a2b_hex(clean_data)
elif filter_type == FilterType.LZWDecode:
return lzwdecode(data)
elif filter_type == FilterType.FlateDecode:
return zlib.decompress(data)
elif filter_type == FilterType.RunLengthDecode:
return rldecode(data)
elif filter_type == FilterType.CCITTFaxDecode:
return ccittfaxdecode(data, **kwrgs)
else:
raise InvalidDecoderOrNotImplemented(filter_type)
# TODO:
# elif filter_type == FilterType.JBIG2Decode:
# pass
# elif filter_type == FilterType.DCTDecode:
# pass
# elif filter_type == FilterType.JPXDecode:
# pass
# elif filter_type == FilterType.Crypt:
# pass
+567
View File
@@ -0,0 +1,567 @@
# CCITT Fax decoder
#
# Bugs: uncompressed mode untested.
#
# cf.
# ITU-T Recommendation T.4
# "Standardization of Group 3 facsimile terminals for document transmission"
# ITU-T Recommendation T.6
# "FACSIMILE CODING SCHEMES AND CODING CONTROL FUNCTIONS FOR GROUP 4 FACSIMILE APPARATUS"
import sys
import array
from typing import Optional
from pdfmajor.execptions import EOFB, InvalidData, ByteSkip
from abc import abstractmethod
def get_bytes(data):
for byte in data:
yield byte
## BitParser
##
class BitParser(object):
def __init__(self):
self._pos = 0
self._state = None
return
@classmethod
def add(klass, root, v, bits):
p = root
b = None
for i in range(len(bits)):
if 0 < i:
if p[b] is None:
p[b] = [None, None]
p = p[b]
if bits[i] == "1":
b = 1
else:
b = 0
p[b] = v
return
def feedbytes(self, data):
for byte in get_bytes(data):
for m in (128, 64, 32, 16, 8, 4, 2, 1):
self._parse_bit(byte & m)
return
def _parse_bit(self, x):
if x:
v = self._state[1]
else:
v = self._state[0]
self._pos += 1
if isinstance(v, list):
self._state = v
else:
self._state = self._accept(v)
return
@abstractmethod
def _accept(self, v):
raise NotImplementedError
## CCITTG4Parser
##
class CCITTG4Parser(BitParser):
MODE = [None, None]
BitParser.add(MODE, 0, "1")
BitParser.add(MODE, +1, "011")
BitParser.add(MODE, -1, "010")
BitParser.add(MODE, "h", "001")
BitParser.add(MODE, "p", "0001")
BitParser.add(MODE, +2, "000011")
BitParser.add(MODE, -2, "000010")
BitParser.add(MODE, +3, "0000011")
BitParser.add(MODE, -3, "0000010")
BitParser.add(MODE, "u", "0000001111")
BitParser.add(MODE, "x1", "0000001000")
BitParser.add(MODE, "x2", "0000001001")
BitParser.add(MODE, "x3", "0000001010")
BitParser.add(MODE, "x4", "0000001011")
BitParser.add(MODE, "x5", "0000001100")
BitParser.add(MODE, "x6", "0000001101")
BitParser.add(MODE, "x7", "0000001110")
BitParser.add(MODE, "e", "000000000001000000000001")
WHITE = [None, None]
BitParser.add(WHITE, 0, "00110101")
BitParser.add(WHITE, 1, "000111")
BitParser.add(WHITE, 2, "0111")
BitParser.add(WHITE, 3, "1000")
BitParser.add(WHITE, 4, "1011")
BitParser.add(WHITE, 5, "1100")
BitParser.add(WHITE, 6, "1110")
BitParser.add(WHITE, 7, "1111")
BitParser.add(WHITE, 8, "10011")
BitParser.add(WHITE, 9, "10100")
BitParser.add(WHITE, 10, "00111")
BitParser.add(WHITE, 11, "01000")
BitParser.add(WHITE, 12, "001000")
BitParser.add(WHITE, 13, "000011")
BitParser.add(WHITE, 14, "110100")
BitParser.add(WHITE, 15, "110101")
BitParser.add(WHITE, 16, "101010")
BitParser.add(WHITE, 17, "101011")
BitParser.add(WHITE, 18, "0100111")
BitParser.add(WHITE, 19, "0001100")
BitParser.add(WHITE, 20, "0001000")
BitParser.add(WHITE, 21, "0010111")
BitParser.add(WHITE, 22, "0000011")
BitParser.add(WHITE, 23, "0000100")
BitParser.add(WHITE, 24, "0101000")
BitParser.add(WHITE, 25, "0101011")
BitParser.add(WHITE, 26, "0010011")
BitParser.add(WHITE, 27, "0100100")
BitParser.add(WHITE, 28, "0011000")
BitParser.add(WHITE, 29, "00000010")
BitParser.add(WHITE, 30, "00000011")
BitParser.add(WHITE, 31, "00011010")
BitParser.add(WHITE, 32, "00011011")
BitParser.add(WHITE, 33, "00010010")
BitParser.add(WHITE, 34, "00010011")
BitParser.add(WHITE, 35, "00010100")
BitParser.add(WHITE, 36, "00010101")
BitParser.add(WHITE, 37, "00010110")
BitParser.add(WHITE, 38, "00010111")
BitParser.add(WHITE, 39, "00101000")
BitParser.add(WHITE, 40, "00101001")
BitParser.add(WHITE, 41, "00101010")
BitParser.add(WHITE, 42, "00101011")
BitParser.add(WHITE, 43, "00101100")
BitParser.add(WHITE, 44, "00101101")
BitParser.add(WHITE, 45, "00000100")
BitParser.add(WHITE, 46, "00000101")
BitParser.add(WHITE, 47, "00001010")
BitParser.add(WHITE, 48, "00001011")
BitParser.add(WHITE, 49, "01010010")
BitParser.add(WHITE, 50, "01010011")
BitParser.add(WHITE, 51, "01010100")
BitParser.add(WHITE, 52, "01010101")
BitParser.add(WHITE, 53, "00100100")
BitParser.add(WHITE, 54, "00100101")
BitParser.add(WHITE, 55, "01011000")
BitParser.add(WHITE, 56, "01011001")
BitParser.add(WHITE, 57, "01011010")
BitParser.add(WHITE, 58, "01011011")
BitParser.add(WHITE, 59, "01001010")
BitParser.add(WHITE, 60, "01001011")
BitParser.add(WHITE, 61, "00110010")
BitParser.add(WHITE, 62, "00110011")
BitParser.add(WHITE, 63, "00110100")
BitParser.add(WHITE, 64, "11011")
BitParser.add(WHITE, 128, "10010")
BitParser.add(WHITE, 192, "010111")
BitParser.add(WHITE, 256, "0110111")
BitParser.add(WHITE, 320, "00110110")
BitParser.add(WHITE, 384, "00110111")
BitParser.add(WHITE, 448, "01100100")
BitParser.add(WHITE, 512, "01100101")
BitParser.add(WHITE, 576, "01101000")
BitParser.add(WHITE, 640, "01100111")
BitParser.add(WHITE, 704, "011001100")
BitParser.add(WHITE, 768, "011001101")
BitParser.add(WHITE, 832, "011010010")
BitParser.add(WHITE, 896, "011010011")
BitParser.add(WHITE, 960, "011010100")
BitParser.add(WHITE, 1024, "011010101")
BitParser.add(WHITE, 1088, "011010110")
BitParser.add(WHITE, 1152, "011010111")
BitParser.add(WHITE, 1216, "011011000")
BitParser.add(WHITE, 1280, "011011001")
BitParser.add(WHITE, 1344, "011011010")
BitParser.add(WHITE, 1408, "011011011")
BitParser.add(WHITE, 1472, "010011000")
BitParser.add(WHITE, 1536, "010011001")
BitParser.add(WHITE, 1600, "010011010")
BitParser.add(WHITE, 1664, "011000")
BitParser.add(WHITE, 1728, "010011011")
BitParser.add(WHITE, 1792, "00000001000")
BitParser.add(WHITE, 1856, "00000001100")
BitParser.add(WHITE, 1920, "00000001101")
BitParser.add(WHITE, 1984, "000000010010")
BitParser.add(WHITE, 2048, "000000010011")
BitParser.add(WHITE, 2112, "000000010100")
BitParser.add(WHITE, 2176, "000000010101")
BitParser.add(WHITE, 2240, "000000010110")
BitParser.add(WHITE, 2304, "000000010111")
BitParser.add(WHITE, 2368, "000000011100")
BitParser.add(WHITE, 2432, "000000011101")
BitParser.add(WHITE, 2496, "000000011110")
BitParser.add(WHITE, 2560, "000000011111")
BLACK = [None, None]
BitParser.add(BLACK, 0, "0000110111")
BitParser.add(BLACK, 1, "010")
BitParser.add(BLACK, 2, "11")
BitParser.add(BLACK, 3, "10")
BitParser.add(BLACK, 4, "011")
BitParser.add(BLACK, 5, "0011")
BitParser.add(BLACK, 6, "0010")
BitParser.add(BLACK, 7, "00011")
BitParser.add(BLACK, 8, "000101")
BitParser.add(BLACK, 9, "000100")
BitParser.add(BLACK, 10, "0000100")
BitParser.add(BLACK, 11, "0000101")
BitParser.add(BLACK, 12, "0000111")
BitParser.add(BLACK, 13, "00000100")
BitParser.add(BLACK, 14, "00000111")
BitParser.add(BLACK, 15, "000011000")
BitParser.add(BLACK, 16, "0000010111")
BitParser.add(BLACK, 17, "0000011000")
BitParser.add(BLACK, 18, "0000001000")
BitParser.add(BLACK, 19, "00001100111")
BitParser.add(BLACK, 20, "00001101000")
BitParser.add(BLACK, 21, "00001101100")
BitParser.add(BLACK, 22, "00000110111")
BitParser.add(BLACK, 23, "00000101000")
BitParser.add(BLACK, 24, "00000010111")
BitParser.add(BLACK, 25, "00000011000")
BitParser.add(BLACK, 26, "000011001010")
BitParser.add(BLACK, 27, "000011001011")
BitParser.add(BLACK, 28, "000011001100")
BitParser.add(BLACK, 29, "000011001101")
BitParser.add(BLACK, 30, "000001101000")
BitParser.add(BLACK, 31, "000001101001")
BitParser.add(BLACK, 32, "000001101010")
BitParser.add(BLACK, 33, "000001101011")
BitParser.add(BLACK, 34, "000011010010")
BitParser.add(BLACK, 35, "000011010011")
BitParser.add(BLACK, 36, "000011010100")
BitParser.add(BLACK, 37, "000011010101")
BitParser.add(BLACK, 38, "000011010110")
BitParser.add(BLACK, 39, "000011010111")
BitParser.add(BLACK, 40, "000001101100")
BitParser.add(BLACK, 41, "000001101101")
BitParser.add(BLACK, 42, "000011011010")
BitParser.add(BLACK, 43, "000011011011")
BitParser.add(BLACK, 44, "000001010100")
BitParser.add(BLACK, 45, "000001010101")
BitParser.add(BLACK, 46, "000001010110")
BitParser.add(BLACK, 47, "000001010111")
BitParser.add(BLACK, 48, "000001100100")
BitParser.add(BLACK, 49, "000001100101")
BitParser.add(BLACK, 50, "000001010010")
BitParser.add(BLACK, 51, "000001010011")
BitParser.add(BLACK, 52, "000000100100")
BitParser.add(BLACK, 53, "000000110111")
BitParser.add(BLACK, 54, "000000111000")
BitParser.add(BLACK, 55, "000000100111")
BitParser.add(BLACK, 56, "000000101000")
BitParser.add(BLACK, 57, "000001011000")
BitParser.add(BLACK, 58, "000001011001")
BitParser.add(BLACK, 59, "000000101011")
BitParser.add(BLACK, 60, "000000101100")
BitParser.add(BLACK, 61, "000001011010")
BitParser.add(BLACK, 62, "000001100110")
BitParser.add(BLACK, 63, "000001100111")
BitParser.add(BLACK, 64, "0000001111")
BitParser.add(BLACK, 128, "000011001000")
BitParser.add(BLACK, 192, "000011001001")
BitParser.add(BLACK, 256, "000001011011")
BitParser.add(BLACK, 320, "000000110011")
BitParser.add(BLACK, 384, "000000110100")
BitParser.add(BLACK, 448, "000000110101")
BitParser.add(BLACK, 512, "0000001101100")
BitParser.add(BLACK, 576, "0000001101101")
BitParser.add(BLACK, 640, "0000001001010")
BitParser.add(BLACK, 704, "0000001001011")
BitParser.add(BLACK, 768, "0000001001100")
BitParser.add(BLACK, 832, "0000001001101")
BitParser.add(BLACK, 896, "0000001110010")
BitParser.add(BLACK, 960, "0000001110011")
BitParser.add(BLACK, 1024, "0000001110100")
BitParser.add(BLACK, 1088, "0000001110101")
BitParser.add(BLACK, 1152, "0000001110110")
BitParser.add(BLACK, 1216, "0000001110111")
BitParser.add(BLACK, 1280, "0000001010010")
BitParser.add(BLACK, 1344, "0000001010011")
BitParser.add(BLACK, 1408, "0000001010100")
BitParser.add(BLACK, 1472, "0000001010101")
BitParser.add(BLACK, 1536, "0000001011010")
BitParser.add(BLACK, 1600, "0000001011011")
BitParser.add(BLACK, 1664, "0000001100100")
BitParser.add(BLACK, 1728, "0000001100101")
BitParser.add(BLACK, 1792, "00000001000")
BitParser.add(BLACK, 1856, "00000001100")
BitParser.add(BLACK, 1920, "00000001101")
BitParser.add(BLACK, 1984, "000000010010")
BitParser.add(BLACK, 2048, "000000010011")
BitParser.add(BLACK, 2112, "000000010100")
BitParser.add(BLACK, 2176, "000000010101")
BitParser.add(BLACK, 2240, "000000010110")
BitParser.add(BLACK, 2304, "000000010111")
BitParser.add(BLACK, 2368, "000000011100")
BitParser.add(BLACK, 2432, "000000011101")
BitParser.add(BLACK, 2496, "000000011110")
BitParser.add(BLACK, 2560, "000000011111")
UNCOMPRESSED = [None, None]
BitParser.add(UNCOMPRESSED, "1", "1")
BitParser.add(UNCOMPRESSED, "01", "01")
BitParser.add(UNCOMPRESSED, "001", "001")
BitParser.add(UNCOMPRESSED, "0001", "0001")
BitParser.add(UNCOMPRESSED, "00001", "00001")
BitParser.add(UNCOMPRESSED, "00000", "000001")
BitParser.add(UNCOMPRESSED, "T00", "00000011")
BitParser.add(UNCOMPRESSED, "T10", "00000010")
BitParser.add(UNCOMPRESSED, "T000", "000000011")
BitParser.add(UNCOMPRESSED, "T100", "000000010")
BitParser.add(UNCOMPRESSED, "T0000", "0000000011")
BitParser.add(UNCOMPRESSED, "T1000", "0000000010")
BitParser.add(UNCOMPRESSED, "T00000", "00000000011")
BitParser.add(UNCOMPRESSED, "T10000", "00000000010")
def __init__(self, width, bytealign=False):
BitParser.__init__(self)
self.width = width
self.bytealign = bytealign
self.reset()
return
def feedbytes(self, data):
for byte in get_bytes(data):
try:
for m in (128, 64, 32, 16, 8, 4, 2, 1):
self._parse_bit(byte & m)
except ByteSkip:
self._accept = self._parse_mode
self._state = self.MODE
except EOFB:
break
return
def _parse_mode(self, mode):
if mode == "p":
self._do_pass()
self._flush_line()
return self.MODE
elif mode == "h":
self._n1 = 0
self._accept = self._parse_horiz1
if self._color:
return self.WHITE
else:
return self.BLACK
elif mode == "u":
self._accept = self._parse_uncompressed
return self.UNCOMPRESSED
elif mode == "e":
raise EOFB
elif isinstance(mode, int):
self._do_vertical(mode)
self._flush_line()
return self.MODE
else:
raise InvalidData(mode)
def _parse_horiz1(self, n):
if n is None:
raise InvalidData
self._n1 += n
if n < 64:
self._n2 = 0
self._color = 1 - self._color
self._accept = self._parse_horiz2
if self._color:
return self.WHITE
else:
return self.BLACK
def _parse_horiz2(self, n):
if n is None:
raise InvalidData
self._n2 += n
if n < 64:
self._color = 1 - self._color
self._accept = self._parse_mode
self._do_horizontal(self._n1, self._n2)
self._flush_line()
return self.MODE
elif self._color:
return self.WHITE
else:
return self.BLACK
def _parse_uncompressed(self, bits):
if not bits:
raise InvalidData
if bits.startswith("T"):
self._accept = self._parse_mode
self._color = int(bits[1])
self._do_uncompressed(bits[2:])
return self.MODE
else:
self._do_uncompressed(bits)
return self.UNCOMPRESSED
def _get_bits(self):
return "".join(str(b) for b in self._curline[: self._curpos])
def _get_refline(self, i):
if i < 0:
return "[]" + "".join(str(b) for b in self._refline)
elif len(self._refline) <= i:
return "".join(str(b) for b in self._refline) + "[]"
else:
return (
"".join(str(b) for b in self._refline[:i])
+ "["
+ str(self._refline[i])
+ "]"
+ "".join(str(b) for b in self._refline[i + 1 :])
)
def reset(self):
self._y = 0
self._curline = array.array("b", [1] * self.width)
self._reset_line()
self._accept = self._parse_mode
self._state = self.MODE
return
def output_line(self, y, bits):
print(y, "".join(str(b) for b in bits))
return
def _reset_line(self):
self._refline = self._curline
self._curline = array.array("b", [1] * self.width)
self._curpos = -1
self._color = 1
return
def _flush_line(self):
if self.width <= self._curpos:
self.output_line(self._y, self._curline)
self._y += 1
self._reset_line()
if self.bytealign:
raise ByteSkip
return
def _do_vertical(self, dx):
x1 = self._curpos + 1
while 1:
if x1 == 0:
if self._color == 1 and self._refline[x1] != self._color:
break
elif x1 == len(self._refline):
break
elif (
self._refline[x1 - 1] == self._color
and self._refline[x1] != self._color
):
break
x1 += 1
x1 += dx
x0 = max(0, self._curpos)
x1 = max(0, min(self.width, x1))
if x1 < x0:
for x in range(x1, x0):
self._curline[x] = self._color
elif x0 < x1:
for x in range(x0, x1):
self._curline[x] = self._color
self._curpos = x1
self._color = 1 - self._color
return
def _do_pass(self):
x1 = self._curpos + 1
while 1:
if x1 == 0:
if self._color == 1 and self._refline[x1] != self._color:
break
elif x1 == len(self._refline):
break
elif (
self._refline[x1 - 1] == self._color
and self._refline[x1] != self._color
):
break
x1 += 1
while 1:
if x1 == 0:
if self._color == 0 and self._refline[x1] == self._color:
break
elif x1 == len(self._refline):
break
elif (
self._refline[x1 - 1] != self._color
and self._refline[x1] == self._color
):
break
x1 += 1
for x in range(self._curpos, x1):
self._curline[x] = self._color
self._curpos = x1
return
def _do_horizontal(self, n1, n2):
if self._curpos < 0:
self._curpos = 0
x = self._curpos
for _ in range(n1):
if len(self._curline) <= x:
break
self._curline[x] = self._color
x += 1
for _ in range(n2):
if len(self._curline) <= x:
break
self._curline[x] = 1 - self._color
x += 1
self._curpos = x
return
def _do_uncompressed(self, bits):
for c in bits:
self._curline[self._curpos] = int(c)
self._curpos += 1
self._flush_line()
return
class CCITTFaxDecoder(CCITTG4Parser):
def __init__(self, width, bytealign=False, reversed=False):
CCITTG4Parser.__init__(self, width, bytealign=bytealign)
self.reversed = reversed
self._buf = b""
return
def close(self):
return self._buf
def output_line(self, y, bits):
bytes = array.array("B", [0] * ((len(bits) + 7) // 8))
if self.reversed:
bits = [1 - b for b in bits]
for (i, b) in enumerate(bits):
if b:
bytes[i // 8] += (128, 64, 32, 16, 8, 4, 2, 1)[i % 8]
self._buf += bytes.tostring()
return
def ccittfaxdecode(
data: bytes,
Columns: int,
EncodedByteAlign: bool = False,
BlackIs1: bool = False,
K: Optional[int] = None,
) -> bytes:
if K == -1:
parser = CCITTFaxDecoder(Columns, bytealign=EncodedByteAlign, reversed=BlackIs1)
else:
raise ValueError(K)
parser.feedbytes(data)
return parser.close()
+88
View File
@@ -0,0 +1,88 @@
from io import BytesIO
from typing import List, Optional
from ..exceptions import DecodeFailed
## LZWDecoder
##
class LZWDecoder(object):
def __init__(self, fp: BytesIO):
self.fp = fp
self.buff: int = 0
self.bpos: int = 8
self.nbits: int = 9
self.table: Optional[List[bytes]] = None
self.prevbuf: Optional[bytes] = None
return
def readbits(self, bits):
v = 0
while 1:
# the number of remaining bits we can get from the current buffer.
r = 8 - self.bpos
if bits <= r:
# |-----8-bits-----|
# |-bpos-|-bits-| |
# | |----r----|
v = (v << bits) | ((self.buff >> (r - bits)) & ((1 << bits) - 1))
self.bpos += bits
break
else:
# |-----8-bits-----|
# |-bpos-|---bits----...
# | |----r----|
v = (v << r) | (self.buff & ((1 << r) - 1))
bits -= r
x = self.fp.read(1)
if not x:
raise EOFError
self.buff = ord(x)
self.bpos = 0
return v
def feed(self, code):
x = b""
if code == 256:
self.table = [bytes([c]) for c in range(256)] # 0-255
self.table.append(None) # 256
self.table.append(None) # 257
self.prevbuf = b""
self.nbits = 9
elif code == 257:
pass
elif not self.prevbuf:
x = self.prevbuf = self.table[code]
else:
if code < len(self.table):
x = self.table[code]
self.table.append(self.prevbuf + x[:1])
elif code == len(self.table):
self.table.append(self.prevbuf + self.prevbuf[:1])
x = self.table[code]
else:
raise DecodeFailed("LZW")
l = len(self.table)
if l == 511:
self.nbits = 10
elif l == 1023:
self.nbits = 11
elif l == 2047:
self.nbits = 12
self.prevbuf = x
return x
def run(self):
while 1:
try:
code = self.readbits(self.nbits)
x = self.feed(code)
except EOFError:
break
yield x
return
# lzwdecode
def lzwdecode(data: bytes):
fp = BytesIO(data)
s = LZWDecoder(fp).run()
return b"".join(s)
+36
View File
@@ -0,0 +1,36 @@
#
# RunLength decoder (Adobe version) implementation based on PDF Reference
# version 1.4 section 3.3.4.
#
# * public domain *
#
def rldecode(data: bytes) -> bytes:
"""
RunLength decoder (Adobe version) implementation based on PDF Reference
version 1.4 section 3.3.4:
The RunLengthDecode filter decodes data that has been encoded in a
simple byte-oriented format based on run length. The encoded data
is a sequence of runs, where each run consists of a length byte
followed by 1 to 128 bytes of data. If the length byte is in the
range 0 to 127, the following length + 1 (1 to 128) bytes are
copied literally during decompression. If length is in the range
129 to 255, the following single byte is to be copied 257 - length
(2 to 128) times during decompression. A length value of 128
denotes EOD.
"""
decoded = b""
i: int = 0
while i < len(data):
length = data[i]
if length == 128:
break
if length >= 0 and length < 128:
for j in range(i + 1, (i + 1) + (length + 1)):
decoded += bytes([data[j]])
i = (i + 1) + (length + 1)
if length > 128:
run: bytes = bytes([data[i + 1]]) * (257 - length)
decoded += run
i = (i + 1) + 1
return decoded
+46
View File
@@ -0,0 +1,46 @@
import enum
from typing import Dict
from ..exceptions import InvalidDecoderOrNotImplemented
class FilterType(enum.Enum):
"""Standard PDF Filter types see PDF spec 1.7 section 7.4 table 6"""
ASCIIHexDecode = enum.auto()
ASCII85Decode = enum.auto()
LZWDecode = enum.auto()
FlateDecode = enum.auto()
RunLengthDecode = enum.auto()
CCITTFaxDecode = enum.auto()
JBIG2Decode = enum.auto()
DCTDecode = enum.auto()
JPXDecode = enum.auto()
Crypt = enum.auto()
_filter_type_name_mapping: Dict[str, FilterType] = {
"FlateDecode": FilterType.FlateDecode,
"Fl": FilterType.FlateDecode,
"LZWDecode": FilterType.LZWDecode,
"LZW": FilterType.LZWDecode,
"ASCII85Decode": FilterType.ASCII85Decode,
"A85": FilterType.ASCII85Decode,
"ASCIIHexDecode": FilterType.ASCIIHexDecode,
"AHx": FilterType.ASCIIHexDecode,
"RunLengthDecode": FilterType.RunLengthDecode,
"RL": FilterType.RunLengthDecode,
"CCITTFaxDecode": FilterType.CCITTFaxDecode,
"CCF": FilterType.CCITTFaxDecode,
"DCTDecode": FilterType.DCTDecode,
"DCT": FilterType.DCTDecode,
"Crypt": FilterType.Crypt,
"JBIG2Decode": FilterType.JBIG2Decode,
"JPXDecode": FilterType.JPXDecode,
}
def detect_filter_type(name: str) -> FilterType:
ftype = _filter_type_name_mapping.get(name, None)
if ftype is None:
raise InvalidDecoderOrNotImplemented(name)
return ftype
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,14 +1,24 @@
#!/usr/bin/env python
from unittest import TestCase
# -*- coding: utf-8 -*-
from pdfmajor.parser_v3.fliters.ccit import CCITTG4Parser
from pdfmajor.parser_v3.fliters.lzw import lzwdecode
from pdfmajor.parser_v3.fliters.rld import rldecode
import unittest
from pdfmajor.parser.PDFStream.ccitt import CCITTG4Parser
## Test cases
##
class TestCCITTG4Parser(unittest.TestCase):
class Filters(TestCase):
def test_lzwdecode(self):
self.assertEqual(
lzwdecode(b"\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01"),
b"\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42",
)
def test_rldecode(self):
self.assertEqual(
rldecode(b"\x05123456\xfa7\x04abcde\x80junk"), b"1234567777777abcde"
)
class CCIT(TestCase):
def get_parser(self, bits):
parser = CCITTG4Parser(len(bits))
parser._curline = [int(c) for c in bits]
@@ -16,60 +26,60 @@ class TestCCITTG4Parser(unittest.TestCase):
return parser
def test_b1(self):
parser = self.get_parser('00000')
parser = self.get_parser("00000")
parser._do_vertical(0)
self.assertEqual(parser._curpos, 0)
return
def test_b2(self):
parser = self.get_parser('10000')
parser = self.get_parser("10000")
parser._do_vertical(-1)
self.assertEqual(parser._curpos, 0)
return
def test_b3(self):
parser = self.get_parser('000111')
parser = self.get_parser("000111")
parser._do_pass()
self.assertEqual(parser._curpos, 3)
self.assertEqual(parser._get_bits(), '111')
self.assertEqual(parser._get_bits(), "111")
return
def test_b4(self):
parser = self.get_parser('00000')
parser = self.get_parser("00000")
parser._do_vertical(+2)
self.assertEqual(parser._curpos, 2)
self.assertEqual(parser._get_bits(), '11')
self.assertEqual(parser._get_bits(), "11")
return
def test_b5(self):
parser = self.get_parser('11111111100')
parser = self.get_parser("11111111100")
parser._do_horizontal(0, 3)
self.assertEqual(parser._curpos, 3)
parser._do_vertical(1)
self.assertEqual(parser._curpos, 10)
self.assertEqual(parser._get_bits(), '0001111111')
self.assertEqual(parser._get_bits(), "0001111111")
return
def test_e1(self):
parser = self.get_parser('10000')
parser = self.get_parser("10000")
parser._do_vertical(0)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(0)
self.assertEqual(parser._curpos, 5)
self.assertEqual(parser._get_bits(), '10000')
self.assertEqual(parser._get_bits(), "10000")
return
def test_e2(self):
parser = self.get_parser('10011')
parser = self.get_parser("10011")
parser._do_vertical(0)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(2)
self.assertEqual(parser._curpos, 5)
self.assertEqual(parser._get_bits(), '10000')
self.assertEqual(parser._get_bits(), "10000")
return
def test_e3(self):
parser = self.get_parser('011111')
parser = self.get_parser("011111")
parser._color = 0
parser._do_vertical(0)
self.assertEqual(parser._color, 1)
@@ -79,85 +89,82 @@ class TestCCITTG4Parser(unittest.TestCase):
self.assertEqual(parser._curpos, 4)
parser._do_vertical(0)
self.assertEqual(parser._curpos, 6)
self.assertEqual(parser._get_bits(), '011100')
self.assertEqual(parser._get_bits(), "011100")
return
def test_e4(self):
parser = self.get_parser('10000')
parser = self.get_parser("10000")
parser._do_vertical(0)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(-2)
self.assertEqual(parser._curpos, 3)
parser._do_vertical(0)
self.assertEqual(parser._curpos, 5)
self.assertEqual(parser._get_bits(), '10011')
self.assertEqual(parser._get_bits(), "10011")
return
def test_e5(self):
parser = self.get_parser('011000')
parser = self.get_parser("011000")
parser._color = 0
parser._do_vertical(0)
self.assertEqual(parser._curpos, 1)
parser._do_vertical(3)
self.assertEqual(parser._curpos, 6)
self.assertEqual(parser._get_bits(), '011111')
self.assertEqual(parser._get_bits(), "011111")
return
def test_e6(self):
parser = self.get_parser('11001')
parser = self.get_parser("11001")
parser._do_pass()
self.assertEqual(parser._curpos, 4)
parser._do_vertical(0)
self.assertEqual(parser._curpos, 5)
self.assertEqual(parser._get_bits(), '11111')
self.assertEqual(parser._get_bits(), "11111")
return
def test_e7(self):
parser = self.get_parser('0000000000')
parser = self.get_parser("0000000000")
parser._curpos = 2
parser._color = 1
parser._do_horizontal(2, 6)
self.assertEqual(parser._curpos, 10)
self.assertEqual(parser._get_bits(), '1111000000')
self.assertEqual(parser._get_bits(), "1111000000")
return
def test_e8(self):
parser = self.get_parser('001100000')
parser = self.get_parser("001100000")
parser._curpos = 1
parser._color = 0
parser._do_vertical(0)
self.assertEqual(parser._curpos, 2)
parser._do_horizontal(7, 0)
self.assertEqual(parser._curpos, 9)
self.assertEqual(parser._get_bits(), '101111111')
self.assertEqual(parser._get_bits(), "101111111")
return
def test_m1(self):
parser = self.get_parser('10101')
parser = self.get_parser("10101")
parser._do_pass()
self.assertEqual(parser._curpos, 2)
parser._do_pass()
self.assertEqual(parser._curpos, 4)
self.assertEqual(parser._get_bits(), '1111')
self.assertEqual(parser._get_bits(), "1111")
return
def test_m2(self):
parser = self.get_parser('101011')
parser = self.get_parser("101011")
parser._do_vertical(-1)
parser._do_vertical(-1)
parser._do_vertical(1)
parser._do_horizontal(1, 1)
self.assertEqual(parser._get_bits(), '011101')
self.assertEqual(parser._get_bits(), "011101")
return
def test_m3(self):
parser = self.get_parser('10111011')
parser = self.get_parser("10111011")
parser._do_vertical(-1)
parser._do_pass()
parser._do_vertical(1)
parser._do_vertical(1)
self.assertEqual(parser._get_bits(), '00000001')
self.assertEqual(parser._get_bits(), "00000001")
return
if __name__ == '__main__':
unittest.main()
-35
View File
@@ -1,35 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from unittest import TestCase, main
#test of various compression/encoding modules (previously in doctests):
from pdfmajor.utils.ascii85 import ascii85decode, asciihexdecode
from pdfmajor.utils.lzw import LZWDecoder, lzwdecode
from pdfmajor.parser.PDFStream import rldecode
import binascii
def hex(b): return binascii.hexlify(b) #encode('hex')
def dehex(b): return binascii.unhexlify(b) #decode('hex')
class TestAscii85(TestCase):
def test_ascii85decode(self):
#The sample string is taken from: http://en.wikipedia.org/w/index.php?title=Ascii85
self.assertEqual(ascii85decode(b'9jqo^BlbD-BleB1DJ+*+F(f,q'),b'Man is distinguished')
self.assertEqual(ascii85decode(b'E,9)oF*2M7/c~>'),b'pleasure.')
def test_asciihexdecode(self):
self.assertEqual(asciihexdecode(b'61 62 2e6364 65'),b'ab.cde')
self.assertEqual(asciihexdecode(b'61 62 2e6364 657>'),b'ab.cdep')
self.assertEqual(asciihexdecode(b'7>'),b'p')
class TestLzw(TestCase):
def test_lzwdecode(self):
self.assertEqual(lzwdecode(b'\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01'),b'\x2d\x2d\x2d\x2d\x2d\x41\x2d\x2d\x2d\x42')
class TestRunlength(TestCase):
def test_rldecode(self):
self.assertEqual(rldecode(b'\x05123456\xfa7\x04abcde\x80junk'),b'1234567777777abcde')
if __name__ == '__main__':
main()
-98
View File
@@ -1,98 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
from unittest import TestCase, main
from io import BytesIO
from pdfmajor.parser.PSStackParser import KWD, LIT, PSBaseParser, PSStackParser
from pdfmajor.execptions import PSEOF
## Simplistic Test cases
##
TESTDATA = br'''%!PS
begin end
" @ #
/a/BCD /Some_Name /foo#5f#xbaa
0 +1 -2 .5 1.234
(abc) () (abc ( def ) ghi)
(def\040\0\0404ghi) (bach\\slask) (foo\nbaa)
(this % is not a comment.)
(foo
baa)
(foo\
baa)
<> <20> < 40 4020 >
<abcd00
12345>
func/a/b{(c)do*}def
[ 1 (z) ! ]
<< /foo (bar) >>
'''
TOKENS = [
(5, KWD(b'begin')), (11, KWD(b'end')), (16, KWD(b'"')), (19, KWD(b'@')),
(21, KWD(b'#')), (23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
(65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'),
(98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'),
(143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'),
(191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'),
(226, KWD(b'func')), (230, LIT('a')), (232, LIT('b')),
(234, KWD(b'{')), (235, b'c'), (238, KWD(b'do*')), (241, KWD(b'}')),
(242, KWD(b'def')), (246, KWD(b'[')), (248, 1), (250, b'z'), (254, KWD(b'!')),
(256, KWD(b']')), (258, KWD(b'<<')), (261, LIT('foo')), (266, b'bar'),
(272, KWD(b'>>'))
]
OBJS = [
(23, LIT('a')), (25, LIT('BCD')), (30, LIT('Some_Name')),
(41, LIT('foo_xbaa')), (54, 0), (56, 1), (59, -2), (62, 0.5),
(65, 1.234), (71, b'abc'), (77, b''), (80, b'abc ( def ) ghi'),
(98, b'def \x00 4ghi'), (118, b'bach\\slask'), (132, b'foo\nbaa'),
(143, b'this % is not a comment.'), (170, b'foo\nbaa'), (180, b'foobaa'),
(191, b''), (194, b' '), (199, b'@@ '), (211, b'\xab\xcd\x00\x124\x05'),
(230, LIT('a')), (232, LIT('b')), (234, [b'c']), (246, [1, b'z']),
(258, {'foo': b'bar'}),
]
class TestPSBaseParser(TestCase):
def get_tokens(self, s):
class MyParser(PSStackParser):
def flush(self):
self.add_results(*self.popall())
parser = MyParser(BytesIO(s))
r = []
try:
while True:
r.append(parser.nexttoken())
except PSEOF:
pass
return r
def get_objects(self, s):
class MyParser(PSStackParser):
def flush(self):
self.add_results(*self.popall())
parser = MyParser(BytesIO(s))
r = []
try:
while True:
r.append(parser.nextobject())
except PSEOF:
pass
return r
def test_1(self):
tokens = self.get_tokens(TESTDATA)
logging.info(tokens)
self.assertEqual(tokens, TOKENS)
return
def test_2(self):
objs = self.get_objects(TESTDATA)
logging.info(objs)
self.assertEqual(objs, OBJS)
return
if __name__ == '__main__':
#import logging,sys,os,six
#logging.basicConfig(level=logging.DEBUG, filename='%s_%d.%d.log'%(os.path.basename(__file__),sys.version_info[0],sys.version_info[1]))
main()