added reverse line reader

This commit is contained in:
ari committed 2020-10-21 17:02:53 -04:00
1 parent 37e567a352
commit b1edcbe51d
2 files changed
+138 -9

No files matched your search

+29 -1
View File
@@ -1,5 +1,5 @@
import io
from typing import NamedTuple
from typing import Iterator, NamedTuple
from pdfmajor.exceptions import PDFMajorException
@@ -83,3 +83,31 @@ class BufferStream:
def __exit__(self):
self.close()
def into_reverse_reader_iter(self) -> Iterator[BufferedBytes]:
"""reads the file in reverse, outputting complete lines
Yields:
BufferedBytes
"""
self.fp.seek(0, 2)
pos = self.fp.tell()
buf = b""
n = -1
while 0 < pos:
prevpos = pos
pos = max(0, pos - self.buffer_size)
self.fp.seek(pos)
s = self.fp.read(prevpos - pos)
if not s:
break
while True:
n = max(s.rfind(b"\r"), s.rfind(b"\n"))
if n == -1:
buf = s + buf
break
yield BufferedBytes(pos=n + 1, data=(s[n + 1 :] + buf))
s = s[:n]
buf = b""
if len(buf) > 0 and pos == 0 and n == -1:
yield BufferedBytes(pos=n + 1, data=buf)
+109 -8
View File
@@ -16,14 +16,15 @@ class UseCase(TestCase):
self.assertLessEqual(len(subbuf), it.buffer_size)
def test_itraises(self):
it = BufferStream(io.BytesIO(b"this"))
list(it) # read everything
with self.assertRaises(StreamEOF):
for _ in it:
pass
with self.assertRaises(EOFError):
for _ in it:
pass
for bufsize in [2, 33, 65, 66, 100, 200]:
it = BufferStream(io.BytesIO(b"this"), buffer_size=bufsize)
list(it) # read everything
with self.assertRaises(StreamEOF):
for _ in it:
pass
with self.assertRaises(EOFError):
for _ in it:
pass
def test_seekd(self):
for bufsize in [2, 33, 65, 66, 100, 200]:
@@ -40,3 +41,103 @@ class UseCase(TestCase):
it.seekd(-it.buffer_size)
if i > 4:
break
def test_reverse_reader(self):
for bufsize in [2, 33, 65, 66, 100, 200, 600, 1000]:
it = BufferStream(
io.BytesIO(
b"""xref
0 32
0000000000 65535 f
0000027259 00000 n
0000000019 00000 n
0000000344 00000 n
0000027402 00000 n
0000000364 00000 n
0000000714 00000 n
0000027545 00000 n
0000000734 00000 n
0000005949 00000 n
0000005970 00000 n
0000006176 00000 n
0000006503 00000 n
0000006700 00000 n
0000011679 00000 n
0000011701 00000 n
0000011905 00000 n
0000012232 00000 n
0000012427 00000 n
0000017804 00000 n
0000017826 00000 n
0000018037 00000 n
0000018364 00000 n
0000018565 00000 n
0000026258 00000 n
0000026280 00000 n
0000026477 00000 n
0000026882 00000 n
0000027141 00000 n
0000027204 00000 n
0000027650 00000 n
0000027747 00000 n
trailer
<</Size 32/Root 30 0 R
/Info 31 0 R
/ID [ <4FCA78642D7C01696E050B630CE27BE4>
<4FCA78642D7C01696E050B630CE27BE4> ]
/DocChecksum /6647D1C1022900091A1C5A662E91248C
>>
startxref
27922
%%EOF"""
),
buffer_size=bufsize,
)
expected = [
b"%%EOF",
b"27922",
b"startxref",
b">>",
b"/DocChecksum /6647D1C1022900091A1C5A662E91248C",
b"<4FCA78642D7C01696E050B630CE27BE4> ]",
b"/ID [ <4FCA78642D7C01696E050B630CE27BE4>",
b"/Info 31 0 R",
b"<</Size 32/Root 30 0 R",
b"trailer",
b"0000027747 00000 n ",
b"0000027650 00000 n ",
b"0000027204 00000 n ",
b"0000027141 00000 n ",
b"0000026882 00000 n ",
b"0000026477 00000 n ",
b"0000026280 00000 n ",
b"0000026258 00000 n ",
b"0000018565 00000 n ",
b"0000018364 00000 n ",
b"0000018037 00000 n ",
b"0000017826 00000 n ",
b"0000017804 00000 n ",
b"0000012427 00000 n ",
b"0000012232 00000 n ",
b"0000011905 00000 n ",
b"0000011701 00000 n ",
b"0000011679 00000 n ",
b"0000006700 00000 n ",
b"0000006503 00000 n ",
b"0000006176 00000 n ",
b"0000005970 00000 n ",
b"0000005949 00000 n ",
b"0000000734 00000 n ",
b"0000027545 00000 n ",
b"0000000714 00000 n ",
b"0000000364 00000 n ",
b"0000027402 00000 n ",
b"0000000344 00000 n ",
b"0000000019 00000 n ",
b"0000027259 00000 n ",
b"0000000000 65535 f ",
b"0 32",
b"xref",
]
for (pos, buf), ebuf in zip(it.into_reverse_reader_iter(), expected):
self.assertEqual(buf, ebuf)