added reverse line reader
This commit is contained in:
1 parent
37e567a352
commit
b1edcbe51d
2 files changed
+138
-9
No files matched your search
@@ -1,5 +1,5 @@
|
||||
import io
|
||||
from typing import NamedTuple
|
||||
from typing import Iterator, NamedTuple
|
||||
from pdfmajor.exceptions import PDFMajorException
|
||||
|
||||
|
||||
@@ -83,3 +83,31 @@ class BufferStream:
|
||||
|
||||
def __exit__(self):
|
||||
self.close()
|
||||
|
||||
def into_reverse_reader_iter(self) -> Iterator[BufferedBytes]:
|
||||
"""reads the file in reverse, outputting complete lines
|
||||
|
||||
Yields:
|
||||
BufferedBytes
|
||||
"""
|
||||
self.fp.seek(0, 2)
|
||||
pos = self.fp.tell()
|
||||
buf = b""
|
||||
n = -1
|
||||
while 0 < pos:
|
||||
prevpos = pos
|
||||
pos = max(0, pos - self.buffer_size)
|
||||
self.fp.seek(pos)
|
||||
s = self.fp.read(prevpos - pos)
|
||||
if not s:
|
||||
break
|
||||
while True:
|
||||
n = max(s.rfind(b"\r"), s.rfind(b"\n"))
|
||||
if n == -1:
|
||||
buf = s + buf
|
||||
break
|
||||
yield BufferedBytes(pos=n + 1, data=(s[n + 1 :] + buf))
|
||||
s = s[:n]
|
||||
buf = b""
|
||||
if len(buf) > 0 and pos == 0 and n == -1:
|
||||
yield BufferedBytes(pos=n + 1, data=buf)
|
||||
+109
-8
@@ -16,14 +16,15 @@ class UseCase(TestCase):
|
||||
self.assertLessEqual(len(subbuf), it.buffer_size)
|
||||
|
||||
def test_itraises(self):
|
||||
it = BufferStream(io.BytesIO(b"this"))
|
||||
list(it) # read everything
|
||||
with self.assertRaises(StreamEOF):
|
||||
for _ in it:
|
||||
pass
|
||||
with self.assertRaises(EOFError):
|
||||
for _ in it:
|
||||
pass
|
||||
for bufsize in [2, 33, 65, 66, 100, 200]:
|
||||
it = BufferStream(io.BytesIO(b"this"), buffer_size=bufsize)
|
||||
list(it) # read everything
|
||||
with self.assertRaises(StreamEOF):
|
||||
for _ in it:
|
||||
pass
|
||||
with self.assertRaises(EOFError):
|
||||
for _ in it:
|
||||
pass
|
||||
|
||||
def test_seekd(self):
|
||||
for bufsize in [2, 33, 65, 66, 100, 200]:
|
||||
@@ -40,3 +41,103 @@ class UseCase(TestCase):
|
||||
it.seekd(-it.buffer_size)
|
||||
if i > 4:
|
||||
break
|
||||
|
||||
def test_reverse_reader(self):
|
||||
for bufsize in [2, 33, 65, 66, 100, 200, 600, 1000]:
|
||||
it = BufferStream(
|
||||
io.BytesIO(
|
||||
b"""xref
|
||||
0 32
|
||||
0000000000 65535 f
|
||||
0000027259 00000 n
|
||||
0000000019 00000 n
|
||||
0000000344 00000 n
|
||||
0000027402 00000 n
|
||||
0000000364 00000 n
|
||||
0000000714 00000 n
|
||||
0000027545 00000 n
|
||||
0000000734 00000 n
|
||||
0000005949 00000 n
|
||||
0000005970 00000 n
|
||||
0000006176 00000 n
|
||||
0000006503 00000 n
|
||||
0000006700 00000 n
|
||||
0000011679 00000 n
|
||||
0000011701 00000 n
|
||||
0000011905 00000 n
|
||||
0000012232 00000 n
|
||||
0000012427 00000 n
|
||||
0000017804 00000 n
|
||||
0000017826 00000 n
|
||||
0000018037 00000 n
|
||||
0000018364 00000 n
|
||||
0000018565 00000 n
|
||||
0000026258 00000 n
|
||||
0000026280 00000 n
|
||||
0000026477 00000 n
|
||||
0000026882 00000 n
|
||||
0000027141 00000 n
|
||||
0000027204 00000 n
|
||||
0000027650 00000 n
|
||||
0000027747 00000 n
|
||||
trailer
|
||||
<</Size 32/Root 30 0 R
|
||||
/Info 31 0 R
|
||||
/ID [ <4FCA78642D7C01696E050B630CE27BE4>
|
||||
<4FCA78642D7C01696E050B630CE27BE4> ]
|
||||
/DocChecksum /6647D1C1022900091A1C5A662E91248C
|
||||
>>
|
||||
startxref
|
||||
27922
|
||||
%%EOF"""
|
||||
),
|
||||
buffer_size=bufsize,
|
||||
)
|
||||
expected = [
|
||||
b"%%EOF",
|
||||
b"27922",
|
||||
b"startxref",
|
||||
b">>",
|
||||
b"/DocChecksum /6647D1C1022900091A1C5A662E91248C",
|
||||
b"<4FCA78642D7C01696E050B630CE27BE4> ]",
|
||||
b"/ID [ <4FCA78642D7C01696E050B630CE27BE4>",
|
||||
b"/Info 31 0 R",
|
||||
b"<</Size 32/Root 30 0 R",
|
||||
b"trailer",
|
||||
b"0000027747 00000 n ",
|
||||
b"0000027650 00000 n ",
|
||||
b"0000027204 00000 n ",
|
||||
b"0000027141 00000 n ",
|
||||
b"0000026882 00000 n ",
|
||||
b"0000026477 00000 n ",
|
||||
b"0000026280 00000 n ",
|
||||
b"0000026258 00000 n ",
|
||||
b"0000018565 00000 n ",
|
||||
b"0000018364 00000 n ",
|
||||
b"0000018037 00000 n ",
|
||||
b"0000017826 00000 n ",
|
||||
b"0000017804 00000 n ",
|
||||
b"0000012427 00000 n ",
|
||||
b"0000012232 00000 n ",
|
||||
b"0000011905 00000 n ",
|
||||
b"0000011701 00000 n ",
|
||||
b"0000011679 00000 n ",
|
||||
b"0000006700 00000 n ",
|
||||
b"0000006503 00000 n ",
|
||||
b"0000006176 00000 n ",
|
||||
b"0000005970 00000 n ",
|
||||
b"0000005949 00000 n ",
|
||||
b"0000000734 00000 n ",
|
||||
b"0000027545 00000 n ",
|
||||
b"0000000714 00000 n ",
|
||||
b"0000000364 00000 n ",
|
||||
b"0000027402 00000 n ",
|
||||
b"0000000344 00000 n ",
|
||||
b"0000000019 00000 n ",
|
||||
b"0000027259 00000 n ",
|
||||
b"0000000000 65535 f ",
|
||||
b"0 32",
|
||||
b"xref",
|
||||
]
|
||||
for (pos, buf), ebuf in zip(it.into_reverse_reader_iter(), expected):
|
||||
self.assertEqual(buf, ebuf)
|
||||
Reference in new issue
Block a user