Skip to content

Commit 9ea431d

Browse files
committed
Small update
1 parent f4b3898 commit 9ea431d

1 file changed

Lines changed: 265 additions & 0 deletions

File tree

pyfoxfile/pyfoxfile.py

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3790,6 +3790,271 @@ def GetDataFromArrayAlt(structure, path, default=None):
37903790
return default
37913791
return element
37923792

3793+
class MultiOpen:
3794+
def __init__(self, *paths, mode="r+b"):
3795+
self.files = [open(p, mode) for p in paths]
3796+
self.sizes = [os.path.getsize(p) for p in paths]
3797+
self.total_size = sum(self.sizes)
3798+
self.position = 0
3799+
3800+
def tell(self):
3801+
return self.position
3802+
3803+
def seek(self, offset, whence=os.SEEK_SET):
3804+
if whence == os.SEEK_SET:
3805+
new_pos = offset
3806+
elif whence == os.SEEK_CUR:
3807+
new_pos = self.position + offset
3808+
elif whence == os.SEEK_END:
3809+
new_pos = self.total_size + offset
3810+
else:
3811+
raise ValueError("Invalid whence")
3812+
3813+
if not (0 <= new_pos <= self.total_size):
3814+
raise ValueError("Seek out of range")
3815+
3816+
self.position = new_pos
3817+
return self.position
3818+
3819+
def _locate_file(self, position):
3820+
cumulative = 0
3821+
for i, size in enumerate(self.sizes):
3822+
if position < cumulative + size:
3823+
return i, position - cumulative
3824+
cumulative += size
3825+
return len(self.files) - 1, self.sizes[-1]
3826+
3827+
def read(self, size=-1):
3828+
if size < 0:
3829+
size = self.total_size - self.position
3830+
3831+
data = bytearray()
3832+
remaining = size
3833+
3834+
while remaining > 0 and self.position < self.total_size:
3835+
idx, offset = self._locate_file(self.position)
3836+
f = self.files[idx]
3837+
f.seek(offset)
3838+
3839+
to_read = min(remaining, self.sizes[idx] - offset)
3840+
chunk = f.read(to_read)
3841+
3842+
if not chunk:
3843+
break
3844+
3845+
data.extend(chunk)
3846+
read_len = len(chunk)
3847+
self.position += read_len
3848+
remaining -= read_len
3849+
3850+
return bytes(data)
3851+
3852+
def write(self, data):
3853+
remaining = len(data)
3854+
written = 0
3855+
3856+
while remaining > 0 and self.position < self.total_size:
3857+
idx, offset = self._locate_file(self.position)
3858+
f = self.files[idx]
3859+
f.seek(offset)
3860+
3861+
to_write = min(remaining, self.sizes[idx] - offset)
3862+
chunk = data[written:written + to_write]
3863+
f.write(chunk)
3864+
f.flush()
3865+
3866+
self.position += to_write
3867+
written += to_write
3868+
remaining -= to_write
3869+
3870+
return written
3871+
3872+
def close(self):
3873+
for f in self.files:
3874+
f.close()
3875+
3876+
class MultiFileRaw(io.RawIOBase):
3877+
"""
3878+
Treat multiple underlying files as one continuous binary stream.
3879+
Works best when all component files already exist and have fixed sizes.
3880+
3881+
- Supports readinto(), read(), write(), seek(), tell()
3882+
- Intended for binary modes: 'rb', 'r+b', 'wb', etc.
3883+
"""
3884+
def __init__(self, paths, mode="r+b"):
3885+
super().__init__()
3886+
if isinstance(paths, (str, bytes, os.PathLike)):
3887+
paths = [paths]
3888+
self._paths = list(paths)
3889+
self._mode = mode
3890+
self._files = [open(p, mode) for p in self._paths]
3891+
self._sizes = [os.path.getsize(p) for p in self._paths]
3892+
self._total = sum(self._sizes)
3893+
self._pos = 0
3894+
self._closed = False
3895+
3896+
# --- Helpers ---
3897+
def _check_open(self):
3898+
if self._closed:
3899+
raise ValueError("I/O operation on closed MultiFileRaw")
3900+
3901+
def _locate(self, pos: int):
3902+
"""Return (file_index, offset_in_that_file) for absolute stream position."""
3903+
# pos in [0, total]
3904+
acc = 0
3905+
for i, sz in enumerate(self._sizes):
3906+
nxt = acc + sz
3907+
if pos < nxt:
3908+
return i, pos - acc
3909+
acc = nxt
3910+
# pos == total -> point at end of last file
3911+
return len(self._files) - 1, self._sizes[-1]
3912+
3913+
# --- io.RawIOBase API ---
3914+
def readable(self):
3915+
return "r" in self._mode or "+" in self._mode
3916+
3917+
def writable(self):
3918+
return any(ch in self._mode for ch in ("w", "a", "+"))
3919+
3920+
def seekable(self):
3921+
return True
3922+
3923+
def tell(self):
3924+
self._check_open()
3925+
return self._pos
3926+
3927+
def seek(self, offset, whence=os.SEEK_SET):
3928+
self._check_open()
3929+
if whence == os.SEEK_SET:
3930+
new = int(offset)
3931+
elif whence == os.SEEK_CUR:
3932+
new = self._pos + int(offset)
3933+
elif whence == os.SEEK_END:
3934+
new = self._total + int(offset)
3935+
else:
3936+
raise ValueError("Invalid whence")
3937+
3938+
if new < 0 or new > self._total:
3939+
raise ValueError("Seek out of range")
3940+
3941+
self._pos = new
3942+
return self._pos
3943+
3944+
def readinto(self, b):
3945+
"""
3946+
Read bytes into a pre-allocated, writable bytes-like object b.
3947+
Returns number of bytes read (0 at EOF).
3948+
"""
3949+
self._check_open()
3950+
if not self.readable():
3951+
raise io.UnsupportedOperation("not readable")
3952+
3953+
mv = memoryview(b).cast("B")
3954+
if len(mv) == 0:
3955+
return 0
3956+
if self._pos >= self._total:
3957+
return 0
3958+
3959+
remaining = len(mv)
3960+
out_off = 0
3961+
3962+
while remaining > 0 and self._pos < self._total:
3963+
idx, off = self._locate(self._pos)
3964+
f = self._files[idx]
3965+
f.seek(off, os.SEEK_SET)
3966+
3967+
can = min(remaining, self._sizes[idx] - off)
3968+
n = f.readinto(mv[out_off:out_off + can])
3969+
if not n:
3970+
break
3971+
3972+
self._pos += n
3973+
out_off += n
3974+
remaining -= n
3975+
3976+
return out_off
3977+
3978+
def read(self, size=-1):
3979+
self._check_open()
3980+
if size is None or size < 0:
3981+
size = self._total - self._pos
3982+
if size == 0 or self._pos >= self._total:
3983+
return b""
3984+
3985+
buf = bytearray(size)
3986+
n = self.readinto(buf)
3987+
return bytes(buf[:n])
3988+
3989+
def write(self, b):
3990+
self._check_open()
3991+
if not self.writable():
3992+
raise io.UnsupportedOperation("not writable")
3993+
3994+
mv = memoryview(b).cast("B")
3995+
total_to_write = len(mv)
3996+
if total_to_write == 0:
3997+
return 0
3998+
3999+
remaining = total_to_write
4000+
in_off = 0
4001+
4002+
# This implementation writes *within existing file extents*.
4003+
# If you want auto-growing into the last file, say so and I’ll adjust.
4004+
while remaining > 0 and self._pos < self._total:
4005+
idx, off = self._locate(self._pos)
4006+
f = self._files[idx]
4007+
f.seek(off, os.SEEK_SET)
4008+
4009+
can = min(remaining, self._sizes[idx] - off)
4010+
n = f.write(mv[in_off:in_off + can])
4011+
if n is None:
4012+
n = can # some file objects may return None; assume full write
4013+
if n <= 0:
4014+
break
4015+
4016+
self._pos += n
4017+
in_off += n
4018+
remaining -= n
4019+
4020+
return total_to_write - remaining
4021+
4022+
def flush(self):
4023+
self._check_open()
4024+
for f in self._files:
4025+
f.flush()
4026+
4027+
def close(self):
4028+
if not self._closed:
4029+
try:
4030+
for f in self._files:
4031+
try:
4032+
f.close()
4033+
except Exception:
4034+
pass
4035+
finally:
4036+
self._closed = True
4037+
super().close()
4038+
4039+
4040+
def multiopen(paths, mode="r+b", buffering=io.DEFAULT_BUFFER_SIZE):
4041+
"""
4042+
Return a buffered, seekable file-like object over multiple files.
4043+
4044+
Examples:
4045+
f = multiopen(["a.bin","b.bin"], "rb")
4046+
f = multiopen(["a.bin","b.bin"], "r+b") # read/write
4047+
"""
4048+
raw = MultiFileRaw(paths, mode=mode)
4049+
4050+
# Choose an appropriate buffered wrapper
4051+
if "r" in mode and "+" not in mode and "w" not in mode and "a" not in mode:
4052+
return io.BufferedReader(raw, buffer_size=buffering)
4053+
if any(ch in mode for ch in ("w", "a")) and "+" not in mode and "r" not in mode:
4054+
return io.BufferedWriter(raw, buffer_size=buffering)
4055+
# default for random read/write
4056+
return io.BufferedRandom(raw, buffer_size=buffering)
4057+
37934058
# ========= pushback-aware delimiter reader =========
37944059
class _DelimiterReader:
37954060
"""

0 commit comments

Comments
 (0)