Skip to content

Commit dc1bc58

Browse files
committed
Update pycatfile.py
1 parent 1423fdb commit dc1bc58

1 file changed

Lines changed: 84 additions & 7 deletions

File tree

pycatfile.py

Lines changed: 84 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,17 @@
7373
except ImportError:
7474
import json
7575

76+
testyaml = False
77+
try:
78+
import oyaml as yaml
79+
testyaml = True
80+
except ImportError:
81+
try:
82+
import yaml
83+
testyaml = True
84+
except ImportError:
85+
testyaml = False
86+
7687
try:
7788
import configparser
7889
except ImportError:
@@ -2619,7 +2630,7 @@ def _load_all_members_spooled(self):
26192630
scanned_leading = 0 # for tolerant header scan
26202631

26212632
while True:
2622-
data = self.file.read(1 << 20) # 1 MiB blocks
2633+
data = self.file.read(__filebuff_size__) # 1 MiB blocks
26232634
if not data:
26242635
if d is not None:
26252636
self._spool.write(d.flush())
@@ -2777,7 +2788,7 @@ def write(self, data):
27772788

27782789
# Buffer and compress in chunks to limit memory
27792790
self._write_buf += data
2780-
if len(self._write_buf) >= (1 << 20): # 1 MiB threshold
2791+
if len(self._write_buf) >= (__filebuff_size__): # 1 MiB threshold
27812792
chunk = self._compressor.compress(bytes(self._write_buf))
27822793
if chunk:
27832794
self.file.write(chunk)
@@ -3082,7 +3093,7 @@ def _load_all_members_spooled(self):
30823093

30833094
self._spool = tempfile.SpooledTemporaryFile(max_size=self.spool_threshold)
30843095

3085-
CHUNK = 1 << 20
3096+
CHUNK = __filebuff_size__
30863097
pending = b""
30873098
d = None
30883099
absolute_offset = 0
@@ -3245,7 +3256,7 @@ def write(self, data):
32453256

32463257
# Stage and compress in chunks
32473258
self._write_buf += data
3248-
if len(self._write_buf) >= (1 << 20): # 1 MiB threshold
3259+
if len(self._write_buf) >= (__filebuff_size__): # 1 MiB threshold
32493260
out = self._compressor.compress(bytes(self._write_buf))
32503261
if out:
32513262
self.file.write(out)
@@ -3698,7 +3709,7 @@ def GetFileChecksum(inbytes, checksumtype="md5", encodedata=True, formatspecs=__
36983709
if CheckSumSupport(algo_key, hashlib_guaranteed):
36993710
h = hashlib.new(algo_key)
37003711
while True:
3701-
chunk = inbytes.read(1 << 20)
3712+
chunk = inbytes.read(__filebuff_size__)
37023713
if not chunk:
37033714
break
37043715
if not isinstance(chunk, (bytes, bytearray, memoryview)):
@@ -4150,6 +4161,28 @@ def ReadFileHeaderDataWithContent(fp, listonly=False, uncompress=True, skipcheck
41504161
fprejsoncontent = ""
41514162
fjsonrawcontent = fprejsoncontent
41524163
fjsoncontent = {}
4164+
elif(testyaml and fjsontype == "yaml"):
4165+
fjsoncontent = {}
4166+
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
4167+
if (fjsonsize > 0):
4168+
try:
4169+
# try base64 → utf-8 → YAML
4170+
fjsonrawcontent = base64.b64decode(fprejsoncontent.encode("UTF-8")).decode("UTF-8")
4171+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4172+
except (binascii.Error, UnicodeDecodeError, yaml.YAMLError):
4173+
try:
4174+
# fall back to treating the bytes as plain text YAML
4175+
fjsonrawcontent = fprejsoncontent
4176+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4177+
except (UnicodeDecodeError, yaml.YAMLError):
4178+
# final fallback: empty
4179+
fprejsoncontent = ""
4180+
fjsonrawcontent = fprejsoncontent
4181+
fjsoncontent = {}
4182+
else:
4183+
fprejsoncontent = ""
4184+
fjsonrawcontent = fprejsoncontent
4185+
fjsoncontent = {}
41534186
elif(fjsontype=="list"):
41544187
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
41554188
flisttmp = MkTempFile()
@@ -4323,6 +4356,28 @@ def ReadFileHeaderDataWithContentToArray(fp, listonly=False, contentasfile=True,
43234356
fprejsoncontent = ""
43244357
fjsonrawcontent = fprejsoncontent
43254358
fjsoncontent = {}
4359+
elif(testyaml and fjsontype == "yaml"):
4360+
fjsoncontent = {}
4361+
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
4362+
if (fjsonsize > 0):
4363+
try:
4364+
# try base64 → utf-8 → YAML
4365+
fjsonrawcontent = base64.b64decode(fprejsoncontent.encode("UTF-8")).decode("UTF-8")
4366+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4367+
except (binascii.Error, UnicodeDecodeError, yaml.YAMLError):
4368+
try:
4369+
# fall back to treating the bytes as plain text YAML
4370+
fjsonrawcontent = fprejsoncontent
4371+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4372+
except (UnicodeDecodeError, yaml.YAMLError):
4373+
# final fallback: empty
4374+
fprejsoncontent = ""
4375+
fjsonrawcontent = fprejsoncontent
4376+
fjsoncontent = {}
4377+
else:
4378+
fprejsoncontent = ""
4379+
fjsonrawcontent = fprejsoncontent
4380+
fjsoncontent = {}
43264381
elif(fjsontype=="list"):
43274382
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
43284383
flisttmp = MkTempFile()
@@ -4509,6 +4564,28 @@ def ReadFileHeaderDataWithContentToList(fp, listonly=False, contentasfile=False,
45094564
fprejsoncontent = ""
45104565
fjsonrawcontent = fprejsoncontent
45114566
fjsoncontent = {}
4567+
elif(testyaml and fjsontype == "yaml"):
4568+
fjsoncontent = {}
4569+
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
4570+
if (fjsonsize > 0):
4571+
try:
4572+
# try base64 → utf-8 → YAML
4573+
fjsonrawcontent = base64.b64decode(fprejsoncontent.encode("UTF-8")).decode("UTF-8")
4574+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4575+
except (binascii.Error, UnicodeDecodeError, yaml.YAMLError):
4576+
try:
4577+
# fall back to treating the bytes as plain text YAML
4578+
fjsonrawcontent = fprejsoncontent
4579+
fjsoncontent = yaml.safe_load(fjsonrawcontent) or {}
4580+
except (UnicodeDecodeError, yaml.YAMLError):
4581+
# final fallback: empty
4582+
fprejsoncontent = ""
4583+
fjsonrawcontent = fprejsoncontent
4584+
fjsoncontent = {}
4585+
else:
4586+
fprejsoncontent = ""
4587+
fjsonrawcontent = fprejsoncontent
4588+
fjsoncontent = {}
45124589
elif(fjsontype=="list"):
45134590
fprejsoncontent = fp.read(fjsonsize).decode("UTF-8")
45144591
flisttmp = MkTempFile()
@@ -8765,7 +8842,7 @@ def ensure_filelike(infile, mode="rb", use_mmap=False, **adapter_kw):
87658842

87668843
# ========= copy helpers =========
87678844

8768-
def fast_copy(infp, outfp, bufsize=1 << 20):
8845+
def fast_copy(infp, outfp, bufsize=__filebuff_size__):
87698846
"""
87708847
Efficient copy from any readable file-like to any writable file-like.
87718848
Uses readinto() when available to avoid extra allocations.
@@ -8809,7 +8886,7 @@ def copy_file_to_mmap_dest(src_path, outfp, chunk_size=__spoolfile_size__):
88098886
shutil.copyfileobj(fp, outfp, length=chunk_size)
88108887

88118888

8812-
def copy_opaque(src, dst, bufsize=1 << 20, grow_step=64 << 20):
8889+
def copy_opaque(src, dst, bufsize=__filebuff_size__, grow_step=64 << 20):
88138890
"""
88148891
Copy opaque bytes from 'src' (any readable file-like) to 'dst'
88158892
(your mmap-backed FileLikeAdapter or any writable file-like).

0 commit comments

Comments
 (0)