Skip to content

Commit f3d66b0

Browse files
committed
Small update
1 parent e6bd17a commit f3d66b0

3 files changed

Lines changed: 124 additions & 74 deletions

File tree

File renamed without changes.
File renamed without changes.

pycatfile/pyfile.py

Lines changed: 124 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import hmac
2626
import json
2727
import stat
28+
import atexit
2829
import shutil
2930
import base64
3031
import logging
@@ -445,6 +446,97 @@ def add_format(reg, key, magic, ext, name=None, ver="001",
445446
"format_extension": ext,
446447
}
447448

449+
# Process-lifetime extract dir (only used when resources aren't on the filesystem)
450+
_EXTRACT_DIR = None
451+
452+
def _get_extract_dir():
453+
global _EXTRACT_DIR
454+
if _EXTRACT_DIR is None:
455+
_EXTRACT_DIR = tempfile.mkdtemp(prefix="gm2k-cfg-")
456+
atexit.register(lambda: shutil.rmtree(_EXTRACT_DIR, ignore_errors=True))
457+
return _EXTRACT_DIR
458+
459+
def _atomic_write(path, data):
460+
tmp = path + ".tmp"
461+
f = open(tmp, "wb")
462+
try:
463+
f.write(data)
464+
finally:
465+
f.close()
466+
467+
try:
468+
# Py3
469+
os.replace(tmp, path)
470+
except Exception:
471+
# Py2 / fallback
472+
try:
473+
if os.path.exists(path):
474+
os.remove(path)
475+
except Exception:
476+
pass
477+
os.rename(tmp, path)
478+
479+
def resource_path(package_name, filename):
480+
"""
481+
Return a REAL filesystem path to a resource inside this package.
482+
483+
- If package is installed normally on disk: returns the existing path (no copy).
484+
- If package is imported from zip/egg: extracts to a temp dir once and returns that path.
485+
- Falls back to pkg_resources if needed.
486+
"""
487+
files = None
488+
try:
489+
try:
490+
from importlib.resources import files as _files # Py3.9+ (and sometimes available)
491+
files = _files
492+
except Exception:
493+
from importlib_resources import files as _files # backport
494+
files = _files
495+
except Exception:
496+
files = None
497+
498+
if files is not None:
499+
try:
500+
ref = files(package_name).joinpath(filename)
501+
502+
# If backed by filesystem, return that path
503+
try:
504+
return os.fspath(ref) # Py3 only
505+
except Exception:
506+
# zipped/non-path traversable -> extract bytes
507+
out = os.path.join(_get_extract_dir(), filename)
508+
if not os.path.exists(out):
509+
data = ref.read_bytes()
510+
_atomic_write(out, data)
511+
return out
512+
except Exception:
513+
pass
514+
515+
# setuptools fallback
516+
try:
517+
import pkg_resources
518+
return pkg_resources.resource_filename(package_name, filename)
519+
except Exception:
520+
pass
521+
522+
# last resort: __file__ relative (works only when not zipped)
523+
mod = sys.modules.get(package_name)
524+
base = os.path.dirname(getattr(mod, "__file__", __file__))
525+
return os.path.join(base, filename)
526+
527+
def resource_dir(package_name, filenames):
528+
"""
529+
Ensure a set of resources exist as real files in one directory.
530+
Returns that directory path.
531+
"""
532+
paths = [resource_path(package_name, fn) for fn in filenames]
533+
return os.path.dirname(paths[0])
534+
535+
filecfgpath = resource_dir(__name__, [
536+
"catfile.ini",
537+
"catfile.json",
538+
])
539+
448540
__file_format_multi_dict__ = {}
449541
__file_format_default__ = "CatFile"
450542
__include_defaults__ = True
@@ -466,9 +558,9 @@ def add_format(reg, key, magic, ext, name=None, ver="001",
466558
__program_name__ = "Py"+__file_format_default__
467559
__use_env_file__ = True
468560
__use_ini_file__ = True
469-
__use_ini_name__ = "catfile.ini"
561+
__use_ini_name__ = os.path.join(filecfgpath, "catfile.ini")
470562
__use_json_file__ = False
471-
__use_json_name__ = "catfile.json"
563+
__use_json_name__ = os.path.join(filecfgpath, "catfile.json")
472564
if(__use_ini_file__ and __use_json_file__):
473565
__use_json_file__ = False
474566
if('PYARCHIVEFILE_CONFIG_FILE' in os.environ and os.path.exists(os.environ['PYARCHIVEFILE_CONFIG_FILE']) and __use_env_file__):
@@ -3304,45 +3396,34 @@ def UncompressFileAlt(fp, formatspecs=__file_format_multi_dict__, filestart=0):
33043396
if IsNestedDict(formatspecs) and kind in formatspecs:
33053397
formatspecs = formatspecs[kind]
33063398

3307-
# Guard against detector side-effects: ensure we're back at filestart
3308-
try:
3309-
src.seek(filestart, 0)
3310-
except Exception:
3311-
pass
3399+
src.seek(filestart, 0)
33123400

33133401
# Build logical stream (or passthrough)
33143402
if kind == "gzip" and "gzip" in compressionsupport:
33153403
wrapped = gzip.GzipFile(fileobj=src, mode="rb")
3404+
wrapped.seek(0, 0)
33163405
elif kind == "bzip2" and ("bzip2" in compressionsupport or "bz2" in compressionsupport):
33173406
wrapped = bz2.BZ2File(src)
3407+
wrapped.seek(0, 0)
33183408
elif kind in ("lzma","xz") and (("lzma" in compressionsupport) or ("xz" in compressionsupport)):
33193409
wrapped = lzma.LZMAFile(src)
3410+
wrapped.seek(0, 0)
33203411
elif kind == "zstd" and ("zstd" in compressionsupport or "zstandard" in compressionsupport):
33213412
if 'zstd' in compressionsupport:
33223413
wrapped = zstd.ZstdFile(src, mode="rb")
3414+
wrapped.seek(0, 0)
33233415
else:
33243416
return False
33253417
elif kind == "lz4" and "lz4" in compressionsupport:
33263418
wrapped = lz4.frame.LZ4FrameFile(src, mode="rb")
3419+
wrapped.seek(0, 0)
33273420
elif kind == "zlib" and "zlib" in compressionsupport:
33283421
wrapped = ZlibFile(fileobj=src, mode="rb")
3422+
wrapped.seek(0, 0)
33293423
else:
33303424
# Passthrough
33313425
wrapped = src
3332-
try:
3333-
wrapped.seek(filestart, 0)
3334-
except Exception:
3335-
pass
3336-
kind = "" # treat as uncompressed for logic below
3337-
3338-
# Positioning: start-of-member for compressed; filestart for passthrough
3339-
try:
3340-
if kind in compressionsupport:
3341-
wrapped.seek(0, 0)
3342-
else:
3343-
wrapped.seek(filestart, 0)
3344-
except Exception:
3345-
pass
3426+
wrapped.seek(filestart, 0)
33463427

33473428
return wrapped
33483429

@@ -3357,33 +3438,34 @@ def UncompressFile(infile, formatspecs=__file_format_multi_dict__, mode="rb",
33573438
# Compressed branches
33583439
if (compresscheck == "gzip" and "gzip" in compressionsupport):
33593440
fp = gzip.open(infile, mode)
3441+
fp.seek(0, 0)
33603442
elif (compresscheck == "bzip2" and "bzip2" in compressionsupport):
33613443
fp = bz2.open(infile, mode)
3444+
fp.seek(0, 0)
33623445
elif (compresscheck == "zstd" and "zstandard" in compressionsupport):
33633446
if 'zstd' in compressionsupport:
33643447
fp = zstd.ZstdFile(infile, mode=mode)
3448+
fp.seek(0, 0)
33653449
else:
33663450
return False
33673451
elif (compresscheck == "lz4" and "lz4" in compressionsupport):
33683452
fp = lz4.frame.open(infile, mode)
3453+
fp.seek(0, 0)
33693454
elif ((compresscheck == "lzma" or compresscheck == "xz") and "xz" in compressionsupport):
33703455
fp = lzma.open(infile, mode)
3456+
fp.seek(0, 0)
33713457
elif (compresscheck == "zlib" and "zlib" in compressionsupport):
33723458
fp = ZlibFile(infile, mode=mode)
3459+
fp.seek(0, 0)
33733460

33743461
# Uncompressed (or unknown): open plain file
33753462
else:
33763463
fp = open(infile, mode)
3464+
fp.seek(filestart, 0)
33773465

33783466
except FileNotFoundError:
33793467
return False
33803468

3381-
# Position to filestart if caller requested it (mainly for fileobj-based headers)
3382-
try:
3383-
fp.seek(0 if compresscheck else filestart, 0)
3384-
except Exception:
3385-
pass
3386-
33873469
return fp
33883470

33893471
def CompressOpenFileAlt(fp, compression="auto", compressionlevel=None,
@@ -4594,12 +4676,6 @@ def ReadFileDataWithContent(fp, filestart=0, listonly=False, contentasfile=False
45944676
return False
45954677
delimiter = formatspecs['format_delimiter']
45964678
curloc = filestart
4597-
try:
4598-
fp.seek(0, 2)
4599-
except (OSError, ValueError):
4600-
SeekToEndOfFile(fp)
4601-
CatSize = fp.tell()
4602-
CatSizeEnd = CatSize
46034679
fp.seek(curloc, 0)
46044680
inheaderver = str(int(formatspecs['format_ver'].replace(".", "")))
46054681
headeroffset = fp.tell()
@@ -4658,6 +4734,8 @@ def ReadFileDataWithContent(fp, filestart=0, listonly=False, contentasfile=False
46584734
break
46594735
flist.append(HeaderOut)
46604736
countnum = countnum + 1
4737+
CatSize = fp.tell()
4738+
CatSizeEnd = CatSize
46614739
return flist
46624740

46634741

@@ -4666,12 +4744,6 @@ def ReadFileDataWithContentToArray(fp, filestart=0, seekstart=0, seekend=0, list
46664744
return False
46674745
delimiter = formatspecs['format_delimiter']
46684746
curloc = filestart
4669-
try:
4670-
fp.seek(0, 2)
4671-
except (OSError, ValueError):
4672-
SeekToEndOfFile(fp)
4673-
CatSize = fp.tell()
4674-
CatSizeEnd = CatSize
46754747
fp.seek(curloc, 0)
46764748
inheaderver = str(int(formatspecs['format_ver'].replace(".", "")))
46774749
headeroffset = fp.tell()
@@ -4833,7 +4905,7 @@ def ReadFileDataWithContentToArray(fp, filestart=0, seekstart=0, seekend=0, list
48334905
return False
48344906
formversions = re.search('(.*?)(\\d+)', formstring).groups()
48354907
fcompresstype = ""
4836-
outlist = {'fnumfiles': fnumfiles, 'ffilestart': filestart, 'fformat': formversions[0], 'fcompression': fcompresstype, 'fencoding': fhencoding, 'fmtime': fheadmtime, 'fctime': fheadctime, 'fversion': formversions[1], 'fostype': fostype, 'fprojectname': fprojectname, 'fimptype': fpythontype, 'fheadersize': fheadsize, 'fsize': CatSizeEnd, 'fnumfields': fnumfields + 2, 'fformatspecs': formatspecs, 'fseeknextfile': fseeknextfile, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'fjsonchecksumtype': fjsonchecksumtype, 'fjsontype': fjsontype, 'fjsonlen': fjsonlen, 'fjsonsize': fjsonsize, 'fjsonrawdata': fjsonrawcontent, 'fjsondata': fjsoncontent, 'fjstart': fjstart, 'fjend': fjend, 'fjsonchecksum': fjsonchecksum, 'frawheader': [formstring] + inheader, 'fextrafields': fnumextrafields, 'fextrafieldsize': fnumextrafieldsize, 'fextradata': fextrafieldslist, 'fvendorfields': fvendorfields, 'fvendordata': fvendorfieldslist, 'ffilelist': []}
4908+
outlist = {'fnumfiles': fnumfiles, 'ffilestart': filestart, 'fformat': formversions[0], 'fcompression': fcompresstype, 'fencoding': fhencoding, 'fmtime': fheadmtime, 'fctime': fheadctime, 'fversion': formversions[1], 'fostype': fostype, 'fprojectname': fprojectname, 'fimptype': fpythontype, 'fheadersize': fheadsize, 'fnumfields': fnumfields + 2, 'fformatspecs': formatspecs, 'fseeknextfile': fseeknextfile, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'fjsonchecksumtype': fjsonchecksumtype, 'fjsontype': fjsontype, 'fjsonlen': fjsonlen, 'fjsonsize': fjsonsize, 'fjsonrawdata': fjsonrawcontent, 'fjsondata': fjsoncontent, 'fjstart': fjstart, 'fjend': fjend, 'fjsonchecksum': fjsonchecksum, 'frawheader': [formstring] + inheader, 'fextrafields': fnumextrafields, 'fextrafieldsize': fnumextrafieldsize, 'fextradata': fextrafieldslist, 'fvendorfields': fvendorfields, 'fvendordata': fvendorfieldslist, 'ffilelist': []}
48374909
if (seekstart < 0) or (seekstart > fnumfiles):
48384910
seekstart = 0
48394911
if (seekend == 0) or (seekend > fnumfiles) or (seekend < seekstart):
@@ -4916,15 +4988,17 @@ def ReadFileDataWithContentToArray(fp, filestart=0, seekstart=0, seekend=0, list
49164988
il = il + 1
49174989
realidnum = 0
49184990
countnum = seekstart
4919-
while (fp.tell() < CatSizeEnd) if seektoend else (countnum < seekend):
4991+
while (countnum < seekend):
49204992
HeaderOut = ReadFileHeaderDataWithContentToArray(fp, listonly, contentasfile, uncompress, skipchecksum, formatspecs, saltkey)
49214993
if(len(HeaderOut) == 0):
49224994
break
49234995
HeaderOut.update({'fid': realidnum, 'fidalt': realidnum})
49244996
outlist['ffilelist'].append(HeaderOut)
49254997
countnum = countnum + 1
49264998
realidnum = realidnum + 1
4927-
outlist.update({'fp': fp})
4999+
CatSize = fp.tell()
5000+
CatSizeEnd = CatSize
5001+
outlist.update({'fp': fp, 'fsize': CatSizeEnd})
49285002
return outlist
49295003

49305004

@@ -4933,12 +5007,6 @@ def ReadFileDataWithContentToList(fp, filestart=0, seekstart=0, seekend=0, listo
49335007
return False
49345008
delimiter = formatspecs['format_delimiter']
49355009
curloc = filestart
4936-
try:
4937-
fp.seek(0, 2)
4938-
except (OSError, ValueError):
4939-
SeekToEndOfFile(fp)
4940-
CatSize = fp.tell()
4941-
CatSizeEnd = CatSize
49425010
fp.seek(curloc, 0)
49435011
inheaderver = str(int(formatspecs['format_ver'].replace(".", "")))
49445012
headeroffset = fp.tell()
@@ -5186,13 +5254,15 @@ def ReadFileDataWithContentToList(fp, filestart=0, seekstart=0, seekend=0, listo
51865254
il = il + 1
51875255
realidnum = 0
51885256
countnum = seekstart
5189-
while (fp.tell() < CatSizeEnd) if seektoend else (countnum < seekend):
5257+
while (countnum < seekend):
51905258
HeaderOut = ReadFileHeaderDataWithContentToList(fp, listonly, contentasfile, uncompress, skipchecksum, formatspecs, saltkey)
51915259
if(len(HeaderOut) == 0):
51925260
break
51935261
outlist.append(HeaderOut)
51945262
countnum = countnum + 1
51955263
realidnum = realidnum + 1
5264+
CatSize = fp.tell()
5265+
CatSizeEnd = CatSize
51965266
return outlist
51975267

51985268
def ReadInFileWithContentToArray(infile, fmttype="auto", filestart=0, seekstart=0, seekend=0, listonly=False, contentasfile=True, uncompress=True, skipchecksum=False, formatspecs=__file_format_multi_dict__, saltkey=None, seektoend=False):
@@ -5274,17 +5344,11 @@ def ReadInFileWithContentToArray(infile, fmttype="auto", filestart=0, seekstart=
52745344
currentfilepos = readfp.tell()
52755345
else:
52765346
infp = UncompressFileAlt(readfp, formatspecs, currentfilepos)
5347+
if(not infp):
5348+
break
52775349
infp.seek(0, 0)
52785350
currentinfilepos = infp.tell()
5279-
try:
5280-
infp.seek(0, 2)
5281-
except (OSError, ValueError):
5282-
SeekToEndOfFile(infp)
5283-
outinfsize = infp.tell()
5284-
infp.seek(currentinfilepos, 0)
52855351
while True:
5286-
if currentinfilepos >= outinfsize: # stop when function signals False
5287-
break
52885352
oldinfppos = infp.tell()
52895353
compresscheck = CheckCompressionType(infp, formatspecs, currentinfilepos, False)
52905354
if(IsNestedDict(formatspecs) and compresscheck in formatspecs):
@@ -5391,17 +5455,11 @@ def ReadInFileWithContentToList(infile, fmttype="auto", filestart=0, seekstart=0
53915455
currentfilepos = readfp.tell()
53925456
else:
53935457
infp = UncompressFileAlt(readfp, formatspecs, currentfilepos)
5458+
if(not infp):
5459+
break
53945460
infp.seek(0, 0)
53955461
currentinfilepos = infp.tell()
5396-
try:
5397-
infp.seek(0, 2)
5398-
except (OSError, ValueError):
5399-
SeekToEndOfFile(infp)
5400-
outinfsize = infp.tell()
5401-
infp.seek(currentinfilepos, 0)
54025462
while True:
5403-
if currentinfilepos >= outinfsize: # stop when function signals False
5404-
break
54055463
oldinfppos = infp.tell()
54065464
compresscheck = CheckCompressionType(infp, formatspecs, currentinfilepos, False)
54075465
if(IsNestedDict(formatspecs) and compresscheck in formatspecs):
@@ -8019,14 +8077,6 @@ def CatFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__file_form
80198077
return False
80208078
fp = UncompressFile(infile, formatspecs, "rb", filestart)
80218079

8022-
try:
8023-
fp.seek(0, 2)
8024-
except (OSError, ValueError):
8025-
SeekToEndOfFile(fp)
8026-
CatSize = fp.tell()
8027-
CatSizeEnd = CatSize
8028-
fp.seek(0)
8029-
fp.seek(curloc, 0)
80308080
if(IsNestedDict(formatspecs)):
80318081
compresschecking = CheckCompressionType(fp, formatspecs, filestart, False)
80328082
if(compresschecking not in formatspecs):
@@ -8123,7 +8173,7 @@ def CatFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__file_form
81238173
if(verbose):
81248174
VerbosePrintOut("")
81258175
# Iterate either until EOF (seektoend) or fixed count
8126-
while (fp.tell() < CatSizeEnd) if seektoend else (il < fnumfiles):
8176+
while (il < fnumfiles):
81278177
outfhstart = fp.tell()
81288178
if(__use_new_style__):
81298179
inheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter'])

0 commit comments

Comments
 (0)