Skip to content

Commit 42a714a

Browse files
jaredmauchjaredmauch
committed
update
1 parent 4763877 commit 42a714a

1 file changed

Lines changed: 63 additions & 87 deletions

File tree

‎bin/check_db‎

Lines changed: 63 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -72,98 +72,76 @@ def load_pickle(fp):
7272
return None
7373

7474

75-
def testfile(dbfile, verbose=False, listname=None):
76-
# Use our safe loader
77-
loadfunc = load_pickle
78-
if dbfile.endswith('.db') or dbfile.endswith('.db.last'):
79-
loadfunc = marshal.load
80-
elif dbfile.endswith('.pck') or dbfile.endswith('.pck.last'):
81-
loadfunc = load_pickle
82-
else:
83-
assert 0
84-
if verbose:
85-
print('Processing file: %s' % dbfile)
86-
print(' File type: %s' % ('marshal' if dbfile.endswith('.db') else 'pickle'))
87-
print(' File size: %d bytes' % os.path.getsize(dbfile))
88-
fp = open(dbfile, 'rb')
75+
def testfile(dbfile, listname=None, verbose=0):
76+
"""Test the integrity of a list's config database file."""
8977
try:
90-
data = loadfunc(fp)
9178
if verbose:
92-
print(' Successfully loaded data')
93-
if isinstance(data, dict):
94-
print(' Number of entries: %d' % len(data))
95-
if 'version' in data:
96-
print(' Database version: %s' % data['version'])
97-
# Add detailed debugging for request.pck files
98-
if dbfile.endswith('request.pck'):
99-
print(' Request data structure:')
100-
for key, value in data.items():
101-
print(' Key: %s' % key)
102-
print(' Value type: %s' % type(value))
103-
if isinstance(value, (str, bytes)):
104-
print(' Value length: %d' % len(value))
105-
if len(str(value)) < 100: # Only print short values
106-
print(' Value: %s' % value)
107-
elif isinstance(value, (list, tuple)):
108-
print(' Number of items: %d' % len(value))
109-
if len(value) > 0:
110-
print(' First item type: %s' % type(value[0]))
111-
if isinstance(value[0], (str, bytes)) and len(str(value[0])) < 100:
112-
print(' First item: %s' % value[0])
113-
# Handle string/bytes conversion with better error handling
114-
if isinstance(data, bytes):
79+
print(' Loading file %s for list %s...' %
80+
(os.path.basename(dbfile), listname or 'unknown'))
81+
if dbfile.endswith('.pck'):
82+
# Try to load the pickle file
11583
try:
116-
data = data.decode('utf-8', 'replace')
117-
except Exception as e:
118-
if verbose:
119-
print(' Warning: Error decoding bytes in %s for list %s: %s' %
120-
(os.path.basename(dbfile), listname or 'unknown', str(e)))
121-
data = str(data)
122-
elif isinstance(data, dict):
123-
new_data = {}
124-
for k, v in data.items():
125-
try:
126-
if isinstance(k, bytes):
127-
k = k.decode('utf-8', 'replace')
128-
if isinstance(v, bytes):
129-
v = v.decode('utf-8', 'replace')
130-
elif isinstance(v, (list, tuple)):
131-
v = list(v)
132-
for i, item in enumerate(v):
133-
if isinstance(item, bytes):
134-
try:
135-
v[i] = item.decode('utf-8', 'replace')
136-
except Exception as e:
137-
if verbose:
138-
print(' Warning: Error decoding list item in %s for list %s: %s' %
139-
(os.path.basename(dbfile), listname or 'unknown', str(e)))
140-
v[i] = str(item)
141-
new_data[k] = v
142-
except Exception as e:
143-
if verbose:
144-
print(' Warning: Error processing dictionary item in %s for list %s: %s' %
145-
(os.path.basename(dbfile), listname or 'unknown', str(e)))
146-
new_data[str(k)] = str(v)
147-
data = new_data
148-
elif isinstance(data, (list, tuple)):
149-
data = list(data)
150-
for i, item in enumerate(data):
151-
if isinstance(item, bytes):
84+
with open(dbfile, 'rb') as fp:
85+
# First try to detect Python 2 pickle
15286
try:
153-
data[i] = item.decode('utf-8', 'replace')
154-
except Exception as e:
87+
fp.seek(0)
88+
header = fp.read(2)
89+
is_py2_pickle = header.startswith(b'c') or header.startswith(b'C')
15590
if verbose:
156-
print(' Warning: Error decoding list item in %s for list %s: %s' %
157-
(os.path.basename(dbfile), listname or 'unknown', str(e)))
158-
data[i] = str(item)
159-
return data
160-
except Exception as e:
91+
print(' Python 2 pickle detected: %s' % ('Yes' if is_py2_pickle else 'No'))
92+
except:
93+
is_py2_pickle = False
94+
95+
# Now load the actual data
96+
fp.seek(0)
97+
data = pickle.load(fp)
98+
if verbose:
99+
# Get pickle version info
100+
fp.seek(0)
101+
version = pickle.format_version
102+
protocol = pickle.HIGHEST_PROTOCOL
103+
print(' Pickle format version: %s' % version)
104+
print(' Pickle protocol: %d' % protocol)
105+
if is_py2_pickle:
106+
print(' WARNING: This file was likely written with Python 2')
107+
print(' String data may need special handling for Python 3 compatibility')
108+
except (EOFError, pickle.UnpicklingError) as e:
109+
print(' Error loading file %s for list %s: %s' %
110+
(os.path.basename(dbfile), listname or 'unknown', str(e)))
111+
# Always print error for request.pck files, even if not verbose
112+
if dbfile.endswith('request.pck'):
113+
print(' File %s for list %s: ERROR - %s' %
114+
(os.path.basename(dbfile), listname or 'unknown', str(e)))
115+
raise
116+
elif dbfile.endswith('.db'):
117+
# Try to load the marshal file
118+
try:
119+
with open(dbfile, 'rb') as fp:
120+
data = marshal.load(fp)
121+
if verbose:
122+
print(' Marshal format version: %d' % marshal.version)
123+
if marshal.version < 2:
124+
print(' WARNING: This file was likely written with Python 2')
125+
print(' String data may need special handling for Python 3 compatibility')
126+
except (EOFError, ValueError) as e:
127+
print(' Error loading file %s for list %s: %s' %
128+
(os.path.basename(dbfile), listname or 'unknown', str(e)))
129+
# Always print error for request.pck files, even if not verbose
130+
if dbfile.endswith('request.pck'):
131+
print(' File %s for list %s: ERROR - %s' %
132+
(os.path.basename(dbfile), listname or 'unknown', str(e)))
133+
raise
161134
if verbose:
162-
print(' Error loading file %s for list %s: %s' %
135+
print(' File %s for list %s: OK' %
136+
(os.path.basename(dbfile), listname or 'unknown'))
137+
except Exception as e:
138+
print(' Error loading file %s for list %s: %s' %
139+
(os.path.basename(dbfile), listname or 'unknown', str(e)))
140+
# Always print error for request.pck files, even if not verbose
141+
if dbfile.endswith('request.pck'):
142+
print(' File %s for list %s: ERROR - %s' %
163143
(os.path.basename(dbfile), listname or 'unknown', str(e)))
164144
raise
165-
finally:
166-
fp.close()
167145

168146

169147
def main():
@@ -223,9 +201,7 @@ def main():
223201
for dbfile in dbfiles:
224202
if os.path.exists(dbfile):
225203
try:
226-
testfile(dbfile, args.verbose, listname)
227-
if args.verbose:
228-
print(' File %s: OK' % os.path.basename(dbfile))
204+
testfile(dbfile, listname, args.verbose)
229205
except Exception as e:
230206
print(' File %s: ERROR - %s' % (os.path.basename(dbfile), str(e)))
231207
elif args.verbose:

0 commit comments

Comments
 (0)