Skip to content

Commit a2b8af0

Browse files
jaredmauchjaredmauch
committed
archiver
1 parent 6c6d41c commit a2b8af0

4 files changed

Lines changed: 70 additions & 2 deletions

File tree

‎Mailman/Archiver/Archiver.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,9 +162,11 @@ def __archive_to_mbox(self, post):
162162
"""Retain a text copy of the message in an mbox file."""
163163
try:
164164
afn = self.ArchiveFileName()
165+
syslog('debug', 'Archiver: Writing to mbox file: %s', afn)
165166
mbox = self.__archive_file(afn)
166167
mbox.AppendMessage(post)
167168
mbox.close()
169+
syslog('debug', 'Archiver: Successfully wrote message to mbox file: %s', afn)
168170
except IOError as msg:
169171
syslog('error', 'Archive file access failure:\n\t%s %s', afn, msg)
170172
raise
@@ -186,16 +188,24 @@ def ExternalArchive(self, ar, txt):
186188
#
187189
def ArchiveMail(self, msg):
188190
"""Store postings in mbox and/or pipermail archive, depending."""
191+
from Mailman import syslog
192+
syslog('debug', 'Archiver: Starting ArchiveMail for list %s', self.internal_name())
193+
189194
# Fork so archival errors won't disrupt normal list delivery
190195
if mm_cfg.ARCHIVE_TO_MBOX == -1:
196+
syslog('debug', 'Archiver: ARCHIVE_TO_MBOX is -1, archiving disabled')
191197
return
198+
199+
syslog('debug', 'Archiver: ARCHIVE_TO_MBOX = %s', mm_cfg.ARCHIVE_TO_MBOX)
192200
#
193201
# We don't need an extra archiver lock here because we know the list
194202
# itself must be locked.
195203
if mm_cfg.ARCHIVE_TO_MBOX in (1, 2):
204+
syslog('debug', 'Archiver: Writing to mbox archive')
196205
self.__archive_to_mbox(msg)
197206
if mm_cfg.ARCHIVE_TO_MBOX == 1:
198207
# Archive to mbox only.
208+
syslog('debug', 'Archiver: ARCHIVE_TO_MBOX = 1, mbox only, returning')
199209
return
200210

201211
txt = msg.as_string()
@@ -205,12 +215,17 @@ def ArchiveMail(self, msg):
205215

206216
# should we use the internal or external archiver?
207217
private_p = self.archive_private
218+
syslog('debug', 'Archiver: archive_private = %s', private_p)
219+
208220
if mm_cfg.PUBLIC_EXTERNAL_ARCHIVER and not private_p:
221+
syslog('debug', 'Archiver: Using public external archiver')
209222
self.ExternalArchive(mm_cfg.PUBLIC_EXTERNAL_ARCHIVER, txt)
210223
elif mm_cfg.PRIVATE_EXTERNAL_ARCHIVER and private_p:
224+
syslog('debug', 'Archiver: Using private external archiver')
211225
self.ExternalArchive(mm_cfg.PRIVATE_EXTERNAL_ARCHIVER, txt)
212226
else:
213227
# use the internal archiver
228+
syslog('debug', 'Archiver: Using internal HyperArch archiver')
214229
f = tempfile.NamedTemporaryFile()
215230
if isinstance(txt, str):
216231
txt = txt.encode('utf-8')
@@ -221,6 +236,7 @@ def ArchiveMail(self, msg):
221236
h.processUnixMailbox(f)
222237
h.close()
223238
f.close()
239+
syslog('debug', 'Archiver: Completed internal archiving')
224240

225241
#
226242
# called from MailList.MailList.Save()

‎Mailman/Handlers/ToArchive.py‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,31 @@
2626

2727

2828
def process(mlist, msg, msgdata):
29+
# DEBUG: Log archiver processing start
30+
from Mailman import syslog
31+
syslog('debug', 'ToArchive: Starting archive processing for list %s', mlist.internal_name())
32+
2933
# short circuits
30-
if msgdata.get('isdigest') or not mlist.archive:
34+
if msgdata.get('isdigest'):
35+
syslog('debug', 'ToArchive: Skipping digest message for list %s', mlist.internal_name())
3136
return
37+
if not mlist.archive:
38+
syslog('debug', 'ToArchive: Archiving disabled for list %s', mlist.internal_name())
39+
return
40+
3241
# Common practice seems to favor "X-No-Archive: yes". No other value for
3342
# this header seems to make sense, so we'll just test for it's presence.
3443
# I'm keeping "X-Archive: no" for backwards compatibility.
35-
if 'x-no-archive' in msg or msg.get('x-archive', '').lower() == 'no':
44+
if 'x-no-archive' in msg:
45+
syslog('debug', 'ToArchive: Skipping message with X-No-Archive header for list %s', mlist.internal_name())
46+
return
47+
if msg.get('x-archive', '').lower() == 'no':
48+
syslog('debug', 'ToArchive: Skipping message with X-Archive: no for list %s', mlist.internal_name())
3649
return
50+
3751
# Send the message to the archiver queue
3852
archq = get_switchboard(mm_cfg.ARCHQUEUE_DIR)
53+
syslog('debug', 'ToArchive: Enqueuing message to archive queue for list %s', mlist.internal_name())
3954
# Send the message to the queue
4055
archq.enqueue(msg, msgdata)
56+
syslog('debug', 'ToArchive: Successfully enqueued message to archive queue for list %s', mlist.internal_name())

‎Mailman/Queue/ArchRunner.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ class ArchRunner(Runner):
3030
QDIR = mm_cfg.ARCHQUEUE_DIR
3131

3232
def _dispose(self, mlist, msg, msgdata):
33+
from Mailman import syslog
34+
syslog('debug', 'ArchRunner: Starting archive processing for list %s', mlist.internal_name())
35+
3336
# Support clobber_date, i.e. setting the date in the archive to the
3437
# received date, not the (potentially bogus) Date: header of the
3538
# original message.
@@ -44,26 +47,34 @@ def _dispose(self, mlist, msg, msgdata):
4447
originaldate = None
4548

4649
receivedtime = formatdate(msgdata['received_time'])
50+
syslog('debug', 'ArchRunner: Original date: %s, Received time: %s', originaldate, receivedtime)
51+
4752
if not originaldate:
4853
clobber = 1
54+
syslog('debug', 'ArchRunner: No original date, will clobber')
4955
elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 1:
5056
clobber = 1
57+
syslog('debug', 'ArchRunner: ARCHIVER_CLOBBER_DATE_POLICY = 1, will clobber')
5158
elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 2:
5259
# what's the timestamp on the original message?
5360
try:
5461
tup = parsedate_tz(originaldate)
5562
now = time.time()
5663
if not tup:
5764
clobber = 1
65+
syslog('debug', 'ArchRunner: Could not parse original date, will clobber')
5866
elif abs(now - mktime_tz(tup)) > \
5967
mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW:
6068
clobber = 1
69+
syslog('debug', 'ArchRunner: Date skew too large, will clobber')
6170
except (ValueError, OverflowError, TypeError):
6271
# The likely cause of this is that the year in the Date: field
6372
# is horribly incorrect, e.g. (from SF bug # 571634):
6473
# Date: Tue, 18 Jun 0102 05:12:09 +0500
6574
# Obviously clobber such dates.
6675
clobber = 1
76+
syslog('debug', 'ArchRunner: Date parsing exception, will clobber')
77+
6778
if clobber:
6879
# Use proper header manipulation methods
6980
if 'date' in msg:
@@ -73,19 +84,33 @@ def _dispose(self, mlist, msg, msgdata):
7384
msg['Date'] = receivedtime
7485
if originaldate:
7586
msg['X-Original-Date'] = originaldate
87+
syslog('debug', 'ArchRunner: Clobbered date headers')
88+
7689
# Always put an indication of when we received the message.
7790
msg['X-List-Received-Date'] = receivedtime
91+
7892
# Now try to get the list lock
93+
syslog('debug', 'ArchRunner: Attempting to lock list %s', mlist.internal_name())
7994
try:
8095
mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
96+
syslog('debug', 'ArchRunner: Successfully locked list %s', mlist.internal_name())
8197
except LockFile.TimeOutError:
8298
# oh well, try again later
99+
syslog('debug', 'ArchRunner: Failed to lock list %s, will retry later', mlist.internal_name())
83100
return 1
101+
84102
try:
85103
# Archiving should be done in the list's preferred language, not
86104
# the sender's language.
87105
i18n.set_language(mlist.preferred_language)
106+
syslog('debug', 'ArchRunner: Calling ArchiveMail for list %s', mlist.internal_name())
88107
mlist.ArchiveMail(msg)
108+
syslog('debug', 'ArchRunner: ArchiveMail completed, saving list %s', mlist.internal_name())
89109
mlist.Save()
110+
syslog('debug', 'ArchRunner: Successfully completed archive processing for list %s', mlist.internal_name())
111+
except Exception as e:
112+
syslog('error', 'ArchRunner: Exception during archive processing for list %s: %s', mlist.internal_name(), e)
113+
raise
90114
finally:
91115
mlist.Unlock()
116+
syslog('debug', 'ArchRunner: Unlocked list %s', mlist.internal_name())

‎Mailman/Queue/Switchboard.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,18 @@ def whichq(self):
100100
return self.__whichq
101101

102102
def enqueue(self, _msg, _metadata={}, **_kws):
103+
from Mailman import syslog
103104
# Calculate the SHA hexdigest of the message to get a unique base
104105
# filename. We're also going to use the digest as a hash into the set
105106
# of parallel qrunner processes.
106107
data = _metadata.copy()
107108
data.update(_kws)
108109
listname = data.get('listname', '--nolist--')
110+
111+
# DEBUG: Log archive queue enqueue
112+
if self.__whichq == mm_cfg.ARCHQUEUE_DIR:
113+
syslog('debug', 'Switchboard: Enqueuing message to archive queue for list %s', listname)
114+
109115
# Get some data for the input to the sha hash
110116
now = time.time()
111117
if SAVE_MSGS_AS_PICKLES and not data.get('_plaintext'):
@@ -163,6 +169,11 @@ def enqueue(self, _msg, _metadata={}, **_kws):
163169
finally:
164170
os.umask(omask)
165171
os.rename(tmpfile, filename)
172+
173+
# DEBUG: Log successful enqueue
174+
if self.__whichq == mm_cfg.ARCHQUEUE_DIR:
175+
syslog('debug', 'Switchboard: Successfully enqueued message to archive queue: %s', filebase)
176+
166177
return filebase
167178

168179
def dequeue(self, filebase):

0 commit comments

Comments
 (0)