From 20e4f7bda377f677fbf2128eaa652ea3b6421599 Mon Sep 17 00:00:00 2001 From: ybadri Date: Wed, 13 Mar 2024 11:21:24 +0000 Subject: [PATCH 1/6] Eliminated Forwarder Delivery intefaces and introduced SmtpSender and SmtpMessage instead. Introduced SmtpResponseDescriptor class. --- .../grey/mailismus/imap/server/IMAP4Task.java | 6 +- .../grey/mailismus/mta/deliver/Client.java | 424 +++++++----------- .../mta/deliver/ClientConfiguration.java | 12 +- .../mailismus/mta/deliver/DeliverTask.java | 2 +- .../grey/mailismus/mta/deliver/Delivery.java | 121 ----- .../grey/mailismus/mta/deliver/Forwarder.java | 330 +++++++++----- .../mta/deliver/QueueBasedMessage.java | 93 ++++ .../mta/deliver/QueueBasedRecipient.java | 35 ++ .../com/grey/mailismus/mta/deliver/Relay.java | 164 ++++--- .../grey/mailismus/mta/deliver/Routing.java | 14 +- .../mailismus/mta/deliver/SharedFields.java | 21 +- .../{ => client}/ConnectionConfig.java | 2 +- .../mta/deliver/client/SmtpMessage.java | 22 + .../client/SmtpMessageDefaultImpl.java | 95 ++++ .../mta/deliver/client/SmtpRelay.java | 156 +++++++ .../client/SmtpResponseDescriptor.java | 90 ++++ .../mta/deliver/client/SmtpSender.java | 34 ++ .../mailismus/mta/queue/QueueManager.java | 2 +- .../grey/mailismus/mta/submit/SubmitTask.java | 6 +- .../mailismus/pop3/client/DownloadTask.java | 6 +- .../grey/mailismus/pop3/server/POP3Task.java | 6 +- .../mailismus/mta/deliver/ForwarderTest.java | 105 +++-- .../mailismus/mta/deliver/RoutingTest.java | 52 +-- .../client/SmtpResponseDescriptorTest.java | 133 ++++++ .../mailismus/mta/queue/BaseManagerTest.java | 2 +- .../grey/mailismus/mta/smtp/DeliveryTest.java | 7 +- .../com/grey/mailismus/pop3/POP3Test.java | 8 +- 27 files changed, 1287 insertions(+), 661 deletions(-) delete mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/Delivery.java create mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java create mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedRecipient.java rename server/src/main/java/com/grey/mailismus/mta/deliver/{ => client}/ConnectionConfig.java (99%) create mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java create mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java create mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpRelay.java create mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java create mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSender.java create mode 100644 server/src/test/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptorTest.java diff --git a/server/src/main/java/com/grey/mailismus/imap/server/IMAP4Task.java b/server/src/main/java/com/grey/mailismus/imap/server/IMAP4Task.java index 3218ca4..f92af3a 100644 --- a/server/src/main/java/com/grey/mailismus/imap/server/IMAP4Task.java +++ b/server/src/main/java/com/grey/mailismus/imap/server/IMAP4Task.java @@ -41,10 +41,10 @@ protected boolean stopNaflet() { } @Override - public void eventIndication(Object obj, String eventId) + public void eventIndication(String eventId, Object evtsrc, Object data) { - if (!(obj instanceof ListenerSet) || !EventListenerNAF.EVENTID_ENTITY_STOPPED.equals(eventId)) { - getDispatcher().getLogger().info("IMAP4Task="+getName()+" discarding unexpected event="+obj.getClass().getName()+"/"+eventId); + if (!(evtsrc instanceof ListenerSet) || !EventListenerNAF.EVENTID_ENTITY_STOPPED.equals(eventId)) { + getDispatcher().getLogger().info("IMAP4Task="+getName()+" discarding unexpected event="+eventId+"/"+evtsrc.getClass().getName()+"/"+data); return; } nafletStopped(); diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java index e91aeae..958d8dc 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java @@ -5,20 +5,19 @@ package com.grey.mailismus.mta.deliver; import java.io.IOException; -import java.io.OutputStream; import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; import com.grey.base.utils.IP; import com.grey.base.utils.TSAP; import com.grey.base.utils.EmailAddress; import com.grey.base.utils.ByteArrayRef; import com.grey.base.utils.ByteChars; -import com.grey.base.utils.FileOps; import com.grey.base.sasl.SaslEntity; import com.grey.base.ExceptionUtils; import com.grey.logging.Logger.LEVEL; @@ -32,7 +31,11 @@ import com.grey.mailismus.Task; import com.grey.mailismus.mta.Protocol; -import com.grey.mailismus.mta.queue.MessageRecip; +import com.grey.mailismus.mta.deliver.client.ConnectionConfig; +import com.grey.mailismus.mta.deliver.client.SmtpMessage; +import com.grey.mailismus.mta.deliver.client.SmtpRelay; +import com.grey.mailismus.mta.deliver.client.SmtpResponseDescriptor; +import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.errors.MailismusException; /** @@ -41,8 +44,7 @@ */ class Client extends CM_Client - implements Delivery.MessageSender, - ResolverDNS.Client, + implements ResolverDNS.Client, TimerNAF.Handler { private static final String LOG_PREFIX = "SMTP-Client"; @@ -61,7 +63,7 @@ private enum PROTO_ACTION {A_CONNECT, A_DISCONNECT, A_HELO, A_EHLO, A_MAILFROM, private static final ByteChars FAILMSG_TMT = new ByteChars("SMTP session timed out"); private static final ByteChars FAILMSG_NOSSL = new ByteChars("Remote MTA doesn't support SSL"); private static final ByteChars FAILMSG_NORECIPS = new ByteChars("No valid recipients specified"); - private static final ByteChars FAILMSG_NOSPOOL = new ByteChars("Message deleted from queue"); + private static final ByteChars FAILMSG_LOCAL = new ByteChars("Local message-handling issue"); private static final int TMRTYPE_SESSIONTMT = 1; private static final int TMRTYPE_DISCON = 2; @@ -75,12 +77,14 @@ private enum PROTO_ACTION {A_CONNECT, A_DISCONNECT, A_HELO, A_EHLO, A_MAILFROM, private static final byte S2_ABORT = 1 << 6; private static final byte S2_CNXLIMIT = (byte)(1 << 7); //this is 8-bit, but overflows Byte.MAX_VALUE - private final Delivery.MessageParams msgparams = new Delivery.MessageParams(); - private final SharedFields shared; - private final TSAP remote_tsap_buf; private final List dnsInfo = new ArrayList<>(); + private final List recipientStatus = new ArrayList<>(); + private final TSAP remote_tsap_buf = new TSAP(); + private final SharedFields shared; + private SmtpMessage smtpMessage; //the current message being sent + private SmtpSender smtpSender; + private SmtpRelay active_relay; private ConnectionConfig conncfg; //config to apply to current connection - private Relay active_relay; private TSAP remote_tsap; private PROTO_STATE pstate; private byte state2; //secondary-state, qualifying some of the pstate phases @@ -88,21 +92,21 @@ private enum PROTO_ACTION {A_CONNECT, A_DISCONNECT, A_HELO, A_EHLO, A_MAILFROM, private TimerNAF tmr_sesstmt; private long alt_tmtprotocol; //if non-zero, this overrides ConnectionConfig.tmtprotocol - used to set longer timeout for DATA phase private int mxptr; //indicates which dnsInfo.rrlist node we're currently connecting/connected to - only valid if dnsInfo non-empty - private int recip_id; //indicates which recipient we're currently awaiting a response for + private int recip_id; //indicates which recipient we're currently awaiting a response for xxx can it be replaced by size of recipStatus list? private int recips_sent; //how many recips we've already sent to server - will run ahead of recip_id in pipelining mode private int okrecips; //number of recipients accepted by server private int dataWait; - private short reply_status; + private SmtpResponseDescriptor replyDescriptor; //xxx does this need to be a global? private short disconnect_status; private int pipe_cap; //max pipeline for current connection - 1 means pipelining not enabled private int pipe_count; //number of requests in current pipelined send private SaslEntity.MECH auth_method; private byte auth_step; + private int msgcnt; private int cnxid; //increments with each incarnation - useful for distinguishing Transcript logs private String pfx_log; - @Override public Delivery.MessageParams getMessageParams() {return msgparams;} - @Override public String getLogID() {return pfx_log;} + public String getLogID() {return pfx_log;} private void setFlag(byte f, boolean b) {if (b) {setFlag(f);} else {clearFlag(f);}} private void setFlag(byte f) {state2 |= f;} @@ -110,14 +114,27 @@ private enum PROTO_ACTION {A_CONNECT, A_DISCONNECT, A_HELO, A_EHLO, A_MAILFROM, private boolean isFlagSet(byte f) {return ((state2 & f) == f);} @Override - protected SSLConfig getSSLConfig() {return (active_relay == null ? conncfg.getAnonSSL() : active_relay.sslconfig);} + protected SSLConfig getSSLConfig() {return (active_relay == null ? conncfg.getAnonSSL() : active_relay.getSslConfig());} - Client(SharedFields shared) { - super(shared.getController().getDispatcher(), shared.getBufferGenerator(), shared.getBufferGenerator()); + Client(SharedFields shared, Dispatcher dsptch) { + super(dsptch, shared.getBufferGenerator(), shared.getBufferGenerator()); pfx_log = LOG_PREFIX+": "; this.shared = shared; - //will need to build addresses at connect time if we don't have a default relay - remote_tsap_buf = (shared.getController().getRouting().haveDefaultRelay() ? null : new TSAP()); + } + + public void startConnection(SmtpMessage msg, SmtpSender sender, Relay relay) throws IOException { + initConnection(); + smtpMessage = msg; + smtpSender = sender; + active_relay = relay; + issueAction(PROTO_ACTION.A_CONNECT, PROTO_STATE.S_CONN); + } + + public boolean stop() { + setFlag(S2_ABORT); + if (pstate == PROTO_STATE.S_DISCON) return (tmr_exit == null); // we're already completely stopped + issueDisconnect(0, "Forcibly halted", null); + return false; } private ConnectionConfig getConnectionConfig(int remote_ip) { @@ -135,20 +152,6 @@ private void transitionState(PROTO_STATE newstate) { pstate = newstate; } - @Override - public void start(Delivery.Controller ctl) throws IOException { - initConnection(); - issueAction(PROTO_ACTION.A_CONNECT, PROTO_STATE.S_CONN); - } - - @Override - public boolean stop() { - setFlag(S2_ABORT); - if (pstate == PROTO_STATE.S_DISCON) return (tmr_exit == null); // we're already completely stopped - issueDisconnect(0, "Forcibly halted"); - return false; - } - @Override public void ioReceived(ByteArrayRef rcvdata) throws IOException { if (pstate == PROTO_STATE.S_DISCON) return; //this method can be called in a loop, so skip it after a disconnect @@ -162,10 +165,6 @@ public void ioDisconnected(CharSequence diagnostic) { raiseSafeEvent(PROTO_EVENT.E_DISCONNECTED, null, diagnostic); } - private PROTO_STATE issueDisconnect(int statuscode, CharSequence diagnostic) { - return issueDisconnect(statuscode, diagnostic, null); - } - private PROTO_STATE issueDisconnect(int statuscode, CharSequence diagnostic, ByteArrayRef failmsg) { if (pstate == PROTO_STATE.S_DISCON) return pstate; CharSequence discmsg = "Disconnect"; @@ -209,11 +208,12 @@ private void dnsLookup(boolean as_host) throws IOException { mxptr = 0; dnsInfo.clear(); setFlag(S2_DNSWAIT); + CharSequence domain = smtpMessage.getRecipients().get(0).getDomain(); ResolverAnswer answer; if (as_host) { - answer = shared.getDnsResolver().resolveHostname(msgparams.getDestination(), this, null, 0); + answer = shared.getDnsResolver().resolveHostname(domain, this, null, 0); } else { - answer = shared.getDnsResolver().resolveMailDomain(msgparams.getDestination(), this, null, 0); + answer = shared.getDnsResolver().resolveMailDomain(domain, this, null, 0); } if (answer != null) dnsResolved(getDispatcher(), answer, null); } @@ -291,20 +291,12 @@ private void eventErrorIndication(Throwable ex, TimerNAF tmr) { } private void issueConnect() throws IOException { - active_relay = msgparams.getRelay(); - if (active_relay != null) remote_tsap = active_relay.tsap; + if (active_relay != null) remote_tsap = active_relay.getAddress(); if (remote_tsap != null) { connect(); return; } - - if (msgparams.getDestination() == null) { - // If we got here this message is not routed, so the recipient domain must be null. Such a message - // should never have been handed to this client, as it is intended solely for remote delivery via SMTP. - connectionFailed(Protocol.REPLYCODE_PERMERR_ADDR, "SMTP client cannot deliver to local mailboxes", null); - return; - } dnsLookup(false); } @@ -328,20 +320,9 @@ private void connect() throws UnknownHostException { raiseSafeEvent(PROTO_EVENT.E_DISCONNECTED, null, null); return; } - TSAP tsap = remote_tsap; - Relay interceptor = shared.getController().getRouting().getInterceptor(); - - if (interceptor != null) { - //active_relay==interceptor will be true if we we are trying additional MX relays because first one failed - if (active_relay == null || active_relay == interceptor || !interceptor.dns_only) { - //leave remote_tsap as is so that we log the IP address we would have connected to - active_relay = interceptor; - tsap = active_relay.tsap; - } - } try { - connect(tsap.sockaddr); + connect(remote_tsap.sockaddr); } catch (Throwable ex) { connectionFailed(0, "connect-error", ex); } @@ -358,18 +339,16 @@ protected void connected(boolean success, CharSequence diag, Throwable exconn) t if (getLogger().isActive(lvl) || (shared.getTranscript() != null)) { TSAP local_tsap = TSAP.get(getLocalIP(), getLocalPort(), shared.getTmpTSAP(), true); if (getLogger().isActive(lvl)) { - StringBuilder sb = shared.getTmpSB(); - sb.setLength(0); + StringBuilder sb = shared.getTmpSB(true); sb.append(pfx_log).append(" connected to "); recordConnection(sb, local_tsap); getLogger().log(lvl, sb); } if (shared.getTranscript() != null) { - Relay rly = msgparams.getRelay(); - CharSequence remote = (rly == null ? msgparams.getDestination() : rly.display()); + StringBuilder sb = shared.getTmpSB(true); shared.getTranscript().connection_out(pfx_log, local_tsap.dotted_ip, local_tsap.port, remote_tsap.dotted_ip, remote_tsap.port, getSystemTime(), - remote, usingSSL()); + peerDescription(sb), usingSSL()); } } eventRaised(PROTO_EVENT.E_CONNECTED, null, null); @@ -381,20 +360,16 @@ remote_tsap.dotted_ip, remote_tsap.port, getSystemTime(), // error) and 'diagnostic' gives the reason. // statuscode zero therefore also means that remote_tsap is non-null, as we have attempted a connection. private void connectionFailed(int statuscode, CharSequence diagnostic, Throwable exconn) throws UnknownHostException { - CharSequence extspid = null; - StringBuilder sb = shared.getTmpSB(); + StringBuilder sb = shared.getTmpSB(true); if (statuscode == 0) { if (shared.getTranscript() != null) { - Relay rly = msgparams.getRelay(); - CharSequence remote = (rly == null ? msgparams.getDestination() : rly.display()); + StringBuilder sb2 = shared.getTmpSB2(true); shared.getTranscript().connection_out(pfx_log, null, 0, remote_tsap.dotted_ip, remote_tsap.port, - getSystemTime(), remote, usingSSL()); + getSystemTime(), peerDescription(sb2), usingSSL()); } LEVEL lvl = (MailismusException.isError(exconn) ? LEVEL.WARN : LEVEL.TRC2); if (getLogger().isActive(lvl)) { - extspid = formatSPID(msgparams.getSPID()); - sb.setLength(0); sb.append(pfx_log).append(" failed to connect to "); recordConnection(sb, null); if (diagnostic != null) sb.append(" - ").append(diagnostic); @@ -424,12 +399,10 @@ private void connectionFailed(int statuscode, CharSequence diagnostic, Throwable disconnect_status = (short)statuscode; setFlag(S2_DOMAIN_ERR); LEVEL lvl = LEVEL.TRC; - StringBuilder sbfail = shared.getTmpSB2(); - sbfail.setLength(0); + StringBuilder sbfail = shared.getTmpSB2(true); peerDescription(sbfail); if (getLogger().isActive(lvl) || (shared.getTranscript() != null)) { - if (extspid == null) extspid = formatSPID(msgparams.getSPID()); StringBuilder sbdisc = shared.getDisconnectMsgBuf(); if (diagnostic == sbdisc) { sb.setLength(0); @@ -437,7 +410,7 @@ private void connectionFailed(int statuscode, CharSequence diagnostic, Throwable diagnostic = sb; } sbdisc.setLength(0); - sbdisc.append("Cannot connect to ").append(sbfail).append(" for msgid=").append(extspid); + sbdisc.append("Cannot connect to ").append(sbfail).append(" for msgid=").append(smtpMessage.getMessageId()); if (diagnostic != null) sbdisc.append(" - ").append(diagnostic); diagnostic = sbdisc; @@ -458,13 +431,13 @@ private void connectionFailed(int statuscode, CharSequence diagnostic, Throwable private void endConnection(CharSequence discmsg, ByteArrayRef failmsg) { LEVEL lvl = LEVEL.TRC2; if (getLogger().isActive(lvl)) { - shared.getTmpSB().setLength(0); - shared.getTmpSB().append(pfx_log).append(" ending with state=").append(pstate).append("/0x").append(Integer.toHexString(state2)); - shared.getTmpSB().append(", remote=").append(remote_tsap).append("/dns=").append(dnsInfo.size()); - shared.getTmpSB().append(", msgcnt=").append(msgparams.messageCount()); - if (discmsg != null) shared.getTmpSB().append(" - reason=").append(discmsg); - if (failmsg != null) shared.getTmpSB().append(" - diagnostic=").append(shared.getTmpBC().populateBytes(failmsg)); - getLogger().log(lvl, shared.getTmpSB()); + StringBuilder sb = shared.getTmpSB(true); + sb.append(pfx_log).append(" ending with state=").append(pstate).append("/0x").append(Integer.toHexString(state2)); + sb.append(", remote=").append(remote_tsap).append("/dns=").append(dnsInfo.size()); + sb.append(", msgcnt=").append(msgcnt); + if (discmsg != null) sb.append(" - reason=").append(discmsg); + if (failmsg != null) sb.append(" - diagnostic=").append(shared.getTmpBC().populateBytes(failmsg)); + getLogger().log(lvl, sb); } if (tmr_sesstmt != null) { tmr_sesstmt.cancel(); @@ -482,7 +455,7 @@ private void endConnection(CharSequence discmsg, ByteArrayRef failmsg) { // If the disconnection is unintentional on the server's part and due to some other issues, then (a) is still a good enough reason // to give up, and since it's impossible for us to know whether the disconnect is intentional or not, the combination of no // EHLO support and an inopportune disconnect is irrecoverable anyway. Message delivery can fail, this one has now failed! - disconnect_status = reply_status; + disconnect_status = replyDescriptor.smtpStatus(); String msg = "Disconnected due to EHLO rejection"; if (discmsg != null) msg += " - "+discmsg; discmsg = msg; @@ -490,12 +463,13 @@ private void endConnection(CharSequence discmsg, ByteArrayRef failmsg) { try { if (failmsg == null && discmsg != null) failmsg = shared.getFailMsgBuffer().populate(discmsg); - setRecipientStatus(-1, disconnect_status, failmsg, false); + SmtpResponseDescriptor rsp = new SmtpResponseDescriptor(disconnect_status, null, null); + setRecipientStatus(-1, rsp); } catch (Exception ex) { getLogger().log(LEVEL.WARN, ex, false, pfx_log+" failed to set final recipients status"); } if (remote_tsap != null && !isFlagSet(S2_CNXLIMIT)) shared.decrementServerConnections(remote_tsap.ip, conncfg); - if (remote_tsap_buf != null) remote_tsap_buf.clear(); //don't erase the statically configured TSAPs! + remote_tsap_buf.clear(); if (isFlagSet(S2_DNSWAIT)) { try { @@ -562,7 +536,7 @@ private PROTO_STATE eventRaised(PROTO_EVENT evt, ByteArrayRef rspdata, CharSeque break; case E_LOCALERROR: - issueDisconnect(Protocol.REPLYCODE_TMPERR_LOCAL, "Local Error - "+discmsg); + issueDisconnect(Protocol.REPLYCODE_TMPERR_LOCAL, "Local Error - "+discmsg, null); break; default: @@ -600,7 +574,7 @@ private PROTO_STATE handleReply(ByteArrayRef rspdata) throws IOException { // in response to anything else then we discard the leading lines and wait for the final one, taking its reply code as the // definitive one. CORRECTION!! AOL sends a multi-line Greeting, so just as well we handle continued replies anywhere. setFlag(S2_REPLYCONTD, rspdata.byteAt(Protocol.REPLY_CODELEN) == Protocol.REPLY_CONTD); - reply_status = getReplyCode(rspdata); + replyDescriptor = SmtpResponseDescriptor.parse(rspdata, false); //xxx need flag for enhanced-status if (isFlagSet(S2_REPLYCONTD)) { if (pstate != PROTO_STATE.S_EHLO) return pstate; } else { @@ -625,28 +599,29 @@ private PROTO_STATE handleReply(ByteArrayRef rspdata) throws IOException { break; case S_EHLO: - if (reply_status != okreply) { + if (replyDescriptor.smtpStatus() != okreply) { if (conncfg.isFallbackHelo()) return issueAction(PROTO_ACTION.A_HELO, PROTO_STATE.S_HELO); } else { if (matchesExtension(rspdata, Protocol.EXT_PIPELINE, false)) { pipe_cap = conncfg.getMaxPipeline(); } else if (matchesExtension(rspdata, Protocol.EXT_STLS, false)) { setFlag(S2_SERVER_STLS); - } else if (active_relay != null && active_relay.auth_enabled && auth_method == null - && (matchesExtension(rspdata, Protocol.EXT_AUTH, false) - || (active_relay.auth_compat && matchesExtension(rspdata, Protocol.EXT_AUTH_COMPAT, true)))) { - // loop through the advertised SASL mechanisms and use the first one we support - int lmt = rspdata.limit(); - int off = rspdata.offset(); - while (off != lmt) { - int off2 = off; - while (rspdata.buffer()[off2] > ' ') off2++; - shared.getTmpLightBC().set(rspdata.buffer(), off, off2-off); - auth_method = shared.getAuthTypesSupported().get(shared.getTmpLightBC()); - //server might advertise EXTERNAL anyway without meaning it unless in SSL mode, so keep scanning for better option - if (auth_method != null && auth_method != SaslEntity.MECH.EXTERNAL) break; - off = off2; - while (rspdata.buffer()[off] == ' ') off++; + } else if (matchesExtension(rspdata, Protocol.EXT_AUTH, false) + || (active_relay.isAuthCompat() && matchesExtension(rspdata, Protocol.EXT_AUTH_COMPAT, true))) { + if (active_relay != null && active_relay.isAuthRequired() && auth_method == null) { + // loop through the advertised SASL mechanisms and use the first one we support + int lmt = rspdata.limit(); + int off = rspdata.offset(); + while (off != lmt) { + int off2 = off; + while (rspdata.buffer()[off2] > ' ') off2++; + shared.getTmpLightBC().set(rspdata.buffer(), off, off2-off); + auth_method = shared.getAuthTypesSupported().get(shared.getTmpLightBC()); + //server might advertise EXTERNAL anyway without meaning it unless in SSL mode, so keep scanning for better option + if (auth_method != null && auth_method != SaslEntity.MECH.EXTERNAL) break; + off = off2; + while (rspdata.buffer()[off] == ' ') off++; + } } } } @@ -658,10 +633,10 @@ private PROTO_STATE handleReply(ByteArrayRef rspdata) throws IOException { break; case S_MAILTO: - if (reply_status == Protocol.REPLYCODE_RECIPMOVING) reply_status = okreply; - setRecipientStatus(recip_id++, reply_status, rspdata, true); + //xxx Need to handle this and 252 if (replyDescriptor.smtpStatus() == Protocol.REPLYCODE_RECIPMOVING) reply_status = okreply; + setRecipientStatus(recip_id++, replyDescriptor); okreply = 0; //an error response at this stage merely invalidates the current recipient, so reply status has now been processed - if (recip_id == msgparams.recipCount()) { + if (recip_id == smtpMessage.getRecipients().size()) { if (okrecips == 0) return issueDisconnect(0, "No valid recipients", FAILMSG_NORECIPS); issueAction(PROTO_ACTION.A_DATA, PROTO_STATE.S_DATA, okreply, null, rspdata); break; @@ -690,8 +665,8 @@ private PROTO_STATE handleReply(ByteArrayRef rspdata) throws IOException { break; case S_AUTH: - if (reply_status != Protocol.REPLYCODE_AUTH_OK && reply_status != Protocol.REPLYCODE_AUTH_CONTD) { - return issueDisconnect(reply_status, "Authentication failed", rspdata); + if (replyDescriptor.smtpStatus() != Protocol.REPLYCODE_AUTH_OK && replyDescriptor.smtpStatus() != Protocol.REPLYCODE_AUTH_CONTD) { + return issueDisconnect(replyDescriptor.smtpStatus(), "Authentication failed", rspdata); } sendAuth(rspdata); break; @@ -706,20 +681,19 @@ private PROTO_STATE handleReply(ByteArrayRef rspdata) throws IOException { } private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int okreply, CharSequence discmsg, ByteArrayRef rspdata) throws IOException { - if (okreply != 0 && reply_status != okreply) { + if (okreply != 0 && replyDescriptor.smtpStatus() != okreply) { //note that okreply will already have been reset in state S_MAILTO LEVEL lvl = LEVEL.TRC; if (getLogger().isActive(lvl)) { int len = rspdata.size(); while (rspdata.buffer()[rspdata.offset()+len-1] < ' ') len--; //strip trailing CRLF - StringBuilder sb = shared.getTmpSB(); - sb.setLength(0); + StringBuilder sb = shared.getTmpSB(true); sb.append(pfx_log).append(" rejected in state=").append(pstate).append(" - "); peerDescription(sb); sb.append(" replied: ").append(shared.getTmpLightBC().set(rspdata.buffer(), rspdata.offset(), len)); getLogger().log(lvl, sb); } - return issueDisconnect(reply_status, "Server rejection", rspdata); + return issueDisconnect(replyDescriptor.smtpStatus(), "Server rejection", rspdata); } if (newstate != null) transitionState(newstate); boolean endpipe = (pipe_count == 0 && dataWait != 0); //changed state, but don't send any more until all pipelined commands are acked @@ -732,13 +706,13 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o break; case A_HELO: - auth_method = (active_relay == null ? null : active_relay.auth_override); + auth_method = (active_relay == null ? null : active_relay.getAuthOverride()); reqbuf = shared.getHeloBuffer(conncfg, false); transmit(reqbuf); break; case A_EHLO: - auth_method = (active_relay == null ? null : active_relay.auth_override); + auth_method = (active_relay == null ? null : active_relay.getAuthOverride()); reqbuf = shared.getHeloBuffer(conncfg, true); transmit(reqbuf); break; @@ -760,23 +734,18 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o break; case A_MAILFROM: //all recips members refer to same message, so they have a common Sender - endpipe = sendPipelinedRequest(SMTPREQ_MAILFROM, false, msgparams.getSender(), null, true); + endpipe = sendPipelinedRequest(SMTPREQ_MAILFROM, false, smtpMessage.getSender(), null, true); if (!endpipe) issueAction(PROTO_ACTION.A_MAILTO, null); break; case A_MAILTO: while (!endpipe) { - if (recips_sent == msgparams.recipCount()) { + if (recips_sent == smtpMessage.getRecipients().size()) { issueAction(PROTO_ACTION.A_DATA, null); break; } - MessageRecip recip = msgparams.getRecipient(recips_sent); - if (recip.spid != msgparams.getSPID()) { - throw new IllegalStateException(pfx_log+" has mismatched SPID on recip "+recips_sent+"=" - +recip.domain_to+"/"+recip.qid - +" - "+Integer.toHexString(recip.spid)+" vs "+Integer.toHexString(msgparams.getSPID())); - } - endpipe = sendPipelinedRequest(SMTPREQ_MAILTO, false, recip.mailbox_to, recip.domain_to, true); + SmtpMessage.Recipient recip = smtpMessage.getRecipients().get(recips_sent); + endpipe = sendPipelinedRequest(SMTPREQ_MAILTO, false, recip.getMailbox(), recip.getDomain(), true); recips_sent++; } break; @@ -793,43 +762,53 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o break; case A_MAILBODY: - Path fh_msg = shared.getController().getQueue().getMessage(msgparams.getSPID(), msgparams.getRecipient(0).qid); - long msgbytes = Files.size(fh_msg); - alt_tmtprotocol = calculateMaxTime(msgbytes, conncfg.getMinRateData()); - if (alt_tmtprotocol < conncfg.getIdleTimeout().toMillis()) alt_tmtprotocol = 0; + Object msgdata = smtpMessage.getData().get(); + Supplier dataErrMsg = () -> null; + long msgbytes = 0; try { - getWriter().transmit(fh_msg); + if (msgdata instanceof Path) { + Path p = (Path)msgdata; + dataErrMsg = () -> (Files.exists(p) ? null : "Spool file missing"); + msgbytes = Files.size(p); + getWriter().transmit(p); + } else if (msgdata instanceof byte[]) { + byte[] b = (byte[])msgdata; + msgbytes = b.length; + getWriter().transmit(b); + } else { + dataErrMsg = () -> "Invalid message data supplied - "+(msgdata == null ? null : msgdata.getClass().getName()); + } } catch (IOException ex) { - if (Files.exists(fh_msg, FileOps.LINKOPTS_NONE)) throw ex; //prob some temporary comms issue - return issueDisconnect(Protocol.REPLYCODE_PERMERR_MISC, "Spool file missing", FAILMSG_NOSPOOL); + if (dataErrMsg.get() == null) + throw ex; //prob some temporary comms issue } + if (dataErrMsg.get() != null) + return issueDisconnect(Protocol.REPLYCODE_PERMERR_MISC, dataErrMsg.get(), FAILMSG_LOCAL); + transmit(shared.getSmtpRequestEOM()); + alt_tmtprotocol = calculateMaxTime(msgbytes, conncfg.getMinRateData()); + if (alt_tmtprotocol < conncfg.getIdleTimeout().toMillis()) alt_tmtprotocol = 0; if (shared.getTranscript() != null) { - shared.getTmpSB().setLength(0); - shared.getTmpSB().append("Sent message-body octets=").append(msgbytes).append(" for msgid="); - shared.getTmpSB().append(formatSPID(msgparams.getSPID())); - shared.getTranscript().event(pfx_log, shared.getTmpSB(), getSystemTime()); + StringBuilder sb = shared.getTmpSB(true); + sb.append("Sent message-body octets=").append(msgbytes).append(" for msgid="); + sb.append(smtpMessage.getMessageId()); + shared.getTranscript().event(pfx_log, sb, getSystemTime()); } break; case A_ENDMESSAGE: - shared.getController().messageCompleted(this); - if (msgparams.recipCount() == 0) return issueAction(PROTO_ACTION.A_QUIT, PROTO_STATE.S_QUIT); - initMessage(); - LEVEL lvl = LEVEL.TRC2; - if (getLogger().isActive(lvl)) { - shared.getTmpSB().setLength(0); - shared.getTmpSB().append(pfx_log).append(" follow-on msg #").append(msgparams.messageCount()).append(" - msgid="); - shared.getTmpSB().append(formatSPID(msgparams.getSPID())).append(", recips=").append(msgparams.recipCount()); - getLogger().log(lvl, shared.getTmpSB()); + smtpMessage = smtpSender.messageCompleted(smtpMessage, ++msgcnt); + if (smtpMessage == null) { + return issueAction(PROTO_ACTION.A_QUIT, PROTO_STATE.S_QUIT); } + initMessage(); issueAction(PROTO_ACTION.A_RESET, PROTO_STATE.S_RESET); break; case A_QUIT: if (conncfg.isSendQuit()) transmit(shared.getSmtpRequestQuit()); - if (!conncfg.isAwaitQuit()) issueDisconnect(0, "A_QUIT"); + if (!conncfg.isAwaitQuit()) issueDisconnect(0, "A_QUIT", null); break; case A_RESET: @@ -837,7 +816,7 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o break; case A_ENDSESSION: - issueDisconnect(0, "A_ENDSESSION"); + issueDisconnect(0, "A_ENDSESSION", null); break; case A_DISCONNECT: @@ -874,8 +853,7 @@ protected void startedSSL() throws IOException { @Override protected void disconnectLingerDone(boolean ok, CharSequence info, Throwable ex) { if (shared.getTranscript() == null) return; - StringBuilder sb = shared.getTmpSB(); - sb.setLength(0); + StringBuilder sb = shared.getTmpSB(true); sb.append("Disconnect linger "); if (ok) { sb.append("completed"); @@ -893,26 +871,26 @@ private PROTO_STATE sendAuth(ByteArrayRef rspdata) throws IOException { int step = auth_step++; if (auth_method == SaslEntity.MECH.PLAIN) { - int finalstep = (active_relay.auth_initrsp ? 1 : 2); + int finalstep = (active_relay.isAuthInitialResponse() ? 1 : 2); auth_done = (step == finalstep); if (step == 0) { rspbuf.append(Protocol.CMDREQ_SASL_PLAIN); - if (active_relay.auth_initrsp) { + if (active_relay.isAuthInitialResponse()) { rspbuf.append(' '); shared.getSaslPlain().init(); - shared.getSaslPlain().setResponse(null, active_relay.usrnam, active_relay.passwd, rspbuf); + shared.getSaslPlain().setResponse(null, active_relay.getUsername(), active_relay.getPassword(), rspbuf); } } else if (!auth_done) { shared.getSaslPlain().init(); - shared.getSaslPlain().setResponse(null, active_relay.usrnam, active_relay.passwd, rspbuf); + shared.getSaslPlain().setResponse(null, active_relay.getUsername(), active_relay.getPassword(), rspbuf); } } else if (auth_method == SaslEntity.MECH.EXTERNAL) { // we send a zero-length response (whether initial or not), to assume the derived authorization ID - int finalstep = (active_relay.auth_initrsp ? 1 : 2); + int finalstep = (active_relay.isAuthInitialResponse() ? 1 : 2); auth_done = (step == finalstep); if (step == 0) { rspbuf.append(Protocol.CMDREQ_SASL_EXTERNAL); - if (active_relay.auth_initrsp) { + if (active_relay.isAuthInitialResponse()) { rspbuf.append(' ').append(Protocol.AUTH_EMPTY); } } else if (!auth_done) { @@ -925,7 +903,7 @@ private PROTO_STATE sendAuth(ByteArrayRef rspdata) throws IOException { } else if (step == 1) { rspdata.advance(Protocol.AUTH_CHALLENGE.length()); //advance past prefix rspdata.incrementSize(-Protocol.EOL.length()); //strip CRLF - shared.getSaslCramMD5().setResponse(active_relay.usrnam, active_relay.passwd, rspdata, rspbuf); + shared.getSaslCramMD5().setResponse(active_relay.getUsername(), ByteChars.valueOf(active_relay.getPassword()), rspdata, rspbuf); } else { auth_done = true; } @@ -942,90 +920,38 @@ private PROTO_STATE sendAuth(ByteArrayRef rspdata) throws IOException { return pstate; } - private void setRecipientStatus(int idx, short statuscode, ByteArrayRef failmsg, boolean remote_rsp) throws IOException { + private void setRecipientStatus(int idx, SmtpResponseDescriptor reply) { if (idx == -1) { // apply this status to all recipients - if (isFlagSet(S2_ABORT)) { - // Any recipients who've already failed remain as failures, but recipients who had been marked as OK revert to - // an unprocessed status, as we've clearly been interrupted before completing the message send. - for (int idx2 = 0; idx2 != msgparams.recipCount(); idx2++) { - MessageRecip recip = msgparams.getRecipient(idx2); - if (recip.smtp_status == Protocol.REPLYCODE_OK) { - recip.qstatus = MessageRecip.STATUS_READY; - } - } - return; - } - - for (int idx2 = 0; idx2 != msgparams.recipCount(); idx2++) { - setRecipientStatus(idx2, statuscode, failmsg, remote_rsp); + if (smtpMessage == null || smtpMessage.getRecipients() == null) + return; //is probably an error condition, so worth checking if we are initialised + for (int idx2 = 0; idx2 != smtpMessage.getRecipients().size(); idx2++) { + setRecipientStatus(idx2, reply); } return; } - MessageRecip recip = msgparams.getRecipient(idx); - Relay interceptor = shared.getController().getRouting().getInterceptor(); - - if (!isFlagSet(S2_ABORT)) recip.qstatus = MessageRecip.STATUS_DONE; - if (interceptor != null) { - //even though we generally log the theoretical destination address, we must audit the actual one - recip.ip_send = interceptor.tsap.ip; - } else { - recip.ip_send = (remote_tsap == null ? 0 : remote_tsap.ip); - } - - // this ensures that perm errors override preliminary temp errors, which in turn override preliminary success - if (statuscode > recip.smtp_status) { - if (recip.smtp_status == Protocol.REPLYCODE_OK) okrecips--; //retract an earlier success - if (statuscode == Protocol.REPLYCODE_OK) okrecips++; //no status had been set yet - recip.smtp_status = statuscode; - - // Set or clear the NDR diagnostic. - // Note that we don't create the diagnostic-message file for NDRs themselves. - if (recip.smtp_status == Protocol.REPLYCODE_OK) { - if (recip.sender != null && recip.retrycnt != 0) { - Path fh = shared.getController().getQueue().getDiagnosticFile(recip.spid, recip.qid); - Exception ex = FileOps.deleteFile(fh); - if (ex != null) getLogger().warn(pfx_log+" failed to delete NDR-diagnostic="+fh.getFileName()+" - "+ex); - } - } else { - LEVEL lvl = LEVEL.TRC2; - if (remote_rsp && getLogger().isActive(lvl)) { - int len = failmsg.size(); - while (failmsg.buffer()[failmsg.offset()+len-1] < ' ') len--; //strip trailing CRLF - StringBuilder sb = shared.getTmpSB(); - sb.setLength(0); - sb.append(pfx_log).append(" rejected on recip ").append(idx+1).append('/').append(msgparams.recipCount()); - sb.append('=').append(msgparams.getRecipient(idx).mailbox_to).append(EmailAddress.DLM_DOM).append(msgparams.getRecipient(idx).domain_to); - sb.append(" - ").append(shared.getTmpLightBC().set(failmsg.buffer(), failmsg.offset(), len)); - getLogger().log(lvl, sb); - } - int diaglen = (failmsg == null ? 0 : failmsg.size()); - if (recip.sender == null || recip.smtp_status <= Protocol.REPLYCODE_OK) diaglen = 0; - while (diaglen != 0 && failmsg.buffer()[failmsg.offset()+diaglen-1] <= ' ') diaglen--; - if (diaglen != 0) { - OutputStream fstrm = null; - try { - fstrm = shared.getController().getQueue().createDiagnosticFile(recip.spid, recip.qid); - if (failmsg.byteAt(0) >= '1' && failmsg.byteAt(0) <= '5') { - //prefix message with the IP address we failed to send to - IP.ip2net(recip.ip_send, shared.getTmpIP(), 0); - fstrm.write(1); - fstrm.write(shared.getTmpIP()); - } - fstrm.write(failmsg.buffer(), failmsg.offset(), diaglen); - } catch (Exception ex) { - getLogger().log(LEVEL.WARN, ex, true, pfx_log+" failed to set failure reason for "+recip); - } finally { - if (fstrm != null) fstrm.close(); - } + SmtpResponseDescriptor prevStatus = (idx >= recipientStatus.size() ? null : recipientStatus.get(idx)); + + if (prevStatus != null) { + //overwriting an earlier status + if (reply.smtpStatus() > prevStatus.smtpStatus()) { + // this ensures that perm errors override preliminary temp errors, which in turn override preliminary success + if (reply.smtpStatus() != Protocol.REPLYCODE_OK && prevStatus.smtpStatus() == Protocol.REPLYCODE_OK) { + okrecips--; //retract an earlier success } + recipientStatus.set(idx, reply); } + } else { + //appending new status - idx is presumably equal to recipientStatus.size() but don't bother checking + if (reply.smtpStatus() == Protocol.REPLYCODE_OK) okrecips++; + recipientStatus.add(reply); } + smtpSender.recipientCompleted(smtpMessage, msgcnt, idx, reply, remote_tsap, isFlagSet(S2_ABORT)); } // NB: We can use shared.pipebuf because the pipeline is always built up and sent within one callback - private boolean sendPipelinedRequest(ByteChars cmd, boolean flush, ByteChars addr, - ByteChars domain, boolean close_brace) throws IOException { + private boolean sendPipelinedRequest(ByteChars cmd, boolean flush, CharSequence addr, + CharSequence domain, boolean close_brace) throws IOException { ByteChars rspbuf = shared.getPipelineBuffer(); if (pipe_count == 0) rspbuf.clear(); rspbuf.append(cmd); @@ -1060,14 +986,6 @@ private void transmit(ByteBuffer xmtbuf) throws IOException { dataWait++; } - private short getReplyCode(ByteArrayRef rsp) { - int off = rsp.offset(); - short statuscode = (short)((rsp.buffer()[off++] - '0') * 100); - statuscode += (short) ((rsp.buffer()[off++] - '0') * 10); - statuscode += (short) (rsp.buffer()[off] - '0'); - return statuscode; - } - private boolean matchesExtension(ByteArrayRef data, char[] cmd, boolean with_equals) { int off = Protocol.REPLY_CODELEN + 1; //+1 for the hyphen or space following the "250" int len = data.size() - off - Protocol.EOL.length(); @@ -1093,11 +1011,13 @@ private void initConnection() { pfx_log = LOG_PREFIX+"/E"+getCMID()+"-"+cnxid; initChannelMonitor(); initMessage(); + smtpMessage = null; dnsInfo.clear(); conncfg = shared.getDefaultConfig(); pipe_cap = 1; remote_tsap = null; active_relay = null; + msgcnt = 0; pstate = PROTO_STATE.S_DISCON; disconnect_status = 0; dataWait = 0; @@ -1111,16 +1031,12 @@ private void initMessage() { recips_sent = 0; okrecips = 0; pipe_count = 0; + recipientStatus.clear(); } - private CharSequence formatSPID(int spid) { - return shared.getController().getQueue().externalSPID(spid); - } - - @Override public short getDomainError() { if (!isFlagSet(S2_DOMAIN_ERR)) return 0; - return msgparams.getRecipient(0).smtp_status; + return recipientStatus.get(0).smtpStatus(); } private void recordConnection(StringBuilder sb, TSAP local_tsap) { @@ -1128,16 +1044,22 @@ private void recordConnection(StringBuilder sb, TSAP local_tsap) { if (local_tsap != null) sb.append(" on ").append(local_tsap.dotted_ip).append(':').append(local_tsap.port); sb.append(" for "); peerDescription(sb); - sb.append(" with msgid=").append(formatSPID(msgparams.getSPID())); - sb.append(", recips=").append(msgparams.recipCount()); + sb.append(" with msgid=").append(smtpMessage.getMessageId()); + sb.append(", recips=").append(smtpMessage.getRecipients().size()); } - private void peerDescription(StringBuilder sb) { - if (msgparams.getRelay() != null) { - sb.append("relay=").append(msgparams.getRelay().display()); - } else { - sb.append("domain=").append(msgparams.getDestination()); + private CharSequence peerDescription(StringBuilder sb) { + if (smtpMessage == null) + return "nullpeer"; + String dlm = ""; + if (smtpMessage.getRecipients() != null && !smtpMessage.getRecipients().isEmpty()) { + sb.append("domain=").append(smtpMessage.getRecipients().get(0).getDomain()); + dlm = "/"; } + if (active_relay != null) { + sb.append(dlm).append("relay=").append(active_relay.getName()).append('=').append(active_relay.getAddress()); + } + return sb; } // given a minimum bits per second, calculate the max time expected to send this many bytes (in milliseconds) @@ -1147,11 +1069,11 @@ private static long calculateMaxTime(long numBytes, long minBPS) { @Override public StringBuilder dumpAppState(StringBuilder sb) { + int cnt = (smtpMessage == null || smtpMessage.getRecipients() == null ? -1 : smtpMessage.getRecipients().size()); if (sb == null) sb = new StringBuilder(); sb.append(pfx_log).append('/').append(pstate).append("/0x").append(Integer.toHexString(state2)).append(": "); peerDescription(sb); - sb.append("; msgcnt=").append(msgparams.messageCount()); - sb.append("; recips=").append(recip_id).append('/').append(msgparams.recipCount()); + sb.append("; recips=").append(recip_id).append('/').append(cnt); return sb; } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/ClientConfiguration.java b/server/src/main/java/com/grey/mailismus/mta/deliver/ClientConfiguration.java index f39bd99..16c8304 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/ClientConfiguration.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/ClientConfiguration.java @@ -1,3 +1,7 @@ +/* + * Copyright 2010-2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ package com.grey.mailismus.mta.deliver; import java.io.IOException; @@ -16,18 +20,17 @@ import com.grey.mailismus.AppConfig; import com.grey.mailismus.Transcript; import com.grey.mailismus.errors.MailismusConfigException; +import com.grey.mailismus.mta.deliver.client.ConnectionConfig; class ClientConfiguration { private static final String LOG_PREFIX = "SMTP-Client-Config"; public static SharedFields createSharedFields(XmlConfig xmlcfg, - Delivery.Controller ctl, + Dispatcher dsptch, ResolverDNS dns, AppConfig appConfig, int max_serverconns) throws IOException, GeneralSecurityException { - Dispatcher dsptch = ctl.getDispatcher(); - boolean fallback_mx_a = xmlcfg.getBool("fallbackMX_A", false); BufferGenerator bufferGenerator = createBufferGenerator(xmlcfg); Transcript transcript = createTranscript(xmlcfg, dsptch); @@ -45,14 +48,13 @@ public static SharedFields createSharedFields(XmlConfig xmlcfg, } SharedFields shared = SharedFields.builder() - .withController(ctl) .withDnsResolver(dns) .withBufferGenerator(bufferGenerator) .withTranscript(transcript) .withDefaultConfig(defaultConfig) .withRemoteConfigs(remotesConfig) .build(); - ctl.getDispatcher().getLogger().info(LOG_PREFIX+": "+bufferGenerator); + dsptch.getLogger().info(LOG_PREFIX+": "+bufferGenerator); return shared; } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/DeliverTask.java b/server/src/main/java/com/grey/mailismus/mta/deliver/DeliverTask.java index ea01bf3..00d0b31 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/DeliverTask.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/DeliverTask.java @@ -37,7 +37,7 @@ protected boolean stopNaflet() { // the only entity we launch is the SMTP Sender, so this must be it ... and that means we're now finished as well @Override - public void eventIndication(Object reportingEntity, String eventId) { + public void eventIndication(String eventId, Object reportingEntity, Object data) { nafletStopped(); } } \ No newline at end of file diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Delivery.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Delivery.java deleted file mode 100644 index 148d398..0000000 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Delivery.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2015-2024 Yusef Badri - All rights reserved. - * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). - */ -package com.grey.mailismus.mta.deliver; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.grey.base.utils.ByteChars; -import com.grey.naf.EventListenerNAF; -import com.grey.naf.reactor.Dispatcher; -import com.grey.naf.reactor.TimerNAF; -import com.grey.mailismus.mta.queue.MessageRecip; -import com.grey.mailismus.mta.queue.QueueManager; - -public interface Delivery -{ - public interface Controller - { - void messageCompleted(MessageSender sender); - void senderCompleted(MessageSender sender); - Dispatcher getDispatcher(); - QueueManager getQueue(); - Routing getRouting(); - } - - - public interface MessageSender - { - void start(Controller ctl) throws IOException; - boolean stop(); - MessageParams getMessageParams(); - short getDomainError(); - String getLogID(); - void setEventListener(EventListenerNAF l); - } - - - static final class MessageParams - { - private final List recips = new ArrayList<>(); - private ByteChars sender; - private ByteChars destdomain; - private Relay relay; - private int spid; - private int msgcnt; - - public int getSPID() {return spid;} - public ByteChars getSender() {return sender;} - public ByteChars getDestination(){return destdomain;} - public Relay getRelay() {return relay;} - public MessageRecip getRecipient(int idx) {return recips.get(idx);} - public int recipCount() {return recips.size();} - int messageCount() {return msgcnt;} - int incrementMessages() {return ++msgcnt;} - - MessageParams init(Relay rly, ByteChars destdom) { - clear(); - relay = rly; - if (relay == null) destdomain = destdom; - return this; - } - - MessageParams clear() { - resetMessage(); - sender = null; - destdomain = null; - relay = null; - msgcnt = 0; - return this; - } - - // This clears per-message state only - MessageParams resetMessage() { - recips.clear(); - spid = 0; - return this; - } - - void addRecipient(MessageRecip recip) { - if (recips.isEmpty()) { - // Need to record these params outside 'recips', as list will get cleared. Obviously every 'recips' member - // will have the same SPID, but the destination domains will vary if in slave-relay or source-routed mode, - // so destdomain may not be meaningful. - // So long as callers are aware of that, it's useful to record destdomain anyway for logging purposes, as - // many messages will only have one recipient. - sender = recip.sender; - spid = recip.spid; - } - recips.add(recip); - } - } - - - // Stats accumulator - can be used to record batch stats, or some other interval - public static final class Stats - { - public int conncnt; //number of SMTP connections - public int sendermsgcnt; //number of SMTP messages - always >=conncnt, depending on whether senders were refilled - public int remotecnt; //number of remote (SMTP) recipients handled (ie. no. of MessageRecips assigned to a MessageSender) - public int remotefailcnt; //number of remote recipients who failed - this is a subset of remotecnt - public int localcnt; //number of local recipients handled (ie. no. of MessageRecips deliver into the MS) - public int localfailcnt; //number of local recipients who failed - this is a subset of localcnt - public long start; - private final TimerNAF.TimeProvider timeProvider; - public Stats(TimerNAF.TimeProvider t) {timeProvider=t; reset();} - public Stats reset() { - start = timeProvider.getRealTime(); - conncnt = sendermsgcnt = remotecnt = remotefailcnt = localcnt = localfailcnt = 0; - return this;} - @Override - public String toString() { - String txt = "DeliveryStats: Conns="+conncnt+", remote-msgs="+sendermsgcnt; - if (localcnt != 0) txt += ", localrecips="+(localcnt-localfailcnt)+"/"+localcnt; - if (remotecnt != 0) txt += ", remoterecips="+(remotecnt-remotefailcnt)+"/"+remotecnt; - return txt+" - time="+(timeProvider.getRealTime()-start)+"ms"; - } - } -} diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java index bd0af37..54d54d6 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java @@ -9,17 +9,20 @@ import java.nio.file.Path; import java.security.GeneralSecurityException; import java.io.IOException; +import java.io.OutputStream; import com.grey.base.collections.GenericFactory; +import com.grey.base.collections.HashedMap; import com.grey.base.collections.HashedMapIntValue; -import com.grey.base.collections.HashedSet; import com.grey.base.collections.ObjectWell; import com.grey.base.config.SysProps; import com.grey.base.config.XmlConfig; import com.grey.base.utils.StringOps; +import com.grey.base.utils.TSAP; import com.grey.base.utils.TimeOps; import com.grey.base.utils.ByteChars; import com.grey.base.utils.EmailAddress; +import com.grey.base.utils.FileOps; import com.grey.base.utils.IP; import com.grey.logging.Logger; import com.grey.logging.Logger.LEVEL; @@ -36,6 +39,9 @@ import com.grey.mailismus.Transcript; import com.grey.mailismus.mta.MTA_Task; import com.grey.mailismus.mta.Protocol; +import com.grey.mailismus.mta.deliver.client.SmtpMessage; +import com.grey.mailismus.mta.deliver.client.SmtpResponseDescriptor; +import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.mta.queue.Cache; import com.grey.mailismus.mta.queue.MessageRecip; import com.grey.mailismus.mta.queue.QueueManager; @@ -43,39 +49,27 @@ import com.grey.mailismus.nafman.Loader; public class Forwarder - implements Delivery.Controller, + implements SmtpSender, TimerNAF.Handler, NafManCommand.Handler { public interface BatchCallback { - void batchCompleted(int qsize, Delivery.Stats stats); + void batchCompleted(int qsize, DeliveryStats deliveryStats); } private static class SenderReaper implements EventListenerNAF { - private final Delivery.Controller ctl; - public SenderReaper(Delivery.Controller ctl) { - this.ctl = ctl; + private final Forwarder fwd; + public SenderReaper(Forwarder fwd) { + this.fwd = fwd; } @Override - public void eventIndication(Object obj, String eventId) { - if (obj instanceof Client && ChannelMonitor.EVENTID_CM_DISCONNECTED.equals(eventId)) { - ctl.senderCompleted((Delivery.MessageSender)obj); + public void eventIndication(String eventId, Object evtsrc, Object data) { + if (evtsrc instanceof Client && ChannelMonitor.EVENTID_CM_DISCONNECTED.equals(eventId)) { + fwd.senderCompleted((Client)evtsrc, false); } } } - private static class ClientFactory implements GenericFactory { - private final SharedFields shared; - public ClientFactory(SharedFields shared) { - this.shared = shared; - } - @Override - public Client factory_create() { - // NB: can't set eventListener here, as it gets cleared on ChannelMonitor.disconnect() - return new Client(shared); - } - } - private static final boolean CHECK_OFFLINE = SysProps.get("grey.mta.smtpclient.offlinecheck", false); private static final int TMRTYPE_QPOLL = 1; private static final int TMRTYPE_KILLSENDERS = 2; @@ -109,8 +103,9 @@ public Client factory_create() { private final EventListenerNAF sendersEventListener; private final BatchCallback batchCallback; private final Cache qcache; - private final ObjectWell sparesenders; - private final HashedSet activesenders = new HashedSet<>(); + private final ObjectWell spareMessageRequests; + private final ObjectWell sparesenders; + private final HashedMap activesenders = new HashedMap<>(); // This maps connection targets (ie. SMTP servers) to the number of simultaneous connections we currently have to them. // The map values can be of type ByteChars (destination domain) or Relay. @@ -128,12 +123,11 @@ public Client factory_create() { // batchStats is logged and reset at the end of each batch, while openStats is accumulated for an open-ended period, // until retrieved and reset by the NAFMAN COUNTERS command (and unlike the running totals below, it is only updated // at the end of each batch) - private final Delivery.Stats batchStats; - private final Delivery.Stats openStats; - private int sendercnt; //number of MessageSenders launched for current batch + private final DeliveryStats batchStats; + private final DeliveryStats openStats; private int pending_recips; //number of entries in current batch which have not yet been handled (qstatus==READY) - // Stats - running totals across all batches + // DeliveryStats - running totals across all batches private int batchcnt; //not incremented for null batches (ie. nothing in queue) private int total_conncnt; //SMTP connections private int total_sendermsgcnt; //SMTP messages (>= total_conncnt and excludes local delivery) @@ -146,12 +140,12 @@ public Client factory_create() { //pre-allocated merely for efficiency private final EmailAddress tmpemaddr = new EmailAddress(); private final StringBuilder tmpsb = new StringBuilder(); + private final StringBuilder tmpsb2 = new StringBuilder(); + private final ByteChars tmpBC = new ByteChars(); + private final byte[] tmpIP = IP.ip2net(0, null, 0); - @Override public Dispatcher getDispatcher() {return dsptch;} - @Override public QueueManager getQueue() {return qmgr;} - @Override public Routing getRouting() {return routing;} - @Override public void senderCompleted(Delivery.MessageSender sender) {senderCompleted(sender, false);} @Override public CharSequence nafmanHandlerID() {return "SMTP-Forwarder";} + public Routing getRouting() {return routing;} // these counts should give the same result public int activeSendersCount() {return activesenders.size();} @@ -163,7 +157,7 @@ public Forwarder(Dispatcher d, MTA_Task task, XmlConfig cfg, EventListenerNAF ev public Forwarder(Dispatcher d, XmlConfig cfg, AppConfig appConfig, QueueManager qm, MessageStore mstore, - EventListenerNAF evtl, GenericFactory senderFactory, + EventListenerNAF evtl, GenericFactory senderFactory, BatchCallback bcb, ResolverDNS dnsResolver) throws IOException, GeneralSecurityException { dsptch = d; @@ -211,19 +205,20 @@ public Forwarder(Dispatcher d, XmlConfig cfg, AppConfig appConfig, } cap_qcache = (int)cfg.getSize("queuecache", cap_qcache); qcache = qmgr.initCache(cap_qcache); - batchStats = new Delivery.Stats(dsptch); - openStats = new Delivery.Stats(dsptch); + batchStats = new DeliveryStats(dsptch); + openStats = new DeliveryStats(dsptch); sendersEventListener = new SenderReaper(this); if (senderFactory == null) { XmlConfig smtpcfg = cfg.getSection("client"); - sharedFields = ClientConfiguration.createSharedFields(smtpcfg, this, dnsResolver, appConfig, max_serverconns); - senderFactory = new ClientFactory(sharedFields); + sharedFields = ClientConfiguration.createSharedFields(smtpcfg, dsptch, dnsResolver, appConfig, max_serverconns); + senderFactory = () -> new Client(sharedFields, dsptch); } else { // sender-factory is only supplied in some test modes, never in production mode sharedFields = null; } - sparesenders = new ObjectWell<>(senderFactory, "SmtpFwd"); + sparesenders = new ObjectWell<>(senderFactory, "SmtpFwdSenders"); + spareMessageRequests = new ObjectWell<>(() -> new QueueBasedMessage(qmgr), "SmtpFwdMessageReqs"); active_serverconns = (max_serverconns == 0 ? null : new HashedMapIntValue<>()); log.info("SMTP-Delivery: slave-relay mode="+routing.modeSlaveRelay()); @@ -276,9 +271,9 @@ public boolean stop() private void stopSenders() { // loop on copy of set to avoid ConcurrentModification from callbacks - List lst = new ArrayList<>(activesenders); + List lst = new ArrayList<>(activesenders.keySet()); for (int idx = 0; idx != lst.size(); idx++) { - Delivery.MessageSender sender = lst.get(idx); + Client sender = lst.get(idx); if (sender.stop()) senderCompleted(sender, true); } } @@ -294,7 +289,7 @@ private void stopped(boolean notify) if (active_serverconns != null) active_serverconns.clear(); qcache.clear(); has_stopped = true; - if (notify && eventListener != null) eventListener.eventIndication(this, EventListenerNAF.EVENTID_ENTITY_STOPPED); + if (notify && eventListener != null) eventListener.eventIndication(EventListenerNAF.EVENTID_ENTITY_STOPPED, this, null); } @Override @@ -354,7 +349,6 @@ private boolean processQueue() throws IOException qmgr.getMessages(qcache, sendDeferred); sendDeferred = false; pending_recips = qcache.size(); - sendercnt = 0; if (pending_recips == 0) { if (batchCallback != null) batchCallback.batchCompleted(0, null); @@ -372,6 +366,7 @@ private boolean processQueue() throws IOException tmpsb.append(" (qtime=").append(qtime).append("ms)"); dsptch.getLogger().log(lvl, tmpsb); } + int existingActive = activeSendersCount(); // scan the cache to load its entries into the Senders, and then initiate them qcache.sort(); @@ -385,6 +380,7 @@ private boolean processQueue() throws IOException long launchtime = time2 - time1; total_launchtime += launchtime; total_sendtime -= (time2 - batchStats.start); //because we will later add the time from batchStats.start onwards + int sendercnt = activeSendersCount() - existingActive; //number of senders launched for current batch if (activeSendersCount() == 0) { cacheProcessed(); @@ -436,21 +432,25 @@ private void processCache() // Current message is for a remote recipient. // Local recipients (null domain) are sorted to the top of the cache, so we've already seen them all. local_done = true; - if (max_simulconns != 0 && activeSendersCount() == max_simulconns) break; //no more connections allowed - Delivery.MessageSender sender = populateSender(null, qslot, qlimit); - if (sender == null) break; + if (max_simulconns != 0 && activeSendersCount() == max_simulconns) { + break; //no more connections allowed + } + QueueBasedMessage msgparams = generateMessage(qslot, qlimit, null); + if (msgparams == null) { + break; + } + Client sender = sparesenders.extract(); //extract() won't return null because this ObjectWell is uncapped + activesenders.put(sender, msgparams); + msgparams.setClient(sender); startSender(sender); } } // If dest_domain is passed in, then we're only interested in cache entries that match that. - private Delivery.MessageSender populateSender(Delivery.MessageSender sender, int qslot, int limit) - { - Delivery.MessageParams msgparams = null; + private QueueBasedMessage generateMessage(int qslot, int limit, QueueBasedMessage msgparams) { Relay sender_relay = null; ByteChars dest_domain = null; - if (sender != null) { - msgparams = sender.getMessageParams(); + if (msgparams != null) { sender_relay = msgparams.getRelay(); dest_domain = msgparams.getDestination(); } @@ -462,16 +462,15 @@ private Delivery.MessageSender populateSender(Delivery.MessageSender sender, int if (spid != 0 && recip.spid != spid) break; //no entries left for this message if (recip.qstatus != MessageRecip.STATUS_READY) continue; Relay recip_relay = getRoute(recip); - if (sender != null) { + + if (msgparams != null) { //these conditions will be met in slave-relay mode, as sender_relay non-null and recip_relay is same if (sender_relay != recip_relay) continue; if (sender_relay == null) { if (!dest_domain.equals(recip.domain_to)) continue; //dest_domain is guaranteed non-null here } - } - - //current MessageRecip matches the criteria so allocate to sender - might have to allocate sender too - if (sender == null) { + } else { + //current MessageRecip matches the criteria so allocate to sender if (max_serverconns != 0) { //recall that this is zero in slave-relay mode (but not only in that mode) Object key = (recip_relay == null ? recip.domain_to : recip_relay); int cnt = active_serverconns.get(key); @@ -479,11 +478,9 @@ private Delivery.MessageSender populateSender(Delivery.MessageSender sender, int active_serverconns.put(key, cnt+1); } //null dest_domain means grab every entry for this SPID, else we're tied to initial recipient domain - sender = sparesenders.extract(); //extract() won't return null because this ObjectWell is uncapped - activesenders.add(sender); - sendercnt++; - msgparams = sender.getMessageParams().init(recip_relay, recip.domain_to); - sender_relay = msgparams.getRelay(); + msgparams = spareMessageRequests.extract(); //won't return null because this ObjectWell is uncapped + msgparams = msgparams.init(recip_relay, recip.domain_to); + sender_relay = recip_relay; dest_domain = msgparams.getDestination(); } spid = recip.spid; @@ -492,61 +489,87 @@ private Delivery.MessageSender populateSender(Delivery.MessageSender sender, int pending_recips--; if (max_msgrecips != 0 && msgparams.recipCount() == max_msgrecips) break; } - return sender; + return msgparams; } - private void startSender(Delivery.MessageSender sender) + private void startSender(Client sender) { try { - sender.setEventListener(sendersEventListener); - sender.start(this); + QueueBasedMessage msg = activesenders.get(sender); + Relay interceptor = routing.getInterceptor(); + Relay relay = msg.getRelay(); + if (relay == null || (interceptor != null && !interceptor.dnsOnly)) relay = interceptor; + sender.setEventListener(sendersEventListener); //gets cleared on ChannelMonitor.disconnect() + sender.startConnection(msg, this, relay); } catch (Throwable ex) { dsptch.getLogger().log(LEVEL.TRC, ex, true, "SMTP-Delivery/batch="+batchcnt+": Failed to start Sender="+sender.getLogID()+"/"+sender); senderCompleted(sender, true); } } - // This method clears the existing recipient set, and then attempt to repopulate it with another msg for same destination - // domain (might be same msg, if it has more recips waiting). - // Caller should check for non-empty recipients on return, to determine if it has anything to send. + // This method processes the result of a message send, and attempts to generate another msg for same destination. + // Returns new message request if we found something to send, else null. @Override - public void messageCompleted(Delivery.MessageSender sender) - { + public SmtpMessage messageCompleted(SmtpMessage smtpmsg, int msgcnt) { + QueueBasedMessage msgparams = (QueueBasedMessage)smtpmsg; + Client sender = msgparams.getClient(); + int domainError = sender.getDomainError(); + long time1 = dsptch.getRealTime(); - boolean active = recordMessageResult(sender); - Delivery.MessageParams msgparams = sender.getMessageParams(); + boolean active = recordMessageResult(msgparams, msgcnt, domainError); msgparams.resetMessage(); - if (!active || inScan || inShutdown || sender.getDomainError() != 0 || msgparams.messageCount() == max_connmsgs) return; + boolean refill = (active && !inScan && !inShutdown && domainError == 0 && msgcnt != max_connmsgs); if (time1 - batchStats.start > max_conntime) { - //don't refill this Sender - dsptch.getLogger().info("SMTP-Delivery/batch="+batchcnt+": Stopping slow Sender at messages="+msgparams.messageCount() - +" - remote="+getPeerText(msgparams)+" - "+sender); + dsptch.getLogger().info("SMTP-Delivery/batch="+batchcnt+": Stopping slow Sender at messages="+msgcnt + +" - remote="+getPeerText(msgparams)+" - "+sender.getLogID()); } else { - populateSender(sender, 0, qcache.size()); + if (refill) { + generateMessage(0, qcache.size(), msgparams); + } } long span = dsptch.getRealTime() - time1; total_launchtime += span; total_sendtime -= span; //because we will later add the time from batchStats.start onwards - } - - // The sender's recipient list will be empty if it completed successfully, as it would have called messageCompleted() - // which clears it. So a non-empty recipient list probably means we have an error condition to report. - public void senderCompleted(Delivery.MessageSender sender, boolean aborted) - { - Delivery.MessageParams msgparams = sender.getMessageParams(); - if (msgparams.recipCount() != 0) recordMessageResult(sender); - activesenders.remove(sender); - if (active_serverconns != null) { + if (msgparams.recipCount() == 0) { Object key = msgparams.getRelay(); if (key == null) key = msgparams.getDestination(); - int cnt = active_serverconns.get(key); - if (--cnt == 0) { - active_serverconns.remove(key); - } else { - active_serverconns.put(key, cnt); + + msgparams.clear(); + spareMessageRequests.store(msgparams); + msgparams = null; + + if (active_serverconns != null) { + int cnt = active_serverconns.get(key); + if (--cnt <= 0) { + active_serverconns.remove(key); + } else { + active_serverconns.put(key, cnt); + } } + activesenders.remove(sender); + + if (!inScan && activeSendersCount() == 0) { + cacheProcessed(); + } + } + return msgparams; + } + + @Override + public void onDisconnect(SmtpMessage smtpmsg, int msgcnt) { + if (smtpmsg != null && smtpmsg.getRecipients() != null && !smtpmsg.getRecipients().isEmpty()) { + messageCompleted(smtpmsg, msgcnt); + } + } + + private void senderCompleted(Client sender, boolean aborted) { + QueueBasedMessage msg = activesenders.get(sender); + int msgcnt = -1; + if (msg != null) { + msgcnt = msg.messageCount(); + messageCompleted(msg, msgcnt); } LEVEL lvl = LEVEL.TRC2; @@ -554,28 +577,95 @@ public void senderCompleted(Delivery.MessageSender sender, boolean aborted) tmpsb.setLength(0); tmpsb.append("SMTP-Delivery/batch=").append(batchcnt).append(": Sender=").append(sender.getLogID()); tmpsb.append(" has ").append(aborted?"aborted":"completed"); - if (msgparams.messageCount() != 1) tmpsb.append(" with msgcnt=").append(msgparams.messageCount()); + tmpsb.append(" with msgcnt=").append(msgcnt); tmpsb.append(" - active-conns=").append(activeSendersCount()).append(", pending-recips=").append(pending_recips); - if (inScan) tmpsb.append("/scanning"); + tmpsb.append("/scanning=").append(inScan); dsptch.getLogger().log(lvl, tmpsb); } - msgparams.clear(); sparesenders.store(sender); - if (inScan) return; //take no further action if within a synchronous callback + } - if (activeSendersCount() == 0) { - cacheProcessed(); + @Override + public boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, SmtpResponseDescriptor status, TSAP remote, boolean aborted) { + SmtpMessage.Recipient recip = msg.getRecipients().get(recipId); + MessageRecip qrecip = ((QueueBasedRecipient)recip).getQueueRecip(); + + if (aborted) { + // Any recipients who've already failed remain as failures, but recipients who had been marked as OK revert to + // an unprocessed status, as we've clearly been interrupted before completing the message send. + if (status.smtpStatus() == Protocol.REPLYCODE_OK) { + qrecip.qstatus = MessageRecip.STATUS_READY; + } + return false; } + qrecip.qstatus = MessageRecip.STATUS_DONE; + qrecip.ip_send = (remote == null ? 0 : remote.ip); + + if (status.smtpStatus() > qrecip.smtp_status) { + // ensures that perm errors override preliminary temp errors, which in turn override preliminary success + qrecip.smtp_status = status.smtpStatus(); + + // Set or clear the NDR diagnostic. + // Note that we don't create the diagnostic-message file for NDRs themselves. + if (status.smtpStatus() == Protocol.REPLYCODE_OK) { + if (qrecip.sender != null && qrecip.retrycnt != 0) { + Path fh = qmgr.getDiagnosticFile(qrecip.spid, qrecip.qid); + Exception ex = FileOps.deleteFile(fh); + if (ex != null) dsptch.getLogger().warn("SMTP-Delivery: Failed to delete NDR-diagnostic="+fh.getFileName()+" - "+ex); + } + } else { + StringBuilder sbStatus = tmpsb2; + sbStatus.setLength(0); + sbStatus.append(status.smtpStatus()); + if (status.enhancedStatus() != null) sbStatus.append(' ').append(status.enhancedStatus()); + sbStatus.append(' ').append(status.message()); + + LEVEL lvl = LEVEL.TRC2; + if (dsptch.getLogger().isActive(lvl)) { + StringBuilder sb = tmpsb; + sb.setLength(0); + sb.append("SMTP-Delivery: Message rejected on recip ").append(recipId+1).append('/').append(msg.getRecipients().size()); + sb.append('=').append(qrecip.mailbox_to).append(EmailAddress.DLM_DOM).append(qrecip.domain_to); + sb.append(" by remote=").append(remote); + sb.append(" - SMTP status=").append(sbStatus); + dsptch.getLogger().log(lvl, sb); + } + + if (qrecip.sender != null && status.smtpStatus() > Protocol.REPLYCODE_OK) { + OutputStream fstrm = null; + try { + fstrm = qmgr.createDiagnosticFile(qrecip.spid, qrecip.qid); + if (remote != null) { + IP.ip2net(remote.ip, tmpIP, 0); + fstrm.write(1); //special marker to introduce IP address + fstrm.write(tmpIP); + } + tmpBC.clear().append(sbStatus); + fstrm.write(tmpBC.buffer(), tmpBC.offset(), tmpBC.size()); + } catch (Exception ex) { + dsptch.getLogger().log(LEVEL.WARN, ex, true, "SMTP-Delivery: Failed to set failure reason for "+qrecip); + } finally { + if (fstrm != null) { + try { + fstrm.close(); + } catch (Exception ex) { + dsptch.getLogger().log(LEVEL.WARN, ex, true, "SMTP-Delivery: Failed to close NDR diagnostic "+qrecip); + } + } + } + } + } + } + return false; } - private boolean recordMessageResult(Delivery.MessageSender sender) + private boolean recordMessageResult(QueueBasedMessage msgparams, int msgcnt, int domain_error) { - Delivery.MessageParams msgparams = sender.getMessageParams(); int recipcnt = msgparams.recipCount(); int processed_cnt = 0; for (int idx = 0; idx != recipcnt; idx++) { - MessageRecip recip = msgparams.getRecipient(idx); + MessageRecip recip = msgparams.getRecipient(idx).getQueueRecip(); if (recip.qstatus == MessageRecip.STATUS_DONE) { processed_cnt++; } else { @@ -591,21 +681,19 @@ private boolean recordMessageResult(Delivery.MessageSender sender) total_remotecnt += processed_cnt; batchStats.sendermsgcnt++; total_sendermsgcnt++; - int sender_msgcnt = msgparams.incrementMessages(); - if (sender_msgcnt == 1) { + if (msgparams.incrementMessages() == 1) { //count connection after first msg (rather than having to decrement after abort, if we incremented prematurely) batchStats.conncnt++; total_conncnt++; } - CharSequence extspid = qmgr.externalSPID(msgparams.getSPID()); - short domain_error = sender.getDomainError(); + CharSequence extspid = msgparams.getMessageId(); LEVEL lvl = LEVEL.TRC2; boolean log = dsptch.getLogger().isActive(lvl); if (log) { tmpsb.setLength(0); - tmpsb.append("SMTP-Delivery/batch=").append(batchcnt).append(": Sender=").append(sender.getLogID()); - tmpsb.append(" has processed msg #").append(sender_msgcnt).append('/').append(total_sendermsgcnt); + tmpsb.append("SMTP-Delivery/batch=").append(batchcnt).append(": Sender=").append(msgparams.getClient().getLogID()); + tmpsb.append(" has processed msg #").append(msgcnt).append('/').append(msgparams.messageCount()).append('/').append(total_sendermsgcnt); tmpsb.append(" - SPID=").append(extspid).append(" from ").append(msgparams.getSender()); tmpsb.append(" for recips=").append(processed_cnt).append('/').append(recipcnt).append('/').append(total_remotecnt); tmpsb.append(" at remote=").append(getPeerText(msgparams)); @@ -616,7 +704,7 @@ private boolean recordMessageResult(Delivery.MessageSender sender) } for (int idx = 0; idx != recipcnt; idx++) { - MessageRecip recip = msgparams.getRecipient(idx); + MessageRecip recip = msgparams.getRecipient(idx).getQueueRecip(); if (log) { tmpsb.append('\n').append(lvl).append(" - Recip=").append(idx+1).append(": "); recip.toString(tmpsb); @@ -641,7 +729,7 @@ private boolean recordMessageResult(Delivery.MessageSender sender) if (getRoute(recip) != msgparams.getRelay()) continue; if (destdomain == null || !destdomain.equals(recip.domain_to)) continue; recip.qstatus = MessageRecip.STATUS_DONE; - recip.smtp_status = domain_error; + recip.smtp_status = (short)domain_error; pending_recips--; } lvl = LEVEL.TRC; @@ -721,10 +809,10 @@ private Relay getRoute(MessageRecip mr) return rt; } - private String getPeerText(Delivery.MessageParams msgparams) + private String getPeerText(QueueBasedMessage msgparams) { Object peer = (routing.modeSlaveRelay() ? "smarthost" : null); - if (peer == null) peer = (msgparams.getRelay()==null?null:msgparams.getRelay().display()); + if (peer == null) peer = (msgparams.getRelay()==null?null:msgparams.getRelay().toString()); if (peer == null) peer = msgparams.getDestination(); return peer.toString(); } @@ -755,4 +843,28 @@ public CharSequence handleNAFManCommand(NafManCommand cmd) } return tmpsb; } + + + public static class DeliveryStats { + public int conncnt; //number of SMTP connections + public int sendermsgcnt; //number of SMTP messages - always >=conncnt, depending on whether senders were refilled + public int remotecnt; //number of remote (SMTP) recipients handled (ie. no. of MessageRecips assigned to a MessageSender) + public int remotefailcnt; //number of remote recipients who failed - this is a subset of remotecnt + public int localcnt; //number of local recipients handled (ie. no. of MessageRecips deliver into the MS) + public int localfailcnt; //number of local recipients who failed - this is a subset of localcnt + public long start; + private final TimerNAF.TimeProvider timeProvider; + public DeliveryStats(TimerNAF.TimeProvider t) {timeProvider=t; reset();} + public DeliveryStats reset() { + start = timeProvider.getRealTime(); + conncnt = sendermsgcnt = remotecnt = remotefailcnt = localcnt = localfailcnt = 0; + return this;} + @Override + public String toString() { + String txt = "DeliveryStats: Conns="+conncnt+", remote-msgs="+sendermsgcnt; + if (localcnt != 0) txt += ", localrecips="+(localcnt-localfailcnt)+"/"+localcnt; + if (remotecnt != 0) txt += ", remoterecips="+(remotecnt-remotefailcnt)+"/"+remotecnt; + return txt+" - time="+(timeProvider.getRealTime()-start)+"ms"; + } + } } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java new file mode 100644 index 0000000..f25d3c5 --- /dev/null +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java @@ -0,0 +1,93 @@ +/* + * Copyright 2015-2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import com.grey.base.utils.ByteChars; +import com.grey.mailismus.mta.deliver.client.SmtpMessage; +import com.grey.mailismus.mta.queue.MessageRecip; +import com.grey.mailismus.mta.queue.QueueManager; + +class QueueBasedMessage implements SmtpMessage { + private final List recips = new ArrayList<>(); + private final QueueManager qmgr; + + private ByteChars sender; + private ByteChars destdomain; + private Relay relay; + private Supplier data; + private String spid; + private Client client; + private int msgcnt; + + @Override public ByteChars getSender() {return sender;} + @Override public List getRecipients() {return recips;} + @Override public Supplier getData() {return data;} + @Override public String getMessageId() {return spid;} + + public ByteChars getDestination() {return destdomain;} + public Relay getRelay() {return relay;} + public QueueBasedRecipient getRecipient(int idx) {return recips.get(idx);} + public int recipCount() {return recips.size();} + public Client getClient() {return client;} + int messageCount() {return msgcnt;} + int incrementMessages() {return ++msgcnt;} + + public QueueBasedMessage(QueueManager qmgr) { + this.qmgr = qmgr; + } + + public QueueBasedMessage init(Relay rly, ByteChars destdom) { + clear(); + relay = rly; + if (relay == null) destdomain = destdom; + return this; + } + + public QueueBasedMessage clear() { + resetMessage(); + sender = null; + destdomain = null; + relay = null; + data = null; + client = null; + msgcnt = 0; + return this; + } + + QueueBasedMessage resetMessage() { + recips.clear(); + spid = null; + return this; + } + + public void addRecipient(MessageRecip qrecip) { + if (recips.isEmpty()) { + // Need to record these params outside 'recips', as list will get cleared. Obviously every 'recips' member + // will have the same SPID, but the destination domains will vary if in slave-relay or source-routed mode, + // so destdomain may not be meaningful. + // So long as callers are aware of that, it's useful to record destdomain anyway for logging purposes, as + // many messages will only have one recipient. + sender = qrecip.sender; + spid = qmgr.externalSPID(qrecip.spid).toString(); + data = () -> qmgr.getMessage(qrecip.spid, qrecip.qid); + + } + QueueBasedRecipient recip = new QueueBasedRecipient(qrecip); + recips.add(recip); + } + + public void setClient(Client client) { + this.client = client; + } + + @Override + public String toString() { + return "QueueBasedMessage[msg="+msgcnt+": sender="+sender+" => "+destdomain+"/"+ recips+ " - relay="+relay+", spid="+spid+"]"; + } +} diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedRecipient.java b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedRecipient.java new file mode 100644 index 0000000..f8b1396 --- /dev/null +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedRecipient.java @@ -0,0 +1,35 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver; + +import com.grey.mailismus.mta.deliver.client.SmtpMessage; +import com.grey.mailismus.mta.queue.MessageRecip; + +class QueueBasedRecipient implements SmtpMessage.Recipient { + private final MessageRecip queueRecip; + + public QueueBasedRecipient(MessageRecip queueRecip) { + this.queueRecip = queueRecip; + } + + @Override + public CharSequence getDomain() { + return getQueueRecip().domain_to; + } + + @Override + public CharSequence getMailbox() { + return getQueueRecip().mailbox_to; + } + + public MessageRecip getQueueRecip() { + return queueRecip; + } + + @Override + public String toString() { + return "QueueBasedRecipient["+queueRecip+"]"; + } +} diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Relay.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Relay.java index 890d0d6..d0f053c 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Relay.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Relay.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import com.grey.base.config.XmlConfig; @@ -14,44 +15,38 @@ import com.grey.base.utils.EmailAddress; import com.grey.base.utils.IP; import com.grey.base.utils.TSAP; -import com.grey.logging.Logger; import com.grey.mailismus.errors.MailismusConfigException; import com.grey.mailismus.mta.Protocol; +import com.grey.mailismus.mta.deliver.client.SmtpRelay; import com.grey.naf.NAFConfig; import com.grey.naf.reactor.config.SSLConfig; /* * This class specifies the connection details for a remote SMTP server we use as a relay for specific domains. */ -public final class Relay +public class Relay extends SmtpRelay { - public final TSAP tsap; - public final SSLConfig sslconfig; - public final boolean auth_enabled; - public final boolean auth_initrsp; - public final boolean auth_compat; //handle Protocol.EXT_AUTH_COMPAT responses from this server - public final SaslEntity.MECH auth_override; - public final String usrnam; - public final ByteChars passwd; - public final boolean dns_only; //only relevant for interceptor mode - true means don't intercept statically configured servers - final ByteChars[] destdomains; - final EmailAddress[] senders; - final IP.Subnet[] sender_ipnets; //if present, sender connect from one of these IPs, to match a source-Relay - private final String relay_string; - private final String display_txt; - - @Override public String toString() {return relay_string;} - public String display() {return display_txt;} - - public Relay(XmlConfig cfg, boolean interceptor, NAFConfig nafcfg, Logger log) throws IOException + public final ByteChars[] destDomains; + public final EmailAddress[] senders; + public final IP.Subnet[] senderIpNets; //if present, sender connect from one of these IPs, to match a source-Relay + public final boolean dnsOnly; //only relevant for interceptor mode - true means don't intercept statically configured servers + private final String relayString; + + public static Relay create(String name, XmlConfig cfg, boolean isInterceptor, NAFConfig nafcfg) throws IOException { + String usrnam = null; + CharSequence passwd = null; + SaslEntity.MECH auth_override = null; + SSLConfig sslconfig = null; + boolean dns_only = false; + List lst_destdoms = new ArrayList<>(); List lst_senders = new ArrayList<>(); List lst_subnets = new ArrayList<>(); - if (interceptor) { + + if (isInterceptor) { dns_only = cfg.getBool("@dns", false); } else { - dns_only = false; String s = cfg.getValue("@destdomains", false, null); if (s != null) { String[] arr = s.split(","); @@ -82,11 +77,12 @@ public Relay(XmlConfig cfg, boolean interceptor, NAFConfig nafcfg, Logger log) t } } } - destdomains = (lst_destdoms.isEmpty() ? null : lst_destdoms.toArray(new ByteChars[lst_destdoms.size()])); - senders = (lst_senders.isEmpty() ? null : lst_senders.toArray(new EmailAddress[lst_senders.size()])); + ByteChars[] destdomains = (lst_destdoms.isEmpty() ? null : lst_destdoms.toArray(new ByteChars[lst_destdoms.size()])); + EmailAddress[] senders = (lst_senders.isEmpty() ? null : lst_senders.toArray(new EmailAddress[lst_senders.size()])); + name = cfg.getValue("@name", true, name); String ipspec = cfg.getValue("@address", true, null); - tsap = TSAP.build(ipspec, Protocol.TCP_PORT, true); + TSAP tsap = TSAP.build(ipspec, Protocol.TCP_PORT, true); if (senders != null) { String s = cfg.getValue("@sendernets", false, null); @@ -100,16 +96,12 @@ public Relay(XmlConfig cfg, boolean interceptor, NAFConfig nafcfg, Logger log) t } } } - sender_ipnets = (lst_subnets.isEmpty() ? null : lst_subnets.toArray(new IP.Subnet[lst_subnets.size()])); - - auth_enabled = cfg.getBool("auth/@enabled", false); - auth_initrsp = cfg.getBool("auth/@initrsp", false); - auth_compat = cfg.getBool("auth/@compat", false); - if (!auth_enabled) { - usrnam = null; - passwd = null; - auth_override = null; - } else { + IP.Subnet[] sender_ipnets = (lst_subnets.isEmpty() ? null : lst_subnets.toArray(new IP.Subnet[lst_subnets.size()])); + + boolean auth_enabled = cfg.getBool("auth/@enabled", false); + boolean auth_initrsp = cfg.getBool("auth/@initrsp", false); + boolean auth_compat = cfg.getBool("auth/@compat", false); + if (auth_enabled) { usrnam = cfg.getValue("auth/username", false, null); passwd = new ByteChars(cfg.getValue("auth/password", false, null)); String val = cfg.getValue("auth/@override", false, null); @@ -117,9 +109,7 @@ public Relay(XmlConfig cfg, boolean interceptor, NAFConfig nafcfg, Logger log) t } XmlConfig sslcfg = cfg.getSection("ssl"); - if (sslcfg == null || !sslcfg.exists()) { - sslconfig = null; - } else { + if (sslcfg != null && sslcfg.exists()) { sslconfig = new SSLConfig.Builder() .withPeerCertName(ipspec) .withIsClient(true) @@ -127,23 +117,85 @@ public Relay(XmlConfig cfg, boolean interceptor, NAFConfig nafcfg, Logger log) t .build(); } - String txt = "SMTP-"+(interceptor ? "Interceptor"+(dns_only?"/DNS":"") : "Relay")+"="; - if (usrnam != null) txt += usrnam+"@"; - txt += tsap; - if (sslconfig != null) txt += "/SSL"; - display_txt = txt; - if (destdomains != null) txt += "=>"+(destdomains.length==1?destdomains[0]:destdomains.length+"/"+lst_destdoms); - if (senders != null) txt += "; Senders="+(senders.length==1?senders[0]:senders.length+"/"+lst_senders); - if (sender_ipnets != null) txt += "; SenderNets="+sender_ipnets.length+"/"+lst_subnets; - relay_string = txt; - - if (log != null) { - log.info(relay_string); - String indent = new String(new char[5]).replace('\0', ' '); - if (auth_enabled) { - log.info(indent+"Authenticate with override="+auth_override+"/initrsp="+auth_initrsp+(auth_compat?"/compat="+true:"")); - } - if (sslconfig != null) log.info(indent+sslconfig); + return builder() + .withName(name) + .withAddress(tsap) + .withSslConfig(sslconfig) + .withAuthRequired(auth_enabled) + .withAuthOverride(auth_override) + .withAuthCompat(auth_compat) + .withAuthInitialResponse(auth_initrsp) + .withUsername(usrnam) + .withPassword(passwd) + .withDestDomains(destdomains) + .withSenders(senders) + .withSenderIpNets(sender_ipnets) + .withDnsOnly(dns_only) + .withInterceptor(isInterceptor) + .build(); + } + + public Relay(Builder bldr) { + super(bldr); + this.dnsOnly = bldr.dnsOnly; + this.destDomains = bldr.destDomains; + this.senders = bldr.senders; + this.senderIpNets = bldr.senderIpNets; + + relayString = super.toString()+"=>ForwarderRelay[" + +(bldr.isInterceptor ? "Interceptor"+(dnsOnly?"/DNS":"")+", " : "") + +"Senders="+(senders==null?"":senders.length+"/"+Arrays.asList(senders)) + +", SenderNets="+(senderIpNets==null?"":senderIpNets.length+"/"+Arrays.asList(senderIpNets)) + +", DestDomain="+(destDomains==null?"":destDomains.length+"/"+Arrays.asList(destDomains)) + +"]"; + } + + @Override public String toString() { + return relayString; + } + + public static Builder builder() { + return new Builder<>(); + } + + + public static class Builder> extends SmtpRelay.Builder { + private boolean dnsOnly; + private ByteChars[] destDomains; + private EmailAddress[] senders; + private IP.Subnet[] senderIpNets; + private boolean isInterceptor; + + private Builder() {} + + public T withDnsOnly(boolean dnsOnly) { + this.dnsOnly = dnsOnly; + return self(); + } + + public T withDestDomains(ByteChars[] destDomains) { + this.destDomains = destDomains; + return self(); + } + + public T withSenders(EmailAddress[] senders) { + this.senders = senders; + return self(); + } + + public T withSenderIpNets(IP.Subnet[] senderIpNets) { + this.senderIpNets = senderIpNets; + return self(); + } + + public T withInterceptor(boolean val) { + this.isInterceptor = val; + return self(); + } + + @Override + public Relay build() { + return new Relay(this); } } } \ No newline at end of file diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Routing.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Routing.java index c484fe1..1ec2f17 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Routing.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Routing.java @@ -35,13 +35,13 @@ public Routing(XmlConfig relaycfg, NAFConfig nafcfg, Logger logger) throws IOExc XmlConfig[] cfgnodes = relaycfg.getSections("relay"); if (cfgnodes != null) { for (int idx = 0; idx != cfgnodes.length; idx++) { - Relay relay = new Relay(cfgnodes[idx], false, nafcfg, logger); - if (relay.destdomains == null && relay.senders == null) { + Relay relay = Relay.create("relay-"+idx, cfgnodes[idx], false, nafcfg); + if (relay.destDomains == null && relay.senders == null) { if (rlysrvr != null) throw new MailismusConfigException("Duplicate general relay: "+relay); rlysrvr = relay; } else { - if (relay.destdomains != null) { - for (ByteChars destdom : relay.destdomains) { + if (relay.destDomains != null) { + for (ByteChars destdom : relay.destDomains) { if (byDestDomain.put(destdom, relay) != null) { throw new MailismusConfigException("Duplicate route for "+destdom+": "+relay); } @@ -69,7 +69,7 @@ public Routing(XmlConfig relaycfg, NAFConfig nafcfg, Logger logger) throws IOExc Relay irly = null; if (cfgnodes != null) { if (cfgnodes.length != 1) throw new MailismusConfigException("Only one interceptor can be defined - found relays/intercept="+cfgnodes.length); - irly = new Relay(cfgnodes[0], true, nafcfg, logger); + irly = Relay.create("interceptor", cfgnodes[0], true, nafcfg); } interceptor = irly; @@ -92,8 +92,8 @@ public Relay getSourceRoute(EmailAddress sender, int ip_sender) { Relay relay = bySrcAddress.get(sender.full); if (relay == null) relay = bySrcDomain.get(sender.domain); - if (relay == null || relay.sender_ipnets == null || ip_sender == 0) return relay; - for (IP.Subnet net : relay.sender_ipnets) { + if (relay == null || relay.senderIpNets == null || ip_sender == 0) return relay; + for (IP.Subnet net : relay.senderIpNets) { if (net.isMember(ip_sender)) return relay; } return null; diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/SharedFields.java b/server/src/main/java/com/grey/mailismus/mta/deliver/SharedFields.java index e51cf40..7190253 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/SharedFields.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/SharedFields.java @@ -26,12 +26,12 @@ import com.grey.mailismus.Task; import com.grey.mailismus.Transcript; import com.grey.mailismus.mta.Protocol; +import com.grey.mailismus.mta.deliver.client.ConnectionConfig; class SharedFields { private final ConnectionConfig defaultConfig; private final List remotesConfig; //alternative configs for specfic remote destinations - private final Delivery.Controller controller; private final ResolverDNS dnsResolver; private final Transcript transcript; private final BufferGenerator bufferGenerator; @@ -67,7 +67,6 @@ class SharedFields { private ByteBuffer tmpNioBuffer; //grows on demand public SharedFields(Builder bldr) throws GeneralSecurityException { - this.controller = bldr.controller; this.dnsResolver = bldr.dnsResolver; this.bufferGenerator = bldr.bufferGenerator; this.transcript = bldr.transcript; @@ -82,7 +81,7 @@ public SharedFields(Builder bldr) throws GeneralSecurityException { } List conncfgs = new ArrayList<>(); - conncfgs.add(defaultConfig); + if (defaultConfig != null) conncfgs.add(defaultConfig); conncfgs.addAll(remotesConfig); for (ConnectionConfig conncfg : conncfgs) { String host = conncfg.getAnnouncehost(); @@ -140,10 +139,6 @@ public List getRemotesConfig() { return remotesConfig; } - public Delivery.Controller getController() { - return controller; - } - public ResolverDNS getDnsResolver() { return dnsResolver; } @@ -216,11 +211,13 @@ public byte[] getTmpIP() { return tmpIP; } - public StringBuilder getTmpSB() { + public StringBuilder getTmpSB(boolean reset) { + if (reset) tmpSB.setLength(0); return tmpSB; } - public StringBuilder getTmpSB2() { + public StringBuilder getTmpSB2(boolean reset) { + if (reset) tmpSB2.setLength(0); return tmpSB2; } @@ -238,7 +235,6 @@ public static Builder builder() { static class Builder { - private Delivery.Controller controller; private ResolverDNS dnsResolver; private BufferGenerator bufferGenerator; private Transcript transcript; @@ -248,11 +244,6 @@ static class Builder { private Builder() { } - public Builder withController(Delivery.Controller ctl) { - this.controller = ctl; - return this; - } - public Builder withDnsResolver(ResolverDNS dns) { this.dnsResolver = dns; return this; diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/ConnectionConfig.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/ConnectionConfig.java similarity index 99% rename from server/src/main/java/com/grey/mailismus/mta/deliver/ConnectionConfig.java rename to server/src/main/java/com/grey/mailismus/mta/deliver/client/ConnectionConfig.java index c41035b..55eaa21 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/ConnectionConfig.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/ConnectionConfig.java @@ -2,7 +2,7 @@ * Copyright 2010-2024 Yusef Badri - All rights reserved. * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). */ -package com.grey.mailismus.mta.deliver; +package com.grey.mailismus.mta.deliver.client; import java.time.Duration; import java.util.ArrayList; diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java new file mode 100644 index 0000000..24a3954 --- /dev/null +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java @@ -0,0 +1,22 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver.client; + +import java.util.List; +import java.util.function.Supplier; + +public interface SmtpMessage { + interface Recipient { + CharSequence getDomain(); + CharSequence getMailbox(); + } + + CharSequence getSender(); + List getRecipients(); + Supplier getData(); + + // This is only used for logging, and is something meaningful to the caller. Maybe the SMTP message-ID, but not necessarily. + CharSequence getMessageId(); +} diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java new file mode 100644 index 0000000..a146bde --- /dev/null +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java @@ -0,0 +1,95 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver.client; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.function.Supplier; + +public class SmtpMessageDefaultImpl implements SmtpMessage { + private final String msgId = UUID.randomUUID().toString(); + private final CharSequence sender; + private final List recipients; + private final Supplier data; //data can be Path or byte[] + + public SmtpMessageDefaultImpl(Builder bldr) { + this.sender = bldr.sender; + this.recipients = bldr.recipients; + this.data = bldr.data; + } + + @Override + public CharSequence getSender() { + return sender; + } + + @Override + public List getRecipients() { + return recipients; + } + + @Override + public Supplier getData() { + return data; + } + + @Override + public CharSequence getMessageId() { + return msgId; + } + + + public static class SmtpRecipientDefaultImpl implements SmtpMessage.Recipient { + private final CharSequence domain; + private final CharSequence mailbox; + + public SmtpRecipientDefaultImpl(CharSequence domain, CharSequence mailbox) { + this.domain = domain; + this.mailbox = mailbox; + } + + @Override + public CharSequence getDomain() { + return domain; + } + + @Override + public CharSequence getMailbox() { + return mailbox; + } + } + + + public static class Builder { + private List recipients = new ArrayList<>(); + private CharSequence sender; + private Supplier data; + + public Builder withSender(CharSequence sender) { + this.sender = sender; + return this; + } + + public Builder withRecipient(Recipient recipient) { + this.recipients.add(recipient); + return this; + } + + public Builder withRecipients(List recipients) { + this.recipients.addAll(recipients); + return this; + } + + public Builder withData(Supplier data) { + this.data = data; + return this; + } + + public SmtpMessageDefaultImpl createSmtpMessage() { + return new SmtpMessageDefaultImpl(this); + } + } +} diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpRelay.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpRelay.java new file mode 100644 index 0000000..b15db25 --- /dev/null +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpRelay.java @@ -0,0 +1,156 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver.client; + +import com.grey.base.sasl.SaslEntity; +import com.grey.base.utils.TSAP; +import com.grey.naf.reactor.config.SSLConfig; + +public class SmtpRelay { + private final CharSequence name; + private final TSAP address; + private final SSLConfig sslConfig; + private final boolean authRequired; + private final SaslEntity.MECH authOverride; //override whatever auth methods the relay offers + private final boolean authCompat; //handle deprecated Protocol.EXT_AUTH_COMPAT responses from this relay + private final boolean authInitialResponse; //send credentials with AUTH command in single request (for supported methods) + private final CharSequence username; + private final CharSequence password; + + public SmtpRelay(Builder bldr) { + this.name = bldr.name; + this.address = bldr.address; + this.sslConfig = bldr.sslConfig; + this.authRequired = bldr.authRequired; + this.authOverride = bldr.authOverride; + this.authCompat = bldr.authCompat; + this.authInitialResponse = bldr.authInitialResponse; + this.username = bldr.username; + this.password = bldr.password; + } + + public CharSequence getName() { + return name; + } + + public TSAP getAddress() { + return address; + } + + public SSLConfig getSslConfig() { + return sslConfig; + } + + public boolean isAuthRequired() { + return authRequired; + } + + public SaslEntity.MECH getAuthOverride() { + return authOverride; + } + + public boolean isAuthCompat() { + return authCompat; + } + + public boolean isAuthInitialResponse() { + return authInitialResponse; + } + + public CharSequence getUsername() { + return username; + } + + public CharSequence getPassword() { + return password; + } + + public static Builder builder() { + return new Builder<>(); + } + + @Override + public String toString() { + return "SmtpRelay[" + +"name=" + name + +", address=" + address + +", authRequired=" + authRequired + +", authOverride=" + authOverride + +", authCompat=" + authCompat + +", authInitialResponse="+ authInitialResponse + +", username=" + username + +", sslConfig="+ sslConfig + +"]"; + } + + + public static class Builder> { + private CharSequence name; + private TSAP address; + private SSLConfig sslConfig; + private boolean authRequired; + private SaslEntity.MECH authOverride; + private boolean authCompat; + private boolean authInitialResponse; + private CharSequence username; + private CharSequence password; + + protected Builder() {} + + public T withName(CharSequence name) { + this.name = name; + return self(); + } + + public T withAddress(TSAP address) { + this.address = address; + return self(); + } + + public T withSslConfig(SSLConfig sslConfig) { + this.sslConfig = sslConfig; + return self(); + } + + public T withAuthRequired(boolean authRequired) { + this.authRequired = authRequired; + return self(); + } + + public T withAuthOverride(SaslEntity.MECH authOverride) { + this.authOverride = authOverride; + return self(); + } + + public T withAuthCompat(boolean authCompat) { + this.authCompat = authCompat; + return self(); + } + + public T withAuthInitialResponse(boolean authInitialResponse) { + this.authInitialResponse = authInitialResponse; + return self(); + } + + public T withUsername(CharSequence username) { + this.username = username; + return self(); + } + + public T withPassword(CharSequence password) { + this.password = password; + return self(); + } + + protected T self() { + @SuppressWarnings("unchecked") T bldr = (T)this; + return bldr; + } + + public SmtpRelay build() { + return new SmtpRelay(this); + } + } +} diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java new file mode 100644 index 0000000..8233211 --- /dev/null +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java @@ -0,0 +1,90 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver.client; + +import com.grey.base.utils.ByteArrayRef; + +public class SmtpResponseDescriptor { + private static final int STATUSCODE_DIGITS = 3; + + private final short smtp_status; //xxx convert to int and make the Protocol constants ints too + private final String enhanced_status; + private final String msg; + + public static SmtpResponseDescriptor parse(ByteArrayRef rspdata, boolean withEnhanced) { + StringBuilder sb = null; + String enhanced = null; + String msg = ""; + + int rsplen = rspdata.size(); + while (rsplen != 0 && rspdata.buffer()[rspdata.offset() + rsplen - 1] <= ' ') { + rsplen--; + } + + if (rsplen < STATUSCODE_DIGITS) + return null; + + int off = rspdata.offset(); + short statuscode = (short)parseDecimal(STATUSCODE_DIGITS, rspdata.buffer(), off); + off += STATUSCODE_DIGITS; + + off++; //skip space + //xxx could also be hyphen (see Google EHLO response) and return null if neither - caller must handle null + if (off >= rsplen) + return new SmtpResponseDescriptor(statuscode, enhanced, msg); + + if (withEnhanced) { + int off2 = off; + while (rspdata.buffer()[off2] != ' ') { + off2++; + if (off2 == rsplen) break; + } + sb = (StringBuilder)rspdata.toString(sb, off, off2 - off); + enhanced = sb.toString(); + off = off2 + 1; + sb.setLength(0); + } + + if (off <= rsplen) { + msg = rspdata.toString(sb, off, rsplen - off).toString(); + } + return new SmtpResponseDescriptor(statuscode, enhanced, msg); + } + + public SmtpResponseDescriptor(short smtp_status, String enhanced_status, String msg) { + this.smtp_status = smtp_status; + this.enhanced_status = enhanced_status; + this.msg = msg; + } + + public short smtpStatus() { + return smtp_status; + } + + public String enhancedStatus() { + return enhanced_status; + } + + public String message() { + return msg; + } + + @Override + public String toString() { + return "SmtpResponse [smtp_status="+smtp_status+", enhanced="+enhanced_status+ ", msg="+msg+"]"; + } + + private static int parseDecimal(int numDigits, byte[] buf, int off) { + int idx = off + numDigits - 1; + int factor = 1; + int val = 0; + + for (int loop = 0; loop != numDigits; loop++) { + val += (buf[idx--] - '0') * factor; + factor *= 10; + } + return val; + } +} \ No newline at end of file diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSender.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSender.java new file mode 100644 index 0000000..62d2ffe --- /dev/null +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSender.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver.client; + +import com.grey.base.utils.TSAP; + +public interface SmtpSender { + /** + * This is called after receiving the response to each individual recipient send (SMTP "RCPT TO" command) + * recipId is an index into SmtpMessage.getRecipients() + * Returns true to indicate that message processing should abort, else false + */ + default boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, SmtpResponseDescriptor status, TSAP remote, boolean aborted) { + return false; + } + + /** + * This is called after receiving the response to the message body (SMTP "DATA" command) + * recipId is an index into SmtpMessage.getRecipients() + * Returns non-null to indicate a new message to be sent on this connection. + */ + default SmtpMessage messageCompleted(SmtpMessage msg, int msgCount) { + return null; + } + + /** + * This is called after the SMTP client disconnects. + * Note that if neither this nor onMessage() is set, the sender receives no feedback on the message delivery. + */ + default void onDisconnect(SmtpMessage finalMsg, int msgCount) { + } +} diff --git a/server/src/main/java/com/grey/mailismus/mta/queue/QueueManager.java b/server/src/main/java/com/grey/mailismus/mta/queue/QueueManager.java index 16c786d..e7383dd 100644 --- a/server/src/main/java/com/grey/mailismus/mta/queue/QueueManager.java +++ b/server/src/main/java/com/grey/mailismus/mta/queue/QueueManager.java @@ -333,7 +333,7 @@ public final boolean endSubmit(SubmitHandle sph, boolean rollback) } } catch (Throwable ex) { success = false; - dsptch.getLogger().log(LEVEL.INFO, ex, true, loglbl+"Submit-ctl failed on SPID="+Integer.toHexString(sph.spid)); + dsptch.getLogger().log(LEVEL.INFO, ex, true, loglbl+"Submit-fwd failed on SPID="+Integer.toHexString(sph.spid)); } if (rollback || !success) { diff --git a/server/src/main/java/com/grey/mailismus/mta/submit/SubmitTask.java b/server/src/main/java/com/grey/mailismus/mta/submit/SubmitTask.java index 2399981..89883b2 100644 --- a/server/src/main/java/com/grey/mailismus/mta/submit/SubmitTask.java +++ b/server/src/main/java/com/grey/mailismus/mta/submit/SubmitTask.java @@ -40,9 +40,9 @@ protected boolean stopNaflet() { } @Override - public void eventIndication(Object obj, String eventId) { - if (!(obj instanceof ListenerSet) || !EventListenerNAF.EVENTID_ENTITY_STOPPED.equals(eventId)) { - getDispatcher().getLogger().info("SubmitTask="+getName()+" discarding unexpected event="+obj.getClass().getName()+"/"+eventId); + public void eventIndication(String eventId, Object evtsrc, Object data) { + if (!(evtsrc instanceof ListenerSet) || !EventListenerNAF.EVENTID_ENTITY_STOPPED.equals(eventId)) { + getDispatcher().getLogger().info("SubmitTask="+getName()+" discarding unexpected event="+eventId+"/"+evtsrc.getClass().getName()+"/"+data); return; } nafletStopped(); diff --git a/server/src/main/java/com/grey/mailismus/pop3/client/DownloadTask.java b/server/src/main/java/com/grey/mailismus/pop3/client/DownloadTask.java index 6f7cb28..716c3e7 100644 --- a/server/src/main/java/com/grey/mailismus/pop3/client/DownloadTask.java +++ b/server/src/main/java/com/grey/mailismus/pop3/client/DownloadTask.java @@ -86,11 +86,11 @@ private void stopped() } @Override - public void eventIndication(Object obj, String eventId) + public void eventIndication(String eventId, Object evtsrc, Object data) { - DownloadClient client = DownloadClient.class.cast(obj); + DownloadClient client = DownloadClient.class.cast(evtsrc); boolean active = activeClients.remove(client); - if (observer != null) observer.eventIndication(client, eventId); + if (observer != null) observer.eventIndication(eventId, client, null); if (inShutdown) { if (activeClients.size() == 0) stopped(); return; diff --git a/server/src/main/java/com/grey/mailismus/pop3/server/POP3Task.java b/server/src/main/java/com/grey/mailismus/pop3/server/POP3Task.java index d143876..2a8db43 100644 --- a/server/src/main/java/com/grey/mailismus/pop3/server/POP3Task.java +++ b/server/src/main/java/com/grey/mailismus/pop3/server/POP3Task.java @@ -41,9 +41,9 @@ protected boolean stopNaflet() { } @Override - public void eventIndication(Object obj, String eventId) { - if (!(obj instanceof ListenerSet) || !EventListenerNAF.EVENTID_ENTITY_STOPPED.equals(eventId)) { - getDispatcher().getLogger().info("POP3Task="+getName()+" discarding unexpected event="+obj.getClass().getName()+"/"+eventId); + public void eventIndication(String eventId, Object evtsrc, Object data) { + if (!(evtsrc instanceof ListenerSet) || !EventListenerNAF.EVENTID_ENTITY_STOPPED.equals(eventId)) { + getDispatcher().getLogger().info("POP3Task="+getName()+" discarding unexpected event="+eventId+"/"+evtsrc.getClass().getName()+"/"+data); return; } nafletStopped(); diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java index 123fbac..0c07c4a 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java @@ -10,6 +10,9 @@ import com.grey.base.utils.EmailAddress; import com.grey.base.utils.IP; import com.grey.base.utils.DynLoader; + +import java.security.GeneralSecurityException; + import com.grey.base.collections.HashedSetInt; import com.grey.naf.ApplicationContextNAF; import com.grey.naf.EventListenerNAF; @@ -21,6 +24,8 @@ import com.grey.mailismus.AppConfig; import com.grey.mailismus.TestSupport; import com.grey.mailismus.mta.Protocol; +import com.grey.mailismus.mta.deliver.client.SmtpMessage; +import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.mta.queue.MessageRecip; import com.grey.mailismus.mta.smtp.MockServerDNS; @@ -147,9 +152,9 @@ public void testSmarthost() throws Exception { String cfgxml = delivxml.replace("x1relay", "relay"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs1, true); - SenderFactory sndrfact = new SenderFactory(expected_senders, false); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, null, false); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, true); } @@ -160,9 +165,9 @@ public void testSenderRefill() throws Exception { String cfgxml = delivxml.replace("x2maxconnections", "maxconnections"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs2, true); - SenderFactory sndrfact = new SenderFactory(expected_senders, expected_refills, true); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, expected_refills, true); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -174,9 +179,9 @@ public void testLeftoverMessages() throws Exception { String cfgxml = delivxml.replace("x2maxconnections", "maxconnections"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs3, true, 1); - SenderFactory sndrfact = new SenderFactory(expected_senders, true); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, null, true); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -190,9 +195,9 @@ public void testServerConnections() throws Exception { String cfgxml = delivxml.replace("x2maxserverconnections", "maxserverconnections"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs4, true, 0); - SenderFactory sndrfact = new SenderFactory(expected_senders, expected_refills, true); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, expected_refills, true); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -203,9 +208,9 @@ public void testMaxServerConnectionsWithSourceRoute() throws Exception { String cfgxml = delivxml.replace("x2maxserverconnections", "maxserverconnections").replace("x2relay","relay"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs4, true, 0); - SenderFactory sndrfact = new SenderFactory(expected_senders, true); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, null, true); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -216,9 +221,9 @@ public void testSourceRouteRefill() throws Exception { String cfgxml = delivxml.replace("x2maxserverconnections", "maxserverconnections").replace("x2relay","relay"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs5, true, 0); - SenderFactory sndrfact = new SenderFactory(expected_senders, expected_refills, true); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, expected_refills, true); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -240,11 +245,11 @@ public void testBulk() throws Exception { } String[][] msgs = lst.toArray(new String[lst.size()][]); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs, true); - SenderFactory sndrfact = new SenderFactory(null, true); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, null, null, true); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); com.grey.mailismus.mta.queue.Cache qc = (com.grey.mailismus.mta.queue.Cache)DynLoader.getField(fwd, "qcache"); org.junit.Assert.assertEquals(2500, qc.capacity()); //the default for non-slaverelay mode - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); org.junit.Assert.assertEquals(maxserverconns * destcnt, sndrfact.senders.size()); for (int idx = 0; idx != sndrfact.senders.size(); idx++) { @@ -274,9 +279,9 @@ public void testBulkMultiRecips() throws Exception { } String[][] msgs = lst.toArray(new String[lst.size()][]); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs, true); - SenderFactory sndrfact = new SenderFactory(null, true); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, null, null, true); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); org.junit.Assert.assertEquals(maxserverconns * destcnt, sndrfact.senders.size()); for (int idx = 0; idx != sndrfact.senders.size(); idx++) { @@ -299,11 +304,11 @@ public void testBulkSmarthost() throws Exception { } String[][] msgs = lst.toArray(new String[lst.size()][]); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs, true); - SenderFactory sndrfact = new SenderFactory(null, true); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, null, null, true); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); com.grey.mailismus.mta.queue.Cache qc = (com.grey.mailismus.mta.queue.Cache)DynLoader.getField(fwd, "qcache"); org.junit.Assert.assertEquals(5000, qc.capacity()); //the default for slaverelay mode - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, true); org.junit.Assert.assertEquals(500, sndrfact.senders.size()); //default maxconnections=500 for (int idx = 0; idx != sndrfact.senders.size(); idx++) { @@ -325,9 +330,9 @@ private void testSenderAllocation(boolean deferred) throws Exception { int[][][] expected_senders = new int[][][]{{{101, 1}, {101, 3}}, {{101, 2}}, {{102, 1}}}; XmlConfig cfg = XmlConfig.makeSection(delivxml, "deliver"); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs1, true); - SenderFactory sndrfact = new SenderFactory(expected_senders, deferred); + SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, null, deferred); fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); - sndrfact.ctl = fwd; + sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -348,7 +353,7 @@ private void exec(MyQueueManager qmgr, SenderFactory sndrfact, boolean slaverela } @Override - public void eventIndication(Object obj, String eventId) + public void eventIndication(String eventId, Object obj, Object data) { if (obj == dsptch) { halted = fwd.stop(); @@ -358,7 +363,7 @@ public void eventIndication(Object obj, String eventId) } @Override - public void batchCompleted(int qsize, Delivery.Stats stats) { + public void batchCompleted(int qsize, Forwarder.DeliveryStats stats) { dsptch.stop(); } @@ -435,27 +440,29 @@ void addQError(String s) { } - private static class MySender - implements Delivery.MessageSender, - TimerNAF.Handler + private static class MySender extends Client { - private final Delivery.MessageParams msgparams = new Delivery.MessageParams(); private final SenderFactory mgr; private final int id; + private QueueBasedMessage msgparams; private TimerNAF tmr_report; private int seqno; //sequence number amongst Sender set - first Sender to be launched is zero public int msgcnt; @Override public String getLogID() {return "MySender-"+id;} - @Override public Delivery.MessageParams getMessageParams() {return msgparams;} @Override public short getDomainError() {return 0;} @Override public void setEventListener(EventListenerNAF l) {} @Override public String toString() {return "MySender="+getLogID();} - public MySender(int id, SenderFactory fact) {this.id=id; mgr=fact;} + public MySender(int id, SenderFactory fact) throws GeneralSecurityException { + super(SharedFields.builder().build(), fact.dsptch); + this.id = id; + mgr = fact; + } @Override - public void start(Delivery.Controller ctl) { + public void startConnection(SmtpMessage msg, SmtpSender sender, Relay relay) { + msgparams = (QueueBasedMessage)msg; seqno = mgr.sender_cnt++; int[][] exp = null; if (mgr.expected_msgs != null) { @@ -473,7 +480,7 @@ private void processMessage(int[][] exp) { if (msgparams.recipCount() != exp.length) mgr.addError(seqno, "recips="+msgparams.recipCount()+" vs "+exp.length); } for (int idx = 0; idx != msgparams.recipCount(); idx++) { - MessageRecip recip = msgparams.getRecipient(idx); + MessageRecip recip = msgparams.getRecipient(idx).getQueueRecip(); if (exp != null) { int exp_spid = exp[idx][0]; int exp_qid = exp[idx][1]; @@ -486,7 +493,7 @@ private void processMessage(int[][] exp) { } msgcnt++; if (mgr.deferred_cb) { - tmr_report = mgr.ctl.getDispatcher().setTimer(0, 0, this, Boolean.TRUE); + tmr_report = getDispatcher().setTimer(0, 0, this, Boolean.TRUE); } else { reportCompletion(true); } @@ -496,7 +503,7 @@ private void reportCompletion(boolean report_msg) { if (report_msg) { int[][] exp = null; if (mgr.expected_refills != null && msgcnt == 1 && seqno < mgr.expected_refills.length) exp = mgr.expected_refills[seqno]; - mgr.ctl.messageCompleted(this); + mgr.fwd.messageCompleted(msgparams, msgcnt); if (msgparams.recipCount() == 0) { if (exp != null) mgr.addError(seqno, "Expected another message after messageCompleted()"); } else { @@ -507,11 +514,11 @@ private void reportCompletion(boolean report_msg) { return; } if (mgr.deferred_cb) { - tmr_report = mgr.ctl.getDispatcher().setTimer(0, 0, this, Boolean.FALSE); + tmr_report = getDispatcher().setTimer(0, 0, this, Boolean.FALSE); return; } } - mgr.ctl.senderCompleted(this); + mgr.fwd.onDisconnect(msgparams, msgcnt); mgr.sender_completion_cnt++; } @@ -539,15 +546,17 @@ public void eventError(TimerNAF tmr, Dispatcher d, Throwable ex) { private static class SenderFactory - implements com.grey.base.collections.GenericFactory, + implements com.grey.base.collections.GenericFactory, Forwarder.BatchCallback, TimerNAF.Handler { + private final Dispatcher dsptch; + private final MyQueueManager qmgr; public final java.util.ArrayList senders = new java.util.ArrayList(); public final int[][][] expected_msgs; //expected messages, per MessageSender and per MessageRecip public final int[][][] expected_refills; //expected message after possible refill in messageCompleted() public final boolean deferred_cb; - public Delivery.Controller ctl; + public Forwarder fwd; public int sender_cnt; public int sender_completion_cnt; public int batch_completion_cnt; @@ -556,32 +565,34 @@ private static class SenderFactory @Override public MySender factory_create() { - MySender sender = new MySender(++next_id, this); + MySender sender; + try { + sender = new MySender(++next_id, this); + } catch (Exception ex) { + throw new RuntimeException(ex); + } senders.add(sender); return sender; } - public SenderFactory(int[][][] exp, boolean deferred) { - this(exp, null, deferred); - } - - public SenderFactory(int[][][] exp, int[][][] refills, boolean deferred) { + public SenderFactory(Dispatcher dsptch, MyQueueManager qmgr, int[][][] exp, int[][][] refills, boolean deferred) { + this.dsptch = dsptch; + this.qmgr = qmgr; expected_msgs = exp; expected_refills = refills; deferred_cb = deferred; } void addError(int id, String s) { - MyQueueManager qmgr = (MyQueueManager)ctl.getQueue(); qmgr.addError("MessageSender="+id+": "+s); } @Override - public void batchCompleted(int qsize, Delivery.Stats stats) { + public void batchCompleted(int qsize, Forwarder.DeliveryStats stats) { if (deferred_cb) { - ctl.getDispatcher().setTimer(0, 0, this); + dsptch.setTimer(0, 0, this); } else { - ctl.getDispatcher().stop(); + dsptch.stop(); } if (expected_msgs != null) { int relaycnt = 0; @@ -606,7 +617,7 @@ public void batchCompleted(int qsize, Delivery.Stats stats) { @Override public void timerIndication(TimerNAF tmr, Dispatcher d) { - ctl.getDispatcher().stop(); + dsptch.stop(); } @Override diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/RoutingTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/RoutingTest.java index 39b04f4..4f73c81 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/RoutingTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/RoutingTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 Yusef Badri - All rights reserved. + * Copyright 2015-2024 Yusef Badri - All rights reserved. * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). */ package com.grey.mailismus.mta.deliver; @@ -65,37 +65,37 @@ public void testDestinationRoutes() throws java.io.IOException verifyDestinationRoute(rt, "anydomain", false, "10.0.0.1", 25001); verifyDestinationRoute(rt, "destdomain1", true, "10.0.0.2", 1025); Relay rly = rt.getRoute(new ByteChars("destdomain1")); - org.junit.Assert.assertFalse(rly.auth_enabled); - org.junit.Assert.assertNull(rly.usrnam); + org.junit.Assert.assertFalse(rly.isAuthRequired()); + org.junit.Assert.assertNull(rly.getUsername()); org.junit.Assert.assertNull(rly.senders); rt = parseRouting(cfgxml_nodflt, false, false, false); verifyDestinationRoute(rt, "anydomain", false, null, 0); verifyDestinationRoute(rt, "destdomain1", true, "10.0.0.2", Protocol.TCP_PORT); rly = rt.getRoute(new ByteChars("destdomain1")); - org.junit.Assert.assertTrue(rly.auth_enabled); - org.junit.Assert.assertEquals("user1", rly.usrnam); - org.junit.Assert.assertEquals("pass1", rly.passwd.toString()); - org.junit.Assert.assertNull(rly.sslconfig); + org.junit.Assert.assertTrue(rly.isAuthRequired()); + org.junit.Assert.assertEquals("user1", rly.getUsername()); + org.junit.Assert.assertEquals("pass1", rly.getPassword().toString()); + org.junit.Assert.assertNull(rly.getSslConfig()); rt = parseRouting(cfgxml_interceptor, false, true, true); verifyDestinationRoute(rt, "anydomain", false, "10.0.0.1", Protocol.TCP_PORT); verifyDestinationRoute(rt, "destdomain1", true, "10.0.0.2", Protocol.TCP_PORT); rly = rt.getInterceptor(); - org.junit.Assert.assertTrue(rly.dns_only); - org.junit.Assert.assertEquals(IP.IP_LOCALHOST, rly.tsap.ip); - org.junit.Assert.assertEquals(2001, rly.tsap.port); + org.junit.Assert.assertTrue(rly.dnsOnly); + org.junit.Assert.assertEquals(IP.IP_LOCALHOST, rly.getAddress().ip); + org.junit.Assert.assertEquals(2001, rly.getAddress().port); org.junit.Assert.assertNull(rly.senders); rt = parseRouting(cfgxml_interceptor_withslave, true, true, true); verifyDestinationRoute(rt, "anydomain", false, "10.0.0.1", Protocol.TCP_PORT); rly = rt.getInterceptor(); - org.junit.Assert.assertFalse(rly.dns_only); - org.junit.Assert.assertEquals(IP.IP_LOCALHOST, rly.tsap.ip); - org.junit.Assert.assertEquals(Protocol.TCP_PORT, rly.tsap.port); - org.junit.Assert.assertNotNull(rly.sslconfig); - org.junit.Assert.assertFalse(rly.sslconfig.isLatent()); - org.junit.Assert.assertFalse(rly.sslconfig.isMandatory()); + org.junit.Assert.assertFalse(rly.dnsOnly); + org.junit.Assert.assertEquals(IP.IP_LOCALHOST, rly.getAddress().ip); + org.junit.Assert.assertEquals(Protocol.TCP_PORT, rly.getAddress().port); + org.junit.Assert.assertNotNull(rly.getSslConfig()); + org.junit.Assert.assertFalse(rly.getSslConfig().isLatent()); + org.junit.Assert.assertFalse(rly.getSslConfig().isMandatory()); org.junit.Assert.assertNull(rly.senders); } @@ -106,18 +106,18 @@ public void testSourceRoutes() throws java.io.IOException // source-address route - only the specific address should match EmailAddress emaddr = new EmailAddress("sender2@srcdomain2").decompose(); Relay rly = rt.getSourceRoute(emaddr, 0); - org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.2"), rly.tsap.ip); + org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.2"), rly.getAddress().ip); emaddr = new EmailAddress("sender3@srcomain2").decompose(); rly = rt.getSourceRoute(emaddr, 0); org.junit.Assert.assertNull(rly); // source-domain route emaddr = new EmailAddress("sender9@srcdomain1").decompose(); rly = rt.getSourceRoute(emaddr, 999); - org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.2"), rly.tsap.ip); + org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.2"), rly.getAddress().ip); // source-address route that overlaps with a source-domain route takes precedence emaddr = new EmailAddress("sender1@srcdomain1").decompose(); rly = rt.getSourceRoute(emaddr, 0); - org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.3"), rly.tsap.ip); + org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.3"), rly.getAddress().ip); // test a completely unknown source domain emaddr = new EmailAddress("anymailbox@anydomain").decompose(); rly = rt.getSourceRoute(emaddr, 0); @@ -126,7 +126,7 @@ public void testSourceRoutes() throws java.io.IOException //now test co-existence with a destination route emaddr = new EmailAddress("sender@srcdomain3").decompose(); rly = rt.getSourceRoute(emaddr, 0); - org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.4"), rly.tsap.ip); + org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.4"), rly.getAddress().ip); Relay rly2 = rt.getRoute(new ByteChars("destdomain1")); org.junit.Assert.assertTrue(rly2 == rly); } @@ -138,17 +138,17 @@ public void testSourceRoutesWithIP() throws java.io.IOException EmailAddress emaddr = new EmailAddress("sender1@srcdomain1").decompose(); //control test - make sure source route matches without an IP restriction Relay rly = rt.getSourceRoute(emaddr, 0); - org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.5"), rly.tsap.ip); + org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.5"), rly.getAddress().ip); rly = rt.getSourceRoute(emaddr, IP.convertDottedIP("192.168.100.1")); - org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.5"), rly.tsap.ip); + org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.5"), rly.getAddress().ip); rly = rt.getSourceRoute(emaddr, IP.convertDottedIP("192.168.0.0")); - org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.5"), rly.tsap.ip); + org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.5"), rly.getAddress().ip); rly = rt.getSourceRoute(emaddr, IP.convertDottedIP("192.167.255.255")); org.junit.Assert.assertNull(rly); rly = rt.getSourceRoute(emaddr, IP.IP_LOCALHOST); - org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.5"), rly.tsap.ip); + org.junit.Assert.assertEquals(IP.convertDottedIP("10.0.0.5"), rly.getAddress().ip); rly = rt.getSourceRoute(emaddr, IP.convertDottedIP("127.0.0.0")); org.junit.Assert.assertNull(rly); } @@ -172,8 +172,8 @@ private void verifyDestinationRoute(Routing rt, String destdom, boolean exp_rt, if (exp_ip == null) { org.junit.Assert.assertNull(rly); } else { - org.junit.Assert.assertEquals(IP.convertDottedIP(exp_ip), rly.tsap.ip); - org.junit.Assert.assertEquals(exp_port, rly.tsap.port); + org.junit.Assert.assertEquals(IP.convertDottedIP(exp_ip), rly.getAddress().ip); + org.junit.Assert.assertEquals(exp_port, rly.getAddress().port); org.junit.Assert.assertNull(rly.senders); } } diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptorTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptorTest.java new file mode 100644 index 0000000..3e56b49 --- /dev/null +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptorTest.java @@ -0,0 +1,133 @@ +package com.grey.mailismus.mta.deliver.client; + +import java.nio.charset.StandardCharsets; + +import com.grey.base.utils.ByteArrayRef; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class SmtpResponseDescriptorTest { + @Test + public void testNoEnhanced() { + String str = "250 All OK"; + ByteArrayRef rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + SmtpResponseDescriptor reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(250, reply.smtpStatus()); + assertEquals("All OK", reply.message()); + assertNull(reply.enhancedStatus()); + + str = "250 2.1.0 OK"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(250, reply.smtpStatus()); + assertEquals("2.1.0 OK", reply.message()); + assertNull(reply.enhancedStatus()); + + str = "250 2.1.0"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(250, reply.smtpStatus()); + assertEquals("2.1.0", reply.message()); + assertNull(reply.enhancedStatus()); + + str = "500"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(500, reply.smtpStatus()); + assertTrue(reply.toString(), reply.message().isEmpty()); + assertNull(reply.enhancedStatus()); + } + + @Test + public void testWithEnhanced() { + String str = "250 All OK"; + ByteArrayRef rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + SmtpResponseDescriptor reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(250, reply.smtpStatus()); + assertEquals("OK", reply.message()); + assertEquals("All", reply.enhancedStatus()); + + str = "250 2.1.0 OK"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(250, reply.smtpStatus()); + assertEquals("OK", reply.message()); + assertEquals("2.1.0", reply.enhancedStatus()); + + str = "250 2.1.0"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(250, reply.smtpStatus()); + assertTrue(reply.toString(), reply.message().isEmpty()); + assertEquals("2.1.0", reply.enhancedStatus()); + + str = "500"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(500, reply.smtpStatus()); + assertTrue(reply.toString(), reply.message().isEmpty()); + assertNull(reply.enhancedStatus()); + } + + @Test + public void testWithEOL() { + String str = "250 All OK\r\n"; + ByteArrayRef rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + SmtpResponseDescriptor reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(250, reply.smtpStatus()); + assertEquals("OK", reply.message()); + assertEquals("All", reply.enhancedStatus()); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(250, reply.smtpStatus()); + assertEquals("All OK", reply.message()); + assertNull(reply.enhancedStatus()); + + str = "250 All OK\n"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(250, reply.smtpStatus()); + assertEquals("OK", reply.message()); + assertEquals("All", reply.enhancedStatus()); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(250, reply.smtpStatus()); + assertEquals("All OK", reply.message()); + assertNull(reply.enhancedStatus()); + + str = "250 All OK"+"".repeat(3); + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(250, reply.smtpStatus()); + assertEquals("OK", reply.message()); + assertEquals("All", reply.enhancedStatus()); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(250, reply.smtpStatus()); + assertEquals("All OK", reply.message()); + assertNull(reply.enhancedStatus()); + } + + @Test + public void testInvalid() { + String str = ""; + ByteArrayRef rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + SmtpResponseDescriptor reply = SmtpResponseDescriptor.parse(rspdata, true); + assertNull(reply); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertNull(reply); + + str = " ".repeat(5); + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertNull(reply); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertNull(reply); + + str = "25"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertNull(reply); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertNull(reply); + } +} diff --git a/server/src/test/java/com/grey/mailismus/mta/queue/BaseManagerTest.java b/server/src/test/java/com/grey/mailismus/mta/queue/BaseManagerTest.java index d4be82e..71af361 100644 --- a/server/src/test/java/com/grey/mailismus/mta/queue/BaseManagerTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/queue/BaseManagerTest.java @@ -215,7 +215,7 @@ public TestManager_SubmitFails(Dispatcher d, XmlConfig qcfg, AppConfig appcfg, S super(d, qcfg, appcfg, name); } @Override - protected boolean storeMessage(SubmitHandle sph) {throw new QException("Simulating ctl-store failure");} + protected boolean storeMessage(SubmitHandle sph) {throw new QException("Simulating fwd-store failure");} } public static class TestManager extends QueueManager diff --git a/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java b/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java index c7ce17d..22ac1e3 100644 --- a/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java @@ -43,7 +43,6 @@ import com.grey.mailismus.mta.submit.filter.api.MessageFilter; import com.grey.mailismus.mta.deliver.Relay; import com.grey.mailismus.mta.deliver.DeliverTask; -import com.grey.mailismus.mta.deliver.Delivery; import com.grey.mailismus.mta.deliver.Forwarder; import com.grey.mailismus.TestSupport; import com.grey.mailismus.ms.maildir.MaildirStore; @@ -422,7 +421,7 @@ private void runtest(MessageSpec[] msgs, int server_submitcnt, int server_spoolc if (interceptor_spec != null) { String ixml = ""; cfg = XmlConfig.makeSection(ixml, "intercept"); - Relay interceptor = new Relay(cfg, true, nafcfg, dsptch.getLogger()); + Relay interceptor = Relay.create("test-interceptor", cfg, true, nafcfg); Object routing = DynLoader.getField(smtp_sender, "routing"); DynLoader.setField(routing, "interceptor", interceptor); } @@ -545,7 +544,7 @@ private void runtest(MessageSpec[] msgs, int server_submitcnt, int server_spoolc } @Override - public void batchCompleted(int qsize, Delivery.Stats stats) + public void batchCompleted(int qsize, Forwarder.DeliveryStats stats) { if (qsize == 0) { if (!stopping) { @@ -616,7 +615,7 @@ public void cancel() {} private static class FwdStats { - final Delivery.Stats stats = new Delivery.Stats(TimeProvider); + final Forwarder.DeliveryStats stats = new Forwarder.DeliveryStats(TimeProvider); int qsize; FwdStats() {} FwdStats(int q, int c, int m) {qsize = q; stats.conncnt = c; stats.sendermsgcnt = m;} diff --git a/server/src/test/java/com/grey/mailismus/pop3/POP3Test.java b/server/src/test/java/com/grey/mailismus/pop3/POP3Test.java index 4ff0ca7..37da8bb 100644 --- a/server/src/test/java/com/grey/mailismus/pop3/POP3Test.java +++ b/server/src/test/java/com/grey/mailismus/pop3/POP3Test.java @@ -279,12 +279,12 @@ public ClientReaper(com.grey.naf.reactor.TimerNAF.Handler th, Dispatcher d) { } @Override - public void eventIndication(Object obj, String eventId) { - if (obj instanceof DownloadTask) { + public void eventIndication(String eventId, Object evtsrc, Object data) { + if (evtsrc instanceof DownloadTask) { dsptch.setTimer(100, TMRTYPE_STOP, observer); //give server time to receive disconnect event return; } - DownloadClient client = (DownloadClient)obj; + DownloadClient client = (DownloadClient)evtsrc; DownloadClient.Results r = new DownloadClient.Results(); r.completed_ok = client.getResults().completed_ok; r.msgcnt = client.getResults().msgcnt; @@ -302,7 +302,7 @@ public TestDownloadTask(String name, Dispatcher d, XmlConfig cfg, int srvport, E @Override protected void nafletStopped() { super.nafletStopped(); - evtl.eventIndication(this, EventListenerNAF.EVENTID_ENTITY_STOPPED); + evtl.eventIndication(EventListenerNAF.EVENTID_ENTITY_STOPPED, this, null); } } } \ No newline at end of file From 3fa134673af7deac9eaf8a4ead8f2115b58689cd Mon Sep 17 00:00:00 2001 From: ybadri Date: Tue, 19 Mar 2024 11:41:53 +0000 Subject: [PATCH 2/6] Refactored some NAF constructor methods - Dispatcher, ApplicationContextNAF and BufferGenerator --- .../java/com/grey/mailismus/AppConfig.java | 2 +- .../java/com/grey/mailismus/DBHandle.java | 2 +- .../main/java/com/grey/mailismus/IPlist.java | 2 +- .../java/com/grey/mailismus/MailismusApp.java | 5 +- .../main/java/com/grey/mailismus/Task.java | 4 +- .../directory/files/CachedFiles.java | 2 +- .../grey/mailismus/imap/server/IMAP4Task.java | 2 +- .../mailismus/imap/server/SharedFields.java | 9 +- .../mailismus/ms/maildir/MaildirStore.java | 6 +- .../grey/mailismus/mta/deliver/Client.java | 4 +- .../mta/deliver/ClientConfiguration.java | 4 +- .../grey/mailismus/mta/deliver/Forwarder.java | 2 +- .../com/grey/mailismus/mta/queue/Spooler.java | 2 +- .../filesystem/FilesysQueue.java | 2 +- .../filesystem_cluster/ClusteredQueue.java | 2 +- .../mailismus/mta/reporting/ReportsTask.java | 2 +- .../com/grey/mailismus/mta/submit/Server.java | 4 +- .../grey/mailismus/mta/submit/SubmitTask.java | 2 +- .../mailismus/pop3/client/DownloadClient.java | 8 +- .../mailismus/pop3/server/POP3Server.java | 2 +- .../grey/mailismus/pop3/server/POP3Task.java | 2 +- .../java/com/grey/mailismus/IPlistTest.java | 30 +++--- .../java/com/grey/mailismus/TestSupport.java | 20 ++-- .../directory/files/FilesDirectoryTest.java | 10 +- .../imap/server/IMAP4ServerTest.java | 16 ++-- .../ms/maildir/MaildirStoreTest.java | 11 ++- .../mailismus/mta/deliver/ClientTest.java | 84 ++++++++++++++++ .../mailismus/mta/deliver/ForwarderTest.java | 18 ++-- .../mailismus/mta/queue/BaseManagerTest.java | 7 +- .../grey/mailismus/mta/queue/ManagerTest.java | 12 +-- .../grey/mailismus/mta/queue/SpoolerTest.java | 4 +- .../ClusterControllerTest.java | 2 +- .../grey/mailismus/mta/smtp/DeliveryTest.java | 15 +-- .../mailismus/mta/submit/GreylistTest.java | 14 +-- .../{smtp => testsupport}/MockServerDNS.java | 8 +- .../mta/testsupport/MockSmtpServer.java | 95 +++++++++++++++++++ .../com/grey/mailismus/pop3/POP3Test.java | 15 ++- 37 files changed, 312 insertions(+), 119 deletions(-) create mode 100644 server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java rename server/src/test/java/com/grey/mailismus/mta/{smtp => testsupport}/MockServerDNS.java (92%) create mode 100644 server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java diff --git a/server/src/main/java/com/grey/mailismus/AppConfig.java b/server/src/main/java/com/grey/mailismus/AppConfig.java index 9f9b586..7babb2e 100755 --- a/server/src/main/java/com/grey/mailismus/AppConfig.java +++ b/server/src/main/java/com/grey/mailismus/AppConfig.java @@ -56,7 +56,7 @@ private AppConfig(String cfgpath, Dispatcher dsptch) throws java.io.IOException announceHost = getAnnounceHost(cfg, hostName); XmlConfig dbcfg = cfg.getSection("database"+XmlConfig.XPATH_ENABLED); - dbType = (dbcfg.exists() ? new DBHandle.Type(dbcfg, dsptch.getApplicationContext().getConfig(), dsptch.getLogger()) : null); + dbType = (dbcfg.exists() ? new DBHandle.Type(dbcfg, dsptch.getApplicationContext().getNafConfig(), dsptch.getLogger()) : null); } public XmlConfig getConfigQueue(String name) diff --git a/server/src/main/java/com/grey/mailismus/DBHandle.java b/server/src/main/java/com/grey/mailismus/DBHandle.java index 08a7291..5826963 100644 --- a/server/src/main/java/com/grey/mailismus/DBHandle.java +++ b/server/src/main/java/com/grey/mailismus/DBHandle.java @@ -160,7 +160,7 @@ public DBHandle(String name, Type dbtype, ApplicationContextNAF appctx, XmlConfi this.name = name; this.dbtype = dbtype; this.log = log; - nafcfg = appctx.getConfig(); + nafcfg = appctx.getNafConfig(); connurl_base = dbtype.connurl.replace(TOKEN_CONNSTR_NAME, name); liveconns = appctx.getNamedItem(getClass().getName()+"-liveconns", () -> new HashedSet<>()); diff --git a/server/src/main/java/com/grey/mailismus/IPlist.java b/server/src/main/java/com/grey/mailismus/IPlist.java index 6f1b539..aea8734 100644 --- a/server/src/main/java/com/grey/mailismus/IPlist.java +++ b/server/src/main/java/com/grey/mailismus/IPlist.java @@ -102,7 +102,7 @@ private IPlist(String name, DBHandle.Type dbtype, XmlConfig cfg, ApplicationCont listname = name; log = logger; - srcpath = appctx.getConfig().getURL(cfg, "sourcefile", null, true, null, getClass()); + srcpath = appctx.getNafConfig().getURL(cfg, "sourcefile", null, true, null, getClass()); memlimit = cfg.getInt("mem_threshold", false, 0); loadfactor = cfg.getInt("hashfactor", false, 10); allow_hostnames = cfg.getBool("hostnames", true); diff --git a/server/src/main/java/com/grey/mailismus/MailismusApp.java b/server/src/main/java/com/grey/mailismus/MailismusApp.java index e1ffc49..c371cfc 100644 --- a/server/src/main/java/com/grey/mailismus/MailismusApp.java +++ b/server/src/main/java/com/grey/mailismus/MailismusApp.java @@ -9,7 +9,6 @@ import com.grey.naf.ApplicationContextNAF; import com.grey.naf.Launcher; import com.grey.mailismus.directory.DirectoryImpl; -import com.grey.logging.Logger; public class MailismusApp extends Launcher @@ -30,13 +29,13 @@ public MailismusApp(String[] args) { } @Override - protected void appExecute(ApplicationContextNAF appctx, int param1, Logger bootlog) throws Exception { + protected void appExecute(ApplicationContextNAF appctx, int param1) throws Exception { if (options.plaintxt != null) { ByteChars plain = new ByteChars(options.plaintxt); char[] digest = DirectoryImpl.passwordHash(plain); System.out.println("Hashed to ["+new String(digest)+"]"); } else { - super.appExecute(appctx, param1, bootlog); + super.appExecute(appctx, param1); } } diff --git a/server/src/main/java/com/grey/mailismus/Task.java b/server/src/main/java/com/grey/mailismus/Task.java index da0fa37..6d78bdb 100644 --- a/server/src/main/java/com/grey/mailismus/Task.java +++ b/server/src/main/java/com/grey/mailismus/Task.java @@ -61,7 +61,7 @@ public Task(String name, Dispatcher d, XmlConfig cfg, DirectoryFactory df, Messa super(name, d, cfg); Logger logger = d.getLogger(); dnsResolver = (dns == null ? null: dns); - NAFConfig nafcfg = d.getApplicationContext().getConfig(); + NAFConfig nafcfg = d.getApplicationContext().getNafConfig(); String cfgfile = nafcfg.getPath(cfg, "configfile", null, false, null, null); if (cfgfile != null) { @@ -135,7 +135,7 @@ public CharSequence handleNAFManCommand(NafManCommand cmd) throws java.io.IOExce public static ResolverDNS createResolverDNS(Dispatcher d) throws UnknownHostException { ResolverConfig rcfg = new ResolverConfig.Builder() - .withXmlConfig(d.getApplicationContext().getConfig().getNode("dnsresolver")) + .withXmlConfig(d.getApplicationContext().getNafConfig().getNode("dnsresolver")) .build(); return ResolverDNS.create(d, rcfg); } diff --git a/server/src/main/java/com/grey/mailismus/directory/files/CachedFiles.java b/server/src/main/java/com/grey/mailismus/directory/files/CachedFiles.java index 82927c0..3a78920 100644 --- a/server/src/main/java/com/grey/mailismus/directory/files/CachedFiles.java +++ b/server/src/main/java/com/grey/mailismus/directory/files/CachedFiles.java @@ -44,7 +44,7 @@ public static synchronized CachedFiles get(com.grey.naf.reactor.Dispatcher d, co private CachedFiles(com.grey.naf.reactor.Dispatcher d, com.grey.base.config.XmlConfig cfg) throws java.io.IOException { dsptch = d; - NAFConfig nafcfg = dsptch.getApplicationContext().getConfig(); + NAFConfig nafcfg = dsptch.getApplicationContext().getNafConfig(); String pthnam = nafcfg.getPath(cfg, "users", null, false, null, getClass()); fh_users = (pthnam == null ? null : new java.io.File(pthnam)); pthnam = nafcfg.getPath(cfg, "domains", null, false, null, getClass()); diff --git a/server/src/main/java/com/grey/mailismus/imap/server/IMAP4Task.java b/server/src/main/java/com/grey/mailismus/imap/server/IMAP4Task.java index f92af3a..6e8ed0f 100644 --- a/server/src/main/java/com/grey/mailismus/imap/server/IMAP4Task.java +++ b/server/src/main/java/com/grey/mailismus/imap/server/IMAP4Task.java @@ -20,7 +20,7 @@ public class IMAP4Task public IMAP4Task(String name, Dispatcher d, XmlConfig cfg) throws java.io.IOException { super(name, d, cfg, DFLT_FACT_DTORY, DFLT_FACT_MS, null); String grpname = "IMAP4Task="+getName(); - ConcurrentListenerConfig[] lcfg = ConcurrentListenerConfig.buildMultiConfig(grpname, d.getApplicationContext().getConfig(), "listeners/listener", taskConfig(), + ConcurrentListenerConfig[] lcfg = ConcurrentListenerConfig.buildMultiConfig(grpname, d.getApplicationContext().getNafConfig(), "listeners/listener", taskConfig(), IMAP4Protocol.TCP_PORT, IMAP4Protocol.TCP_SSLPORT, IMAP4Server.Factory.class, null); listeners = new ListenerSet(grpname, getDispatcher(), this, this, lcfg); if (listeners.configured() != 0) registerDirectoryOps(com.grey.mailismus.nafman.Loader.PREF_DTORY_IMAP4S); diff --git a/server/src/main/java/com/grey/mailismus/imap/server/SharedFields.java b/server/src/main/java/com/grey/mailismus/imap/server/SharedFields.java index 7cdf130..a1b533d 100644 --- a/server/src/main/java/com/grey/mailismus/imap/server/SharedFields.java +++ b/server/src/main/java/com/grey/mailismus/imap/server/SharedFields.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2018 Yusef Badri - All rights reserved. + * Copyright 2013-2024 Yusef Badri - All rights reserved. * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). */ package com.grey.mailismus.imap.server; @@ -9,6 +9,7 @@ import com.grey.mailismus.imap.IMAP4Protocol; import com.grey.mailismus.imap.server.Defs.EnvelopeHeader; import com.grey.mailismus.imap.server.Defs.FetchOpDef; +import com.grey.naf.BufferGenerator; import com.grey.mailismus.imap.server.Defs.FetchOp; /* @@ -114,7 +115,7 @@ public SharedFields(com.grey.base.config.XmlConfig cfg, com.grey.naf.reactor.Dis capa_idle = cfg.getBool("capa_idle", true); delay_chanclose = cfg.getTime("delay_close", 0); maximapbuf = cfg.getInt("maxtransmitbuf", true, 8 * 1024); - bufspec = new com.grey.naf.BufferGenerator(cfg, "niobuffers", 16 * 1024, 16 * 1024); + bufspec = BufferGenerator.create(cfg, "niobuffers", 16 * 1024, 16 * 1024); transcript = com.grey.mailismus.Transcript.create(dsptch, cfg, "transcript"); full_transcript = cfg.getBool("transcript/@full", false); @@ -131,7 +132,7 @@ public SharedFields(com.grey.base.config.XmlConfig cfg, com.grey.naf.reactor.Dis long timeval = cfg.getTime("newmailfreq", TimeOps.parseMilliTime("20s")); interval_newmail = Math.max(timeval, minval); - String pthnam_msgflags = cfg.getValue("keywords_map", true, dsptch.getApplicationContext().getConfig().getPathVar()+"/imap/imapkeywords"); + String pthnam_msgflags = cfg.getValue("keywords_map", true, dsptch.getApplicationContext().getNafConfig().getPathVar()+"/imap/imapkeywords"); boolean dynkwords = cfg.getBool("keywords_dyn", true); msgFlags = new MessageFlags(pthnam_msgflags, dynkwords); @@ -211,7 +212,7 @@ public SharedFields(com.grey.base.config.XmlConfig cfg, com.grey.naf.reactor.Dis imap4rsp_bye_timeout = com.grey.mailismus.Task.constBuffer(IMAP4Protocol.STATUS_UNTAGGED+IMAP4Protocol.STATUS_BYE+" idle timeout"+IMAP4Protocol.EOL); imap4rsp_contd_ready = com.grey.mailismus.Task.constBuffer(IMAP4Protocol.STATUS_CONTD+"Ready"+IMAP4Protocol.EOL); - String pthnam = dsptch.getApplicationContext().getConfig().getPathTemp()+"/imap4server"; + String pthnam = dsptch.getApplicationContext().getNafConfig().getPathTemp()+"/imap4server"; stagingDir = new java.io.File(pthnam); } diff --git a/server/src/main/java/com/grey/mailismus/ms/maildir/MaildirStore.java b/server/src/main/java/com/grey/mailismus/ms/maildir/MaildirStore.java index 2b92bb8..b8d4d2f 100644 --- a/server/src/main/java/com/grey/mailismus/ms/maildir/MaildirStore.java +++ b/server/src/main/java/com/grey/mailismus/ms/maildir/MaildirStore.java @@ -82,7 +82,7 @@ public MaildirStore(com.grey.naf.reactor.Dispatcher d, com.grey.base.config.XmlC char dflt_colon = (SysProps.isWindows ? '+' : ':'); String hostname = java.net.InetAddress.getLocalHost().getCanonicalHostName(); - path_users = dsptch.getApplicationContext().getConfig().getPath(cfg, "userpath", null, true, null, getClass()); + path_users = dsptch.getApplicationContext().getNafConfig().getPath(cfg, "userpath", null, true, null, getClass()); path_maildir = cfg.getValue("mailpath", true, "Maildir"); dotstuffing = cfg.getBool("dotstuffing", false); mailismus_delivery = cfg.getBool("exclusive", false); @@ -108,8 +108,8 @@ public MaildirStore(com.grey.naf.reactor.Dispatcher d, com.grey.base.config.XmlC if (!virtual_users) dsptch.getLogger().info("MS-Maildir: chmod tree ["+chmod_tree+"] - msgfile="+chmod_msgfile); //make sure the Maildir suffix chars are acceptable for this platform - FileOps.ensureDirExists(d.getApplicationContext().getConfig().getPathTemp()); - java.io.File fh1 = new java.io.File(d.getApplicationContext().getConfig().getPathTemp()+"/ms_"+Thread.currentThread().getId()+".test"+FLAGS_MARKER+"x"); + FileOps.ensureDirExists(d.getApplicationContext().getNafConfig().getPathTemp()); + java.io.File fh1 = new java.io.File(d.getApplicationContext().getNafConfig().getPathTemp()+"/ms_"+Thread.currentThread().getId()+".test"+FLAGS_MARKER+"x"); java.io.File fh2 = new java.io.File(fh1.getParentFile(), fh1.getName()+"y"); FileOps.deleteFile(fh1); FileOps.deleteFile(fh2); diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java index 958d8dc..903bb61 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java @@ -122,7 +122,7 @@ private enum PROTO_ACTION {A_CONNECT, A_DISCONNECT, A_HELO, A_EHLO, A_MAILFROM, this.shared = shared; } - public void startConnection(SmtpMessage msg, SmtpSender sender, Relay relay) throws IOException { + public void startConnection(SmtpMessage msg, SmtpSender sender, SmtpRelay relay) throws IOException { initConnection(); smtpMessage = msg; smtpSender = sender; @@ -521,7 +521,7 @@ private PROTO_STATE eventRaised(PROTO_EVENT evt, ByteArrayRef rspdata, CharSeque case E_REPLY: // Note that if we act on the start of a reply before we've seen the end of it, there is an infinitesmal chance that we will send // our next request prematurely, and look like a slammer. I say infinitesmal because it's almost certain that the entire reply is - // already in transit even if we've only seen part of it so far, but setting a larger rcvbufsiz would mean this never happens anyway. + // already in transit even if we've only seen part of it so far, but setting a larger recvBufSiz would mean this never happens anyway. // The largest reply I've seen so far is Hotmail's 308-byte greeting. boolean discardThis = isFlagSet(S2_DISCARD); //discard this chunk of data? setFlag(S2_DISCARD, rspdata.byteAt(rspdata.size() - 1) != '\n'); //discard next received chunk? diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/ClientConfiguration.java b/server/src/main/java/com/grey/mailismus/mta/deliver/ClientConfiguration.java index 16c8304..f490180 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/ClientConfiguration.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/ClientConfiguration.java @@ -89,7 +89,7 @@ private static ConnectionConfig createConnectionConfig(XmlConfig xmlcfg, if (sslcfg != null && sslcfg.exists()) { anonssl = new SSLConfig.Builder() .withIsClient(true) - .withXmlConfig(sslcfg, dsptch.getApplicationContext().getConfig()) + .withXmlConfig(sslcfg, dsptch.getApplicationContext().getNafConfig()) .build(); } @@ -114,7 +114,7 @@ private static ConnectionConfig createConnectionConfig(XmlConfig xmlcfg, } private static BufferGenerator createBufferGenerator(XmlConfig xmlcfg) { - BufferGenerator bufferGenerator = new BufferGenerator(xmlcfg, "niobuffers", 256, 128); + BufferGenerator bufferGenerator = BufferGenerator.create(xmlcfg, "niobuffers", 256, 128); if (bufferGenerator.rcvbufsiz < 40) throw new MailismusConfigException(LOG_PREFIX+": rcvbuf is too small - "+bufferGenerator); return bufferGenerator; } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java index 54d54d6..15f1f34 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java @@ -168,7 +168,7 @@ public Forwarder(Dispatcher d, XmlConfig cfg, AppConfig appConfig, Logger log = dsptch.getLogger(); audit = Audit.create("MTA-Delivery", "audit", dsptch, cfg); XmlConfig relaycfg = cfg.getSection("relays"); - routing = new Routing(relaycfg, dsptch.getApplicationContext().getConfig(), log); + routing = new Routing(relaycfg, dsptch.getApplicationContext().getNafConfig(), log); int cap_qcache = 2500; int _max_simulconns = cap_qcache; diff --git a/server/src/main/java/com/grey/mailismus/mta/queue/Spooler.java b/server/src/main/java/com/grey/mailismus/mta/queue/Spooler.java index 4cadfb7..5390454 100644 --- a/server/src/main/java/com/grey/mailismus/mta/queue/Spooler.java +++ b/server/src/main/java/com/grey/mailismus/mta/queue/Spooler.java @@ -69,7 +69,7 @@ public interface SPID_Counter poolSubmitHandles = new ObjectWell(SubmitHandle.class, "Spooler-"+name); spidbuilder.setLength(EXTSPIDSIZE); - NAFConfig nafcfg = appctx.getConfig(); + NAFConfig nafcfg = appctx.getNafConfig(); String rootpath = nafcfg.getPath(cfg, "rootpath", null, false, nafcfg.getPathVar()+"/spool", null); int loadfactor = cfg.getInt("silofactor", false, 5); bufsiz_submit = (int)cfg.getSize("bufsize", "16K"); diff --git a/server/src/main/java/com/grey/mailismus/mta/queue/queue_providers/filesystem/FilesysQueue.java b/server/src/main/java/com/grey/mailismus/mta/queue/queue_providers/filesystem/FilesysQueue.java index 23382fc..29d5d73 100644 --- a/server/src/main/java/com/grey/mailismus/mta/queue/queue_providers/filesystem/FilesysQueue.java +++ b/server/src/main/java/com/grey/mailismus/mta/queue/queue_providers/filesystem/FilesysQueue.java @@ -59,7 +59,7 @@ public FilesysQueue(com.grey.naf.reactor.Dispatcher d, com.grey.base.config.XmlC throws java.io.IOException { super(d, qcfg, name); - NAFConfig nafcfg = dsptch.getApplicationContext().getConfig(); + NAFConfig nafcfg = dsptch.getApplicationContext().getNafConfig(); String queuepath = nafcfg.getPath(qcfg, "rootpath", null, false, nafcfg.getPathVar()+"/queue", null); maxDeferredIgnore = qcfg.getTime("deferred_maxignore", "5m"); maxFilesList = qcfg.getInt("maxfileslist", false, 0); diff --git a/server/src/main/java/com/grey/mailismus/mta/queue/queue_providers/filesystem_cluster/ClusteredQueue.java b/server/src/main/java/com/grey/mailismus/mta/queue/queue_providers/filesystem_cluster/ClusteredQueue.java index 1dd4c48..3715521 100644 --- a/server/src/main/java/com/grey/mailismus/mta/queue/queue_providers/filesystem_cluster/ClusteredQueue.java +++ b/server/src/main/java/com/grey/mailismus/mta/queue/queue_providers/filesystem_cluster/ClusteredQueue.java @@ -68,7 +68,7 @@ public final class ClusteredQueue public ClusteredQueue(Dispatcher d, XmlConfig qcfg, AppConfig appcfg, String name) throws java.io.IOException { super(d, qcfg, name); - NAFConfig nafcfg = dsptch.getApplicationContext().getConfig(); + NAFConfig nafcfg = dsptch.getApplicationContext().getNafConfig(); String queuepath = nafcfg.getPath(qcfg, "rootpath", null, false, nafcfg.getPathVar()+"/queue", null); maxClusterSize = (int)qcfg.getSize("maxclustersize", "256K"); maxDeferredIgnore = qcfg.getTime("deferred_maxignore", "5m"); diff --git a/server/src/main/java/com/grey/mailismus/mta/reporting/ReportsTask.java b/server/src/main/java/com/grey/mailismus/mta/reporting/ReportsTask.java index 4be9610..f10de67 100644 --- a/server/src/main/java/com/grey/mailismus/mta/reporting/ReportsTask.java +++ b/server/src/main/java/com/grey/mailismus/mta/reporting/ReportsTask.java @@ -121,7 +121,7 @@ public ReportsTask(String name, com.grey.naf.reactor.Dispatcher d, com.grey.base interval_housekeep = taskcfg.getTime("interval_housekeeping", TimeOps.parseMilliTime("24h")); delay_start = taskcfg.getTime("delay_start", TimeOps.parseMilliTime("20s")); statuszero = (short)taskcfg.getInt("statuszero", false, 0); - archiveDirectory = getDispatcher().getApplicationContext().getConfig().getPath(taskcfg, "ndr_copies_folder", null, false, null, null); + archiveDirectory = getDispatcher().getApplicationContext().getNafConfig().getPath(taskcfg, "ndr_copies_folder", null, false, null, null); String ndr_recip = taskcfg.getValue("ndr_recip_redirect", false, null); String[] ndr_bcc = taskcfg.getTuple("ndr_recips_additional", "|", false, null); diff --git a/server/src/main/java/com/grey/mailismus/mta/submit/Server.java b/server/src/main/java/com/grey/mailismus/mta/submit/Server.java index 9e2c72d..1f5ce08 100644 --- a/server/src/main/java/com/grey/mailismus/mta/submit/Server.java +++ b/server/src/main/java/com/grey/mailismus/mta/submit/Server.java @@ -409,7 +409,7 @@ public SharedFields(XmlConfig cfg, Dispatcher dsptch, MTA_Task task, spf_sender_rewrite = cfg.getBool("spf_sender_rewrite", true); stats_start = dsptch.getRealTime(); discard_msgid = stats_start; - netbufs = new com.grey.naf.BufferGenerator(cfg, "niobuffers", 4*1024, 128); + netbufs = com.grey.naf.BufferGenerator.create(cfg, "niobuffers", 4*1024, 128); if (netbufs.rcvbufsiz < 80) throw new MailismusConfigException(logpfx+"recvbuf="+netbufs.rcvbufsiz+" is too small"); // read the per-connection config @@ -431,7 +431,7 @@ public SharedFields(XmlConfig cfg, Dispatcher dsptch, MTA_Task task, // the existence of a relay as proof that a domain is known a priori to be legit, and shouldn't be subjected to // DNS-Validation or relay-restriction checks. // Doesn't matter if a relay domain duplicates a local one, as recipient-validation tests if local domain first. - routing = new Routing(appConfig.getConfigRelays(), dsptch.getApplicationContext().getConfig(), null); + routing = new Routing(appConfig.getConfigRelays(), dsptch.getApplicationContext().getNafConfig(), null); // Set up blacklisting, if configured String xpath = "blacklist"+XmlConfig.XPATH_ENABLED; diff --git a/server/src/main/java/com/grey/mailismus/mta/submit/SubmitTask.java b/server/src/main/java/com/grey/mailismus/mta/submit/SubmitTask.java index 89883b2..915dc96 100644 --- a/server/src/main/java/com/grey/mailismus/mta/submit/SubmitTask.java +++ b/server/src/main/java/com/grey/mailismus/mta/submit/SubmitTask.java @@ -19,7 +19,7 @@ public final class SubmitTask public SubmitTask(String name, Dispatcher dsptch, XmlConfig cfg) throws java.io.IOException { super(name, dsptch, cfg, DFLT_FACT_DTORY, null, DFLT_FACT_QUEUE, createResolverDNS(dsptch)); String grpname = "SubmitTask="+getName(); - ConcurrentListenerConfig[] lcfg = ConcurrentListenerConfig.buildMultiConfig(grpname, dsptch.getApplicationContext().getConfig(), "listeners/listener", taskConfig(), + ConcurrentListenerConfig[] lcfg = ConcurrentListenerConfig.buildMultiConfig(grpname, dsptch.getApplicationContext().getNafConfig(), "listeners/listener", taskConfig(), com.grey.mailismus.mta.Protocol.TCP_PORT, com.grey.mailismus.mta.Protocol.TCP_SSLPORT, Server.Factory.class, null); listeners = new ListenerSet(grpname, dsptch, this, this, lcfg); if (listeners.configured() != 0) registerQueueOps(com.grey.mailismus.nafman.Loader.PREF_SHOWQ_SUBMIT); diff --git a/server/src/main/java/com/grey/mailismus/pop3/client/DownloadClient.java b/server/src/main/java/com/grey/mailismus/pop3/client/DownloadClient.java index a1d23ce..ae28a8b 100644 --- a/server/src/main/java/com/grey/mailismus/pop3/client/DownloadClient.java +++ b/server/src/main/java/com/grey/mailismus/pop3/client/DownloadClient.java @@ -94,7 +94,7 @@ public Common(XmlConfig cfg, com.grey.naf.reactor.Dispatcher dsptch, com.grey.ma appConfig = task.getAppConfig(); tmtprotocol = cfg.getTime("timeout", com.grey.base.utils.TimeOps.parseMilliTime("4m")); delay_chanclose = cfg.getTime("delay_close", 0); - bufspec = new com.grey.naf.BufferGenerator(cfg, "niobuffers", 4*1024, 128); //always line-buffered + bufspec = com.grey.naf.BufferGenerator.create(cfg, "niobuffers", 4*1024, 128); //always line-buffered audit = com.grey.mailismus.Audit.create("POP3-Download", "audit", dsptch, cfg); transcript = com.grey.mailismus.Transcript.create(dsptch, cfg, "transcript"); ms = task.getMS(); @@ -110,7 +110,7 @@ public Common(XmlConfig cfg, com.grey.naf.reactor.Dispatcher dsptch, com.grey.ma reqbuf_stls = com.grey.mailismus.Task.constBuffer(POP3Protocol.CMDREQ_STLS+POP3Protocol.EOL); reqbuf_capa = com.grey.mailismus.Task.constBuffer(POP3Protocol.CMDREQ_CAPA+POP3Protocol.EOL); - String pthnam = dsptch.getApplicationContext().getConfig().getPathTemp()+"/pop3/downloadclient"; + String pthnam = dsptch.getApplicationContext().getNafConfig().getPathTemp()+"/pop3/downloadclient"; stagingDir = new java.io.File(pthnam); String pfx = "POP3-Clients: "; @@ -204,7 +204,7 @@ public DownloadClient(com.grey.naf.reactor.Dispatcher d, DownloadClient.Common c if (cfg.getBool("@omitreceivedheader", false)) fcfg |= CFG_OMITRCVHDR; if (cfg.getBool("@fulltrans", false)) fcfg |= CFG_FULLTRANS; - String pthnam = d.getApplicationContext().getConfig().getPath(cfg, "downloads_directory", null, false, null, null); + String pthnam = d.getApplicationContext().getNafConfig().getPath(cfg, "downloads_directory", null, false, null, null); if (pthnam != null) pthnam = pthnam.replace("%U%", localdest); dh_download = (pthnam == null ? null : new java.io.File(pthnam)); @@ -221,7 +221,7 @@ public DownloadClient(com.grey.naf.reactor.Dispatcher d, DownloadClient.Common c sslconfig = new SSLConfig.Builder() .withPeerCertName(srvname) .withIsClient(true) - .withXmlConfig(sslcfg, getDispatcher().getApplicationContext().getConfig()) + .withXmlConfig(sslcfg, getDispatcher().getApplicationContext().getNafConfig()) .build(); } // and finalise the server address, which depends on our SSL mode diff --git a/server/src/main/java/com/grey/mailismus/pop3/server/POP3Server.java b/server/src/main/java/com/grey/mailismus/pop3/server/POP3Server.java index c5fafcb..bcb8158 100644 --- a/server/src/main/java/com/grey/mailismus/pop3/server/POP3Server.java +++ b/server/src/main/java/com/grey/mailismus/pop3/server/POP3Server.java @@ -136,7 +136,7 @@ public SharedFields(com.grey.base.config.XmlConfig cfg, com.grey.naf.reactor.Dis expire = cfg.getInt("expire", true, -1); tmtprotocol = cfg.getTime("timeout", com.grey.base.utils.TimeOps.parseMilliTime("2m")); //NB: RFC-1939 says at least 10 mins delay_chanclose = cfg.getTime("delay_close", 0); - bufspec = new com.grey.naf.BufferGenerator(cfg, "niobuffers", 256, 128); + bufspec = com.grey.naf.BufferGenerator.create(cfg, "niobuffers", 256, 128); transcript = com.grey.mailismus.Transcript.create(dsptch, cfg, "transcript"); authtypes_enabled = configureAuthTypes("authtypes", cfg, true, POP3Protocol.AUTHTYPE.values(), null, logpfx); diff --git a/server/src/main/java/com/grey/mailismus/pop3/server/POP3Task.java b/server/src/main/java/com/grey/mailismus/pop3/server/POP3Task.java index 2a8db43..1925137 100644 --- a/server/src/main/java/com/grey/mailismus/pop3/server/POP3Task.java +++ b/server/src/main/java/com/grey/mailismus/pop3/server/POP3Task.java @@ -20,7 +20,7 @@ public final class POP3Task public POP3Task(String name, Dispatcher d, XmlConfig cfg) throws java.io.IOException { super(name, d, cfg, DFLT_FACT_DTORY, DFLT_FACT_MS, null); String grpname = "POP3Task="+getName(); - ConcurrentListenerConfig[] lcfg = ConcurrentListenerConfig.buildMultiConfig(grpname, d.getApplicationContext().getConfig(), "listeners/listener", taskConfig(), + ConcurrentListenerConfig[] lcfg = ConcurrentListenerConfig.buildMultiConfig(grpname, d.getApplicationContext().getNafConfig(), "listeners/listener", taskConfig(), POP3Protocol.TCP_PORT, POP3Protocol.TCP_SSLPORT, POP3Server.Factory.class, null); listeners = new ListenerSet(grpname, getDispatcher(), this, this, lcfg); if (listeners.configured() != 0) registerDirectoryOps(com.grey.mailismus.nafman.Loader.PREF_DTORY_POP3S); diff --git a/server/src/test/java/com/grey/mailismus/IPlistTest.java b/server/src/test/java/com/grey/mailismus/IPlistTest.java index 3081faa..e632fea 100644 --- a/server/src/test/java/com/grey/mailismus/IPlistTest.java +++ b/server/src/test/java/com/grey/mailismus/IPlistTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2011-2021 Yusef Badri - All rights reserved. + * Copyright 2011-2024 Yusef Badri - All rights reserved. * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). */ package com.grey.mailismus; @@ -48,8 +48,8 @@ public class IPlistTest public void testMemorySync() throws java.io.IOException, java.net.URISyntaxException { String cfgpath = TestSupport.getResourcePath("/mtanaf.xml", getClass()); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true); - XmlConfig cfg = appctx.getConfig().getNode("iplist"); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true, logger); + XmlConfig cfg = appctx.getNafConfig().getNode("iplist"); org.junit.Assert.assertNotNull(cfg); IPlist iplist = new IPlist("test_iplist_mem_sync", null, cfg, appctx, logger); org.junit.Assert.assertTrue(iplist.allowHostnames()); @@ -69,13 +69,13 @@ public void testMemorySync() throws java.io.IOException, java.net.URISyntaxExcep public void testMemoryAsync() throws java.io.IOException, java.net.URISyntaxException { String cfgpath = TestSupport.getResourcePath("/mtanaf.xml", getClass()); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true); - XmlConfig dcfg = appctx.getConfig().getDispatcher(dname); - DispatcherConfig def = new DispatcherConfig.Builder().withXmlConfig(dcfg).build(); - Dispatcher dsptch = Dispatcher.create(appctx, def, logger); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true, logger); + XmlConfig dcfg = appctx.getNafConfig().getDispatcherConfigNode(dname); + DispatcherConfig def = DispatcherConfig.builder().withXmlConfig(dcfg).withAppContext(appctx).build(); + Dispatcher dsptch = Dispatcher.create(def); boolean ok = false; try { - XmlConfig cfg = appctx.getConfig().getNode("iplist"); + XmlConfig cfg = appctx.getNafConfig().getNode("iplist"); org.junit.Assert.assertNotNull(cfg); IPlist iplist = new IPlist("test_iplist_mem_async", null, cfg, dsptch); org.junit.Assert.assertTrue(iplist.allowHostnames()); @@ -120,9 +120,9 @@ public void testDBSync() throws java.io.IOException, java.net.URISyntaxException org.junit.Assume.assumeTrue(TestSupport.HAVE_DBDRIVERS); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "/iplist"); String cfgpath = TestSupport.getResourcePath("/mtanaf.xml", getClass()); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true, logger); DBHandle.Type dbtype = setup_dbtype; //null means we will fail, but want to report the failure - if (dbtype == null) dbtype = new DBHandle.Type(cfg, appctx.getConfig(), logger); + if (dbtype == null) dbtype = new DBHandle.Type(cfg, appctx.getNafConfig(), logger); IPlist iplist = new IPlist("test_iplist_db_sync", dbtype, cfg, appctx, logger); org.junit.Assert.assertFalse(iplist.allowHostnames()); commonChecks(iplist, EXPSIZE_DBTEST); @@ -144,14 +144,14 @@ public void testDBAsync() throws java.io.IOException, java.net.URISyntaxExceptio org.junit.Assume.assumeTrue(TestSupport.HAVE_DBDRIVERS); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "/iplist"); String cfgpath = TestSupport.getResourcePath("/mtanaf.xml", getClass()); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true); - XmlConfig dcfg = appctx.getConfig().getDispatcher(dname); - DispatcherConfig def = new DispatcherConfig.Builder().withXmlConfig(dcfg).build(); - Dispatcher dsptch = Dispatcher.create(appctx, def, logger); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true, logger); + XmlConfig dcfg = appctx.getNafConfig().getDispatcherConfigNode(dname); + DispatcherConfig def = DispatcherConfig.builder().withXmlConfig(dcfg).withAppContext(appctx).build(); + Dispatcher dsptch = Dispatcher.create(def); boolean ok = false; try { DBHandle.Type dbtype = setup_dbtype; //null means we will fail, but want to report the failure - if (dbtype == null) dbtype = new DBHandle.Type(cfg, appctx.getConfig(), logger); + if (dbtype == null) dbtype = new DBHandle.Type(cfg, appctx.getNafConfig(), logger); IPlist iplist = new IPlist("test_iplist_db", dbtype, cfg, dsptch); org.junit.Assert.assertFalse(iplist.allowHostnames()); commonChecks(iplist, EXPSIZE_DBTEST); diff --git a/server/src/test/java/com/grey/mailismus/TestSupport.java b/server/src/test/java/com/grey/mailismus/TestSupport.java index 15ae3da..fd60e8e 100644 --- a/server/src/test/java/com/grey/mailismus/TestSupport.java +++ b/server/src/test/java/com/grey/mailismus/TestSupport.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2021 Yusef Badri - All rights reserved. + * Copyright 2012-2024 Yusef Badri - All rights reserved. * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). */ package com.grey.mailismus; @@ -7,6 +7,7 @@ import com.grey.base.config.SysProps; import com.grey.base.utils.DynLoader; import com.grey.base.utils.FileOps; +import com.grey.logging.Logger; import com.grey.base.ExceptionUtils; import com.grey.naf.ApplicationContextNAF; import com.grey.naf.NAFConfig; @@ -39,19 +40,24 @@ public static String initPaths(Class clss) { return rootpath; } - public static ApplicationContextNAF createApplicationContext(String name, String cfgpath, boolean withNafman) { + public static ApplicationContextNAF createApplicationContext(String name, String cfgpath, boolean withNafman, Logger bootLogger) { NAFConfig nafcfg = new NAFConfig.Builder().withConfigFile(cfgpath).build(); - return createApplicationContext(name, nafcfg, withNafman); + return createApplicationContext(name, nafcfg, withNafman, bootLogger); } - public static ApplicationContextNAF createApplicationContext(String name, boolean withNafman) { + public static ApplicationContextNAF createApplicationContext(String name, boolean withNafman, Logger bootLogger) { NAFConfig nafcfg = new NAFConfig.Builder().withBasePort(NAFConfig.RSVPORT_ANON).build(); - return createApplicationContext(name, nafcfg, withNafman); + return createApplicationContext(name, nafcfg, withNafman, bootLogger); } - public static ApplicationContextNAF createApplicationContext(String name, NAFConfig nafcfg, boolean withNafman) { + public static ApplicationContextNAF createApplicationContext(String name, NAFConfig nafcfg, boolean withNafman, Logger bootLogger) { NafManConfig nafmanConfig = (withNafman ? new NafManConfig.Builder(nafcfg).build() : null); - return ApplicationContextNAF.create(name, nafcfg, nafmanConfig); + return ApplicationContextNAF.builder() + .withName(name) + .withNafConfig(nafcfg) + .withNafManConfig(nafmanConfig) + .withBootLogger(bootLogger) + .build(); } public static void setTime(Dispatcher d, long millisecs) { diff --git a/server/src/test/java/com/grey/mailismus/directory/files/FilesDirectoryTest.java b/server/src/test/java/com/grey/mailismus/directory/files/FilesDirectoryTest.java index fe9f2f7..f8ac75c 100644 --- a/server/src/test/java/com/grey/mailismus/directory/files/FilesDirectoryTest.java +++ b/server/src/test/java/com/grey/mailismus/directory/files/FilesDirectoryTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2021 Yusef Badri - All rights reserved. + * Copyright 2012-2024 Yusef Badri - All rights reserved. * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). */ package com.grey.mailismus.directory.files; @@ -13,6 +13,7 @@ import com.grey.mailismus.directory.DirectoryImpl; import com.grey.naf.ApplicationContextNAF; import com.grey.naf.NAFConfig; +import com.grey.naf.reactor.config.DispatcherConfig; public class FilesDirectoryTest { @@ -35,7 +36,7 @@ public class FilesDirectoryTest +" role1 : locuser2 : somebody@elsewhere.com\n" +" role2 : role1 : somebody2@elsewhere.com"; - private final ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true); + private final ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true, logger); private com.grey.naf.reactor.Dispatcher dsptch; private String pthnam_users; private String pthnam_aliases; @@ -198,7 +199,8 @@ private com.grey.base.config.XmlConfig setup(boolean hashed) throws java.io.IOEx FileOps.deleteDirectory(dh_work); org.junit.Assert.assertFalse(dh_work.exists()); FileOps.ensureDirExists(dh_work); - dsptch = com.grey.naf.reactor.Dispatcher.create(appctx, new com.grey.naf.reactor.config.DispatcherConfig.Builder().build(), logger); + DispatcherConfig def = DispatcherConfig.builder().withAppContext(appctx).build(); + dsptch = com.grey.naf.reactor.Dispatcher.create(def); String cfgxml = cfgxml_directory; String local_users = local_users_hashed; @@ -206,7 +208,7 @@ private com.grey.base.config.XmlConfig setup(boolean hashed) throws java.io.IOEx cfgxml = cfgxml.replace("xplainpass", "plainpass"); local_users = local_users_plain; } - NAFConfig nafcfg = appctx.getConfig(); + NAFConfig nafcfg = appctx.getNafConfig(); com.grey.base.config.XmlConfig cfg = com.grey.base.config.XmlConfig.makeSection(cfgxml, "directory"); String pthnam = nafcfg.getPath(cfg, "domains", null, true, null, getClass()); java.io.File fh = new java.io.File(pthnam); diff --git a/server/src/test/java/com/grey/mailismus/imap/server/IMAP4ServerTest.java b/server/src/test/java/com/grey/mailismus/imap/server/IMAP4ServerTest.java index 2fa4382..e185ec3 100644 --- a/server/src/test/java/com/grey/mailismus/imap/server/IMAP4ServerTest.java +++ b/server/src/test/java/com/grey/mailismus/imap/server/IMAP4ServerTest.java @@ -13,6 +13,7 @@ import com.grey.naf.ApplicationContextNAF; import com.grey.naf.NAFConfig; import com.grey.naf.reactor.Dispatcher; +import com.grey.naf.reactor.config.DispatcherConfig; import com.grey.mailismus.TestSupport; import com.grey.mailismus.imap.IMAP4Protocol; @@ -34,7 +35,7 @@ public class IMAP4ServerTest private static final String UNTAG_NOCHECK = "_JUNIT_NOASSERT_"; //untagged response is present, but not to be checked - private static final ApplicationContextNAF appctx = TestSupport.createApplicationContext("IMAP4ServerTest", true); + private static final ApplicationContextNAF appctx = TestSupport.createApplicationContext("IMAP4ServerTest", true, null); private Dispatcher dsptch; private IMAP4Task srvtask; private TSAP srvaddr; @@ -314,7 +315,7 @@ private void testMIME() throws java.io.IOException, java.net.URISyntaxException private void testOddMessages() throws java.io.IOException, java.net.URISyntaxException { - String pthnam = dsptch.getApplicationContext().getConfig().getPathTemp()+"/badmsg1"; + String pthnam = dsptch.getApplicationContext().getNafConfig().getPathTemp()+"/badmsg1"; java.io.File fh = new java.io.File(pthnam); int exists_cnt = 1; int recent_cnt = 0; @@ -376,16 +377,15 @@ private void testOddMessages() throws java.io.IOException, java.net.URISyntaxExc private void startServer() throws java.io.IOException { // create a disposable Dispatcher first, just to identify and clean up the working directories that will be used - dsptch = Dispatcher.create(appctx, new com.grey.naf.reactor.config.DispatcherConfig.Builder().build(), com.grey.logging.Factory.getLogger("no-such-logger")); - NAFConfig nafcfg = dsptch.getApplicationContext().getConfig(); + DispatcherConfig def = DispatcherConfig.builder().withAppContext(appctx).build(); + dsptch = Dispatcher.create(def); + NAFConfig nafcfg = dsptch.getApplicationContext().getNafConfig(); FileOps.deleteDirectory(nafcfg.getPathVar()); FileOps.deleteDirectory(nafcfg.getPathTemp()); FileOps.deleteDirectory(nafcfg.getPathLogs()); // now create the real Dispatcher - com.grey.naf.reactor.config.DispatcherConfig def = new com.grey.naf.reactor.config.DispatcherConfig.Builder() - .withSurviveHandlers(false) - .build(); - dsptch = Dispatcher.create(appctx, def, com.grey.logging.Factory.getLogger("no-such-logger")); + def = def.mutate().withSurviveHandlers(false).build(); + dsptch = Dispatcher.create(def); // set up the IMAP server XmlConfig cfg = XmlConfig.makeSection(nafxml_server, "x"); diff --git a/server/src/test/java/com/grey/mailismus/ms/maildir/MaildirStoreTest.java b/server/src/test/java/com/grey/mailismus/ms/maildir/MaildirStoreTest.java index ab928fb..c05d157 100644 --- a/server/src/test/java/com/grey/mailismus/ms/maildir/MaildirStoreTest.java +++ b/server/src/test/java/com/grey/mailismus/ms/maildir/MaildirStoreTest.java @@ -11,12 +11,14 @@ import com.grey.mailismus.ms.MessageStore; import com.grey.mailismus.ms.MessageStoreFactory; import com.grey.naf.ApplicationContextNAF; +import com.grey.naf.reactor.Dispatcher; +import com.grey.naf.reactor.config.DispatcherConfig; public class MaildirStoreTest { private static final String workdir = TestSupport.initPaths(MaildirStoreTest.class)+"/work"; private static final com.grey.logging.Logger logger = com.grey.logging.Factory.getLoggerNoEx(""); - private static final ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true); + private static final ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true, logger); private final String mscfgxml = "" +"" @@ -176,7 +178,8 @@ private com.grey.base.config.XmlConfig setup(boolean withDirectory, boolean disa FileOps.deleteDirectory(dh_work); org.junit.Assert.assertFalse(dh_work.exists()); FileOps.ensureDirExists(dh_work); - dsptch = com.grey.naf.reactor.Dispatcher.create(appctx, new com.grey.naf.reactor.config.DispatcherConfig.Builder().build(), logger); + DispatcherConfig def = DispatcherConfig.builder().withAppContext(appctx).build(); + dsptch = Dispatcher.create(def); String cfgxml = mscfgxml; if (!withDirectory) { @@ -189,11 +192,11 @@ private com.grey.base.config.XmlConfig setup(boolean withDirectory, boolean disa if (!dotstuffed) cfgxml = cfgxml.replace("dotstuffing>", "xdotstuffing>"); com.grey.base.config.XmlConfig cfg = com.grey.base.config.XmlConfig.makeSection(cfgxml, "message_store"); if (!withDirectory) return cfg; - String pthnam = appctx.getConfig().getPath(cfg, "directory/domains", null, true, null, getClass()); + String pthnam = appctx.getNafConfig().getPath(cfg, "directory/domains", null, true, null, getClass()); java.io.File fh = new java.io.File(pthnam); FileOps.ensureDirExists(fh.getParentFile()); FileOps.writeTextFile(fh, local_domains, false); - pthnam = appctx.getConfig().getPath(cfg, "directory/users", null, true, null, getClass()); + pthnam = appctx.getNafConfig().getPath(cfg, "directory/users", null, true, null, getClass()); FileOps.writeTextFile(pthnam, local_users); return cfg; } diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java new file mode 100644 index 0000000..2b81d69 --- /dev/null +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java @@ -0,0 +1,84 @@ +package com.grey.mailismus.mta.deliver; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.net.Socket; + +import com.grey.mailismus.mta.deliver.client.SmtpRelay; +import com.grey.mailismus.mta.testsupport.MockSmtpServer; +import com.grey.naf.BufferGenerator; +import com.grey.naf.reactor.Dispatcher; +import com.grey.naf.reactor.config.DispatcherConfig; +import com.grey.base.utils.TSAP; +import com.grey.base.utils.TimeOps; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/*xxxx Need to test: HELO, EHLO, EHLO rejected, SASL, STLS, some/all recips rejected, msg rejected, quit error, connection-loss in mid session + * Also a test with DNS enabled, but interceptor in place to ensure we connect to mock server anyway - how do we verify the DNS lookups? + * Interceptor would be in SharedFields if we bring it back + */ +public class ClientTest { + private MockSmtpServer srvr; + + @Before + public void setup() throws Exception { + srvr = new MockSmtpServer(null); + srvr.start(); + } + + @After + public void teardown() throws Exception { + if (srvr != null) { + srvr.stop(); + Thread.sleep(100); + } + } + + //xxx @Test + public void testSanity() throws Exception { + Socket sock = srvr.connect(); + OutputStream ostrm = sock.getOutputStream(); + PrintWriter sendStream = new PrintWriter(ostrm, true); + InputStream istrm = sock.getInputStream(); + InputStreamReader rdr = new InputStreamReader(istrm); + BufferedReader recvStream = new BufferedReader(rdr); + sendStream.println("Hello I am the client"); + String s = recvStream.readLine(); + System.out.println("Client received ["+s+"]"); + sock.close(); //xxx srvr.closeSession(sock.getPort()); + Thread.sleep(500); + } + + @Test + public void testSuccessfulSend() throws Exception { + BufferGenerator.BufferConfig bufcfg = new BufferGenerator.BufferConfig(256, true, null, null); + BufferGenerator bufgen = new BufferGenerator(bufcfg); + TSAP remote = TSAP.build("127.0.0.1", srvr.getServicePort()); + + DispatcherConfig def = DispatcherConfig.builder() + .withName("test-dispatcher1") + .withSurviveHandlers(false) + .build(); + Dispatcher dsptch = Dispatcher.create(def); + + SharedFields shared = SharedFields.builder() + .withBufferGenerator(bufgen) + .build(); + SmtpRelay relay = SmtpRelay.builder() + .withName("test-relay1") + .withAddress(remote) + .build(); + Client clnt = new Client(shared, dsptch); + + //xxx clnt.startConnection(null, null, relay); + dsptch.start(); + Dispatcher.STOPSTATUS stopsts = dsptch.waitStopped(TimeOps.MSECS_PER_SECOND*10L, true); + System.out.println("xxx client="+clnt+", relay="+relay+", stopsts="+stopsts); + } +} diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java index 0c07c4a..8b1e97b 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java @@ -25,9 +25,10 @@ import com.grey.mailismus.TestSupport; import com.grey.mailismus.mta.Protocol; import com.grey.mailismus.mta.deliver.client.SmtpMessage; +import com.grey.mailismus.mta.deliver.client.SmtpRelay; import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.mta.queue.MessageRecip; -import com.grey.mailismus.mta.smtp.MockServerDNS; +import com.grey.mailismus.mta.testsupport.MockServerDNS; // Note that the older DeliveryTest suite includes a lot of test cases designed to test the Forwarder as well as the // SMTP client and server entities, but it is a blunter interest, and this class allows a more precise exploration of @@ -96,7 +97,7 @@ public class ForwarderTest @org.junit.BeforeClass public static void beforeClass() throws java.io.IOException { - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true, logger); mockserver = new MockServerDNS(appctx); mockserver.start(); } @@ -113,17 +114,18 @@ public void setup() throws java.io.IOException { +"" +"" +""; - com.grey.naf.reactor.config.DispatcherConfig def = new com.grey.naf.reactor.config.DispatcherConfig.Builder() + XmlConfig xmlcfg = XmlConfig.makeSection(nafxml, "/naf"); + NAFConfig nafcfg = new NAFConfig.Builder().withXmlConfig(xmlcfg).build(); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, nafcfg, true, logger); + com.grey.naf.reactor.config.DispatcherConfig def = com.grey.naf.reactor.config.DispatcherConfig.builder() .withName("utest_fwd_"+testname) .withSurviveHandlers(false) + .withAppContext(appctx) .build(); - XmlConfig xmlcfg = XmlConfig.makeSection(nafxml, "/naf"); - NAFConfig nafcfg = new NAFConfig.Builder().withXmlConfig(xmlcfg).build(); ResolverConfig rcfg = new ResolverConfig.Builder() .withXmlConfig(nafcfg.getNode("dnsresolver")) .build(); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, nafcfg, true); - dsptch = Dispatcher.create(appctx, def, logger); + dsptch = Dispatcher.create(def); dsptch.registerEventListener(this); dnsResolver = ResolverDNS.create(dsptch, rcfg); } @@ -461,7 +463,7 @@ public MySender(int id, SenderFactory fact) throws GeneralSecurityException { } @Override - public void startConnection(SmtpMessage msg, SmtpSender sender, Relay relay) { + public void startConnection(SmtpMessage msg, SmtpSender sender, SmtpRelay relay) { msgparams = (QueueBasedMessage)msg; seqno = mgr.sender_cnt++; int[][] exp = null; diff --git a/server/src/test/java/com/grey/mailismus/mta/queue/BaseManagerTest.java b/server/src/test/java/com/grey/mailismus/mta/queue/BaseManagerTest.java index 71af361..576ad13 100644 --- a/server/src/test/java/com/grey/mailismus/mta/queue/BaseManagerTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/queue/BaseManagerTest.java @@ -24,14 +24,15 @@ public class BaseManagerTest { private static final Logger logger = com.grey.logging.Factory.getLoggerNoEx("qmgrbasetest"); private final String testrootpath = TestSupport.initPaths(getClass()); - private final ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true); + private final ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true, logger); private final Dispatcher dsptch; private QueueManager qmgr; public BaseManagerTest() throws Exception { - dsptch = Dispatcher.create(appctx, new DispatcherConfig.Builder().withName("qmgrbasetest").build(), logger); - FileOps.ensureDirExists(appctx.getConfig().getPathTemp()); + DispatcherConfig def = DispatcherConfig.builder().withName("qmgrbasetest").withAppContext(appctx).build(); + dsptch = Dispatcher.create(def); + FileOps.ensureDirExists(appctx.getNafConfig().getPathTemp()); } @org.junit.After diff --git a/server/src/test/java/com/grey/mailismus/mta/queue/ManagerTest.java b/server/src/test/java/com/grey/mailismus/mta/queue/ManagerTest.java index 45e402a..110e0b7 100644 --- a/server/src/test/java/com/grey/mailismus/mta/queue/ManagerTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/queue/ManagerTest.java @@ -69,10 +69,10 @@ public void init() throws java.io.IOException, java.net.URISyntaxException String qcfgxml_extra = getQueueConfig(); if (qcfgxml_extra != null) qcfgxml = qcfgxml.replace("", qcfgxml_extra+""); String dname = "qmgrtest-"+getQueueClass().getName(); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true); - DispatcherConfig def = new DispatcherConfig.Builder().withName(dname).build(); - dsptch = Dispatcher.create(appctx, def, logger); - FileOps.ensureDirExists(dsptch.getApplicationContext().getConfig().getPathTemp()); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true, logger); + DispatcherConfig def = DispatcherConfig.builder().withName(dname).withAppContext(appctx).build(); + dsptch = Dispatcher.create(def); + FileOps.ensureDirExists(dsptch.getApplicationContext().getNafConfig().getPathTemp()); appcfg = createAppConfig(dsptch, hasDatabase()); qmgr = createManager(qcfgxml, "utest"); org.junit.Assert.assertEquals(getQueueClass(), qmgr.getClass()); @@ -694,7 +694,7 @@ private int submitMessage(boolean rollback, java.util.ArrayList re private void verifyExport(int spid, int qid, long expectsize) throws java.io.IOException { - String exportpath = dsptch.getApplicationContext().getConfig().getPathVar()+"/exports"; + String exportpath = dsptch.getApplicationContext().getNafConfig().getPathVar()+"/exports"; java.nio.file.Path fh = qmgr.exportMessage(spid, qid, exportpath); String pthnam = fh.toAbsolutePath().toString(); exportpath = java.nio.file.Paths.get(exportpath).toAbsolutePath().toString(); @@ -755,7 +755,7 @@ private static AppConfig createAppConfig(Dispatcher dsptch, boolean withDB) thro } String dbcfg = (withDB ? "" : ""); String cfgtxt = ""+dbcfg+""; - String cfgfile = dsptch.getApplicationContext().getConfig().getPathTemp()+"/mailismus-conf.xml"; + String cfgfile = dsptch.getApplicationContext().getNafConfig().getPathTemp()+"/mailismus-conf.xml"; FileOps.writeTextFile(cfgfile, cfgtxt); return AppConfig.get(cfgfile, dsptch); } diff --git a/server/src/test/java/com/grey/mailismus/mta/queue/SpoolerTest.java b/server/src/test/java/com/grey/mailismus/mta/queue/SpoolerTest.java index b8bb7e3..250fded 100644 --- a/server/src/test/java/com/grey/mailismus/mta/queue/SpoolerTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/queue/SpoolerTest.java @@ -24,8 +24,8 @@ public class SpoolerTest public SpoolerTest() throws Exception { String cfgpath = TestSupport.getResourcePath("/mtanaf.xml", getClass()); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true); - FileOps.deleteDirectory(appctx.getConfig().getPathVar()); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true, logger); + FileOps.deleteDirectory(appctx.getNafConfig().getPathVar()); spool = new Spooler(appctx, XmlConfig.getSection(cfgpath, "naf"), logger, "utest"); java.nio.file.Path pth = (java.nio.file.Path) DynLoader.getField(spool, "dhroot"); dhroot = pth.toFile(); diff --git a/server/src/test/java/com/grey/mailismus/mta/queue/queue_providers/filesystem_cluster/ClusterControllerTest.java b/server/src/test/java/com/grey/mailismus/mta/queue/queue_providers/filesystem_cluster/ClusterControllerTest.java index 54f31b5..918afc0 100644 --- a/server/src/test/java/com/grey/mailismus/mta/queue/queue_providers/filesystem_cluster/ClusterControllerTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/queue/queue_providers/filesystem_cluster/ClusterControllerTest.java @@ -21,7 +21,7 @@ public class ClusterControllerTest { protected static final com.grey.logging.Logger logger = com.grey.logging.Factory.getLoggerNoEx("cctltest"); private final String rootpath = TestSupport.initPaths(getClass()); - private final ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true); + private final ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, true, logger); private final ClusterController cctl; private final ConcurrentHashMap cmap; private final Clock clock = Clock.systemUTC(); diff --git a/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java b/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java index 22ac1e3..131875e 100644 --- a/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java @@ -41,6 +41,7 @@ import com.grey.mailismus.mta.submit.filter.api.FilterFactory; import com.grey.mailismus.mta.submit.filter.api.FilterResultsHandler; import com.grey.mailismus.mta.submit.filter.api.MessageFilter; +import com.grey.mailismus.mta.testsupport.MockServerDNS; import com.grey.mailismus.mta.deliver.Relay; import com.grey.mailismus.mta.deliver.DeliverTask; import com.grey.mailismus.mta.deliver.Forwarder; @@ -97,7 +98,7 @@ public static void beforeClass() throws java.io.IOException SysProps.setAppEnv("MAILISMUS_TEST_PORT_SMARTHOST", String.valueOf(TSAP.getVacantPort())); System.out.println("DeliverTest App Env = "+SysProps.getAppEnv()); - ApplicationContextNAF appctx = TestSupport.createApplicationContext("DeliveryTest-MockServerDNS", true); + ApplicationContextNAF appctx = TestSupport.createApplicationContext("DeliveryTest-MockServerDNS", true, null); mockserverDNS = new MockServerDNS(appctx); mockserverDNS.start(); } @@ -367,23 +368,23 @@ private void runtest(MessageSpec[] msgs, int server_submitcnt, int server_spoolc XmlConfig xmlcfg = XmlConfig.makeSection(nafxml, "/naf"); NAFConfig nafcfg = new NAFConfig.Builder().withXmlConfig(xmlcfg).build(); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, nafcfg, true); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, nafcfg, true, null); Clock clock = Clock.systemUTC(); // create a disposable Dispatcher first, just to identify and clean up the working directories that will be used - com.grey.logging.Logger logger = com.grey.logging.Factory.getLogger("no-such-logger"); - DispatcherConfig dcfg = new DispatcherConfig.Builder().withName("DeliveryTest-preliminary").build(); - dsptch = Dispatcher.create(appctx, dcfg, logger); + DispatcherConfig dcfg = DispatcherConfig.builder().withName("DeliveryTest-preliminary").withAppContext(appctx).build(); + dsptch = Dispatcher.create(dcfg); FileOps.deleteDirectory(nafcfg.getPathVar()); FileOps.deleteDirectory(nafcfg.getPathTemp()); FileOps.deleteDirectory(nafcfg.getPathLogs()); // now create the real Dispatcher - DispatcherConfig def = new DispatcherConfig.Builder() + DispatcherConfig def = DispatcherConfig.builder() .withName("DeliveryTest") .withSurviveHandlers(false) .withClock(clock) + .withAppContext(appctx) .build(); - dsptch = Dispatcher.create(appctx, def, logger); + dsptch = Dispatcher.create(def); AppConfig appcfg = AppConfig.get(nafcfg.getPath(pthnam_appcfg, null), dsptch); // Inject the messages into the queue for Forwarder to pick up. diff --git a/server/src/test/java/com/grey/mailismus/mta/submit/GreylistTest.java b/server/src/test/java/com/grey/mailismus/mta/submit/GreylistTest.java index cb8e27b..95cedc7 100644 --- a/server/src/test/java/com/grey/mailismus/mta/submit/GreylistTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/submit/GreylistTest.java @@ -46,13 +46,13 @@ public void testSuite() throws java.io.IOException, java.net.URISyntaxException { org.junit.Assume.assumeTrue(TestSupport.HAVE_DBDRIVERS); String cfgpath = TestSupport.getResourcePath("/mtanaf.xml", getClass()); - ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true); - FileOps.deleteDirectory(appctx.getConfig().getPathVar()); - XmlConfig dcfg = appctx.getConfig().getDispatcher("testmtadispatcher1"); - DispatcherConfig def = new DispatcherConfig.Builder().withXmlConfig(dcfg).build(); - Dispatcher dsptch = Dispatcher.create(appctx, def, logger); + ApplicationContextNAF appctx = TestSupport.createApplicationContext(null, cfgpath, true, logger); + FileOps.deleteDirectory(appctx.getNafConfig().getPathVar()); + XmlConfig dcfg = appctx.getNafConfig().getDispatcherConfigNode("testmtadispatcher1"); + DispatcherConfig def = DispatcherConfig.builder().withXmlConfig(dcfg).withAppContext(appctx).build(); + Dispatcher dsptch = Dispatcher.create(def); - String pthnam = dsptch.getApplicationContext().getConfig().getPathVar()+"/"+WHITELIST_FILE; + String pthnam = dsptch.getApplicationContext().getNafConfig().getPathVar()+"/"+WHITELIST_FILE; java.io.File fh = new java.io.File(pthnam); FileOps.writeTextFile(fh, WHITELIST_IP, false); @@ -76,7 +76,7 @@ private int runTests(Dispatcher dsptch, String xml, boolean withWhitelist, int p throws java.io.IOException { XmlConfig cfg = XmlConfig.makeSection(xml, "/greylist"); - final DBHandle.Type dbtype = new DBHandle.Type(cfg, dsptch.getApplicationContext().getConfig(), dsptch.getLogger()); + final DBHandle.Type dbtype = new DBHandle.Type(cfg, dsptch.getApplicationContext().getNafConfig(), dsptch.getLogger()); grylst = new Greylist(dsptch, dbtype, cfg); int cnt = grylst.reset(); org.junit.Assert.assertEquals(prevtotal, cnt); diff --git a/server/src/test/java/com/grey/mailismus/mta/smtp/MockServerDNS.java b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockServerDNS.java similarity index 92% rename from server/src/test/java/com/grey/mailismus/mta/smtp/MockServerDNS.java rename to server/src/test/java/com/grey/mailismus/mta/testsupport/MockServerDNS.java index 7d1785c..ba109e6 100644 --- a/server/src/test/java/com/grey/mailismus/mta/smtp/MockServerDNS.java +++ b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockServerDNS.java @@ -2,7 +2,7 @@ * Copyright 2015-2021 Yusef Badri - All rights reserved. * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). */ -package com.grey.mailismus.mta.smtp; +package com.grey.mailismus.mta.testsupport; import com.grey.base.utils.ByteChars; import com.grey.base.utils.TimeOps; @@ -29,12 +29,12 @@ public class MockServerDNS public MockServerDNS(ApplicationContextNAF appctx) throws java.io.IOException { populateAnswers(); - com.grey.logging.Logger logger = com.grey.logging.Factory.getLogger("no-such-logger"); - com.grey.naf.reactor.config.DispatcherConfig def = new com.grey.naf.reactor.config.DispatcherConfig.Builder() + com.grey.naf.reactor.config.DispatcherConfig def = com.grey.naf.reactor.config.DispatcherConfig.builder() .withName("Mock-DNS-Server") .withSurviveHandlers(false) + .withAppContext(appctx) .build(); - dsptch = Dispatcher.create(appctx, def, logger); //pass in null logger to get logging output + dsptch = Dispatcher.create(def); DnsServerConfig.Builder bldr = new DnsServerConfig.Builder(); bldr.getListenerConfig().withPort(0).withInterface("127.0.0.1"); diff --git a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java new file mode 100644 index 0000000..8e9f5de --- /dev/null +++ b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java @@ -0,0 +1,95 @@ +package com.grey.mailismus.mta.testsupport; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Map; + +public class MockSmtpServer +{ + private final ServerSocket srvrSocket; + private volatile boolean shutdown; + + public MockSmtpServer(Map expectSend) throws IOException { + srvrSocket = new ServerSocket(0, 10, InetAddress.getLoopbackAddress()); + } + + public void start() { + Runnable r = () -> runService(); + Thread t = new Thread(r); + t.start(); + } + + public void stop() throws IOException { + shutdown = true; + srvrSocket.close(); + } + + public int getServicePort() { + return srvrSocket.getLocalPort(); + } + + public Socket connect() throws IOException { + return new Socket(InetAddress.getLoopbackAddress(), srvrSocket.getLocalPort()); + } + + private void runService() { + for (;;) { + try { + handleConnection(); + } catch (Throwable ex) { + if (!shutdown) System.out.println("Mock SMTP server error - "+ex); + break; + } + } + System.out.println("SMTP server terminating - "+srvrSocket); + } + + private void handleConnection() throws IOException { + Socket sock = srvrSocket.accept(); + System.out.println("SMTP session accepted - "+sock); + SmtpSession session = new SmtpSession(sock); + Runnable r = () -> session.serveSession(); + Thread t = new Thread(r); + t.start(); + } + + + private static class SmtpSession { + private final Socket sock; + private final BufferedReader recvStream; + private final PrintWriter sendStream; + + public SmtpSession(Socket sock) throws IOException { + this.sock = sock; + InputStream istrm = sock.getInputStream(); + InputStreamReader rdr = new InputStreamReader(istrm); + recvStream = new BufferedReader(rdr); + OutputStream ostrm = sock.getOutputStream(); + sendStream = new PrintWriter(ostrm, true); + } + + public void serveSession() { + try { + serviceLoop(); + } catch (Throwable ex) { + System.out.println("SMTP session exiting on error - "+ex); + } + System.out.println("SMTP session terminating - "+sock); + } + + private void serviceLoop() throws IOException { + String request; + while ((request = recvStream.readLine()) != null) { + System.out.println("xxx session received ["+request+"]"); + sendStream.println("Hello I am the server"); + } + } + } +} \ No newline at end of file diff --git a/server/src/test/java/com/grey/mailismus/pop3/POP3Test.java b/server/src/test/java/com/grey/mailismus/pop3/POP3Test.java index 37da8bb..9f55e9c 100644 --- a/server/src/test/java/com/grey/mailismus/pop3/POP3Test.java +++ b/server/src/test/java/com/grey/mailismus/pop3/POP3Test.java @@ -47,7 +47,7 @@ public class POP3Test private static final String SIZEPFX = "Size="; - private static final ApplicationContextNAF appctx = TestSupport.createApplicationContext("POP3Test", true); + private static final ApplicationContextNAF appctx = TestSupport.createApplicationContext("POP3Test", true, null); private Dispatcher dsptch; private boolean dsptch_failed; @@ -149,23 +149,22 @@ private DownloadClient.Results runtest(String cid, int sid, String[] messages, b throws java.io.IOException, java.security.GeneralSecurityException { // create a disposable Dispatcher first, just to identify and clean up the working directories that will be used - dsptch = Dispatcher.create(appctx, new DispatcherConfig.Builder().build(), com.grey.logging.Factory.getLogger("no-such-logger")); - NAFConfig nafcfg = dsptch.getApplicationContext().getConfig(); + DispatcherConfig def = DispatcherConfig.builder().withAppContext(appctx).build(); + dsptch = Dispatcher.create(def); + NAFConfig nafcfg = dsptch.getApplicationContext().getNafConfig(); FileOps.deleteDirectory(nafcfg.getPathVar()); FileOps.deleteDirectory(nafcfg.getPathTemp()); FileOps.deleteDirectory(nafcfg.getPathLogs()); // now create the real Dispatcher - com.grey.naf.reactor.config.DispatcherConfig def = new com.grey.naf.reactor.config.DispatcherConfig.Builder() - .withSurviveHandlers(false) - .build(); - dsptch = Dispatcher.create(appctx, def, com.grey.logging.Factory.getLogger("no-such-logger")); + def = def.mutate().withSurviveHandlers(false).build(); + dsptch = Dispatcher.create(def); // set up the POP3 server XmlConfig cfg = XmlConfig.makeSection(nafxml_server, "x"); com.grey.mailismus.Task stask = new com.grey.mailismus.Task("utest_pop3s", dsptch, cfg, Task.DFLT_FACT_DTORY, Task.DFLT_FACT_MS, null); if (dotstuffing) DynLoader.setField(stask.getMS(), "dotstuffing", true); String grpname = "utest_pop3s_listeners"; - ConcurrentListenerConfig[] lcfg = ConcurrentListenerConfig.buildMultiConfig(grpname, appctx.getConfig(), "listeners/listener", stask.taskConfig(), 0, 0, POP3Server.Factory.class, null); + ConcurrentListenerConfig[] lcfg = ConcurrentListenerConfig.buildMultiConfig(grpname, appctx.getNafConfig(), "listeners/listener", stask.taskConfig(), 0, 0, POP3Server.Factory.class, null); ListenerSet lstnrs = new ListenerSet(grpname, dsptch, stask, null, lcfg); int srvport = (connectfail ? 0 : lstnrs.getListener(sid).getPort()); From 5c961c17ab323a7cef1503a7a512f10501ad58a0 Mon Sep 17 00:00:00 2001 From: ybadri Date: Wed, 3 Apr 2024 10:53:49 +0100 Subject: [PATCH 3/6] Completed MockSmtpServer and got a basic SMTP client test to run all the way through --- .../grey/mailismus/mta/deliver/Client.java | 52 +++++---- .../grey/mailismus/mta/deliver/Forwarder.java | 5 +- .../mta/deliver/QueueBasedMessage.java | 4 +- .../mailismus/mta/deliver/SharedFields.java | 5 +- .../mta/deliver/client/SmtpMessage.java | 3 + .../client/SmtpMessageDefaultImpl.java | 26 ++++- .../client/SmtpResponseDescriptor.java | 4 +- .../mailismus/mta/queue/MessageRecip.java | 2 + .../java/com/grey/mailismus/IPlistTest.java | 4 + .../mailismus/mta/deliver/ClientTest.java | 106 ++++++++++++++--- .../mailismus/mta/deliver/ForwarderTest.java | 23 ++-- .../mta/testsupport/MockSmtpServer.java | 108 +++++++++++++++--- 12 files changed, 271 insertions(+), 71 deletions(-) diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java index 903bb61..657028d 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java @@ -11,9 +11,9 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.function.Supplier; import com.grey.base.utils.IP; +import com.grey.base.utils.StringOps; import com.grey.base.utils.TSAP; import com.grey.base.utils.EmailAddress; import com.grey.base.utils.ByteArrayRef; @@ -154,9 +154,11 @@ private void transitionState(PROTO_STATE newstate) { @Override public void ioReceived(ByteArrayRef rcvdata) throws IOException { + System.out.println("xxx Client.ioReceived: state="+pstate+"/"+state2+" - "+rcvdata.toString(null).toString().strip()); if (pstate == PROTO_STATE.S_DISCON) return; //this method can be called in a loop, so skip it after a disconnect if (shared.getTranscript() != null) shared.getTranscript().data_in(pfx_log, rcvdata, getSystemTime()); alt_tmtprotocol = 0; + //xxx parse SmtpDescriptor here and issue disconnect if null, citing invalid response eventRaised(PROTO_EVENT.E_REPLY, rcvdata, null); } @@ -169,14 +171,10 @@ private PROTO_STATE issueDisconnect(int statuscode, CharSequence diagnostic, Byt if (pstate == PROTO_STATE.S_DISCON) return pstate; CharSequence discmsg = "Disconnect"; - if (shared.getTranscript() != null) { - // We will transcript this at the actual point of closing the connection. - // The POP3 client does this more cleanly, as it doesn't finalise the message until it transcripts it. - if (diagnostic != null) { - shared.getDisconnectMsgBuf().setLength(0); - shared.getDisconnectMsgBuf().append(discmsg).append(" - ").append(diagnostic); - discmsg = shared.getDisconnectMsgBuf(); - } + if (diagnostic != null) { + shared.getDisconnectMsgBuf().setLength(0); + shared.getDisconnectMsgBuf().append(discmsg).append(" - ").append(diagnostic); + discmsg = shared.getDisconnectMsgBuf(); } if (pstate == PROTO_STATE.S_RESET) statuscode = 0; // failed on transition to new message, so don't assign the blame to its recips disconnect_status = (short)statuscode; @@ -429,6 +427,7 @@ private void connectionFailed(int statuscode, CharSequence diagnostic, Throwable // consist of a reject response from the remote server or a locally generated problem description (provisional because // we don't decide here whether any failure is transient or final). private void endConnection(CharSequence discmsg, ByteArrayRef failmsg) { + System.out.println("xxx endConnection="+discmsg+" - failmsg="+(failmsg==null?null:failmsg.toString(null).toString().strip())); LEVEL lvl = LEVEL.TRC2; if (getLogger().isActive(lvl)) { StringBuilder sb = shared.getTmpSB(true); @@ -463,6 +462,7 @@ private void endConnection(CharSequence discmsg, ByteArrayRef failmsg) { try { if (failmsg == null && discmsg != null) failmsg = shared.getFailMsgBuffer().populate(discmsg); + //xxx failmsg unused here, but used to get passed to setRecipientStatus() as failmsg SmtpResponseDescriptor rsp = new SmtpResponseDescriptor(disconnect_status, null, null); setRecipientStatus(-1, rsp); } catch (Exception ex) { @@ -502,6 +502,7 @@ private PROTO_STATE raiseSafeEvent(PROTO_EVENT evt, ByteArrayRef rspdata, CharSe } private PROTO_STATE eventRaised(PROTO_EVENT evt, ByteArrayRef rspdata, CharSequence discmsg) throws IOException { + System.out.println("xxx eventRaised="+evt+" with rspdata="+(rspdata==null?null:rspdata.toString(null).toString().strip())+" and discmsg="+discmsg); switch (evt) { case E_CONNECTED: @@ -763,28 +764,35 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o case A_MAILBODY: Object msgdata = smtpMessage.getData().get(); - Supplier dataErrMsg = () -> null; + String dataErrMsg = null; long msgbytes = 0; try { - if (msgdata instanceof Path) { - Path p = (Path)msgdata; - dataErrMsg = () -> (Files.exists(p) ? null : "Spool file missing"); - msgbytes = Files.size(p); - getWriter().transmit(p); - } else if (msgdata instanceof byte[]) { + if (msgdata instanceof byte[]) { byte[] b = (byte[])msgdata; msgbytes = b.length; getWriter().transmit(b); + } else if (msgdata instanceof CharSequence) { + byte[] b = msgdata.toString().getBytes(StringOps.DFLT_CHARSET); + msgbytes = b.length; + getWriter().transmit(b); + } else if (msgdata instanceof Path) { + Path p = (Path)msgdata; + msgbytes = Files.size(p); + getWriter().transmit(p); } else { - dataErrMsg = () -> "Invalid message data supplied - "+(msgdata == null ? null : msgdata.getClass().getName()); + dataErrMsg = "Invalid message data supplied - "+(msgdata == null ? null : msgdata.getClass().getName()); } } catch (IOException ex) { - if (dataErrMsg.get() == null) - throw ex; //prob some temporary comms issue + if (msgdata instanceof Path && !Files.exists((Path)msgdata)) { + dataErrMsg = "Spool file missing"; + } else { + return issueDisconnect(Protocol.REPLYCODE_TMPERR_LOCAL, "IO error "+ex.getMessage(), FAILMSG_LOCAL); + } } - if (dataErrMsg.get() != null) - return issueDisconnect(Protocol.REPLYCODE_PERMERR_MISC, dataErrMsg.get(), FAILMSG_LOCAL); + if (dataErrMsg != null) //xxx need to return this error to caller + return issueDisconnect(Protocol.REPLYCODE_PERMERR_MISC, dataErrMsg, FAILMSG_LOCAL); + transmit(Protocol.EOL_BC); transmit(shared.getSmtpRequestEOM()); alt_tmtprotocol = calculateMaxTime(msgbytes, conncfg.getMinRateData()); if (alt_tmtprotocol < conncfg.getIdleTimeout().toMillis()) alt_tmtprotocol = 0; @@ -921,6 +929,7 @@ private PROTO_STATE sendAuth(ByteArrayRef rspdata) throws IOException { } private void setRecipientStatus(int idx, SmtpResponseDescriptor reply) { + System.out.println("xxx Client.setRecipientStatus="+idx+" - "+reply); if (idx == -1) { // apply this status to all recipients if (smtpMessage == null || smtpMessage.getRecipients() == null) @@ -932,6 +941,7 @@ private void setRecipientStatus(int idx, SmtpResponseDescriptor reply) { } SmtpResponseDescriptor prevStatus = (idx >= recipientStatus.size() ? null : recipientStatus.get(idx)); + //xxx REPLYCODE_OK is not only one, 251&252 are also ok - we need isRecipientStatusOk(SmtpResponseDescriptor) if (prevStatus != null) { //overwriting an earlier status if (reply.smtpStatus() > prevStatus.smtpStatus()) { diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java index 15f1f34..d7c6685 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java @@ -587,6 +587,7 @@ private void senderCompleted(Client sender, boolean aborted) { @Override public boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, SmtpResponseDescriptor status, TSAP remote, boolean aborted) { + System.out.println("xxx Forwarder.recipientCompleted-"+((QueueBasedMessage)msg).getClient()+": recip="+msgCount+":"+recipId+"/"+status+" on "+remote+" with aborted="+aborted+" - msg="+msg); SmtpMessage.Recipient recip = msg.getRecipients().get(recipId); MessageRecip qrecip = ((QueueBasedRecipient)recip).getQueueRecip(); @@ -617,8 +618,8 @@ public boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, Sm StringBuilder sbStatus = tmpsb2; sbStatus.setLength(0); sbStatus.append(status.smtpStatus()); - if (status.enhancedStatus() != null) sbStatus.append(' ').append(status.enhancedStatus()); - sbStatus.append(' ').append(status.message()); + if (!StringOps.isBlank(status.enhancedStatus())) sbStatus.append(' ').append(status.enhancedStatus()); + if (!StringOps.isBlank(status.message())) sbStatus.append(' ').append(status.message()); LEVEL lvl = LEVEL.TRC2; if (dsptch.getLogger().isActive(lvl)) { diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java index f25d3c5..0a746e8 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java @@ -88,6 +88,8 @@ public void setClient(Client client) { @Override public String toString() { - return "QueueBasedMessage[msg="+msgcnt+": sender="+sender+" => "+destdomain+"/"+ recips+ " - relay="+relay+", spid="+spid+"]"; + return getClass().getSimpleName()+"[msg="+msgcnt+": sender="+sender + +" => "+destdomain+"/"+(recips==null?null:recips.size()+"/"+recips) + + " - relay="+relay+", spid="+spid+" - sender="+client+"]"; } } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/SharedFields.java b/server/src/main/java/com/grey/mailismus/mta/deliver/SharedFields.java index 7190253..06178a6 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/SharedFields.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/SharedFields.java @@ -70,7 +70,7 @@ public SharedFields(Builder bldr) throws GeneralSecurityException { this.dnsResolver = bldr.dnsResolver; this.bufferGenerator = bldr.bufferGenerator; this.transcript = bldr.transcript; - this.defaultConfig = bldr.defaultConfig; + this.defaultConfig = (bldr.defaultConfig == null ? ConnectionConfig.builder().build() : bldr.defaultConfig ); this.remotesConfig = Collections.unmodifiableList(bldr.remoteConfigs); authTypesSupported = new HashMap<>(); @@ -241,8 +241,7 @@ static class Builder { private ConnectionConfig defaultConfig; private final List remoteConfigs = new ArrayList<>(); - private Builder() { - } + private Builder() {} public Builder withDnsResolver(ResolverDNS dns) { this.dnsResolver = dns; diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java index 24a3954..baf94ee 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java @@ -8,6 +8,7 @@ import java.util.function.Supplier; public interface SmtpMessage { + //xxx scrap this interface and Client can parse out domain part where it needs to interface Recipient { CharSequence getDomain(); CharSequence getMailbox(); @@ -19,4 +20,6 @@ interface Recipient { // This is only used for logging, and is something meaningful to the caller. Maybe the SMTP message-ID, but not necessarily. CharSequence getMessageId(); + + static SmtpMessageDefaultImpl.Builder builder() {return SmtpMessageDefaultImpl.builder();} } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java index a146bde..a6ec193 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java @@ -41,6 +41,20 @@ public CharSequence getMessageId() { return msgId; } + @Override + public String toString() { + return getClass().getSimpleName()+"[" + +"msgId=" + msgId + +", sender=" + sender + +", recipients=" + recipients + +", data=" + data + +"]"; + } + + public static Builder builder() { + return new Builder(); + } + public static class SmtpRecipientDefaultImpl implements SmtpMessage.Recipient { private final CharSequence domain; @@ -60,6 +74,14 @@ public CharSequence getDomain() { public CharSequence getMailbox() { return mailbox; } + + @Override + public String toString() { + return getClass().getSimpleName()+"[" + +"domain=" + domain + +", mailbox="+ mailbox + +"]"; + } } @@ -68,6 +90,8 @@ public static class Builder { private CharSequence sender; private Supplier data; + private Builder() {} + public Builder withSender(CharSequence sender) { this.sender = sender; return this; @@ -88,7 +112,7 @@ public Builder withData(Supplier data) { return this; } - public SmtpMessageDefaultImpl createSmtpMessage() { + public SmtpMessageDefaultImpl build() { return new SmtpMessageDefaultImpl(this); } } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java index 8233211..50dd860 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java @@ -31,7 +31,7 @@ public static SmtpResponseDescriptor parse(ByteArrayRef rspdata, boolean withEnh off += STATUSCODE_DIGITS; off++; //skip space - //xxx could also be hyphen (see Google EHLO response) and return null if neither - caller must handle null + //xxx could also be hyphen (see Google EHLO response) and return null if neither if (off >= rsplen) return new SmtpResponseDescriptor(statuscode, enhanced, msg); @@ -73,7 +73,7 @@ public String message() { @Override public String toString() { - return "SmtpResponse [smtp_status="+smtp_status+", enhanced="+enhanced_status+ ", msg="+msg+"]"; + return "SmtpResponse[smtp_status="+smtp_status+", enhanced="+enhanced_status+ ", msg="+msg+"]"; } private static int parseDecimal(int numDigits, byte[] buf, int off) { diff --git a/server/src/main/java/com/grey/mailismus/mta/queue/MessageRecip.java b/server/src/main/java/com/grey/mailismus/mta/queue/MessageRecip.java index d82c360..d5e9d36 100644 --- a/server/src/main/java/com/grey/mailismus/mta/queue/MessageRecip.java +++ b/server/src/main/java/com/grey/mailismus/mta/queue/MessageRecip.java @@ -70,12 +70,14 @@ public String toString() { public StringBuilder toString(StringBuilder sb) { if (sb == null) sb = new StringBuilder(80); + sb.append(getClass().getName()).append('['); sb.append(sender).append("=>").append(mailbox_to); if (domain_to != null) sb.append('@').append(domain_to); sb.append("/spid=0x"); IntValue.encodeHex(spid & ByteOps.INTMASK, true, sb).append("/qid=").append(qid); sb.append("/status=").append(status(qstatus)).append(':').append(smtp_status); if (retrycnt != 0) sb.append("/retries=").append(retrycnt); + sb.append(']'); return sb; } diff --git a/server/src/test/java/com/grey/mailismus/IPlistTest.java b/server/src/test/java/com/grey/mailismus/IPlistTest.java index e632fea..79ef302 100644 --- a/server/src/test/java/com/grey/mailismus/IPlistTest.java +++ b/server/src/test/java/com/grey/mailismus/IPlistTest.java @@ -89,6 +89,8 @@ public void testMemoryAsync() throws java.io.IOException, java.net.URISyntaxExce @Override public Dispatcher getDispatcher() {return dsptch;} @Override + public void startDispatcherRunnable() {} + @Override public boolean stopDispatcherRunnable() { for (TimerNAF t : timers) { t.cancel(); @@ -164,6 +166,8 @@ public void testDBAsync() throws java.io.IOException, java.net.URISyntaxExceptio @Override public Dispatcher getDispatcher() {return dsptch;} @Override + public void startDispatcherRunnable() {} + @Override public boolean stopDispatcherRunnable() { for (TimerNAF t : timers) { t.cancel(); diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java index 2b81d69..9c6061c 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java @@ -1,16 +1,29 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ package com.grey.mailismus.mta.deliver; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintWriter; import java.net.Socket; +import java.security.GeneralSecurityException; +import java.util.HashMap; +import java.util.Map; +import com.grey.mailismus.mta.deliver.client.SmtpMessage; +import com.grey.mailismus.mta.deliver.client.SmtpMessageDefaultImpl; import com.grey.mailismus.mta.deliver.client.SmtpRelay; +import com.grey.mailismus.mta.deliver.client.SmtpResponseDescriptor; +import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.mta.testsupport.MockSmtpServer; import com.grey.naf.BufferGenerator; import com.grey.naf.reactor.Dispatcher; +import com.grey.naf.reactor.DispatcherRunnable; import com.grey.naf.reactor.config.DispatcherConfig; import com.grey.base.utils.TSAP; import com.grey.base.utils.TimeOps; @@ -19,30 +32,29 @@ import org.junit.Before; import org.junit.Test; -/*xxxx Need to test: HELO, EHLO, EHLO rejected, SASL, STLS, some/all recips rejected, msg rejected, quit error, connection-loss in mid session - * Also a test with DNS enabled, but interceptor in place to ensure we connect to mock server anyway - how do we verify the DNS lookups? - * Interceptor would be in SharedFields if we bring it back +/*xxxx Need to test: HELO, EHLO, EHLO rejected, SASL, STLS, some/all recips rejected, msg rejected, quit error, bad greeting, connection-loss in mid session */ public class ClientTest { - private MockSmtpServer srvr; + private MockSmtpServer smtpServer; @Before public void setup() throws Exception { - srvr = new MockSmtpServer(null); - srvr.start(); + smtpServer = new MockSmtpServer(); + smtpServer.start(); } @After public void teardown() throws Exception { - if (srvr != null) { - srvr.stop(); + if (smtpServer != null) { + smtpServer.stop(); Thread.sleep(100); } } //xxx @Test public void testSanity() throws Exception { - Socket sock = srvr.connect(); + smtpServer.configure(null, "250 ok", "250 msg o", "220 Greetings"); + Socket sock = smtpServer.connect(); OutputStream ostrm = sock.getOutputStream(); PrintWriter sendStream = new PrintWriter(ostrm, true); InputStream istrm = sock.getInputStream(); @@ -51,34 +63,94 @@ public void testSanity() throws Exception { sendStream.println("Hello I am the client"); String s = recvStream.readLine(); System.out.println("Client received ["+s+"]"); - sock.close(); //xxx srvr.closeSession(sock.getPort()); + sock.close(); Thread.sleep(500); } - @Test + //xxx @Test public void testSuccessfulSend() throws Exception { + SmtpMessage msg = SmtpMessage.builder() + .withSender("sender1@srcdom1.com") + .withRecipient(new SmtpMessageDefaultImpl.SmtpRecipientDefaultImpl("dstdom1.com", "recip1")) + .withData(() -> "test msgbody1") + .build(); + Map sequences = new HashMap<>(); + sequences.put("HELO", "220 server.com ESMTP"); + sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); + sequences.put("DATA", "354 start body"); + sequences.put("QUIT", "250 quit ok"); + for (SmtpMessage.Recipient recip : msg.getRecipients()) { + sequences.put("RCPT TO: <"+recip.getMailbox()+"@"+recip.getDomain()+">", "250 recip ok"); + } + smtpServer.configure(sequences, "220 Greetings", "250 msg ok", "250 ok"); + sendMessage(msg); + } + + private void sendMessage(SmtpMessage msg) throws IOException, GeneralSecurityException { BufferGenerator.BufferConfig bufcfg = new BufferGenerator.BufferConfig(256, true, null, null); BufferGenerator bufgen = new BufferGenerator(bufcfg); - TSAP remote = TSAP.build("127.0.0.1", srvr.getServicePort()); + TSAP remote = TSAP.build("127.0.0.1", smtpServer.getServicePort()); DispatcherConfig def = DispatcherConfig.builder() .withName("test-dispatcher1") .withSurviveHandlers(false) .build(); Dispatcher dsptch = Dispatcher.create(def); - + SharedFields shared = SharedFields.builder() .withBufferGenerator(bufgen) .build(); SmtpRelay relay = SmtpRelay.builder() - .withName("test-relay1") + .withName("test-relay-mockserver") .withAddress(remote) .build(); Client clnt = new Client(shared, dsptch); - - //xxx clnt.startConnection(null, null, relay); + + TestSmtpSender sender = new TestSmtpSender(dsptch, clnt, msg, relay); + dsptch.loadRunnable(sender); dsptch.start(); Dispatcher.STOPSTATUS stopsts = dsptch.waitStopped(TimeOps.MSECS_PER_SECOND*10L, true); System.out.println("xxx client="+clnt+", relay="+relay+", stopsts="+stopsts); } -} + + + private static class TestSmtpSender implements SmtpSender, DispatcherRunnable { + private final Dispatcher dsptch; + private final Client clnt; + private final SmtpMessage msg; + private final SmtpRelay relay; + + public TestSmtpSender(Dispatcher dsptch, Client clnt, SmtpMessage msg, SmtpRelay relay) { + this.dsptch = dsptch; + this.clnt = clnt; + this.msg = msg; + this.relay = relay; + } + @Override + public boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, SmtpResponseDescriptor status, TSAP remote, boolean aborted) { + System.out.println("xxx TestSmtpSender.recipientCompleted: recip="+msgCount+":"+recipId+"/"+status+" on "+remote+" with aborted="+aborted+" - msg="+msg); + return false; + } + @Override + public SmtpMessage messageCompleted(SmtpMessage msg, int msgCount) { + System.out.println("xxx TestSmtpSender.messageCompleted: msg="+msgCount+"/"+msg); + return null; + } + @Override + public void onDisconnect(SmtpMessage msg, int msgCount) { + System.out.println("xxx TestSmtpSender.onDisconnect: msg="+msgCount+"/"+msg); + } + @Override + public String getName() { + return getClass().getName(); + } + @Override + public Dispatcher getDispatcher() { + return dsptch; + } + @Override + public void startDispatcherRunnable() throws IOException { + clnt.startConnection(msg, this, relay); + } + } +} \ No newline at end of file diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java index 8b1e97b..d7c7ede 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java @@ -4,15 +4,16 @@ */ package com.grey.mailismus.mta.deliver; +import java.io.IOException; +import java.security.GeneralSecurityException; + import com.grey.base.config.XmlConfig; import com.grey.base.utils.ByteChars; import com.grey.base.utils.TimeOps; import com.grey.base.utils.EmailAddress; +import com.grey.base.utils.FileOps; import com.grey.base.utils.IP; import com.grey.base.utils.DynLoader; - -import java.security.GeneralSecurityException; - import com.grey.base.collections.HashedSetInt; import com.grey.naf.ApplicationContextNAF; import com.grey.naf.EventListenerNAF; @@ -122,11 +123,11 @@ public void setup() throws java.io.IOException { .withSurviveHandlers(false) .withAppContext(appctx) .build(); + dsptch = Dispatcher.create(def); + dsptch.registerEventListener(this); ResolverConfig rcfg = new ResolverConfig.Builder() .withXmlConfig(nafcfg.getNode("dnsresolver")) .build(); - dsptch = Dispatcher.create(def); - dsptch.registerEventListener(this); dnsResolver = ResolverDNS.create(dsptch, rcfg); } @@ -145,7 +146,7 @@ public void testRealClient_PermErr() throws Exception { MyQueueManager qmgr = new MyQueueManager(dsptch, msgs, false); AppConfig appcfg = AppConfig.get("", dsptch); fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, null, this, dnsResolver); - exec(qmgr, null, false); + exec(qmgr, null, false, 4); //4 NDRs (3 for one SPID, 1 for another) } @org.junit.Test @@ -338,7 +339,9 @@ private void testSenderAllocation(boolean deferred) throws Exception { exec(qmgr, sndrfact, false); } - private void exec(MyQueueManager qmgr, SenderFactory sndrfact, boolean slaverelay) { + private void exec(MyQueueManager qmgr, SenderFactory sndrfact, boolean slaverelay, int expectedSpoolFiles) throws IOException { + String nafPathVar = dsptch.getApplicationContext().getNafConfig().getPathVar(); + FileOps.deleteDirectory(nafPathVar); org.junit.Assert.assertEquals(slaverelay, fwd.getRouting().modeSlaveRelay()); fwd.start(); dsptch.start(); @@ -352,6 +355,12 @@ private void exec(MyQueueManager qmgr, SenderFactory sndrfact, boolean slaverela org.junit.Assert.assertEquals(sndrfact.sender_cnt, sndrfact.sender_completion_cnt); org.junit.Assert.assertEquals(1, sndrfact.batch_completion_cnt); } + org.junit.Assert.assertEquals(expectedSpoolFiles, FileOps.countFiles(new java.io.File(nafPathVar+"/spool"), true)); + FileOps.deleteDirectory(nafPathVar); + } + + private void exec(MyQueueManager qmgr, SenderFactory sndrfact, boolean slaverelay) throws IOException { + exec(qmgr, sndrfact, slaverelay, 0); } @Override diff --git a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java index 8e9f5de..a84487e 100644 --- a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java +++ b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java @@ -1,3 +1,7 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ package com.grey.mailismus.mta.testsupport; import java.io.BufferedReader; @@ -5,18 +9,32 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; -import java.io.PrintWriter; import java.net.InetAddress; import java.net.ServerSocket; import java.net.Socket; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public class MockSmtpServer { + private static final String EOL_SMTP = "\r\n"; + private static final String SENDSEQ_NOREPLY = "-"; + + private static final String EXPECT_ACTION_DISCONNECT = "_DISC_"; + + private final Map> transcriptsByPort = new ConcurrentHashMap<>();//keyed on remote-dotted-ip:remote-port private final ServerSocket srvrSocket; private volatile boolean shutdown; - public MockSmtpServer(Map expectSend) throws IOException { + private final Map expectSendSequences = new ConcurrentHashMap<>(); + private volatile String greetingResponse; + private volatile String messageResponse; + private volatile String defaultResponse; + + public MockSmtpServer() throws IOException { srvrSocket = new ServerSocket(0, 10, InetAddress.getLoopbackAddress()); } @@ -35,6 +53,14 @@ public int getServicePort() { return srvrSocket.getLocalPort(); } + public void configure(Map expectSend, String initrsp, String msgrsp, String dfltrsp) { + expectSendSequences.clear(); + if (expectSend != null) expectSendSequences.putAll(expectSend); + greetingResponse = initrsp; + messageResponse = msgrsp; + defaultResponse = dfltrsp; + } + public Socket connect() throws IOException { return new Socket(InetAddress.getLoopbackAddress(), srvrSocket.getLocalPort()); } @@ -44,17 +70,18 @@ private void runService() { try { handleConnection(); } catch (Throwable ex) { - if (!shutdown) System.out.println("Mock SMTP server error - "+ex); + if (!shutdown) System.out.println("SMTP-Server error handing connections - "+ex); break; } } - System.out.println("SMTP server terminating - "+srvrSocket); + System.out.println("SMTP-Server terminating - "+srvrSocket); } private void handleConnection() throws IOException { Socket sock = srvrSocket.accept(); - System.out.println("SMTP session accepted - "+sock); - SmtpSession session = new SmtpSession(sock); + String key = sock.getInetAddress().getHostAddress()+":"+sock.getPort(); + List transcript = transcriptsByPort.computeIfAbsent(key, k-> new ArrayList<>()); + SmtpSession session = new SmtpSession(sock, expectSendSequences, greetingResponse, messageResponse, defaultResponse, transcript); Runnable r = () -> session.serveSession(); Thread t = new Thread(r); t.start(); @@ -64,32 +91,79 @@ private void handleConnection() throws IOException { private static class SmtpSession { private final Socket sock; private final BufferedReader recvStream; - private final PrintWriter sendStream; + private final OutputStream sendStream; + private final Map expectSendSequences; + private final String greetingResponse; + private final String messageResponse; + private final String defaultResponse; + private final List transcript; + private boolean inDataTransfer; - public SmtpSession(Socket sock) throws IOException { + public SmtpSession(Socket sock, Map expectSend, String initrsp, String msgrsp, String dfltrsp, List transcript) throws IOException { + this.expectSendSequences = new HashMap<>(); + if (expectSend != null) expectSendSequences.putAll(expectSend); + this.greetingResponse = initrsp; + this.messageResponse = msgrsp; + this.defaultResponse = dfltrsp; + this.transcript = transcript; this.sock = sock; InputStream istrm = sock.getInputStream(); InputStreamReader rdr = new InputStreamReader(istrm); recvStream = new BufferedReader(rdr); - OutputStream ostrm = sock.getOutputStream(); - sendStream = new PrintWriter(ostrm, true); + sendStream = sock.getOutputStream(); } public void serveSession() { + transcript.add("SMTP-Session accepted - "+sock); try { - serviceLoop(); + sessionDialogue(); } catch (Throwable ex) { - System.out.println("SMTP session exiting on error - "+ex); + transcript.add("SMTP-Session error handling session - "+ex); } - System.out.println("SMTP session terminating - "+sock); + transcript.add("SMTP-Session terminating - "+sock); + + try { + sock.close(); + } catch (Throwable ex) { + transcript.add("SMTP-Session error closing connection - "+ex); + } + System.out.println("TRANSCRIPT:\n"+String.join("\n", transcript)); } - private void serviceLoop() throws IOException { + private void sessionDialogue() throws IOException { + sendResponse(greetingResponse); String request; while ((request = recvStream.readLine()) != null) { - System.out.println("xxx session received ["+request+"]"); - sendStream.println("Hello I am the server"); + transcript.add("RECV: "+request); + String rsp; + if (inDataTransfer) { + if (!request.equals(".")) { + continue; + } + inDataTransfer = false; + rsp = messageResponse; + } else { + rsp = expectSendSequences.get(request); + if (EXPECT_ACTION_DISCONNECT.equals(rsp)) { + break; + } + if (request.equalsIgnoreCase("DATA")) { + inDataTransfer = true; + } + } + sendResponse(rsp); } } + + private void sendResponse(String rsp) throws IOException { + if (rsp == null) + rsp = defaultResponse; + if (SENDSEQ_NOREPLY.equals(rsp)) + return; + transcript.add("XMIT: "+rsp); + rsp += EOL_SMTP; + sendStream.write(rsp.getBytes()); + sendStream.flush(); + } } -} \ No newline at end of file +} From f30b5a543428a73554d40aa41fcc537660059aa4 Mon Sep 17 00:00:00 2001 From: ybadri Date: Wed, 3 Apr 2024 16:02:29 +0100 Subject: [PATCH 4/6] Got rid of SmtpMessage.Recipient interface - is now just a CharSequence --- .../grey/mailismus/mta/deliver/Client.java | 48 +++++----- .../grey/mailismus/mta/deliver/Forwarder.java | 6 +- .../mta/deliver/QueueBasedRecipient.java | 44 ++++++--- .../mta/deliver/client/SmtpMessage.java | 8 +- .../client/SmtpMessageDefaultImpl.java | 39 ++------ .../mailismus/mta/queue/MessageRecip.java | 3 +- .../mailismus/mta/deliver/ClientTest.java | 7 +- .../mta/deliver/QueueBasedRecipientTest.java | 90 +++++++++++++++++++ .../grey/mailismus/mta/smtp/DeliveryTest.java | 3 + 9 files changed, 169 insertions(+), 79 deletions(-) create mode 100644 server/src/test/java/com/grey/mailismus/mta/deliver/QueueBasedRecipientTest.java diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java index 657028d..9b38c60 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java @@ -148,8 +148,8 @@ private ConnectionConfig getConnectionConfig(int remote_ip) { return shared.getDefaultConfig(); } - private void transitionState(PROTO_STATE newstate) { - pstate = newstate; + private void transitionState(PROTO_STATE newState) { + pstate = newState; } @Override @@ -206,7 +206,7 @@ private void dnsLookup(boolean as_host) throws IOException { mxptr = 0; dnsInfo.clear(); setFlag(S2_DNSWAIT); - CharSequence domain = smtpMessage.getRecipients().get(0).getDomain(); + CharSequence domain = parseDomain(smtpMessage.getRecipients().get(0)); ResolverAnswer answer; if (as_host) { answer = shared.getDnsResolver().resolveHostname(domain, this, null, 0); @@ -427,7 +427,6 @@ private void connectionFailed(int statuscode, CharSequence diagnostic, Throwable // consist of a reject response from the remote server or a locally generated problem description (provisional because // we don't decide here whether any failure is transient or final). private void endConnection(CharSequence discmsg, ByteArrayRef failmsg) { - System.out.println("xxx endConnection="+discmsg+" - failmsg="+(failmsg==null?null:failmsg.toString(null).toString().strip())); LEVEL lvl = LEVEL.TRC2; if (getLogger().isActive(lvl)) { StringBuilder sb = shared.getTmpSB(true); @@ -502,7 +501,7 @@ private PROTO_STATE raiseSafeEvent(PROTO_EVENT evt, ByteArrayRef rspdata, CharSe } private PROTO_STATE eventRaised(PROTO_EVENT evt, ByteArrayRef rspdata, CharSequence discmsg) throws IOException { - System.out.println("xxx eventRaised="+evt+" with rspdata="+(rspdata==null?null:rspdata.toString(null).toString().strip())+" and discmsg="+discmsg); + System.out.println("xxx eventRaised="+evt+"/"+pstate+" with rspdata="+(rspdata==null?null:rspdata.toString(null).toString().strip())+" and discmsg="+discmsg); switch (evt) { case E_CONNECTED: @@ -681,7 +680,7 @@ private PROTO_STATE handleReply(ByteArrayRef rspdata) throws IOException { return pstate; } - private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int okreply, CharSequence discmsg, ByteArrayRef rspdata) throws IOException { + private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newState, int okreply, CharSequence discmsg, ByteArrayRef rspdata) throws IOException { if (okreply != 0 && replyDescriptor.smtpStatus() != okreply) { //note that okreply will already have been reset in state S_MAILTO LEVEL lvl = LEVEL.TRC; @@ -696,7 +695,7 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o } return issueDisconnect(replyDescriptor.smtpStatus(), "Server rejection", rspdata); } - if (newstate != null) transitionState(newstate); + if (newState != null) transitionState(newState); boolean endpipe = (pipe_count == 0 && dataWait != 0); //changed state, but don't send any more until all pipelined commands are acked ByteBuffer reqbuf; @@ -735,7 +734,7 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o break; case A_MAILFROM: //all recips members refer to same message, so they have a common Sender - endpipe = sendPipelinedRequest(SMTPREQ_MAILFROM, false, smtpMessage.getSender(), null, true); + endpipe = sendPipelinedRequest(SMTPREQ_MAILFROM, false, smtpMessage.getSender(), true); if (!endpipe) issueAction(PROTO_ACTION.A_MAILTO, null); break; @@ -745,8 +744,7 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o issueAction(PROTO_ACTION.A_DATA, null); break; } - SmtpMessage.Recipient recip = smtpMessage.getRecipients().get(recips_sent); - endpipe = sendPipelinedRequest(SMTPREQ_MAILTO, false, recip.getMailbox(), recip.getDomain(), true); + endpipe = sendPipelinedRequest(SMTPREQ_MAILTO, false, smtpMessage.getRecipients().get(recips_sent), true); recips_sent++; } break; @@ -755,7 +753,7 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o if (isFlagSet(S2_SENT_DATACMD)) break; //in case we pipelined it before receiving all recip responses if (pipe_count != 0) { //in pipelined send-ahead mode - forego canned ByteBuffer, to piggyback reply on the buffered pipeline - sendPipelinedRequest(Protocol.CMDREQ_DATA, true, null, null, false); + sendPipelinedRequest(Protocol.CMDREQ_DATA, true, null, false); } else { transmit(shared.getSmtpRequestData()); } @@ -848,8 +846,8 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate, int o return pstate; } - private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newstate) throws IOException { - return issueAction(action, newstate, 0, null, null); + private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newState) throws IOException { + return issueAction(action, newState, 0, null, null); } @Override @@ -929,7 +927,6 @@ private PROTO_STATE sendAuth(ByteArrayRef rspdata) throws IOException { } private void setRecipientStatus(int idx, SmtpResponseDescriptor reply) { - System.out.println("xxx Client.setRecipientStatus="+idx+" - "+reply); if (idx == -1) { // apply this status to all recipients if (smtpMessage == null || smtpMessage.getRecipients() == null) @@ -960,15 +957,13 @@ private void setRecipientStatus(int idx, SmtpResponseDescriptor reply) { } // NB: We can use shared.pipebuf because the pipeline is always built up and sent within one callback - private boolean sendPipelinedRequest(ByteChars cmd, boolean flush, CharSequence addr, - CharSequence domain, boolean close_brace) throws IOException { + private boolean sendPipelinedRequest(ByteChars cmd, boolean flush, CharSequence param, boolean close_brace) throws IOException { ByteChars rspbuf = shared.getPipelineBuffer(); if (pipe_count == 0) rspbuf.clear(); rspbuf.append(cmd); - if (addr != null) { - rspbuf.append(addr); - if (domain != null) rspbuf.append(EmailAddress.DLM_DOM).append(domain); + if (param != null) { + rspbuf.append(param); } if (close_brace) rspbuf.append(CLOSE_ANGLE); rspbuf.append(Protocol.EOL_BC); @@ -1063,7 +1058,8 @@ private CharSequence peerDescription(StringBuilder sb) { return "nullpeer"; String dlm = ""; if (smtpMessage.getRecipients() != null && !smtpMessage.getRecipients().isEmpty()) { - sb.append("domain=").append(smtpMessage.getRecipients().get(0).getDomain()); + CharSequence domain = parseDomain(smtpMessage.getRecipients().get(0)); + sb.append("domain=").append(domain); dlm = "/"; } if (active_relay != null) { @@ -1072,6 +1068,18 @@ private CharSequence peerDescription(StringBuilder sb) { return sb; } + private static CharSequence parseDomain(CharSequence emailAddress) { + int pos = 0; + int lmt = emailAddress.length(); + while (pos < lmt - 1) { + if (emailAddress.charAt(pos) == EmailAddress.DLM_DOM) { + return emailAddress.subSequence(pos+1, lmt); + } + pos++; + } + return null; + } + // given a minimum bits per second, calculate the max time expected to send this many bytes (in milliseconds) private static long calculateMaxTime(long numBytes, long minBPS) { return (numBytes * minBPS) / (8_000); diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java index d7c6685..8f8135c 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java @@ -502,7 +502,7 @@ private void startSender(Client sender) sender.setEventListener(sendersEventListener); //gets cleared on ChannelMonitor.disconnect() sender.startConnection(msg, this, relay); } catch (Throwable ex) { - dsptch.getLogger().log(LEVEL.TRC, ex, true, "SMTP-Delivery/batch="+batchcnt+": Failed to start Sender="+sender.getLogID()+"/"+sender); + dsptch.getLogger().log(LEVEL.INFO, ex, true, "SMTP-Delivery/batch="+batchcnt+": Failed to start Sender="+sender.getLogID()+"/"+sender); senderCompleted(sender, true); } } @@ -588,8 +588,8 @@ private void senderCompleted(Client sender, boolean aborted) { @Override public boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, SmtpResponseDescriptor status, TSAP remote, boolean aborted) { System.out.println("xxx Forwarder.recipientCompleted-"+((QueueBasedMessage)msg).getClient()+": recip="+msgCount+":"+recipId+"/"+status+" on "+remote+" with aborted="+aborted+" - msg="+msg); - SmtpMessage.Recipient recip = msg.getRecipients().get(recipId); - MessageRecip qrecip = ((QueueBasedRecipient)recip).getQueueRecip(); + QueueBasedRecipient recip = (QueueBasedRecipient)msg.getRecipients().get(recipId); + MessageRecip qrecip = recip.getQueueRecip(); if (aborted) { // Any recipients who've already failed remain as failures, but recipients who had been marked as OK revert to diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedRecipient.java b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedRecipient.java index f8b1396..8a0622a 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedRecipient.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedRecipient.java @@ -4,32 +4,56 @@ */ package com.grey.mailismus.mta.deliver; -import com.grey.mailismus.mta.deliver.client.SmtpMessage; +import com.grey.base.utils.EmailAddress; import com.grey.mailismus.mta.queue.MessageRecip; -class QueueBasedRecipient implements SmtpMessage.Recipient { +// This is just a wrapper that adapts MessageRecip for SmtpMessage +class QueueBasedRecipient implements CharSequence { private final MessageRecip queueRecip; public QueueBasedRecipient(MessageRecip queueRecip) { this.queueRecip = queueRecip; } + public MessageRecip getQueueRecip() { + return queueRecip; + } + @Override - public CharSequence getDomain() { - return getQueueRecip().domain_to; + public String toString() { + CharSequence mbx = queueRecip.mailbox_to; + CharSequence dom = queueRecip.domain_to; + if (mbx == null || mbx.length() == 0) return ""; + if (dom == null || dom.length() == 0) return mbx.toString(); + StringBuilder sb = new StringBuilder(mbx).append(EmailAddress.DLM_DOM).append(dom); + return sb.toString(); } @Override - public CharSequence getMailbox() { - return getQueueRecip().mailbox_to; + public int length() { + CharSequence mbx = queueRecip.mailbox_to; + CharSequence dom = queueRecip.domain_to; + int len = (mbx == null ? 0 : mbx.length()); + if (dom != null) len += dom.length() + 1; //include the "@" + return len; } - public MessageRecip getQueueRecip() { - return queueRecip; + @Override + public char charAt(int idx) { + CharSequence mbx = queueRecip.mailbox_to; + int len_mbx = (mbx == null ? 0 : mbx.length()); + if (idx < len_mbx) return mbx.charAt(idx); + if (idx >= length()) return toString().charAt(idx); + if (idx == len_mbx) return EmailAddress.DLM_DOM; + return queueRecip.domain_to.charAt(idx - len_mbx - 1); } @Override - public String toString() { - return "QueueBasedRecipient["+queueRecip+"]"; + public CharSequence subSequence(int start, int end) { + CharSequence mbx = queueRecip.mailbox_to; + int len_mbx = (mbx == null ? 0 : mbx.length()); + if (start == 0 && end == len_mbx) return mbx; + if (start == len_mbx + 1 && end == length()) return queueRecip.domain_to; + return toString().subSequence(start, end); } } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java index baf94ee..65362aa 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java @@ -8,14 +8,8 @@ import java.util.function.Supplier; public interface SmtpMessage { - //xxx scrap this interface and Client can parse out domain part where it needs to - interface Recipient { - CharSequence getDomain(); - CharSequence getMailbox(); - } - CharSequence getSender(); - List getRecipients(); + List getRecipients(); Supplier getData(); // This is only used for logging, and is something meaningful to the caller. Maybe the SMTP message-ID, but not necessarily. diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java index a6ec193..6894358 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java @@ -12,7 +12,7 @@ public class SmtpMessageDefaultImpl implements SmtpMessage { private final String msgId = UUID.randomUUID().toString(); private final CharSequence sender; - private final List recipients; + private final List recipients; private final Supplier data; //data can be Path or byte[] public SmtpMessageDefaultImpl(Builder bldr) { @@ -27,7 +27,7 @@ public CharSequence getSender() { } @Override - public List getRecipients() { + public List getRecipients() { return recipients; } @@ -56,37 +56,8 @@ public static Builder builder() { } - public static class SmtpRecipientDefaultImpl implements SmtpMessage.Recipient { - private final CharSequence domain; - private final CharSequence mailbox; - - public SmtpRecipientDefaultImpl(CharSequence domain, CharSequence mailbox) { - this.domain = domain; - this.mailbox = mailbox; - } - - @Override - public CharSequence getDomain() { - return domain; - } - - @Override - public CharSequence getMailbox() { - return mailbox; - } - - @Override - public String toString() { - return getClass().getSimpleName()+"[" - +"domain=" + domain - +", mailbox="+ mailbox - +"]"; - } - } - - public static class Builder { - private List recipients = new ArrayList<>(); + private List recipients = new ArrayList<>(); private CharSequence sender; private Supplier data; @@ -97,12 +68,12 @@ public Builder withSender(CharSequence sender) { return this; } - public Builder withRecipient(Recipient recipient) { + public Builder withRecipient(CharSequence recipient) { this.recipients.add(recipient); return this; } - public Builder withRecipients(List recipients) { + public Builder withRecipients(List recipients) { this.recipients.addAll(recipients); return this; } diff --git a/server/src/main/java/com/grey/mailismus/mta/queue/MessageRecip.java b/server/src/main/java/com/grey/mailismus/mta/queue/MessageRecip.java index d5e9d36..26501bd 100644 --- a/server/src/main/java/com/grey/mailismus/mta/queue/MessageRecip.java +++ b/server/src/main/java/com/grey/mailismus/mta/queue/MessageRecip.java @@ -5,6 +5,7 @@ package com.grey.mailismus.mta.queue; import com.grey.base.utils.ByteOps; +import com.grey.base.utils.EmailAddress; import com.grey.base.utils.IntValue; /* @@ -72,7 +73,7 @@ public StringBuilder toString(StringBuilder sb) { if (sb == null) sb = new StringBuilder(80); sb.append(getClass().getName()).append('['); sb.append(sender).append("=>").append(mailbox_to); - if (domain_to != null) sb.append('@').append(domain_to); + if (domain_to != null) sb.append(EmailAddress.DLM_DOM).append(domain_to); sb.append("/spid=0x"); IntValue.encodeHex(spid & ByteOps.INTMASK, true, sb).append("/qid=").append(qid); sb.append("/status=").append(status(qstatus)).append(':').append(smtp_status); diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java index 9c6061c..480e4a4 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java @@ -16,7 +16,6 @@ import java.util.Map; import com.grey.mailismus.mta.deliver.client.SmtpMessage; -import com.grey.mailismus.mta.deliver.client.SmtpMessageDefaultImpl; import com.grey.mailismus.mta.deliver.client.SmtpRelay; import com.grey.mailismus.mta.deliver.client.SmtpResponseDescriptor; import com.grey.mailismus.mta.deliver.client.SmtpSender; @@ -71,7 +70,7 @@ public void testSanity() throws Exception { public void testSuccessfulSend() throws Exception { SmtpMessage msg = SmtpMessage.builder() .withSender("sender1@srcdom1.com") - .withRecipient(new SmtpMessageDefaultImpl.SmtpRecipientDefaultImpl("dstdom1.com", "recip1")) + .withRecipient("recip1@dstdom1.com") .withData(() -> "test msgbody1") .build(); Map sequences = new HashMap<>(); @@ -79,8 +78,8 @@ public void testSuccessfulSend() throws Exception { sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); sequences.put("DATA", "354 start body"); sequences.put("QUIT", "250 quit ok"); - for (SmtpMessage.Recipient recip : msg.getRecipients()) { - sequences.put("RCPT TO: <"+recip.getMailbox()+"@"+recip.getDomain()+">", "250 recip ok"); + for (CharSequence recip : msg.getRecipients()) { + sequences.put("RCPT TO: <"+recip+">", "250 recip ok"); } smtpServer.configure(sequences, "220 Greetings", "250 msg ok", "250 ok"); sendMessage(msg); diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/QueueBasedRecipientTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/QueueBasedRecipientTest.java new file mode 100644 index 0000000..4b99343 --- /dev/null +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/QueueBasedRecipientTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import java.util.function.Function; + +import com.grey.base.utils.ByteChars; +import com.grey.mailismus.mta.queue.MessageRecip; + +public class QueueBasedRecipientTest { + @Test + public void testWithDomain() { + MessageRecip mrecip = new MessageRecip(); + mrecip.mailbox_to = ByteChars.valueOf("abc"); + mrecip.domain_to = ByteChars.valueOf("xyz"); + String exp = "abc@xyz"; + QueueBasedRecipient qrecip = new QueueBasedRecipient(mrecip); + assertSame(mrecip, qrecip.getQueueRecip()); + assertEquals(exp, qrecip.toString()); + assertEquals(exp.length(), qrecip.length()); + + assertEquals('a', qrecip.charAt(0)); + assertEquals('b', qrecip.charAt(1)); + assertEquals('c', qrecip.charAt(2)); + assertEquals('@', qrecip.charAt(3)); + assertEquals('x', qrecip.charAt(4)); + assertEquals('y', qrecip.charAt(5)); + assertEquals('z', qrecip.charAt(6)); + verifyOOB(qrecip, (cs) -> cs.charAt(7)); + + assertEquals("", qrecip.subSequence(0, 0)); + assertEquals("", qrecip.subSequence(6, 6)); + assertEquals("", qrecip.subSequence(7, 7)); + assertEquals("", qrecip.subSequence(2, 2)); + assertEquals("", qrecip.subSequence(5, 5)); + assertEquals("a", qrecip.subSequence(0, 1)); + assertEquals("abc", qrecip.subSequence(0, 3).toString()); + assertEquals("abc@", qrecip.subSequence(0, 4).toString()); + assertEquals("xyz", qrecip.subSequence(4, 7).toString()); + assertEquals(exp, qrecip.subSequence(0, 7).toString()); + assertEquals("c@x", qrecip.subSequence(2, 5).toString()); + assertEquals("@", qrecip.subSequence(3, 4).toString()); + verifyOOB(qrecip, (cs) -> cs.subSequence(6,8)); + } + @Test + public void testWithoutDomain() { + MessageRecip mrecip = new MessageRecip(); + mrecip.mailbox_to = ByteChars.valueOf("abc"); + String exp = "abc"; + QueueBasedRecipient qrecip = new QueueBasedRecipient(mrecip); + assertSame(mrecip, qrecip.getQueueRecip()); + assertEquals(exp, qrecip.toString()); + assertEquals(exp.length(), qrecip.length()); + + assertEquals('a', qrecip.charAt(0)); + assertEquals('b', qrecip.charAt(1)); + assertEquals('c', qrecip.charAt(2)); + verifyOOB(qrecip, (cs) -> cs.charAt(3)); + + assertEquals("", qrecip.subSequence(0, 0)); + assertEquals("", qrecip.subSequence(1, 1)); + assertEquals("", qrecip.subSequence(2, 2)); + assertEquals("", qrecip.subSequence(3, 3)); + assertEquals("a", qrecip.subSequence(0, 1)); + assertEquals("bc", qrecip.subSequence(1, 3)); + assertEquals("abc", qrecip.subSequence(0, 3).toString()); + verifyOOB(qrecip, (cs) -> cs.subSequence(0,4)); + } + + private static void verifyOOB(QueueBasedRecipient qrecip, Function func) { + try { + Object result = func.apply(qrecip); + fail("Expected to fail on qrecip out-of-bounds="+result); + } catch (Exception ex) { + try { + Object result = func.apply(qrecip.toString()); + fail("Expected to fail on string out-of-bounds="+result); + } catch (Exception ex2) { + assertEquals(ex2.getClass(), ex.getClass()); + assertEquals(ex2.getMessage(), ex.getMessage()); + } + } + } +} diff --git a/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java b/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java index 131875e..e4cf37e 100644 --- a/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/smtp/DeliveryTest.java @@ -16,6 +16,7 @@ import com.grey.base.utils.ByteChars; import com.grey.base.utils.IP; import com.grey.base.utils.TSAP; +import com.grey.logging.Logger; import com.grey.base.utils.EmailAddress; import com.grey.base.utils.DynLoader; import com.grey.base.collections.Circulist; @@ -374,6 +375,7 @@ private void runtest(MessageSpec[] msgs, int server_submitcnt, int server_spoolc // create a disposable Dispatcher first, just to identify and clean up the working directories that will be used DispatcherConfig dcfg = DispatcherConfig.builder().withName("DeliveryTest-preliminary").withAppContext(appctx).build(); dsptch = Dispatcher.create(dcfg); + dsptch.getLogger().setLevel(Logger.LEVEL.TRC5); FileOps.deleteDirectory(nafcfg.getPathVar()); FileOps.deleteDirectory(nafcfg.getPathTemp()); FileOps.deleteDirectory(nafcfg.getPathLogs()); @@ -386,6 +388,7 @@ private void runtest(MessageSpec[] msgs, int server_submitcnt, int server_spoolc .build(); dsptch = Dispatcher.create(def); AppConfig appcfg = AppConfig.get(nafcfg.getPath(pthnam_appcfg, null), dsptch); + dsptch.getLogger().setLevel(Logger.LEVEL.TRC5); // Inject the messages into the queue for Forwarder to pick up. // Email addresses would be lower-cased by SMTP server before being submitted to Queue, so do same with our test data. From 21a01cd35570bd908d7887da3f05f254cc7b4488 Mon Sep 17 00:00:00 2001 From: ybadri Date: Thu, 4 Apr 2024 16:56:10 +0100 Subject: [PATCH 5/6] Introduced SmtpSendResult and refactored SmtpSender callbacks --- pkg/src/main/scripts/mta.bat | 3 +- pkg/src/main/scripts/mta.sh | 3 +- .../java/com/grey/mailismus/AppConfig.java | 2 +- .../grey/mailismus/mta/deliver/Client.java | 32 ++- .../grey/mailismus/mta/deliver/Forwarder.java | 53 ++--- .../mta/deliver/QueueBasedMessage.java | 8 +- .../mta/deliver/client/SmtpMessage.java | 3 + .../client/SmtpMessageDefaultImpl.java | 4 +- .../client/SmtpResponseDescriptor.java | 2 +- .../mta/deliver/client/SmtpSendResult.java | 120 +++++++++++ .../mta/deliver/client/SmtpSender.java | 25 +-- .../mailismus/mta/deliver/ClientTest.java | 202 +++++++++++++++--- .../mailismus/mta/deliver/ForwarderTest.java | 56 +++-- .../mta/testsupport/MockServerDNS.java | 2 +- .../mta/testsupport/MockSmtpServer.java | 4 +- 15 files changed, 398 insertions(+), 121 deletions(-) create mode 100644 server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSendResult.java diff --git a/pkg/src/main/scripts/mta.bat b/pkg/src/main/scripts/mta.bat index 984df07..2cf9821 100644 --- a/pkg/src/main/scripts/mta.bat +++ b/pkg/src/main/scripts/mta.bat @@ -1,4 +1,4 @@ -:: Copyright 2010-2018 Yusef Badri - All rights reserved. +:: Copyright 2010-2024 Yusef Badri - All rights reserved. :: Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). @ECHO OFF @@ -6,6 +6,7 @@ set CMD=%1 set MTAJAR=mailismus-server-${project.version}.jar set NAFCFG=conf\naf.xml set JVM="%JAVA_HOME%"\bin\java +set GREYNAF_BASEPORT=13000 if "%JAVA_HOME%" == "" ( set JVM=java diff --git a/pkg/src/main/scripts/mta.sh b/pkg/src/main/scripts/mta.sh index 8e0312d..7c0fe73 100644 --- a/pkg/src/main/scripts/mta.sh +++ b/pkg/src/main/scripts/mta.sh @@ -1,11 +1,12 @@ #!/bin/sh -# Copyright 2010-2018 Yusef Badri - All rights reserved. +# Copyright 2010-2024 Yusef Badri - All rights reserved. # Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). cd `dirname $0`/.. CMD=$1 MTAJAR=mailismus-server-${project.version}.jar NAFCFG=conf/naf.xml +export GREYNAF_BASEPORT=13000 JVM="$JAVA_HOME"/bin/java if [ ! -x "$JVM" ]; then JVM=java; fi diff --git a/server/src/main/java/com/grey/mailismus/AppConfig.java b/server/src/main/java/com/grey/mailismus/AppConfig.java index 7babb2e..26f05f3 100755 --- a/server/src/main/java/com/grey/mailismus/AppConfig.java +++ b/server/src/main/java/com/grey/mailismus/AppConfig.java @@ -49,7 +49,7 @@ public static AppConfig get(String cfgpath, Dispatcher d) { private AppConfig(String cfgpath, Dispatcher dsptch) throws java.io.IOException { hostName = java.net.InetAddress.getLocalHost().getCanonicalHostName(); - configRoot = (cfgpath.isEmpty() ? XmlConfig.BLANKCFG : XmlConfig.getSection(cfgpath, "mailserver")); + configRoot = (cfgpath == null || cfgpath.isEmpty() ? XmlConfig.BLANKCFG : XmlConfig.getSection(cfgpath, "mailserver")); XmlConfig cfg = configRoot.getSection("application"); productName = cfg.getValue("prodname", true, "Mailismus"); diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java index 9b38c60..be8b10d 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java @@ -35,6 +35,7 @@ import com.grey.mailismus.mta.deliver.client.SmtpMessage; import com.grey.mailismus.mta.deliver.client.SmtpRelay; import com.grey.mailismus.mta.deliver.client.SmtpResponseDescriptor; +import com.grey.mailismus.mta.deliver.client.SmtpSendResult; import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.errors.MailismusException; @@ -47,6 +48,7 @@ class Client implements ResolverDNS.Client, TimerNAF.Handler { + //xxx need to catch Throwable on all calls to user code - messageCompleted() and data Supplier private static final String LOG_PREFIX = "SMTP-Client"; private enum PROTO_STATE {S_DISCON, S_CONN, S_READY, S_AUTH, S_STLS, S_HELO, S_EHLO, S_MAILFROM, S_MAILTO, S_DATA, S_MAILBODY, @@ -94,7 +96,7 @@ private enum PROTO_ACTION {A_CONNECT, A_DISCONNECT, A_HELO, A_EHLO, A_MAILFROM, private int mxptr; //indicates which dnsInfo.rrlist node we're currently connecting/connected to - only valid if dnsInfo non-empty private int recip_id; //indicates which recipient we're currently awaiting a response for xxx can it be replaced by size of recipStatus list? private int recips_sent; //how many recips we've already sent to server - will run ahead of recip_id in pipelining mode - private int okrecips; //number of recipients accepted by server + private int okrecips; //number of recipients accepted by server xxx surely we can count recipientStatus when needed private int dataWait; private SmtpResponseDescriptor replyDescriptor; //xxx does this need to be a global? private short disconnect_status; @@ -467,6 +469,20 @@ private void endConnection(CharSequence discmsg, ByteArrayRef failmsg) { } catch (Exception ex) { getLogger().log(LEVEL.WARN, ex, false, pfx_log+" failed to set final recipients status"); } + + if (smtpMessage != null) { + //xxx need to set errorResponse and some diagnostics + SmtpSendResult result = SmtpSendResult.builder() + .withMessage(smtpMessage) + .withMessageCount(msgcnt) + .withRecipientStatus(recipientStatus) + .withRemoteAddress(remote_tsap) + .withErrorDiagnostic("state="+pstate) + .build(); + smtpSender.messageCompleted(result); + //xxx need to indicate they can't give us another msg, and maybe throw if they do + } + if (remote_tsap != null && !isFlagSet(S2_CNXLIMIT)) shared.decrementServerConnections(remote_tsap.ip, conncfg); remote_tsap_buf.clear(); @@ -780,14 +796,14 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newState, int o } else { dataErrMsg = "Invalid message data supplied - "+(msgdata == null ? null : msgdata.getClass().getName()); } - } catch (IOException ex) { + } catch (Throwable ex) { if (msgdata instanceof Path && !Files.exists((Path)msgdata)) { dataErrMsg = "Spool file missing"; } else { return issueDisconnect(Protocol.REPLYCODE_TMPERR_LOCAL, "IO error "+ex.getMessage(), FAILMSG_LOCAL); } } - if (dataErrMsg != null) //xxx need to return this error to caller + if (dataErrMsg != null) //xxx need to return this error to caller, maybe an SmtpSendResult.localError field return issueDisconnect(Protocol.REPLYCODE_PERMERR_MISC, dataErrMsg, FAILMSG_LOCAL); transmit(Protocol.EOL_BC); @@ -804,7 +820,14 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newState, int o break; case A_ENDMESSAGE: - smtpMessage = smtpSender.messageCompleted(smtpMessage, ++msgcnt); + msgcnt++; + SmtpSendResult result = SmtpSendResult.builder() + .withMessage(smtpMessage) + .withMessageCount(msgcnt) + .withRecipientStatus(recipientStatus) + .withRemoteAddress(remote_tsap) + .build(); + smtpMessage = smtpSender.messageCompleted(result); if (smtpMessage == null) { return issueAction(PROTO_ACTION.A_QUIT, PROTO_STATE.S_QUIT); } @@ -953,7 +976,6 @@ private void setRecipientStatus(int idx, SmtpResponseDescriptor reply) { if (reply.smtpStatus() == Protocol.REPLYCODE_OK) okrecips++; recipientStatus.add(reply); } - smtpSender.recipientCompleted(smtpMessage, msgcnt, idx, reply, remote_tsap, isFlagSet(S2_ABORT)); } // NB: We can use shared.pipebuf because the pipeline is always built up and sent within one callback diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java index 8f8135c..47b3895 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java @@ -41,6 +41,7 @@ import com.grey.mailismus.mta.Protocol; import com.grey.mailismus.mta.deliver.client.SmtpMessage; import com.grey.mailismus.mta.deliver.client.SmtpResponseDescriptor; +import com.grey.mailismus.mta.deliver.client.SmtpSendResult; import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.mta.queue.Cache; import com.grey.mailismus.mta.queue.MessageRecip; @@ -208,15 +209,14 @@ public Forwarder(Dispatcher d, XmlConfig cfg, AppConfig appConfig, batchStats = new DeliveryStats(dsptch); openStats = new DeliveryStats(dsptch); - sendersEventListener = new SenderReaper(this); + XmlConfig smtpcfg = cfg.getSection("client"); + sharedFields = ClientConfiguration.createSharedFields(smtpcfg, dsptch, dnsResolver, appConfig, max_serverconns); if (senderFactory == null) { - XmlConfig smtpcfg = cfg.getSection("client"); - sharedFields = ClientConfiguration.createSharedFields(smtpcfg, dsptch, dnsResolver, appConfig, max_serverconns); - senderFactory = () -> new Client(sharedFields, dsptch); - } else { // sender-factory is only supplied in some test modes, never in production mode - sharedFields = null; + senderFactory = () -> new Client(sharedFields, dsptch); } + + sendersEventListener = new SenderReaper(this); sparesenders = new ObjectWell<>(senderFactory, "SmtpFwdSenders"); spareMessageRequests = new ObjectWell<>(() -> new QueueBasedMessage(qmgr), "SmtpFwdMessageReqs"); active_serverconns = (max_serverconns == 0 ? null : new HashedMapIntValue<>()); @@ -510,10 +510,15 @@ private void startSender(Client sender) // This method processes the result of a message send, and attempts to generate another msg for same destination. // Returns new message request if we found something to send, else null. @Override - public SmtpMessage messageCompleted(SmtpMessage smtpmsg, int msgcnt) { - QueueBasedMessage msgparams = (QueueBasedMessage)smtpmsg; + public SmtpMessage messageCompleted(SmtpSendResult msgResult) { + QueueBasedMessage msgparams = (QueueBasedMessage)msgResult.getMessage(); Client sender = msgparams.getClient(); int domainError = sender.getDomainError(); + int msgcnt = msgResult.getMessageCount(); + + for (int idx = 0; idx != msgResult.getRecipientStatus().size(); idx++) { + recipientCompleted(msgparams, idx, msgResult.getRecipientStatus().get(idx), msgResult.getRemoteAddress()); + } long time1 = dsptch.getRealTime(); boolean active = recordMessageResult(msgparams, msgcnt, domainError); @@ -557,27 +562,12 @@ public SmtpMessage messageCompleted(SmtpMessage smtpmsg, int msgcnt) { return msgparams; } - @Override - public void onDisconnect(SmtpMessage smtpmsg, int msgcnt) { - if (smtpmsg != null && smtpmsg.getRecipients() != null && !smtpmsg.getRecipients().isEmpty()) { - messageCompleted(smtpmsg, msgcnt); - } - } - private void senderCompleted(Client sender, boolean aborted) { - QueueBasedMessage msg = activesenders.get(sender); - int msgcnt = -1; - if (msg != null) { - msgcnt = msg.messageCount(); - messageCompleted(msg, msgcnt); - } - LEVEL lvl = LEVEL.TRC2; if (dsptch.getLogger().isActive(lvl)) { tmpsb.setLength(0); tmpsb.append("SMTP-Delivery/batch=").append(batchcnt).append(": Sender=").append(sender.getLogID()); tmpsb.append(" has ").append(aborted?"aborted":"completed"); - tmpsb.append(" with msgcnt=").append(msgcnt); tmpsb.append(" - active-conns=").append(activeSendersCount()).append(", pending-recips=").append(pending_recips); tmpsb.append("/scanning=").append(inScan); dsptch.getLogger().log(lvl, tmpsb); @@ -585,20 +575,10 @@ private void senderCompleted(Client sender, boolean aborted) { sparesenders.store(sender); } - @Override - public boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, SmtpResponseDescriptor status, TSAP remote, boolean aborted) { - System.out.println("xxx Forwarder.recipientCompleted-"+((QueueBasedMessage)msg).getClient()+": recip="+msgCount+":"+recipId+"/"+status+" on "+remote+" with aborted="+aborted+" - msg="+msg); + private void recipientCompleted(SmtpMessage msg, int recipId, SmtpResponseDescriptor status, TSAP remote) { QueueBasedRecipient recip = (QueueBasedRecipient)msg.getRecipients().get(recipId); MessageRecip qrecip = recip.getQueueRecip(); - if (aborted) { - // Any recipients who've already failed remain as failures, but recipients who had been marked as OK revert to - // an unprocessed status, as we've clearly been interrupted before completing the message send. - if (status.smtpStatus() == Protocol.REPLYCODE_OK) { - qrecip.qstatus = MessageRecip.STATUS_READY; - } - return false; - } qrecip.qstatus = MessageRecip.STATUS_DONE; qrecip.ip_send = (remote == null ? 0 : remote.ip); @@ -657,7 +637,6 @@ public boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, Sm } } } - return false; } private boolean recordMessageResult(QueueBasedMessage msgparams, int msgcnt, int domain_error) @@ -832,8 +811,8 @@ public CharSequence handleNAFManCommand(NafManCommand cmd) tmpsb.append("
SMTP Recipients: OK=").append(openStats.remotecnt-openStats.remotefailcnt).append("; Fail=").append(openStats.remotefailcnt); tmpsb.append("
Local Recipients: OK=").append(openStats.localcnt-openStats.localfailcnt).append("; Fail=").append(openStats.localfailcnt); tmpsb.append("
Current SMTP Connections: ").append(activeConnectionsCount()); - if (active_serverconns != null) tmpsb.append(" (Peers=").append(activeSendersCount()) - .append('/').append(active_serverconns == null ? 0 : active_serverconns.size()).append(')'); + if (active_serverconns != null) + tmpsb.append(" (Peers=").append(activeSendersCount()).append('/').append(active_serverconns.size()).append(')'); if (StringOps.stringAsBool(cmd.getArg(NafManCommand.ATTR_RESET))) openStats.reset(); } else if (cmd.getCommandDef().code.equals(Loader.CMD_SENDQ)) { if (tmr_qpoll != null) tmr_qpoll.reset(0); diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java index 0a746e8..eec5e27 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/QueueBasedMessage.java @@ -88,8 +88,10 @@ public void setClient(Client client) { @Override public String toString() { - return getClass().getSimpleName()+"[msg="+msgcnt+": sender="+sender - +" => "+destdomain+"/"+(recips==null?null:recips.size()+"/"+recips) - + " - relay="+relay+", spid="+spid+" - sender="+client+"]"; + return getClass().getSimpleName()+"[msg="+msgcnt+", spid="+spid + +": sender="+sender+" => "+destdomain+"/"+recips.size()+"/"+recips + +" - relay="+relay + +" - client="+client + +"]"; } } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java index 65362aa..b0c306f 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessage.java @@ -15,5 +15,8 @@ public interface SmtpMessage { // This is only used for logging, and is something meaningful to the caller. Maybe the SMTP message-ID, but not necessarily. CharSequence getMessageId(); + // continue on to send the message, as long as at least one recipient is accepted + //xxx boolean allowBadRecipients(); + static SmtpMessageDefaultImpl.Builder builder() {return SmtpMessageDefaultImpl.builder();} } diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java index 6894358..c0e3ea4 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpMessageDefaultImpl.java @@ -46,7 +46,7 @@ public String toString() { return getClass().getSimpleName()+"[" +"msgId=" + msgId +", sender=" + sender - +", recipients=" + recipients + +", recipients=" + (recipients==null ? null : recipients.size()+"/"+recipients) +", data=" + data +"]"; } @@ -57,7 +57,7 @@ public static Builder builder() { public static class Builder { - private List recipients = new ArrayList<>(); + private final List recipients = new ArrayList<>(); private CharSequence sender; private Supplier data; diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java index 50dd860..4235157 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java @@ -73,7 +73,7 @@ public String message() { @Override public String toString() { - return "SmtpResponse[smtp_status="+smtp_status+", enhanced="+enhanced_status+ ", msg="+msg+"]"; + return "SmtpResponse[smtp_status="+smtp_status+", enhanced="+enhanced_status+", msg="+msg+"]"; } private static int parseDecimal(int numDigits, byte[] buf, int off) { diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSendResult.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSendResult.java new file mode 100644 index 0000000..2945353 --- /dev/null +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSendResult.java @@ -0,0 +1,120 @@ +/* + * Copyright 2024 Yusef Badri - All rights reserved. + * Mailismus is distributed under the terms of the GNU Affero General Public License, Version 3 (AGPLv3). + */ +package com.grey.mailismus.mta.deliver.client; + +import java.util.ArrayList; +import java.util.List; + +import com.grey.base.utils.TSAP; + +public class SmtpSendResult { + //xxx do we need errorResponse if recipient status has error? + private final int messageCount; //order of this message on a connection where multiple sent - becomes 1 on sending 1st msg + private final SmtpMessage message; + private final List recipientStatus; + private final SmtpResponseDescriptor errorResponse; //non-null means message failed on this response + private final String errorDiagnostic; //qualifies errorResponse, indicates what stage we failed at + private final TSAP remoteAddress; + + private SmtpSendResult(Builder bldr) { + this.messageCount = bldr.messageCount; + this.message = bldr.message; + this.recipientStatus = bldr.recipientStatus; + this.errorResponse = bldr.errorResponse; + this.errorDiagnostic = bldr.errorDiagnostic; + this.remoteAddress = bldr.remoteAddress; + } + + public int getMessageCount() { + return messageCount; + } + + public SmtpMessage getMessage() { + return message; + } + + public List getRecipientStatus() { + return recipientStatus; + } + + public SmtpResponseDescriptor getErrorResponse() { + return errorResponse; + } + + public String getErrorDiagnostic() { + return errorDiagnostic; + } + + public TSAP getRemoteAddress() { + return remoteAddress; + } + + @Override + public String toString() { + return "SmtpSendResult[" + +"messageCount=" + messageCount + +", message="+ message + +", recipientStatus=" + (recipientStatus==null ? null : recipientStatus.size()+"/"+recipientStatus) + +", errorResponse=" + errorResponse + +", errorDiagnostic="+ errorDiagnostic + +", remoteAddress=" + remoteAddress + +"]"; + } + + public static Builder builder() { + return new Builder(); + } + + + public static class Builder { + private final List recipientStatus = new ArrayList<>(); + private int messageCount; + private SmtpMessage message; + private SmtpResponseDescriptor errorResponse; + private String errorDiagnostic; + private TSAP remoteAddress; + + private Builder() {} + + public Builder withMessageCount(int messageCount) { + this.messageCount = messageCount; + return this; + } + + public Builder withMessage(SmtpMessage message) { + this.message = message; + return this; + } + + public Builder withRecipientStatus(SmtpResponseDescriptor recipientStatus) { + this.recipientStatus.add(recipientStatus); + return this; + } + + public Builder withRecipientStatus(List recipientStatus) { + this.recipientStatus.addAll(recipientStatus); + return this; + } + + public Builder withErrorResponse(SmtpResponseDescriptor errorResponse) { + this.errorResponse = errorResponse; + return this; + } + + public Builder withErrorDiagnostic(String errorDiagnostic) { + this.errorDiagnostic = errorDiagnostic; + return this; + } + + public Builder withRemoteAddress(TSAP remoteAddress) { + this.remoteAddress = remoteAddress; + return this; + } + + public SmtpSendResult build() { + return new SmtpSendResult(this); + } + } +} diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSender.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSender.java index 62d2ffe..b752370 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSender.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSender.java @@ -4,31 +4,10 @@ */ package com.grey.mailismus.mta.deliver.client; -import com.grey.base.utils.TSAP; - public interface SmtpSender { /** - * This is called after receiving the response to each individual recipient send (SMTP "RCPT TO" command) - * recipId is an index into SmtpMessage.getRecipients() - * Returns true to indicate that message processing should abort, else false - */ - default boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, SmtpResponseDescriptor status, TSAP remote, boolean aborted) { - return false; - } - - /** - * This is called after receiving the response to the message body (SMTP "DATA" command) - * recipId is an index into SmtpMessage.getRecipients() + * This is called after receiving the response to the message body (following the SMTP "DATA" command) * Returns non-null to indicate a new message to be sent on this connection. */ - default SmtpMessage messageCompleted(SmtpMessage msg, int msgCount) { - return null; - } - - /** - * This is called after the SMTP client disconnects. - * Note that if neither this nor onMessage() is set, the sender receives no feedback on the message delivery. - */ - default void onDisconnect(SmtpMessage finalMsg, int msgCount) { - } + SmtpMessage messageCompleted(SmtpSendResult result); } diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java index 480e4a4..d7df2fd 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java @@ -4,6 +4,11 @@ */ package com.grey.mailismus.mta.deliver; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -12,26 +17,34 @@ import java.io.PrintWriter; import java.net.Socket; import java.security.GeneralSecurityException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import com.grey.mailismus.mta.deliver.client.ConnectionConfig; import com.grey.mailismus.mta.deliver.client.SmtpMessage; import com.grey.mailismus.mta.deliver.client.SmtpRelay; -import com.grey.mailismus.mta.deliver.client.SmtpResponseDescriptor; +import com.grey.mailismus.mta.deliver.client.SmtpSendResult; import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.mta.testsupport.MockSmtpServer; +import com.grey.naf.ApplicationContextNAF; import com.grey.naf.BufferGenerator; +import com.grey.naf.EventListenerNAF; +import com.grey.naf.reactor.ChannelMonitor; import com.grey.naf.reactor.Dispatcher; import com.grey.naf.reactor.DispatcherRunnable; +import com.grey.naf.reactor.TimerNAF; import com.grey.naf.reactor.config.DispatcherConfig; import com.grey.base.utils.TSAP; import com.grey.base.utils.TimeOps; +import com.grey.logging.Logger; import org.junit.After; import org.junit.Before; import org.junit.Test; -/*xxxx Need to test: HELO, EHLO, EHLO rejected, SASL, STLS, some/all recips rejected, msg rejected, quit error, bad greeting, connection-loss in mid session +/*xxxx Need to test: SASL, STLS, some/all recips rejected, msg rejected, quit error, connection-loss in mid session */ public class ClientTest { private MockSmtpServer smtpServer; @@ -50,42 +63,159 @@ public void teardown() throws Exception { } } - //xxx @Test + @Test public void testSanity() throws Exception { - smtpServer.configure(null, "250 ok", "250 msg o", "220 Greetings"); + smtpServer.configure(null, "Greetings", "msg ok", "default rsp"); Socket sock = smtpServer.connect(); OutputStream ostrm = sock.getOutputStream(); PrintWriter sendStream = new PrintWriter(ostrm, true); InputStream istrm = sock.getInputStream(); InputStreamReader rdr = new InputStreamReader(istrm); BufferedReader recvStream = new BufferedReader(rdr); + + assertEquals("Greetings", recvStream.readLine()); sendStream.println("Hello I am the client"); - String s = recvStream.readLine(); - System.out.println("Client received ["+s+"]"); + assertEquals("default rsp", recvStream.readLine()); sock.close(); - Thread.sleep(500); } - //xxx @Test + @Test public void testSuccessfulSend() throws Exception { + verifySuccessfulSend(true, null, true); + verifySuccessfulSend(true, "announce-host1", true); + verifySuccessfulSend(false, null, false); + verifySuccessfulSend(false, "announce-host1", false); + verifySuccessfulSend(false, null, true); + verifySuccessfulSend(false, "announce-host1", true); + } + + @Test + public void testBadGreeting() throws Exception { + verifyBadGreeting(true); + verifyBadGreeting(false); + } + + @Test + public void testHeloRejected() throws Exception { + verifyHeloRejected(true, false); + verifyHeloRejected(true, true); + verifyHeloRejected(false, false); + verifyHeloRejected(false, true); + } + + private void verifySuccessfulSend(boolean sayHelo, String announceHost, boolean rejectEhlo) throws IOException, GeneralSecurityException { + ConnectionConfig conncfg = ConnectionConfig.builder() + .withSayHelo(sayHelo) + .withAnnouncehost(announceHost) + .withFallbackHelo(rejectEhlo) + .build(); + SmtpMessage msg = SmtpMessage.builder() + .withSender("sender1@srcdom1.com") + .withRecipient("recip1@dstdom1.com") + .withRecipient("recip2@dstdom1.com") + .withData(() -> "test msgbody1") + .build(); + + Map sequences = new HashMap<>(); + String heloName = (announceHost==null?"":" "+announceHost); + if (sayHelo) { + sequences.put("HELO"+heloName, "250 hello to you too"); + } else { + if (rejectEhlo) { + sequences.put("HELO"+heloName, "250 hello to you too"); + } else { + sequences.put("EHLO"+heloName, "250 hello to you too"); + } + } + sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); + sequences.put("DATA", "354 start body"); + sequences.put("QUIT", "250 quit ok"); + for (CharSequence recip : msg.getRecipients()) { + sequences.put("RCPT TO:<"+recip+">", "250 recip ok"); + } + smtpServer.configure(sequences, "220 Greetings", "250 msg ok", "550 bad"); + + TestSmtpSender sender = sendMessage(msg, conncfg); + List results = sender.getResults(); + assertEquals(1, results.size()); + SmtpSendResult result = results.get(0); + assertEquals(1, result.getMessageCount()); + assertSame(msg, result.getMessage()); + assertNull(result.getErrorResponse()); + assertEquals(2, result.getRecipientStatus().size()); + assertEquals(250, result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(250, result.getRecipientStatus().get(1).smtpStatus()); + assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); + } + + private void verifyBadGreeting(boolean sayHelo) throws IOException, GeneralSecurityException { + ConnectionConfig conncfg = ConnectionConfig.builder() + .withSayHelo(sayHelo) + .build(); + SmtpMessage msg = SmtpMessage.builder() + .withSender("sender1@srcdom1.com") + .withRecipient("recip1@dstdom1.com") + .withData(() -> "test msgbody1") + .build(); + + Map sequences = new HashMap<>(); + sequences.put("HELO", "250 hello to you too"); + sequences.put("EHLO", "250 hello to you too"); + sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); + sequences.put("DATA", "354 start body"); + sequences.put("QUIT", "250 quit ok"); + for (CharSequence recip : msg.getRecipients()) { + sequences.put("RCPT TO:<"+recip+">", "250 recip ok"); + } + smtpServer.configure(sequences, "421 Go away", "250 msg ok", "550 bad"); + + TestSmtpSender sender = sendMessage(msg, conncfg); + List results = sender.getResults(); + assertEquals(1, results.size()); + SmtpSendResult result = results.get(0); + assertEquals(0, result.getMessageCount()); + assertSame(msg, result.getMessage()); + //xxx check errorResponse + assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_READY")); + assertEquals(1, result.getRecipientStatus().size()); + assertEquals(421, result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); + } + + private void verifyHeloRejected(boolean sayHelo, boolean fallbackHelo) throws IOException, GeneralSecurityException { + ConnectionConfig conncfg = ConnectionConfig.builder() + .withSayHelo(sayHelo) + .withFallbackHelo(fallbackHelo) + .build(); SmtpMessage msg = SmtpMessage.builder() .withSender("sender1@srcdom1.com") .withRecipient("recip1@dstdom1.com") .withData(() -> "test msgbody1") .build(); + Map sequences = new HashMap<>(); - sequences.put("HELO", "220 server.com ESMTP"); sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); sequences.put("DATA", "354 start body"); sequences.put("QUIT", "250 quit ok"); for (CharSequence recip : msg.getRecipients()) { - sequences.put("RCPT TO: <"+recip+">", "250 recip ok"); + sequences.put("RCPT TO:<"+recip+">", "250 recip ok"); } - smtpServer.configure(sequences, "220 Greetings", "250 msg ok", "250 ok"); - sendMessage(msg); + smtpServer.configure(sequences, "220 Greetings", "250 msg ok", "550 bad"); + + TestSmtpSender sender = sendMessage(msg, conncfg); + List results = sender.getResults(); + assertEquals(1, results.size()); + SmtpSendResult result = results.get(0); + assertEquals(0, result.getMessageCount()); + assertSame(msg, result.getMessage()); + //xxx check errorResponse + assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_"+(sayHelo?"HELO":(fallbackHelo?"HELO":"EHLO")))); + assertEquals(1, result.getRecipientStatus().size()); + assertEquals(550, result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); } - private void sendMessage(SmtpMessage msg) throws IOException, GeneralSecurityException { + private TestSmtpSender sendMessage(SmtpMessage msg, ConnectionConfig conncfg) throws IOException, GeneralSecurityException { BufferGenerator.BufferConfig bufcfg = new BufferGenerator.BufferConfig(256, true, null, null); BufferGenerator bufgen = new BufferGenerator(bufcfg); TSAP remote = TSAP.build("127.0.0.1", smtpServer.getServicePort()); @@ -95,8 +225,10 @@ private void sendMessage(SmtpMessage msg) throws IOException, GeneralSecurityExc .withSurviveHandlers(false) .build(); Dispatcher dsptch = Dispatcher.create(def); + dsptch.getLogger().setLevel(Logger.LEVEL.TRC5); SharedFields shared = SharedFields.builder() + .withDefaultConfig(conncfg) .withBufferGenerator(bufgen) .build(); SmtpRelay relay = SmtpRelay.builder() @@ -106,14 +238,22 @@ private void sendMessage(SmtpMessage msg) throws IOException, GeneralSecurityExc Client clnt = new Client(shared, dsptch); TestSmtpSender sender = new TestSmtpSender(dsptch, clnt, msg, relay); + TestSupervisor supervisor = new TestSupervisor(); + clnt.setEventListener(supervisor); + dsptch.loadRunnable(sender); dsptch.start(); Dispatcher.STOPSTATUS stopsts = dsptch.waitStopped(TimeOps.MSECS_PER_SECOND*10L, true); - System.out.println("xxx client="+clnt+", relay="+relay+", stopsts="+stopsts); + assertEquals(Dispatcher.STOPSTATUS.STOPPED, stopsts); + assertTrue(dsptch.completedOK()); + assertEquals(0, shared.getActiveServerConnections()); + ApplicationContextNAF.unregister(dsptch.getApplicationContext()); + return sender; } private static class TestSmtpSender implements SmtpSender, DispatcherRunnable { + private final List results = new ArrayList<>(); private final Dispatcher dsptch; private final Client clnt; private final SmtpMessage msg; @@ -126,20 +266,13 @@ public TestSmtpSender(Dispatcher dsptch, Client clnt, SmtpMessage msg, SmtpRelay this.relay = relay; } @Override - public boolean recipientCompleted(SmtpMessage msg, int msgCount, int recipId, SmtpResponseDescriptor status, TSAP remote, boolean aborted) { - System.out.println("xxx TestSmtpSender.recipientCompleted: recip="+msgCount+":"+recipId+"/"+status+" on "+remote+" with aborted="+aborted+" - msg="+msg); - return false; - } - @Override - public SmtpMessage messageCompleted(SmtpMessage msg, int msgCount) { - System.out.println("xxx TestSmtpSender.messageCompleted: msg="+msgCount+"/"+msg); + public SmtpMessage messageCompleted(SmtpSendResult result) { + synchronized (results) { + results.add(result); + } return null; } @Override - public void onDisconnect(SmtpMessage msg, int msgCount) { - System.out.println("xxx TestSmtpSender.onDisconnect: msg="+msgCount+"/"+msg); - } - @Override public String getName() { return getClass().getName(); } @@ -151,5 +284,24 @@ public Dispatcher getDispatcher() { public void startDispatcherRunnable() throws IOException { clnt.startConnection(msg, this, relay); } + public List getResults() { + synchronized (results) { + return results; + } + } + } + + private static class TestSupervisor implements EventListenerNAF, TimerNAF.Handler { + @Override + public void eventIndication(String eventId, Object eventSource, Object eventData) { + if (eventSource instanceof Client && ChannelMonitor.EVENTID_CM_DISCONNECTED.equals(eventId)) { + Dispatcher d = ((Client)eventSource).getDispatcher(); + d.setTimer(100, 1, this); + } + } + @Override + public void timerIndication(TimerNAF tmr, Dispatcher d) { + d.stop(); + } } } \ No newline at end of file diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java index d7c7ede..43ee6da 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ForwarderTest.java @@ -10,6 +10,7 @@ import com.grey.base.config.XmlConfig; import com.grey.base.utils.ByteChars; import com.grey.base.utils.TimeOps; +import com.grey.logging.Logger; import com.grey.base.utils.EmailAddress; import com.grey.base.utils.FileOps; import com.grey.base.utils.IP; @@ -27,6 +28,7 @@ import com.grey.mailismus.mta.Protocol; import com.grey.mailismus.mta.deliver.client.SmtpMessage; import com.grey.mailismus.mta.deliver.client.SmtpRelay; +import com.grey.mailismus.mta.deliver.client.SmtpSendResult; import com.grey.mailismus.mta.deliver.client.SmtpSender; import com.grey.mailismus.mta.queue.MessageRecip; import com.grey.mailismus.mta.testsupport.MockServerDNS; @@ -124,6 +126,7 @@ public void setup() throws java.io.IOException { .withAppContext(appctx) .build(); dsptch = Dispatcher.create(def); + dsptch.getLogger().setLevel(Logger.LEVEL.TRC5); dsptch.registerEventListener(this); ResolverConfig rcfg = new ResolverConfig.Builder() .withXmlConfig(nafcfg.getNode("dnsresolver")) @@ -139,7 +142,7 @@ public void testRealClient_PermErr() throws Exception { //Add some more messages for final domain that will be leftover due to maxconnections=3 but will still get marked as //done and failed, as the error for the recips we did try was domain-wide. java.util.List rdlst = java.util.Arrays.asList(msgs1); - java.util.ArrayList lst = new java.util.ArrayList(rdlst); //asList() returns a read-only list + java.util.ArrayList lst = new java.util.ArrayList<>(rdlst); //asList() returns a read-only list lst.add(new String[]{"sender1", "recip11@domain2", "201", "1", "192.168.101.1"}); lst.add(new String[]{"sender1", "recip12@domain2", "202", "1", "192.168.101.1"}); String[][] msgs = lst.toArray(new String[lst.size()][]); @@ -154,9 +157,10 @@ public void testSmarthost() throws Exception { int[][][] expected_senders = new int[][][]{{{101, 1}, {101, 2}, {101, 3}}, {{102, 1}}}; String cfgxml = delivxml.replace("x1relay", "relay"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs1, true); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, null, false); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); sndrfact.fwd = fwd; exec(qmgr, sndrfact, true); } @@ -167,9 +171,10 @@ public void testSenderRefill() throws Exception { int[][][] expected_refills = new int[][][]{null, {{103, 1}}, null}; String cfgxml = delivxml.replace("x2maxconnections", "maxconnections"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs2, true); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, expected_refills, true); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -181,9 +186,10 @@ public void testLeftoverMessages() throws Exception { int[][][] expected_senders = new int[][][]{{{101, 1}, {101, 3}}, {{102, 1}}}; String cfgxml = delivxml.replace("x2maxconnections", "maxconnections"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs3, true, 1); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, null, true); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -197,9 +203,10 @@ public void testServerConnections() throws Exception { int[][][] expected_refills = new int[][][]{null, {{103, 1}}}; String cfgxml = delivxml.replace("x2maxserverconnections", "maxserverconnections"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs4, true, 0); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, expected_refills, true); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -210,9 +217,10 @@ public void testMaxServerConnectionsWithSourceRoute() throws Exception { int[][][] expected_senders = new int[][][]{{{101, 1}}, {{102, 1}, {102,2}}, {{103, 1}}}; String cfgxml = delivxml.replace("x2maxserverconnections", "maxserverconnections").replace("x2relay","relay"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs4, true, 0); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, null, true); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -223,9 +231,10 @@ public void testSourceRouteRefill() throws Exception { int[][][] expected_refills = new int[][][]{{{104, 1}}, {{103, 1}, {103, 2}}}; String cfgxml = delivxml.replace("x2maxserverconnections", "maxserverconnections").replace("x2relay","relay"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs5, true, 0); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, expected_refills, true); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -238,7 +247,7 @@ public void testBulk() throws Exception { int destcnt = 5; int maxserverconns = cfg.getInt("maxserverconnections", true, 0); org.junit.Assert.assertEquals(50, maxserverconns); - java.util.ArrayList lst = new java.util.ArrayList(); + java.util.ArrayList lst = new java.util.ArrayList<>(); int spid = 0; for (int loop = 0; loop != 500; loop++) { //5x500 matches the expected cache capacity for (int dom = 1; dom <= destcnt; dom++) { @@ -247,9 +256,10 @@ public void testBulk() throws Exception { } } String[][] msgs = lst.toArray(new String[lst.size()][]); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs, true); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, null, null, true); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); com.grey.mailismus.mta.queue.Cache qc = (com.grey.mailismus.mta.queue.Cache)DynLoader.getField(fwd, "qcache"); org.junit.Assert.assertEquals(2500, qc.capacity()); //the default for non-slaverelay mode sndrfact.fwd = fwd; @@ -267,7 +277,7 @@ public void testBulkMultiRecips() throws Exception { int destcnt = 5; int maxserverconns = cfg.getInt("maxserverconnections", true, 0); org.junit.Assert.assertEquals(50, maxserverconns); - java.util.ArrayList lst = new java.util.ArrayList(); + java.util.ArrayList lst = new java.util.ArrayList<>(); int spid = 0; for (int loop = 0; loop != 250; loop++) { //5x250x2 matches the expected cache capacity for (int dom = 1; dom <= destcnt; dom++) { @@ -281,9 +291,10 @@ public void testBulkMultiRecips() throws Exception { } } String[][] msgs = lst.toArray(new String[lst.size()][]); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs, true); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, null, null, true); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); org.junit.Assert.assertEquals(maxserverconns * destcnt, sndrfact.senders.size()); @@ -297,7 +308,7 @@ public void testBulkSmarthost() throws Exception { String cfgxml = delivxml.replace("x1relay", "relay"); XmlConfig cfg = XmlConfig.makeSection(cfgxml, "deliver"); int destcnt = 5; - java.util.ArrayList lst = new java.util.ArrayList(); + java.util.ArrayList lst = new java.util.ArrayList<>(); int spid = 0; for (int loop = 0; loop != 1000; loop++) { //5x1000 matches the expected cache capacity for (int dom = 1; dom <= destcnt; dom++) { @@ -306,9 +317,10 @@ public void testBulkSmarthost() throws Exception { } } String[][] msgs = lst.toArray(new String[lst.size()][]); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs, true); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, null, null, true); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); com.grey.mailismus.mta.queue.Cache qc = (com.grey.mailismus.mta.queue.Cache)DynLoader.getField(fwd, "qcache"); org.junit.Assert.assertEquals(5000, qc.capacity()); //the default for slaverelay mode sndrfact.fwd = fwd; @@ -332,9 +344,10 @@ public void testSenderAllocation_Deferred() throws Exception { private void testSenderAllocation(boolean deferred) throws Exception { int[][][] expected_senders = new int[][][]{{{101, 1}, {101, 3}}, {{101, 2}}, {{102, 1}}}; XmlConfig cfg = XmlConfig.makeSection(delivxml, "deliver"); + AppConfig appcfg = AppConfig.get(null, dsptch); MyQueueManager qmgr = new MyQueueManager(dsptch, msgs1, true); SenderFactory sndrfact = new SenderFactory(dsptch, qmgr, expected_senders, null, deferred); - fwd = new Forwarder(dsptch, cfg, null, qmgr, null, this, sndrfact, sndrfact, dnsResolver); + fwd = new Forwarder(dsptch, cfg, appcfg, qmgr, null, this, sndrfact, sndrfact, dnsResolver); sndrfact.fwd = fwd; exec(qmgr, sndrfact, false); } @@ -513,8 +526,14 @@ private void processMessage(int[][] exp) { private void reportCompletion(boolean report_msg) { if (report_msg) { int[][] exp = null; - if (mgr.expected_refills != null && msgcnt == 1 && seqno < mgr.expected_refills.length) exp = mgr.expected_refills[seqno]; - mgr.fwd.messageCompleted(msgparams, msgcnt); + if (mgr.expected_refills != null && msgcnt == 1 && seqno < mgr.expected_refills.length) { + exp = mgr.expected_refills[seqno]; + } + SmtpSendResult result = SmtpSendResult.builder() + .withMessage(msgparams) + .withMessageCount(msgcnt) + .build(); + mgr.fwd.messageCompleted(result); if (msgparams.recipCount() == 0) { if (exp != null) mgr.addError(seqno, "Expected another message after messageCompleted()"); } else { @@ -529,7 +548,6 @@ private void reportCompletion(boolean report_msg) { return; } } - mgr.fwd.onDisconnect(msgparams, msgcnt); mgr.sender_completion_cnt++; } @@ -546,7 +564,7 @@ public boolean stop() { public void timerIndication(TimerNAF tmr, Dispatcher d) { tmr_report = null; Boolean msg_only = (Boolean)tmr.getAttachment(); - reportCompletion(msg_only.booleanValue()); + reportCompletion(msg_only); } @Override @@ -563,7 +581,7 @@ private static class SenderFactory { private final Dispatcher dsptch; private final MyQueueManager qmgr; - public final java.util.ArrayList senders = new java.util.ArrayList(); + public final java.util.ArrayList senders = new java.util.ArrayList<>(); public final int[][][] expected_msgs; //expected messages, per MessageSender and per MessageRecip public final int[][][] expected_refills; //expected message after possible refill in messageCompleted() public final boolean deferred_cb; diff --git a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockServerDNS.java b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockServerDNS.java index ba109e6..59fca54 100644 --- a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockServerDNS.java +++ b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockServerDNS.java @@ -76,7 +76,7 @@ public void dnsResolveQuestion(int qid, byte qtype, ByteChars qn, boolean recurs private void store(int rrtype, String name, ResourceData[][] data) { HashedMap map = answers.get(rrtype); if (map == null) { - map = new HashedMap(); + map = new HashedMap<>(); answers.put(rrtype, map); } map.put(name, data); diff --git a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java index a84487e..cecfc26 100644 --- a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java +++ b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java @@ -39,7 +39,7 @@ public MockSmtpServer() throws IOException { } public void start() { - Runnable r = () -> runService(); + Runnable r = this::runService; Thread t = new Thread(r); t.start(); } @@ -82,7 +82,7 @@ private void handleConnection() throws IOException { String key = sock.getInetAddress().getHostAddress()+":"+sock.getPort(); List transcript = transcriptsByPort.computeIfAbsent(key, k-> new ArrayList<>()); SmtpSession session = new SmtpSession(sock, expectSendSequences, greetingResponse, messageResponse, defaultResponse, transcript); - Runnable r = () -> session.serveSession(); + Runnable r = session::serveSession; Thread t = new Thread(r); t.start(); } From b15ae03965bfed92351e59a9c2bb0aadd977ae8d Mon Sep 17 00:00:00 2001 From: ybadri Date: Fri, 5 Apr 2024 16:47:23 +0100 Subject: [PATCH 6/6] Completed Client tests on Mock SMTP server. Add support for response continuation to SmtpResponseDescriptor. --- .../grey/mailismus/mta/deliver/Client.java | 14 +- .../grey/mailismus/mta/deliver/Forwarder.java | 28 +- .../mta/deliver/client/SmtpRelay.java | 1 + .../client/SmtpResponseDescriptor.java | 52 ++- .../mta/deliver/client/SmtpSendResult.java | 2 +- .../mailismus/mta/deliver/ClientTest.java | 408 ++++++++++++++++-- .../client/SmtpResponseDescriptorTest.java | 92 +++- .../mta/testsupport/MockSmtpServer.java | 40 +- 8 files changed, 547 insertions(+), 90 deletions(-) diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java index be8b10d..3d8fe55 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Client.java @@ -71,7 +71,7 @@ private enum PROTO_ACTION {A_CONNECT, A_DISCONNECT, A_HELO, A_EHLO, A_MAILFROM, private static final int TMRTYPE_DISCON = 2; private static final byte S2_DNSWAIT = 1 << 0; - private static final byte S2_REPLYCONTD = 1 << 1; + private static final byte S2_REPLYCONTD = 1 << 1; //xxx can be replaced by SmtpStatusDescriptor.isContinued() private static final byte S2_DISCARD = 1 << 2; //discarding remainder of an excessively long reply (we're only interested in initial part) private static final byte S2_SENT_DATACMD = 1 << 3; private static final byte S2_DOMAIN_ERR = 1 << 4; //apply error status to entire domain, not just a particular message recipient @@ -160,7 +160,6 @@ public void ioReceived(ByteArrayRef rcvdata) throws IOException { if (pstate == PROTO_STATE.S_DISCON) return; //this method can be called in a loop, so skip it after a disconnect if (shared.getTranscript() != null) shared.getTranscript().data_in(pfx_log, rcvdata, getSystemTime()); alt_tmtprotocol = 0; - //xxx parse SmtpDescriptor here and issue disconnect if null, citing invalid response eventRaised(PROTO_EVENT.E_REPLY, rcvdata, null); } @@ -464,7 +463,7 @@ private void endConnection(CharSequence discmsg, ByteArrayRef failmsg) { try { if (failmsg == null && discmsg != null) failmsg = shared.getFailMsgBuffer().populate(discmsg); //xxx failmsg unused here, but used to get passed to setRecipientStatus() as failmsg - SmtpResponseDescriptor rsp = new SmtpResponseDescriptor(disconnect_status, null, null); + SmtpResponseDescriptor rsp = new SmtpResponseDescriptor(disconnect_status, null, false, null); setRecipientStatus(-1, rsp); } catch (Exception ex) { getLogger().log(LEVEL.WARN, ex, false, pfx_log+" failed to set final recipients status"); @@ -589,8 +588,11 @@ private PROTO_STATE handleReply(ByteArrayRef rspdata) throws IOException { // We don't expect continued replies for any command except EHLO, but they are always legal, so if we do receive a continued reply // in response to anything else then we discard the leading lines and wait for the final one, taking its reply code as the // definitive one. CORRECTION!! AOL sends a multi-line Greeting, so just as well we handle continued replies anywhere. + replyDescriptor = (rspdata == null ? null : SmtpResponseDescriptor.parse(rspdata, false)); //xxx need flag for enhanced-status + if (replyDescriptor == null) { + return issueDisconnect(0, "Invalid response - "+(rspdata==null?null:rspdata.toString(null)), null); + } setFlag(S2_REPLYCONTD, rspdata.byteAt(Protocol.REPLY_CODELEN) == Protocol.REPLY_CONTD); - replyDescriptor = SmtpResponseDescriptor.parse(rspdata, false); //xxx need flag for enhanced-status if (isFlagSet(S2_REPLYCONTD)) { if (pstate != PROTO_STATE.S_EHLO) return pstate; } else { @@ -722,7 +724,7 @@ private PROTO_STATE issueAction(PROTO_ACTION action, PROTO_STATE newState, int o break; case A_HELO: - auth_method = (active_relay == null ? null : active_relay.getAuthOverride()); + auth_method = (active_relay == null ? null : active_relay.getAuthOverride());//xxx can this be set in startConnection() ? reqbuf = shared.getHeloBuffer(conncfg, false); transmit(reqbuf); break; @@ -1013,7 +1015,7 @@ private void transmit(ByteBuffer xmtbuf) throws IOException { dataWait++; } - private boolean matchesExtension(ByteArrayRef data, char[] cmd, boolean with_equals) { + private boolean matchesExtension(ByteArrayRef data, char[] cmd, boolean with_equals) {//xxx could just check SmtpResponseDescriptor.message int off = Protocol.REPLY_CODELEN + 1; //+1 for the hyphen or space following the "250" int len = data.size() - off - Protocol.EOL.length(); if (len < cmd.length) return false; diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java index 47b3895..1881c2a 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/Forwarder.java @@ -507,6 +507,19 @@ private void startSender(Client sender) } } + private void senderCompleted(Client sender, boolean aborted) { + LEVEL lvl = LEVEL.TRC2; + if (dsptch.getLogger().isActive(lvl)) { + tmpsb.setLength(0); + tmpsb.append("SMTP-Delivery/batch=").append(batchcnt).append(": Sender=").append(sender.getLogID()); + tmpsb.append(" has ").append(aborted?"aborted":"completed"); + tmpsb.append(" - active-conns=").append(activeSendersCount()).append(", pending-recips=").append(pending_recips); + tmpsb.append("/scanning=").append(inScan); + dsptch.getLogger().log(lvl, tmpsb); + } + sparesenders.store(sender); + } + // This method processes the result of a message send, and attempts to generate another msg for same destination. // Returns new message request if we found something to send, else null. @Override @@ -562,24 +575,11 @@ public SmtpMessage messageCompleted(SmtpSendResult msgResult) { return msgparams; } - private void senderCompleted(Client sender, boolean aborted) { - LEVEL lvl = LEVEL.TRC2; - if (dsptch.getLogger().isActive(lvl)) { - tmpsb.setLength(0); - tmpsb.append("SMTP-Delivery/batch=").append(batchcnt).append(": Sender=").append(sender.getLogID()); - tmpsb.append(" has ").append(aborted?"aborted":"completed"); - tmpsb.append(" - active-conns=").append(activeSendersCount()).append(", pending-recips=").append(pending_recips); - tmpsb.append("/scanning=").append(inScan); - dsptch.getLogger().log(lvl, tmpsb); - } - sparesenders.store(sender); - } - private void recipientCompleted(SmtpMessage msg, int recipId, SmtpResponseDescriptor status, TSAP remote) { QueueBasedRecipient recip = (QueueBasedRecipient)msg.getRecipients().get(recipId); MessageRecip qrecip = recip.getQueueRecip(); - qrecip.qstatus = MessageRecip.STATUS_DONE; + qrecip.qstatus = MessageRecip.STATUS_DONE;//xxx should not be if cnx loss resulted in smtpstatus=0 qrecip.ip_send = (remote == null ? 0 : remote.ip); if (status.smtpStatus() > qrecip.smtp_status) { diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpRelay.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpRelay.java index b15db25..f418f64 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpRelay.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpRelay.java @@ -86,6 +86,7 @@ public String toString() { } + // This is only genericised to support subclassing, there are no generic types in here public static class Builder> { private CharSequence name; private TSAP address; diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java index 4235157..3c18767 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptor.java @@ -5,19 +5,24 @@ package com.grey.mailismus.mta.deliver.client; import com.grey.base.utils.ByteArrayRef; +import com.grey.mailismus.mta.Protocol; public class SmtpResponseDescriptor { private static final int STATUSCODE_DIGITS = 3; + private static final int STATUSCODE_INVALID = 9999; private final short smtp_status; //xxx convert to int and make the Protocol constants ints too private final String enhanced_status; private final String msg; + private final boolean continued; public static SmtpResponseDescriptor parse(ByteArrayRef rspdata, boolean withEnhanced) { StringBuilder sb = null; String enhanced = null; String msg = ""; + boolean contd = false; + // strip end-of-line chars (and any other trailing white space) int rsplen = rspdata.size(); while (rsplen != 0 && rspdata.buffer()[rspdata.offset() + rsplen - 1] <= ' ') { rsplen--; @@ -26,16 +31,32 @@ public static SmtpResponseDescriptor parse(ByteArrayRef rspdata, boolean withEnh if (rsplen < STATUSCODE_DIGITS) return null; + // parse the basic status code int off = rspdata.offset(); - short statuscode = (short)parseDecimal(STATUSCODE_DIGITS, rspdata.buffer(), off); + int statusCode = parseDecimal(STATUSCODE_DIGITS, rspdata.buffer(), off); + if (statusCode == STATUSCODE_INVALID) return null; off += STATUSCODE_DIGITS; + int statusClass = statusCode / 100; + int mark = off; + + // Skip past following spaces - I've seen GMail insert an extra space to represent the missing enhanced code for a 354 reply + while (off < rsplen) { + if (off == mark && rspdata.buffer()[off] == Protocol.REPLY_CONTD) { + contd = true; + withEnhanced = false; + } else if (rspdata.buffer()[off] != ' ') { + break; + } + off++; + } - off++; //skip space - //xxx could also be hyphen (see Google EHLO response) and return null if neither - if (off >= rsplen) - return new SmtpResponseDescriptor(statuscode, enhanced, msg); + if (off >= rsplen) //is just a bare status code + return new SmtpResponseDescriptor((short)statusCode, enhanced, contd, msg); + if (off == mark) { //there were extra chars at end of status code + return null; + } - if (withEnhanced) { + if (withEnhanced && statusClass != 3) { int off2 = off; while (rspdata.buffer()[off2] != ' ') { off2++; @@ -48,14 +69,15 @@ public static SmtpResponseDescriptor parse(ByteArrayRef rspdata, boolean withEnh } if (off <= rsplen) { - msg = rspdata.toString(sb, off, rsplen - off).toString(); + msg = rspdata.toString(sb, off, rsplen - off).toString().trim(); } - return new SmtpResponseDescriptor(statuscode, enhanced, msg); + return new SmtpResponseDescriptor((short)statusCode, enhanced, contd, msg); } - public SmtpResponseDescriptor(short smtp_status, String enhanced_status, String msg) { + public SmtpResponseDescriptor(short smtp_status, String enhanced_status, boolean continued, String msg) { this.smtp_status = smtp_status; this.enhanced_status = enhanced_status; + this.continued = continued; this.msg = msg; } @@ -71,9 +93,14 @@ public String message() { return msg; } + public boolean isContinued() { + return continued; + } + @Override public String toString() { - return "SmtpResponse[smtp_status="+smtp_status+", enhanced="+enhanced_status+", msg="+msg+"]"; + String contd = (isContinued() ? "/continued" : ""); + return "SmtpResponse[smtp_status="+smtp_status+contd+", enhanced="+enhanced_status+", msg="+msg+"]"; } private static int parseDecimal(int numDigits, byte[] buf, int off) { @@ -82,7 +109,10 @@ private static int parseDecimal(int numDigits, byte[] buf, int off) { int val = 0; for (int loop = 0; loop != numDigits; loop++) { - val += (buf[idx--] - '0') * factor; + int digit = buf[idx--] - '0'; + if (digit < 0 || digit > 9) + return STATUSCODE_INVALID; + val += (digit * factor); factor *= 10; } return val; diff --git a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSendResult.java b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSendResult.java index 2945353..eaf885c 100644 --- a/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSendResult.java +++ b/server/src/main/java/com/grey/mailismus/mta/deliver/client/SmtpSendResult.java @@ -10,7 +10,7 @@ import com.grey.base.utils.TSAP; public class SmtpSendResult { - //xxx do we need errorResponse if recipient status has error? + //xxx errorResponse unused, do we need it if recipient status has error? Maybe rename errorDiagnostic to diagnostic to be more general private final int messageCount; //order of this message on a connection where multiple sent - becomes 1 on sending 1st msg private final SmtpMessage message; private final List recipientStatus; diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java index d7df2fd..8b894ff 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/ClientTest.java @@ -17,6 +17,7 @@ import java.io.PrintWriter; import java.net.Socket; import java.security.GeneralSecurityException; +import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -36,16 +37,14 @@ import com.grey.naf.reactor.DispatcherRunnable; import com.grey.naf.reactor.TimerNAF; import com.grey.naf.reactor.config.DispatcherConfig; +import com.grey.base.sasl.SaslEntity; import com.grey.base.utils.TSAP; -import com.grey.base.utils.TimeOps; import com.grey.logging.Logger; import org.junit.After; import org.junit.Before; import org.junit.Test; -/*xxxx Need to test: SASL, STLS, some/all recips rejected, msg rejected, quit error, connection-loss in mid session - */ public class ClientTest { private MockSmtpServer smtpServer; @@ -58,7 +57,11 @@ public void setup() throws Exception { @After public void teardown() throws Exception { if (smtpServer != null) { - smtpServer.stop(); + try { + smtpServer.stop(); + } catch (Throwable ex) { + ex.printStackTrace(System.out); + } Thread.sleep(100); } } @@ -89,18 +92,139 @@ public void testSuccessfulSend() throws Exception { verifySuccessfulSend(false, "announce-host1", true); } + @Test + public void testPartialRecipients() throws Exception {//xxx need to test this with and without allowBadRecipients + verifyPartialRecipients(false); + verifyPartialRecipients(true); + } + + @Test + public void testEhloExtensions() throws Exception { + ConnectionConfig conncfg = ConnectionConfig.builder() + .build(); + SmtpMessage msg = SmtpMessage.builder() + .withSender("sender1@srcdom1.com") + .withRecipient("recip1@dstdom1.com") + .withRecipient("recip2@dstdom1.com") + .withData(() -> "test msgbody1") + .build(); + + Map sequences = new HashMap<>(); + sequences.put("EHLO", "250-hello to you too\r\n" + +"250-STARTTLS\r\n" + +"250-AUTH PLAIN LOGIN\r\n" + +"250-ENHANCEDSTATUSCODES\r\n" + +"250-PIPELINING\r\n" + +"250 8BITMIME"); + sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 2.1.0 sender ok"); + sequences.put("DATA", "354 start body"); + sequences.put("QUIT", "221 2.0.0 quit ok"); + for (CharSequence recip : msg.getRecipients()) { + sequences.put("RCPT TO:<"+recip+">", "250 2.1.5 recip ok"); + } + smtpServer.configure(sequences, "220 Greetings", "250 2.6.0 msg ok", "550 5.9.9 xbad"); + + TestSmtpSender sender = sendMessage(msg, conncfg, 0, null); + List results = sender.getResults(); + assertEquals(1, results.size()); + SmtpSendResult result = results.get(0); + assertEquals(1, result.getMessageCount()); + assertSame(msg, result.getMessage()); + assertNull(result.getErrorResponse()); + assertNull(result.getErrorDiagnostic()); + assertEquals(2, result.getRecipientStatus().size()); + assertEquals(250, result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(250, result.getRecipientStatus().get(1).smtpStatus()); + //xxx check enhanced status codes here once that is implemented + assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); + } + @Test public void testBadGreeting() throws Exception { - verifyBadGreeting(true); - verifyBadGreeting(false); + verifyHeloRejected(true, false, true); + verifyHeloRejected(false, false, true); } @Test public void testHeloRejected() throws Exception { - verifyHeloRejected(true, false); - verifyHeloRejected(true, true); - verifyHeloRejected(false, false); - verifyHeloRejected(false, true); + verifyHeloRejected(true, false, false); + verifyHeloRejected(true, true, false); + verifyHeloRejected(false, false, false); + verifyHeloRejected(false, true, false); + } + + @Test + public void testRejections() throws Exception { + verifyRejection("MAILFROM", false); + verifyRejection("DATA", false); + verifyRejection("MSGBODY", false); + verifyRejection("QUIT", false); + // repeat with unparseable invalid responses, rather than valid rejections + verifyRejection("MAILFROM", true); + verifyRejection("DATA", true); + verifyRejection("MSGBODY", true); + verifyRejection("QUIT", true); + } + + @Test + public void testConnectionFailure() throws Exception { + ConnectionConfig conncfg = ConnectionConfig.builder() + .build(); + SmtpMessage msg = SmtpMessage.builder() + .withSender("sender1@srcdom1.com") + .withRecipient("recip1@dstdom1.com") + .withData(() -> "test msgbody1") + .build(); + TestSmtpSender sender = sendMessage(msg, conncfg, smtpServer.getServicePort()+1, null); + List results = sender.getResults(); + assertEquals(1, results.size()); + SmtpSendResult result = results.get(0); + assertEquals(0, result.getMessageCount()); + assertSame(msg, result.getMessage()); + assertNull(result.getErrorResponse()); + assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_CONN")); + assertEquals(1, result.getRecipientStatus().size()); + assertEquals(421, result.getRecipientStatus().get(0).smtpStatus()); //xxx should really be a 49x in line with cnx loss and timeout + assertEquals(smtpServer.getServicePort()+1, result.getRemoteAddress().port); + } + + @Test + public void testConnectionLoss() throws Exception { + verifyConnectionLoss("GREET", false); + verifyConnectionLoss("HELO", false); + verifyConnectionLoss("DATA", false); + verifyConnectionLoss("MSGBODY", false); + verifyConnectionLoss("QUIT", false); + } + + @Test + public void testTimeouts() throws Exception { + verifyConnectionLoss("GREET", true); + verifyConnectionLoss("HELO", true); + verifyConnectionLoss("DATA", true); + verifyConnectionLoss("MSGBODY", true); + verifyConnectionLoss("QUIT", true); + } + + @Test + public void testAuth() throws Exception { + // test successful auth + verifyAuth(SaslEntity.MECH.PLAIN, false, false); + verifyAuth(SaslEntity.MECH.PLAIN, true, false); + verifyAuth(SaslEntity.MECH.CRAM_MD5, true, false);//verifies that CRAM-MD5 ignores initial-response setting + verifyAuth(null, false, false); //should default to SASL-Plain, given the ordering of the response + verifyAuth(null, true, false); + + // test unsuccessful auth + verifyAuth(SaslEntity.MECH.PLAIN, false, true); + verifyAuth(SaslEntity.MECH.PLAIN, true, true); + verifyAuth(SaslEntity.MECH.CRAM_MD5, true, true); + verifyAuth(null, false, true); + verifyAuth(null, true, true); + + // test unsupported server auth mechanisms - may fail or pass depending + verifyAuth(SaslEntity.MECH.PLAIN, true, null); + verifyAuth(null, true, null); } private void verifySuccessfulSend(boolean sayHelo, String announceHost, boolean rejectEhlo) throws IOException, GeneralSecurityException { @@ -129,63 +253,109 @@ private void verifySuccessfulSend(boolean sayHelo, String announceHost, boolean } sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); sequences.put("DATA", "354 start body"); - sequences.put("QUIT", "250 quit ok"); + sequences.put("QUIT", "221 quit ok"); for (CharSequence recip : msg.getRecipients()) { - sequences.put("RCPT TO:<"+recip+">", "250 recip ok"); + sequences.put("RCPT TO:<"+recip+">", "250 recip ok"); //xxx need to test 25x responses as well } smtpServer.configure(sequences, "220 Greetings", "250 msg ok", "550 bad"); - TestSmtpSender sender = sendMessage(msg, conncfg); + TestSmtpSender sender = sendMessage(msg, conncfg, 0, null); List results = sender.getResults(); assertEquals(1, results.size()); SmtpSendResult result = results.get(0); assertEquals(1, result.getMessageCount()); assertSame(msg, result.getMessage()); assertNull(result.getErrorResponse()); + assertNull(result.getErrorDiagnostic()); assertEquals(2, result.getRecipientStatus().size()); assertEquals(250, result.getRecipientStatus().get(0).smtpStatus()); assertEquals(250, result.getRecipientStatus().get(1).smtpStatus()); assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); } - private void verifyBadGreeting(boolean sayHelo) throws IOException, GeneralSecurityException { + private void verifyPartialRecipients(boolean allFail) throws IOException, GeneralSecurityException { ConnectionConfig conncfg = ConnectionConfig.builder() - .withSayHelo(sayHelo) .build(); SmtpMessage msg = SmtpMessage.builder() .withSender("sender1@srcdom1.com") .withRecipient("recip1@dstdom1.com") + .withRecipient("recip2@dstdom1.com") .withData(() -> "test msgbody1") .build(); Map sequences = new HashMap<>(); - sequences.put("HELO", "250 hello to you too"); sequences.put("EHLO", "250 hello to you too"); sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); sequences.put("DATA", "354 start body"); - sequences.put("QUIT", "250 quit ok"); + sequences.put("QUIT", "221 quit ok"); + sequences.put("RCPT TO:<"+msg.getRecipients().get(0)+">", "553 really bad recip"); + sequences.put("RCPT TO:<"+msg.getRecipients().get(1)+">", allFail? "400 bad recip" : "250 recip ok"); + smtpServer.configure(sequences, "220 Greetings", "250 msg ok", "550 bad"); + + TestSmtpSender sender = sendMessage(msg, conncfg, 0, null); + List results = sender.getResults(); + assertEquals(1, results.size()); + SmtpSendResult result = results.get(0); + assertEquals(allFail ? 0 : 1, result.getMessageCount()); + assertSame(msg, result.getMessage()); + assertNull(result.getErrorResponse()); + if (allFail) { + assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_MAILTO")); + } else { + assertNull(result.toString(), result.getErrorDiagnostic()); + } + assertEquals(2, result.getRecipientStatus().size()); + assertEquals(553, result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(allFail ? 400 : 250, result.getRecipientStatus().get(1).smtpStatus()); + assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); + } + + private void verifyHeloRejected(boolean sayHelo, boolean fallbackHelo, boolean badGreeting) throws IOException, GeneralSecurityException { + ConnectionConfig conncfg = ConnectionConfig.builder() + .withSayHelo(sayHelo) + .withFallbackHelo(fallbackHelo) + .build(); + SmtpMessage msg = SmtpMessage.builder() + .withSender("sender1@srcdom1.com") + .withRecipient("recip1@dstdom1.com") + .withData(() -> "test msgbody1") + .build(); + + Map sequences = new HashMap<>(); + String greeting = "220 Greetings"; + String errorState = (sayHelo?"HELO":(fallbackHelo?"HELO":"EHLO")); + int errorStatus = 550; //the default response code + if (badGreeting) { + //we'll fail before the Helo stage, but configure all later stages to work + greeting = "421 Go away"; + errorStatus = 421; + errorState = "READY"; + sequences.put("HELO", "250 hello to you too"); + sequences.put("EHLO", "250 hello to you too"); + } + sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); + sequences.put("DATA", "354 start body"); + sequences.put("QUIT", "221 quit ok"); for (CharSequence recip : msg.getRecipients()) { sequences.put("RCPT TO:<"+recip+">", "250 recip ok"); } - smtpServer.configure(sequences, "421 Go away", "250 msg ok", "550 bad"); + smtpServer.configure(sequences, greeting, "250 msg ok", "550 bad"); - TestSmtpSender sender = sendMessage(msg, conncfg); + TestSmtpSender sender = sendMessage(msg, conncfg, 0, null); List results = sender.getResults(); assertEquals(1, results.size()); SmtpSendResult result = results.get(0); assertEquals(0, result.getMessageCount()); assertSame(msg, result.getMessage()); - //xxx check errorResponse - assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_READY")); + assertNull(result.getErrorResponse()); + assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_"+errorState)); assertEquals(1, result.getRecipientStatus().size()); - assertEquals(421, result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(errorStatus, result.getRecipientStatus().get(0).smtpStatus()); assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); } - private void verifyHeloRejected(boolean sayHelo, boolean fallbackHelo) throws IOException, GeneralSecurityException { + private void verifyRejection(String stage, boolean invalidRsp) throws IOException, GeneralSecurityException { ConnectionConfig conncfg = ConnectionConfig.builder() - .withSayHelo(sayHelo) - .withFallbackHelo(fallbackHelo) .build(); SmtpMessage msg = SmtpMessage.builder() .withSender("sender1@srcdom1.com") @@ -194,31 +364,174 @@ private void verifyHeloRejected(boolean sayHelo, boolean fallbackHelo) throws IO .build(); Map sequences = new HashMap<>(); + sequences.put("EHLO", "250 hello to you too"); + populateSequence(stage.equals("MAILFROM"), !invalidRsp, "MAIL FROM:<"+msg.getSender()+">", "250 sender ok", "abc blah", sequences); + populateSequence(stage.equals("DATA"), !invalidRsp, "DATA", "354 start body", "ab1 blah", sequences); + populateSequence(stage.equals("QUIT"), !invalidRsp, "QUIT", "221 quit ok", "a", sequences); + for (CharSequence recip : msg.getRecipients()) { + sequences.put("RCPT TO:<"+recip+">", "250 recip ok"); + } + String msgrsp = stage.equals("MSGBODY") ? (invalidRsp ? "25" : null) : "250 msg ok"; + smtpServer.configure(sequences, "220 Greetings", msgrsp, "550 bad"); + + Map errorStates = new HashMap<>(); + errorStates.put("MAILFROM", "MAILFROM"); + errorStates.put("DATA", "DATA"); + errorStates.put("MSGBODY", "MAILBODY"); + int expectedStatus = (invalidRsp ? (stage.equals("MAILFROM") ? 0 : 250) : 550); //xxx BUG: invalidrsp status should be 4xx not 0 + + TestSmtpSender sender = sendMessage(msg, conncfg, 0, null); + List results = sender.getResults(); + assertEquals(1, results.size()); + SmtpSendResult result = results.get(0); + if (stage.equals("QUIT")) { + // QUIT is different because we've already received a successful SmtpSendResult + expectedStatus = 250; + assertEquals(1, result.getMessageCount()); + assertNull(result.toString(), result.getErrorDiagnostic()); + } else { + assertEquals(0, result.getMessageCount()); + assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_"+errorStates.get(stage))); + } + assertNull(result.getErrorResponse()); + assertSame(msg, result.getMessage()); + assertEquals(1, result.getRecipientStatus().size()); + assertEquals(expectedStatus, result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); + } + + private void verifyConnectionLoss(String stage, boolean isTimeout) throws IOException, GeneralSecurityException { + String errorSequence = (isTimeout ? MockSmtpServer.EXPECT_ACTION_TIMEOUT : MockSmtpServer.EXPECT_ACTION_DISCONNECT); + ConnectionConfig conncfg = ConnectionConfig.builder() + .withIdleTimeout(Duration.ofSeconds(1)) + .build(); + SmtpMessage msg = SmtpMessage.builder() + .withSender("sender1@srcdom1.com") + .withRecipient("recip1@dstdom1.com") + .withData(() -> "test msgbody1") + .build(); + + Map sequences = new HashMap<>(); + sequences.put("EHLO", stage.equals("HELO") ? errorSequence : "250 hello to you too"); sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 sender ok"); - sequences.put("DATA", "354 start body"); - sequences.put("QUIT", "250 quit ok"); + sequences.put("DATA", stage.equals("DATA") ? errorSequence : "354 start body"); + sequences.put("QUIT", stage.equals("QUIT") ? errorSequence : "221 quit ok"); for (CharSequence recip : msg.getRecipients()) { sequences.put("RCPT TO:<"+recip+">", "250 recip ok"); } + String greeting = stage.equals("GREET") ? errorSequence : "220 Greetings"; + String msgrsp = stage.equals("MSGBODY") ? errorSequence : "250 msg ok"; + smtpServer.configure(sequences, greeting, msgrsp, "550 bad"); + + Map errorStates = new HashMap<>(); + errorStates.put("GREET", "READY"); + errorStates.put("HELO", "EHLO"); + errorStates.put("DATA", "DATA"); + errorStates.put("MSGBODY", "MAILBODY"); + Map recipStatus = new HashMap<>(); + if (isTimeout) { + recipStatus.put("GREET", 421); + recipStatus.put("HELO", 421); + recipStatus.put("DATA", 421); + recipStatus.put("MSGBODY", 421); + //xxx don't use 421, use 499 with local errmsg or special 49x codes + } else { + recipStatus.put("GREET", 0); + recipStatus.put("HELO", 0); + recipStatus.put("DATA", 250); + recipStatus.put("MSGBODY", 250); + //xxx BUG: Wrong to report success, smtp-status needs to end up as 499 or have local errmsg that allows same treatment + } + recipStatus.put("QUIT", 250); //because we've already received a successful SmtpSendResult + + TestSmtpSender sender = sendMessage(msg, conncfg, 0, null); + List results = sender.getResults(); + assertEquals(1, results.size()); + SmtpSendResult result = results.get(0); + if (errorStates.containsKey(stage)) { + assertEquals(0, result.getMessageCount()); + assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_"+errorStates.get(stage))); + } else { + // QUIT is different because we've already received a successful SmtpSendResult + assertEquals(1, result.getMessageCount()); + assertNull(result.toString(), result.getErrorDiagnostic()); + } + assertNull(result.getErrorResponse()); + assertSame(msg, result.getMessage()); + assertEquals(1, result.getRecipientStatus().size()); + assertEquals(result.toString(), recipStatus.get(stage).intValue(), result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); + } + + private void verifyAuth(SaslEntity.MECH authOverride, boolean initrsp, Boolean rejected) throws IOException, GeneralSecurityException { + SmtpRelay relay = SmtpRelay.builder() + .withAuthRequired(true) + .withAuthOverride(authOverride) + .withUsername("username1") + .withPassword("password1") + .withAuthInitialResponse(initrsp) + .withAuthCompat(true) + .build(); + ConnectionConfig conncfg = ConnectionConfig.builder() + .build(); + SmtpMessage msg = SmtpMessage.builder() + .withSender("sender1@srcdom1.com") + .withRecipient("recip1@dstdom1.com") + .withData(() -> "test msgbody1") + .build(); + + Map sequences = new HashMap<>(); + String authOptions = (rejected == null ? "LOGIN" : "LOGIN PLAIN CRAM-MD5"); //LOGIN is not supported by our client + sequences.put("EHLO", "250-hello to you too\r\n" + +"250-STARTTLS\r\n" + +"250-AUTH "+authOptions+"\r\n" + +"250-ENHANCEDSTATUSCODES\r\n" + +"250-PIPELINING\r\n" + +"250 8BITMIME"); + sequences.put("MAIL FROM:<"+msg.getSender()+">", "250 2.1.0 sender ok"); + sequences.put("DATA", "354 start body"); + sequences.put("QUIT", "221 2.0.0 quit ok"); + for (CharSequence recip : msg.getRecipients()) { + sequences.put("RCPT TO:<"+recip+">", "250 2.1.5 recip ok"); + } + + // Auth dialogues - see https://mailtrap.io/blog/smtp-auth/ + // If proposed auth mechanisms not supported, Client should without auth unless an override mechanism was specified + boolean authFail = (rejected != null && rejected || rejected == null && authOverride != null); + if (authOverride == SaslEntity.MECH.CRAM_MD5) { + sequences.put("AUTH CRAM-MD5", "334 PFQyMl9TTVRQXzQ0XzkzMzk1Ml8xNzEyMzI2NzAyMDgxXzMyODM5MEBWT05ERksyWkQzPg=="); + if (!authFail) sequences.put("dXNlcm5hbWUxIDMwZDU4NmUxYzdiY2VhZjU5ZWZlOTVhNDJhMWNkNjA0", "235 2.1.3 auth-cram-md5 ok"); + } else { + if (initrsp) { + if (!authFail) sequences.put("AUTH PLAIN AHVzZXJuYW1lMQBwYXNzd29yZDE=", "235 2.0.0 auth ok"); + } else { + sequences.put("AUTH PLAIN", "334"); + if (!authFail) sequences.put("AHVzZXJuYW1lMQBwYXNzd29yZDE=", "235 2.1.3 auth-plain ok"); + } + } smtpServer.configure(sequences, "220 Greetings", "250 msg ok", "550 bad"); - TestSmtpSender sender = sendMessage(msg, conncfg); + TestSmtpSender sender = sendMessage(msg, conncfg, 0, relay); List results = sender.getResults(); assertEquals(1, results.size()); SmtpSendResult result = results.get(0); - assertEquals(0, result.getMessageCount()); + assertEquals(authFail ? 0 : 1, result.getMessageCount()); assertSame(msg, result.getMessage()); - //xxx check errorResponse - assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_"+(sayHelo?"HELO":(fallbackHelo?"HELO":"EHLO")))); + assertNull(result.getErrorResponse()); + if (authFail) { + assertTrue(result.toString(), result.getErrorDiagnostic().contains("state=S_AUTH")); + } else { + assertNull(result.toString(), result.getErrorDiagnostic()); + } assertEquals(1, result.getRecipientStatus().size()); - assertEquals(550, result.getRecipientStatus().get(0).smtpStatus()); + assertEquals(authFail ? 550 : 250, result.getRecipientStatus().get(0).smtpStatus()); assertEquals(smtpServer.getServicePort(), result.getRemoteAddress().port); } - private TestSmtpSender sendMessage(SmtpMessage msg, ConnectionConfig conncfg) throws IOException, GeneralSecurityException { + private TestSmtpSender sendMessage(SmtpMessage msg, ConnectionConfig conncfg, int port, SmtpRelay authRelay) throws IOException, GeneralSecurityException { BufferGenerator.BufferConfig bufcfg = new BufferGenerator.BufferConfig(256, true, null, null); BufferGenerator bufgen = new BufferGenerator(bufcfg); - TSAP remote = TSAP.build("127.0.0.1", smtpServer.getServicePort()); + TSAP remote = TSAP.build("127.0.0.1", port == 0 ? smtpServer.getServicePort() : port); DispatcherConfig def = DispatcherConfig.builder() .withName("test-dispatcher1") @@ -231,10 +544,19 @@ private TestSmtpSender sendMessage(SmtpMessage msg, ConnectionConfig conncfg) th .withDefaultConfig(conncfg) .withBufferGenerator(bufgen) .build(); - SmtpRelay relay = SmtpRelay.builder() + SmtpRelay.Builder bldr = SmtpRelay.builder() .withName("test-relay-mockserver") - .withAddress(remote) - .build(); + .withAddress(remote); + if (authRelay != null) { + bldr = bldr.withAuthRequired(authRelay.isAuthRequired()) + .withAuthCompat(authRelay.isAuthCompat()) + .withAuthInitialResponse(authRelay.isAuthInitialResponse()) + .withAuthOverride(authRelay.getAuthOverride()) + .withUsername(authRelay.getUsername()) + .withPassword(authRelay.getPassword()) + .withSslConfig(authRelay.getSslConfig()); + } + SmtpRelay relay = bldr.build(); Client clnt = new Client(shared, dsptch); TestSmtpSender sender = new TestSmtpSender(dsptch, clnt, msg, relay); @@ -243,7 +565,7 @@ private TestSmtpSender sendMessage(SmtpMessage msg, ConnectionConfig conncfg) th dsptch.loadRunnable(sender); dsptch.start(); - Dispatcher.STOPSTATUS stopsts = dsptch.waitStopped(TimeOps.MSECS_PER_SECOND*10L, true); + Dispatcher.STOPSTATUS stopsts = dsptch.waitStopped(Duration.ofSeconds(10).toMillis(), true); assertEquals(Dispatcher.STOPSTATUS.STOPPED, stopsts); assertTrue(dsptch.completedOK()); assertEquals(0, shared.getActiveServerConnections()); @@ -251,6 +573,16 @@ private TestSmtpSender sendMessage(SmtpMessage msg, ConnectionConfig conncfg) th return sender; } + private static void populateSequence(boolean altSequence, boolean omit, String xmit, String recv, String altRsp, Map sequences) { + if (altSequence) { + if (!omit) { + sequences.put(xmit, altRsp); + } + } else { + sequences.put(xmit, recv); + } + } + private static class TestSmtpSender implements SmtpSender, DispatcherRunnable { private final List results = new ArrayList<>(); diff --git a/server/src/test/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptorTest.java b/server/src/test/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptorTest.java index 3e56b49..c2bd1b6 100644 --- a/server/src/test/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptorTest.java +++ b/server/src/test/java/com/grey/mailismus/mta/deliver/client/SmtpResponseDescriptorTest.java @@ -17,6 +17,7 @@ public void testNoEnhanced() { assertEquals(250, reply.smtpStatus()); assertEquals("All OK", reply.message()); assertNull(reply.enhancedStatus()); + assertFalse(reply.isContinued()); str = "250 2.1.0 OK"; rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); @@ -24,6 +25,7 @@ public void testNoEnhanced() { assertEquals(250, reply.smtpStatus()); assertEquals("2.1.0 OK", reply.message()); assertNull(reply.enhancedStatus()); + assertFalse(reply.isContinued()); str = "250 2.1.0"; rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); @@ -31,6 +33,7 @@ public void testNoEnhanced() { assertEquals(250, reply.smtpStatus()); assertEquals("2.1.0", reply.message()); assertNull(reply.enhancedStatus()); + assertFalse(reply.isContinued()); str = "500"; rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); @@ -38,6 +41,15 @@ public void testNoEnhanced() { assertEquals(500, reply.smtpStatus()); assertTrue(reply.toString(), reply.message().isEmpty()); assertNull(reply.enhancedStatus()); + assertFalse(reply.isContinued()); + + str = "500 "; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(500, reply.smtpStatus()); + assertTrue(reply.toString(), reply.message().isEmpty()); + assertNull(reply.enhancedStatus()); + assertFalse(reply.isContinued()); } @Test @@ -46,29 +58,77 @@ public void testWithEnhanced() { ByteArrayRef rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); SmtpResponseDescriptor reply = SmtpResponseDescriptor.parse(rspdata, true); assertEquals(250, reply.smtpStatus()); - assertEquals("OK", reply.message()); assertEquals("All", reply.enhancedStatus()); + assertEquals("OK", reply.message()); + assertFalse(reply.isContinued()); str = "250 2.1.0 OK"; rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); reply = SmtpResponseDescriptor.parse(rspdata, true); assertEquals(250, reply.smtpStatus()); - assertEquals("OK", reply.message()); assertEquals("2.1.0", reply.enhancedStatus()); + assertEquals("OK", reply.message()); + assertFalse(reply.isContinued()); str = "250 2.1.0"; rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); reply = SmtpResponseDescriptor.parse(rspdata, true); assertEquals(250, reply.smtpStatus()); - assertTrue(reply.toString(), reply.message().isEmpty()); assertEquals("2.1.0", reply.enhancedStatus()); + assertTrue(reply.toString(), reply.message().isEmpty()); + assertFalse(reply.isContinued()); + + str = "300 2.1.0"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(300, reply.smtpStatus()); + assertNull(reply.enhancedStatus()); + assertEquals("2.1.0", reply.message()); + assertFalse(reply.isContinued()); str = "500"; rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); reply = SmtpResponseDescriptor.parse(rspdata, true); assertEquals(500, reply.smtpStatus()); + assertNull(reply.enhancedStatus()); assertTrue(reply.toString(), reply.message().isEmpty()); + assertFalse(reply.isContinued()); + } + + @Test + public void testContinued() { + String str = "250-All OK"; + ByteArrayRef rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + SmtpResponseDescriptor reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(250, reply.smtpStatus()); + assertTrue(reply.isContinued()); assertNull(reply.enhancedStatus()); + assertEquals("All OK", reply.message()); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(250, reply.smtpStatus()); + assertTrue(reply.isContinued()); + assertNull(reply.enhancedStatus()); + assertEquals("All OK", reply.message()); + + str = "250-"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertEquals(250, reply.smtpStatus()); + assertTrue(reply.isContinued()); + assertNull(reply.enhancedStatus()); + assertTrue(reply.toString(), reply.message().isEmpty()); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertEquals(250, reply.smtpStatus()); + assertTrue(reply.isContinued()); + assertNull(reply.enhancedStatus()); + assertTrue(reply.toString(), reply.message().isEmpty()); + + str = "2501-"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertNull(reply); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertNull(reply); } @Test @@ -83,6 +143,7 @@ public void testWithEOL() { assertEquals(250, reply.smtpStatus()); assertEquals("All OK", reply.message()); assertNull(reply.enhancedStatus()); + assertFalse(reply.isContinued()); str = "250 All OK\n"; rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); @@ -90,10 +151,12 @@ public void testWithEOL() { assertEquals(250, reply.smtpStatus()); assertEquals("OK", reply.message()); assertEquals("All", reply.enhancedStatus()); + assertFalse(reply.isContinued()); reply = SmtpResponseDescriptor.parse(rspdata, false); assertEquals(250, reply.smtpStatus()); assertEquals("All OK", reply.message()); assertNull(reply.enhancedStatus()); + assertFalse(reply.isContinued()); str = "250 All OK"+"".repeat(3); rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); @@ -101,10 +164,12 @@ public void testWithEOL() { assertEquals(250, reply.smtpStatus()); assertEquals("OK", reply.message()); assertEquals("All", reply.enhancedStatus()); + assertFalse(reply.isContinued()); reply = SmtpResponseDescriptor.parse(rspdata, false); assertEquals(250, reply.smtpStatus()); assertEquals("All OK", reply.message()); assertNull(reply.enhancedStatus()); + assertFalse(reply.isContinued()); } @Test @@ -129,5 +194,26 @@ public void testInvalid() { assertNull(reply); reply = SmtpResponseDescriptor.parse(rspdata, false); assertNull(reply); + + str = "1234"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertNull(reply); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertNull(reply); + + str = "abc"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertNull(reply); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertNull(reply); + + str = "25c"; + rspdata = new ByteArrayRef(str.getBytes(StandardCharsets.UTF_8)); + reply = SmtpResponseDescriptor.parse(rspdata, true); + assertNull(reply); + reply = SmtpResponseDescriptor.parse(rspdata, false); + assertNull(reply); } } diff --git a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java index cecfc26..03fbd75 100644 --- a/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java +++ b/server/src/test/java/com/grey/mailismus/mta/testsupport/MockSmtpServer.java @@ -20,10 +20,10 @@ public class MockSmtpServer { - private static final String EOL_SMTP = "\r\n"; - private static final String SENDSEQ_NOREPLY = "-"; + public static final String EXPECT_ACTION_DISCONNECT = "_DISCONNECT_"; + public static final String EXPECT_ACTION_TIMEOUT = "_TIMEOUT_"; - private static final String EXPECT_ACTION_DISCONNECT = "_DISC_"; + private static final String EOL_SMTP = "\r\n"; private final Map> transcriptsByPort = new ConcurrentHashMap<>();//keyed on remote-dotted-ip:remote-port private final ServerSocket srvrSocket; @@ -32,7 +32,7 @@ public class MockSmtpServer private final Map expectSendSequences = new ConcurrentHashMap<>(); private volatile String greetingResponse; private volatile String messageResponse; - private volatile String defaultResponse; + private volatile String defaultResponse; public MockSmtpServer() throws IOException { srvrSocket = new ServerSocket(0, 10, InetAddress.getLoopbackAddress()); @@ -56,8 +56,8 @@ public int getServicePort() { public void configure(Map expectSend, String initrsp, String msgrsp, String dfltrsp) { expectSendSequences.clear(); if (expectSend != null) expectSendSequences.putAll(expectSend); - greetingResponse = initrsp; - messageResponse = msgrsp; + greetingResponse = initrsp; + messageResponse = msgrsp; defaultResponse = dfltrsp; } @@ -93,8 +93,8 @@ private static class SmtpSession { private final BufferedReader recvStream; private final OutputStream sendStream; private final Map expectSendSequences; - private final String greetingResponse; - private final String messageResponse; + private final String greetingResponse; + private final String messageResponse; private final String defaultResponse; private final List transcript; private boolean inDataTransfer; @@ -102,7 +102,7 @@ private static class SmtpSession { public SmtpSession(Socket sock, Map expectSend, String initrsp, String msgrsp, String dfltrsp, List transcript) throws IOException { this.expectSendSequences = new HashMap<>(); if (expectSend != null) expectSendSequences.putAll(expectSend); - this.greetingResponse = initrsp; + this.greetingResponse = initrsp; this.messageResponse = msgrsp; this.defaultResponse = dfltrsp; this.transcript = transcript; @@ -131,7 +131,11 @@ public void serveSession() { } private void sessionDialogue() throws IOException { - sendResponse(greetingResponse); + if (EXPECT_ACTION_DISCONNECT.equals(greetingResponse)) { + return; + } + sendResponse(greetingResponse); + String request; while ((request = recvStream.readLine()) != null) { transcript.add("RECV: "+request); @@ -144,22 +148,24 @@ private void sessionDialogue() throws IOException { rsp = messageResponse; } else { rsp = expectSendSequences.get(request); - if (EXPECT_ACTION_DISCONNECT.equals(rsp)) { - break; - } if (request.equalsIgnoreCase("DATA")) { inDataTransfer = true; } } + + if (EXPECT_ACTION_DISCONNECT.equals(rsp)) { + break; + } sendResponse(rsp); } } private void sendResponse(String rsp) throws IOException { - if (rsp == null) - rsp = defaultResponse; - if (SENDSEQ_NOREPLY.equals(rsp)) - return; + if (rsp == null) + rsp = defaultResponse; + if (EXPECT_ACTION_TIMEOUT.equals(rsp)) { + return; //skip our response, so client times out + } transcript.add("XMIT: "+rsp); rsp += EOL_SMTP; sendStream.write(rsp.getBytes());