diff --git a/app/controllers/web/admin/index.js b/app/controllers/web/admin/index.js index ed96558fd3..2d077c1840 100644 --- a/app/controllers/web/admin/index.js +++ b/app/controllers/web/admin/index.js @@ -14,6 +14,7 @@ const emails = require('./emails'); const inquiries = require('./inquiries'); const payments = require('./payments'); const jobs = require('./jobs'); +const spam = require('./spam'); module.exports = { analytics, @@ -26,5 +27,6 @@ module.exports = { emails, inquiries, payments, - jobs + jobs, + spam }; diff --git a/app/controllers/web/admin/spam.js b/app/controllers/web/admin/spam.js new file mode 100644 index 0000000000..c8a4fb501b --- /dev/null +++ b/app/controllers/web/admin/spam.js @@ -0,0 +1,185 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const paginate = require('koa-ctx-paginate'); +const dayjs = require('dayjs-with-plugins'); + +const getMongoQuery = require('#helpers/get-mongo-query'); +const { Logs } = require('#models'); + +const MAX_TIME_MS = 10000; + +// Cap count at 10,000 to improve performance on large collections +const MAX_COUNT_LIMIT = 10_000; + +async function list(ctx) { + const userQuery = getMongoQuery(ctx); + + // Build base query: always filter to spam bounce category + const query = { ...userQuery, bounce_category: 'spam' }; + + // Parse filter params + const { ip, hostname, start_date, end_date } = ctx.query; + + if (ip) query['meta.app.ip'] = ip; + if (hostname) query['meta.app.hostname'] = hostname; + + // Date range filter (defaults to last 7 days) + const endDate = end_date ? dayjs(end_date).endOf('day').toDate() : new Date(); + const startDate = start_date + ? dayjs(start_date).startOf('day').toDate() + : dayjs(endDate).subtract(7, 'day').toDate(); + + query.created_at = { $gte: startDate, $lte: endDate }; + + // Determine time bucket format based on range span + const rangeDays = dayjs(endDate).diff(dayjs(startDate), 'day'); + const dateFormat = rangeDays <= 3 ? '%Y-%m-%dT%H:00' : '%Y-%m-%d'; + + // Queries for 24h/7d/30d summary counts + const now = new Date(); + const baseMatch = { bounce_category: 'spam' }; + if (ip) baseMatch['meta.app.ip'] = ip; + if (hostname) baseMatch['meta.app.hostname'] = hostname; + + const [ + logs, + itemCount, + timeline, + topIPs, + topErrors, + topHostnames, + count24h, + count7d, + count30d + ] = await Promise.all([ + // Paginated logs + // eslint-disable-next-line unicorn/no-array-callback-reference + Logs.find(query) + .hint({ bounce_category: 1, domains: 1, created_at: 1 }) + .limit(ctx.query.limit) + .skip(ctx.paginate.skip) + .sort(ctx.query.sort || '-created_at') + .select( + 'created_at meta.app.ip meta.app.hostname err.message err.responseCode err.statusCode err.status message domains' + ) + .lean() + .maxTimeMS(MAX_TIME_MS) + .exec(), + // Item count + Logs.countDocuments(query, { + hint: { bounce_category: 1, domains: 1, created_at: 1 }, + maxTimeMS: MAX_TIME_MS + }), + // Spam over time + Logs.aggregate([ + { $match: query }, + { + $group: { + _id: { $dateToString: { format: dateFormat, date: '$created_at' } }, + count: { $sum: 1 } + } + }, + { $sort: { _id: 1 } } + ]).option({ + hint: { bounce_category: 1, domains: 1, created_at: 1 }, + maxTimeMS: MAX_TIME_MS + }), + // Top IPs + Logs.aggregate([ + { $match: query }, + { $group: { _id: '$meta.app.ip', count: { $sum: 1 } } }, + { $sort: { count: -1 } }, + { $limit: 10 } + ]).option({ + hint: { bounce_category: 1, domains: 1, created_at: 1 }, + maxTimeMS: MAX_TIME_MS + }), + // Top error messages + Logs.aggregate([ + { $match: query }, + { $group: { _id: '$err.message', count: { $sum: 1 } } }, + { $sort: { count: -1 } }, + { $limit: 10 } + ]).option({ + hint: { bounce_category: 1, domains: 1, created_at: 1 }, + maxTimeMS: MAX_TIME_MS + }), + // Top hostnames + Logs.aggregate([ + { $match: query }, + { $group: { _id: '$meta.app.hostname', count: { $sum: 1 } } }, + { $sort: { count: -1 } }, + { $limit: 10 } + ]).option({ + hint: { bounce_category: 1, domains: 1, created_at: 1 }, + maxTimeMS: MAX_TIME_MS + }), + // 24h count + Logs.countDocuments( + { + ...baseMatch, + created_at: { $gte: dayjs(now).subtract(24, 'hour').toDate() } + }, + { + hint: { bounce_category: 1, domains: 1, created_at: 1 }, + maxTimeMS: MAX_TIME_MS + } + ), + // 7d count + Logs.countDocuments( + { + ...baseMatch, + created_at: { $gte: dayjs(now).subtract(7, 'day').toDate() } + }, + { + hint: { bounce_category: 1, domains: 1, created_at: 1 }, + maxTimeMS: MAX_TIME_MS + } + ), + // 30d count + Logs.countDocuments( + { + ...baseMatch, + created_at: { $gte: dayjs(now).subtract(30, 'day').toDate() } + }, + { + hint: { bounce_category: 1, domains: 1, created_at: 1 }, + maxTimeMS: MAX_TIME_MS + } + ) + ]); + + // Cap the item count to prevent UI issues with very large result sets + const cappedItemCount = Math.min(itemCount, MAX_COUNT_LIMIT); + const pageCount = Math.ceil(cappedItemCount / ctx.query.limit); + + const renderData = { + logs, + pageCount, + itemCount: cappedItemCount, + timeline, + topIPs, + topErrors, + topHostnames, + count24h, + count7d, + count30d, + startDate: dayjs(startDate).format('YYYY-MM-DD'), + endDate: dayjs(endDate).format('YYYY-MM-DD'), + filterIP: ip || '', + filterHostname: hostname || '', + pages: paginate.getArrayPages(ctx)(6, pageCount, ctx.query.page) + }; + + if (ctx.accepts('html')) return ctx.render('admin/spam', renderData); + + const table = await ctx.render('admin/spam/_table', renderData); + ctx.body = { table }; +} + +module.exports = { + list +}; diff --git a/app/views/admin/spam/_table.pug b/app/views/admin/spam/_table.pug new file mode 100644 index 0000000000..79182394b8 --- /dev/null +++ b/app/views/admin/spam/_table.pug @@ -0,0 +1,67 @@ +include ../../_sort-header +include ../../_pagination + +table.table.table-hover.table-bordered.table-sm + thead.thead-dark + tr + th(scope="col") + +sortHeader('created_at', 'Created', '#table-spam') + th(scope="col") + +sortHeader('meta.app.ip', 'IP', '#table-spam') + th(scope="col") + +sortHeader('meta.app.hostname', 'Hostname', '#table-spam') + th.align-middle(scope="col")= t("Error Message") + th.align-middle.text-center(scope="col")= t("Response Code") + th.align-middle(scope="col")= t("Domain") + th.text-center.align-middle(scope="col")= t("Actions") + tbody + if logs.length === 0 + tr + td.alert.alert-info(colspan="7") + = t("No spam logs found for that search.") + else + each log in logs + tr + td.align-middle.text-center.dayjs( + data-time=new Date(log.created_at).getTime() + ) + = dayjs(log.created_at).tz(user.timezone === 'Etc/Unknown' ? 'UTC' : user.timezone).format("M/D/YY h:mm A z") + td.align-middle + if log.meta && log.meta.app && log.meta.app.ip + code= log.meta.app.ip + td.align-middle + if log.meta && log.meta.app && log.meta.app.hostname + code= log.meta.app.hostname + td.align-middle.small + if log.err && log.err.message + != ansiHTML(log.err.message) + else + != ansiHTML(log.message) + td.align-middle.text-center + - let statusCode; + if log.err && log.err.responseCode + - statusCode = log.err.responseCode; + else if log.err && log.err.statusCode + - statusCode = log.err.statusCode; + else if log.err && log.err.status + - statusCode = log.err.status; + + if statusCode + - let badgeClass = "badge-success"; + if statusCode >= 500 + - badgeClass = "badge-danger"; + else if statusCode >= 400 + - badgeClass = "badge-warning"; + else if statusCode >= 300 + - badgeClass = "badge-primary"; + span.badge.badge-pill(class=badgeClass)= statusCode + td.align-middle + if log.domains && log.domains.length > 0 + each domain in log.domains + span.badge.badge-secondary.mr-1= domain + td.align-middle.text-center + a.btn.btn-dark.btn-sm( + href=l(`/admin/logs/${log.id}`), + target="_blank" + ): i.fa.fa-search ++paginate('#table-spam') diff --git a/app/views/admin/spam/index.pug b/app/views/admin/spam/index.pug new file mode 100644 index 0000000000..6e32e72826 --- /dev/null +++ b/app/views/admin/spam/index.pug @@ -0,0 +1,161 @@ +extends ../../layout + +block append scripts + script( + defer, + src=manifest("js/admin-spam.js"), + integrity=manifest("js/admin-spam.js", "integrity"), + crossorigin="anonymous" + ) + +block body + .container-fluid.py-3 + .row.mt-1 + .col + include ../../_breadcrumbs + //- Summary cards + .row.mb-3 + .col-md-4 + .card.text-center + .card-body + .h5.card-title= t("Last 24 Hours") + p.display-4.mb-0= count24h.toLocaleString() + .col-md-4 + .card.text-center + .card-body + .h5.card-title= t("Last 7 Days") + p.display-4.mb-0= count7d.toLocaleString() + .col-md-4 + .card.text-center + .card-body + .h5.card-title= t("Last 30 Days") + p.display-4.mb-0= count30d.toLocaleString() + + //- Filter form + form.ajax-form.table-ajax-form.card.mb-3( + action=ctx.path, + method="GET", + data-table="#table-spam", + data-search-params="mongodb_query,ip,hostname,start_date,end_date" + ) + .card-body + .h5.card-title= t("Filter Spam Logs") + .form-row + .form-group.col-md-3 + label(for="input-ip")= t("IP Address") + input#input-ip.form-control( + type="text", + value=filterIP, + name="ip", + placeholder="e.g. 1.2.3.4" + ) + .form-group.col-md-3 + label(for="input-hostname")= t("Hostname") + input#input-hostname.form-control( + type="text", + value=filterHostname, + name="hostname", + placeholder="e.g. mx1.example.com" + ) + .form-group.col-md-3 + label(for="input-start-date")= t("Start Date") + input#input-start-date.form-control( + type="date", + value=startDate, + name="start_date" + ) + .form-group.col-md-3 + label(for="input-end-date")= t("End Date") + input#input-end-date.form-control( + type="date", + value=endDate, + name="end_date" + ) + .form-group + label(for="textarea-mongodb-query") MongoDB query + textarea#textarea-mongodb-query.form-control( + name="mongodb_query", + rows=2, + placeholder="MongoDB query" + )= ctx.query.mongodb_query || "" + small.form-text.text-muted!= t('See mongodb-query-parser for more insight.') + button.btn.btn-success(type="submit")= t("Search") + + //- Spam over time chart + .card.mb-3 + .card-body + h5.card-title= t("Spam Over Time") + #spam-timeline-chart(data-chart=JSON.stringify(timeline)) + .text-center.py-5 + i.fa.fa-spin.fa-spinner.fa-2x.text-muted + + //- Top IPs and Top Hostnames + .row.mb-3 + .col-md-6 + .card.h-100 + .card-body + .h5.card-title= t("Top Offending IPs") + if topIPs.length === 0 + p.text-muted= t("No data available.") + else + table.table.table-sm.table-bordered.mb-0 + thead.thead-dark + tr + th= t("IP Address") + th.text-right= t("Count") + th.text-center= t("Filter") + tbody + each item in topIPs + tr + td: code= item._id || "N/A" + td.text-right= item.count.toLocaleString() + td.text-center + if item._id + a.btn.btn-dark.btn-sm( + href=`${ctx.path}?ip=${encodeURIComponent(item._id)}` + ): i.fa.fa-filter + .col-md-6 + .card.h-100 + .card-body + .h5.card-title= t("Top Hostnames") + if topHostnames.length === 0 + p.text-muted= t("No data available.") + else + table.table.table-sm.table-bordered.mb-0 + thead.thead-dark + tr + th= t("Hostname") + th.text-right= t("Count") + th.text-center= t("Filter") + tbody + each item in topHostnames + tr + td: code= item._id || "N/A" + td.text-right= item.count.toLocaleString() + td.text-center + if item._id + a.btn.btn-dark.btn-sm( + href=`${ctx.path}?hostname=${encodeURIComponent(item._id)}` + ): i.fa.fa-filter + + //- Top Error Messages + .card.mb-3 + .card-body + .h5.card-title= t("Top Error Messages") + if topErrors.length === 0 + p.text-muted= t("No data available.") + else + table.table.table-sm.table-bordered.mb-0 + thead.thead-dark + tr + th= t("Error Message") + th.text-right= t("Count") + tbody + each item in topErrors + tr + td.small: code= item._id || "N/A" + td.text-right= item.count.toLocaleString() + + //- Spam logs table + #table-spam + include ./_table diff --git a/assets/js/admin-spam.js b/assets/js/admin-spam.js new file mode 100644 index 0000000000..fd4c6bfaa2 --- /dev/null +++ b/assets/js/admin-spam.js @@ -0,0 +1,146 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +/* eslint-disable prefer-object-spread */ + +const $ = require('jquery'); +const Apex = require('apexcharts'); + +const charts = {}; + +function getThemeMode() { + if ( + window.matchMedia && + window.matchMedia('(prefers-color-scheme: dark)').matches + ) { + return 'dark'; + } + + return 'light'; +} + +function getCommonOptions() { + const isDark = getThemeMode() === 'dark'; + return { + chart: { + background: 'transparent', + toolbar: { + show: true, + tools: { + download: true, + selection: false, + zoom: false, + zoomin: false, + zoomout: false, + pan: false, + reset: false + } + }, + animations: { + enabled: true, + easing: 'easeinout', + speed: 500 + } + }, + theme: { + mode: isDark ? 'dark' : 'light' + }, + grid: { + borderColor: isDark ? '#404040' : '#e0e0e0' + }, + tooltip: { + theme: isDark ? 'dark' : 'light' + } + }; +} + +function initSpamTimelineChart() { + const $element = $('#spam-timeline-chart'); + if ($element.length === 0) return; + + const data = $element.data('chart'); + if (!data || data.length === 0) { + $element.html( + '

No spam data for the selected period.

' + ); + return; + } + + const common = getCommonOptions(); + const options = Object.assign({}, common, { + chart: Object.assign({}, common.chart, { type: 'area', height: 300 }), + series: [ + { + name: 'Spam Bounces', + data: data.map(function (d) { + return d.count; + }) + } + ], + xaxis: { + categories: data.map(function (d) { + return d._id; + }), + labels: { + rotate: -45, + rotateAlways: data.length > 20 + } + }, + colors: ['#dc3545'], + fill: { + type: 'gradient', + gradient: { + shadeIntensity: 1, + opacityFrom: 0.7, + opacityTo: 0.3 + } + }, + dataLabels: { + enabled: false + }, + yaxis: { + title: { + text: 'Count' + }, + min: 0 + }, + tooltip: { + y: { + formatter(val) { + return val + ' spam bounces'; + } + } + } + }); + + $element.empty(); + const chart = new Apex($element.get(0), options); + chart.render(); + charts['spam-timeline-chart'] = chart; +} + +function initCharts() { + initSpamTimelineChart(); +} + +$(document).ready(function () { + initCharts(); +}); + +if (window.matchMedia) { + window + .matchMedia('(prefers-color-scheme: dark)') + .addEventListener('change', function () { + for (const key of Object.keys(charts)) { + if (charts[key] && typeof charts[key].destroy === 'function') { + charts[key].destroy(); + } + + delete charts[key]; + } + + initCharts(); + }); +} diff --git a/routes/web/admin.js b/routes/web/admin.js index 881a4aedb4..2f08ddc669 100644 --- a/routes/web/admin.js +++ b/routes/web/admin.js @@ -89,6 +89,9 @@ router .get('/logs', paginate.middleware(10, 50), web.admin.logs.list) .get('/logs/:id', web.admin.logs.retrieve) + // spam + .get('/spam', paginate.middleware(10, 50), web.admin.spam.list) + // jobs .get('/jobs', paginate.middleware(10, 50), web.admin.jobs.list) .get('/jobs/detail/:name', web.admin.jobs.jobDetail)