🐛 Fixed incorrect date filtering on SQLite

fixes #23441

- date columns are stored as "YYYY-MM-DD HH:MM:SS", but absolute date values in filters were passed through untouched while only relative dates (now-30d) were normalized
- on SQLite, datetimes are stored as text and compared lexically, so the "T" in an ISO value sorts after the stored space separator and greater/less-than filters returned the wrong rows
- normalizes date-column filter values to the stored UTC format before the query is built, leaving non-date columns and unparseable values untouched

Author: frenck

ghost/core/core/server/models/base/bookshelf.js

@@ -20,6 +20,11 @@ ghostBookshelf.plugin(plugins.customQuery);
 // Load the Ghost filter plugin, which handles applying a 'filter' to findPage requests
 ghostBookshelf.plugin(plugins.filter);
 
+// Normalize absolute date values in filters to the database date format, so date
+// comparisons behave consistently across SQLite and MySQL. Must come after the
+// filter plugin, which it wraps.
+ghostBookshelf.plugin(require('./plugins/date-filter'));
+
 // Load the Ghost filter plugin, which handles applying a 'order' to findPage requests
 ghostBookshelf.plugin(plugins.order);
 

ghost/core/core/server/models/base/plugins/date-filter.js

@@ -0,0 +1,136 @@
+const moment = require('moment');
+const {chainTransformers} = require('@tryghost/mongo-utils');
+const schemaTables = require('../../../data/schema/schema');
+
+// Date columns are stored as "YYYY-MM-DD HH:MM:SS" (UTC). NQL normalizes relative
+// dates (e.g. `now-30d`) to that format, but absolute values from a filter
+// (e.g. `published_at:>'2025-02-27T19:03:00.000-05:00'`) are passed through as-is.
+// On SQLite, datetimes are stored as text and compared lexically, so the "T"
+// sorts after the space separator and the comparison returns the wrong rows.
+// We normalize those values to the stored format before the query is built.
+// See https://github.com/TryGhost/Ghost/issues/23441
+const ACCEPTED_DATE_FORMATS = [moment.ISO_8601, 'YYYY-MM-DD HH:mm:ss'];
+const DB_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+
+// Columns to treat as dates, keyed by column name. A name only qualifies when
+// it is a `dateTime` in every table that has it, so we never normalize a value
+// for a same-named column of another type. Filters resolve by column name (the
+// trailing segment), which keeps relation-qualified fields like `tags.created_at`
+// working without needing the model's table.
+let dateColumns = null;
+const getDateColumns = () => {
+    if (!dateColumns) {
+        const otherColumns = new Set();
+        dateColumns = new Set();
+
+        for (const columns of Object.values(schemaTables)) {
+            for (const [name, spec] of Object.entries(columns)) {
+                if (spec && spec.type === 'dateTime') {
+                    dateColumns.add(name);
+                } else if (spec && spec.type) {
+                    otherColumns.add(name);
+                }
+            }
+        }
+
+        otherColumns.forEach(name => dateColumns.delete(name));
+    }
+    return dateColumns;
+};
+
+const isDateColumn = (key) => {
+    const column = key.includes('.') ? key.slice(key.lastIndexOf('.') + 1) : key;
+    return getDateColumns().has(column);
+};
+
+// Reformat a single value to the database date format. Non-strings and values we
+// can't parse as a date are returned untouched, so unexpected input is never
+// corrupted.
+const normalizeValue = (value) => {
+    if (typeof value !== 'string') {
+        return value;
+    }
+
+    const parsed = moment.utc(value, ACCEPTED_DATE_FORMATS, true);
+    return parsed.isValid() ? parsed.format(DB_DATE_FORMAT) : value;
+};
+
+// An operator map like `{$gt: ...}`: a plain object whose keys are all operators.
+// Anything else that happens to be an object (e.g. a `Date`) is not one and must
+// be left untouched rather than reduced to `{}`.
+const isOperatorMap = (value) => {
+    if (!value || Object.prototype.toString.call(value) !== '[object Object]') {
+        return false;
+    }
+
+    const keys = Object.keys(value);
+    return keys.length > 0 && keys.every(key => key.charAt(0) === '$');
+};
+
+// A field value is a plain value (equality), an array (e.g. `$in`), or an operator
+// map (e.g. `{$gt: ...}`).
+const normalizeFieldValue = (value) => {
+    if (Array.isArray(value)) {
+        return value.map(normalizeValue);
+    }
+
+    if (isOperatorMap(value)) {
+        const result = {};
+        for (const [operator, operatorValue] of Object.entries(value)) {
+            result[operator] = Array.isArray(operatorValue)
+                ? operatorValue.map(normalizeValue)
+                : normalizeValue(operatorValue);
+        }
+        return result;
+    }
+
+    return normalizeValue(value);
+};
+
+// Walk the parsed mongo-JSON filter, normalizing date column values and recursing
+// into `$and`/`$or` groups.
+const normalizeDateFilters = (node) => {
+    if (Array.isArray(node)) {
+        return node.map(normalizeDateFilters);
+    }
+
+    if (!node || typeof node !== 'object') {
+        return node;
+    }
+
+    const result = {};
+    for (const [key, value] of Object.entries(node)) {
+        if (key.charAt(0) === '$') {
+            result[key] = normalizeDateFilters(value);
+        } else if (isDateColumn(key)) {
+            result[key] = normalizeFieldValue(value);
+        } else {
+            result[key] = value;
+        }
+    }
+    return result;
+};
+
+/**
+ * Normalizes absolute date values in NQL filters to the database date format, so
+ * date comparisons behave the same on SQLite and MySQL. Wraps
+ * `applyDefaultAndCustomFilters` and chains the date transformer after any
+ * transformer the caller supplied.
+ *
+ * @param {import('bookshelf')} Bookshelf
+ */
+module.exports = function (Bookshelf) {
+    const parentApply = Bookshelf.Model.prototype.applyDefaultAndCustomFilters;
+
+    Bookshelf.Model = Bookshelf.Model.extend({
+        applyDefaultAndCustomFilters: function applyDefaultAndCustomFilters(options = {}) {
+            const mongoTransformer = options.mongoTransformer
+                ? chainTransformers(options.mongoTransformer, normalizeDateFilters)
+                : normalizeDateFilters;
+
+            return parentApply.call(this, Object.assign({}, options, {mongoTransformer}));
+        }
+    });
+};
+
+module.exports.normalizeDateFilters = normalizeDateFilters;

ghost/core/test/integration/model/post-date-filter.test.js

@@ -0,0 +1,57 @@
+const assert = require('node:assert/strict');
+const testUtils = require('../../utils');
+const models = require('../../../core/server/models');
+
+const context = testUtils.context.owner;
+const markdownToMobiledoc = testUtils.DataGenerator.markdownToMobiledoc;
+
+// Regression test for https://github.com/TryGhost/Ghost/issues/23441
+// On SQLite an absolute ISO date in a filter used to sort after the stored
+// "YYYY-MM-DD HH:MM:SS" format (because "T" > " "), returning the wrong rows.
+describe('Integration: Post date filtering', function () {
+    const early = new Date(Date.UTC(2025, 5, 15, 9, 0, 0)); // same day, before the boundary
+    const late = new Date(Date.UTC(2025, 5, 15, 12, 0, 0)); // same day, after the boundary
+    const later = new Date(Date.UTC(2025, 11, 31, 23, 0, 0)); // a later day
+
+    const addPost = (title, publishedAt) => models.Post.add({
+        status: 'published',
+        title,
+        published_at: publishedAt,
+        mobiledoc: markdownToMobiledoc('content')
+    }, context);
+
+    beforeAll(testUtils.teardownDb);
+    beforeAll(testUtils.setup('users:roles'));
+    beforeAll(async function () {
+        await addPost('early-same-day', early);
+        await addPost('late-same-day', late);
+        await addPost('later-day', later);
+    });
+    afterAll(testUtils.teardownDb);
+
+    const titlesFor = async (filter) => {
+        const result = await models.Post.findPage({filter, status: 'all'});
+        return result.data.map(post => post.get('title')).sort();
+    };
+
+    it('includes a same-day post that is after a "greater than" ISO boundary', async function () {
+        // Boundary is 2025-06-15 10:00:00 UTC. "late-same-day" (12:00) is after it.
+        const titles = await titlesFor("published_at:>'2025-06-15T10:00:00.000Z'");
+
+        assert.deepEqual(titles, ['late-same-day', 'later-day']);
+    });
+
+    it('excludes a same-day post that is before a "less than" ISO boundary', async function () {
+        // Boundary is 2025-06-15 10:00:00 UTC. Only "early-same-day" (09:00) is before it.
+        const titles = await titlesFor("published_at:<'2025-06-15T10:00:00.000Z'");
+
+        assert.deepEqual(titles, ['early-same-day']);
+    });
+
+    it('handles an ISO boundary with a timezone offset', async function () {
+        // 2025-06-15T05:00:00-05:00 is 2025-06-15 10:00:00 UTC, same boundary as above.
+        const titles = await titlesFor("published_at:>'2025-06-15T05:00:00.000-05:00'");
+
+        assert.deepEqual(titles, ['late-same-day', 'later-day']);
+    });
+});

ghost/core/test/unit/server/models/base/date-filter.test.js

@@ -0,0 +1,110 @@
+const assert = require('node:assert/strict');
+const {normalizeDateFilters} = require('../../../../../core/server/models/base/plugins/date-filter');
+
+describe('Models: date-filter', function () {
+    describe('normalizeDateFilters', function () {
+        it('normalizes an ISO date with a timezone offset on a date column to UTC db format', function () {
+            const result = normalizeDateFilters({
+                published_at: {$gt: '2025-02-27T19:03:00.000-05:00'}
+            });
+
+            assert.deepEqual(result, {
+                published_at: {$gt: '2025-02-28 00:03:00'}
+            });
+        });
+
+        it('normalizes a Zulu ISO date on a date column', function () {
+            const result = normalizeDateFilters({
+                published_at: {$lt: '2025-02-27T19:03:00Z'}
+            });
+
+            assert.deepEqual(result, {
+                published_at: {$lt: '2025-02-27 19:03:00'}
+            });
+        });
+
+        it('normalizes an equality value on a date column', function () {
+            const result = normalizeDateFilters({
+                created_at: '2025-02-27T19:03:00.000Z'
+            });
+
+            assert.deepEqual(result, {
+                created_at: '2025-02-27 19:03:00'
+            });
+        });
+
+        it('leaves values already in db format untouched', function () {
+            const filter = {published_at: {$gt: '2025-02-27 19:03:00'}};
+
+            assert.deepEqual(normalizeDateFilters(filter), {
+                published_at: {$gt: '2025-02-27 19:03:00'}
+            });
+        });
+
+        it('does not touch non-date columns even when the value looks like a date', function () {
+            const filter = {slug: '2025-02-27', title: {$ne: '2025-02-27T19:03:00Z'}};
+
+            assert.deepEqual(normalizeDateFilters(filter), filter);
+        });
+
+        it('leaves unparseable values on a date column untouched', function () {
+            const filter = {published_at: {$gt: 'not-a-date'}};
+
+            assert.deepEqual(normalizeDateFilters(filter), filter);
+        });
+
+        it('leaves a non-plain object value (e.g. a Date) on a date column untouched', function () {
+            const date = new Date('2025-02-27T19:03:00Z');
+            const result = normalizeDateFilters({published_at: date});
+
+            assert.equal(result.published_at, date);
+        });
+
+        it('normalizes arrays of values ($in)', function () {
+            const result = normalizeDateFilters({
+                published_at: {$in: ['2025-02-27T19:03:00Z', '2025-03-01T00:00:00Z']}
+            });
+
+            assert.deepEqual(result, {
+                published_at: {$in: ['2025-02-27 19:03:00', '2025-03-01 00:00:00']}
+            });
+        });
+
+        it('recurses into $and / $or groups', function () {
+            const result = normalizeDateFilters({
+                $and: [
+                    {published_at: {$gt: '2025-02-27T19:03:00Z'}},
+                    {$or: [
+                        {featured: true},
+                        {updated_at: {$lt: '2025-03-01T00:00:00Z'}}
+                    ]}
+                ]
+            });
+
+            assert.deepEqual(result, {
+                $and: [
+                    {published_at: {$gt: '2025-02-27 19:03:00'}},
+                    {$or: [
+                        {featured: true},
+                        {updated_at: {$lt: '2025-03-01 00:00:00'}}
+                    ]}
+                ]
+            });
+        });
+
+        it('resolves relation-qualified date columns by column name', function () {
+            const result = normalizeDateFilters({
+                'posts.published_at': {$gt: '2025-02-27T19:03:00Z'}
+            });
+
+            assert.deepEqual(result, {
+                'posts.published_at': {$gt: '2025-02-27 19:03:00'}
+            });
+        });
+
+        it('returns primitive nodes unchanged', function () {
+            assert.equal(normalizeDateFilters(null), null);
+            assert.equal(normalizeDateFilters('string'), 'string');
+        });
+    });
+});