From 7a6bb477e646acd8d48bcafdab01e5a51ee054ee Mon Sep 17 00:00:00 2001 From: Kris Powell Date: Wed, 19 Nov 2025 10:51:47 +1100 Subject: [PATCH 1/6] Add datetime: null when an item has start_datetime and end_datetime but no datetime. --- src/pgstac/sql/003a_items.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pgstac/sql/003a_items.sql b/src/pgstac/sql/003a_items.sql index 88924d17..2b1cf217 100644 --- a/src/pgstac/sql/003a_items.sql +++ b/src/pgstac/sql/003a_items.sql @@ -145,6 +145,12 @@ BEGIN fields ); + IF (output->'properties' ? 'start_datetime') + AND (output->'properties' ? 'end_datetime') + AND NOT (output->'properties' ? 'datetime') THEN + output := jsonb_set(output, '{properties,datetime}', 'null'::jsonb); + END IF; + RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; From cebe263ec88ddab0d296ff9f8a3bdf43beee89d1 Mon Sep 17 00:00:00 2001 From: Kris Powell Date: Tue, 3 Feb 2026 14:10:11 +1100 Subject: [PATCH 2/6] Move datetime:null fix to jsonb content_hydrate to cover all hydration paths --- src/pgstac/sql/003a_items.sql | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/pgstac/sql/003a_items.sql b/src/pgstac/sql/003a_items.sql index 2b1cf217..680caf07 100644 --- a/src/pgstac/sql/003a_items.sql +++ b/src/pgstac/sql/003a_items.sql @@ -115,11 +115,21 @@ CREATE OR REPLACE FUNCTION content_hydrate( _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ - SELECT merge_jsonb( - jsonb_fields(_item, fields), - jsonb_fields(_base_item, fields) +DECLARE + output jsonb; +BEGIN + output := merge_jsonb( + jsonb_fields(_item, fields), + jsonb_fields(_base_item, fields) ); -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + IF (output->'properties' ? 'start_datetime') + AND (output->'properties' ? 'end_datetime') + AND NOT (output->'properties' ? 'datetime') THEN + output := jsonb_set(output, '{properties,datetime}', 'null'::jsonb); + END IF; + RETURN output; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; @@ -145,12 +155,6 @@ BEGIN fields ); - IF (output->'properties' ? 'start_datetime') - AND (output->'properties' ? 'end_datetime') - AND NOT (output->'properties' ? 'datetime') THEN - output := jsonb_set(output, '{properties,datetime}', 'null'::jsonb); - END IF; - RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; From ddc398785e19495ab4194eafed6ae778b6de7992 Mon Sep 17 00:00:00 2001 From: Kris Powell Date: Tue, 3 Feb 2026 14:13:11 +1100 Subject: [PATCH 3/6] Add pgtap and basic tests for datetime:null hydration --- src/pgstac/tests/basic/hydration.sql | 19 +++++++++++++++++++ src/pgstac/tests/basic/hydration.sql.out | 20 ++++++++++++++++++++ src/pgstac/tests/pgtap/003_items.sql | 19 +++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 src/pgstac/tests/basic/hydration.sql create mode 100644 src/pgstac/tests/basic/hydration.sql.out diff --git a/src/pgstac/tests/basic/hydration.sql b/src/pgstac/tests/basic/hydration.sql new file mode 100644 index 00000000..881441e0 --- /dev/null +++ b/src/pgstac/tests/basic/hydration.sql @@ -0,0 +1,19 @@ +-- Test STAC datetime:null compliance during hydration +SET ROLE pgstac_ingest; + +-- Setup: collection with empty base_item +INSERT INTO collections (content) VALUES ('{"id":"pgstactest-hydration"}'); + +-- Create item with start/end_datetime but no datetime +SELECT create_item('{ + "id": "temporal-range-item", + "collection": "pgstactest-hydration", + "geometry": {"type": "Point", "coordinates": [0, 0]}, + "properties": { + "start_datetime": "2026-01-01T00:00:00Z", + "end_datetime": "2026-01-31T23:00:00Z" + } +}'); + +-- Verify hydrated item has datetime:null (STAC compliance) +SELECT get_item('temporal-range-item', 'pgstactest-hydration')->'properties'->'datetime'; \ No newline at end of file diff --git a/src/pgstac/tests/basic/hydration.sql.out b/src/pgstac/tests/basic/hydration.sql.out new file mode 100644 index 00000000..ee844942 --- /dev/null +++ b/src/pgstac/tests/basic/hydration.sql.out @@ -0,0 +1,20 @@ +-- Test STAC datetime:null compliance during hydration +SET ROLE pgstac_ingest; +SET +-- Setup: collection with empty base_item +INSERT INTO collections (content) VALUES ('{"id":"pgstactest-hydration"}'); +INSERT 0 1 +-- Create item with start/end_datetime but no datetime +SELECT create_item('{ + "id": "temporal-range-item", + "collection": "pgstactest-hydration", + "geometry": {"type": "Point", "coordinates": [0, 0]}, + "properties": { + "start_datetime": "2026-01-01T00:00:00Z", + "end_datetime": "2026-01-31T23:00:00Z" + } +}'); + +-- Verify hydrated item has datetime:null (STAC compliance) +SELECT get_item('temporal-range-item', 'pgstactest-hydration')->'properties'->'datetime'; + null diff --git a/src/pgstac/tests/pgtap/003_items.sql b/src/pgstac/tests/pgtap/003_items.sql index ddebf80a..8f4355c0 100644 --- a/src/pgstac/tests/pgtap/003_items.sql +++ b/src/pgstac/tests/pgtap/003_items.sql @@ -52,3 +52,22 @@ SELECT results_eq($$ $$, 'Test delete_item function' ); + +-- content_hydrate: STAC spec requires datetime:null when temporal range present +SELECT results_eq( + $$ SELECT (content_hydrate( + '{"properties": {"start_datetime": "2026-01-01T00:00:00Z", "end_datetime": "2026-01-31T23:00:00Z"}}'::jsonb, + '{"properties": {}}'::jsonb + )->'properties') ? 'datetime' $$, + $$ SELECT true $$, + 'content_hydrate adds datetime key when start/end_datetime present' +); + +SELECT results_eq( + $$ SELECT content_hydrate( + '{"properties": {"datetime": "2026-01-15T12:00:00Z", "start_datetime": "2026-01-01T00:00:00Z", "end_datetime": "2026-01-31T23:00:00Z"}}'::jsonb, + '{"properties": {}}'::jsonb + )->'properties'->>'datetime' $$, + $$ SELECT '2026-01-15T12:00:00Z' $$, + 'content_hydrate preserves existing datetime' +); From 12c7d4be4358216550a51b9966aefea81ae0ce70 Mon Sep 17 00:00:00 2001 From: Kris Powell Date: Tue, 3 Feb 2026 14:24:50 +1100 Subject: [PATCH 4/6] Update CHANGELOG.md with fix and linked issue --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ced94bc4..490f71ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [UNRELEASED] +### Fixed +- Add datetime: null when an item has start_datetime and end_datetime but no datetime. Fixes (#158) + ## [v0.9.8] ### Fixed - Allow array as q parameter for full text search From 9379f5c95acd7663a1355548e6bb69b28b363030 Mon Sep 17 00:00:00 2001 From: Kris Powell Date: Wed, 11 Feb 2026 17:22:12 +1100 Subject: [PATCH 5/6] Revert content_hydrate datetime:null patch in favour of root-cause fix --- src/pgstac/sql/003a_items.sql | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/pgstac/sql/003a_items.sql b/src/pgstac/sql/003a_items.sql index 680caf07..88924d17 100644 --- a/src/pgstac/sql/003a_items.sql +++ b/src/pgstac/sql/003a_items.sql @@ -115,21 +115,11 @@ CREATE OR REPLACE FUNCTION content_hydrate( _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ -DECLARE - output jsonb; -BEGIN - output := merge_jsonb( - jsonb_fields(_item, fields), - jsonb_fields(_base_item, fields) + SELECT merge_jsonb( + jsonb_fields(_item, fields), + jsonb_fields(_base_item, fields) ); - IF (output->'properties' ? 'start_datetime') - AND (output->'properties' ? 'end_datetime') - AND NOT (output->'properties' ? 'datetime') THEN - output := jsonb_set(output, '{properties,datetime}', 'null'::jsonb); - END IF; - RETURN output; -END; -$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; From 3b69add38dd880f6a00a460cc76e28b5772aad83 Mon Sep 17 00:00:00 2001 From: Kris Powell Date: Wed, 11 Feb 2026 17:26:41 +1100 Subject: [PATCH 6/6] Preserve JSON null values in merge_jsonb and strip_jsonb --- src/pgstac/sql/001a_jsonutils.sql | 53 ++++++++++-------------- src/pgstac/tests/basic/hydration.sql | 12 ++++-- src/pgstac/tests/basic/hydration.sql.out | 11 +++-- src/pgstac/tests/pgtap/003_items.sql | 22 +++++----- 4 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/pgstac/sql/001a_jsonutils.sql b/src/pgstac/sql/001a_jsonutils.sql index 9fcb2375..2ff94ac6 100644 --- a/src/pgstac/sql/001a_jsonutils.sql +++ b/src/pgstac/sql/001a_jsonutils.sql @@ -108,12 +108,7 @@ BEGIN THEN RETURN j; ELSE - includes := includes || ( - CASE WHEN j ? 'collection' THEN - '["id","collection"]' - ELSE - '["id"]' - END)::jsonb; + includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; @@ -151,21 +146,19 @@ CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL - WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b + WHEN _a IS NULL THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( - SELECT - jsonb_strip_nulls( - jsonb_object_agg( - key, - merge_jsonb(a.value, b.value) - ) - ) - FROM - jsonb_each(coalesce(_a,'{}'::jsonb)) as a - FULL JOIN - jsonb_each(coalesce(_b,'{}'::jsonb)) as b - USING (key) + SELECT jsonb_object_agg(key, val) + FROM ( + SELECT key, merge_jsonb(a.value, b.value) AS val + FROM + jsonb_each(coalesce(_a,'{}'::jsonb)) as a + FULL JOIN + jsonb_each(coalesce(_b,'{}'::jsonb)) as b + USING (key) + ) sub + WHERE val IS NOT NULL ) WHEN jsonb_typeof(_a) = 'array' @@ -196,18 +189,16 @@ CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( - SELECT - jsonb_strip_nulls( - jsonb_object_agg( - key, - strip_jsonb(a.value, b.value) - ) - ) - FROM - jsonb_each(_a) as a - FULL JOIN - jsonb_each(_b) as b - USING (key) + SELECT jsonb_object_agg(key, val) + FROM ( + SELECT key, strip_jsonb(a.value, b.value) AS val + FROM + jsonb_each(_a) as a + FULL JOIN + jsonb_each(_b) as b + USING (key) + ) sub + WHERE val IS NOT NULL ) WHEN jsonb_typeof(_a) = 'array' diff --git a/src/pgstac/tests/basic/hydration.sql b/src/pgstac/tests/basic/hydration.sql index 881441e0..88508762 100644 --- a/src/pgstac/tests/basic/hydration.sql +++ b/src/pgstac/tests/basic/hydration.sql @@ -1,19 +1,23 @@ --- Test STAC datetime:null compliance during hydration +-- Test that JSON null values survive the dehydrate/hydrate round-trip SET ROLE pgstac_ingest; -- Setup: collection with empty base_item INSERT INTO collections (content) VALUES ('{"id":"pgstactest-hydration"}'); --- Create item with start/end_datetime but no datetime +-- Create item with explicit datetime:null (STAC-compliant temporal range) SELECT create_item('{ "id": "temporal-range-item", "collection": "pgstactest-hydration", "geometry": {"type": "Point", "coordinates": [0, 0]}, "properties": { + "datetime": null, "start_datetime": "2026-01-01T00:00:00Z", "end_datetime": "2026-01-31T23:00:00Z" } }'); --- Verify hydrated item has datetime:null (STAC compliance) -SELECT get_item('temporal-range-item', 'pgstactest-hydration')->'properties'->'datetime'; \ No newline at end of file +-- Verify datetime:null is preserved in stored content +SELECT content->'properties'->'datetime' FROM items WHERE id='temporal-range-item'; + +-- Verify datetime:null is preserved after hydration +SELECT get_item('temporal-range-item', 'pgstactest-hydration')->'properties'->'datetime'; diff --git a/src/pgstac/tests/basic/hydration.sql.out b/src/pgstac/tests/basic/hydration.sql.out index ee844942..cbdc12a1 100644 --- a/src/pgstac/tests/basic/hydration.sql.out +++ b/src/pgstac/tests/basic/hydration.sql.out @@ -1,20 +1,25 @@ --- Test STAC datetime:null compliance during hydration +-- Test that JSON null values survive the dehydrate/hydrate round-trip SET ROLE pgstac_ingest; SET -- Setup: collection with empty base_item INSERT INTO collections (content) VALUES ('{"id":"pgstactest-hydration"}'); INSERT 0 1 --- Create item with start/end_datetime but no datetime +-- Create item with explicit datetime:null (STAC-compliant temporal range) SELECT create_item('{ "id": "temporal-range-item", "collection": "pgstactest-hydration", "geometry": {"type": "Point", "coordinates": [0, 0]}, "properties": { + "datetime": null, "start_datetime": "2026-01-01T00:00:00Z", "end_datetime": "2026-01-31T23:00:00Z" } }'); --- Verify hydrated item has datetime:null (STAC compliance) +-- Verify datetime:null is preserved in stored content +SELECT content->'properties'->'datetime' FROM items WHERE id='temporal-range-item'; + null + +-- Verify datetime:null is preserved after hydration SELECT get_item('temporal-range-item', 'pgstactest-hydration')->'properties'->'datetime'; null diff --git a/src/pgstac/tests/pgtap/003_items.sql b/src/pgstac/tests/pgtap/003_items.sql index 8f4355c0..c4efdcac 100644 --- a/src/pgstac/tests/pgtap/003_items.sql +++ b/src/pgstac/tests/pgtap/003_items.sql @@ -53,21 +53,21 @@ SELECT results_eq($$ 'Test delete_item function' ); --- content_hydrate: STAC spec requires datetime:null when temporal range present +-- merge_jsonb and strip_jsonb must preserve JSON null values SELECT results_eq( - $$ SELECT (content_hydrate( - '{"properties": {"start_datetime": "2026-01-01T00:00:00Z", "end_datetime": "2026-01-31T23:00:00Z"}}'::jsonb, + $$ SELECT merge_jsonb( + '{"properties": {"datetime": null, "start_datetime": "2026-01-01T00:00:00Z", "end_datetime": "2026-01-31T23:00:00Z"}}'::jsonb, '{"properties": {}}'::jsonb - )->'properties') ? 'datetime' $$, - $$ SELECT true $$, - 'content_hydrate adds datetime key when start/end_datetime present' + )->'properties'->'datetime' $$, + $$ SELECT 'null'::jsonb $$, + 'merge_jsonb preserves explicit JSON null values' ); SELECT results_eq( - $$ SELECT content_hydrate( - '{"properties": {"datetime": "2026-01-15T12:00:00Z", "start_datetime": "2026-01-01T00:00:00Z", "end_datetime": "2026-01-31T23:00:00Z"}}'::jsonb, + $$ SELECT strip_jsonb( + '{"properties": {"datetime": null, "start_datetime": "2026-01-01T00:00:00Z", "end_datetime": "2026-01-31T23:00:00Z"}}'::jsonb, '{"properties": {}}'::jsonb - )->'properties'->>'datetime' $$, - $$ SELECT '2026-01-15T12:00:00Z' $$, - 'content_hydrate preserves existing datetime' + )->'properties'->'datetime' $$, + $$ SELECT 'null'::jsonb $$, + 'strip_jsonb preserves explicit JSON null values' );