A task to move from sqlite to postgres

Author: mike-bestcat

src/alib/py/framework/jupiter/framework/appform/webapi/exceptions.py

@@ -1,6 +1,7 @@
 """Standard exceptions for the web API."""
 
 from json import JSONDecodeError
+import logging
 
 from jupiter.framework.appform.webapi.exception import (
     WebApiError,

src/alib/py/framework/jupiter/framework/realm/standard.py

@@ -820,6 +820,36 @@ def encode(self, value: _CompositeValueT) -> RealmThing:
         return result
 
 
+def _database_composite_json_maybe_parse(value: RealmThing) -> RealmThing:
+    """Normalize DB payloads for :class:`CompositeValue` decode.
+
+    VARCHAR / text columns and some SQLite→Postgres loaders expose JSON objects
+    as a single ``str`` (sometimes nested JSON-as-string). Accept ``dict``,
+    ``Mapping``, or parseable JSON text; otherwise return ``value`` unchanged so
+    callers can raise a precise error.
+    """
+    node: RealmThing = value
+    for _ in range(12):
+        if isinstance(node, dict):
+            return node
+        if isinstance(node, Mapping):
+            return dict(node)
+        if isinstance(node, (bytes, bytearray)):
+            node = node.decode()
+            continue
+        if isinstance(node, str):
+            stripped = node.strip()
+            if stripped == "" or stripped.lower() == "null":
+                return node
+            try:
+                node = json.loads(node)
+            except json.JSONDecodeError:
+                return node
+            continue
+        break
+    return node
+
+
 class _StandardCompositeValueDecoder(
     RealmDecoder[_CompositeValueT, _RealmT], Generic[_CompositeValueT, _RealmT]
 ):
@@ -840,6 +870,8 @@ def __init__(
         self._realm = realm
 
     def decode(self, value: RealmThing) -> _CompositeValueT:
+        if self._realm is DatabaseRealm:
+            value = _database_composite_json_maybe_parse(value)
         if not isinstance(value, dict):
             raise RealmDecodingError(
                 f"Expected value for {self._the_type.__name__} to be a dict object"

src/core/jupiter/core/application/impl/postgres_repository.py

@@ -24,7 +24,7 @@
 from jupiter.core.big_plans.name import BigPlanName
 from jupiter.core.chores.name import ChoreName
 from jupiter.core.common.entity_icon import EntityIconDatabaseDecoder
-from jupiter.core.common.recurring_task_period import RecurringTaskPeriod
+from jupiter.core.common.recurring_task_gen_params import RecurringTaskGenParams
 from jupiter.core.common.sub.contacts.sub.contact.name import ContactName
 from jupiter.core.common.sub.inbox_tasks.name import InboxTaskName
 from jupiter.core.habits.name import HabitName
@@ -373,12 +373,7 @@ async def find_all_habit_summaries(
     ) -> list[HabitSummary]:
         """Find all summaries about habits."""
         query = """
-            select
-                ref_id,
-                name,
-                is_key,
-                gen_params->>'period' as period,
-                aspect_ref_id
+            select ref_id, name, is_key, gen_params, aspect_ref_id
             from habit
             where habit_collection_ref_id = :parent_ref_id
         """
@@ -399,8 +394,9 @@ async def find_all_habit_summaries(
                 name=_HABIT_NAME_DECODER.decode(row["name"]),
                 is_key=row["is_key"],
                 period=self._realm_codec_registry.db_decode(
-                    RecurringTaskPeriod, row["period"]
-                ),
+                    RecurringTaskGenParams,
+                    row["gen_params"],
+                ).period,
                 aspect_ref_id=_ENTITY_ID_DECODER.decode(str(row["aspect_ref_id"])),
             )
             for row in result

src/core/jupiter/core/search/use_case/search_index_backfill_do_all.py

@@ -338,13 +338,13 @@ async def _execute(
                             entity_type,
                             s.ref_id,
                         )
-                        LOGGER.info(
-                            "Indexing %s:%s => %s time=%s",
-                            tag,
-                            s.ref_id,
-                            object_id,
-                            s.last_modified_time.value,
-                        )
+                        # LOGGER.info(
+                        #     "Indexing %s:%s => %s time=%s",
+                        #     tag,
+                        #     s.ref_id,
+                        #     object_id,
+                        #     s.last_modified_time.value,
+                        # )
 
                         indexed_here += 1
 

src/core/migrations/postgres/versions/2026_05_04_18_60_initial_reset.py

@@ -246,12 +246,19 @@ def upgrade() -> None:
             big_plan_cnt INTEGER NOT NULL,
             created_time TIMESTAMP WITH TIME ZONE NOT NULL,
             last_modified_time TIMESTAMP WITH TIME ZONE NOT NULL,
-            PRIMARY KEY (score_log_ref_id, period, timeline),
             FOREIGN KEY (score_log_ref_id) REFERENCES gamification_score_log (ref_id)
         )
     """
     )
 
+    op.execute(
+        """
+        CREATE UNIQUE INDEX uq_gamification_score_stats_natural_key
+            ON gamification_score_stats (score_log_ref_id, period, timeline)
+            NULLS NOT DISTINCT
+    """
+    )
+
     op.execute(
         """
         CREATE INDEX ix_gamification_score_stats_score_log_ref_id_period_created_time
@@ -271,12 +278,24 @@ def upgrade() -> None:
             big_plan_cnt INTEGER NOT NULL,
             created_time TIMESTAMP WITH TIME ZONE NOT NULL,
             last_modified_time TIMESTAMP WITH TIME ZONE NOT NULL,
-            PRIMARY KEY (score_log_ref_id, period, timeline, sub_period),
             FOREIGN KEY (score_log_ref_id) REFERENCES gamification_score_log (ref_id)
         )
     """
     )
 
+    op.execute(
+        """
+        CREATE UNIQUE INDEX uq_gamification_score_period_best_natural_key
+            ON gamification_score_period_best (
+                score_log_ref_id,
+                period,
+                timeline,
+                sub_period
+            )
+            NULLS NOT DISTINCT
+    """
+    )
+
     # -------------------------------------------------------------------------
     # Vacation
     # -------------------------------------------------------------------------
@@ -526,7 +545,7 @@ def upgrade() -> None:
             archived_time TIMESTAMP WITH TIME ZONE,
             smart_list_collection_ref_id INTEGER,
             name VARCHAR(100) NOT NULL,
-            icon VARCHAR(4),
+            icon VARCHAR(64),
             archival_reason VARCHAR,
             CONSTRAINT pk_smart_list PRIMARY KEY (ref_id),
             CONSTRAINT fk_smart_list_smart_list_collection_ref_id_smart_list_collection
@@ -832,7 +851,7 @@ def upgrade() -> None:
             last_modified_time TIMESTAMP WITH TIME ZONE NOT NULL,
             archived_time TIMESTAMP WITH TIME ZONE,
             doc_collection_ref_id INTEGER NOT NULL,
-            name VARCHAR(64) NOT NULL,
+            name VARCHAR(255) NOT NULL,
             archival_reason VARCHAR,
             idempotency_key VARCHAR(36),
             parent_dir_ref_id INTEGER NOT NULL,
@@ -899,7 +918,7 @@ def upgrade() -> None:
             metric_collection_ref_id INTEGER,
             name VARCHAR NOT NULL,
             metric_unit VARCHAR,
-            icon VARCHAR(4),
+            icon VARCHAR(64),
             collection_params JSONB,
             archival_reason VARCHAR,
             is_key BOOLEAN NOT NULL,
@@ -1288,7 +1307,6 @@ def upgrade() -> None:
             ON mutation_entity_event (entity_ref_id, name, timestamp, session_index)
     """
     )
-
     op.execute(
         """
         CREATE INDEX ix_mutation_entity_event_entity_type_entity_ref_id_timestamp
@@ -2497,7 +2515,7 @@ def upgrade() -> None:
             archived_time TIMESTAMP WITH TIME ZONE,
             schedule_external_sync_log_ref_id INTEGER NOT NULL,
             name VARCHAR(64) NOT NULL,
-            source VARCHAR(16) NOT NULL,
+            source VARCHAR(64) NOT NULL,
             filter_schedule_stream_ref_id JSONB,
             opened BOOLEAN NOT NULL,
             per_stream_results JSONB NOT NULL,

src/core/migrations/sqlite/versions/2026_05_04_18_60_initial_reset.py

@@ -1,6 +1,6 @@
 """Initial reset of the database.
 
-Revision ID: 6e5f8bc8fcd2
+Revision ID: 8d7c3e9f2a14
 Revises: None
 Create Date: 2026-05-04 18:60:00.000000
 
@@ -9,7 +9,7 @@
 from alembic import op
 
 
-revision = "6e5f8bc8fcd2"
+revision = "8d7c3e9f2a14"
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -526,7 +526,7 @@ def upgrade() -> None:
             archived_time DATETIME,
             smart_list_collection_ref_id INTEGER,
             name VARCHAR(100) NOT NULL,
-            icon VARCHAR(4),
+            icon VARCHAR(64),
             archival_reason VARCHAR,
             CONSTRAINT pk_smart_list PRIMARY KEY (ref_id),
             CONSTRAINT fk_smart_list_smart_list_collection_ref_id_smart_list_collection
@@ -832,7 +832,7 @@ def upgrade() -> None:
             last_modified_time DATETIME NOT NULL,
             archived_time DATETIME,
             doc_collection_ref_id INTEGER NOT NULL,
-            name VARCHAR(64) NOT NULL,
+            name VARCHAR(255) NOT NULL,
             archival_reason VARCHAR,
             idempotency_key VARCHAR(36),
             parent_dir_ref_id INTEGER NOT NULL,
@@ -899,7 +899,7 @@ def upgrade() -> None:
             metric_collection_ref_id INTEGER,
             name VARCHAR NOT NULL,
             metric_unit VARCHAR,
-            icon VARCHAR(4),
+            icon VARCHAR(64),
             collection_params JSON,
             archival_reason VARCHAR,
             is_key BOOLEAN NOT NULL,
@@ -2490,7 +2490,7 @@ def upgrade() -> None:
             archived_time DATETIME,
             schedule_external_sync_log_ref_id INTEGER NOT NULL,
             name VARCHAR(64) NOT NULL,
-            source VARCHAR(16) NOT NULL,
+            source VARCHAR(64) NOT NULL,
             filter_schedule_stream_ref_id JSON,
             opened BOOLEAN NOT NULL,
             per_stream_results JSON NOT NULL,