TimescaleDB local to cloud sync, wrong DBC warning

.github/workflows/offline-bundle.yml

@@ -0,0 +1,83 @@
+name: Offline Docker Bundle
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Git ref to package (branch, tag, SHA)'
+        required: true
+        default: 'timescale-sync'
+
+  release:
+    types: [published]
+
+  push:
+    branches: [ timescale-sync, main, dev ]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  package-offline:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.ref || github.ref }}
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build cloud-sync image
+        run: |
+          docker compose -f universal-telemetry-software/deploy/docker-compose.macbook-base.yml build cloud-sync
+
+      - name: Pull all required images
+        run: |
+          docker pull ghcr.io/western-formula-racing/daq-radio/universal-telemetry:latest
+          docker pull ghcr.io/western-formula-racing/daq-radio/pecan:latest
+          docker pull timescale/timescaledb:latest-pg16
+          docker pull redis:8.2
+          docker pull bluenviron/mediamtx:latest
+          docker pull grafana/grafana:latest
+
+      - name: Save images as tarball
+        run: |
+          cd universal-telemetry-software/deploy
+          docker save \
+            ghcr.io/western-formula-racing/daq-radio/universal-telemetry:latest \
+            ghcr.io/western-formula-racing/daq-radio/pecan:latest \
+            timescale/timescaledb:latest-pg16 \
+            redis:8.2 \
+            bluenviron/mediamtx:latest \
+            deploy-cloud-sync:latest \
+            grafana/grafana:latest \
+            -o offline/wfr-docker-images.tar
+
+      - name: Upload as workflow artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: wfr-docker-images-offline
+          path: universal-telemetry-software/deploy/offline/wfr-docker-images.tar
+          retention-days: 30
+
+      - name: Upload to Release
+        if: github.event_name == 'release'
+        uses: softprops/action-gh-release@v1
+        with:
+          files: universal-telemetry-software/deploy/offline/wfr-docker-images.tar
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

pecan/src/pages/Trace.tsx

@@ -458,7 +458,6 @@ function Trace() {
 
   const handleClear = useCallback(() => {
     clearTrace();
-    frozenRef.current = [];
   }, [clearTrace]);
 
   // Handle Easter Egg Trigger

server/installer/sandbox/Dockerfile

@@ -12,7 +12,9 @@ RUN python -c "from langchain_community.embeddings import FastEmbedEmbeddings; F
 # Copy application code
 COPY code_generator.py .
 COPY stats_report.py .
-COPY prompt-guide.txt prompt-guide.txt.example ./
+COPY prompt-guide.txt.example ./
+# prompt-guide.txt is optional (local customization); fall back to example if absent
+RUN if [ ! -f prompt-guide.txt ]; then cp prompt-guide.txt.example prompt-guide.txt; fi
 
 # Create directories for generated code, ChromaDB, and cache
 RUN mkdir -p /app/generated /app/chroma_db /app/cache

universal-telemetry-software/checklist/SETUP_CARD.tex

@@ -545,7 +545,7 @@ \section*{9 -- TIMESCALE LOGGING (MacBook Only)}
   TimescaleDB runs on MacBook's \cmd{macbook-base} stack only. \\
   RPi base (\cmd{rpi-base}) does NOT write to DB -- it relays to MacBook via UDP/TCP. \\[1mm]
   Connect with \cmd{psql}: \cmd{psql postgresql://wfr:wfr\_password@localhost:5432/wfr} \\
-  Check tables: \cmd{\textbackslash dt} \quad Check data: \cmd{SELECT COUNT(*) FROM wfr26test\_base;} \\
+  Check tables: \cmd{\textbackslash dt} \quad Check data: \cmd{SELECT COUNT(*) FROM wfr26base;} \\
   Grafana: \cmd{http://localhost:8087} \quad user: \cmd{admin} \quad pass: \cmd{admin} (or as set in \cmd{.env}) \\
 \end{tabular}
 
@@ -555,7 +555,7 @@ \section*{9 -- TIMESCALE LOGGING (MacBook Only)}
 \noindent
 \begin{tabular}{p{126mm}}
   \cmd{docker logs daq-telemetry $\vert$ grep "table="} \\[1mm]
-  \textbf{PASS:} \cmd{table=WFR26test\_base} (or as set by \cmd{TIMESCALE\_TABLE}) \\
+  \textbf{PASS:} \cmd{table=wfr26base} (or as set by \cmd{TIMESCALE\_TABLE}) \\
   \textbf{FAIL:} \cmd{ENABLE\_TIMESCALE\_LOGGING=true} not set, or TimescaleDB not reachable \\
   \textit{Schema:} wide format -- one row per CAN message, each decoded signal as a column. \\
   \textit{Table is created automatically on first write.} \\
@@ -635,7 +635,7 @@ \section*{10 -- TIMESYNCALEDB BRIDGE (Base Only)}
   It reads decoded CAN frames from Redis and writes them directly to the server stack's \\
   TimescaleDB over the network (writes to \cmd{POSTGRES\_DSN}). No local TimescaleDB. \\[1mm]
   Wide format: one row per CAN message, all signals as columns. \\
-  Table: \cmd{WFR26test\_base} (MacBook) or as set by \cmd{TIMESCALE\_TABLE}. \\
+  Table: \cmd{wfr26base} (MacBook) or as set by \cmd{TIMESCALE\_TABLE}. \\
 \end{tabular}
 
 \vspace{1mm}
@@ -710,6 +710,7 @@ \section*{14 -- KEY PORTS}
   5006 & TCP & Packet retransmission \\
   5432 & TCP & TimescaleDB (MacBook only) \\
   6379 & TCP & Redis \\
+  8092 & HTTP & Cloud Sync dashboard (MacBook only) \\
   8080 & HTTP & Status monitoring page \\
   8087 & HTTP & Grafana dashboards (MacBook) \\
   9080 & WebSocket & PECAN telemetry feed (plain WS) \\
@@ -741,7 +742,7 @@ \section*{15 -- ENV VAR QUICKREF}
   ENABLE\_VIDEO & false & Video streaming \\
   ENABLE\_AUDIO & false & Audio streaming \\
   ENABLE\_TIMESCALE\_LOGGING & false & Log telemetry to TimescaleDB (MacBook only) \\
-  TIMESCALE\_TABLE & WFR26test\_base & Table name for telemetry writes \\
+  TIMESCALE\_TABLE & wfr26base & Table name for telemetry writes \\
   POSTGRES\_DSN & (local) & TimescaleDB connection string (MacBook: local; RPi: points to MacBook) \\
   ENABLE\_WS\_RELAY & false & Start WS relay (ws\_relay.py) on port 9089 \\
   RELAY\_UPSTREAM\_WS & ws://127.0.0.1:9080 & Upstream WS URL for relay \\

universal-telemetry-software/cloud-sync/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+WORKDIR /app
+
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+
+COPY config.py sync.py app.py ./
+COPY static/ ./static/
+
+EXPOSE 8092
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8092/api/status')"
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8092"]

universal-telemetry-software/cloud-sync/app.py

@@ -0,0 +1,215 @@
+import logging
+import time
+from datetime import datetime, timezone
+
+from fastapi import BackgroundTasks, FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
+
+import config
+from sync import SyncEngine
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(name)s %(levelname)s %(message)s",
+)
+logger = logging.getLogger("cloud-sync")
+
+app = FastAPI(title="WFR Cloud Sync")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+app.mount("/static", StaticFiles(directory="static"), name="static")
+
+engine = SyncEngine()
+
+# ── Sync state (module-level, single process) ─────────────────────────────────
+
+_sync_state: dict = {
+    "running": False,
+    "rows_done": 0,
+    "rows_total": 0,
+    "last_sync_iso": None,      # ISO timestamp of last completed sync
+    "last_sync_rows": None,     # row count of last completed sync
+    "last_sync_elapsed": None,  # seconds
+    "last_error": None,
+    # cached from last status call so /api/status is fast
+    "_cloud_cursor": None,
+    "_unsynced_count": None,
+    "_unsynced_ts": 0.0,        # monotonic time of last unsynced_count fetch
+}
+
+_UNSYNCED_CACHE_TTL = 30.0  # seconds
+
+
+@app.get("/")
+def root():
+    return RedirectResponse(url="/static/index.html")
+
+
+@app.get("/api/status")
+def status():
+    # Local count — always fresh (fast local query)
+    try:
+        local_count = engine.get_local_count()
+    except Exception as e:
+        local_count = None
+        logger.warning(f"get_local_count failed: {e}")
+
+    # Unsynced count — cached with TTL to avoid hammering cloud on every poll
+    now = time.monotonic()
+    if now - _sync_state["_unsynced_ts"] > _UNSYNCED_CACHE_TTL and not _sync_state["running"]:
+        try:
+            cursor = engine.get_cloud_cursor()
+            _sync_state["_cloud_cursor"] = cursor.isoformat() if cursor else None
+            _sync_state["_unsynced_count"] = engine.get_unsynced_count(cursor)
+            _sync_state["_unsynced_ts"] = now
+        except Exception as e:
+            logger.warning(f"unsynced_count fetch failed: {e}")
+
+    cloud_configured = bool(config.CLOUD_POSTGRES_DSN)
+
+    return {
+        "local_count": local_count,
+        "local_table": config.LOCAL_TABLE,
+        "cloud_table": engine.cloud_table,
+        "cloud_configured": cloud_configured,
+        "cloud_cursor": _sync_state["_cloud_cursor"],
+        "unsynced_count": _sync_state["_unsynced_count"],
+        "last_sync_iso": _sync_state["last_sync_iso"],
+        "last_sync_rows": _sync_state["last_sync_rows"],
+        "last_sync_elapsed": _sync_state["last_sync_elapsed"],
+        "last_error": _sync_state["last_error"],
+        "sync_running": _sync_state["running"],
+    }
+
+
+@app.post("/api/check-cloud")
+def check_cloud():
+    result = engine.check_cloud_connection()
+    return result
+
+
+@app.post("/api/sync")
+def trigger_sync(background_tasks: BackgroundTasks):
+    if _sync_state["running"]:
+        raise HTTPException(status_code=409, detail="Sync already in progress")
+
+    if not config.CLOUD_POSTGRES_DSN:
+        raise HTTPException(status_code=400, detail="CLOUD_POSTGRES_DSN not configured")
+
+    _sync_state["running"] = True
+    _sync_state["rows_done"] = 0
+    _sync_state["rows_total"] = 0
+    _sync_state["last_error"] = None
+
+    background_tasks.add_task(_run_sync)
+    return {"status": "started"}
+
+
+@app.get("/api/sync-status")
+def sync_status():
+    return {
+        "running": _sync_state["running"],
+        "rows_done": _sync_state["rows_done"],
+        "rows_total": _sync_state["rows_total"],
+        "last_sync_iso": _sync_state["last_sync_iso"],
+        "last_sync_rows": _sync_state["last_sync_rows"],
+        "last_sync_elapsed": _sync_state["last_sync_elapsed"],
+        "last_error": _sync_state["last_error"],
+    }
+
+
+def _progress_cb(rows_done: int, rows_total: int) -> None:
+    _sync_state["rows_done"] = rows_done
+    _sync_state["rows_total"] = rows_total
+
+
+class SelectTablePayload(BaseModel):
+    table: str
+
+
+class CreateTablePayload(BaseModel):
+    table: str
+
+
+@app.get("/api/local-tables")
+def list_local_tables():
+    """List existing local tables (tables matching ^wfr[0-9] on the local DB)."""
+    tables = engine.list_local_tables()
+    return {"tables": tables, "current": engine.local_table}
+
+
+@app.post("/api/select-local-table")
+def select_local_table(payload: SelectTablePayload):
+    """Switch the active local source table for the next sync."""
+    if _sync_state["running"]:
+        raise HTTPException(status_code=409, detail="Cannot change table while sync is running")
+    name = payload.table.lower().strip()
+    if not name:
+        raise HTTPException(status_code=400, detail="Table name is required")
+    engine.local_table = name
+    # Invalidate unsynced cache
+    _sync_state["_unsynced_ts"] = 0.0
+    _sync_state["_unsynced_count"] = None
+    _sync_state["_cloud_cursor"] = None
+    return {"selected": name}
+
+
+@app.get("/api/cloud-tables")
+def list_cloud_tables():
+    """List existing cloud tables (tables matching ^wfr[0-9] on the cloud DB)."""
+    tables = engine.list_cloud_tables()
+    return {"tables": tables, "current": engine.cloud_table}
+
+
+@app.post("/api/cloud-tables")
+def create_cloud_table(payload: CreateTablePayload):
+    """Create a new cloud hypertable."""
+    name = payload.table.lower().strip()
+    if not name:
+        raise HTTPException(status_code=400, detail="Table name is required")
+    try:
+        engine.create_cloud_table(name)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    # Invalidate unsynced cache
+    _sync_state["_unsynced_ts"] = 0.0
+    return {"created": name}
+
+
+@app.post("/api/select-table")
+def select_table(payload: SelectTablePayload):
+    """Switch the active cloud table for the next sync."""
+    if _sync_state["running"]:
+        raise HTTPException(status_code=409, detail="Cannot change table while sync is running")
+    name = payload.table.lower().strip()
+    if not name:
+        raise HTTPException(status_code=400, detail="Table name is required")
+    engine.cloud_table = name
+    # Invalidate unsynced cache so next /api/status recalculates
+    _sync_state["_unsynced_ts"] = 0.0
+    _sync_state["_unsynced_count"] = None
+    _sync_state["_cloud_cursor"] = None
+    return {"selected": name}
+
+
+def _run_sync() -> None:
+    try:
+        result = engine.sync(progress_cb=_progress_cb)
+        _sync_state["last_sync_iso"] = datetime.now(timezone.utc).isoformat()
+        _sync_state["last_sync_rows"] = result["rows_synced"]
+        _sync_state["last_sync_elapsed"] = result["elapsed_s"]
+        _sync_state["last_error"] = None
+        # Invalidate unsynced cache
+        _sync_state["_unsynced_ts"] = 0.0
+    except Exception as e:
+        logger.error(f"Sync failed: {e}")
+        _sync_state["last_error"] = str(e)
+    finally:
+        _sync_state["running"] = False

universal-telemetry-software/cloud-sync/config.py

@@ -0,0 +1,12 @@
+import os
+
+LOCAL_POSTGRES_DSN = os.getenv(
+    "LOCAL_POSTGRES_DSN",
+    "postgresql://wfr:wfr_password@timescaledb:5432/wfr",
+)
+LOCAL_TABLE = os.getenv("LOCAL_TABLE", "wfr26base").lower()
+
+CLOUD_POSTGRES_DSN = os.getenv("CLOUD_POSTGRES_DSN", "")
+CLOUD_TABLE = os.getenv("CLOUD_TABLE", "wfr26").lower()
+
+SYNC_BATCH_SIZE = int(os.getenv("SYNC_BATCH_SIZE", "5000"))

universal-telemetry-software/cloud-sync/requirements.txt

@@ -0,0 +1,3 @@
+fastapi==0.115.4
+uvicorn[standard]==0.32.1
+psycopg2-binary>=2.9.9