Merge branch 'main' into review/issue-471-ai-resource-importers

Author: Bornunique911

.env.example

@@ -38,6 +38,9 @@ LOGIN_ALLOWED_DOMAINS=example.com
 # GCP
 
 GCP_NATIVE=false
+GEMINI_API_KEY=your-gemini-api-key
+VERTEX_CHAT_MODEL=gemini-2.0-flash
+VERTEX_EMBED_CONTENT_MODEL=embedding-001
 
 # Spreadsheet Auth
 

Makefile

@@ -25,6 +25,10 @@ docker-redis:
 	docker start cre-redis-stack 2>/dev/null ||\
 	docker run -d --name cre-redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
 
+docker-postgres:
+	docker start cre-postgres 2>/dev/null ||\
+	docker run -d --name cre-postgres -e POSTGRES_PASSWORD=password -e POSTGRES_USER=cre -e POSTGRES_DB=cre -p 5432:5432 postgres
+
 start-containers: docker-neo4j docker-redis
 
 start-worker:
@@ -132,4 +136,32 @@ import-neo4j:
 backfill-gap-analysis:
 	RUN_COUNT=8 bash ./scripts/backfill_gap_analysis.sh
 
+sync-gap-analysis-table-local:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	python scripts/sync_gap_analysis_table.py \
+		--from-sqlite "$(CURDIR)/standards_cache.sqlite" \
+		--to-postgres "postgresql://cre:password@127.0.0.1:5432/cre" \
+		--require-local-destination
+
+verify-ga-complete-prod:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	python scripts/verify_ga_completeness.py \
+		--base-url "https://opencre.org" \
+		--output-json "$(CURDIR)/tmp/prod-ga-completeness.json"
+
+verify-ga-parity-local:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	export NEO4J_URL="$${NEO4J_URL:-bolt://neo4j:password@127.0.0.1:7687}"; \
+	export PYTHONPATH="$(CURDIR)"; \
+	python scripts/verify_ga_postgres_neo_parity.py --output-json "$(CURDIR)/tmp/local-ga-parity.json"
+
+backfill-gap-analysis-sync:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export FLASK_APP="$(CURDIR)/cre.py"; \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	export NEO4J_URL="$${NEO4J_URL:-bolt://neo4j:password@127.0.0.1:7687}"; \
+	python cre.py --cache_file "$$CRE_CACHE_FILE" --populate_neo4j_db && \
+	python cre.py --cache_file "$$CRE_CACHE_FILE" --ga_backfill_missing --ga_backfill_no_queue
+
 all: clean lint test dev dev-run

application/cmd/cre_main.py

@@ -13,6 +13,7 @@
 import hashlib
 import json as _json
 from rq import Queue, job, exceptions
+from sqlalchemy import not_
 
 from application.utils.external_project_parsers.base_parser import BaseParser
 from application.utils.external_project_parsers.parsers import master_spreadsheet_parser
@@ -649,11 +650,15 @@ def download_gap_analysis_from_upstream(cache: str) -> None:
                     )
                     if res.status_code == 200:
                         tojson = res.json()
-                        if tojson.get("result"):
-                            cache_key = gap_analysis.make_resources_key([sa, sb])
-                            collection.add_gap_analysis_result(
-                                cache_key, _json.dumps({"result": tojson.get("result")})
-                            )
+                        if "result" not in tojson:
+                            continue
+                        payload = _json.dumps({"result": tojson.get("result")})
+                        if not gap_analysis.primary_gap_analysis_payload_is_material(
+                            payload
+                        ):
+                            continue
+                        cache_key = gap_analysis.make_resources_key([sa, sb])
+                        collection.add_gap_analysis_result(cache_key, payload)
                     bar()
         else:
             for sa, sb in pairs:
@@ -662,18 +667,33 @@ def download_gap_analysis_from_upstream(cache: str) -> None:
                 )
                 if res.status_code == 200:
                     tojson = res.json()
-                    if tojson.get("result"):
-                        cache_key = gap_analysis.make_resources_key([sa, sb])
-                        collection.add_gap_analysis_result(
-                            cache_key, _json.dumps({"result": tojson.get("result")})
-                        )
+                    if "result" not in tojson:
+                        continue
+                    payload = _json.dumps({"result": tojson.get("result")})
+                    if not gap_analysis.primary_gap_analysis_payload_is_material(
+                        payload
+                    ):
+                        continue
+                    cache_key = gap_analysis.make_resources_key([sa, sb])
+                    collection.add_gap_analysis_result(cache_key, payload)
 
 
 def _missing_ga_pairs(collection: db.Node_collection) -> List[Tuple[str, str]]:
-    standards = sorted(collection.standards())
+    from application.utils.ga_parity import ga_matrix_standard_names
+
+    standards = ga_matrix_standard_names(collection)
+    rows = (
+        collection.session.query(
+            db.GapAnalysisResults.cache_key, db.GapAnalysisResults.ga_object
+        )
+        .filter(not_(db.GapAnalysisResults.cache_key.like("% >> %->%")))
+        .all()
+    )
     existing = {
-        key
-        for (key,) in collection.session.query(db.GapAnalysisResults.cache_key).all()
+        str(key)
+        for key, payload in rows
+        if key
+        and gap_analysis.primary_gap_analysis_payload_is_material(str(payload or ""))
     }
     missing: List[Tuple[str, str]] = []
     for sa in standards:

application/database/db.py

@@ -21,7 +21,7 @@
 from flask import json as flask_json
 from sqlalchemy.orm import aliased
 from flask_sqlalchemy.model import DefaultMeta
-from sqlalchemy import func, delete, cast as sql_cast, literal
+from sqlalchemy import func, delete, cast as sql_cast, literal, or_
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.exc import OperationalError, IntegrityError
 
@@ -42,9 +42,11 @@
 from application.defs import cre_defs
 from application.utils import file
 from application.utils.gap_analysis import (
+    gap_analysis_cache_key_is_primary,
     get_path_score,
     make_resources_key,
     make_subresources_key,
+    primary_gap_analysis_payload_is_material,
 )
 
 
@@ -2300,10 +2302,16 @@ def add_embedding(
             return existing
 
     def gap_analysis_exists(self, cache_key) -> bool:
-        q = self.session.query(GapAnalysisResults).filter(
-            GapAnalysisResults.cache_key == cache_key
+        row = (
+            self.session.query(GapAnalysisResults)
+            .filter(GapAnalysisResults.cache_key == cache_key)
+            .first()
         )
-        return self.session.query(q.exists()).scalar()
+        if row is None:
+            return False
+        if gap_analysis_cache_key_is_primary(cache_key):
+            return primary_gap_analysis_payload_is_material(row.ga_object)
+        return True
 
     def get_gap_analysis_result(self, cache_key) -> str:
         logger.info(f"looking for gap analysis with cache key: {cache_key}")
@@ -2318,7 +2326,16 @@ def get_gap_analysis_result(self, cache_key) -> str:
         logger.info(f"did not find gap analysis with cache key: {cache_key}")
 
     def add_gap_analysis_result(self, cache_key: str, ga_object: str):
-        if not self.gap_analysis_exists(cache_key):
+        existing = (
+            self.session.query(GapAnalysisResults)
+            .filter(GapAnalysisResults.cache_key == cache_key)
+            .first()
+        )
+        if existing:
+            existing.ga_object = ga_object
+            self.session.add(existing)
+            self.session.commit()
+        else:
             logger.info(f"adding gap analysis result with cache key: {cache_key}")
             res = GapAnalysisResults(cache_key=cache_key, ga_object=ga_object)
             self.session.add(res)
@@ -2490,6 +2507,20 @@ def gap_analysis(
             grouped_paths[key] = {"start": node, "paths": {}, "extra": 0}
             extra_paths_dict[key] = {"paths": {}}
 
+    # Paths may start from CRE nodes that were not included in ``base_standard`` (or
+    # ``base_standard`` entries were skipped due to empty ids). Seed roots from each
+    # path's start so we never drop Neo paths or hit KeyError in the merge loop below.
+    for path in paths:
+        start_doc = path.get("start")
+        if start_doc is None:
+            continue
+        key = getattr(start_doc, "id", "") or ""
+        if not key:
+            continue
+        if key not in grouped_paths:
+            grouped_paths[key] = {"start": start_doc, "paths": {}, "extra": 0}
+            extra_paths_dict[key] = {"paths": {}}
+
     for path in paths:
         key = path["start"].id
         end_key = path["end"].id
@@ -2525,6 +2556,30 @@ def gap_analysis(
 
     if cache_key == "":
         cache_key = make_resources_key(node_names)
+    if not grouped_paths:
+        if gap_analysis_cache_key_is_primary(cache_key):
+            stale = (
+                cre_db.session.query(GapAnalysisResults)
+                .filter(
+                    or_(
+                        GapAnalysisResults.cache_key == cache_key,
+                        GapAnalysisResults.cache_key.like(cache_key + "->%"),
+                    )
+                )
+                .all()
+            )
+            for row in stale:
+                cre_db.session.delete(row)
+            if stale:
+                cre_db.session.commit()
+        logger.warning(
+            "Not persisting gap analysis for %s: grouped_paths is empty "
+            "(Neo likely had no parseable base standard or no paths). "
+            "Re-run after Neo is populated so the pair is not locked in as an empty cache.",
+            cache_key,
+        )
+        return (node_names, grouped_paths, extra_paths_dict)
+
     logger.info(f"got gap analysis paths for {'>>>'.join(node_names)}, storing result")
     cre_db.add_gap_analysis_result(
         cache_key=cache_key, ga_object=flask_json.dumps({"result": grouped_paths})