Enhance map analysis by enforcing two standards input and improving Redis fallback handling

Author: Bornunique911

application/web/web_main.py

@@ -48,13 +48,6 @@
 
 
 ITEMS_PER_PAGE = 20
-OWASP_TOP10_2025_DATA_FILE = (
-    pathlib.Path(__file__).resolve().parent.parent
-    / "utils"
-    / "external_project_parsers"
-    / "data"
-    / "owasp_top10_2025.json"
-)
 OPENCRE_STANDARD_NAME = "OpenCRE"
 
 app = Blueprint(
@@ -422,8 +415,12 @@ def map_analysis() -> Any:
         posthog.capture(f"map_analysis", f"standards:{standards}")
 
     database = db.Node_collection()
+    if len(standards) < 2:
+        abort(400, "Please provide two standards")
+    standards = standards[:2]
     standards_hash = gap_analysis.make_resources_key(standards)
 
+    # ----- PR #825: OpenCRE fast path -----
     if OPENCRE_STANDARD_NAME in standards:
         direct_gap_analysis = _build_direct_cre_overlap_map_analysis(
             standards, standards_hash, database
@@ -432,48 +429,69 @@ def map_analysis() -> Any:
             return jsonify(direct_gap_analysis)
         abort(404, "No direct overlap found for requested standards")
 
-    # First, check if we have cached results in the database
-    if database.gap_analysis_exists(standards_hash):
-        gap_analysis_result = database.get_gap_analysis_result(standards_hash)
-        if gap_analysis_result:
-            return jsonify(flask_json.loads(gap_analysis_result))
-
-    # On Heroku (read-only), check if standards exist before attempting Redis/queue operations
-    is_heroku = os.environ.get("DYNO") is not None
-    if is_heroku:
-        # Check if all requested standards exist
-        try:
-            existing_standards = database.standards()
-            if isinstance(existing_standards, (list, tuple, set)):
-                existing_lower = {str(s).lower() for s in existing_standards}
-                missing = [s for s in standards if str(s).lower() not in existing_lower]
-                if missing:
-                    logger.info(
-                        f"On Heroku: gap analysis request {standards_hash} references "
-                        f"standards that do not exist: {', '.join(missing)}, returning 404"
-                    )
-                    abort(
-                        404, f"One or more standards do not exist: {', '.join(missing)}"
-                    )
-        except Exception as exc:
-            # If we can't verify standards, log but don't fail (defensive)
-            logger.warning(f"Could not verify standards existence on Heroku: {exc}")
-
-    # If calculations are disabled, return 404
-    if os.environ.get("CRE_NO_CALCULATE_GAP_ANALYSIS"):
-        logger.info(
-            f"Gap analysis calculations are disabled by CRE_NO_CALCULATE_GAP_ANALYSIS; "
-            f"refusing to schedule new job for {standards_hash}"
+    # ----- upstream: cached result -----
+    cache_key = standards_hash
+    if database.gap_analysis_exists(cache_key):
+        cached = database.get_gap_analysis_result(cache_key=cache_key)
+        if cached:
+            parsed = json.loads(cached)
+            if "result" in parsed:
+                return jsonify({"result": parsed.get("result")})
+
+    # ----- upstream: Heroku guard -----
+    if os.environ.get("HEROKU"):
+        abort(404, "No such Cache")
+
+    # ----- upstream: Redis / RQ path with synchronous fallback -----
+    db_url = os.environ.get("CRE_CACHE_FILE") or os.environ.get("PROD_DATABASE_URL")
+    if not db_url:
+        db_url = str(getattr(getattr(database.session, "bind", None), "url", ""))
+    try:
+        conn = redis.connect()
+        ga_queue_name = os.environ.get("CRE_GA_QUEUE_NAME", "ga")
+        q = Queue(name=ga_queue_name, connection=conn)
+        inflight_key = f"ga:inflight:{cache_key}"
+        inflight_job_id_raw = conn.get(inflight_key)
+        inflight_job_id = (
+            inflight_job_id_raw.decode("utf-8")
+            if isinstance(inflight_job_id_raw, bytes)
+            else str(inflight_job_id_raw) if inflight_job_id_raw else ""
         )
-        abort(404, "Gap analysis calculations are disabled")
-
-    # Now call schedule() which will handle Redis/queue operations
-    gap_analysis_dict = gap_analysis.schedule(standards, database)
-    if "result" in gap_analysis_dict:
-        return jsonify(gap_analysis_dict)
-    if gap_analysis_dict.get("error"):
-        abort(404)
-    return jsonify({"job_id": gap_analysis_dict.get("job_id")})
+        if inflight_job_id:
+            try:
+                inflight_job = job.Job.fetch(id=inflight_job_id, connection=conn)
+                if inflight_job.get_status() in (
+                    job.JobStatus.QUEUED,
+                    job.JobStatus.STARTED,
+                ):
+                    return jsonify({"job_id": inflight_job_id})
+            except exceptions.NoSuchJobError:
+                conn.delete(inflight_key)
+
+        j = q.enqueue_call(
+            description=f"{standards[0]}->{standards[1]}",
+            func=cre_main.run_gap_pair_job,
+            kwargs={
+                "importing_name": standards[0],
+                "peer_name": standards[1],
+                "db_connection_str": db_url,
+            },
+            timeout=gap_analysis.GAP_ANALYSIS_TIMEOUT,
+        )
+        conn.set(inflight_key, str(j.id))
+        conn.expire(inflight_key, _ga_timeout_seconds())
+        return jsonify({"job_id": str(j.id)})
+    except Exception as exc:
+        logger.warning(
+            "Redis/RQ unavailable in map_analysis for %s; using synchronous fallback: %s",
+            cache_key,
+            exc,
+        )
+        try:
+            return jsonify(_compute_ga_without_redis(database, standards))
+        except Exception as fallback_exc:
+            logger.exception("Synchronous GA fallback failed for %s", cache_key)
+            abort(503, f"Gap analysis unavailable: {fallback_exc}")
 
 
 @app.route("/rest/v1/map_analysis_weak_links", methods=["GET"])
@@ -1357,4 +1375,4 @@ def import_from_cre_csv() -> Any:
 
 
 if __name__ == "__main__":
-    app.run(use_reloader=False, debug=True)
+    app.run(use_reloader=False, debug=True)