Merge branch 'main' into feature/issue-469-opencre-mapanalysis

Signed-off-by: Bornunique911 <69379200+Bornunique911@users.noreply.github.com>

Author: Bornunique911

.env.example

@@ -38,6 +38,9 @@ LOGIN_ALLOWED_DOMAINS=example.com
 # GCP
 
 GCP_NATIVE=false
+GEMINI_API_KEY=your-gemini-api-key
+VERTEX_CHAT_MODEL=gemini-2.0-flash
+VERTEX_EMBED_CONTENT_MODEL=embedding-001
 
 # Spreadsheet Auth
 

.github/workflows/linter.yml

@@ -3,6 +3,29 @@ name: Lint Code Base
 
 on: [push, pull_request]
 jobs:
+  frontend:
+    name: Frontend Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'yarn'
+      - name: Install frontend dependencies
+        run: |
+          for i in 1 2 3; do
+            yarn install --frozen-lockfile --network-timeout 120000 && exit 0
+            echo "yarn install failed (attempt $i/3), retrying..."
+            sleep $((i * 10))
+          done
+          echo "yarn install failed after 3 attempts"
+          exit 1
+      - name: Check frontend formatting
+        run: yarn prettier --check "application/frontend/src/**/*.{ts,tsx,js}"
+
   build:
     name: Lint Code Base
     runs-on: ubuntu-latest

.prettierignore

@@ -6,5 +6,10 @@ yarn-error.log
 cypress/videos
 cypress/screenshots
 venv/
+.venv/
+.mypy_cache/
+.pytest_cache/
+.cursor/
+tmp/
 static/
 vendor/

Makefile

@@ -8,6 +8,8 @@ prod-run:
 docker-neo4j-rm:
 	docker stop cre-neo4j
 	docker rm -f cre-neo4j
+	docker volume inspect cre_neo4j_data >/dev/null 2>&1 || docker volume create cre_neo4j_data
+	docker volume inspect cre_neo4j_logs >/dev/null 2>&1 || docker volume create cre_neo4j_logs
 	docker volume rm cre_neo4j_data
 	docker volume rm cre_neo4j_logs
 	# rm -rf .neo4j
@@ -23,6 +25,10 @@ docker-redis:
 	docker start cre-redis-stack 2>/dev/null ||\
 	docker run -d --name cre-redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
 
+docker-postgres:
+	docker start cre-postgres 2>/dev/null ||\
+	docker run -d --name cre-postgres -e POSTGRES_PASSWORD=password -e POSTGRES_USER=cre -e POSTGRES_DB=cre -p 5432:5432 postgres
+
 start-containers: docker-neo4j docker-redis
 
 start-worker:
@@ -55,7 +61,7 @@ e2e:
 test:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	export FLASK_APP="$(CURDIR)/cre.py" &&\
-	flask routes && flask test
+	flask routes && python -m unittest discover -s application/tests -p "*_test.py"
 
 cover:
 	. ./venv/bin/activate && FLASK_APP=cre.py FLASK_CONFIG=testing flask test --cover
@@ -127,7 +133,35 @@ import-neo4j:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	export FLASK_APP="$(CURDIR)/cre.py" && python cre.py --populate_neo4j_db
 
-preload-map-analysis:
-	$(shell RUN_COUNT=5 bash ./scripts/preload_gap_analysis.sh)
+backfill-gap-analysis:
+	RUN_COUNT=8 bash ./scripts/backfill_gap_analysis.sh
+
+sync-gap-analysis-table-local:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	python scripts/sync_gap_analysis_table.py \
+		--from-sqlite "$(CURDIR)/standards_cache.sqlite" \
+		--to-postgres "postgresql://cre:password@127.0.0.1:5432/cre" \
+		--require-local-destination
+
+verify-ga-complete-prod:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	python scripts/verify_ga_completeness.py \
+		--base-url "https://opencre.org" \
+		--output-json "$(CURDIR)/tmp/prod-ga-completeness.json"
+
+verify-ga-parity-local:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	export NEO4J_URL="$${NEO4J_URL:-bolt://neo4j:password@127.0.0.1:7687}"; \
+	export PYTHONPATH="$(CURDIR)"; \
+	python scripts/verify_ga_postgres_neo_parity.py --output-json "$(CURDIR)/tmp/local-ga-parity.json"
+
+backfill-gap-analysis-sync:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export FLASK_APP="$(CURDIR)/cre.py"; \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	export NEO4J_URL="$${NEO4J_URL:-bolt://neo4j:password@127.0.0.1:7687}"; \
+	python cre.py --cache_file "$$CRE_CACHE_FILE" --populate_neo4j_db && \
+	python cre.py --cache_file "$$CRE_CACHE_FILE" --ga_backfill_missing --ga_backfill_no_queue
 
 all: clean lint test dev dev-run

README.md

@@ -68,7 +68,11 @@ Download the latest CRE graph from upstream by running:
 make upstream-sync
 ```
 
-Keep in mind that until [Issue #534](https://github.com/OWASP/OpenCRE/issues/534) is fixed you won't have access to gap analysis results locally.
+You can precompute local gap-analysis cache after imports with:
+
+```bash
+make backfill-gap-analysis
+```
 
 To run CRE locally then you can do:
 
@@ -116,6 +120,21 @@ Some features like Gap Analysis require a neo4j DB running, you can start this w
 make docker-neo4j
 ```
 
+To run only missing gap-analysis pair backfill (without starting Flask), use:
+
+```bash
+RUN_COUNT=8 bash scripts/backfill_gap_analysis.sh
+```
+
+To sync local Postgres data into a Heroku app (staging or prod), use:
+
+```bash
+APP_NAME=stagingopencreorg \
+SOURCE_DB_URL="postgresql://cre:password@127.0.0.1:5432/cre" \
+SYNC_TABLES=gap_analysis \
+bash scripts/push-local-postgres-to-heroku.sh --gap_analysis
+```
+
 Environment variables for app to connect to neo4jDB (default):
 
 * `NEO4J_URL` (neo4j//neo4j:password@localhost:7687)

application/__init__.py

@@ -31,6 +31,10 @@ def create_app(mode: str = "production", conf: any = None) -> Any:
         app.config.from_object(config[mode])
     else:
         app.config.from_object(conf)
+    if os.environ.get("SQLALCHEMY_DATABASE_URI"):
+        app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get(
+            "SQLALCHEMY_DATABASE_URI"
+        )
     GOOGLE_CLIENT_SECRET = os.environ.get("GOOGLE_CLIENT_SECRET")
     app.secret_key = GOOGLE_CLIENT_SECRET
     if os.environ.get("NO_LOGIN"):