Merge branch 'main' into bugfix/tb-issue-analysis-filtering-old-dates-out

Author: epipav

backend/package.json

@@ -128,7 +128,7 @@
     "pg": "^8.7.3",
     "pg-promise": "^11.4.3",
     "sanitize-html": "^2.7.1",
-    "sequelize": "6.21.2",
+    "sequelize": "6.37.8",
     "sequelize-cli-typescript": "^3.2.0-c",
     "slack-block-builder": "^2.7.2",
     "socket.io": "^4.5.4",

backend/src/api/member/index.ts

@@ -9,7 +9,6 @@ export default (app) => {
   app.put(`/member/:id`, safeWrap(require('./memberUpdate').default))
   app.delete(`/member`, safeWrap(require('./memberDestroy').default))
   app.post(`/member/autocomplete`, safeWrap(require('./memberAutocomplete').default))
-  app.get(`/member/active`, safeWrap(require('./memberActiveList').default))
   app.get(`/member/bot-suggestions`, safeWrap(require('./memberBotSuggestionsList').default))
 
   app.get(`/member/:id`, safeWrap(require('./memberFind').default))

backend/src/api/member/memberActiveList.ts

@@ -6,7 +6,6 @@ export default (app) => {
   app.put(`/organization/:id`, safeWrap(require('./organizationUpdate').default))
   app.delete(`/organization`, safeWrap(require('./organizationDestroy').default))
   app.post(`/organization/autocomplete`, safeWrap(require('./organizationAutocomplete').default))
-  app.get(`/organization/active`, safeWrap(require('./organizationActiveList').default))
   app.get(`/organization/:id`, safeWrap(require('./organizationFind').default))
 
   app.put(`/organization/:organizationId/merge`, safeWrap(require('./organizationMerge').default))

backend/src/api/organization/index.ts

@@ -0,0 +1,11 @@
+DROP INDEX IF EXISTS "ix_evaluatedProjects_onboarded";
+DROP INDEX IF EXISTS "ix_evaluatedProjects_evaluationScore";
+DROP INDEX IF EXISTS "ix_evaluatedProjects_evaluationStatus";
+DROP INDEX IF EXISTS "uix_evaluatedProjects_projectCatalogId";
+DROP TABLE IF EXISTS "evaluatedProjects";
+
+DROP INDEX IF EXISTS "ix_projectCatalog_syncedAt";
+DROP INDEX IF EXISTS "ix_projectCatalog_lfCriticalityScore";
+DROP INDEX IF EXISTS "ix_projectCatalog_ossfCriticalityScore";
+DROP INDEX IF EXISTS "uix_projectCatalog_repoUrl";
+DROP TABLE IF EXISTS "projectCatalog";

backend/src/api/organization/organizationActiveList.ts

@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS "idx_memberIdentities_verified_email_lower_value";

backend/src/database/migrations/U1770653666__add-automatic_projects_discovery-tables.sql

@@ -0,0 +1,42 @@
+-- Project Catalog: candidate projects discovered from OSSF Criticality Score and other sources
+CREATE TABLE IF NOT EXISTS "projectCatalog" (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    "projectSlug" VARCHAR(255) NOT NULL,
+    "repoName" VARCHAR(255) NOT NULL,
+    "repoUrl" VARCHAR(1024) NOT NULL,
+    "ossfCriticalityScore" DOUBLE PRECISION,
+    "lfCriticalityScore" DOUBLE PRECISION,
+    "syncedAt" TIMESTAMP WITH TIME ZONE DEFAULT NULL,
+    "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX "uix_projectCatalog_repoUrl" ON "projectCatalog" ("repoUrl");
+CREATE INDEX "ix_projectCatalog_ossfCriticalityScore" ON "projectCatalog" ("ossfCriticalityScore" DESC NULLS LAST);
+CREATE INDEX "ix_projectCatalog_lfCriticalityScore" ON "projectCatalog" ("lfCriticalityScore" DESC NULLS LAST);
+CREATE INDEX "ix_projectCatalog_syncedAt" ON "projectCatalog" ("syncedAt");
+
+-- Evaluated Projects: AI evaluation results linked to catalog entries
+CREATE TABLE IF NOT EXISTS "evaluatedProjects" (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    "projectCatalogId" UUID NOT NULL REFERENCES "projectCatalog"(id) ON DELETE CASCADE,
+    "evaluationStatus" VARCHAR(50) NOT NULL DEFAULT 'pending',
+    "evaluationScore" DOUBLE PRECISION,
+    "evaluation" JSONB,
+    "evaluationReason" TEXT,
+    "evaluatedAt" TIMESTAMP WITH TIME ZONE,
+    "starsCount" INTEGER,
+    "forksCount" INTEGER,
+    "commitsCount" INTEGER,
+    "pullRequestsCount" INTEGER,
+    "issuesCount" INTEGER,
+    "onboarded" BOOLEAN NOT NULL DEFAULT FALSE,
+    "onboardedAt" TIMESTAMP WITH TIME ZONE,
+    "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX "uix_evaluatedProjects_projectCatalogId" ON "evaluatedProjects" ("projectCatalogId");
+CREATE INDEX "ix_evaluatedProjects_evaluationStatus" ON "evaluatedProjects" ("evaluationStatus");
+CREATE INDEX "ix_evaluatedProjects_evaluationScore" ON "evaluatedProjects" ("evaluationScore" DESC NULLS LAST);
+CREATE INDEX "ix_evaluatedProjects_onboarded" ON "evaluatedProjects" ("onboarded");

backend/src/database/migrations/U1774363114__add-btree-index-for-member-identities-email-lookup.sql

@@ -0,0 +1,20 @@
+-- The existing idx_memberIdentities_email_verified_trgm covers the right partial
+-- conditions (verified = true, type = 'email', deletedAt IS NULL) but is a GIN
+-- trigram index -- PostgreSQL will not use it for equality joins on
+-- lower(mi.value) against input emails, for example:
+--
+--   JOIN ON lower(mi.value) = input_email
+--   WHERE mi.verified = true AND mi.type = 'email' AND mi."deletedAt" IS NULL
+--
+-- The only usable B-tree index (idx_memberIdentities_lower_value) only carries
+-- the partial condition deletedAt IS NULL, so PG must heap-fetch every matched
+-- row to re-check verified and type, which degrades as the table grows.
+--
+-- This B-tree partial index lets the planner do a direct nested-loop index scan
+-- (one lookup per input email) with no extra heap fetches for verified/type.
+
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_memberIdentities_verified_email_lower_value"
+    ON "memberIdentities" (lower(value))
+    WHERE verified = true
+      AND type = 'email'
+      AND "deletedAt" IS NULL;