From d79e3aad4b72b8e5ac1e3f25480d7b3dc6e7bf5c Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 10 Oct 2025 10:23:09 +0100 Subject: [PATCH 1/7] fix: remove heavy query --- .../integrationProgressRepository.ts | 82 ++++++++++--------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/backend/src/database/repositories/integrationProgressRepository.ts b/backend/src/database/repositories/integrationProgressRepository.ts index d1207c1a66..1703bc0279 100644 --- a/backend/src/database/repositories/integrationProgressRepository.ts +++ b/backend/src/database/repositories/integrationProgressRepository.ts @@ -95,47 +95,51 @@ class IntegrationProgressRepository { }): Promise { const tb = new TinybirdClient() - const promises: Promise<{ data: Counter }>[] = [ - queryActivitiesCounter( - IntegrationProgressRepository.createPayloadWithActivityType(['star'], repos, segments), - tb, - ), - queryActivitiesCounter( - IntegrationProgressRepository.createPayloadWithActivityType(['unstar'], repos, segments), - tb, - ), - queryActivitiesCounter( - { - ...IntegrationProgressRepository.createPayloadWithActivityType(['fork'], repos, segments), - indirectFork: 1, - }, - tb, - ), - queryActivitiesCounter( - IntegrationProgressRepository.createPayloadWithActivityType( - ['issues-opened'], - repos, - segments, - ), - tb, - ), - queryActivitiesCounter( - IntegrationProgressRepository.createPayloadWithActivityType( - ['pull_request-opened'], - repos, - segments, - ), - tb, - ), - ] - - const result = await Promise.all(promises) + // const promises: Promise<{ data: Counter }>[] = [ + // queryActivitiesCounter( + // IntegrationProgressRepository.createPayloadWithActivityType(['star'], repos, segments), + // tb, + // ), + // queryActivitiesCounter( + // IntegrationProgressRepository.createPayloadWithActivityType(['unstar'], repos, segments), + // tb, + // ), + // queryActivitiesCounter( + // { + // ...IntegrationProgressRepository.createPayloadWithActivityType(['fork'], repos, segments), + // indirectFork: 1, + // }, + // tb, + // ), + // queryActivitiesCounter( + // IntegrationProgressRepository.createPayloadWithActivityType( + // ['issues-opened'], + // repos, + // segments, + // ), + // tb, + // ), + // queryActivitiesCounter( + // IntegrationProgressRepository.createPayloadWithActivityType( + // ['pull_request-opened'], + // repos, + // segments, + // ), + // tb, + // ), + // ] + + // const result = await Promise.all(promises) return { - stars: (result[0]?.data?.[0]?.count ?? 0) - (result[1]?.data?.[0]?.count ?? 0), - forks: result[2]?.data?.[0]?.count ?? 0, - totalIssues: result[3]?.data?.[0]?.count ?? 0, - totalPRs: result[4]?.data?.[0]?.count ?? 0, + // stars: (result[0]?.data?.[0]?.count ?? 0) - (result[1]?.data?.[0]?.count ?? 0), + // forks: result[2]?.data?.[0]?.count ?? 0, + // totalIssues: result[3]?.data?.[0]?.count ?? 0, + // totalPRs: result[4]?.data?.[0]?.count ?? 0, + stars: 0, + forks: 0, + totalIssues: 0, + totalPRs: 0, } } From 47909eb38980c0a3232976d17472be5d43ed2084 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 13 Oct 2025 16:31:25 +0200 Subject: [PATCH 2/7] fix: comment out also the tinybird client --- .../src/database/repositories/integrationProgressRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/database/repositories/integrationProgressRepository.ts b/backend/src/database/repositories/integrationProgressRepository.ts index 1703bc0279..b4832c0b6c 100644 --- a/backend/src/database/repositories/integrationProgressRepository.ts +++ b/backend/src/database/repositories/integrationProgressRepository.ts @@ -93,7 +93,7 @@ class IntegrationProgressRepository { repos: Repos segments: string[] }): Promise { - const tb = new TinybirdClient() + // const tb = new TinybirdClient() // const promises: Promise<{ data: Counter }>[] = [ // queryActivitiesCounter( From abe33ef90b2280a93d2dfc7d595c5d8317e03a11 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 13 Oct 2025 17:25:40 +0200 Subject: [PATCH 3/7] fix: lint --- .../database/repositories/integrationProgressRepository.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/backend/src/database/repositories/integrationProgressRepository.ts b/backend/src/database/repositories/integrationProgressRepository.ts index b4832c0b6c..4c5a5f2eea 100644 --- a/backend/src/database/repositories/integrationProgressRepository.ts +++ b/backend/src/database/repositories/integrationProgressRepository.ts @@ -1,11 +1,6 @@ import { QueryTypes } from 'sequelize' - -import { queryActivitiesCounter } from '@crowd/data-access-layer' -import { Counter, TinybirdClient } from '@crowd/data-access-layer/src/database' - import { Repos } from '@/serverless/integrations/types/regularTypes' import { GitHubStats } from '@/serverless/integrations/usecases/github/rest/getRemoteStats' - import { IRepositoryOptions } from './IRepositoryOptions' import SequelizeRepository from './sequelizeRepository' From b4bccdaac3361d622ebadeaa20d5b1baac6787f7 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 13 Oct 2025 17:34:29 +0200 Subject: [PATCH 4/7] fix: lint --- .../database/repositories/integrationProgressRepository.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/backend/src/database/repositories/integrationProgressRepository.ts b/backend/src/database/repositories/integrationProgressRepository.ts index 4c5a5f2eea..37de8aac5c 100644 --- a/backend/src/database/repositories/integrationProgressRepository.ts +++ b/backend/src/database/repositories/integrationProgressRepository.ts @@ -81,10 +81,7 @@ class IntegrationProgressRepository { return (result[0] as any).id as string } - static async getDbStatsForGithub({ - repos, - segments, - }: { + static async getDbStatsForGithub({}: { repos: Repos segments: string[] }): Promise { From 027924edd440bf2770f0eff754b05fa548715605 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 13 Oct 2025 17:57:39 +0200 Subject: [PATCH 5/7] fix: lint --- .../repositories/integrationProgressRepository.ts | 7 +++---- backend/src/services/integrationService.ts | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/backend/src/database/repositories/integrationProgressRepository.ts b/backend/src/database/repositories/integrationProgressRepository.ts index 37de8aac5c..8c8fdf2588 100644 --- a/backend/src/database/repositories/integrationProgressRepository.ts +++ b/backend/src/database/repositories/integrationProgressRepository.ts @@ -1,6 +1,8 @@ import { QueryTypes } from 'sequelize' + import { Repos } from '@/serverless/integrations/types/regularTypes' import { GitHubStats } from '@/serverless/integrations/usecases/github/rest/getRemoteStats' + import { IRepositoryOptions } from './IRepositoryOptions' import SequelizeRepository from './sequelizeRepository' @@ -81,10 +83,7 @@ class IntegrationProgressRepository { return (result[0] as any).id as string } - static async getDbStatsForGithub({}: { - repos: Repos - segments: string[] - }): Promise { + static async getDbStatsForGithub(): Promise { // const tb = new TinybirdClient() // const promises: Promise<{ data: Counter }>[] = [ diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index b919140b0a..e79d44e3f0 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -2195,10 +2195,10 @@ export default class IntegrationService { this.options.log.debug( `Evaluating cache for repos: ${repos.map((r) => r.name).join(',')} and segments: ${segments}`, ) - cachedStats = await IntegrationProgressRepository.getDbStatsForGithub({ - repos, - segments, - }) + // cachedStats = await IntegrationProgressRepository.getDbStatsForGithub({ + // repos, + // segments, + // }) this.options.log.debug(`Caching data: ${JSON.stringify(cachedStats)}`) // cache for 1 minute From b7fb6b32a6332356d67df9896683197fd711ca8e Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 13 Oct 2025 18:01:08 +0200 Subject: [PATCH 6/7] fix: lint --- backend/src/services/integrationService.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index e79d44e3f0..8d66904cbe 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -2200,11 +2200,10 @@ export default class IntegrationService { // segments, // }) - this.options.log.debug(`Caching data: ${JSON.stringify(cachedStats)}`) + // this.options.log.debug(`Caching data: ${JSON.stringify(cachedStats)}`) // cache for 1 minute - await cacheDb.set(key, JSON.stringify(cachedStats), 60) + // await cacheDb.set(key, JSON.stringify(cachedStats), 60) } else { - this.options.log.debug(`Cache data found: ${JSON.stringify(cachedStats)}`) cachedStats = JSON.parse(cachedStats) } return cachedStats as GitHubStats From dd740ef114e1ff6337ed6fa3e5fe6f0b74d54898 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 14 Oct 2025 12:00:49 +0200 Subject: [PATCH 7/7] fix: stats should be set to 0 --- backend/src/services/integrationService.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 8d66904cbe..ec52aace16 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -2195,14 +2195,11 @@ export default class IntegrationService { this.options.log.debug( `Evaluating cache for repos: ${repos.map((r) => r.name).join(',')} and segments: ${segments}`, ) - // cachedStats = await IntegrationProgressRepository.getDbStatsForGithub({ - // repos, - // segments, - // }) + cachedStats = await IntegrationProgressRepository.getDbStatsForGithub() - // this.options.log.debug(`Caching data: ${JSON.stringify(cachedStats)}`) + this.options.log.debug(`Caching data: ${JSON.stringify(cachedStats)}`) // cache for 1 minute - // await cacheDb.set(key, JSON.stringify(cachedStats), 60) + await cacheDb.set(key, JSON.stringify(cachedStats), 60) } else { cachedStats = JSON.parse(cachedStats) }