Skip to content

Commit 96ba7de

Browse files
themaroltthemarolt
authored
chore: refactor sync insights project to common layer and use it in nango worker as well (CM-786) (#3628)
Signed-off-by: Uroš Marolt <uros@marolt.me>
1 parent 7eb6a89 commit 96ba7de

13 files changed

Lines changed: 426 additions & 381 deletions

File tree

‎backend/src/api/collections/segmentsRepositoriesGet.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
import { CollectionService } from '@/services/collectionService'
1+
import { findRepositoriesForSegment } from '@crowd/data-access-layer/src/integrations'
2+
3+
import SequelizeRepository from '@/database/repositories/sequelizeRepository'
24

35
import Permissions from '../../security/permissions'
46
import PermissionChecker from '../../services/user/permissionChecker'
@@ -18,8 +20,8 @@ import PermissionChecker from '../../services/user/permissionChecker'
1820
export default async (req, res) => {
1921
new PermissionChecker(req).validateHas(Permissions.values.collectionRead)
2022

21-
const service = new CollectionService(req)
22-
const payload = await service.findRepositoriesForSegment(req.params.id)
23+
const qx = SequelizeRepository.getQueryExecutor(req)
24+
const payload = await findRepositoriesForSegment(qx, req.params.id)
2325

2426
await req.responseHandler.success(req, res, payload)
2527
}

‎backend/src/database/repositories/segmentRepository.ts‎

Lines changed: 0 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -921,122 +921,6 @@ class SegmentRepository extends RepositoryBase<
921921

922922
return result[0].segment_name as string
923923
}
924-
925-
async getGithubMappedRepos(segmentId: string) {
926-
const transaction = SequelizeRepository.getTransaction(this.options)
927-
const tenantId = this.options.currentTenant.id
928-
929-
const result = await this.options.database.sequelize.query(
930-
`
931-
select
932-
r.url as url
933-
from
934-
"githubRepos" r
935-
where r."segmentId" = :segmentId
936-
and r."tenantId" = :tenantId
937-
and r."deletedAt" is null
938-
order by r.url
939-
`,
940-
{
941-
replacements: {
942-
segmentId,
943-
tenantId,
944-
},
945-
type: QueryTypes.SELECT,
946-
transaction,
947-
},
948-
)
949-
950-
return result
951-
}
952-
953-
async getGitlabMappedRepos(segmentId: string) {
954-
const transaction = SequelizeRepository.getTransaction(this.options)
955-
const tenantId = this.options.currentTenant.id
956-
957-
const result = await this.options.database.sequelize.query(
958-
`
959-
select
960-
r.url as url
961-
from
962-
"gitlabRepos" r
963-
where r."segmentId" = :segmentId
964-
and r."tenantId" = :tenantId
965-
and r."deletedAt" is null
966-
order by r.url
967-
`,
968-
{
969-
replacements: {
970-
segmentId,
971-
tenantId,
972-
},
973-
type: QueryTypes.SELECT,
974-
transaction,
975-
},
976-
)
977-
978-
return result
979-
}
980-
981-
async getGithubRepoUrlsMappedToOtherSegments(urls: string[], segmentId: string) {
982-
if (!urls || urls.length === 0) {
983-
return []
984-
}
985-
986-
const transaction = SequelizeRepository.getTransaction(this.options)
987-
const tenantId = this.options.currentTenant.id
988-
989-
const rows = await this.options.database.sequelize.query(
990-
`
991-
select distinct
992-
r."url" as "url"
993-
from
994-
"githubRepos" r
995-
where
996-
r."tenantId" = :tenantId
997-
and r."url" in (:urls)
998-
and r."deletedAt" is null
999-
and r."segmentId" <> :segmentId
1000-
`,
1001-
{
1002-
replacements: { tenantId, urls, segmentId },
1003-
type: QueryTypes.SELECT,
1004-
transaction,
1005-
},
1006-
)
1007-
1008-
return rows.map((r) => r.url)
1009-
}
1010-
1011-
async getGitlabRepoUrlsMappedToOtherSegments(urls: string[], segmentId: string) {
1012-
if (!urls || urls.length === 0) {
1013-
return []
1014-
}
1015-
1016-
const transaction = SequelizeRepository.getTransaction(this.options)
1017-
const tenantId = this.options.currentTenant.id
1018-
1019-
const rows = await this.options.database.sequelize.query(
1020-
`
1021-
select distinct
1022-
r."url" as "url"
1023-
from
1024-
"gitlabRepos" r
1025-
where
1026-
r."tenantId" = :tenantId
1027-
and r."url" in (:urls)
1028-
and r."deletedAt" is null
1029-
and r."segmentId" <> :segmentId
1030-
`,
1031-
{
1032-
replacements: { tenantId, urls, segmentId },
1033-
type: QueryTypes.SELECT,
1034-
transaction,
1035-
},
1036-
)
1037-
1038-
return rows.map((r) => r.url)
1039-
}
1040924
}
1041925

1042926
export default SegmentRepository

‎backend/src/services/collectionService.ts‎

Lines changed: 1 addition & 158 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,7 @@ import {
2525
updateCollection,
2626
updateInsightsProject,
2727
} from '@crowd/data-access-layer/src/collections'
28-
import {
29-
fetchIntegrationById,
30-
fetchIntegrationsForSegment,
31-
removePlainGitHubRepoMapping,
32-
} from '@crowd/data-access-layer/src/integrations'
28+
import { fetchIntegrationsForSegment } from '@crowd/data-access-layer/src/integrations'
3329
import { QueryFilter } from '@crowd/data-access-layer/src/query'
3430
import {
3531
ICreateRepositoryGroup,
@@ -492,89 +488,6 @@ export class CollectionService extends LoggerBase {
492488
return listRepositoryGroups(qx, { insightsProjectId })
493489
}
494490

495-
async findRepositoriesForSegment(segmentId: string) {
496-
return SequelizeRepository.withTx(this.options, async (tx) => {
497-
const qx = SequelizeRepository.getQueryExecutor({ ...this.options, transaction: tx })
498-
const integrations = await fetchIntegrationsForSegment(qx, segmentId)
499-
500-
// Initialize result with platform arrays
501-
const result: Record<string, Array<{ url: string; label: string }>> = {
502-
git: [],
503-
github: [],
504-
gitlab: [],
505-
gerrit: [],
506-
}
507-
508-
const addToResult = (platform: PlatformType, fullUrl: string, label: string) => {
509-
const platformKey = platform.toLowerCase()
510-
if (!result[platformKey].some((item) => item.url === fullUrl)) {
511-
result[platformKey].push({ url: fullUrl, label })
512-
}
513-
}
514-
515-
// Add mapped repositories to GitHub platform
516-
const segmentRepository = new SegmentRepository({ ...this.options, transaction: tx })
517-
const githubMappedRepos = await segmentRepository.getGithubMappedRepos(segmentId)
518-
const gitlabMappedRepos = await segmentRepository.getGitlabMappedRepos(segmentId)
519-
520-
for (const repo of [...githubMappedRepos, ...gitlabMappedRepos]) {
521-
const url = repo.url
522-
try {
523-
const parsedUrl = new URL(url)
524-
if (parsedUrl.hostname === 'github.com') {
525-
const label = parsedUrl.pathname.slice(1) // removes leading '/'
526-
addToResult(PlatformType.GITHUB, url, label)
527-
}
528-
if (parsedUrl.hostname === 'gitlab.com') {
529-
const label = parsedUrl.pathname.slice(1) // removes leading '/'
530-
addToResult(PlatformType.GITLAB, url, label)
531-
}
532-
} catch (err) {
533-
// Do nothing
534-
}
535-
}
536-
537-
for (const i of integrations) {
538-
if (i.platform === PlatformType.GIT) {
539-
for (const r of (i.settings as any).remotes) {
540-
try {
541-
const url = new URL(r)
542-
let label = r
543-
544-
if (url.hostname === 'gitlab.com') {
545-
label = url.pathname.slice(1)
546-
} else if (url.hostname === 'github.com') {
547-
label = url.pathname.slice(1)
548-
}
549-
550-
addToResult(i.platform, r, label)
551-
} catch {
552-
this.options.log.warn(`Invalid URL in remotes: ${r}`)
553-
}
554-
}
555-
}
556-
557-
if (i.platform === PlatformType.GITLAB) {
558-
for (const group of Object.values((i.settings as any).groupProjects) as any[]) {
559-
for (const r of group) {
560-
const label = r.path_with_namespace
561-
const fullUrl = `https://gitlab.com/${label}`
562-
addToResult(i.platform, fullUrl, label)
563-
}
564-
}
565-
}
566-
567-
if (i.platform === PlatformType.GERRIT) {
568-
for (const r of (i.settings as any).remote.repoNames) {
569-
addToResult(i.platform, `${(i.settings as any).remote.orgURL}/q/project:${r}`, r)
570-
}
571-
}
572-
}
573-
574-
return result
575-
})
576-
}
577-
578491
static isSingleRepoOrg(orgs: GithubIntegrationSettings['orgs']): boolean {
579492
return (
580493
Array.isArray(orgs) &&
@@ -721,74 +634,4 @@ export class CollectionService extends LoggerBase {
721634

722635
return result
723636
}
724-
725-
static extractGithubRepoSlug(url: string): any {
726-
const parsedUrl = new URL(url)
727-
const pathname = parsedUrl.pathname
728-
const parts = pathname.split('/').filter(Boolean)
729-
730-
if (parts.length >= 2) {
731-
return `${parts[0]}/${parts[1]}`
732-
}
733-
734-
throw new Error('Invalid GitHub URL format')
735-
}
736-
737-
async findNangoRepositoriesToBeRemoved(integrationId: string): Promise<string[]> {
738-
return SequelizeRepository.withTx(this.options, async (tx) => {
739-
const qx = SequelizeRepository.getQueryExecutor({ ...this.options, transaction: tx })
740-
const integration = await fetchIntegrationById(qx, integrationId)
741-
742-
if (!integration || integration.platform !== PlatformType.GITHUB_NANGO) {
743-
return []
744-
}
745-
746-
const repoSlugs = new Set<string>()
747-
const settings = integration.settings as any
748-
const reposToBeRemoved = []
749-
750-
if (!settings.nangoMapping) {
751-
return []
752-
}
753-
754-
if (settings.orgs) {
755-
for (const org of settings.orgs) {
756-
for (const repo of org.repos ?? []) {
757-
repoSlugs.add(CollectionService.extractGithubRepoSlug(repo.url))
758-
}
759-
}
760-
}
761-
762-
if (settings.repos) {
763-
for (const repo of settings.repos) {
764-
repoSlugs.add(CollectionService.extractGithubRepoSlug(repo.url))
765-
}
766-
}
767-
// determine which connections to delete if needed
768-
for (const mappedRepo of Object.values(settings.nangoMapping) as {
769-
owner: string
770-
repoName: string
771-
}[]) {
772-
if (!repoSlugs.has(`${mappedRepo.owner}/${mappedRepo.repoName}`)) {
773-
reposToBeRemoved.push(`https://github.com/${mappedRepo.owner}/${mappedRepo.repoName}`)
774-
}
775-
}
776-
777-
return reposToBeRemoved
778-
})
779-
}
780-
781-
async unmapGithubRepo(integrationId: string, repo: string): Promise<void> {
782-
return SequelizeRepository.withTx(this.options, async (tx) => {
783-
const qx = SequelizeRepository.getQueryExecutor({ ...this.options, transaction: tx })
784-
await removePlainGitHubRepoMapping(qx, this.options.redis, integrationId, repo)
785-
})
786-
}
787-
788-
async unmapGitlabRepo(integrationId: string, repo: string): Promise<void> {
789-
return SequelizeRepository.withTx(this.options, async (tx) => {
790-
const qx = SequelizeRepository.getQueryExecutor({ ...this.options, transaction: tx })
791-
await removePlainGitHubRepoMapping(qx, this.options.redis, integrationId, repo)
792-
})
793-
}
794637
}

0 commit comments

Comments
 (0)