GitHub integration: check repo can be queried before starting parsing (#47)

Co-authored-by: Joan Reyero <joan@crowd.dev>

Author: epipav

backend/src/serverless/integrations/coordinators/devtoCoordinator.ts

@@ -19,7 +19,7 @@ async function devtoCoordinator(): Promise<void> {
       integrationId: integration.id,
       tenant: integration.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {},
     }
 

backend/src/serverless/integrations/coordinators/discordCoordinator.ts

@@ -16,7 +16,7 @@ async function discordCoordinator(): Promise<void> {
       sleep: Math.floor(Math.random() * 1200),
       tenant: integration.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {
         guildId: integration.integrationIdentifier,
         channels: integration.settings.channels || [],

backend/src/serverless/integrations/coordinators/slackCoordinator.ts

@@ -16,7 +16,7 @@ async function slackCoordinator(): Promise<void> {
       sleep: 0,
       tenant: integration.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {
         channels: integration.settings.channels || [],
       },

backend/src/serverless/integrations/coordinators/twitterCoordinator.ts

@@ -16,7 +16,7 @@ async function twitterCoordinator(): Promise<void> {
       sleep: 0,
       tenant: integration.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {
         profileId: integration.integrationIdentifier,
         hashtags: integration.settings.hashtags,

backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts

@@ -15,7 +15,7 @@ async function twitterReachCoordinator(): Promise<void> {
       sleep: 0,
       tenant: microservice.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {
         profileId: microservice.microserviceIdentifier,
       },

backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts

@@ -5,35 +5,35 @@ describe('Integrations worker static tests', () => {
   const endpoints = ['endpoint1', 'endpoint2', 'endpoint3']
   describe('Get iterator tests', () => {
     it('Should return the endpoints to iterate for a standard state', async () => {
-      const state = { endpoint: 'endpoint2', page: 'here' }
+      const state = { endpoint: 'endpoint2', page: 'here', endpoints: [] }
       const iterator = BaseIterator.getEndPointsIterator(endpoints, state)
 
       expect(iterator).toStrictEqual(['endpoint2', 'endpoint3'])
     })
 
     it('Should return the endpoints to iterate for a state with no page', async () => {
-      const state = { endpoint: 'endpoint2', page: '' }
+      const state = { endpoint: 'endpoint2', page: '', endpoints: [] }
       const iterator = BaseIterator.getEndPointsIterator(endpoints, state)
 
       expect(iterator).toStrictEqual(['endpoint2', 'endpoint3'])
     })
 
     it('Should return the whole endpoints when the state is in the first endpoint with a page', async () => {
-      const state = { endpoint: 'endpoint1', page: 'here' }
+      const state = { endpoint: 'endpoint1', page: 'here', endpoints: [] }
       const iterator = BaseIterator.getEndPointsIterator(endpoints, state)
 
       expect(iterator).toStrictEqual(endpoints)
     })
 
     it('Should return the last endpoint when the state is in the last endpoint', async () => {
-      const state = { endpoint: 'endpoint3', page: '' }
+      const state = { endpoint: 'endpoint3', page: '', endpoints: [] }
       const iterator = BaseIterator.getEndPointsIterator(endpoints, state)
 
       expect(iterator).toStrictEqual(['endpoint3'])
     })
 
     it('Should throw an error when state.endpoint is not in endpoints', async () => {
-      const state = { endpoint: 'endpoint4', page: '' }
+      const state = { endpoint: 'endpoint4', page: '', endpoints: [] }
       try {
         BaseIterator.getEndPointsIterator(endpoints, state)
       } catch (error: any) {
@@ -48,17 +48,20 @@ describe('Integrations worker static tests', () => {
       const state = BaseIterator.initState(endpoints, {
         endpoint: '',
         page: '',
+        endpoints: [],
       })
       expect(state).toStrictEqual({
         endpoint: 'endpoint1',
         page: '',
+        endpoints: ['endpoint1', 'endpoint2', 'endpoint3'],
       })
     })
 
     it('Should return the given state', async () => {
       const givenState = {
         endpoint: 'endpoint2',
         page: 'here',
+        endpoints: [],
       }
       const state = BaseIterator.initState(endpoints, givenState)
       expect(state).toStrictEqual(givenState)

backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts

@@ -76,6 +76,7 @@ async function getDevtoIterator(articles: DevtoArticle[]) {
     {
       endpoint: '',
       page: '',
+      endpoints: [],
     },
     false,
   )

backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts

@@ -589,7 +589,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('1', recentRecord)
@@ -605,7 +605,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('4', recentRecord)
@@ -621,7 +621,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('3', recentRecord)
@@ -637,7 +637,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('1', oldRecord)
@@ -653,7 +653,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('3', oldRecord)
@@ -669,7 +669,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('4', oldRecord)
@@ -685,7 +685,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOverGeneral = iter.isEndpointFinished('4', {}, [])

backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts

@@ -38,6 +38,7 @@ async function getGithubIterator(repos: Repos, options: IRepositoryOptions) {
     {
       endpoint: '',
       page: '',
+      endpoints: [],
     },
     true,
   )