diff --git a/scripts/fetchData.ts b/scripts/fetchData.ts index c9eb0f1b..8f395261 100644 --- a/scripts/fetchData.ts +++ b/scripts/fetchData.ts @@ -1,7 +1,9 @@ import { GraphQLClient, gql } from 'graphql-request'; import fs from 'fs'; import path from 'path'; -import { AllDataQuery } from '../generated/types'; +import { ProjectsPageQuery, StaticDataQuery } from '../generated/types'; + +type FetchedData = StaticDataQuery & { publicProjects: ProjectsPageQuery['publicProjects'] }; const datadir = path.join(__dirname, '../fullData'); const baseUrl = process.env.MAPSWIPE_API_ENDPOINT || 'http://localhost:8000/'; @@ -12,7 +14,7 @@ const pipelineType = process.env.PIPELINE_TYPE; const graphQLClient = new GraphQLClient(GRAPHQL_ENDPOINT); -const dummyData: AllDataQuery = { +const dummyData: FetchedData = { publicProjects: { results: [], totalCount: 0, @@ -29,15 +31,17 @@ const dummyData: AllDataQuery = { globalExportAssets: [], }; -const query = gql` - query AllData { +const PROJECT_PAGE_SIZE = 500; + +const projectsQuery = gql` + query ProjectsPage($limit: Int!, $offset: Int!) { publicProjects( filters: { status: { inList: [PUBLISHED, FINISHED], }, }, - pagination: { limit: 9999 }, + pagination: { limit: $limit, offset: $offset }, ) { results { id @@ -175,6 +179,11 @@ const query = gql` } totalCount } + } +`; + +const staticQuery = gql` + query StaticData { communityStats { id totalContributors @@ -225,8 +234,32 @@ async function getCsrfTokenValue() { return undefined; } +const MAX_RETRIES = 3; + +async function requestWithRetry( + requestFn: () => Promise, +): Promise { + for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { + try { + // eslint-disable-next-line no-await-in-loop + return await requestFn(); + } catch (err: unknown) { + const status = (err as { response?: { status?: number } })?.response?.status; + const isServerError = status !== undefined && status >= 500; + if (isServerError && attempt < MAX_RETRIES) { + // eslint-disable-next-line no-console + console.warn(`Request failed with ${status} (attempt ${attempt}/${MAX_RETRIES}). Retrying...`); + } else { + throw err; + } + } + } + // unreachable, but satisfies the return type + throw new Error('Unexpected end of requestWithRetry'); +} + async function fetchAndWriteData() { - let data = {} as AllDataQuery; + let data = {} as FetchedData; if (pipelineType === 'ci') { data = dummyData; } else { @@ -245,7 +278,33 @@ async function fetchAndWriteData() { graphQLClient.setHeader('X-CSRFToken', csrfTokenValue); graphQLClient.setHeader('Cookie', `${COOKIE_NAME}=${csrfTokenValue}`); graphQLClient.setHeader('Referer', referer); - data = (await graphQLClient.request(query)) as AllDataQuery; + + const staticData = (await requestWithRetry(() => graphQLClient.request(staticQuery))) as Pick< + FetchedData, + 'communityStats' | 'publicOrganizations' | 'globalExportAssets' + >; + + const allProjects: FetchedData['publicProjects']['results'] = []; + let totalCount = Infinity; + // eslint-disable-next-line no-await-in-loop + for (let offset = 0; offset < totalCount; offset += PROJECT_PAGE_SIZE) { + // eslint-disable-next-line no-await-in-loop + const page = (await requestWithRetry(() => graphQLClient.request(projectsQuery, { + limit: PROJECT_PAGE_SIZE, + offset, + }))) as Pick; + allProjects.push(...page.publicProjects.results); + totalCount = page.publicProjects.totalCount; + console.log(`Fetched ${allProjects.length}/${totalCount} projects`); + } + + data = { + ...staticData, + publicProjects: { + results: allProjects, + totalCount, + }, + }; } // ensure the `data` directory exists diff --git a/src/components/ProjectsMap/index.tsx b/src/components/ProjectsMap/index.tsx index 4efed400..d7c72a8d 100644 --- a/src/components/ProjectsMap/index.tsx +++ b/src/components/ProjectsMap/index.tsx @@ -29,11 +29,11 @@ import { import GestureHandler from 'components/LeafletGestureHandler'; import Link from 'components/Link'; -import { AllDataQuery } from 'generated/types'; +import { ProjectsPageQuery } from 'generated/types'; import styles from './styles.module.css'; -type PublicProject = NonNullable['results']>[number]; +type PublicProject = ProjectsPageQuery['publicProjects']['results'][number]; const pathOptions: { [key in ProjectStatus]?: CircleMarkerOptions diff --git a/src/pages/[locale]/data/index.tsx b/src/pages/[locale]/data/index.tsx index 782af27c..1321a868 100644 --- a/src/pages/[locale]/data/index.tsx +++ b/src/pages/[locale]/data/index.tsx @@ -50,11 +50,13 @@ import useDebouncedValue from 'hooks/useDebouncedValue'; import { GlobalExportAssets } from 'utils/queries'; import data from 'fullData/staticData.json'; -import { AllDataQuery } from 'generated/types'; +import { ProjectsPageQuery, StaticDataQuery } from 'generated/types'; import i18nextConfig from '@/next-i18next.config'; import styles from './styles.module.css'; +type AllDataQuery = StaticDataQuery & { publicProjects: ProjectsPageQuery['publicProjects'] }; + type PublicProjects = NonNullable['results']>; type PublicProject = PublicProjects[number]; diff --git a/src/pages/[locale]/index.tsx b/src/pages/[locale]/index.tsx index 12732a74..db9f341c 100644 --- a/src/pages/[locale]/index.tsx +++ b/src/pages/[locale]/index.tsx @@ -27,10 +27,12 @@ import data from 'fullData/staticData.json'; import i18nextConfig from '@/next-i18next.config'; -import { AllDataQuery } from 'generated/types'; +import { ProjectsPageQuery, StaticDataQuery } from 'generated/types'; import styles from './styles.module.css'; +type AllDataQuery = StaticDataQuery & { publicProjects: ProjectsPageQuery['publicProjects'] }; + async function getAllData() { // FIXME: This should be inferred return data as AllDataQuery; diff --git a/src/pages/[locale]/projects/[id].tsx b/src/pages/[locale]/projects/[id].tsx index 2c3c1886..ea7c6cdc 100644 --- a/src/pages/[locale]/projects/[id].tsx +++ b/src/pages/[locale]/projects/[id].tsx @@ -52,11 +52,13 @@ import { } from 'utils/chart'; import { UrlInfo } from 'utils/queries'; -import { AllDataQuery } from 'generated/types'; +import { ProjectsPageQuery, StaticDataQuery } from 'generated/types'; import i18nextConfig from '@/next-i18next.config'; import styles from './styles.module.css'; +type AllDataQuery = StaticDataQuery & { publicProjects: ProjectsPageQuery['publicProjects'] }; + type PublicProjects = NonNullable['results']>; async function getAllProjects() { return (data as AllDataQuery)?.publicProjects?.results as unknown as PublicProjects;