@@ -4,16 +4,17 @@ import chunk from 'lodash/chunk';
44import fs from 'node:fs' ;
55import path from 'node:path' ;
66
7+ import debugGithubRepos from '~/debug-github-repos.json' ;
8+ import githubRepos from '~/react-native-libraries.json' ;
9+ import { Library } from '~/types' ;
10+ import { isLaterThan , TimeRange } from '~/util/datetime' ;
11+ import { isEmptyOrNull } from '~/util/strings' ;
12+
713import { calculateDirectoryScore , calculatePopularityScore } from './calculate-score' ;
814import { fetchGithubData , fetchGithubRateLimit , loadGitHubLicenses } from './fetch-github-data' ;
915import { fetchNpmData , fetchNpmDataBulk } from './fetch-npm-data' ;
1016import fetchReadmeImages from './fetch-readme-images' ;
1117import { fillNpmName , hasMismatchedPackageData , sleep } from './helpers' ;
12- import debugGithubRepos from '../debug-github-repos.json' ;
13- import githubRepos from '../react-native-libraries.json' ;
14- import { Library } from '../types' ;
15- import { isLaterThan , TimeRange } from '../util/datetime' ;
16- import { isEmptyOrNull } from '../util/strings' ;
1718
1819// Uses debug-github-repos.json instead, so we have less repositories to crunch
1920// each time we run the script
@@ -91,7 +92,7 @@ async function buildAndScoreData() {
9192 data . map ( async project => {
9293 if ( ! project . template ) {
9394 if ( project . npmPkg . startsWith ( '@' ) ) {
94- await sleep ( Math . max ( Math . random ( ) * 2500 ) ) ;
95+ await sleep ( Math . random ( ) * 10000 ) ;
9596 return fetchNpmData ( project ) ;
9697 } else {
9798 bulkList . push ( project . npmPkg ) ;
@@ -214,7 +215,11 @@ async function buildAndScoreData() {
214215 const existingData = libraries . map ( lib => lib . npmPkg ) ;
215216 const newData = data . map ( lib => lib . npmPkg ) ;
216217 const missingData = existingData . filter ( npmPkg => ! newData . includes ( npmPkg ) ) ;
217- const currentData = [ ...libraries . filter ( lib => missingData . includes ( lib . npmPkg ) ) , ...data ] ;
218+
219+ const existingPackages = DATASET . map ( fillNpmName ) . map ( lib => lib . npmPkg ) ;
220+ const dataToFill = missingData . filter ( npmPkg => ! existingPackages . includes ( npmPkg ) ) ;
221+
222+ const currentData = [ ...libraries . filter ( lib => dataToFill . includes ( lib . npmPkg ) ) , ...data ] ;
218223
219224 const dataWithFallback = currentData . map ( entry =>
220225 Object . keys ( entry . npm ) . length > 0
@@ -225,10 +230,11 @@ async function buildAndScoreData() {
225230 latestData . libraries . find ( prevEntry => entry . npmPkg === prevEntry . npmPkg ) ?. npm ?? { } ,
226231 }
227232 ) ;
233+ const finalData = dataWithFallback . filter ( npmPkg => ! existingPackages . includes ( npmPkg ) ) ;
228234
229235 fileContent = JSON . stringify (
230236 {
231- libraries : dataWithFallback ,
237+ libraries : finalData ,
232238 topics : topicCounts ,
233239 topicsList : Object . keys ( topicCounts ) . sort ( ) ,
234240 } ,
0 commit comments