Skip to content

Commit ab76a33

Browse files
mremre
authored
Load GitHub tools.json file locally (#86)
* Phase 1: Data Layer Modernization This commit introduces build-time static data generation, replacing runtime GitHub API calls for tools data. Key changes: ## New Build System - Added scripts/build-data.ts: Fetches and consolidates tools data from static-analysis and dynamic-analysis repos at build time - Added prebuild hook in package.json to run build-data before next build - Data is stored in data/tools.json, data/tags.json, data/tool-stats.json, and data/tag-stats.json (all gitignored as they're generated) ## New Static Data Utilities - utils/static-data.ts: Core utility for reading pre-built static data - utils/tools.ts: Simplified tools API using static data - utils/tools-with-votes.ts: Merges tools with votes data - utils/firebase-votes.ts: Simplified Firebase votes fetching - utils/tags.ts: Tags utility using static data - utils/filters.ts: Filtering utility (moved from utils-api) - utils/stats.ts: View statistics utility ## Updated Pages - pages/index.tsx: Uses new static data utilities - pages/tool/[slug].tsx: Uses new static data utilities - pages/tag/[slug].tsx: Uses new static data utilities ## Build/Deploy Updates - Dockerfile: Removed manual tools.json download (now handled by build-data) - deploy.yml: Updated to hash generated data files for cache busting ## Benefits - No runtime GitHub API calls for tools data - Faster page generation (data is pre-fetched) - Simplified data flow - Old utils-api code preserved for backwards compatibility Note: API routes and old utils-api code are preserved for now. The /tools page still uses API routes which will be addressed in Phase 2 with InstantSearch integration. * Phase 1: Data Layer Modernization This commit introduces build-time static data generation, replacing runtime GitHub API calls for tools data. Key changes: ## New Build System - Added scripts/build-data.ts: Fetches and consolidates tools data from static-analysis and dynamic-analysis repos at build time - Added prebuild hook in package.json to run build-data before next build - Data is stored in data/tools.json, data/tags.json, data/tool-stats.json, and data/tag-stats.json (all gitignored as they're generated) ## New Static Data Utilities - utils/static-data.ts: Core utility for reading pre-built static data - utils/tools.ts: Simplified tools API using static data - utils/tools-with-votes.ts: Merges tools with votes data - utils/firebase-votes.ts: Simplified Firebase votes fetching - utils/tags.ts: Tags utility using static data - utils/filters.ts: Filtering utility (moved from utils-api) - utils/stats.ts: View statistics utility ## Updated Pages - pages/index.tsx: Uses new static data utilities - pages/tool/[slug].tsx: Uses new static data utilities - pages/tag/[slug].tsx: Uses new static data utilities ## Build/Deploy Updates - Dockerfile: Removed manual tools.json download (now handled by build-data) - deploy.yml: Updated to hash generated data files for cache busting ## Benefits - No runtime GitHub API calls for tools data - Faster page generation (data is pre-fetched) - Simplified data flow - Old utils-api code preserved for backwards compatibility Note: API routes and old utils-api code are preserved for now. The /tools page still uses API routes which will be addressed in Phase 2 with InstantSearch integration. * Phase 1: Data Layer Modernization This commit introduces build-time static data generation, replacing runtime GitHub API calls for tools data. Key changes: ## New Build System - Added scripts/build-data.ts: Fetches and consolidates tools data from static-analysis and dynamic-analysis repos at build time - Added prebuild hook in package.json to run build-data before next build - Data is stored in data/tools.json, data/tags.json, data/tool-stats.json, and data/tag-stats.json (all gitignored as they're generated) ## New Repository Classes (lib/repositories/) - ToolsRepository: Singleton for accessing tools data with caching - TagsRepository: Singleton for accessing tags and descriptions - StatsRepository: Singleton for tool/tag view statistics - VotesRepository: Singleton for Firebase votes fetching - ToolsFilter: Fluent filter class for querying tools by various criteria ## Updated Pages - pages/index.tsx: Uses StatsRepository and VotesRepository - pages/tool/[slug].tsx: Uses ToolsRepository and VotesRepository - pages/tag/[slug].tsx: Uses TagsRepository, ToolsRepository, and ToolsFilter ## Build/Deploy Updates - Dockerfile: Removed manual tools.json download (now handled by build-data) - deploy.yml: Updated to hash generated data files for cache busting - tsconfig.json: Added @lib/* path alias ## Benefits - No runtime GitHub API calls for tools data - Faster page generation (data is pre-fetched) - Idiomatic TypeScript with proper class-based repositories - Singleton pattern ensures data is loaded once and cached - Old utils-api code preserved for backwards compatibility Note: API routes and old utils-api code are preserved for now. The /tools page still uses API routes which will be addressed in Phase 2 with InstantSearch integration.
1 parent e3a1bc7 commit ab76a33

16 files changed

Lines changed: 1292 additions & 122 deletions

File tree

‎.github/workflows/deploy.yml‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -87,29 +87,24 @@ jobs:
8787
app_id: ${{ secrets.GH_APP_ID }}
8888
private_key: ${{ secrets.GH_APP_PRIVATE_KEY }}
8989

90-
# We want to redeploy, whenever the tools.json file changes
91-
# so make sure to hash the file and use it as a tag for the Docker image
92-
- name: 'Download tools.json File'
93-
run: curl -sL https://github.com/analysis-tools-dev/static-analysis/raw/master/data/api/tools.json -o ./tools.json
94-
95-
- name: 'Generate Hash of tools.json File'
96-
id: tools_json_hash
97-
run: echo "tools_json_hash=$(sha256sum tools.json | cut -c1-7)" >> $GITHUB_ENV
90+
# Hash the generated tools.json from the build (created by npm run build-data)
91+
# to ensure we redeploy when tools data changes
92+
- name: 'Generate Hash of built tools data'
93+
run: |
94+
echo "tools_hash=$(sha256sum data/tools.json | cut -c1-7)" >> $GITHUB_ENV
9895
99-
# Also take screenshots.json into account, which is at
100-
# https://github.com/analysis-tools-dev/assets/blob/master/screenshots.json
96+
# Also take screenshots.json into account for cache busting
10197
- name: 'Download screenshots.json File'
10298
run: curl -sL https://github.com/analysis-tools-dev/assets/raw/master/screenshots.json -o ./screenshots.json
10399

104100
- name: 'Generate Hash of screenshots.json File'
105-
id: screenshots_json_hash
106-
run: echo "screenshots_json_hash=$(sha256sum screenshots.json | cut -c1-7)" >> $GITHUB_ENV
101+
run: echo "screenshots_hash=$(sha256sum screenshots.json | cut -c1-7)" >> $GITHUB_ENV
107102

108-
# Image hash is a combination of the hashes
103+
# Image hash is a combination of commit + data hashes
109104
- name: 'Set IMAGE_NAME hash'
110105
run: |
111106
short_hash=$(echo "${{ github.sha }}" | cut -c1-7)
112-
echo "IMAGE_NAME=${{ env.IMAGE_NAME }}:$short_hash-${{ env.tools_json_hash }}-${{ env.screenshots_json_hash }}" >> $GITHUB_ENV
107+
echo "IMAGE_NAME=${{ env.IMAGE_NAME }}:$short_hash-${{ env.tools_hash }}-${{ env.screenshots_hash }}" >> $GITHUB_ENV
113108
114109
- name: 'Build Docker Image'
115110
env:

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,5 +46,12 @@ algolia-index.js
4646

4747
credentials.json
4848

49+
# Generated data files (created by npm run build-data)
50+
/data/tools.json
51+
/data/tags.json
52+
/data/tool-stats.json
53+
/data/tag-stats.json
4954

5055
.vscode
56+
57+
TODO.md

‎Dockerfile‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,10 @@ ARG PROJECT_ID
1111

1212
COPY . /src
1313

14-
# Download tools.json directly in Dockerfile and detect changes
15-
# This is done to ensure that we redeploy the app whenever the tools.json changes
16-
ADD https://raw.githubusercontent.com/analysis-tools-dev/static-analysis/master/data/api/tools.json /src/data/api/tools.json
17-
14+
# Build runs npm run build-data (prebuild hook) which fetches tools data
15+
# from GitHub repos and generates static JSON files, then runs next build
1816
RUN npm run build
19-
RUN rm /src/credentials.json /src/data/api/tools.json
17+
RUN rm /src/credentials.json
2018

2119
FROM node:20
2220
WORKDIR /src

‎lib/repositories/StatsRepository.ts‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
/**
2+
* StatsRepository
3+
*
4+
* Repository class for accessing tool and tag statistics from static JSON files.
5+
* Data is pre-built at build time by scripts/build-data.ts.
6+
*/
7+
8+
import * as fs from 'fs';
9+
import * as path from 'path';
10+
import type { StatsApiData, VotesApiData } from 'utils/types';
11+
import type { Tool, ToolsByLanguage } from '@components/tools/types';
12+
import { ToolsRepository } from './ToolsRepository';
13+
import { sortByVote } from 'utils/votes';
14+
15+
export class StatsRepository {
16+
private static instance: StatsRepository | null = null;
17+
private toolStatsData: StatsApiData | null = null;
18+
private tagStatsData: StatsApiData | null = null;
19+
20+
private readonly toolStatsPath: string;
21+
private readonly tagStatsPath: string;
22+
23+
private constructor() {
24+
const dataDir = path.join(process.cwd(), 'data');
25+
this.toolStatsPath = path.join(dataDir, 'tool-stats.json');
26+
this.tagStatsPath = path.join(dataDir, 'tag-stats.json');
27+
}
28+
29+
static getInstance(): StatsRepository {
30+
if (!StatsRepository.instance) {
31+
StatsRepository.instance = new StatsRepository();
32+
}
33+
return StatsRepository.instance;
34+
}
35+
36+
private loadToolStats(): StatsApiData {
37+
if (this.toolStatsData) {
38+
return this.toolStatsData;
39+
}
40+
41+
if (!fs.existsSync(this.toolStatsPath)) {
42+
console.warn(
43+
'Static tool stats not found. Run `npm run build-data` first.',
44+
);
45+
return {};
46+
}
47+
48+
const content = fs.readFileSync(this.toolStatsPath, 'utf-8');
49+
this.toolStatsData = JSON.parse(content) as StatsApiData;
50+
return this.toolStatsData;
51+
}
52+
53+
private loadTagStats(): StatsApiData {
54+
if (this.tagStatsData) {
55+
return this.tagStatsData;
56+
}
57+
58+
if (!fs.existsSync(this.tagStatsPath)) {
59+
console.warn(
60+
'Static tag stats not found. Run `npm run build-data` first.',
61+
);
62+
return {};
63+
}
64+
65+
const content = fs.readFileSync(this.tagStatsPath, 'utf-8');
66+
this.tagStatsData = JSON.parse(content) as StatsApiData;
67+
return this.tagStatsData;
68+
}
69+
70+
getToolStats(): StatsApiData {
71+
return this.loadToolStats();
72+
}
73+
74+
getTagStats(): StatsApiData {
75+
return this.loadTagStats();
76+
}
77+
78+
getToolViewCount(toolId: string): number {
79+
const stats = this.loadToolStats();
80+
return stats[toolId] || 0;
81+
}
82+
83+
getTagViewCount(tag: string): number {
84+
const stats = this.loadTagStats();
85+
return stats[tag] || 0;
86+
}
87+
88+
getLanguageStats(): ToolsByLanguage {
89+
const tagStats = this.loadTagStats();
90+
91+
return Object.entries(tagStats)
92+
.sort(([, a], [, b]) => b - a)
93+
.reduce(
94+
(result, [key, value]) => ({
95+
...result,
96+
[key]: {
97+
views: value,
98+
formatters: [],
99+
linters: [],
100+
},
101+
}),
102+
{} as ToolsByLanguage,
103+
);
104+
}
105+
106+
getPopularLanguageStats(votes?: VotesApiData | null): ToolsByLanguage {
107+
const toolsRepo = ToolsRepository.getInstance();
108+
const tools = toolsRepo.withVotes(votes || null);
109+
const languageStats = this.getLanguageStats();
110+
111+
for (const [toolId, tool] of Object.entries(tools)) {
112+
const isSingleLanguage = tool.languages.length <= 2;
113+
114+
if (isSingleLanguage && tool.languages.length > 0) {
115+
const language = tool.languages[0];
116+
117+
if (languageStats[language]) {
118+
const toolObj: Tool = {
119+
id: toolId,
120+
...tool,
121+
votes: tool.votes || 0,
122+
} as Tool;
123+
124+
if (tool.categories.includes('formatter')) {
125+
languageStats[language].formatters.push(toolObj);
126+
}
127+
if (tool.categories.includes('linter')) {
128+
languageStats[language].linters.push(toolObj);
129+
}
130+
131+
languageStats[language].formatters.sort(sortByVote);
132+
languageStats[language].linters.sort(sortByVote);
133+
134+
if (languageStats[language].formatters.length > 3) {
135+
languageStats[language].formatters.pop();
136+
}
137+
if (languageStats[language].linters.length > 3) {
138+
languageStats[language].linters.pop();
139+
}
140+
}
141+
}
142+
}
143+
144+
// Filter out languages with no tools
145+
for (const language of Object.keys(languageStats)) {
146+
if (
147+
languageStats[language].formatters.length === 0 &&
148+
languageStats[language].linters.length === 0
149+
) {
150+
delete languageStats[language];
151+
}
152+
}
153+
154+
return languageStats;
155+
}
156+
157+
getMostViewedTools(votes?: VotesApiData | null): Tool[] {
158+
const toolsRepo = ToolsRepository.getInstance();
159+
const tools = toolsRepo.withVotes(votes || null);
160+
const toolStats = this.loadToolStats();
161+
162+
const mostViewedToolIds = Object.keys(toolStats);
163+
164+
return mostViewedToolIds
165+
.map((id) => {
166+
const tool = tools[id];
167+
if (!tool) {
168+
return null;
169+
}
170+
171+
return {
172+
id,
173+
...tool,
174+
votes: tool.votes || 0,
175+
views: toolStats[id],
176+
} as Tool & { views: number };
177+
})
178+
.filter((tool): tool is Tool & { views: number } => tool !== null);
179+
}
180+
181+
clearCache(): void {
182+
this.toolStatsData = null;
183+
this.tagStatsData = null;
184+
}
185+
}

0 commit comments

Comments
 (0)