Skip to content

Commit 88fc09c

Browse files
ZJvandeWegZJvandeWeg
authored
Merge pull request #4356 from FlowFuse/integrations
Generate Integration Pages
2 parents a67b51c + f988ec7 commit 88fc09c

6 files changed

Lines changed: 719 additions & 58 deletions

File tree

‎.eleventy.js‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,15 @@ module.exports = function(eleventyConfig) {
162162
return JSON.stringify(content)
163163
});
164164

165+
eleventyConfig.addFilter("fromJson", (content) => {
166+
try {
167+
return JSON.parse(content);
168+
} catch (e) {
169+
console.error("Error parsing JSON:", e);
170+
return content;
171+
}
172+
});
173+
165174
eleventyConfig.addFilter("head", (array, n) => {
166175
if( n < 0 ) {
167176
return array.slice(n);
@@ -198,6 +207,81 @@ module.exports = function(eleventyConfig) {
198207
return spacetime(new Date(dateObj)).format('{date} {month-short}, {year}')
199208
});
200209

210+
// Filter to safely convert values to Date objects
211+
eleventyConfig.addFilter('toDate', value => {
212+
if (!value) return new Date();
213+
if (value instanceof Date) return value;
214+
return new Date(value);
215+
});
216+
217+
eleventyConfig.addFilter('formatNumber', num => {
218+
if (num === undefined || num === null) return '0';
219+
return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
220+
});
221+
222+
eleventyConfig.addFilter('md', (content) => {
223+
if (!content) return '';
224+
const md = new markdownIt({
225+
html: true,
226+
})
227+
.use(markdownItAnchor, {
228+
permalink: markdownItAnchor.permalink.headerLink()
229+
});
230+
return md.render(content);
231+
});
232+
233+
eleventyConfig.addFilter('stripFirstH1', (str) => {
234+
if (!str) return str;
235+
236+
// Remove the first h1 heading from the content to avoid duplicate h1 tags
237+
// This is typically the package name which is already shown in the page header
238+
return str.replace(/<h1[^>]*>.*?<\/h1>/, '');
239+
});
240+
241+
eleventyConfig.addFilter('rewriteIntegrationLinks', (str, integration) => {
242+
if (!str) return str;
243+
244+
// Convert relative links in README to absolute links
245+
const matcher = /((href|src)="([^"]*))"/g;
246+
let match;
247+
const result = str.replace(matcher, (fullMatch, group1, attr, url) => {
248+
// Skip absolute URLs and mailto links
249+
if (/^(http|https|mailto:)/.test(url)) {
250+
return fullMatch;
251+
}
252+
253+
// Skip pure anchors (same-page links)
254+
if (url.startsWith('#')) {
255+
return fullMatch;
256+
}
257+
258+
// Convert relative links to repository links if available
259+
if (integration.repository && integration.repository.url) {
260+
const repoUrl = integration.repository.url
261+
.replace('git+', '')
262+
.replace('.git', '')
263+
.replace('git://', 'https://');
264+
265+
// Handle different types of relative paths
266+
if (url.startsWith('./') || url.startsWith('../')) {
267+
const cleanUrl = url.replace(/^\.\.?\//, '');
268+
return `${attr}="${repoUrl}/blob/master/${cleanUrl}"`;
269+
} else if (url.startsWith('/')) {
270+
// Repository-relative paths (e.g., /CHANGELOG.md)
271+
const cleanUrl = url.replace(/^\//, '');
272+
return `${attr}="${repoUrl}/blob/master/${cleanUrl}"`;
273+
} else if (!url.startsWith('#')) {
274+
// Simple relative paths without prefix
275+
return `${attr}="${repoUrl}/blob/master/${url}"`;
276+
}
277+
}
278+
279+
return fullMatch;
280+
});
281+
282+
return result;
283+
});
284+
201285
eleventyConfig.addFilter('duration', mins => {
202286
if (mins > 60) {
203287
const hrs = Math.floor(mins/60)

‎src/_data/integrations.js‎

Lines changed: 209 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,46 +2,226 @@ const EleventyFetch = require("@11ty/eleventy-fetch");
22
const certifiedNodes = require("./certifiedNodes");
33

44
module.exports = async () => {
5-
console.log("Loading Integrations...")
5+
console.log("Loading Integrations...");
66
const api = "https://ff-integrations.flowfuse.cloud/api/nodes";
77

8+
const cacheDuration = "1h";
9+
810
const response = await EleventyFetch(api, {
9-
duration: "4h", // ensure we've gathered new data every 4 hours
11+
duration: cacheDuration,
1012
type: "json"
1113
});
1214

15+
// Get certified nodes first
1316
const nodes = await certifiedNodes();
1417
const ffNodesMap = nodes.reduce((acc, node) => {
1518
acc[node.id] = node;
1619
return acc;
1720
}, {});
18-
19-
// TODO: Overlap certified nodes here
20-
const data = response.catalogue.map((node) => {
21-
if (ffNodesMap[node._id]) {
22-
node.ffCertified = true
23-
}
24-
if (!node.categories) {
25-
node.categories = []
21+
22+
// Sort by weekly downloads and get top 50 nodes
23+
const topNodes = response.catalogue
24+
.sort((a, b) => b.downloads.week - a.downloads.week)
25+
.slice(0, 50); // Limit to top 50 downloaded nodes
26+
27+
// Create a map of top nodes by ID for quick lookup
28+
const topNodesMap = topNodes.reduce((acc, node) => {
29+
acc[node._id] = node;
30+
return acc;
31+
}, {});
32+
33+
// Merge: ensure all certified nodes are included
34+
// Add any certified nodes that aren't in the top 50
35+
const certifiedNodeIds = Object.keys(ffNodesMap);
36+
certifiedNodeIds.forEach(certifiedId => {
37+
if (!topNodesMap[certifiedId]) {
38+
// Find the certified node in the full catalogue
39+
const certifiedNode = response.catalogue.find(n => n._id === certifiedId);
40+
if (certifiedNode) {
41+
topNodes.push(certifiedNode);
42+
}
2643
}
27-
// map to ensure we have unique collection names
28-
node.categories = node.categories.map(category => {
29-
return category.includes('catalogue') ? category : 'catalogue_' + category
44+
});
45+
46+
const data = Promise.all(
47+
topNodes.map(async (node) => {
48+
// Mark FlowFuse certified nodes
49+
if (ffNodesMap[node._id]) {
50+
node.ffCertified = true;
51+
}
52+
53+
// Ensure categories exist
54+
if (!node.categories) {
55+
node.categories = [];
56+
}
57+
58+
// Ensure unique catalogue-based collection names
59+
node.categories = node.categories.map(category =>
60+
category.includes("catalogue")
61+
? category
62+
: "catalogue_" + category
63+
);
64+
65+
if (!node.categories.includes("catalogue")) {
66+
node.categories.push("catalogue");
67+
}
68+
69+
// Fetch full npm node details (readme, etc.)
70+
try {
71+
const nodeDetails = await EleventyFetch(
72+
`https://registry.npmjs.org/${node._id}`,
73+
{
74+
duration: cacheDuration,
75+
type: "json"
76+
}
77+
);
78+
79+
// Extract additional metadata
80+
node.author = nodeDetails.author;
81+
node.maintainers = nodeDetails.maintainers || [];
82+
node.homepage = nodeDetails.homepage;
83+
node.bugs = nodeDetails.bugs;
84+
node.repository = nodeDetails.repository;
85+
node.time = nodeDetails.time;
86+
node.lastUpdated = nodeDetails.time?.modified || nodeDetails.time?.[node.version];
87+
node.created = nodeDetails.time?.created;
88+
// Extract license from npm registry
89+
node.license = nodeDetails.license || nodeDetails.versions?.[node.version]?.license;
90+
91+
// Extract GitHub info if repository is GitHub
92+
if (nodeDetails.repository?.url) {
93+
const repoUrl = nodeDetails.repository.url
94+
.replace('git+', '')
95+
.replace('.git', '')
96+
.replace('git://', 'https://');
97+
98+
const githubMatch = repoUrl.match(/github\.com\/([^\/]+)\/([^\/]+)/);
99+
if (githubMatch) {
100+
node.githubOwner = githubMatch[1];
101+
node.githubRepo = githubMatch[2];
102+
103+
// Try to fetch examples from GitHub
104+
try {
105+
const examplesUrl = `https://api.github.com/repos/${node.githubOwner}/${node.githubRepo}/contents/examples`;
106+
const examplesResponse = await EleventyFetch(examplesUrl, {
107+
duration: cacheDuration,
108+
type: "json",
109+
fetchOptions: {
110+
headers: {
111+
'User-Agent': 'FlowFuse-Website'
112+
}
113+
}
114+
});
115+
116+
// Filter for .json files (Node-RED flows)
117+
if (Array.isArray(examplesResponse)) {
118+
const exampleFiles = examplesResponse
119+
.filter(file => file.name.endsWith('.json') && file.type === 'file');
120+
121+
// Fetch the actual flow content for each example
122+
node.examples = await Promise.all(
123+
exampleFiles.map(async (file) => {
124+
try {
125+
// Fetch the raw flow JSON content
126+
const flowContent = await EleventyFetch(file.download_url, {
127+
duration: cacheDuration,
128+
type: "text",
129+
fetchOptions: {
130+
headers: {
131+
'User-Agent': 'FlowFuse-Website'
132+
}
133+
}
134+
});
135+
136+
return {
137+
name: file.name.replace('.json', ''), // Remove .json extension for display
138+
path: file.path,
139+
url: file.html_url,
140+
downloadUrl: file.download_url,
141+
flow: flowContent // Store the actual flow JSON as string
142+
};
143+
} catch (err) {
144+
console.error(`Failed to fetch flow content for ${file.name}:`, err.message);
145+
// Return without flow content if fetch fails
146+
return {
147+
name: file.name.replace('.json', ''),
148+
path: file.path,
149+
url: file.html_url,
150+
downloadUrl: file.download_url
151+
};
152+
}
153+
})
154+
);
155+
}
156+
} catch (err) {
157+
// Examples folder doesn't exist or API error - this is fine, just skip
158+
node.examples = [];
159+
}
160+
}
161+
}
162+
163+
if (nodeDetails.readme) {
164+
// Fix relative image paths to use GitHub raw content
165+
node.readme = nodeDetails.readme
166+
// Fix relative image paths in markdown style
167+
.replace(
168+
/!\[(.*?)\]\((?!https?:\/\/)([^)]+)\)/g,
169+
(match, alt, imagePath) => {
170+
// If we have GitHub info, construct the raw GitHub URL
171+
if (node.githubOwner && node.githubRepo && imagePath) {
172+
// Clean up the path - remove leading ./ or ../
173+
const cleanPath = imagePath.replace(/^(\.\.\/)+/, '').replace(/^\.\//, '');
174+
// Use the default branch (usually main or master)
175+
const rawUrl = `https://raw.githubusercontent.com/${node.githubOwner}/${node.githubRepo}/master/${cleanPath}`;
176+
return `![${alt}](${rawUrl})`;
177+
}
178+
// If no GitHub info, return the match as-is (will be broken, but at least visible)
179+
return match;
180+
}
181+
)
182+
// Fix relative image paths in HTML img tags
183+
.replace(
184+
/<img([^>]*?)src=["']((?!https?:\/\/)(\.\.\/)?(\.\/)?[^"']+)["']([^>]*?)>/gi,
185+
(match, before, src, after) => {
186+
// If we have GitHub info, construct the raw GitHub URL
187+
if (node.githubOwner && node.githubRepo) {
188+
// Clean up the path - remove leading ./ or ../
189+
const cleanPath = src.replace(/^(\.\.\/)+/, '').replace(/^\.\//, '');
190+
// Use the default branch (usually main or master)
191+
const rawUrl = `https://raw.githubusercontent.com/${node.githubOwner}/${node.githubRepo}/master/${cleanPath}`;
192+
return `<img${before}src="${rawUrl}"${after}>`;
193+
}
194+
// If no GitHub info, return the match as-is
195+
return match;
196+
}
197+
);
198+
} else {
199+
node.readme = "";
200+
}
201+
202+
console.log(`Loaded readme for ${node._id}`);
203+
} catch (err) {
204+
// Only log non-404 errors to avoid cluttering console with missing packages
205+
if (!err.message || !err.message.includes('404')) {
206+
console.error(`Failed to load readme for ${node._id}`, err);
207+
}
208+
node.readme = "";
209+
}
210+
211+
return node;
30212
})
31-
if (node.categories.indexOf("catalogue") === -1) {
32-
node.categories.push("catalogue")
33-
}
34-
return node
35-
}).sort((a, b) => {
36-
if (a.ffCertified && !b.ffCertified) {
37-
return -1
38-
}
39-
if (!a.ffCertified && b.ffCertified) {
40-
return 1
41-
}
42-
return a.name.localeCompare(b.name)
43-
})
44-
console.log("Loaded Integrations.")
213+
).then((nodes) =>
214+
nodes
215+
.sort((a, b) => {
216+
// Certified nodes first
217+
if (a.ffCertified && !b.ffCertified) return -1;
218+
if (!a.ffCertified && b.ffCertified) return 1;
219+
220+
// Then by weekly downloads (descending)
221+
return b.downloads.week - a.downloads.week;
222+
})
223+
);
45224

46-
return data
47-
}
225+
console.log("Loaded Integrations.");
226+
return data;
227+
};

0 commit comments

Comments
 (0)