Skip to content

Commit 4ddbc04

Browse files
KristjanESPERANTOKristjanESPERANTO
committed
feat(modules): implement skipped modules handling and validation
1 parent 560f008 commit 4ddbc04

8 files changed

Lines changed: 268 additions & 84 deletions

File tree

‎container/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ set -e
1010
git config --global --add safe.directory /workspace
1111
git log -1
1212
export GITHUB_TOKEN="${GITHUB_TOKEN}"
13+
pip install requests
1314
npm clean-install
1415
node --run all
1516
EOF

‎docs/index.css‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -605,3 +605,51 @@ input:checked + .slider::before {
605605
padding: 0.5rem;
606606
cursor: pointer;
607607
}
608+
609+
/* Styling for skipped/failed modules */
610+
.card.skipped {
611+
border-left: 4px solid #dc3545;
612+
background-image: repeating-linear-gradient(140deg, #0000, #f002 2%);
613+
opacity: 0.85;
614+
}
615+
616+
.card.skipped .header {
617+
background-color: #fdeaea;
618+
border-bottom: 1px solid #f5c6cb;
619+
}
620+
621+
.card.skipped .header .title {
622+
color: #721c24;
623+
}
624+
625+
.card.skipped .error-message {
626+
color: #dc3545;
627+
font-weight: 500;
628+
padding: 0.5rem;
629+
background-color: #f8d7da;
630+
border: 1px solid #f5c6cb;
631+
border-radius: 4px;
632+
margin: 0.5rem;
633+
font-size: 0.9rem;
634+
}
635+
636+
/* Dark mode styles for skipped modules */
637+
body.dark .card.skipped {
638+
background-color: #2d1b1b;
639+
border-left-color: #dc3545;
640+
}
641+
642+
body.dark .card.skipped .header {
643+
background-color: #3d2222;
644+
border-bottom-color: #5a2828;
645+
}
646+
647+
body.dark .card.skipped .header .title {
648+
color: #ff6b6b;
649+
}
650+
651+
body.dark .card.skipped .error-message {
652+
background-color: #4a2626;
653+
border-color: #5a2828;
654+
color: #ff8a80;
655+
}

‎docs/script.js‎

Lines changed: 45 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
let allModules = [];
2+
let skippedModules = [];
23
let filteredModuleList = [];
34
const cardTemplate = document.getElementById("card-template");
45
const resetButton = document.getElementById("reset-button");
@@ -33,6 +34,23 @@ function toggleMenu () {
3334
function createCard (moduleData) {
3435
const card = document.importNode(cardTemplate.content, true);
3536

37+
// Skipped module special handling
38+
if (moduleData.skipped) {
39+
card.querySelector(".card").classList.add("skipped");
40+
card.querySelector(".name").textContent = moduleData.name || "Unknown Module";
41+
card.querySelector(".name").href = moduleData.url || "#";
42+
card.querySelector(".description").innerHTML = `<span style='color:red;font-weight:bold'>Error: Module could not be loaded.</span><br>${moduleData.error || "Unknown Error"}`;
43+
// Remove other info sections
44+
card.querySelector(".maintainer").textContent = moduleData.maintainer || "?";
45+
[".stars", ".tags", ".img-container", ".info", ".outdated-note"].forEach((selector) => {
46+
const element = card.querySelector(selector);
47+
if (element) {
48+
element.remove();
49+
}
50+
});
51+
return card;
52+
}
53+
3654
/* Set the header data */
3755
card.querySelector(".name").href = moduleData.url;
3856
card.querySelector(".name").textContent = moduleData.name;
@@ -132,7 +150,7 @@ function updateModuleCardContainer () {
132150
let moduleCounter = filteredModuleList.length;
133151

134152
filteredModuleList.forEach((moduleData) => {
135-
if (!moduleData.outdated || showOutdated.checked) {
153+
if ((!moduleData.outdated || showOutdated.checked) && (!moduleData.skipped || showOutdated.checked)) {
136154
try {
137155
const cardNode = createCard(moduleData);
138156
if (cardNode) {
@@ -229,7 +247,8 @@ function displayStatistics (data) {
229247

230248
function filterBySearchText (searchText) {
231249
const searchLower = searchText.toLowerCase();
232-
filteredModuleList = allModules.filter((card) => {
250+
const allModulesList = allModules.concat(skippedModules);
251+
filteredModuleList = allModulesList.filter((card) => {
233252
const cardText = card.text
234253
? card.text.toLowerCase()
235254
: "";
@@ -295,8 +314,8 @@ function filterByTag (tag) {
295314
function addCategoryFilter () {
296315
const categoryFilter = document.getElementById("category-filter");
297316
const categories = [...new Set(allModules.map((module) => module.category))];
298-
299317
categories.sort();
318+
categories.push("Problematic Modules");
300319

301320
categories.forEach((category) => {
302321
const option = document.createElement("option");
@@ -308,15 +327,15 @@ function addCategoryFilter () {
308327
categoryFilter.addEventListener("change", () => {
309328
const selectedCategory = categoryFilter.value;
310329
if (selectedCategory === "all") {
311-
filteredModuleList = allModules;
330+
filteredModuleList = allModules.concat(skippedModules);
331+
} else if (selectedCategory === "Problematic Modules") {
332+
filteredModuleList = skippedModules;
312333
} else {
313334
filteredModuleList = allModules.filter((module) => module.category === selectedCategory);
314335
}
315336

316337
searchInput.value = "";
317-
318338
removeSelectedMarkingFromTagsAndCards();
319-
320339
updateModuleCardContainer();
321340
});
322341
}
@@ -338,7 +357,7 @@ moduleCardContainer.addEventListener("click", (event) => {
338357
resetButton.addEventListener("click", () => {
339358
resetCategoryFilter();
340359
const root = document.querySelector(":root");
341-
filteredModuleList = allModules;
360+
filteredModuleList = allModules.concat(skippedModules);
342361
searchInput.value = "";
343362
showOutdated.checked = true;
344363
removeSelectedMarkingFromTagsAndCards();
@@ -360,7 +379,7 @@ searchInput.addEventListener("input", () => {
360379
if (searchInput.value) {
361380
filterBySearchText(searchInput.value);
362381
} else {
363-
filteredModuleList = allModules;
382+
filteredModuleList = allModules.concat(skippedModules);
364383
updateModuleCardContainer();
365384
}
366385
}, 180);
@@ -413,15 +432,27 @@ async function initiate () {
413432
const response = await fetch(modulesFile);
414433
const data = await response.json();
415434
allModules = data;
416-
filteredModuleList = data;
417-
sortData(sortDropdown.value);
418-
updateModuleCardContainer();
419-
displayTagButtonContainer();
420-
addCategoryFilter();
421435
} catch (error) {
422-
console.error("Error fetching data:", error);
436+
allModules = [];
437+
console.error("Error fetching modules:", error);
438+
}
439+
440+
// Load skipped modules
441+
try {
442+
const skippedRes = await fetch("data/skipped_modules.json");
443+
const skippedRaw = await skippedRes.json();
444+
skippedModules = skippedRaw.map((moduleObj) => ({...moduleObj, skipped: true, defaultSortWeight: 1000, lastCommit: ""}));
445+
} catch {
446+
skippedModules = [];
423447
}
424448

449+
filteredModuleList = allModules.concat(skippedModules);
450+
sortData(sortDropdown.value);
451+
updateModuleCardContainer();
452+
displayTagButtonContainer();
453+
addCategoryFilter();
454+
455+
// Load statistics
425456
const statisticsFile = "data/stats.json";
426457
try {
427458
const response = await fetch(statisticsFile);

‎scripts/check_modules.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ def check_modules():
273273
}
274274

275275
modules_json_file = open(
276-
"./docs/data/modules.stage.4.json", encoding="utf-8")
276+
"./docs/data/modules.stage.5.json", encoding="utf-8")
277277
modules = json.load(modules_json_file)
278278
stats = {
279279
"moduleCounter": 0,
@@ -287,7 +287,7 @@ def check_modules():
287287

288288
markdown_output_modules = ""
289289

290-
for module in modules:
290+
for module in modules["modules"]:
291291
module["defaultSortWeight"] = 0
292292
stats["moduleCounter"] += 1
293293

@@ -528,7 +528,7 @@ def check_modules():
528528
module["issues"] = False
529529

530530
# Lift modules with many stars in the default sort order.
531-
module["defaultSortWeight"] = module["defaultSortWeight"] - (module['stars'] // 20)
531+
module["defaultSortWeight"] = module["defaultSortWeight"] - (module.get('stars', 0) // 20)
532532

533533
# Modules with few stars shouldn't be too far up in the default sort order. So we give them a minimum value of one.
534534
if module.get('stars', 0) < 3:

‎scripts/check_modules_js.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
import fs from "node:fs";
22
// Disabled: import {isMinified} from "./utils.js";
33

4-
fs.readFile("./docs/data/modules.stage.3.json", "utf8", (err, data) => {
4+
fs.readFile("./docs/data/modules.stage.4.json", "utf8", (err, data) => {
55
if (err) {
66
console.error(err);
77
return;
88
}
99

1010
const modules = JSON.parse(data);
1111

12-
modules.forEach((module) => {
12+
modules.modules.forEach((module) => {
1313
const filePath = `./modules/${module.name}-----${module.maintainer}/${module.name}.js`;
1414

1515
if (fs.existsSync(filePath)) {
@@ -27,5 +27,5 @@ fs.readFile("./docs/data/modules.stage.3.json", "utf8", (err, data) => {
2727
}
2828
});
2929

30-
fs.writeFileSync("./docs/data/modules.stage.4.json", JSON.stringify(modules, null, 2));
30+
fs.writeFileSync("./docs/data/modules.stage.5.json", JSON.stringify(modules, null, 2));
3131
});

‎scripts/create_module_list.js‎

Lines changed: 63 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import fs from "node:fs";
22
import {marked} from "marked";
3+
import process from "node:process";
34
import sanitizeHtml from "sanitize-html";
45

56
async function fetchMarkdownData () {
@@ -28,6 +29,7 @@ async function createModuleList () {
2829
const markdown = await fetchMarkdownData();
2930
const moduleList = [];
3031
let category = "";
32+
const missingRepoErrors = [];
3133

3234
for (const line of markdown.split("\n")) {
3335
if (line.startsWith("### ")) {
@@ -45,53 +47,60 @@ async function createModuleList () {
4547
if (parts.length === 5 || parts.length === 6) {
4648
const issues = [];
4749

48-
const url = parts[1].match(/\[(.*?)\]\((.*?)\)/u)[2].trim();
49-
if (
50-
!url.startsWith("https://github.com") &&
51-
!url.startsWith("https://gitlab.com") &&
52-
!url.startsWith("https://bitbucket.org")
53-
) {
54-
issues.push(`URL: Neither a valid GitHub nor a valid GitLab URL: ${url}.`);
50+
const repoCell = parts[1];
51+
const repoMatch = repoCell.match(/\[(.*?)\]\((.*?)\)/u);
52+
53+
if (repoMatch === null) {
54+
missingRepoErrors.push(line);
55+
} else {
56+
const url = repoMatch[2].trim();
57+
if (
58+
!url.startsWith("https://github.com") &&
59+
!url.startsWith("https://gitlab.com") &&
60+
!url.startsWith("https://bitbucket.org")
61+
) {
62+
issues.push(`URL: Neither a valid GitHub nor a valid GitLab URL: ${url}.`);
63+
}
64+
65+
const maintainer = url.split("/")[3];
66+
const name = url.split("/")[4];
67+
68+
const id = `${maintainer}/${name}`;
69+
70+
const maintainerLinked = parts[2].match(/\[(.*?)\]\((.*?)\)/u);
71+
let maintainerURL = "";
72+
if (maintainerLinked !== null) {
73+
maintainerURL = maintainerLinked[2];
74+
}
75+
76+
const descriptionMarkdown = parts[3];
77+
const descriptionHtml = marked.parseInline(descriptionMarkdown);
78+
const descriptionHtmlATarget = descriptionHtml.replaceAll(
79+
"<a href=",
80+
"<a target=\"_blank\" href="
81+
);
82+
const description = sanitizeHtml(descriptionHtmlATarget);
83+
84+
const module = {
85+
name,
86+
category,
87+
url,
88+
id,
89+
maintainer,
90+
maintainerURL,
91+
description,
92+
issues
93+
};
94+
95+
if (parts.length === 6) {
96+
const outdatedMarkdown = parts[4];
97+
const outdatedHtml = marked.parseInline(outdatedMarkdown);
98+
const outdated = sanitizeHtml(outdatedHtml);
99+
module.outdated = outdated;
100+
}
101+
102+
moduleList.push(module);
55103
}
56-
57-
const maintainer = url.split("/")[3];
58-
const name = url.split("/")[4];
59-
60-
const id = `${maintainer}/${name}`;
61-
62-
const maintainerLinked = parts[2].match(/\[(.*?)\]\((.*?)\)/u);
63-
let maintainerURL = "";
64-
if (maintainerLinked !== null) {
65-
maintainerURL = maintainerLinked[2];
66-
}
67-
68-
const descriptionMarkdown = parts[3];
69-
const descriptionHtml = marked.parseInline(descriptionMarkdown);
70-
const descriptionHtmlATarget = descriptionHtml.replaceAll(
71-
"<a href=",
72-
"<a target=\"_blank\" href="
73-
);
74-
const description = sanitizeHtml(descriptionHtmlATarget);
75-
76-
const module = {
77-
name,
78-
category,
79-
url,
80-
id,
81-
maintainer,
82-
maintainerURL,
83-
description,
84-
issues
85-
};
86-
87-
if (parts.length === 6) {
88-
const outdatedMarkdown = parts[4];
89-
const outdatedHtml = marked.parseInline(outdatedMarkdown);
90-
const outdated = sanitizeHtml(outdatedHtml);
91-
module.outdated = outdated;
92-
}
93-
94-
moduleList.push(module);
95104
}
96105
}
97106
}
@@ -102,6 +111,10 @@ async function createModuleList () {
102111
modules: sortedModuleList
103112
};
104113

114+
if (missingRepoErrors.length > 0) {
115+
throw new Error(`[create_module_list] Missing repository link in ${missingRepoErrors.length} line(s):\n${missingRepoErrors.join("\n")}`);
116+
}
117+
105118

106119
fs.writeFileSync(
107120
"./docs/data/modules.stage.1.json",
@@ -110,4 +123,7 @@ async function createModuleList () {
110123
);
111124
}
112125

113-
createModuleList();
126+
createModuleList().catch((error) => {
127+
console.error(error);
128+
process.exit(1);
129+
});

0 commit comments

Comments
 (0)