build: abort download task on abnormal status code

Author: JLHwung

data/resources.js

@@ -870,12 +870,12 @@ const resources = [
 		emoji: 'https://unicode.org/Public/17.0.0/ucd/emoji/emoji-data.txt',
 		// Emoji_Keycap_Sequence, Emoji_Flag_Sequence, Emoji_Modifier_Sequence
 		'emoji-sequences':
-			'https://www.unicode.org/Public/17.0.0/emoji/emoji-sequences.txt',
+			'https://unicode.org/Public/17.0.0/emoji/emoji-sequences.txt',
 		// Emoji_ZWJ_Sequence
 		'emoji-zwj-sequences':
-			'https://www.unicode.org/Public/17.0.0/emoji/emoji-zwj-sequences.txt',
+			'https://unicode.org/Public/17.0.0/emoji/emoji-zwj-sequences.txt',
 		// Emoji_Test (not an official property)
-		'emoji-test': 'https://www.unicode.org/Public/17.0.0/emoji/emoji-test.txt',
+		'emoji-test': 'https://unicode.org/Public/17.0.0/emoji/emoji-test.txt',
 	},
 ];
 

package.json

@@ -46,7 +46,6 @@
 		"regenerate": "^1.4.2",
 		"unicode-loose-match": "^2.8.0",
 		"unicode-property-aliases": "^2.1.0",
-		"unicode-property-value-aliases": "^3.9.0",
-		"when": "^3.7.8"
+		"unicode-property-value-aliases": "^3.9.0"
 	}
 }

scripts/download.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const fs = require('fs');
-const guard = require('when/guard');
 const path = require('path');
 const { Readable } = require('stream');
 const { finished } = require('stream/promises');
@@ -17,12 +16,35 @@ const download = async function(url, version, type) {
 	);
 	console.log(' ', url, '→', path.basename(file));
 	//console.log(`curl ${url} > data/${path.basename(file)};`);
+	if (!res.ok) {
+		throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`);
+	}
 	return finished(
 		Readable.fromWeb(res.body).pipe(fs.createWriteStream(file))
 	);
 };
+
+async function parallelLimit(tasks, limit) {
+	const results = new Array(tasks.length);
+	const iterator = tasks.entries();
+
+	async function worker() {
+		for (const [index, task] of iterator) {
+			results[index] = await task();
+		}
+	}
+
+	const workers = new Array(limit).fill(null).map(() => worker());
+	await Promise.all(workers);
+	return results;
+}
+
+const tasks = [];
+const addDownloadTask = (url, version, type) => tasks.push(() => download(url, version, type));
+
 // Limit maximum parallelism to something reasonable
-const guardedDownload = guard(guard.n(PARALLEL_REQUEST_LIMIT), download);
+parallelLimit(tasks, PARALLEL_REQUEST_LIMIT);
+const guardedDownload = () => parallelLimit(tasks, PARALLEL_REQUEST_LIMIT);
 
 console.log('Downloading resources…');
 
@@ -53,10 +75,12 @@ const TYPES = [
 
 for (const resource of resources) {
 	const version = resource.version;
-	guardedDownload(resource.main, version, 'database');
+	addDownloadTask(resource.main, version, 'database');
 	for (const type of TYPES) {
 		if (resource[type]) {
-			guardedDownload(resource[type], version, type);
+			addDownloadTask(resource[type], version, type);
 		}
 	}
 }
+
+guardedDownload();