Enhance download-tools script to fetch gdb (#381)

* Enhance download-tools script to fetch gdb

* Fixup: Enforce creation of download path

* Fixup: Remove duplicate doc files

* Replace outdated decompress with fast-extract module

* add back extract-zip for Windows
* some yarn.lock clean up from prev merge

---------

Signed-off-by: Jens Reinecke <jens.reinecke@arm.com>
Co-authored-by: Jens Reinecke <jens.reinecke@arm.com>

Author: JonatanAntoni

.vscode/launch.json

@@ -25,8 +25,9 @@
             "name": "Run download-tools.ts",
             "skipFiles": ["<node_internals>/**"],
             "program": "${workspaceFolder}/scripts/download-tools.ts",
-            "runtimeArgs": ["-r", "ts-node/register"],
-            "args": ["pyocd"]
+            "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/ts-node",
+            "runtimeArgs": [],
+            "args": ["-f", "gdb"]
         },
         {
             "type": "node",

package.json

@@ -118,6 +118,8 @@
     "eslint": "^9.30.1",
     "eslint-plugin-security": "^3.0.1",
     "extract-zip": "^2.0.1",
+    "fast-extract": "^1.7.2",
+    "file-type": "^21.0.0",
     "globby": "^14.1.0",
     "jest": "^30.0.4",
     "jest-html-reporter": "^4.3.0",
@@ -139,6 +141,7 @@
     "yargs": "^18.0.0"
   },
   "cmsis": {
-    "pyocd": "pyocd/pyOCD@0.37.0"
+    "pyocd": "pyocd/pyOCD@0.37.0",
+    "gdb": "14.3.1"
   }
 }

scripts/download-tools.ts

@@ -21,9 +21,11 @@ import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } fr
 import path from 'path';
 import { downloadFile } from './file-download';
 import yargs from 'yargs';
-import extractZip from 'extract-zip';
 import { execSync } from 'child_process';
 import { hideBin } from 'yargs/helpers';
+import console from 'console';
+import fastExtract from 'fast-extract';
+import extractZip from 'extract-zip';
 
 // OS/architecture pairs from vsce --publish
 type VsceTarget = 'win32-x64' | 'win32-arm64' | 'linux-x64' | 'linux-arm64' | 'darwin-x64' | 'darwin-arm64';
@@ -41,6 +43,7 @@ type ToolOptions = {
 
 const TOOLS = {
     'pyocd': downloadPyOCD,
+    'gdb': downloadGDB,
 };
 
 const PACKAGE_JSON = path.resolve(__dirname, '../package.json');
@@ -104,8 +107,7 @@ async function retrieveSha256(url?: string, token?: string) {
     }
     return undefined;
 }
-
-async function download(url: string, options?: ToolOptions & { cache_key?: string }) {
+async function downloadAsset(url: string, options?: ToolOptions & { cache_key?: string }) {
     const cachePath = (options?.cache && options?.cache_key) ? path.join(options.cache, options.cache_key) : undefined;
     if (cachePath && existsSync(cachePath)) {
         console.debug(`Found asset in cache ${cachePath} ...`);
@@ -121,7 +123,7 @@ async function download(url: string, options?: ToolOptions & { cache_key?: strin
 
     const extractPath = cachePath ?? downloadFilePath.replace('.zip', '');
     console.debug(`Extracting to ${extractPath} ...`);
-    await extractZip(downloadFilePath, { dir: extractPath }).catch(error => {
+    await fastExtract(downloadFilePath, extractPath).catch(error => {
         throw new Error(`Failed to extract ${url}`, { cause: error });
     });
 
@@ -180,7 +182,7 @@ async function downloadPyOCD(target: VsceTarget, dest: string, options?: ToolOpt
         cache: options?.cache, 
         cache_key: sha256sum ? `cmsis-pyocd-${version}-${sha256sum}` : undefined
     };
-    const { mode, path: extractPath } = await download(asset.url, dloptions);
+    const { mode, path: extractPath } = await downloadAsset(asset.url, dloptions);
 
     if (existsSync(destPath)) {
         console.debug(`Removing existing ${destPath} ...`);
@@ -202,6 +204,93 @@ async function downloadPyOCD(target: VsceTarget, dest: string, options?: ToolOpt
     }
 }
 
+async function downloadGDB(target: VsceTarget, dest: string, options?: ToolOptions) {
+
+    const jsonVersion = getVersionFromPackageJson(PACKAGE_JSON, 'gdb');
+
+    if (!jsonVersion) {
+        throw new Error('GDB version not found in package.json');
+    }
+
+    const destPath = path.join(dest, 'gdb');
+    const versionFilePath = path.join(destPath, 'version.txt');
+    const targetFilePath = path.join(destPath, 'target.txt');
+
+    if (!options?.force && existsSync(versionFilePath) && existsSync(targetFilePath)) {
+        const hasVersion = readFileSync(versionFilePath, { encoding: 'utf8' });
+        const hasTarget = readFileSync(targetFilePath, { encoding: 'utf8' });
+
+        if (jsonVersion === hasVersion && target === hasTarget) {
+            console.log(`GDB version ${jsonVersion} (${target}) already available.`);
+            return;
+        }
+    }
+
+    console.log(`Downloading GDB version ${jsonVersion} (${target}) ...`);
+
+    const { build, ext }  = {
+        'win32-x64': { build: 'mingw-w64-x86_64', ext: 'zip' },
+        'win32-arm64': { build: 'mingw-w64-x86_64', ext: 'zip' },
+        'linux-x64': { build: 'x86_64', ext: 'tar.xz'},
+        'linux-arm64': { build: 'aarch64', ext: 'tar.xz'},
+        'darwin-x64': { build: 'darwin-arm64', ext: 'tar.xz'},
+        'darwin-arm64': { build: 'darwin-arm64', ext: 'tar.xz'},
+    }[target];
+
+    const asset_name = `arm-gnu-toolchain-${build}-arm-none-eabi-gdb.${ext}`;
+    const url = `https://artifacts.tools.arm.com/arm-none-eabi-gdb/${jsonVersion}/${asset_name}`;
+
+    const tempfile = await import('tempfile');
+    
+    const { mode, downloadPath } = (options?.cache)
+        ? { mode: 'cache', downloadPath: options.cache }
+        : { mode: 'temp', downloadPath: tempfile.default() };
+    const downloadFilePath = path.join(downloadPath, asset_name);
+
+    if (options?.force === false && mode === 'cache' && existsSync(downloadFilePath)) {
+        console.debug(`Found GDB asset in cache ${downloadFilePath} ...`);
+    } else {
+        console.debug(`Downloading ${url} ...`);
+        mkdirSync(downloadPath, { recursive: true });
+        await downloadFile(url, downloadFilePath)
+    }
+
+    if (existsSync(destPath)) {
+        console.debug(`Removing existing ${destPath} ...`);
+        rmSync(destPath, { recursive: true, force: true });
+    }
+
+    console.debug(`Extracting ${downloadFilePath} to ${destPath} ...`);
+    mkdirSync(dest, { recursive: true });
+
+    const {fileTypeFromFile} = await import('file-type');
+
+    const fileType  = await fileTypeFromFile(downloadFilePath);
+
+    if (nodeOs.platform() === 'win32') {
+        await extractZip(downloadFilePath, { dir: destPath }).catch(error => {
+            throw new Error(`Failed to extract ${url}`, { cause: error });
+        });
+    } else {
+        await fastExtract(downloadFilePath, destPath, { strip: 1, type: fileType?.ext }).catch(error => {
+            throw new Error(`Failed to extract ${downloadFilePath}`, { cause: error });
+        });
+    }
+
+
+    // Remove doc directory as it contains duplicate files (names differ only in case)
+    // which are not supported by ZIP (VSIX) archives
+    rmSync(path.join(destPath, 'share', 'doc'), { recursive: true, force: true });
+
+    if (mode === 'temp') {
+        console.debug(`Removing temporary ${downloadPath} ...`);
+        rmSync(downloadPath, { recursive: true, force: true });
+    }
+
+    writeFileSync(versionFilePath, jsonVersion, { encoding: 'utf8' });
+    writeFileSync(targetFilePath, target, { encoding: 'utf8' });
+}
+
 async function main() {
     // Get Yarn cache directory
     const yarnCacheDir = execSync('yarn cache dir').toString().trim();
@@ -256,11 +345,11 @@ async function main() {
     }
 
     for (const tool of new Set(tools)) {
-        TOOLS[tool](target, dest, { cache: cacheFolder(cache), force });
+        await TOOLS[tool](target, dest, { cache: cacheFolder(cache), force });
     }
 }
 
 main().catch(error => {
     console.error(error);
-    process.exit(1);
+    process.exitCode = 1;
 });

scripts/file-download.ts

@@ -29,11 +29,11 @@ export const downloadFile: DownloadFile = (url, outputPath, token?) => new Promi
     };
 
     const req = https.request(url, requestOptions, res => {
-        if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location) {
-            return downloadFile(res.headers.location, outputPath, token).then(resolve, reject);
-        }
-
-        if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
+        if (res.statusCode !== undefined && (res.statusCode < 200 || res.statusCode >= 300)) {
+            res.destroy();
+            if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location) {
+                return downloadFile(res.headers.location, outputPath, token).then(resolve, reject);
+            }
             return reject(new Error(`Status Code: ${res.statusCode}`));
         }