Download devfile stack extra files during component initialization

vrubezhny · vrubezhny · commit c4f5fe393085 · 2026-07-07T12:44:25.000+02:00
Add functionality to download additional stack files (Dockerfile, Kubernetes manifests, etc.) when creating components from devfile registry sources. This brings odoInit to feature parity with the real odo CLI. Key Changes: - Add DevfileRegistry.downloadStackExtraFiles() API for downloading stack files referenced in devfile components (dockerfile.uri, kubernetes.uri) - Download files directly from GitHub devfile registry repository via HTTPS (no ORAS/OCI dependencies needed) - Implement two-level caching: in-memory (ExecutionContext) + filesystem (~/.local/state/vs-openshift-tools/devfile-registry-cache/) - Integrate into odoInit to download files after devfile resolution - Add 5 comprehensive integration tests covering all scenarios Implementation Details: - Files are downloaded from: https://raw.githubusercontent.com/devfile/registry/main/stacks/{name}/{version}/{uri} - Cache location follows XDG Base Directory spec on Linux/macOS and is cross-platform compatible with Windows - Downloads are optional and don't fail init if files are unavailable - Language-agnostic solution works for all stacks (Go, Python, Java, etc.) Testing: - Integration tests verify: basic download, caching, multiple files, error handling, and devfiles without extra files - All cross-platform path operations use Node.js standard APIs (path.join, os.homedir, os.tmpdir) Fixes the issue where components created with odoInit were missing deployment-related files that real odo init creates. Signed-off-by: Victor Rubezhny <vrubezhny@redhat.com> Assisted-By: Claude Sonnet 4.5 <noreply@anthropic.com>
diff --git a/src/devfile-registry/devfileRegistryWrapper.ts b/src/devfile-registry/devfileRegistryWrapper.ts
@@ -10,6 +10,68 @@ import { OdoPreference } from '../odo/odoPreference';
 import { ExecutionContext } from '../util/utils';
 import { DevfileData, DevfileInfo } from './devfileInfo';
 
+// Extension ID is used for cache directory naming
+// Using a constant allows easy updates if extension name changes
+const EXTENSION_CACHE_DIR = 'vs-openshift-tools';
+
+/**
+ * Get the base directory for devfile registry cache.
+ * Can be overridden via VSCODE_OPENSHIFT_CACHE_ROOT environment variable.
+ * Default: ~/.local/state/vs-openshift-tools/devfile-registry-cache
+ *
+ * Returns null if cache directory cannot be created or accessed.
+ */
+async function getDevfileCacheBaseDir(): Promise<string | null> {
+    const os = require('os');
+    const path = require('path');
+
+    // Get cache root - can be overridden via environment variable
+    // This allows the same env var to be used by tools.ts and other cache consumers
+    const cacheRoot = process.env.VSCODE_OPENSHIFT_CACHE_ROOT ||
+        path.join(os.homedir(), '.local', 'state');
+
+    const cacheDir = path.join(
+        cacheRoot,
+        EXTENSION_CACHE_DIR,
+        'devfile-registry-cache'
+    );
+
+    // Validate cache directory
+    try {
+        const fs = await import('fs/promises');
+
+        // Try to create the directory if it doesn't exist
+        await fs.mkdir(cacheDir, { recursive: true });
+
+        // Check if it's actually a directory
+        const stats = await fs.stat(cacheDir);
+        if (!stats.isDirectory()) {
+            // Path exists but is a file, not a directory - try fallback
+            const fallback = path.join(os.tmpdir(), EXTENSION_CACHE_DIR, 'devfile-registry-cache');
+            await fs.mkdir(fallback, { recursive: true });
+            return fallback;
+        }
+
+        // Test write permissions by creating a temporary test file
+        const testFile = path.join(cacheDir, '.write-test');
+        await fs.writeFile(testFile, '', 'utf-8');
+        await fs.unlink(testFile);
+
+        return cacheDir;
+    } catch (error) {
+        // Try fallback to temp directory
+        try {
+            const fallback = path.join(os.tmpdir(), EXTENSION_CACHE_DIR, 'devfile-registry-cache');
+            const fs = await import('fs/promises');
+            await fs.mkdir(fallback, { recursive: true });
+            return fallback;
+        } catch {
+            // No cache available - return null
+            return null;
+        }
+    }
+}
+
 export const DEVFILE_VERSION_LATEST: string = 'latest';
 
 /**
@@ -19,6 +81,7 @@ export class DevfileRegistry {
     private static instance: DevfileRegistry;
 
     private executionContext: ExecutionContext = new ExecutionContext();
+    private cacheWarningShown: boolean = false;
 
     public static get Instance(): DevfileRegistry {
         if (!DevfileRegistry.instance) {
@@ -157,7 +220,8 @@ export class DevfileRegistry {
             const options = { signal, timeout };
             let result: string = '';
             request(url, options, (response) => {
-                if (response.statusCode < 500) {
+                // Only accept 2xx status codes as success
+                if (response.statusCode >= 200 && response.statusCode < 300) {
                     response.on('data', (d) => {
                         result = result.concat(d);
                     });
@@ -170,7 +234,7 @@ export class DevfileRegistry {
                         }
                     });
                 } else {
-                    reject(new Error(`Connect error: ${response.statusMessage}`));
+                    reject(new Error(`HTTP ${response.statusCode}: ${response.statusMessage}`));
                 }
             }).on('error', (e) => {
                 reject(new Error(`Connect error: ${e}`));
@@ -180,6 +244,215 @@ export class DevfileRegistry {
         });
     }
 
+    /**
+     * Download extra stack files (Dockerfile, Kubernetes manifests, etc.) referenced in the devfile.
+     * These files are stored in the devfile registry GitHub repository alongside the devfile.
+     *
+     * Files are cached in the filesystem to avoid repeated downloads.
+     *
+     * @param registryUrl Devfile Registry URL
+     * @param stackName Stack name (e.g., 'go', 'nodejs')
+     * @param version Stack version (e.g., '2.6.0')
+     * @param resolvedDevfile The resolved devfile (with parents merged)
+     * @param projectPath The project path where files should be written
+     * @returns Promise that resolves when all files are downloaded and written
+     * @throws Error if any file download or write fails
+     */
+    public async downloadStackExtraFiles(
+        registryUrl: string,
+        stackName: string,
+        version: string,
+        resolvedDevfile: any,
+        projectPath: string
+    ): Promise<void> {
+        const uriPaths = this.extractUriPaths(resolvedDevfile);
+
+        if (uriPaths.length === 0) {
+            return;
+        }
+
+        // Download and write each file, collecting any errors
+        const results = await Promise.allSettled(
+            uriPaths.map(async (uriPath) => {
+                const content = await this.getStackFile(registryUrl, stackName, version, uriPath);
+                if (!content) {
+                    throw new Error(`Failed to download ${uriPath} from ${stackName}:${version}`);
+                }
+                await this.writeStackFile(projectPath, uriPath, content);
+                return uriPath;
+            })
+        );
+
+        // Check for failures and report them
+        const failures = results.filter((r): r is PromiseRejectedResult => r.status === 'rejected');
+        if (failures.length > 0) {
+            const errorMessages = failures.map(f => f.reason?.message || String(f.reason)).join('; ');
+            throw new Error(`Failed to download ${failures.length} file(s): ${errorMessages}`);
+        }
+    }
+
+    /**
+     * Extract all URI paths from devfile components (dockerfile.uri, kubernetes.uri, etc.)
+     */
+    private extractUriPaths(devfile: any): string[] {
+        const uriPaths: string[] = [];
+
+        if (!devfile.components) {
+            return uriPaths;
+        }
+
+        for (const component of devfile.components) {
+            // Check for dockerfile uri in image components
+            if (component.image?.dockerfile?.uri) {
+                uriPaths.push(component.image.dockerfile.uri);
+            }
+
+            // Check for kubernetes/openshift uri
+            if (component.kubernetes?.uri) {
+                uriPaths.push(component.kubernetes.uri);
+            }
+
+            if (component.openshift?.uri) {
+                uriPaths.push(component.openshift.uri);
+            }
+        }
+
+        return uriPaths;
+    }
+
+    /**
+     * Get a stack file from cache or download from GitHub registry repository
+     * @throws Error if download fails
+     */
+    private async getStackFile(
+        registryUrl: string,
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string | null> {
+        const cacheKey = ExecutionContext.key(`stack-file:${registryUrl}:${stackName}:${version}:${uriPath}`);
+
+        // Check memory cache first
+        if (this.executionContext && this.executionContext.has(cacheKey)) {
+            return this.executionContext.get(cacheKey);
+        }
+
+        // Check filesystem cache
+        const cachedContent = await this.getFromFileCache(stackName, version, uriPath);
+        if (cachedContent) {
+            this.executionContext.set(cacheKey, cachedContent);
+            return cachedContent;
+        }
+
+        // Download from GitHub registry repository
+        try {
+            const content = await this.downloadFromRegistryRepo(stackName, version, uriPath);
+
+            // Cache in memory and filesystem
+            this.executionContext.set(cacheKey, content);
+            await this.saveToFileCache(stackName, version, uriPath, content);
+
+            return content;
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to download ${uriPath}: ${errorMessage}`);
+        }
+    }
+
+    /**
+     * Download a file from the devfile registry GitHub repository
+     */
+    private async downloadFromRegistryRepo(
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string> {
+        const url = `https://raw.githubusercontent.com/devfile/registry/main/stacks/${stackName}/${version}/${uriPath}`;
+        return DevfileRegistry._get(url);
+    }
+
+    /**
+     * Get cached file content from filesystem.
+     * Cache location can be overridden via VSCODE_OPENSHIFT_CACHE_ROOT env var.
+     * Default: ~/.local/state/vs-openshift-tools/devfile-registry-cache
+     */
+    private async getFromFileCache(
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string | null> {
+        try {
+            const fs = await import('fs/promises');
+            const path = await import('path');
+
+            const cacheBaseDir = await getDevfileCacheBaseDir();
+            if (!cacheBaseDir) {
+                // Warn user once that caching is unavailable
+                if (!this.cacheWarningShown) {
+                    this.cacheWarningShown = true;
+                    // eslint-disable-next-line no-console
+                    console.warn('[vscode-openshift-tools] Devfile registry cache is unavailable. Files will be downloaded without caching. Set VSCODE_OPENSHIFT_CACHE_ROOT to a writable directory to enable caching.');
+                }
+                return null;
+            }
+
+            const cacheDir = path.join(cacheBaseDir, stackName, version);
+            const filePath = path.join(cacheDir, uriPath);
+
+            const content = await fs.readFile(filePath, 'utf-8');
+            return content;
+        } catch {
+            return null;
+        }
+    }
+
+    /**
+     * Save file content to filesystem cache.
+     * Cache location can be overridden via VSCODE_OPENSHIFT_CACHE_ROOT env var.
+     * Default: ~/.local/state/vs-openshift-tools/devfile-registry-cache
+     */
+    private async saveToFileCache(
+        stackName: string,
+        version: string,
+        uriPath: string,
+        content: string
+    ): Promise<void> {
+        try {
+            const fs = await import('fs/promises');
+            const path = await import('path');
+
+            const cacheBaseDir = await getDevfileCacheBaseDir();
+            if (!cacheBaseDir) {
+                // No cache available - silently skip (warning already shown in getFromFileCache)
+                return;
+            }
+
+            const cacheDir = path.join(cacheBaseDir, stackName, version);
+            const filePath = path.join(cacheDir, uriPath);
+
+            await fs.mkdir(path.dirname(filePath), { recursive: true });
+            await fs.writeFile(filePath, content, 'utf-8');
+        } catch {
+            // Ignore cache write errors - cache is optional
+        }
+    }
+
+    /**
+     * Write a stack file to the project directory
+     */
+    private async writeStackFile(
+        projectPath: string,
+        uriPath: string,
+        content: string
+    ): Promise<void> {
+        const fs = await import('fs/promises');
+        const path = await import('path');
+
+        const fullPath = path.join(projectPath, uriPath);
+        await fs.mkdir(path.dirname(fullPath), { recursive: true });
+        await fs.writeFile(fullPath, content, 'utf-8');
+    }
+
     /**
      * Clears the Execution context as well as all cached data
      */
diff --git a/src/devfile/init.ts b/src/devfile/init.ts
@@ -54,6 +54,11 @@ export async function odoInit(options: OdoInitOptions) {
     const resolvedDevfile = await resolveDevfile(workingDevfile);
 
     logInfo(ctx, 'Devfile resolved');
+    logInfo(ctx, 'Downloading stack extra files...');
+
+    await downloadStackExtraFiles(resolvedDevfile, acquired, ctx);
+
+    logInfo(ctx, 'Stack extra files downloaded');
     logInfo(ctx, 'Validating devfile...');
 
     // analysis-only copy
@@ -684,7 +689,7 @@ function interpolateDevfileVariables(devfile: Devfile): Devfile {
 }
 
 async function materializeZipStarter(url: string, projectPath: string) {
-    const tmpZip = path.join(projectPath, '.odo-starter.zip');
+    const tmpZip = path.join(projectPath, '.devfile-starter.zip');
 
     try {
         await DownloadUtil.downloadFile(url, tmpZip);
@@ -694,3 +699,31 @@ async function materializeZipStarter(url: string, projectPath: string) {
         await fs.unlink(tmpZip).catch(() => undefined);
     }
 }
+
+async function downloadStackExtraFiles(devfile: Devfile, acquired: AcquiredDevfile, ctx: InitContext) {
+    // Only download extra files if the devfile came from a registry
+    if (!acquired.provenance) {
+        return;
+    }
+
+    const { url: registryUrl, stack: stackName, version } = acquired.provenance;
+
+    try {
+        logInfo(ctx, `Downloading stack files for ${stackName}:${version}`);
+
+        await DevfileRegistry.Instance.downloadStackExtraFiles(
+            registryUrl,
+            stackName,
+            version,
+            devfile,
+            ctx.projectPath
+        );
+
+        logInfo(ctx, 'Stack files downloaded successfully');
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        logError(ctx, `Failed to download stack extra files: ${errorMessage}`);
+        logInfo(ctx, 'Component initialization will continue without extra files');
+        // Don't fail the init - extra files are optional
+    }
+}
diff --git a/test/integration/devfileRegistryWrapper.test.ts b/test/integration/devfileRegistryWrapper.test.ts
