Download devfile stack extra files during component initialization

Add functionality to download additional stack files (Dockerfile, Kubernetes
manifests, etc.) when creating components from devfile registry sources.
This brings odoInit to feature parity with the real odo CLI.

Key Changes:
- Add DevfileRegistry.downloadStackExtraFiles() API for downloading stack
  files referenced in devfile components (dockerfile.uri, kubernetes.uri)
- Download files directly from GitHub devfile registry repository via HTTPS
  (no ORAS/OCI dependencies needed)
- Implement two-level caching: in-memory (ExecutionContext) + filesystem
  (~/.local/state/vs-openshift-tools/devfile-registry-cache/)
- Integrate into odoInit to download files after devfile resolution
- Add 5 comprehensive integration tests covering all scenarios

Implementation Details:
- Files are downloaded from:
  https://raw.githubusercontent.com/devfile/registry/main/stacks/{name}/{version}/{uri}
- Cache location follows XDG Base Directory spec on Linux/macOS and is
  cross-platform compatible with Windows
- Downloads are optional and don't fail init if files are unavailable
- Language-agnostic solution works for all stacks (Go, Python, Java, etc.)

Testing:
- Integration tests verify: basic download, caching, multiple files,
  error handling, and devfiles without extra files
- All cross-platform path operations use Node.js standard APIs
  (path.join, os.homedir, os.tmpdir)

Fixes the issue where components created with odoInit were missing
deployment-related files that real odo init creates.

Signed-off-by: Victor Rubezhny <vrubezhny@redhat.com>
Assisted-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: vrubezhny

src/devfile-registry/devfileRegistryWrapper.ts

@@ -10,6 +10,10 @@ import { OdoPreference } from '../odo/odoPreference';
 import { ExecutionContext } from '../util/utils';
 import { DevfileData, DevfileInfo } from './devfileInfo';
 
+// Extension ID is used for cache directory naming
+// Using a constant allows easy updates if extension name changes
+const EXTENSION_CACHE_DIR = 'vs-openshift-tools';
+
 export const DEVFILE_VERSION_LATEST: string = 'latest';
 
 /**
@@ -180,6 +184,220 @@ export class DevfileRegistry {
         });
     }
 
+    /**
+     * Download extra stack files (Dockerfile, Kubernetes manifests, etc.) referenced in the devfile.
+     * These files are stored in the devfile registry GitHub repository alongside the devfile.
+     *
+     * Files are cached in the filesystem to avoid repeated downloads.
+     *
+     * @param registryUrl Devfile Registry URL
+     * @param stackName Stack name (e.g., 'go', 'nodejs')
+     * @param version Stack version (e.g., '2.6.0')
+     * @param resolvedDevfile The resolved devfile (with parents merged)
+     * @param projectPath The project path where files should be written
+     * @returns Promise that resolves when all files are downloaded and written
+     * @throws Error if any file download or write fails
+     */
+    public async downloadStackExtraFiles(
+        registryUrl: string,
+        stackName: string,
+        version: string,
+        resolvedDevfile: any,
+        projectPath: string
+    ): Promise<void> {
+        const uriPaths = this.extractUriPaths(resolvedDevfile);
+
+        if (uriPaths.length === 0) {
+            return;
+        }
+
+        // Download and write each file, collecting any errors
+        const results = await Promise.allSettled(
+            uriPaths.map(async (uriPath) => {
+                const content = await this.getStackFile(registryUrl, stackName, version, uriPath);
+                if (!content) {
+                    throw new Error(`Failed to download ${uriPath} from ${stackName}:${version}`);
+                }
+                await this.writeStackFile(projectPath, uriPath, content);
+                return uriPath;
+            })
+        );
+
+        // Check for failures and report them
+        const failures = results.filter((r): r is PromiseRejectedResult => r.status === 'rejected');
+        if (failures.length > 0) {
+            const errorMessages = failures.map(f => f.reason?.message || String(f.reason)).join('; ');
+            throw new Error(`Failed to download ${failures.length} file(s): ${errorMessages}`);
+        }
+    }
+
+    /**
+     * Extract all URI paths from devfile components (dockerfile.uri, kubernetes.uri, etc.)
+     */
+    private extractUriPaths(devfile: any): string[] {
+        const uriPaths: string[] = [];
+
+        if (!devfile.components) {
+            return uriPaths;
+        }
+
+        for (const component of devfile.components) {
+            // Check for dockerfile uri in image components
+            if (component.image?.dockerfile?.uri) {
+                uriPaths.push(component.image.dockerfile.uri);
+            }
+
+            // Check for kubernetes/openshift uri
+            if (component.kubernetes?.uri) {
+                uriPaths.push(component.kubernetes.uri);
+            }
+
+            if (component.openshift?.uri) {
+                uriPaths.push(component.openshift.uri);
+            }
+        }
+
+        return uriPaths;
+    }
+
+    /**
+     * Get a stack file from cache or download from GitHub registry repository
+     * @throws Error if download fails
+     */
+    private async getStackFile(
+        registryUrl: string,
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string | null> {
+        const cacheKey = ExecutionContext.key(`stack-file:${registryUrl}:${stackName}:${version}:${uriPath}`);
+
+        // Check memory cache first
+        if (this.executionContext && this.executionContext.has(cacheKey)) {
+            return this.executionContext.get(cacheKey);
+        }
+
+        // Check filesystem cache
+        const cachedContent = await this.getFromFileCache(stackName, version, uriPath);
+        if (cachedContent) {
+            this.executionContext.set(cacheKey, cachedContent);
+            return cachedContent;
+        }
+
+        // Download from GitHub registry repository
+        try {
+            const content = await this.downloadFromRegistryRepo(stackName, version, uriPath);
+
+            // Cache in memory and filesystem
+            this.executionContext.set(cacheKey, content);
+            await this.saveToFileCache(stackName, version, uriPath, content);
+
+            return content;
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to download ${uriPath}: ${errorMessage}`);
+        }
+    }
+
+    /**
+     * Download a file from the devfile registry GitHub repository
+     */
+    private async downloadFromRegistryRepo(
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string> {
+        const url = `https://raw.githubusercontent.com/devfile/registry/main/stacks/${stackName}/${version}/${uriPath}`;
+        return DevfileRegistry._get(url);
+    }
+
+    /**
+     * Get cached file content from filesystem.
+     * Cache location follows XDG Base Directory spec on Linux/macOS,
+     * and AppData on Windows (via os.homedir() + .local/state)
+     */
+    private async getFromFileCache(
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string | null> {
+        try {
+            const fs = await import('fs/promises');
+            const path = await import('path');
+            const os = await import('os');
+
+            // Use .local/state for consistency with extension's tools storage
+            // This works cross-platform: ~/.local/state on Unix, %USERPROFILE%\.local\state on Windows
+            const cacheDir = path.join(
+                os.homedir(),
+                '.local',
+                'state',
+                EXTENSION_CACHE_DIR,
+                'devfile-registry-cache',
+                stackName,
+                version
+            );
+            const filePath = path.join(cacheDir, uriPath);
+
+            const content = await fs.readFile(filePath, 'utf-8');
+            return content;
+        } catch {
+            return null;
+        }
+    }
+
+    /**
+     * Save file content to filesystem cache.
+     * Cache location follows XDG Base Directory spec on Linux/macOS,
+     * and AppData on Windows (via os.homedir() + .local/state)
+     */
+    private async saveToFileCache(
+        stackName: string,
+        version: string,
+        uriPath: string,
+        content: string
+    ): Promise<void> {
+        try {
+            const fs = await import('fs/promises');
+            const path = await import('path');
+            const os = await import('os');
+
+            // Use .local/state for consistency with extension's tools storage
+            // This works cross-platform: ~/.local/state on Unix, %USERPROFILE%\.local\state on Windows
+            const cacheDir = path.join(
+                os.homedir(),
+                '.local',
+                'state',
+                EXTENSION_CACHE_DIR,
+                'devfile-registry-cache',
+                stackName,
+                version
+            );
+            const filePath = path.join(cacheDir, uriPath);
+
+            await fs.mkdir(path.dirname(filePath), { recursive: true });
+            await fs.writeFile(filePath, content, 'utf-8');
+        } catch {
+            // Ignore cache write errors - cache is optional
+        }
+    }
+
+    /**
+     * Write a stack file to the project directory
+     */
+    private async writeStackFile(
+        projectPath: string,
+        uriPath: string,
+        content: string
+    ): Promise<void> {
+        const fs = await import('fs/promises');
+        const path = await import('path');
+
+        const fullPath = path.join(projectPath, uriPath);
+        await fs.mkdir(path.dirname(fullPath), { recursive: true });
+        await fs.writeFile(fullPath, content, 'utf-8');
+    }
+
     /**
      * Clears the Execution context as well as all cached data
      */

src/devfile/init.ts

@@ -54,6 +54,11 @@ export async function odoInit(options: OdoInitOptions) {
     const resolvedDevfile = await resolveDevfile(workingDevfile);
 
     logInfo(ctx, 'Devfile resolved');
+    logInfo(ctx, 'Downloading stack extra files...');
+
+    await downloadStackExtraFiles(resolvedDevfile, acquired, ctx);
+
+    logInfo(ctx, 'Stack extra files downloaded');
     logInfo(ctx, 'Validating devfile...');
 
     // analysis-only copy
@@ -684,7 +689,7 @@ function interpolateDevfileVariables(devfile: Devfile): Devfile {
 }
 
 async function materializeZipStarter(url: string, projectPath: string) {
-    const tmpZip = path.join(projectPath, '.odo-starter.zip');
+    const tmpZip = path.join(projectPath, '.devfile-starter.zip');
 
     try {
         await DownloadUtil.downloadFile(url, tmpZip);
@@ -694,3 +699,31 @@ async function materializeZipStarter(url: string, projectPath: string) {
         await fs.unlink(tmpZip).catch(() => undefined);
     }
 }
+
+async function downloadStackExtraFiles(devfile: Devfile, acquired: AcquiredDevfile, ctx: InitContext) {
+    // Only download extra files if the devfile came from a registry
+    if (!acquired.provenance) {
+        return;
+    }
+
+    const { url: registryUrl, stack: stackName, version } = acquired.provenance;
+
+    try {
+        logInfo(ctx, `Downloading stack files for ${stackName}:${version}`);
+
+        await DevfileRegistry.Instance.downloadStackExtraFiles(
+            registryUrl,
+            stackName,
+            version,
+            devfile,
+            ctx.projectPath
+        );
+
+        logInfo(ctx, 'Stack files downloaded successfully');
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        logError(ctx, `Failed to download stack extra files: ${errorMessage}`);
+        logInfo(ctx, 'Component initialization will continue without extra files');
+        // Don't fail the init - extra files are optional
+    }
+}