Download devfile stack extra files during component initialization

src/devfile-registry/devfileRegistryWrapper.ts

@@ -10,6 +10,10 @@ import { OdoPreference } from '../odo/odoPreference';
 import { ExecutionContext } from '../util/utils';
 import { DevfileData, DevfileInfo } from './devfileInfo';
 
+// Extension ID is used for cache directory naming
+// Using a constant allows easy updates if extension name changes
+const EXTENSION_CACHE_DIR = 'vs-openshift-tools';
+
 export const DEVFILE_VERSION_LATEST: string = 'latest';
 
 /**
@@ -180,6 +184,220 @@ export class DevfileRegistry {
         });
     }
 
+    /**
+     * Download extra stack files (Dockerfile, Kubernetes manifests, etc.) referenced in the devfile.
+     * These files are stored in the devfile registry GitHub repository alongside the devfile.
+     *
+     * Files are cached in the filesystem to avoid repeated downloads.
+     *
+     * @param registryUrl Devfile Registry URL
+     * @param stackName Stack name (e.g., 'go', 'nodejs')
+     * @param version Stack version (e.g., '2.6.0')
+     * @param resolvedDevfile The resolved devfile (with parents merged)
+     * @param projectPath The project path where files should be written
+     * @returns Promise that resolves when all files are downloaded and written
+     * @throws Error if any file download or write fails
+     */
+    public async downloadStackExtraFiles(
+        registryUrl: string,
+        stackName: string,
+        version: string,
+        resolvedDevfile: any,
+        projectPath: string
+    ): Promise<void> {
+        const uriPaths = this.extractUriPaths(resolvedDevfile);
+
+        if (uriPaths.length === 0) {
+            return;
+        }
+
+        // Download and write each file, collecting any errors
+        const results = await Promise.allSettled(
+            uriPaths.map(async (uriPath) => {
+                const content = await this.getStackFile(registryUrl, stackName, version, uriPath);
+                if (!content) {
+                    throw new Error(`Failed to download ${uriPath} from ${stackName}:${version}`);
+                }
+                await this.writeStackFile(projectPath, uriPath, content);
+                return uriPath;
+            })
+        );
+
+        // Check for failures and report them
+        const failures = results.filter((r): r is PromiseRejectedResult => r.status === 'rejected');
+        if (failures.length > 0) {
+            const errorMessages = failures.map(f => f.reason?.message || String(f.reason)).join('; ');
+            throw new Error(`Failed to download ${failures.length} file(s): ${errorMessages}`);
+        }
+    }
+
+    /**
+     * Extract all URI paths from devfile components (dockerfile.uri, kubernetes.uri, etc.)
+     */
+    private extractUriPaths(devfile: any): string[] {
+        const uriPaths: string[] = [];
+
+        if (!devfile.components) {
+            return uriPaths;
+        }
+
+        for (const component of devfile.components) {
+            // Check for dockerfile uri in image components
+            if (component.image?.dockerfile?.uri) {
+                uriPaths.push(component.image.dockerfile.uri);
+            }
+
+            // Check for kubernetes/openshift uri
+            if (component.kubernetes?.uri) {
+                uriPaths.push(component.kubernetes.uri);
+            }
+
+            if (component.openshift?.uri) {
+                uriPaths.push(component.openshift.uri);
+            }
+        }
+
+        return uriPaths;
+    }
+
+    /**
+     * Get a stack file from cache or download from GitHub registry repository
+     * @throws Error if download fails
+     */
+    private async getStackFile(
+        registryUrl: string,
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string | null> {
+        const cacheKey = ExecutionContext.key(`stack-file:${registryUrl}:${stackName}:${version}:${uriPath}`);
+
+        // Check memory cache first
+        if (this.executionContext && this.executionContext.has(cacheKey)) {
+            return this.executionContext.get(cacheKey);
+        }
+
+        // Check filesystem cache
+        const cachedContent = await this.getFromFileCache(stackName, version, uriPath);
+        if (cachedContent) {
+            this.executionContext.set(cacheKey, cachedContent);
+            return cachedContent;
+        }
+
+        // Download from GitHub registry repository
+        try {
+            const content = await this.downloadFromRegistryRepo(stackName, version, uriPath);
+
+            // Cache in memory and filesystem
+            this.executionContext.set(cacheKey, content);
+            await this.saveToFileCache(stackName, version, uriPath, content);
+
+            return content;
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to download ${uriPath}: ${errorMessage}`);
+        }
+    }
+
+    /**
+     * Download a file from the devfile registry GitHub repository
+     */
+    private async downloadFromRegistryRepo(
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string> {
+        const url = `https://raw.githubusercontent.com/devfile/registry/main/stacks/${stackName}/${version}/${uriPath}`;
+        return DevfileRegistry._get(url);
+    }
+
+    /**
+     * Get cached file content from filesystem.
+     * Cache location follows XDG Base Directory spec on Linux/macOS,
+     * and AppData on Windows (via os.homedir() + .local/state)
+     */
+    private async getFromFileCache(
+        stackName: string,
+        version: string,
+        uriPath: string
+    ): Promise<string | null> {
+        try {
+            const fs = await import('fs/promises');
+            const path = await import('path');
+            const os = await import('os');
+
+            // Use .local/state for consistency with extension's tools storage
+            // This works cross-platform: ~/.local/state on Unix, %USERPROFILE%\.local\state on Windows
+            const cacheDir = path.join(
+                os.homedir(),
+                '.local',
+                'state',
+                EXTENSION_CACHE_DIR,
+                'devfile-registry-cache',
+                stackName,
+                version
+            );
+            const filePath = path.join(cacheDir, uriPath);
+
+            const content = await fs.readFile(filePath, 'utf-8');
+            return content;
+        } catch {
+            return null;
+        }
+    }
+
+    /**
+     * Save file content to filesystem cache.
+     * Cache location follows XDG Base Directory spec on Linux/macOS,
+     * and AppData on Windows (via os.homedir() + .local/state)
+     */
+    private async saveToFileCache(
+        stackName: string,
+        version: string,
+        uriPath: string,
+        content: string
+    ): Promise<void> {
+        try {
+            const fs = await import('fs/promises');
+            const path = await import('path');
+            const os = await import('os');
+
+            // Use .local/state for consistency with extension's tools storage
+            // This works cross-platform: ~/.local/state on Unix, %USERPROFILE%\.local\state on Windows
+            const cacheDir = path.join(
+                os.homedir(),
+                '.local',
+                'state',
+                EXTENSION_CACHE_DIR,
+                'devfile-registry-cache',
+                stackName,
+                version
+            );
+            const filePath = path.join(cacheDir, uriPath);
+
+            await fs.mkdir(path.dirname(filePath), { recursive: true });
+            await fs.writeFile(filePath, content, 'utf-8');
+        } catch {
+            // Ignore cache write errors - cache is optional
+        }
+    }
+
+    /**
+     * Write a stack file to the project directory
+     */
+    private async writeStackFile(
+        projectPath: string,
+        uriPath: string,
+        content: string
+    ): Promise<void> {
+        const fs = await import('fs/promises');
+        const path = await import('path');
+
+        const fullPath = path.join(projectPath, uriPath);
+        await fs.mkdir(path.dirname(fullPath), { recursive: true });
+        await fs.writeFile(fullPath, content, 'utf-8');
+    }
+
     /**
      * Clears the Execution context as well as all cached data
      */

src/devfile/init.ts

@@ -54,6 +54,11 @@ export async function odoInit(options: OdoInitOptions) {
     const resolvedDevfile = await resolveDevfile(workingDevfile);
 
     logInfo(ctx, 'Devfile resolved');
+    logInfo(ctx, 'Downloading stack extra files...');
+
+    await downloadStackExtraFiles(resolvedDevfile, acquired, ctx);
+
+    logInfo(ctx, 'Stack extra files downloaded');
     logInfo(ctx, 'Validating devfile...');
 
     // analysis-only copy
@@ -684,7 +689,7 @@ function interpolateDevfileVariables(devfile: Devfile): Devfile {
 }
 
 async function materializeZipStarter(url: string, projectPath: string) {
-    const tmpZip = path.join(projectPath, '.odo-starter.zip');
+    const tmpZip = path.join(projectPath, '.devfile-starter.zip');
 
     try {
         await DownloadUtil.downloadFile(url, tmpZip);
@@ -694,3 +699,31 @@ async function materializeZipStarter(url: string, projectPath: string) {
         await fs.unlink(tmpZip).catch(() => undefined);
     }
 }
+
+async function downloadStackExtraFiles(devfile: Devfile, acquired: AcquiredDevfile, ctx: InitContext) {
+    // Only download extra files if the devfile came from a registry
+    if (!acquired.provenance) {
+        return;
+    }
+
+    const { url: registryUrl, stack: stackName, version } = acquired.provenance;
+
+    try {
+        logInfo(ctx, `Downloading stack files for ${stackName}:${version}`);
+
+        await DevfileRegistry.Instance.downloadStackExtraFiles(
+            registryUrl,
+            stackName,
+            version,
+            devfile,
+            ctx.projectPath
+        );
+
+        logInfo(ctx, 'Stack files downloaded successfully');
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        logError(ctx, `Failed to download stack extra files: ${errorMessage}`);
+        logInfo(ctx, 'Component initialization will continue without extra files');
+        // Don't fail the init - extra files are optional
+    }
+}