From eed7b879ead28dae25cd2b743b4e2fadebc59d5a Mon Sep 17 00:00:00 2001 From: Victor Rubezhny Date: Tue, 7 Jul 2026 02:26:59 +0200 Subject: [PATCH] Download devfile stack extra files during component initialization Add functionality to download additional stack files (Dockerfile, Kubernetes manifests, etc.) when creating components from devfile registry sources. This brings odoInit to feature parity with the real odo CLI. Key Changes: - Add DevfileRegistry.downloadStackExtraFiles() API for downloading stack files referenced in devfile components (dockerfile.uri, kubernetes.uri) - Download files directly from GitHub devfile registry repository via HTTPS (no ORAS/OCI dependencies needed) - Implement two-level caching: in-memory (ExecutionContext) + filesystem (~/.local/state/vs-openshift-tools/devfile-registry-cache/) - Integrate into odoInit to download files after devfile resolution - Add 5 comprehensive integration tests covering all scenarios Implementation Details: - Files are downloaded from: https://raw.githubusercontent.com/devfile/registry/main/stacks/{name}/{version}/{uri} - Cache location follows XDG Base Directory spec on Linux/macOS and is cross-platform compatible with Windows - Downloads are optional and don't fail init if files are unavailable - Language-agnostic solution works for all stacks (Go, Python, Java, etc.) Testing: - Integration tests verify: basic download, caching, multiple files, error handling, and devfiles without extra files - All cross-platform path operations use Node.js standard APIs (path.join, os.homedir, os.tmpdir) Fixes the issue where components created with odoInit were missing deployment-related files that real odo init creates. Signed-off-by: Victor Rubezhny Assisted-By: Claude Sonnet 4.5 --- .../devfileRegistryWrapper.ts | 218 ++++++++++++++ src/devfile/init.ts | 35 ++- .../devfileRegistryWrapper.test.ts | 280 ++++++++++++++++++ 3 files changed, 532 insertions(+), 1 deletion(-) diff --git a/src/devfile-registry/devfileRegistryWrapper.ts b/src/devfile-registry/devfileRegistryWrapper.ts index d7f2a5d6e..cc34e9cb8 100644 --- a/src/devfile-registry/devfileRegistryWrapper.ts +++ b/src/devfile-registry/devfileRegistryWrapper.ts @@ -10,6 +10,10 @@ import { OdoPreference } from '../odo/odoPreference'; import { ExecutionContext } from '../util/utils'; import { DevfileData, DevfileInfo } from './devfileInfo'; +// Extension ID is used for cache directory naming +// Using a constant allows easy updates if extension name changes +const EXTENSION_CACHE_DIR = 'vs-openshift-tools'; + export const DEVFILE_VERSION_LATEST: string = 'latest'; /** @@ -180,6 +184,220 @@ export class DevfileRegistry { }); } + /** + * Download extra stack files (Dockerfile, Kubernetes manifests, etc.) referenced in the devfile. + * These files are stored in the devfile registry GitHub repository alongside the devfile. + * + * Files are cached in the filesystem to avoid repeated downloads. + * + * @param registryUrl Devfile Registry URL + * @param stackName Stack name (e.g., 'go', 'nodejs') + * @param version Stack version (e.g., '2.6.0') + * @param resolvedDevfile The resolved devfile (with parents merged) + * @param projectPath The project path where files should be written + * @returns Promise that resolves when all files are downloaded and written + * @throws Error if any file download or write fails + */ + public async downloadStackExtraFiles( + registryUrl: string, + stackName: string, + version: string, + resolvedDevfile: any, + projectPath: string + ): Promise { + const uriPaths = this.extractUriPaths(resolvedDevfile); + + if (uriPaths.length === 0) { + return; + } + + // Download and write each file, collecting any errors + const results = await Promise.allSettled( + uriPaths.map(async (uriPath) => { + const content = await this.getStackFile(registryUrl, stackName, version, uriPath); + if (!content) { + throw new Error(`Failed to download ${uriPath} from ${stackName}:${version}`); + } + await this.writeStackFile(projectPath, uriPath, content); + return uriPath; + }) + ); + + // Check for failures and report them + const failures = results.filter((r): r is PromiseRejectedResult => r.status === 'rejected'); + if (failures.length > 0) { + const errorMessages = failures.map(f => f.reason?.message || String(f.reason)).join('; '); + throw new Error(`Failed to download ${failures.length} file(s): ${errorMessages}`); + } + } + + /** + * Extract all URI paths from devfile components (dockerfile.uri, kubernetes.uri, etc.) + */ + private extractUriPaths(devfile: any): string[] { + const uriPaths: string[] = []; + + if (!devfile.components) { + return uriPaths; + } + + for (const component of devfile.components) { + // Check for dockerfile uri in image components + if (component.image?.dockerfile?.uri) { + uriPaths.push(component.image.dockerfile.uri); + } + + // Check for kubernetes/openshift uri + if (component.kubernetes?.uri) { + uriPaths.push(component.kubernetes.uri); + } + + if (component.openshift?.uri) { + uriPaths.push(component.openshift.uri); + } + } + + return uriPaths; + } + + /** + * Get a stack file from cache or download from GitHub registry repository + * @throws Error if download fails + */ + private async getStackFile( + registryUrl: string, + stackName: string, + version: string, + uriPath: string + ): Promise { + const cacheKey = ExecutionContext.key(`stack-file:${registryUrl}:${stackName}:${version}:${uriPath}`); + + // Check memory cache first + if (this.executionContext && this.executionContext.has(cacheKey)) { + return this.executionContext.get(cacheKey); + } + + // Check filesystem cache + const cachedContent = await this.getFromFileCache(stackName, version, uriPath); + if (cachedContent) { + this.executionContext.set(cacheKey, cachedContent); + return cachedContent; + } + + // Download from GitHub registry repository + try { + const content = await this.downloadFromRegistryRepo(stackName, version, uriPath); + + // Cache in memory and filesystem + this.executionContext.set(cacheKey, content); + await this.saveToFileCache(stackName, version, uriPath, content); + + return content; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to download ${uriPath}: ${errorMessage}`); + } + } + + /** + * Download a file from the devfile registry GitHub repository + */ + private async downloadFromRegistryRepo( + stackName: string, + version: string, + uriPath: string + ): Promise { + const url = `https://raw.githubusercontent.com/devfile/registry/main/stacks/${stackName}/${version}/${uriPath}`; + return DevfileRegistry._get(url); + } + + /** + * Get cached file content from filesystem. + * Cache location follows XDG Base Directory spec on Linux/macOS, + * and AppData on Windows (via os.homedir() + .local/state) + */ + private async getFromFileCache( + stackName: string, + version: string, + uriPath: string + ): Promise { + try { + const fs = await import('fs/promises'); + const path = await import('path'); + const os = await import('os'); + + // Use .local/state for consistency with extension's tools storage + // This works cross-platform: ~/.local/state on Unix, %USERPROFILE%\.local\state on Windows + const cacheDir = path.join( + os.homedir(), + '.local', + 'state', + EXTENSION_CACHE_DIR, + 'devfile-registry-cache', + stackName, + version + ); + const filePath = path.join(cacheDir, uriPath); + + const content = await fs.readFile(filePath, 'utf-8'); + return content; + } catch { + return null; + } + } + + /** + * Save file content to filesystem cache. + * Cache location follows XDG Base Directory spec on Linux/macOS, + * and AppData on Windows (via os.homedir() + .local/state) + */ + private async saveToFileCache( + stackName: string, + version: string, + uriPath: string, + content: string + ): Promise { + try { + const fs = await import('fs/promises'); + const path = await import('path'); + const os = await import('os'); + + // Use .local/state for consistency with extension's tools storage + // This works cross-platform: ~/.local/state on Unix, %USERPROFILE%\.local\state on Windows + const cacheDir = path.join( + os.homedir(), + '.local', + 'state', + EXTENSION_CACHE_DIR, + 'devfile-registry-cache', + stackName, + version + ); + const filePath = path.join(cacheDir, uriPath); + + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, 'utf-8'); + } catch { + // Ignore cache write errors - cache is optional + } + } + + /** + * Write a stack file to the project directory + */ + private async writeStackFile( + projectPath: string, + uriPath: string, + content: string + ): Promise { + const fs = await import('fs/promises'); + const path = await import('path'); + + const fullPath = path.join(projectPath, uriPath); + await fs.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.writeFile(fullPath, content, 'utf-8'); + } + /** * Clears the Execution context as well as all cached data */ diff --git a/src/devfile/init.ts b/src/devfile/init.ts index c0a060866..183098756 100644 --- a/src/devfile/init.ts +++ b/src/devfile/init.ts @@ -54,6 +54,11 @@ export async function odoInit(options: OdoInitOptions) { const resolvedDevfile = await resolveDevfile(workingDevfile); logInfo(ctx, 'Devfile resolved'); + logInfo(ctx, 'Downloading stack extra files...'); + + await downloadStackExtraFiles(resolvedDevfile, acquired, ctx); + + logInfo(ctx, 'Stack extra files downloaded'); logInfo(ctx, 'Validating devfile...'); // analysis-only copy @@ -684,7 +689,7 @@ function interpolateDevfileVariables(devfile: Devfile): Devfile { } async function materializeZipStarter(url: string, projectPath: string) { - const tmpZip = path.join(projectPath, '.odo-starter.zip'); + const tmpZip = path.join(projectPath, '.devfile-starter.zip'); try { await DownloadUtil.downloadFile(url, tmpZip); @@ -694,3 +699,31 @@ async function materializeZipStarter(url: string, projectPath: string) { await fs.unlink(tmpZip).catch(() => undefined); } } + +async function downloadStackExtraFiles(devfile: Devfile, acquired: AcquiredDevfile, ctx: InitContext) { + // Only download extra files if the devfile came from a registry + if (!acquired.provenance) { + return; + } + + const { url: registryUrl, stack: stackName, version } = acquired.provenance; + + try { + logInfo(ctx, `Downloading stack files for ${stackName}:${version}`); + + await DevfileRegistry.Instance.downloadStackExtraFiles( + registryUrl, + stackName, + version, + devfile, + ctx.projectPath + ); + + logInfo(ctx, 'Stack files downloaded successfully'); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logError(ctx, `Failed to download stack extra files: ${errorMessage}`); + logInfo(ctx, 'Component initialization will continue without extra files'); + // Don't fail the init - extra files are optional + } +} diff --git a/test/integration/devfileRegistryWrapper.test.ts b/test/integration/devfileRegistryWrapper.test.ts index 5e572df1f..d07dafe86 100644 --- a/test/integration/devfileRegistryWrapper.test.ts +++ b/test/integration/devfileRegistryWrapper.test.ts @@ -5,6 +5,9 @@ import { expect } from 'chai'; import { suite, suiteSetup } from 'mocha'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import * as os from 'os'; import { DevfileRegistry } from '../../src/devfile-registry/devfileRegistryWrapper'; import { OdoPreference } from '../../src/odo/odoPreference'; @@ -78,4 +81,281 @@ suite('Devfile Registry Wrapper tests', function () { expect(registryNames).to.not.contain(TEST_REGISTRY_NAME); }); }); + + suite('Stack Extra Files Download', function () { + let testProjectPath: string; + + suiteSetup(async function () { + // Create temporary test directory + testProjectPath = await fs.mkdtemp(path.join(os.tmpdir(), 'devfile-test-stack-files-')); + }); + + suiteTeardown(async function () { + // Cleanup test directory + try { + await fs.rm(testProjectPath, { recursive: true, force: true }); + } catch { + // ignore cleanup errors + } + }); + + test('downloadStackExtraFiles() - downloads docker and kubernetes files', async function () { + this.timeout(30000); // Increase timeout for network requests + + const resolvedDevfile = { + schemaVersion: '2.2.2', + metadata: { + name: 'go-test' + }, + components: [ + { + name: 'build', + image: { + imageName: 'go-image:latest', + dockerfile: { + uri: 'docker/Dockerfile', + buildContext: '.', + rootRequired: false + } + } + }, + { + name: 'deploy', + kubernetes: { + uri: 'kubernetes/deploy.yaml', + endpoints: [ + { name: 'http-8081', targetPort: 8081 } + ] + } + } + ] + }; + + await DevfileRegistry.Instance.downloadStackExtraFiles( + OdoPreference.DEFAULT_DEVFILE_REGISTRY_URL, + 'go', + '2.6.0', + resolvedDevfile, + testProjectPath + ); + + // Verify Dockerfile was downloaded + const dockerfilePath = path.join(testProjectPath, 'docker', 'Dockerfile'); + const dockerfileExists = await fs.access(dockerfilePath).then(() => true).catch(() => false); + expect(dockerfileExists).to.be.true; + + const dockerfileContent = await fs.readFile(dockerfilePath, 'utf-8'); + expect(dockerfileContent).to.contain('FROM'); + expect(dockerfileContent).to.contain('go-toolset'); + + // Verify Kubernetes manifest was downloaded + const deployYamlPath = path.join(testProjectPath, 'kubernetes', 'deploy.yaml'); + const deployYamlExists = await fs.access(deployYamlPath).then(() => true).catch(() => false); + expect(deployYamlExists).to.be.true; + + const deployYamlContent = await fs.readFile(deployYamlPath, 'utf-8'); + expect(deployYamlContent).to.contain('kind: Service'); + expect(deployYamlContent).to.contain('kind: Deployment'); + }); + + test('downloadStackExtraFiles() - handles devfile with no extra files', async function () { + this.timeout(10000); + + const devfileWithNoExtraFiles = { + schemaVersion: '2.2.2', + metadata: { + name: 'simple-component' + }, + components: [ + { + name: 'runtime', + container: { + image: 'node:18', + mountSources: true + } + } + ] + }; + + // Use separate directory for this test + const testPath = await fs.mkdtemp(path.join(os.tmpdir(), 'devfile-test-no-files-')); + + // Should not throw error when no URIs to download + await DevfileRegistry.Instance.downloadStackExtraFiles( + OdoPreference.DEFAULT_DEVFILE_REGISTRY_URL, + 'nodejs', + '3.0.0', + devfileWithNoExtraFiles, + testPath + ); + + // No files should be created + const dockerDir = path.join(testPath, 'docker'); + const dockerDirExists = await fs.access(dockerDir).then(() => true).catch(() => false); + expect(dockerDirExists).to.be.false; + + // Cleanup + await fs.rm(testPath, { recursive: true, force: true }); + }); + + test('downloadStackExtraFiles() - uses filesystem cache', async function () { + this.timeout(30000); + + // Cache follows extension's pattern: ~/.local/state/vs-openshift-tools/ + const cacheDir = path.join( + os.homedir(), + '.local', + 'state', + 'vs-openshift-tools', + 'devfile-registry-cache', + 'go', + '2.6.0' + ); + + // Clean cache first + try { + await fs.rm(cacheDir, { recursive: true, force: true }); + } catch { + // ignore + } + + const resolvedDevfile = { + components: [ + { + name: 'build', + image: { + dockerfile: { + uri: 'docker/Dockerfile' + } + } + } + ] + }; + + // First download - should create cache + await DevfileRegistry.Instance.downloadStackExtraFiles( + OdoPreference.DEFAULT_DEVFILE_REGISTRY_URL, + 'go', + '2.6.0', + resolvedDevfile, + testProjectPath + ); + + // Verify cache file was created + const cachedFilePath = path.join(cacheDir, 'docker', 'Dockerfile'); + const cacheExists = await fs.access(cachedFilePath).then(() => true).catch(() => false); + expect(cacheExists, `Cache file should exist at ${cachedFilePath}`).to.be.true; + + // Second download - should use cache (faster) + const testPath2 = await fs.mkdtemp(path.join(os.tmpdir(), 'devfile-test-cache-')); + + const startTime = Date.now(); + await DevfileRegistry.Instance.downloadStackExtraFiles( + OdoPreference.DEFAULT_DEVFILE_REGISTRY_URL, + 'go', + '2.6.0', + resolvedDevfile, + testPath2 + ); + const duration = Date.now() - startTime; + + // Cached download should be very fast (< 100ms) + expect(duration).to.be.lessThan(100); + + // Verify file was written to second location + const file2Path = path.join(testPath2, 'docker', 'Dockerfile'); + const file2Exists = await fs.access(file2Path).then(() => true).catch(() => false); + expect(file2Exists).to.be.true; + + // Cleanup + await fs.rm(testPath2, { recursive: true, force: true }); + }); + + test('downloadStackExtraFiles() - handles multiple file types', async function () { + this.timeout(30000); + + const testPath = await fs.mkdtemp(path.join(os.tmpdir(), 'devfile-test-multi-')); + + const devfileWithMultipleUris = { + components: [ + { + name: 'build', + image: { + dockerfile: { + uri: 'docker/Dockerfile' + } + } + }, + { + name: 'k8s-deploy', + kubernetes: { + uri: 'kubernetes/deploy.yaml' + } + } + ] + }; + + await DevfileRegistry.Instance.downloadStackExtraFiles( + OdoPreference.DEFAULT_DEVFILE_REGISTRY_URL, + 'go', + '2.6.0', + devfileWithMultipleUris, + testPath + ); + + // Both files should exist + const dockerfileExists = await fs.access(path.join(testPath, 'docker', 'Dockerfile')) + .then(() => true).catch(() => false); + const deployExists = await fs.access(path.join(testPath, 'kubernetes', 'deploy.yaml')) + .then(() => true).catch(() => false); + + expect(dockerfileExists).to.be.true; + expect(deployExists).to.be.true; + + // Cleanup + await fs.rm(testPath, { recursive: true, force: true }); + }); + + test('downloadStackExtraFiles() - throws error for invalid files', async function () { + this.timeout(10000); + + const devfileWithInvalidUri = { + components: [ + { + name: 'build', + image: { + dockerfile: { + uri: 'nonexistent/file.txt' + } + } + } + ] + }; + + // Use separate directory for this test + const testPath = await fs.mkdtemp(path.join(os.tmpdir(), 'devfile-test-error-')); + + // Should throw error with meaningful message + try { + await DevfileRegistry.Instance.downloadStackExtraFiles( + OdoPreference.DEFAULT_DEVFILE_REGISTRY_URL, + 'go', + '2.6.0', + devfileWithInvalidUri, + testPath + ); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error.message).to.include('Failed to download'); + expect(error.message).to.include('nonexistent/file.txt'); + } + + const nonexistentPath = path.join(testPath, 'nonexistent', 'file.txt'); + const fileExists = await fs.access(nonexistentPath).then(() => true).catch(() => false); + expect(fileExists).to.be.false; + + // Cleanup + await fs.rm(testPath, { recursive: true, force: true }); + }); + }); });