From 80a4c276c45aa04d192e11ef43c23cdb2eaf69dd Mon Sep 17 00:00:00 2001 From: Chris Mendoza Date: Tue, 31 Mar 2026 17:47:15 -0400 Subject: [PATCH 1/5] Add LSP Client and long running tests in tools --- .gitignore | 2 + package.json | 1 + tools/long-running/Config.ts | 72 ++++ tools/long-running/Monitoring.ts | 129 ++++++++ tools/long-running/Templates.ts | 36 ++ tools/long-running/TestOrchestrator.ts | 197 +++++++++++ tools/long-running/run-long-running-test.ts | 20 ++ .../long-running/testers/CompletionTester.ts | 55 +++ tools/long-running/testers/HoverTester.ts | 89 +++++ tools/long-running/testers/TesterCommon.ts | 27 ++ tools/long-running/testers/TesterTypes.ts | 31 ++ tools/lsp-client/LspClient.ts | 313 ++++++++++++++++++ tools/lsp-client/LspConnectionInterface.ts | 8 + tools/lsp-client/index.ts | 3 + tools/lsp-client/types.ts | 49 +++ 15 files changed, 1032 insertions(+) create mode 100644 tools/long-running/Config.ts create mode 100644 tools/long-running/Monitoring.ts create mode 100644 tools/long-running/Templates.ts create mode 100644 tools/long-running/TestOrchestrator.ts create mode 100644 tools/long-running/run-long-running-test.ts create mode 100644 tools/long-running/testers/CompletionTester.ts create mode 100644 tools/long-running/testers/HoverTester.ts create mode 100644 tools/long-running/testers/TesterCommon.ts create mode 100644 tools/long-running/testers/TesterTypes.ts create mode 100644 tools/lsp-client/LspClient.ts create mode 100644 tools/lsp-client/LspConnectionInterface.ts create mode 100644 tools/lsp-client/index.ts create mode 100644 tools/lsp-client/types.ts diff --git a/.gitignore b/.gitignore index 5b8af9f..28bbc37 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ .DS_Store tools/* !tools/*.ts +!tools/lsp-client/ +!tools/long-running/ **/.aws-cfn-storage /oss-attribution /tmp-tst diff --git a/package.json b/package.json index 3c94edd..371e07e 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "test:integration": "cross-env NODE_ENV=test vitest run --config vitest.integration.config.ts", "test:unit": "cross-env NODE_ENV=test vitest run --config vitest.unit.config.ts", "test:leaks": "cross-env NODE_ENV=test vitest run --pool=forks --logHeapUsage", + "test:long-running": "tsx tools/long-running/run-long-running-test.ts", "lint": "eslint --cache --cache-location node_modules/.cache/eslint --max-warnings 0 .", "lint:fix": "npm run lint -- --fix", "build:go:dev": "GOPROXY=direct go build -C cfn-init/cmd -v -o ../../bundle/development/bin/cfn-init", diff --git a/tools/long-running/Config.ts b/tools/long-running/Config.ts new file mode 100644 index 0000000..e18cbc6 --- /dev/null +++ b/tools/long-running/Config.ts @@ -0,0 +1,72 @@ +export interface StandaloneConfig { + duration: string; + maxRetries: number; + responseTimeout: number; + standalonePath: string; +} + +function parseSimpleArgs(): Partial { + const args: Partial = {}; + + for (let i = 0; i < process.argv.length; i++) { + const arg = process.argv[i]; + const nextArg = process.argv[i + 1]; + + if ((arg === '--duration' || arg === '-d') && nextArg) { + args.duration = nextArg; + i++; + } else if (arg === '--max-retries' && nextArg) { + args.maxRetries = Number.parseInt(nextArg); + i++; + } else if (arg === '--response-timeout' && nextArg) { + args.responseTimeout = Number.parseInt(nextArg); + i++; + } else if (arg === '--standalone-path' && nextArg) { + args.standalonePath = nextArg; + i++; + } + } + + return args; +} + +export function parseStandaloneConfig(): StandaloneConfig { + // Start with environment variables (npm script support) + const envConfig = { + duration: process.env.LONG_RUNNING_DURATION ?? '4h', + maxRetries: Number.parseInt(process.env.MAX_RETRIES ?? '3'), + responseTimeout: Number.parseInt(process.env.RESPONSE_TIMEOUT ?? '5000'), + standalonePath: process.env.STANDALONE_PATH ?? './cfn-lsp-server-standalone.js', + }; + + // Override with command line arguments if provided + const cliArgs = parseSimpleArgs(); + + return { + ...envConfig, + ...cliArgs, + }; +} + +export function parseDuration(duration: string): number { + const match = duration.match(/^(\d+)([hms])$/); + if (!match) throw new Error(`Invalid duration format: ${duration}`); + + const value = Number.parseInt(match[1]); + const unit = match[2]; + + switch (unit) { + case 'h': { + return value * 60 * 60 * 1000; + } + case 'm': { + return value * 60 * 1000; + } + case 's': { + return value * 1000; + } + default: { + throw new Error(`Invalid duration unit: ${unit}`); + } + } +} diff --git a/tools/long-running/Monitoring.ts b/tools/long-running/Monitoring.ts new file mode 100644 index 0000000..fe72cf2 --- /dev/null +++ b/tools/long-running/Monitoring.ts @@ -0,0 +1,129 @@ +import { OperationType, getTesterConfig } from './testers/TesterTypes'; + +export type StandaloneTestMetrics = { + operationsAttempted: number; + operationsFailed: number; + averageDuration: number | null; + minDuration: number | null; + maxDuration: number | null; + lastDuration: number | null; +}; + +const createEmptyMetrics = (): StandaloneTestMetrics => ({ + operationsAttempted: 0, + operationsFailed: 0, + averageDuration: null, + minDuration: null, + maxDuration: null, + lastDuration: null, +}); + +const metrics: Record = {} as Record; +for (const operationType of Object.values(OperationType)) { + metrics[operationType] = createEmptyMetrics(); +} + +export function recordOperation(duration: number, success: boolean, operationType: OperationType): void { + const metric = metrics[operationType]; + metric.operationsAttempted++; + + if (success) { + const successfulOps = metric.operationsAttempted - metric.operationsFailed; + metric.averageDuration = + metric.averageDuration === null + ? duration + : (metric.averageDuration * (successfulOps - 1) + duration) / successfulOps; + metric.minDuration = metric.minDuration === null ? duration : Math.min(metric.minDuration, duration); + metric.maxDuration = metric.maxDuration === null ? duration : Math.max(metric.maxDuration, duration); + metric.lastDuration = duration; + } else { + metric.operationsFailed++; + } +} + +let startTime: number; + +export function initializeMonitoring(): void { + startTime = Date.now(); +} + +export function logProgress(): void { + const totalOps = Object.values(metrics).reduce((sum, m) => sum + m.operationsAttempted, 0); + const totalFailed = Object.values(metrics).reduce((sum, m) => sum + m.operationsFailed, 0); + const elapsed = Date.now() - startTime; + const elapsedMinutes = Math.round(elapsed / 60_000); + + console.log('Progress Report'); + console.log(` Runtime: ${elapsedMinutes} minutes`); + console.log(` Total Operations: ${totalOps}`); + console.log(` Successful: ${totalOps - totalFailed}`); + console.log(` Failed: ${totalFailed}`); + console.log(` Success Rate: ${totalOps > 0 ? (((totalOps - totalFailed) / totalOps) * 100).toFixed(1) : 0}%`); + + // Per-operation breakdown + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, StandaloneTestMetrics][]) { + if (metric.operationsAttempted > 0) { + const successRate = ( + ((metric.operationsAttempted - metric.operationsFailed) / metric.operationsAttempted) * + 100 + ).toFixed(1); + console.log( + ` ${operationType}: ${metric.operationsAttempted} ops, ${metric.operationsFailed} failed (${successRate}% success)`, + ); + } + } +} + +export function generateFinalReport(testStartTime: number): void { + const runtime = Date.now() - testStartTime; + const totalOps = Object.values(metrics).reduce((sum, m) => sum + m.operationsAttempted, 0); + const totalFailed = Object.values(metrics).reduce((sum, m) => sum + m.operationsFailed, 0); + + console.log('Final Test Report'); + console.log('='.repeat(50)); + console.log(`Runtime: ${Math.round(runtime / 1000 / 60)} minutes`); + console.log(`Total Operations: ${totalOps}`); + console.log(`Successful: ${totalOps - totalFailed}`); + console.log(`Failed: ${totalFailed}`); + console.log(`Success Rate: ${totalOps > 0 ? (((totalOps - totalFailed) / totalOps) * 100).toFixed(2) : 0}%`); + + // Per-operation breakdown + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, StandaloneTestMetrics][]) { + if (metric.operationsAttempted > 0) { + console.log(`${operationType}:`); + console.log(` Operations: ${metric.operationsAttempted}`); + console.log(` Failed: ${metric.operationsFailed}`); + console.log(` Avg Duration: ${metric.averageDuration?.toFixed(2) ?? 'N/A'}ms`); + console.log(` Max Duration: ${metric.maxDuration?.toFixed(2) ?? 'N/A'}ms`); + console.log(` Min Duration: ${metric.minDuration?.toFixed(2) ?? 'N/A'}ms`); + } + } + console.log('='.repeat(50)); +} + +export function checkPerformanceDegradation(): void { + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, StandaloneTestMetrics][]) { + const config = getTesterConfig(operationType); + + if (metric.averageDuration !== null && metric.averageDuration > config.avgDurationLimitMs) { + throw new Error( + `${operationType} average duration ${metric.averageDuration.toFixed(1)}ms exceeds limit ${config.avgDurationLimitMs}ms`, + ); + } + + if (metric.maxDuration !== null && metric.maxDuration > config.maxDurationLimitMs) { + throw new Error( + `${operationType} max duration ${metric.maxDuration.toFixed(1)}ms exceeds limit ${config.maxDurationLimitMs}ms`, + ); + } + } + + // Basic memory check + const memUsage = process.memoryUsage(); + const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024); + + if (heapUsedMB > 2000) { + // 2GB threshold + console.warn(`High memory usage detected: ${heapUsedMB}MB heap used`); + } +} diff --git a/tools/long-running/Templates.ts b/tools/long-running/Templates.ts new file mode 100644 index 0000000..8e1df73 --- /dev/null +++ b/tools/long-running/Templates.ts @@ -0,0 +1,36 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; + +const cache = new Map(); + +const loadTemplate = (name: string): string => { + if (!cache.has(name)) { + const templatePath = join(__dirname, '../../tst/resources/templates', name); + cache.set(name, readFileSync(templatePath, 'utf8')); + } + return cache.get(name)!; +}; + +export interface StandaloneTemplateConfig { + name: string; + content: string; +} + +export const STANDALONE_TEMPLATE_CONFIGS: StandaloneTemplateConfig[] = [ + { + name: 'sample.yaml', + content: loadTemplate('sample_template.yaml'), + }, + { + name: 'simple.yaml', + content: loadTemplate('simple.yaml'), + }, + { + name: 'comprehensive.yaml', + content: loadTemplate('comprehensive.yaml'), + }, + { + name: 'condition-usage.yaml', + content: loadTemplate('condition-usage.yaml'), + }, +]; diff --git a/tools/long-running/TestOrchestrator.ts b/tools/long-running/TestOrchestrator.ts new file mode 100644 index 0000000..ab3ff01 --- /dev/null +++ b/tools/long-running/TestOrchestrator.ts @@ -0,0 +1,197 @@ +import { LspClient } from '../lsp-client/LspClient'; +import { parseStandaloneConfig, parseDuration } from './Config'; +import { initializeMonitoring, logProgress, checkPerformanceDegradation } from './Monitoring'; +import { HoverTester } from './testers/HoverTester'; +import { CompletionTester } from './testers/CompletionTester'; +import { STANDALONE_TEMPLATE_CONFIGS } from './Templates'; +import { AwsRegion } from '../../src/utils/Region'; +import { WaitFor } from '../../tst/utils/Utils'; +import { existsSync } from 'fs'; + +export class TestOrchestrator { + private client!: LspClient; + private readonly config = parseStandaloneConfig(); + private startTime!: number; + private endTime!: number; + private hoverTester!: HoverTester; + private completionTester!: CompletionTester; + + private readonly templates = STANDALONE_TEMPLATE_CONFIGS; + + private readonly testRegions = Object.values(AwsRegion); + + async initialize(): Promise { + console.log('Starting CloudFormation Language Server Standalone Long-Running Tests'); + console.log(`Duration: ${this.config.duration}`); + console.log(`Max retries: ${this.config.maxRetries}`); + console.log(`Response timeout: ${this.config.responseTimeout}ms`); + console.log(`Standalone path: ${this.config.standalonePath}`); + + // Verify standalone bundle exists + if (!existsSync(this.config.standalonePath)) { + throw new Error(`Standalone bundle not found at: ${this.config.standalonePath}`); + } + + // Initialize LSP client + this.client = new LspClient({ + serverPath: this.config.standalonePath, + mode: 'ipc', + clientId: 'standalone-long-running-test', + telemetryEnabled: false, + }); + + await this.client.initialize(); + console.log('LSP client initialized'); + + // Initialize testers + this.hoverTester = new HoverTester(this.client); + this.completionTester = new CompletionTester(this.client); + + console.log(`Loaded ${this.templates.length} templates`); + + // Wait for full system readiness before loading schemas + await this.waitForSystemReadiness(); + + await this.loadAllRegionSchemas(); + + initializeMonitoring(); + console.log('Initialization complete'); + } + + private async waitForSystemReadiness(): Promise { + console.log('Waiting for core services readiness'); + + await this.client.waitForReadiness(); + } + + async runTests(): Promise { + console.log('Starting test execution phase'); + + const durationMs = parseDuration(this.config.duration); + this.startTime = Date.now(); + this.endTime = this.startTime + durationMs; + + let cycleCount = 0; + let successCount = 0; + let errorCount = 0; + let lastProgressLog = Date.now(); + const progressInterval = 5 * 60 * 1000; // 5 minutes + + while (Date.now() < this.endTime) { + cycleCount++; + + try { + await this.executeTestCycle(); + successCount++; + + checkPerformanceDegradation(); + + if (Date.now() - lastProgressLog > progressInterval) { + logProgress(); + lastProgressLog = Date.now(); + } + } catch (error) { + errorCount++; + console.error(`Test cycle ${cycleCount} failed:`, error); + + // Fail fast if too many errors + if (errorCount > 10) { + throw new Error(`Too many errors (${errorCount}), failing fast`); + } + } + + // Brief pause between cycles + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + console.log(`Test execution completed after ${cycleCount} cycles`); + console.log(`Results: ${successCount} success, ${errorCount} errors`); + } + + async cleanup(): Promise { + if (this.client) { + await this.client.shutdown(); + } + } + + private async executeTestCycle(): Promise { + // Test all regions (switch region for each cycle) + for (const region of this.testRegions) { + await this.switchToRegion(region); + + // Test all templates for this region + for (const template of this.templates) { + const uri = `file:///test/${template.name}`; + + try { + await this.client.openDocument(uri, template.content); + + await this.validateLsp(uri); + + // Revert document to original state after tests + await this.client.updateDocument(uri, 6, template.content); + } finally { + try { + await this.client.closeDocument(uri); + } catch (error) { + console.warn(`Failed to close document ${uri}:`, error); + } + } + } + } + } + + private async loadAllRegionSchemas(): Promise { + // Check what regions are already available from LspClient + const alreadyAvailable = [...this.client.readyRegions]; + const unavailableRegions = this.testRegions.filter((region) => !this.client.readyRegions.has(region)); + + console.log(`Schema status: ${alreadyAvailable.length} available, ${unavailableRegions.length} unavailable`); + console.log(`Available region schemas: ${alreadyAvailable.join(', ')}`); + + if (unavailableRegions.length > 0) { + console.log(`Loading the following region schemas: ${unavailableRegions.join(', ')}`); + } + + for (const region of unavailableRegions) { + await this.switchToRegion(region); + await this.waitForRegionSchemas(region); + } + + console.log('Regional schema loading complete'); + } + + private async waitForRegionSchemas(region: string): Promise { + try { + await WaitFor.waitFor( + () => { + if (!this.client.readyRegions.has(region)) { + throw new Error(`Region ${region} schemas not ready yet`); + } + }, + 30_000, // 30 second timeout + 500, // Check every 500ms + ); + } catch { + console.warn(`Timeout waiting for ${region} schemas, proceeding anyway`); + } + } + + private async switchToRegion(region: string): Promise { + // Store the new configuration + await this.client.changeConfiguration({ + settings: { + 'aws.cloudformation': { + profile: { + region, + }, + }, + }, + }); + } + + private async validateLsp(uri: string): Promise { + await this.hoverTester.testAllScenarios(uri); + await this.completionTester.testAllScenarios(uri); + } +} diff --git a/tools/long-running/run-long-running-test.ts b/tools/long-running/run-long-running-test.ts new file mode 100644 index 0000000..9b3b988 --- /dev/null +++ b/tools/long-running/run-long-running-test.ts @@ -0,0 +1,20 @@ +import { TestOrchestrator } from './TestOrchestrator'; +import { generateFinalReport } from './Monitoring'; + +async function main(): Promise { + const orchestrator = new TestOrchestrator(); + + try { + await orchestrator.initialize(); + await orchestrator.runTests(); + generateFinalReport(Date.now()); + } catch (error) { + generateFinalReport(Date.now()); + console.error('Standalone test failed:', error); + throw error; + } finally { + await orchestrator.cleanup(); + } +} + +void main(); diff --git a/tools/long-running/testers/CompletionTester.ts b/tools/long-running/testers/CompletionTester.ts new file mode 100644 index 0000000..f7180d4 --- /dev/null +++ b/tools/long-running/testers/CompletionTester.ts @@ -0,0 +1,55 @@ +import { LspClient } from '../../lsp-client/LspClient'; +import { StandaloneTester, OperationType } from './TesterTypes'; +import { retryOperationWithPerformance } from './TesterCommon'; + +export class CompletionTester implements StandaloneTester { + constructor(private readonly client: LspClient) {} + + private validateCompletionItems(result: any, requiredLabels: string[], context: string): void { + if (!result?.items || !Array.isArray(result.items) || result.items.length === 0) { + throw new Error(`${context} returned no items`); + } + + const labels = new Set(result.items.map((item: any) => item.label as string)); + for (const required of requiredLabels) { + if (!labels.has(required)) { + throw new Error(`${context} missing ${required}`); + } + } + } + + async testAllScenarios(uri: string): Promise { + // Test 1: Top-level completion using full document update + const basicTemplate = `AWSTemplateFormatVersion: '2010-09-09' +`; + + await this.client.updateDocument(uri, 4, basicTemplate); + + await retryOperationWithPerformance( + () => this.client.completion(uri, 1, 0), + (result: any) => this.validateCompletionItems(result, ['Resources', 'Parameters'], 'Top-level completion'), + OperationType.COMPLETION, + ); + + // Test 2: Property completion in Resources using incremental update + const resourceSection = ` +Resources: + MyBucket: + Type: AWS::S3::Bucket + Properties: + `; + + await this.client.updateDocument(uri, 5, [ + { + range: { start: { line: 1, character: 0 }, end: { line: 1, character: 0 } }, + text: resourceSection, + }, + ]); + + await retryOperationWithPerformance( + () => this.client.completion(uri, 6, 6), + (result: any) => this.validateCompletionItems(result, ['BucketName', 'Tags'], 'S3 bucket completion'), + OperationType.COMPLETION, + ); + } +} diff --git a/tools/long-running/testers/HoverTester.ts b/tools/long-running/testers/HoverTester.ts new file mode 100644 index 0000000..2f0d382 --- /dev/null +++ b/tools/long-running/testers/HoverTester.ts @@ -0,0 +1,89 @@ +import { LspClient } from '../../lsp-client/LspClient'; +import { StandaloneTester, OperationType } from './TesterTypes'; +import { retryOperationWithPerformance } from './TesterCommon'; + +export class HoverTester implements StandaloneTester { + constructor(private readonly client: LspClient) {} + + private extractHoverContent(hoverResult: any): string { + if (typeof hoverResult.contents === 'string') { + return hoverResult.contents as string; + } else if (Array.isArray(hoverResult.contents)) { + return hoverResult.contents.length > 0 ? JSON.stringify(hoverResult.contents) : ''; + } else if ( + hoverResult.contents && + typeof hoverResult.contents === 'object' && + 'value' in hoverResult.contents + ) { + return hoverResult.contents.value as string; + } + return ''; + } + + private validateHoverContent(content: string, patterns: string[]): void { + if (!content || content.length === 0) { + throw new Error('Hover content is empty'); + } + + const lowerContent = content.toLowerCase(); + for (const pattern of patterns) { + if (!lowerContent.includes(pattern.toLowerCase())) { + throw new Error(`Hover content missing expected pattern: ${pattern}`); + } + } + } + + async testAllScenarios(uri: string): Promise { + // Test 1: Hover on resource type using full document update + const s3Template = `AWSTemplateFormatVersion: '2010-09-09' +Resources: + MyResource: + Type: AWS::S3::Bucket + Properties: + BucketName: TestName +`; + + await this.client.updateDocument(uri, 2, s3Template); + + await retryOperationWithPerformance( + () => this.client.hover(uri, 3, 15), + (result: any) => { + if (!result?.contents) { + throw new Error('Hover on resource type returned no content'); + } + + const content = this.extractHoverContent(result); + this.validateHoverContent(content, ['aws::s3::bucket', 'bucket', 's3']); + }, + OperationType.HOVER, + ); + + // Test 2: Hover on property after adding Parameters section using incremental update + const parametersSection = ` +Parameters: + MyParam: + Type: String + Default: TestValue +`; + + await this.client.updateDocument(uri, 3, [ + { + range: { start: { line: 6, character: 0 }, end: { line: 6, character: 0 } }, + text: parametersSection, + }, + ]); + + await retryOperationWithPerformance( + () => this.client.hover(uri, 8, 10), + (result: any) => { + if (!result?.contents) { + throw new Error('Hover on parameter returned no content'); + } + + const content = this.extractHoverContent(result); + this.validateHoverContent(content, ['parameter', 'string']); + }, + OperationType.HOVER, + ); + } +} diff --git a/tools/long-running/testers/TesterCommon.ts b/tools/long-running/testers/TesterCommon.ts new file mode 100644 index 0000000..c73beba --- /dev/null +++ b/tools/long-running/testers/TesterCommon.ts @@ -0,0 +1,27 @@ +import { recordOperation } from '../Monitoring'; +import { WaitFor } from '../../../tst/utils/Utils'; +import { OperationType, getTesterConfig } from './TesterTypes'; + +const RETRY_INTERVAL_MS = 250; + +export async function retryOperationWithPerformance( + operation: () => Promise, + validate: (result: T) => void, + operationType: OperationType, +): Promise { + const config = getTesterConfig(operationType); + let responseTime: number = 0; + + await WaitFor.waitFor( + async () => { + const startTime = performance.now(); + const result = await operation(); + responseTime = performance.now() - startTime; + validate(result); + }, + config.retryTimeoutMs, + RETRY_INTERVAL_MS, + ); + + recordOperation(responseTime, true, operationType); +} diff --git a/tools/long-running/testers/TesterTypes.ts b/tools/long-running/testers/TesterTypes.ts new file mode 100644 index 0000000..f3be2d3 --- /dev/null +++ b/tools/long-running/testers/TesterTypes.ts @@ -0,0 +1,31 @@ +export enum OperationType { + HOVER = 'hover', + COMPLETION = 'completion', +} + +export interface StandaloneTester { + testAllScenarios(uri: string): Promise; +} + +export interface TesterConfig { + retryTimeoutMs: number; + avgDurationLimitMs: number; + maxDurationLimitMs: number; +} + +export const TESTER_CONFIG: Record = { + [OperationType.HOVER]: { + retryTimeoutMs: 3000, + avgDurationLimitMs: 150, + maxDurationLimitMs: 3000, + }, + [OperationType.COMPLETION]: { + retryTimeoutMs: 5000, + avgDurationLimitMs: 300, + maxDurationLimitMs: 5000, + }, +}; + +export function getTesterConfig(operationType: OperationType): TesterConfig { + return TESTER_CONFIG[operationType]; +} diff --git a/tools/lsp-client/LspClient.ts b/tools/lsp-client/LspClient.ts new file mode 100644 index 0000000..60d83e8 --- /dev/null +++ b/tools/lsp-client/LspClient.ts @@ -0,0 +1,313 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +import { spawn, ChildProcess } from 'child_process'; +import { + createMessageConnection, + MessageConnection, + StreamMessageReader, + StreamMessageWriter, + IPCMessageReader, + IPCMessageWriter, + TextDocumentContentChangeEvent, +} from 'vscode-languageserver-protocol/node'; +import { randomBytes, randomUUID } from 'crypto'; +import { LspClientConfig, ReadinessFlags, ExtendedInitializeParams } from './types'; +import { LspConnection } from './LspConnectionInterface'; +import { WaitFor } from '../../tst/utils/Utils'; + +/** + * Common LSP client for CloudFormation Language Server testing. + * Handles server startup, LSP protocol communication, and readiness detection. + */ +export class LspClient implements LspConnection { + protected serverProcess?: ChildProcess; + protected connection?: MessageConnection; + protected readinessFlags: ReadinessFlags = { + cfnLint: false, + cfnGuard: false, + }; + + public readonly clientId: string; + public readonly createdAt: number; + protected readonly encryptionKey: Buffer; + protected isShutdown = false; + protected currentWorkspaceConfig: Record[] = [{}]; + public readyRegions = new Set(); + + constructor(protected config: LspClientConfig) { + this.clientId = config.clientId ?? `lsp-client-${randomUUID()}`; + this.createdAt = performance.now(); + this.encryptionKey = randomBytes(32); + } + + async initialize(): Promise { + console.log('LspClient: Starting initialization...'); + + // 1. Start server process + const args = this.config.mode === 'ipc' ? ['--node-ipc'] : ['--stdio']; + console.log(`LspClient: Spawning server with args: node ${this.config.serverPath} ${args.join(' ')}`); + + this.serverProcess = spawn('node', [this.config.serverPath, ...args], { + stdio: this.config.mode === 'ipc' ? ['pipe', 'pipe', 'pipe', 'ipc'] : ['pipe', 'pipe', 'pipe'], + env: { ...process.env, ...this.config.env }, + }); + + console.log(`LspClient: Server process spawned with PID: ${this.serverProcess.pid}`); + + // 2. Setup output monitoring for readiness detection + this.attachOutputListeners(); + + // 3. Create LSP connection + console.log('LspClient: Creating LSP connection...'); + const reader = + this.config.mode === 'ipc' + ? new IPCMessageReader(this.serverProcess) + : new StreamMessageReader(this.serverProcess.stdout!); + + const writer = + this.config.mode === 'ipc' + ? new IPCMessageWriter(this.serverProcess) + : new StreamMessageWriter(this.serverProcess.stdin!); + + this.connection = createMessageConnection(reader, writer); + + // Handle workspace/configuration requests from server + + this.connection.onRequest('workspace/configuration', (params: any) => { + // Extract the specific configuration section requested + if (params?.items?.length > 0) { + const results = params.items.map((item: any) => { + if (item.section === 'aws.cloudformation') { + // Return just the CloudFormation config part + const fullConfig = this.currentWorkspaceConfig[0] ?? {}; + return (fullConfig as any)['aws.cloudformation'] ?? {}; + } + return {}; + }); + return results; + } + return this.currentWorkspaceConfig; + }); + + this.connection.listen(); + console.log('LspClient: LSP connection created and listening'); + + // 4. Perform LSP handshake + console.log('LspClient: Performing LSP handshake...'); + try { + await this.performHandshake(); + console.log('LspClient: LSP handshake completed'); + } catch (error) { + console.error('LspClient: LSP handshake failed:', error); + throw error; + } + } + + private readonly onServerOutput = (data: Buffer) => { + const output = data.toString().trim(); + + // Readiness detection + if (output.includes('cfn-lint version')) { + this.readinessFlags.cfnLint = true; + } + if (output.includes('Loading rules from')) { + this.readinessFlags.cfnGuard = true; + } + + // Region-specific schema loading + const regionSchemaMatch = output.match(/public schemas downloaded for ([a-z0-9-]+)/); + if (regionSchemaMatch) { + this.readyRegions.add(regionSchemaMatch[1]); + } + + // Log filtering + const suppressLevels = this.config.suppressLogLevels ?? ['INFO', 'DEBUG']; + const shouldSuppress = suppressLevels.some((level) => output.includes(`${level}:`)); + + if (!shouldSuppress) { + console.error(`[LSP Server]: ${output}`); + } + }; + + protected attachOutputListeners(): void { + this.serverProcess!.stdout?.on('data', this.onServerOutput); + this.serverProcess!.stderr?.on('data', this.onServerOutput); + + this.serverProcess!.on('exit', (code, signal) => { + if (signal) { + console.log(`[LSP Server]: Process terminated with signal ${signal}`); + } else { + console.log(`[LSP Server]: Process exited with code ${code}`); + } + }); + + this.serverProcess!.on('error', (error) => { + console.error(`[LSP Server]: Process error:`, error); + }); + } + + protected async performHandshake(): Promise { + const initParams: ExtendedInitializeParams = { + processId: process.pid, + rootUri: 'file:///test/workspace', + capabilities: { + textDocument: { + hover: { dynamicRegistration: true }, + completion: { dynamicRegistration: true }, + }, + }, + clientInfo: { + name: 'CFN LSP Test Client', + version: '1.0.0', + }, + initializationOptions: { + aws: { + clientInfo: { + extension: { + name: 'aws.cloudformation.lsp.test', + version: '1.0.0', + }, + clientId: this.clientId, + }, + telemetryEnabled: this.config.telemetryEnabled ?? true, + storageDir: this.config.storageDir, + encryption: { + key: this.encryptionKey.toString('base64'), + mode: 'JWT', + }, + ...(this.config.featureFlags && { + featureFlags: this.config.featureFlags, + }), + }, + }, + }; + + console.log('LspClient: Sending initialize request...'); + try { + await Promise.race([ + this.connection!.sendRequest('initialize', initParams), + new Promise((_resolve, reject) => setTimeout(() => reject(new Error('Initialize timeout')), 30_000)), + ]); + console.log('LspClient: Initialize request completed'); + + console.log('LspClient: Sending initialized notification'); + await this.connection!.sendNotification('initialized', {}); + console.log('LspClient: Initialized notification sent'); + } catch (error) { + console.error('LspClient: Handshake error:', error); + throw error; + } + } + + async openDocument(uri: string, content: string): Promise { + await this.connection!.sendNotification('textDocument/didOpen', { + textDocument: { + uri, + languageId: 'yaml', + version: 1, + text: content, + }, + }); + } + + async updateDocument( + uri: string, + version: number, + changes: string | TextDocumentContentChangeEvent[], + ): Promise { + const contentChanges = + typeof changes === 'string' + ? [{ text: changes }] // Full replacement + : changes; // Incremental changes + + await this.connection!.sendNotification('textDocument/didChange', { + textDocument: { + uri, + version, + }, + contentChanges, + }); + } + + async closeDocument(uri: string): Promise { + await this.connection!.sendNotification('textDocument/didClose', { + textDocument: { uri }, + }); + } + + async hover(uri: string, line: number, character: number): Promise { + return await this.connection!.sendRequest('textDocument/hover', { + textDocument: { uri }, + position: { line, character }, + }); + } + + async completion(uri: string, line: number, character: number): Promise { + return await this.connection!.sendRequest('textDocument/completion', { + textDocument: { uri }, + position: { line, character }, + }); + } + + async changeConfiguration(params: { settings: any }): Promise { + // Store the new configuration + if (params.settings) { + const currentConfig = this.currentWorkspaceConfig[0] ?? {}; + this.currentWorkspaceConfig = [{ ...currentConfig, ...params.settings }]; + } + + // Send the configuration change notification + await this.sendNotification('workspace/didChangeConfiguration', params); + } + + async sendRequest(method: string, params: any): Promise { + return await this.connection!.sendRequest(method, params); + } + + async sendNotification(method: string, params: any): Promise { + return await this.connection!.sendNotification(method, params); + } + + onNotification(method: string, handler: (params: any) => void): void { + this.connection!.onNotification(method, handler); + } + + onRequest(method: string, handler: (params: any) => any): void { + this.connection!.onRequest(method, handler); + } + + async waitForReadiness(timeoutMs: number = 30_000): Promise { + await WaitFor.waitFor( + () => { + if (!this.readinessFlags.cfnLint || !this.readinessFlags.cfnGuard) { + throw new Error('Lint and Guard services not ready yet'); + } + console.log('Lint and Guard services are ready'); + }, + timeoutMs, + 500, // Check every 500ms + ); + } + + get readiness(): ReadinessFlags { + return { ...this.readinessFlags }; + } + + /** Shutdown the LSP server */ + async shutdown(): Promise { + if (this.isShutdown) return; + this.isShutdown = true; + + try { + if (this.connection) { + await this.connection.sendRequest('shutdown', {}); + await this.connection.sendNotification('exit', {}); + } + } catch (e) { + console.warn('Error during LSP shutdown:', e); + } + + if (this.serverProcess) { + this.serverProcess.kill(); + } + } +} diff --git a/tools/lsp-client/LspConnectionInterface.ts b/tools/lsp-client/LspConnectionInterface.ts new file mode 100644 index 0000000..cdacbf6 --- /dev/null +++ b/tools/lsp-client/LspConnectionInterface.ts @@ -0,0 +1,8 @@ +export interface LspConnection { + initialize(): Promise; + sendRequest(method: string, params: any): Promise; + sendNotification(method: string, params: any): Promise; + onNotification(method: string, handler: (params: any) => void): void; + onRequest(method: string, handler: (params: any) => any): void; + shutdown(): Promise; +} diff --git a/tools/lsp-client/index.ts b/tools/lsp-client/index.ts new file mode 100644 index 0000000..e45baff --- /dev/null +++ b/tools/lsp-client/index.ts @@ -0,0 +1,3 @@ +export { LspClient } from './LspClient'; +export { LspConnection } from './LspConnectionInterface'; +export { LspClientConfig, ReadinessFlags } from './types'; diff --git a/tools/lsp-client/types.ts b/tools/lsp-client/types.ts new file mode 100644 index 0000000..9eebcdc --- /dev/null +++ b/tools/lsp-client/types.ts @@ -0,0 +1,49 @@ +import { InitializeParams } from 'vscode-languageserver-protocol'; + +export interface LspClientConfig { + serverPath: string; + mode: 'stdio' | 'ipc'; + clientId?: string; + storageDir?: string; + env?: NodeJS.ProcessEnv; + initTimeout?: number; + telemetryEnabled?: boolean; + featureFlags?: FeatureFlagType; + suppressLogLevels?: string[]; +} + +export interface ReadinessFlags { + cfnLint: boolean; + cfnGuard: boolean; +} + +export type ClientInfo = { + name: string; + version: string; +}; + +export type AwsMetadata = { + clientInfo?: { + extension: ClientInfo; + clientId: string; + }; + telemetryEnabled?: boolean; + storageDir?: string; + encryption?: { + key: string; + mode: string; + }; + featureFlags?: FeatureFlagType; +}; + +export type FeatureFlagType = { + refreshIntervalMs?: number; + dynamicRefreshIntervalMs?: number; +}; + +export interface ExtendedInitializeParams extends InitializeParams { + initializationOptions?: { + aws?: AwsMetadata; + [key: string]: unknown; + }; +} From fe6c1b8ed6ca58a353859c7ef23a77d1f15576d4 Mon Sep 17 00:00:00 2001 From: Chris Mendoza Date: Thu, 2 Apr 2026 14:20:58 -0400 Subject: [PATCH 2/5] Address comments --- .gitignore | 5 +- package.json | 2 +- tools/long-running/Templates.ts | 36 -------- tools/lsp-client/index.ts | 3 - tools/lsp-client/types.ts | 49 ----------- tools/{lsp-client => lspClient}/LspClient.ts | 84 ++++++++++--------- .../LspConnectionInterface.ts | 0 tools/lspClient/types.ts | 19 +++++ tools/{long-running => stability}/Config.ts | 18 ++-- .../{long-running => stability}/Monitoring.ts | 74 ++++++---------- tools/stability/Templates.ts | 25 ++++++ .../TestOrchestrator.ts | 58 +++++++------ .../runStabilityTest.ts} | 2 +- .../testers/CompletionTester.ts | 6 +- .../testers/HoverTester.ts | 6 +- .../testers/TesterCommon.ts | 2 +- .../testers/TesterTypes.ts | 2 +- 17 files changed, 165 insertions(+), 226 deletions(-) delete mode 100644 tools/long-running/Templates.ts delete mode 100644 tools/lsp-client/index.ts delete mode 100644 tools/lsp-client/types.ts rename tools/{lsp-client => lspClient}/LspClient.ts (80%) rename tools/{lsp-client => lspClient}/LspConnectionInterface.ts (100%) create mode 100644 tools/lspClient/types.ts rename tools/{long-running => stability}/Config.ts (77%) rename tools/{long-running => stability}/Monitoring.ts (51%) create mode 100644 tools/stability/Templates.ts rename tools/{long-running => stability}/TestOrchestrator.ts (79%) rename tools/{long-running/run-long-running-test.ts => stability/runStabilityTest.ts} (89%) rename tools/{long-running => stability}/testers/CompletionTester.ts (91%) rename tools/{long-running => stability}/testers/HoverTester.ts (94%) rename tools/{long-running => stability}/testers/TesterCommon.ts (93%) rename tools/{long-running => stability}/testers/TesterTypes.ts (95%) diff --git a/.gitignore b/.gitignore index 28bbc37..5f81b35 100644 --- a/.gitignore +++ b/.gitignore @@ -9,9 +9,8 @@ *.node .DS_Store tools/* -!tools/*.ts -!tools/lsp-client/ -!tools/long-running/ +!tools/*/ +!tools/**/*.ts **/.aws-cfn-storage /oss-attribution /tmp-tst diff --git a/package.json b/package.json index 371e07e..5538ea2 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "test:integration": "cross-env NODE_ENV=test vitest run --config vitest.integration.config.ts", "test:unit": "cross-env NODE_ENV=test vitest run --config vitest.unit.config.ts", "test:leaks": "cross-env NODE_ENV=test vitest run --pool=forks --logHeapUsage", - "test:long-running": "tsx tools/long-running/run-long-running-test.ts", + "test:stability": "tsx tools/stability/runStabilityTest.ts", "lint": "eslint --cache --cache-location node_modules/.cache/eslint --max-warnings 0 .", "lint:fix": "npm run lint -- --fix", "build:go:dev": "GOPROXY=direct go build -C cfn-init/cmd -v -o ../../bundle/development/bin/cfn-init", diff --git a/tools/long-running/Templates.ts b/tools/long-running/Templates.ts deleted file mode 100644 index 8e1df73..0000000 --- a/tools/long-running/Templates.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { readFileSync } from 'fs'; -import { join } from 'path'; - -const cache = new Map(); - -const loadTemplate = (name: string): string => { - if (!cache.has(name)) { - const templatePath = join(__dirname, '../../tst/resources/templates', name); - cache.set(name, readFileSync(templatePath, 'utf8')); - } - return cache.get(name)!; -}; - -export interface StandaloneTemplateConfig { - name: string; - content: string; -} - -export const STANDALONE_TEMPLATE_CONFIGS: StandaloneTemplateConfig[] = [ - { - name: 'sample.yaml', - content: loadTemplate('sample_template.yaml'), - }, - { - name: 'simple.yaml', - content: loadTemplate('simple.yaml'), - }, - { - name: 'comprehensive.yaml', - content: loadTemplate('comprehensive.yaml'), - }, - { - name: 'condition-usage.yaml', - content: loadTemplate('condition-usage.yaml'), - }, -]; diff --git a/tools/lsp-client/index.ts b/tools/lsp-client/index.ts deleted file mode 100644 index e45baff..0000000 --- a/tools/lsp-client/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { LspClient } from './LspClient'; -export { LspConnection } from './LspConnectionInterface'; -export { LspClientConfig, ReadinessFlags } from './types'; diff --git a/tools/lsp-client/types.ts b/tools/lsp-client/types.ts deleted file mode 100644 index 9eebcdc..0000000 --- a/tools/lsp-client/types.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { InitializeParams } from 'vscode-languageserver-protocol'; - -export interface LspClientConfig { - serverPath: string; - mode: 'stdio' | 'ipc'; - clientId?: string; - storageDir?: string; - env?: NodeJS.ProcessEnv; - initTimeout?: number; - telemetryEnabled?: boolean; - featureFlags?: FeatureFlagType; - suppressLogLevels?: string[]; -} - -export interface ReadinessFlags { - cfnLint: boolean; - cfnGuard: boolean; -} - -export type ClientInfo = { - name: string; - version: string; -}; - -export type AwsMetadata = { - clientInfo?: { - extension: ClientInfo; - clientId: string; - }; - telemetryEnabled?: boolean; - storageDir?: string; - encryption?: { - key: string; - mode: string; - }; - featureFlags?: FeatureFlagType; -}; - -export type FeatureFlagType = { - refreshIntervalMs?: number; - dynamicRefreshIntervalMs?: number; -}; - -export interface ExtendedInitializeParams extends InitializeParams { - initializationOptions?: { - aws?: AwsMetadata; - [key: string]: unknown; - }; -} diff --git a/tools/lsp-client/LspClient.ts b/tools/lspClient/LspClient.ts similarity index 80% rename from tools/lsp-client/LspClient.ts rename to tools/lspClient/LspClient.ts index 60d83e8..dc1dd80 100644 --- a/tools/lsp-client/LspClient.ts +++ b/tools/lspClient/LspClient.ts @@ -9,32 +9,33 @@ import { IPCMessageWriter, TextDocumentContentChangeEvent, } from 'vscode-languageserver-protocol/node'; -import { randomBytes, randomUUID } from 'crypto'; -import { LspClientConfig, ReadinessFlags, ExtendedInitializeParams } from './types'; +import { randomBytes } from 'crypto'; +import { CompactEncrypt } from 'jose'; +import { LspClientConfig, InitializationFlags } from './types'; +import { ExtendedInitializeParams } from '../../src/server/InitParams'; +import { IamCredentials } from '../../src/auth/AwsLspAuthTypes'; import { LspConnection } from './LspConnectionInterface'; import { WaitFor } from '../../tst/utils/Utils'; /** * Common LSP client for CloudFormation Language Server testing. - * Handles server startup, LSP protocol communication, and readiness detection. + * Handles server startup, LSP protocol communication, and external service initialization detection. */ export class LspClient implements LspConnection { protected serverProcess?: ChildProcess; protected connection?: MessageConnection; - protected readinessFlags: ReadinessFlags = { + protected initialization: InitializationFlags = { cfnLint: false, cfnGuard: false, }; - public readonly clientId: string; public readonly createdAt: number; - protected readonly encryptionKey: Buffer; + private readonly encryptionKey: Buffer; protected isShutdown = false; - protected currentWorkspaceConfig: Record[] = [{}]; - public readyRegions = new Set(); + protected workspaceConfig: Record[] = [{}]; + protected availableRegions = new Set(); constructor(protected config: LspClientConfig) { - this.clientId = config.clientId ?? `lsp-client-${randomUUID()}`; this.createdAt = performance.now(); this.encryptionKey = randomBytes(32); } @@ -53,7 +54,7 @@ export class LspClient implements LspConnection { console.log(`LspClient: Server process spawned with PID: ${this.serverProcess.pid}`); - // 2. Setup output monitoring for readiness detection + // 2. Setup output monitoring for external service initialization detection this.attachOutputListeners(); // 3. Create LSP connection @@ -78,14 +79,14 @@ export class LspClient implements LspConnection { const results = params.items.map((item: any) => { if (item.section === 'aws.cloudformation') { // Return just the CloudFormation config part - const fullConfig = this.currentWorkspaceConfig[0] ?? {}; + const fullConfig = this.workspaceConfig[0] ?? {}; return (fullConfig as any)['aws.cloudformation'] ?? {}; } return {}; }); return results; } - return this.currentWorkspaceConfig; + return this.workspaceConfig; }); this.connection.listen(); @@ -105,18 +106,18 @@ export class LspClient implements LspConnection { private readonly onServerOutput = (data: Buffer) => { const output = data.toString().trim(); - // Readiness detection + // external service initialization detection if (output.includes('cfn-lint version')) { - this.readinessFlags.cfnLint = true; + this.initialization.cfnLint = true; } if (output.includes('Loading rules from')) { - this.readinessFlags.cfnGuard = true; + this.initialization.cfnGuard = true; } // Region-specific schema loading const regionSchemaMatch = output.match(/public schemas downloaded for ([a-z0-9-]+)/); if (regionSchemaMatch) { - this.readyRegions.add(regionSchemaMatch[1]); + this.availableRegions.add(regionSchemaMatch[1]); } // Log filtering @@ -155,28 +156,20 @@ export class LspClient implements LspConnection { completion: { dynamicRegistration: true }, }, }, - clientInfo: { - name: 'CFN LSP Test Client', - version: '1.0.0', - }, + clientInfo: this.config.clientInfo, initializationOptions: { aws: { clientInfo: { - extension: { - name: 'aws.cloudformation.lsp.test', - version: '1.0.0', - }, - clientId: this.clientId, + extension: this.config.extensionInfo, + clientId: this.config.clientId, }, - telemetryEnabled: this.config.telemetryEnabled ?? true, + telemetryEnabled: this.config.telemetryEnabled, storageDir: this.config.storageDir, encryption: { key: this.encryptionKey.toString('base64'), mode: 'JWT', }, - ...(this.config.featureFlags && { - featureFlags: this.config.featureFlags, - }), + featureFlags: this.config.featureFlags, }, }, }; @@ -251,8 +244,8 @@ export class LspClient implements LspConnection { async changeConfiguration(params: { settings: any }): Promise { // Store the new configuration if (params.settings) { - const currentConfig = this.currentWorkspaceConfig[0] ?? {}; - this.currentWorkspaceConfig = [{ ...currentConfig, ...params.settings }]; + const currentConfig = this.workspaceConfig[0] ?? {}; + this.workspaceConfig = [{ ...currentConfig, ...params.settings }]; } // Send the configuration change notification @@ -275,24 +268,37 @@ export class LspClient implements LspConnection { this.connection!.onRequest(method, handler); } - async waitForReadiness(timeoutMs: number = 30_000): Promise { + async waitForExternalServiceInitialization(): Promise { + console.log('Waiting for lint and guard initialization'); + await WaitFor.waitFor( () => { - if (!this.readinessFlags.cfnLint || !this.readinessFlags.cfnGuard) { - throw new Error('Lint and Guard services not ready yet'); + if (!this.initialization.cfnLint || !this.initialization.cfnGuard) { + throw new Error('Lint and Guard services not initialized'); } - console.log('Lint and Guard services are ready'); + console.log('Lint and Guard services are initialized'); }, - timeoutMs, + 30_000, 500, // Check every 500ms ); } - get readiness(): ReadinessFlags { - return { ...this.readinessFlags }; + getAvailableRegions(): ReadonlySet { + return this.availableRegions; + } + + async updateCredentials(credentials: IamCredentials): Promise { + const payload = new TextEncoder().encode(JSON.stringify({ data: credentials })); + const jwt = await new CompactEncrypt(payload) + .setProtectedHeader({ alg: 'dir', enc: 'A256GCM' }) + .encrypt(this.encryptionKey); + + await this.connection!.sendRequest('aws/credentials/iam/update', { + data: jwt, + encrypted: true, + }); } - /** Shutdown the LSP server */ async shutdown(): Promise { if (this.isShutdown) return; this.isShutdown = true; diff --git a/tools/lsp-client/LspConnectionInterface.ts b/tools/lspClient/LspConnectionInterface.ts similarity index 100% rename from tools/lsp-client/LspConnectionInterface.ts rename to tools/lspClient/LspConnectionInterface.ts diff --git a/tools/lspClient/types.ts b/tools/lspClient/types.ts new file mode 100644 index 0000000..55a19b5 --- /dev/null +++ b/tools/lspClient/types.ts @@ -0,0 +1,19 @@ +import { ClientInfo, AwsMetadata } from '../../src/server/InitParams'; + +export interface LspClientConfig { + serverPath: string; + mode: 'stdio' | 'ipc'; + clientId: string; + clientInfo: ClientInfo; + extensionInfo: ClientInfo; + telemetryEnabled: boolean; + featureFlags: NonNullable; + storageDir?: string; + env?: NodeJS.ProcessEnv; + suppressLogLevels?: string[]; +} + +export interface InitializationFlags { + cfnLint: boolean; + cfnGuard: boolean; +} diff --git a/tools/long-running/Config.ts b/tools/stability/Config.ts similarity index 77% rename from tools/long-running/Config.ts rename to tools/stability/Config.ts index e18cbc6..7511f4b 100644 --- a/tools/long-running/Config.ts +++ b/tools/stability/Config.ts @@ -1,12 +1,12 @@ -export interface StandaloneConfig { +export interface Config { duration: string; maxRetries: number; responseTimeout: number; - standalonePath: string; + path: string; } -function parseSimpleArgs(): Partial { - const args: Partial = {}; +function parseSimpleArgs(): Partial { + const args: Partial = {}; for (let i = 0; i < process.argv.length; i++) { const arg = process.argv[i]; @@ -21,8 +21,8 @@ function parseSimpleArgs(): Partial { } else if (arg === '--response-timeout' && nextArg) { args.responseTimeout = Number.parseInt(nextArg); i++; - } else if (arg === '--standalone-path' && nextArg) { - args.standalonePath = nextArg; + } else if (arg === '--path' && nextArg) { + args.path = nextArg; i++; } } @@ -30,13 +30,13 @@ function parseSimpleArgs(): Partial { return args; } -export function parseStandaloneConfig(): StandaloneConfig { +export function parseConfig(): Config { // Start with environment variables (npm script support) const envConfig = { - duration: process.env.LONG_RUNNING_DURATION ?? '4h', + duration: process.env.STABILITY_TEST_DURATION ?? '4h', maxRetries: Number.parseInt(process.env.MAX_RETRIES ?? '3'), responseTimeout: Number.parseInt(process.env.RESPONSE_TIMEOUT ?? '5000'), - standalonePath: process.env.STANDALONE_PATH ?? './cfn-lsp-server-standalone.js', + path: process.env.STANDALONE_PATH ?? './cfn-lsp-server-standalone.js', }; // Override with command line arguments if provided diff --git a/tools/long-running/Monitoring.ts b/tools/stability/Monitoring.ts similarity index 51% rename from tools/long-running/Monitoring.ts rename to tools/stability/Monitoring.ts index fe72cf2..1bb0a4f 100644 --- a/tools/long-running/Monitoring.ts +++ b/tools/stability/Monitoring.ts @@ -1,44 +1,40 @@ import { OperationType, getTesterConfig } from './testers/TesterTypes'; -export type StandaloneTestMetrics = { - operationsAttempted: number; - operationsFailed: number; +export type TestMetrics = { + operations: number; averageDuration: number | null; minDuration: number | null; maxDuration: number | null; lastDuration: number | null; + durations: number[]; }; -const createEmptyMetrics = (): StandaloneTestMetrics => ({ - operationsAttempted: 0, - operationsFailed: 0, +const createEmptyMetrics = (): TestMetrics => ({ + operations: 0, averageDuration: null, minDuration: null, maxDuration: null, lastDuration: null, + durations: [], }); -const metrics: Record = {} as Record; +const metrics: Record = {} as Record; for (const operationType of Object.values(OperationType)) { metrics[operationType] = createEmptyMetrics(); } -export function recordOperation(duration: number, success: boolean, operationType: OperationType): void { +export function recordOperation(duration: number, operationType: OperationType): void { const metric = metrics[operationType]; - metric.operationsAttempted++; - - if (success) { - const successfulOps = metric.operationsAttempted - metric.operationsFailed; - metric.averageDuration = - metric.averageDuration === null - ? duration - : (metric.averageDuration * (successfulOps - 1) + duration) / successfulOps; - metric.minDuration = metric.minDuration === null ? duration : Math.min(metric.minDuration, duration); - metric.maxDuration = metric.maxDuration === null ? duration : Math.max(metric.maxDuration, duration); - metric.lastDuration = duration; - } else { - metric.operationsFailed++; - } + metric.operations++; + + metric.averageDuration = + metric.averageDuration === null + ? duration + : (metric.averageDuration * (metric.operations - 1) + duration) / metric.operations; + metric.minDuration = metric.minDuration === null ? duration : Math.min(metric.minDuration, duration); + metric.maxDuration = metric.maxDuration === null ? duration : Math.max(metric.maxDuration, duration); + metric.lastDuration = duration; + metric.durations.push(duration); } let startTime: number; @@ -48,51 +44,35 @@ export function initializeMonitoring(): void { } export function logProgress(): void { - const totalOps = Object.values(metrics).reduce((sum, m) => sum + m.operationsAttempted, 0); - const totalFailed = Object.values(metrics).reduce((sum, m) => sum + m.operationsFailed, 0); + const totalOps = Object.values(metrics).reduce((sum, m) => sum + m.operations, 0); const elapsed = Date.now() - startTime; const elapsedMinutes = Math.round(elapsed / 60_000); console.log('Progress Report'); console.log(` Runtime: ${elapsedMinutes} minutes`); console.log(` Total Operations: ${totalOps}`); - console.log(` Successful: ${totalOps - totalFailed}`); - console.log(` Failed: ${totalFailed}`); - console.log(` Success Rate: ${totalOps > 0 ? (((totalOps - totalFailed) / totalOps) * 100).toFixed(1) : 0}%`); + console.log(` Operations: ${totalOps}`); // Per-operation breakdown - for (const [operationType, metric] of Object.entries(metrics) as [OperationType, StandaloneTestMetrics][]) { - if (metric.operationsAttempted > 0) { - const successRate = ( - ((metric.operationsAttempted - metric.operationsFailed) / metric.operationsAttempted) * - 100 - ).toFixed(1); - console.log( - ` ${operationType}: ${metric.operationsAttempted} ops, ${metric.operationsFailed} failed (${successRate}% success)`, - ); + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, TestMetrics][]) { + if (metric.operations > 0) { + console.log(` ${operationType}: ${metric.operations} operations`); } } } export function generateFinalReport(testStartTime: number): void { const runtime = Date.now() - testStartTime; - const totalOps = Object.values(metrics).reduce((sum, m) => sum + m.operationsAttempted, 0); - const totalFailed = Object.values(metrics).reduce((sum, m) => sum + m.operationsFailed, 0); console.log('Final Test Report'); console.log('='.repeat(50)); console.log(`Runtime: ${Math.round(runtime / 1000 / 60)} minutes`); - console.log(`Total Operations: ${totalOps}`); - console.log(`Successful: ${totalOps - totalFailed}`); - console.log(`Failed: ${totalFailed}`); - console.log(`Success Rate: ${totalOps > 0 ? (((totalOps - totalFailed) / totalOps) * 100).toFixed(2) : 0}%`); // Per-operation breakdown - for (const [operationType, metric] of Object.entries(metrics) as [OperationType, StandaloneTestMetrics][]) { - if (metric.operationsAttempted > 0) { + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, TestMetrics][]) { + if (metric.operations > 0) { console.log(`${operationType}:`); - console.log(` Operations: ${metric.operationsAttempted}`); - console.log(` Failed: ${metric.operationsFailed}`); + console.log(` Operations: ${metric.operations}`); console.log(` Avg Duration: ${metric.averageDuration?.toFixed(2) ?? 'N/A'}ms`); console.log(` Max Duration: ${metric.maxDuration?.toFixed(2) ?? 'N/A'}ms`); console.log(` Min Duration: ${metric.minDuration?.toFixed(2) ?? 'N/A'}ms`); @@ -102,7 +82,7 @@ export function generateFinalReport(testStartTime: number): void { } export function checkPerformanceDegradation(): void { - for (const [operationType, metric] of Object.entries(metrics) as [OperationType, StandaloneTestMetrics][]) { + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, TestMetrics][]) { const config = getTesterConfig(operationType); if (metric.averageDuration !== null && metric.averageDuration > config.avgDurationLimitMs) { diff --git a/tools/stability/Templates.ts b/tools/stability/Templates.ts new file mode 100644 index 0000000..fc001a6 --- /dev/null +++ b/tools/stability/Templates.ts @@ -0,0 +1,25 @@ +import { Templates } from '../../tst/utils/TemplateUtils'; + +export interface TemplateConfig { + name: string; + content: string; +} + +export const TEMPLATE_CONFIGS: TemplateConfig[] = [ + { + name: 'sample.yaml', + content: Templates.sample.yaml.contents, + }, + { + name: 'simple.yaml', + content: Templates.simple.yaml.contents, + }, + { + name: 'comprehensive.yaml', + content: Templates.comprehensive.yaml.contents, + }, + { + name: 'condition-usage.yaml', + content: Templates.conditionUsage.yaml.contents, + }, +]; diff --git a/tools/long-running/TestOrchestrator.ts b/tools/stability/TestOrchestrator.ts similarity index 79% rename from tools/long-running/TestOrchestrator.ts rename to tools/stability/TestOrchestrator.ts index ab3ff01..35051bb 100644 --- a/tools/long-running/TestOrchestrator.ts +++ b/tools/stability/TestOrchestrator.ts @@ -1,43 +1,52 @@ -import { LspClient } from '../lsp-client/LspClient'; -import { parseStandaloneConfig, parseDuration } from './Config'; +import { LspClient } from '../lspClient/LspClient'; +import { parseConfig, parseDuration } from './Config'; import { initializeMonitoring, logProgress, checkPerformanceDegradation } from './Monitoring'; import { HoverTester } from './testers/HoverTester'; import { CompletionTester } from './testers/CompletionTester'; -import { STANDALONE_TEMPLATE_CONFIGS } from './Templates'; +import { TEMPLATE_CONFIGS } from './Templates'; import { AwsRegion } from '../../src/utils/Region'; import { WaitFor } from '../../tst/utils/Utils'; import { existsSync } from 'fs'; export class TestOrchestrator { private client!: LspClient; - private readonly config = parseStandaloneConfig(); + private readonly config = parseConfig(); private startTime!: number; private endTime!: number; private hoverTester!: HoverTester; private completionTester!: CompletionTester; - private readonly templates = STANDALONE_TEMPLATE_CONFIGS; + private readonly templates = TEMPLATE_CONFIGS; private readonly testRegions = Object.values(AwsRegion); async initialize(): Promise { - console.log('Starting CloudFormation Language Server Standalone Long-Running Tests'); + console.log('Starting CloudFormation Language Server Long-Running Tests'); console.log(`Duration: ${this.config.duration}`); console.log(`Max retries: ${this.config.maxRetries}`); console.log(`Response timeout: ${this.config.responseTimeout}ms`); - console.log(`Standalone path: ${this.config.standalonePath}`); + console.log(`Standalone path: ${this.config.path}`); // Verify standalone bundle exists - if (!existsSync(this.config.standalonePath)) { - throw new Error(`Standalone bundle not found at: ${this.config.standalonePath}`); + if (!existsSync(this.config.path)) { + throw new Error(`Standalone bundle not found at: ${this.config.path}`); } // Initialize LSP client this.client = new LspClient({ - serverPath: this.config.standalonePath, + serverPath: this.config.path, mode: 'ipc', - clientId: 'standalone-long-running-test', + clientId: 'stability-test', + clientInfo: { + name: 'CFN LSP Stability Test', + version: '1.0.0', + }, + extensionInfo: { + name: 'aws.cloudformation.lsp.stability-test', + version: '1.0.0', + }, telemetryEnabled: false, + featureFlags: {}, }); await this.client.initialize(); @@ -49,8 +58,7 @@ export class TestOrchestrator { console.log(`Loaded ${this.templates.length} templates`); - // Wait for full system readiness before loading schemas - await this.waitForSystemReadiness(); + await this.client.waitForExternalServiceInitialization(); await this.loadAllRegionSchemas(); @@ -58,12 +66,6 @@ export class TestOrchestrator { console.log('Initialization complete'); } - private async waitForSystemReadiness(): Promise { - console.log('Waiting for core services readiness'); - - await this.client.waitForReadiness(); - } - async runTests(): Promise { console.log('Starting test execution phase'); @@ -73,7 +75,6 @@ export class TestOrchestrator { let cycleCount = 0; let successCount = 0; - let errorCount = 0; let lastProgressLog = Date.now(); const progressInterval = 5 * 60 * 1000; // 5 minutes @@ -91,13 +92,10 @@ export class TestOrchestrator { lastProgressLog = Date.now(); } } catch (error) { - errorCount++; console.error(`Test cycle ${cycleCount} failed:`, error); - // Fail fast if too many errors - if (errorCount > 10) { - throw new Error(`Too many errors (${errorCount}), failing fast`); - } + // Fail fast - throw immediately on any error + throw new Error(`Long-running test failed on cycle ${cycleCount}: ${error}`); } // Brief pause between cycles @@ -105,7 +103,7 @@ export class TestOrchestrator { } console.log(`Test execution completed after ${cycleCount} cycles`); - console.log(`Results: ${successCount} success, ${errorCount} errors`); + console.log(`Results: ${successCount} success, 0 errors`); } async cleanup(): Promise { @@ -143,8 +141,8 @@ export class TestOrchestrator { private async loadAllRegionSchemas(): Promise { // Check what regions are already available from LspClient - const alreadyAvailable = [...this.client.readyRegions]; - const unavailableRegions = this.testRegions.filter((region) => !this.client.readyRegions.has(region)); + const alreadyAvailable = [...this.client.getAvailableRegions()]; + const unavailableRegions = this.testRegions.filter((region) => !this.client.getAvailableRegions().has(region)); console.log(`Schema status: ${alreadyAvailable.length} available, ${unavailableRegions.length} unavailable`); console.log(`Available region schemas: ${alreadyAvailable.join(', ')}`); @@ -165,8 +163,8 @@ export class TestOrchestrator { try { await WaitFor.waitFor( () => { - if (!this.client.readyRegions.has(region)) { - throw new Error(`Region ${region} schemas not ready yet`); + if (!this.client.getAvailableRegions().has(region)) { + throw new Error(`Region ${region} schemas not loaded yet`); } }, 30_000, // 30 second timeout diff --git a/tools/long-running/run-long-running-test.ts b/tools/stability/runStabilityTest.ts similarity index 89% rename from tools/long-running/run-long-running-test.ts rename to tools/stability/runStabilityTest.ts index 9b3b988..c3ba3e4 100644 --- a/tools/long-running/run-long-running-test.ts +++ b/tools/stability/runStabilityTest.ts @@ -10,7 +10,7 @@ async function main(): Promise { generateFinalReport(Date.now()); } catch (error) { generateFinalReport(Date.now()); - console.error('Standalone test failed:', error); + console.error('Test failed:', error); throw error; } finally { await orchestrator.cleanup(); diff --git a/tools/long-running/testers/CompletionTester.ts b/tools/stability/testers/CompletionTester.ts similarity index 91% rename from tools/long-running/testers/CompletionTester.ts rename to tools/stability/testers/CompletionTester.ts index f7180d4..6b199a3 100644 --- a/tools/long-running/testers/CompletionTester.ts +++ b/tools/stability/testers/CompletionTester.ts @@ -1,8 +1,8 @@ -import { LspClient } from '../../lsp-client/LspClient'; -import { StandaloneTester, OperationType } from './TesterTypes'; +import { LspClient } from '../../lspClient/LspClient'; +import { OperationTester, OperationType } from './TesterTypes'; import { retryOperationWithPerformance } from './TesterCommon'; -export class CompletionTester implements StandaloneTester { +export class CompletionTester implements OperationTester { constructor(private readonly client: LspClient) {} private validateCompletionItems(result: any, requiredLabels: string[], context: string): void { diff --git a/tools/long-running/testers/HoverTester.ts b/tools/stability/testers/HoverTester.ts similarity index 94% rename from tools/long-running/testers/HoverTester.ts rename to tools/stability/testers/HoverTester.ts index 2f0d382..ce75259 100644 --- a/tools/long-running/testers/HoverTester.ts +++ b/tools/stability/testers/HoverTester.ts @@ -1,8 +1,8 @@ -import { LspClient } from '../../lsp-client/LspClient'; -import { StandaloneTester, OperationType } from './TesterTypes'; +import { LspClient } from '../../lspClient/LspClient'; +import { OperationTester, OperationType } from './TesterTypes'; import { retryOperationWithPerformance } from './TesterCommon'; -export class HoverTester implements StandaloneTester { +export class HoverTester implements OperationTester { constructor(private readonly client: LspClient) {} private extractHoverContent(hoverResult: any): string { diff --git a/tools/long-running/testers/TesterCommon.ts b/tools/stability/testers/TesterCommon.ts similarity index 93% rename from tools/long-running/testers/TesterCommon.ts rename to tools/stability/testers/TesterCommon.ts index c73beba..3faab00 100644 --- a/tools/long-running/testers/TesterCommon.ts +++ b/tools/stability/testers/TesterCommon.ts @@ -23,5 +23,5 @@ export async function retryOperationWithPerformance( RETRY_INTERVAL_MS, ); - recordOperation(responseTime, true, operationType); + recordOperation(responseTime, operationType); } diff --git a/tools/long-running/testers/TesterTypes.ts b/tools/stability/testers/TesterTypes.ts similarity index 95% rename from tools/long-running/testers/TesterTypes.ts rename to tools/stability/testers/TesterTypes.ts index f3be2d3..ab5d1f1 100644 --- a/tools/long-running/testers/TesterTypes.ts +++ b/tools/stability/testers/TesterTypes.ts @@ -3,7 +3,7 @@ export enum OperationType { COMPLETION = 'completion', } -export interface StandaloneTester { +export interface OperationTester { testAllScenarios(uri: string): Promise; } From 037256e951762cacc270ce97b86b38312c8b7c5b Mon Sep 17 00:00:00 2001 From: Chris Mendoza Date: Fri, 3 Apr 2026 12:39:21 -0400 Subject: [PATCH 3/5] Modify LspConnection inteface --- package.json | 2 +- tools/lspClient/LspClient.ts | 3 +-- tools/lspClient/LspConnection.ts | 28 +++++++++++++++++++++++ tools/lspClient/LspConnectionInterface.ts | 8 ------- tools/lspClient/types.ts | 19 --------------- 5 files changed, 30 insertions(+), 30 deletions(-) create mode 100644 tools/lspClient/LspConnection.ts delete mode 100644 tools/lspClient/LspConnectionInterface.ts delete mode 100644 tools/lspClient/types.ts diff --git a/package.json b/package.json index 5538ea2..b339f75 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "test:integration": "cross-env NODE_ENV=test vitest run --config vitest.integration.config.ts", "test:unit": "cross-env NODE_ENV=test vitest run --config vitest.unit.config.ts", "test:leaks": "cross-env NODE_ENV=test vitest run --pool=forks --logHeapUsage", - "test:stability": "tsx tools/stability/runStabilityTest.ts", + "test:stability": "cross-env NODE_ENV=test tsx tools/stability/runStabilityTest.ts", "lint": "eslint --cache --cache-location node_modules/.cache/eslint --max-warnings 0 .", "lint:fix": "npm run lint -- --fix", "build:go:dev": "GOPROXY=direct go build -C cfn-init/cmd -v -o ../../bundle/development/bin/cfn-init", diff --git a/tools/lspClient/LspClient.ts b/tools/lspClient/LspClient.ts index dc1dd80..33cf744 100644 --- a/tools/lspClient/LspClient.ts +++ b/tools/lspClient/LspClient.ts @@ -11,10 +11,9 @@ import { } from 'vscode-languageserver-protocol/node'; import { randomBytes } from 'crypto'; import { CompactEncrypt } from 'jose'; -import { LspClientConfig, InitializationFlags } from './types'; +import { LspClientConfig, LspConnection, InitializationFlags } from './LspConnection'; import { ExtendedInitializeParams } from '../../src/server/InitParams'; import { IamCredentials } from '../../src/auth/AwsLspAuthTypes'; -import { LspConnection } from './LspConnectionInterface'; import { WaitFor } from '../../tst/utils/Utils'; /** diff --git a/tools/lspClient/LspConnection.ts b/tools/lspClient/LspConnection.ts new file mode 100644 index 0000000..42e7fae --- /dev/null +++ b/tools/lspClient/LspConnection.ts @@ -0,0 +1,28 @@ +import { ClientInfo, AwsMetadata } from '../../src/server/InitParams'; + +export interface LspConnection { + initialize(): Promise; + sendRequest(method: string, params: any): Promise; + sendNotification(method: string, params: any): Promise; + onNotification(method: string, handler: (params: any) => void): void; + onRequest(method: string, handler: (params: any) => any): void; + shutdown(): Promise; +} + +export type LspClientConfig = { + serverPath: string; + mode: 'stdio' | 'ipc'; + clientId: string; + clientInfo: ClientInfo; + extensionInfo: ClientInfo; + telemetryEnabled: boolean; + featureFlags: NonNullable; + storageDir?: string; + env?: NodeJS.ProcessEnv; + suppressLogLevels?: string[]; +}; + +export type InitializationFlags = { + cfnLint: boolean; + cfnGuard: boolean; +}; diff --git a/tools/lspClient/LspConnectionInterface.ts b/tools/lspClient/LspConnectionInterface.ts deleted file mode 100644 index cdacbf6..0000000 --- a/tools/lspClient/LspConnectionInterface.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface LspConnection { - initialize(): Promise; - sendRequest(method: string, params: any): Promise; - sendNotification(method: string, params: any): Promise; - onNotification(method: string, handler: (params: any) => void): void; - onRequest(method: string, handler: (params: any) => any): void; - shutdown(): Promise; -} diff --git a/tools/lspClient/types.ts b/tools/lspClient/types.ts deleted file mode 100644 index 55a19b5..0000000 --- a/tools/lspClient/types.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { ClientInfo, AwsMetadata } from '../../src/server/InitParams'; - -export interface LspClientConfig { - serverPath: string; - mode: 'stdio' | 'ipc'; - clientId: string; - clientInfo: ClientInfo; - extensionInfo: ClientInfo; - telemetryEnabled: boolean; - featureFlags: NonNullable; - storageDir?: string; - env?: NodeJS.ProcessEnv; - suppressLogLevels?: string[]; -} - -export interface InitializationFlags { - cfnLint: boolean; - cfnGuard: boolean; -} From 93752210a07621e3fc1ba73791e311c1b72321a4 Mon Sep 17 00:00:00 2001 From: Chris Mendoza Date: Wed, 22 Apr 2026 16:30:01 -0400 Subject: [PATCH 4/5] Use system status handler --- .github/workflows/post-beta-long-running.yml | 88 ++++++++++++++++++++ tools/lspClient/LspClient.ts | 42 ++++------ tools/lspClient/LspConnection.ts | 5 -- tools/stability/TestOrchestrator.ts | 88 +++++++++++++------- 4 files changed, 159 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/post-beta-long-running.yml diff --git a/.github/workflows/post-beta-long-running.yml b/.github/workflows/post-beta-long-running.yml new file mode 100644 index 0000000..ff8dbfb --- /dev/null +++ b/.github/workflows/post-beta-long-running.yml @@ -0,0 +1,88 @@ +name: Post-Beta Long Running Tests +run-name: Long Running Tests for ${{ github.event.release.tag_name }} + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + check-beta-release: + runs-on: ubuntu-latest + outputs: + is-beta: ${{ steps.check.outputs.is-beta }} + steps: + - name: Check if beta release + id: check + run: | + TAG="${{ github.event.release.tag_name }}" + if [[ "$TAG" =~ -beta$ ]]; then + echo "is-beta=true" >> $GITHUB_OUTPUT + else + echo "is-beta=false" >> $GITHUB_OUTPUT + fi + + long-running-tests: + needs: check-beta-release + if: needs.check-beta-release.outputs.is-beta == 'true' + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + node-version: [18, 20, 22] + runs-on: ${{ matrix.os }} + timeout-minutes: 360 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.release.tag_name }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Download release standalone + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG="${{ github.event.release.tag_name }}" + case "${{ matrix.os }}" in + ubuntu-latest) PLATFORM="linux" ;; + windows-latest) PLATFORM="win32" ;; + macos-latest) PLATFORM="darwin" ;; + esac + PATTERN="*${PLATFORM}*x64*node${{ matrix.node-version }}*.zip" + gh release download "$TAG" --pattern "$PATTERN" + + - name: Extract standalone bundle + run: | + if [[ "${{ matrix.os }}" == "windows-latest" ]]; then + unzip *.zip + else + unzip *.zip + chmod +x cfn-lsp-server-standalone.js + fi + + - name: Test standalone bundle + run: node cfn-lsp-server-standalone.js --version + + - name: Install dependencies + run: npm ci + + - name: Run long-running stability tests + env: + STABILITY_TEST_DURATION: 4h + STANDALONE_PATH: ./cfn-lsp-server-standalone.js + run: npm run test:stability + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: long-running-test-results-${{ matrix.os }} + path: | + test-results.json + test-logs.txt + if-no-files-found: ignore diff --git a/tools/lspClient/LspClient.ts b/tools/lspClient/LspClient.ts index 33cf744..b43ec5d 100644 --- a/tools/lspClient/LspClient.ts +++ b/tools/lspClient/LspClient.ts @@ -11,9 +11,10 @@ import { } from 'vscode-languageserver-protocol/node'; import { randomBytes } from 'crypto'; import { CompactEncrypt } from 'jose'; -import { LspClientConfig, LspConnection, InitializationFlags } from './LspConnection'; +import { LspClientConfig, LspConnection } from './LspConnection'; import { ExtendedInitializeParams } from '../../src/server/InitParams'; import { IamCredentials } from '../../src/auth/AwsLspAuthTypes'; +import { GetSystemStatusResponse } from '../../src/protocol/LspSystemHandlers'; import { WaitFor } from '../../tst/utils/Utils'; /** @@ -23,16 +24,11 @@ import { WaitFor } from '../../tst/utils/Utils'; export class LspClient implements LspConnection { protected serverProcess?: ChildProcess; protected connection?: MessageConnection; - protected initialization: InitializationFlags = { - cfnLint: false, - cfnGuard: false, - }; public readonly createdAt: number; private readonly encryptionKey: Buffer; protected isShutdown = false; protected workspaceConfig: Record[] = [{}]; - protected availableRegions = new Set(); constructor(protected config: LspClientConfig) { this.createdAt = performance.now(); @@ -105,21 +101,7 @@ export class LspClient implements LspConnection { private readonly onServerOutput = (data: Buffer) => { const output = data.toString().trim(); - // external service initialization detection - if (output.includes('cfn-lint version')) { - this.initialization.cfnLint = true; - } - if (output.includes('Loading rules from')) { - this.initialization.cfnGuard = true; - } - - // Region-specific schema loading - const regionSchemaMatch = output.match(/public schemas downloaded for ([a-z0-9-]+)/); - if (regionSchemaMatch) { - this.availableRegions.add(regionSchemaMatch[1]); - } - - // Log filtering + // Log filtering - keep for debugging const suppressLevels = this.config.suppressLogLevels ?? ['INFO', 'DEBUG']; const shouldSuppress = suppressLevels.some((level) => output.includes(`${level}:`)); @@ -268,11 +250,15 @@ export class LspClient implements LspConnection { } async waitForExternalServiceInitialization(): Promise { - console.log('Waiting for lint and guard initialization'); + console.log('Waiting for lint and guard initialization via SystemHandler...'); await WaitFor.waitFor( - () => { - if (!this.initialization.cfnLint || !this.initialization.cfnGuard) { + async () => { + const status = await this.getSystemStatus(); + console.log( + `Service status: cfnLint=${status.cfnLintReady.ready}, cfnGuard=${status.cfnGuardReady.ready}`, + ); + if (!status.cfnLintReady.ready || !status.cfnGuardReady.ready) { throw new Error('Lint and Guard services not initialized'); } console.log('Lint and Guard services are initialized'); @@ -282,10 +268,6 @@ export class LspClient implements LspConnection { ); } - getAvailableRegions(): ReadonlySet { - return this.availableRegions; - } - async updateCredentials(credentials: IamCredentials): Promise { const payload = new TextEncoder().encode(JSON.stringify({ data: credentials })); const jwt = await new CompactEncrypt(payload) @@ -298,6 +280,10 @@ export class LspClient implements LspConnection { }); } + async getSystemStatus(): Promise { + return await this.sendRequest('aws/system/status', {}); + } + async shutdown(): Promise { if (this.isShutdown) return; this.isShutdown = true; diff --git a/tools/lspClient/LspConnection.ts b/tools/lspClient/LspConnection.ts index 42e7fae..a1cd6ff 100644 --- a/tools/lspClient/LspConnection.ts +++ b/tools/lspClient/LspConnection.ts @@ -21,8 +21,3 @@ export type LspClientConfig = { env?: NodeJS.ProcessEnv; suppressLogLevels?: string[]; }; - -export type InitializationFlags = { - cfnLint: boolean; - cfnGuard: boolean; -}; diff --git a/tools/stability/TestOrchestrator.ts b/tools/stability/TestOrchestrator.ts index 35051bb..f06e8bd 100644 --- a/tools/stability/TestOrchestrator.ts +++ b/tools/stability/TestOrchestrator.ts @@ -18,7 +18,9 @@ export class TestOrchestrator { private readonly templates = TEMPLATE_CONFIGS; - private readonly testRegions = Object.values(AwsRegion); + private readonly testRegions = Object.values(AwsRegion).filter( + (region) => region !== AwsRegion.ME_SOUTH_1 && region !== AwsRegion.ME_CENTRAL_1, + ); async initialize(): Promise { console.log('Starting CloudFormation Language Server Long-Running Tests'); @@ -58,7 +60,24 @@ export class TestOrchestrator { console.log(`Loaded ${this.templates.length} templates`); - await this.client.waitForExternalServiceInitialization(); + // Wait for all system components to be ready + console.log('Waiting for system components to be ready...'); + await WaitFor.waitFor( + async () => { + const status = await this.client.getSystemStatus(); + if ( + !status.settingsReady.ready || + !status.schemasReady.ready || + !status.cfnLintReady.ready || + !status.cfnGuardReady.ready + ) { + throw new Error('System not ready'); + } + }, + 30_000, + 1000, + ); + console.log('All system components ready'); await this.loadAllRegionSchemas(); @@ -140,42 +159,32 @@ export class TestOrchestrator { } private async loadAllRegionSchemas(): Promise { - // Check what regions are already available from LspClient - const alreadyAvailable = [...this.client.getAvailableRegions()]; - const unavailableRegions = this.testRegions.filter((region) => !this.client.getAvailableRegions().has(region)); - - console.log(`Schema status: ${alreadyAvailable.length} available, ${unavailableRegions.length} unavailable`); - console.log(`Available region schemas: ${alreadyAvailable.join(', ')}`); - - if (unavailableRegions.length > 0) { - console.log(`Loading the following region schemas: ${unavailableRegions.join(', ')}`); - } + console.log('Loading schemas for all regions...'); - for (const region of unavailableRegions) { + for (const region of this.testRegions) { await this.switchToRegion(region); - await this.waitForRegionSchemas(region); + + // Wait for schemas to be ready after region switch + try { + await WaitFor.waitFor( + async () => { + const status = await this.client.getSystemStatus(); + if (!status.schemasReady.ready) { + throw new Error(`Schemas not ready for region ${region}`); + } + }, + 30_000, + 200, + ); + } catch (error) { + console.warn(`Failed to load schemas for region ${region}, continuing anyway:`, error); + } } console.log('Regional schema loading complete'); } - private async waitForRegionSchemas(region: string): Promise { - try { - await WaitFor.waitFor( - () => { - if (!this.client.getAvailableRegions().has(region)) { - throw new Error(`Region ${region} schemas not loaded yet`); - } - }, - 30_000, // 30 second timeout - 500, // Check every 500ms - ); - } catch { - console.warn(`Timeout waiting for ${region} schemas, proceeding anyway`); - } - } - - private async switchToRegion(region: string): Promise { + private async switchToRegion(region: AwsRegion): Promise { // Store the new configuration await this.client.changeConfiguration({ settings: { @@ -186,6 +195,23 @@ export class TestOrchestrator { }, }, }); + + // Wait for settings to be applied with correct region + await WaitFor.waitFor( + async () => { + const status = await this.client.getSystemStatus(); + if (!status.settingsReady.ready) { + throw new Error('Settings not ready after region change'); + } + if (status.currentSettings.profile.region !== region) { + throw new Error( + `Region not applied: expected ${region}, got ${status.currentSettings.profile.region}`, + ); + } + }, + 5000, + 100, + ); // Reduced timeout and faster polling } private async validateLsp(uri: string): Promise { From 183a4c0cd6759e357fcbef1dce4a2c602f81845a Mon Sep 17 00:00:00 2001 From: Chris Mendoza Date: Tue, 28 Apr 2026 14:58:43 -0400 Subject: [PATCH 5/5] Linting --- tools/lspClient/LspClient.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/lspClient/LspClient.ts b/tools/lspClient/LspClient.ts index b43ec5d..1681ebd 100644 --- a/tools/lspClient/LspClient.ts +++ b/tools/lspClient/LspClient.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ import { spawn, ChildProcess } from 'child_process'; import { createMessageConnection,