diff --git a/.github/workflows/post-beta-long-running.yml b/.github/workflows/post-beta-long-running.yml new file mode 100644 index 0000000..ff8dbfb --- /dev/null +++ b/.github/workflows/post-beta-long-running.yml @@ -0,0 +1,88 @@ +name: Post-Beta Long Running Tests +run-name: Long Running Tests for ${{ github.event.release.tag_name }} + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + check-beta-release: + runs-on: ubuntu-latest + outputs: + is-beta: ${{ steps.check.outputs.is-beta }} + steps: + - name: Check if beta release + id: check + run: | + TAG="${{ github.event.release.tag_name }}" + if [[ "$TAG" =~ -beta$ ]]; then + echo "is-beta=true" >> $GITHUB_OUTPUT + else + echo "is-beta=false" >> $GITHUB_OUTPUT + fi + + long-running-tests: + needs: check-beta-release + if: needs.check-beta-release.outputs.is-beta == 'true' + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + node-version: [18, 20, 22] + runs-on: ${{ matrix.os }} + timeout-minutes: 360 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.release.tag_name }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Download release standalone + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG="${{ github.event.release.tag_name }}" + case "${{ matrix.os }}" in + ubuntu-latest) PLATFORM="linux" ;; + windows-latest) PLATFORM="win32" ;; + macos-latest) PLATFORM="darwin" ;; + esac + PATTERN="*${PLATFORM}*x64*node${{ matrix.node-version }}*.zip" + gh release download "$TAG" --pattern "$PATTERN" + + - name: Extract standalone bundle + run: | + if [[ "${{ matrix.os }}" == "windows-latest" ]]; then + unzip *.zip + else + unzip *.zip + chmod +x cfn-lsp-server-standalone.js + fi + + - name: Test standalone bundle + run: node cfn-lsp-server-standalone.js --version + + - name: Install dependencies + run: npm ci + + - name: Run long-running stability tests + env: + STABILITY_TEST_DURATION: 4h + STANDALONE_PATH: ./cfn-lsp-server-standalone.js + run: npm run test:stability + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: long-running-test-results-${{ matrix.os }} + path: | + test-results.json + test-logs.txt + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 5b8af9f..5f81b35 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,8 @@ *.node .DS_Store tools/* -!tools/*.ts +!tools/*/ +!tools/**/*.ts **/.aws-cfn-storage /oss-attribution /tmp-tst diff --git a/package.json b/package.json index 3c94edd..b339f75 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "test:integration": "cross-env NODE_ENV=test vitest run --config vitest.integration.config.ts", "test:unit": "cross-env NODE_ENV=test vitest run --config vitest.unit.config.ts", "test:leaks": "cross-env NODE_ENV=test vitest run --pool=forks --logHeapUsage", + "test:stability": "cross-env NODE_ENV=test tsx tools/stability/runStabilityTest.ts", "lint": "eslint --cache --cache-location node_modules/.cache/eslint --max-warnings 0 .", "lint:fix": "npm run lint -- --fix", "build:go:dev": "GOPROXY=direct go build -C cfn-init/cmd -v -o ../../bundle/development/bin/cfn-init", diff --git a/tools/lspClient/LspClient.ts b/tools/lspClient/LspClient.ts new file mode 100644 index 0000000..1681ebd --- /dev/null +++ b/tools/lspClient/LspClient.ts @@ -0,0 +1,303 @@ +import { spawn, ChildProcess } from 'child_process'; +import { + createMessageConnection, + MessageConnection, + StreamMessageReader, + StreamMessageWriter, + IPCMessageReader, + IPCMessageWriter, + TextDocumentContentChangeEvent, +} from 'vscode-languageserver-protocol/node'; +import { randomBytes } from 'crypto'; +import { CompactEncrypt } from 'jose'; +import { LspClientConfig, LspConnection } from './LspConnection'; +import { ExtendedInitializeParams } from '../../src/server/InitParams'; +import { IamCredentials } from '../../src/auth/AwsLspAuthTypes'; +import { GetSystemStatusResponse } from '../../src/protocol/LspSystemHandlers'; +import { WaitFor } from '../../tst/utils/Utils'; + +/** + * Common LSP client for CloudFormation Language Server testing. + * Handles server startup, LSP protocol communication, and external service initialization detection. + */ +export class LspClient implements LspConnection { + protected serverProcess?: ChildProcess; + protected connection?: MessageConnection; + + public readonly createdAt: number; + private readonly encryptionKey: Buffer; + protected isShutdown = false; + protected workspaceConfig: Record[] = [{}]; + + constructor(protected config: LspClientConfig) { + this.createdAt = performance.now(); + this.encryptionKey = randomBytes(32); + } + + async initialize(): Promise { + console.log('LspClient: Starting initialization...'); + + // 1. Start server process + const args = this.config.mode === 'ipc' ? ['--node-ipc'] : ['--stdio']; + console.log(`LspClient: Spawning server with args: node ${this.config.serverPath} ${args.join(' ')}`); + + this.serverProcess = spawn('node', [this.config.serverPath, ...args], { + stdio: this.config.mode === 'ipc' ? ['pipe', 'pipe', 'pipe', 'ipc'] : ['pipe', 'pipe', 'pipe'], + env: { ...process.env, ...this.config.env }, + }); + + console.log(`LspClient: Server process spawned with PID: ${this.serverProcess.pid}`); + + // 2. Setup output monitoring for external service initialization detection + this.attachOutputListeners(); + + // 3. Create LSP connection + console.log('LspClient: Creating LSP connection...'); + const reader = + this.config.mode === 'ipc' + ? new IPCMessageReader(this.serverProcess) + : new StreamMessageReader(this.serverProcess.stdout!); + + const writer = + this.config.mode === 'ipc' + ? new IPCMessageWriter(this.serverProcess) + : new StreamMessageWriter(this.serverProcess.stdin!); + + this.connection = createMessageConnection(reader, writer); + + // Handle workspace/configuration requests from server + + this.connection.onRequest('workspace/configuration', (params: any) => { + // Extract the specific configuration section requested + if (params?.items?.length > 0) { + const results = params.items.map((item: any) => { + if (item.section === 'aws.cloudformation') { + // Return just the CloudFormation config part + const fullConfig = this.workspaceConfig[0] ?? {}; + return (fullConfig as any)['aws.cloudformation'] ?? {}; + } + return {}; + }); + return results; + } + return this.workspaceConfig; + }); + + this.connection.listen(); + console.log('LspClient: LSP connection created and listening'); + + // 4. Perform LSP handshake + console.log('LspClient: Performing LSP handshake...'); + try { + await this.performHandshake(); + console.log('LspClient: LSP handshake completed'); + } catch (error) { + console.error('LspClient: LSP handshake failed:', error); + throw error; + } + } + + private readonly onServerOutput = (data: Buffer) => { + const output = data.toString().trim(); + + // Log filtering - keep for debugging + const suppressLevels = this.config.suppressLogLevels ?? ['INFO', 'DEBUG']; + const shouldSuppress = suppressLevels.some((level) => output.includes(`${level}:`)); + + if (!shouldSuppress) { + console.error(`[LSP Server]: ${output}`); + } + }; + + protected attachOutputListeners(): void { + this.serverProcess!.stdout?.on('data', this.onServerOutput); + this.serverProcess!.stderr?.on('data', this.onServerOutput); + + this.serverProcess!.on('exit', (code, signal) => { + if (signal) { + console.log(`[LSP Server]: Process terminated with signal ${signal}`); + } else { + console.log(`[LSP Server]: Process exited with code ${code}`); + } + }); + + this.serverProcess!.on('error', (error) => { + console.error(`[LSP Server]: Process error:`, error); + }); + } + + protected async performHandshake(): Promise { + const initParams: ExtendedInitializeParams = { + processId: process.pid, + rootUri: 'file:///test/workspace', + capabilities: { + textDocument: { + hover: { dynamicRegistration: true }, + completion: { dynamicRegistration: true }, + }, + }, + clientInfo: this.config.clientInfo, + initializationOptions: { + aws: { + clientInfo: { + extension: this.config.extensionInfo, + clientId: this.config.clientId, + }, + telemetryEnabled: this.config.telemetryEnabled, + storageDir: this.config.storageDir, + encryption: { + key: this.encryptionKey.toString('base64'), + mode: 'JWT', + }, + featureFlags: this.config.featureFlags, + }, + }, + }; + + console.log('LspClient: Sending initialize request...'); + try { + await Promise.race([ + this.connection!.sendRequest('initialize', initParams), + new Promise((_resolve, reject) => setTimeout(() => reject(new Error('Initialize timeout')), 30_000)), + ]); + console.log('LspClient: Initialize request completed'); + + console.log('LspClient: Sending initialized notification'); + await this.connection!.sendNotification('initialized', {}); + console.log('LspClient: Initialized notification sent'); + } catch (error) { + console.error('LspClient: Handshake error:', error); + throw error; + } + } + + async openDocument(uri: string, content: string): Promise { + await this.connection!.sendNotification('textDocument/didOpen', { + textDocument: { + uri, + languageId: 'yaml', + version: 1, + text: content, + }, + }); + } + + async updateDocument( + uri: string, + version: number, + changes: string | TextDocumentContentChangeEvent[], + ): Promise { + const contentChanges = + typeof changes === 'string' + ? [{ text: changes }] // Full replacement + : changes; // Incremental changes + + await this.connection!.sendNotification('textDocument/didChange', { + textDocument: { + uri, + version, + }, + contentChanges, + }); + } + + async closeDocument(uri: string): Promise { + await this.connection!.sendNotification('textDocument/didClose', { + textDocument: { uri }, + }); + } + + async hover(uri: string, line: number, character: number): Promise { + return await this.connection!.sendRequest('textDocument/hover', { + textDocument: { uri }, + position: { line, character }, + }); + } + + async completion(uri: string, line: number, character: number): Promise { + return await this.connection!.sendRequest('textDocument/completion', { + textDocument: { uri }, + position: { line, character }, + }); + } + + async changeConfiguration(params: { settings: any }): Promise { + // Store the new configuration + if (params.settings) { + const currentConfig = this.workspaceConfig[0] ?? {}; + this.workspaceConfig = [{ ...currentConfig, ...params.settings }]; + } + + // Send the configuration change notification + await this.sendNotification('workspace/didChangeConfiguration', params); + } + + async sendRequest(method: string, params: any): Promise { + return await this.connection!.sendRequest(method, params); + } + + async sendNotification(method: string, params: any): Promise { + return await this.connection!.sendNotification(method, params); + } + + onNotification(method: string, handler: (params: any) => void): void { + this.connection!.onNotification(method, handler); + } + + onRequest(method: string, handler: (params: any) => any): void { + this.connection!.onRequest(method, handler); + } + + async waitForExternalServiceInitialization(): Promise { + console.log('Waiting for lint and guard initialization via SystemHandler...'); + + await WaitFor.waitFor( + async () => { + const status = await this.getSystemStatus(); + console.log( + `Service status: cfnLint=${status.cfnLintReady.ready}, cfnGuard=${status.cfnGuardReady.ready}`, + ); + if (!status.cfnLintReady.ready || !status.cfnGuardReady.ready) { + throw new Error('Lint and Guard services not initialized'); + } + console.log('Lint and Guard services are initialized'); + }, + 30_000, + 500, // Check every 500ms + ); + } + + async updateCredentials(credentials: IamCredentials): Promise { + const payload = new TextEncoder().encode(JSON.stringify({ data: credentials })); + const jwt = await new CompactEncrypt(payload) + .setProtectedHeader({ alg: 'dir', enc: 'A256GCM' }) + .encrypt(this.encryptionKey); + + await this.connection!.sendRequest('aws/credentials/iam/update', { + data: jwt, + encrypted: true, + }); + } + + async getSystemStatus(): Promise { + return await this.sendRequest('aws/system/status', {}); + } + + async shutdown(): Promise { + if (this.isShutdown) return; + this.isShutdown = true; + + try { + if (this.connection) { + await this.connection.sendRequest('shutdown', {}); + await this.connection.sendNotification('exit', {}); + } + } catch (e) { + console.warn('Error during LSP shutdown:', e); + } + + if (this.serverProcess) { + this.serverProcess.kill(); + } + } +} diff --git a/tools/lspClient/LspConnection.ts b/tools/lspClient/LspConnection.ts new file mode 100644 index 0000000..a1cd6ff --- /dev/null +++ b/tools/lspClient/LspConnection.ts @@ -0,0 +1,23 @@ +import { ClientInfo, AwsMetadata } from '../../src/server/InitParams'; + +export interface LspConnection { + initialize(): Promise; + sendRequest(method: string, params: any): Promise; + sendNotification(method: string, params: any): Promise; + onNotification(method: string, handler: (params: any) => void): void; + onRequest(method: string, handler: (params: any) => any): void; + shutdown(): Promise; +} + +export type LspClientConfig = { + serverPath: string; + mode: 'stdio' | 'ipc'; + clientId: string; + clientInfo: ClientInfo; + extensionInfo: ClientInfo; + telemetryEnabled: boolean; + featureFlags: NonNullable; + storageDir?: string; + env?: NodeJS.ProcessEnv; + suppressLogLevels?: string[]; +}; diff --git a/tools/stability/Config.ts b/tools/stability/Config.ts new file mode 100644 index 0000000..7511f4b --- /dev/null +++ b/tools/stability/Config.ts @@ -0,0 +1,72 @@ +export interface Config { + duration: string; + maxRetries: number; + responseTimeout: number; + path: string; +} + +function parseSimpleArgs(): Partial { + const args: Partial = {}; + + for (let i = 0; i < process.argv.length; i++) { + const arg = process.argv[i]; + const nextArg = process.argv[i + 1]; + + if ((arg === '--duration' || arg === '-d') && nextArg) { + args.duration = nextArg; + i++; + } else if (arg === '--max-retries' && nextArg) { + args.maxRetries = Number.parseInt(nextArg); + i++; + } else if (arg === '--response-timeout' && nextArg) { + args.responseTimeout = Number.parseInt(nextArg); + i++; + } else if (arg === '--path' && nextArg) { + args.path = nextArg; + i++; + } + } + + return args; +} + +export function parseConfig(): Config { + // Start with environment variables (npm script support) + const envConfig = { + duration: process.env.STABILITY_TEST_DURATION ?? '4h', + maxRetries: Number.parseInt(process.env.MAX_RETRIES ?? '3'), + responseTimeout: Number.parseInt(process.env.RESPONSE_TIMEOUT ?? '5000'), + path: process.env.STANDALONE_PATH ?? './cfn-lsp-server-standalone.js', + }; + + // Override with command line arguments if provided + const cliArgs = parseSimpleArgs(); + + return { + ...envConfig, + ...cliArgs, + }; +} + +export function parseDuration(duration: string): number { + const match = duration.match(/^(\d+)([hms])$/); + if (!match) throw new Error(`Invalid duration format: ${duration}`); + + const value = Number.parseInt(match[1]); + const unit = match[2]; + + switch (unit) { + case 'h': { + return value * 60 * 60 * 1000; + } + case 'm': { + return value * 60 * 1000; + } + case 's': { + return value * 1000; + } + default: { + throw new Error(`Invalid duration unit: ${unit}`); + } + } +} diff --git a/tools/stability/Monitoring.ts b/tools/stability/Monitoring.ts new file mode 100644 index 0000000..1bb0a4f --- /dev/null +++ b/tools/stability/Monitoring.ts @@ -0,0 +1,109 @@ +import { OperationType, getTesterConfig } from './testers/TesterTypes'; + +export type TestMetrics = { + operations: number; + averageDuration: number | null; + minDuration: number | null; + maxDuration: number | null; + lastDuration: number | null; + durations: number[]; +}; + +const createEmptyMetrics = (): TestMetrics => ({ + operations: 0, + averageDuration: null, + minDuration: null, + maxDuration: null, + lastDuration: null, + durations: [], +}); + +const metrics: Record = {} as Record; +for (const operationType of Object.values(OperationType)) { + metrics[operationType] = createEmptyMetrics(); +} + +export function recordOperation(duration: number, operationType: OperationType): void { + const metric = metrics[operationType]; + metric.operations++; + + metric.averageDuration = + metric.averageDuration === null + ? duration + : (metric.averageDuration * (metric.operations - 1) + duration) / metric.operations; + metric.minDuration = metric.minDuration === null ? duration : Math.min(metric.minDuration, duration); + metric.maxDuration = metric.maxDuration === null ? duration : Math.max(metric.maxDuration, duration); + metric.lastDuration = duration; + metric.durations.push(duration); +} + +let startTime: number; + +export function initializeMonitoring(): void { + startTime = Date.now(); +} + +export function logProgress(): void { + const totalOps = Object.values(metrics).reduce((sum, m) => sum + m.operations, 0); + const elapsed = Date.now() - startTime; + const elapsedMinutes = Math.round(elapsed / 60_000); + + console.log('Progress Report'); + console.log(` Runtime: ${elapsedMinutes} minutes`); + console.log(` Total Operations: ${totalOps}`); + console.log(` Operations: ${totalOps}`); + + // Per-operation breakdown + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, TestMetrics][]) { + if (metric.operations > 0) { + console.log(` ${operationType}: ${metric.operations} operations`); + } + } +} + +export function generateFinalReport(testStartTime: number): void { + const runtime = Date.now() - testStartTime; + + console.log('Final Test Report'); + console.log('='.repeat(50)); + console.log(`Runtime: ${Math.round(runtime / 1000 / 60)} minutes`); + + // Per-operation breakdown + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, TestMetrics][]) { + if (metric.operations > 0) { + console.log(`${operationType}:`); + console.log(` Operations: ${metric.operations}`); + console.log(` Avg Duration: ${metric.averageDuration?.toFixed(2) ?? 'N/A'}ms`); + console.log(` Max Duration: ${metric.maxDuration?.toFixed(2) ?? 'N/A'}ms`); + console.log(` Min Duration: ${metric.minDuration?.toFixed(2) ?? 'N/A'}ms`); + } + } + console.log('='.repeat(50)); +} + +export function checkPerformanceDegradation(): void { + for (const [operationType, metric] of Object.entries(metrics) as [OperationType, TestMetrics][]) { + const config = getTesterConfig(operationType); + + if (metric.averageDuration !== null && metric.averageDuration > config.avgDurationLimitMs) { + throw new Error( + `${operationType} average duration ${metric.averageDuration.toFixed(1)}ms exceeds limit ${config.avgDurationLimitMs}ms`, + ); + } + + if (metric.maxDuration !== null && metric.maxDuration > config.maxDurationLimitMs) { + throw new Error( + `${operationType} max duration ${metric.maxDuration.toFixed(1)}ms exceeds limit ${config.maxDurationLimitMs}ms`, + ); + } + } + + // Basic memory check + const memUsage = process.memoryUsage(); + const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024); + + if (heapUsedMB > 2000) { + // 2GB threshold + console.warn(`High memory usage detected: ${heapUsedMB}MB heap used`); + } +} diff --git a/tools/stability/Templates.ts b/tools/stability/Templates.ts new file mode 100644 index 0000000..fc001a6 --- /dev/null +++ b/tools/stability/Templates.ts @@ -0,0 +1,25 @@ +import { Templates } from '../../tst/utils/TemplateUtils'; + +export interface TemplateConfig { + name: string; + content: string; +} + +export const TEMPLATE_CONFIGS: TemplateConfig[] = [ + { + name: 'sample.yaml', + content: Templates.sample.yaml.contents, + }, + { + name: 'simple.yaml', + content: Templates.simple.yaml.contents, + }, + { + name: 'comprehensive.yaml', + content: Templates.comprehensive.yaml.contents, + }, + { + name: 'condition-usage.yaml', + content: Templates.conditionUsage.yaml.contents, + }, +]; diff --git a/tools/stability/TestOrchestrator.ts b/tools/stability/TestOrchestrator.ts new file mode 100644 index 0000000..f06e8bd --- /dev/null +++ b/tools/stability/TestOrchestrator.ts @@ -0,0 +1,221 @@ +import { LspClient } from '../lspClient/LspClient'; +import { parseConfig, parseDuration } from './Config'; +import { initializeMonitoring, logProgress, checkPerformanceDegradation } from './Monitoring'; +import { HoverTester } from './testers/HoverTester'; +import { CompletionTester } from './testers/CompletionTester'; +import { TEMPLATE_CONFIGS } from './Templates'; +import { AwsRegion } from '../../src/utils/Region'; +import { WaitFor } from '../../tst/utils/Utils'; +import { existsSync } from 'fs'; + +export class TestOrchestrator { + private client!: LspClient; + private readonly config = parseConfig(); + private startTime!: number; + private endTime!: number; + private hoverTester!: HoverTester; + private completionTester!: CompletionTester; + + private readonly templates = TEMPLATE_CONFIGS; + + private readonly testRegions = Object.values(AwsRegion).filter( + (region) => region !== AwsRegion.ME_SOUTH_1 && region !== AwsRegion.ME_CENTRAL_1, + ); + + async initialize(): Promise { + console.log('Starting CloudFormation Language Server Long-Running Tests'); + console.log(`Duration: ${this.config.duration}`); + console.log(`Max retries: ${this.config.maxRetries}`); + console.log(`Response timeout: ${this.config.responseTimeout}ms`); + console.log(`Standalone path: ${this.config.path}`); + + // Verify standalone bundle exists + if (!existsSync(this.config.path)) { + throw new Error(`Standalone bundle not found at: ${this.config.path}`); + } + + // Initialize LSP client + this.client = new LspClient({ + serverPath: this.config.path, + mode: 'ipc', + clientId: 'stability-test', + clientInfo: { + name: 'CFN LSP Stability Test', + version: '1.0.0', + }, + extensionInfo: { + name: 'aws.cloudformation.lsp.stability-test', + version: '1.0.0', + }, + telemetryEnabled: false, + featureFlags: {}, + }); + + await this.client.initialize(); + console.log('LSP client initialized'); + + // Initialize testers + this.hoverTester = new HoverTester(this.client); + this.completionTester = new CompletionTester(this.client); + + console.log(`Loaded ${this.templates.length} templates`); + + // Wait for all system components to be ready + console.log('Waiting for system components to be ready...'); + await WaitFor.waitFor( + async () => { + const status = await this.client.getSystemStatus(); + if ( + !status.settingsReady.ready || + !status.schemasReady.ready || + !status.cfnLintReady.ready || + !status.cfnGuardReady.ready + ) { + throw new Error('System not ready'); + } + }, + 30_000, + 1000, + ); + console.log('All system components ready'); + + await this.loadAllRegionSchemas(); + + initializeMonitoring(); + console.log('Initialization complete'); + } + + async runTests(): Promise { + console.log('Starting test execution phase'); + + const durationMs = parseDuration(this.config.duration); + this.startTime = Date.now(); + this.endTime = this.startTime + durationMs; + + let cycleCount = 0; + let successCount = 0; + let lastProgressLog = Date.now(); + const progressInterval = 5 * 60 * 1000; // 5 minutes + + while (Date.now() < this.endTime) { + cycleCount++; + + try { + await this.executeTestCycle(); + successCount++; + + checkPerformanceDegradation(); + + if (Date.now() - lastProgressLog > progressInterval) { + logProgress(); + lastProgressLog = Date.now(); + } + } catch (error) { + console.error(`Test cycle ${cycleCount} failed:`, error); + + // Fail fast - throw immediately on any error + throw new Error(`Long-running test failed on cycle ${cycleCount}: ${error}`); + } + + // Brief pause between cycles + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + console.log(`Test execution completed after ${cycleCount} cycles`); + console.log(`Results: ${successCount} success, 0 errors`); + } + + async cleanup(): Promise { + if (this.client) { + await this.client.shutdown(); + } + } + + private async executeTestCycle(): Promise { + // Test all regions (switch region for each cycle) + for (const region of this.testRegions) { + await this.switchToRegion(region); + + // Test all templates for this region + for (const template of this.templates) { + const uri = `file:///test/${template.name}`; + + try { + await this.client.openDocument(uri, template.content); + + await this.validateLsp(uri); + + // Revert document to original state after tests + await this.client.updateDocument(uri, 6, template.content); + } finally { + try { + await this.client.closeDocument(uri); + } catch (error) { + console.warn(`Failed to close document ${uri}:`, error); + } + } + } + } + } + + private async loadAllRegionSchemas(): Promise { + console.log('Loading schemas for all regions...'); + + for (const region of this.testRegions) { + await this.switchToRegion(region); + + // Wait for schemas to be ready after region switch + try { + await WaitFor.waitFor( + async () => { + const status = await this.client.getSystemStatus(); + if (!status.schemasReady.ready) { + throw new Error(`Schemas not ready for region ${region}`); + } + }, + 30_000, + 200, + ); + } catch (error) { + console.warn(`Failed to load schemas for region ${region}, continuing anyway:`, error); + } + } + + console.log('Regional schema loading complete'); + } + + private async switchToRegion(region: AwsRegion): Promise { + // Store the new configuration + await this.client.changeConfiguration({ + settings: { + 'aws.cloudformation': { + profile: { + region, + }, + }, + }, + }); + + // Wait for settings to be applied with correct region + await WaitFor.waitFor( + async () => { + const status = await this.client.getSystemStatus(); + if (!status.settingsReady.ready) { + throw new Error('Settings not ready after region change'); + } + if (status.currentSettings.profile.region !== region) { + throw new Error( + `Region not applied: expected ${region}, got ${status.currentSettings.profile.region}`, + ); + } + }, + 5000, + 100, + ); // Reduced timeout and faster polling + } + + private async validateLsp(uri: string): Promise { + await this.hoverTester.testAllScenarios(uri); + await this.completionTester.testAllScenarios(uri); + } +} diff --git a/tools/stability/runStabilityTest.ts b/tools/stability/runStabilityTest.ts new file mode 100644 index 0000000..c3ba3e4 --- /dev/null +++ b/tools/stability/runStabilityTest.ts @@ -0,0 +1,20 @@ +import { TestOrchestrator } from './TestOrchestrator'; +import { generateFinalReport } from './Monitoring'; + +async function main(): Promise { + const orchestrator = new TestOrchestrator(); + + try { + await orchestrator.initialize(); + await orchestrator.runTests(); + generateFinalReport(Date.now()); + } catch (error) { + generateFinalReport(Date.now()); + console.error('Test failed:', error); + throw error; + } finally { + await orchestrator.cleanup(); + } +} + +void main(); diff --git a/tools/stability/testers/CompletionTester.ts b/tools/stability/testers/CompletionTester.ts new file mode 100644 index 0000000..6b199a3 --- /dev/null +++ b/tools/stability/testers/CompletionTester.ts @@ -0,0 +1,55 @@ +import { LspClient } from '../../lspClient/LspClient'; +import { OperationTester, OperationType } from './TesterTypes'; +import { retryOperationWithPerformance } from './TesterCommon'; + +export class CompletionTester implements OperationTester { + constructor(private readonly client: LspClient) {} + + private validateCompletionItems(result: any, requiredLabels: string[], context: string): void { + if (!result?.items || !Array.isArray(result.items) || result.items.length === 0) { + throw new Error(`${context} returned no items`); + } + + const labels = new Set(result.items.map((item: any) => item.label as string)); + for (const required of requiredLabels) { + if (!labels.has(required)) { + throw new Error(`${context} missing ${required}`); + } + } + } + + async testAllScenarios(uri: string): Promise { + // Test 1: Top-level completion using full document update + const basicTemplate = `AWSTemplateFormatVersion: '2010-09-09' +`; + + await this.client.updateDocument(uri, 4, basicTemplate); + + await retryOperationWithPerformance( + () => this.client.completion(uri, 1, 0), + (result: any) => this.validateCompletionItems(result, ['Resources', 'Parameters'], 'Top-level completion'), + OperationType.COMPLETION, + ); + + // Test 2: Property completion in Resources using incremental update + const resourceSection = ` +Resources: + MyBucket: + Type: AWS::S3::Bucket + Properties: + `; + + await this.client.updateDocument(uri, 5, [ + { + range: { start: { line: 1, character: 0 }, end: { line: 1, character: 0 } }, + text: resourceSection, + }, + ]); + + await retryOperationWithPerformance( + () => this.client.completion(uri, 6, 6), + (result: any) => this.validateCompletionItems(result, ['BucketName', 'Tags'], 'S3 bucket completion'), + OperationType.COMPLETION, + ); + } +} diff --git a/tools/stability/testers/HoverTester.ts b/tools/stability/testers/HoverTester.ts new file mode 100644 index 0000000..ce75259 --- /dev/null +++ b/tools/stability/testers/HoverTester.ts @@ -0,0 +1,89 @@ +import { LspClient } from '../../lspClient/LspClient'; +import { OperationTester, OperationType } from './TesterTypes'; +import { retryOperationWithPerformance } from './TesterCommon'; + +export class HoverTester implements OperationTester { + constructor(private readonly client: LspClient) {} + + private extractHoverContent(hoverResult: any): string { + if (typeof hoverResult.contents === 'string') { + return hoverResult.contents as string; + } else if (Array.isArray(hoverResult.contents)) { + return hoverResult.contents.length > 0 ? JSON.stringify(hoverResult.contents) : ''; + } else if ( + hoverResult.contents && + typeof hoverResult.contents === 'object' && + 'value' in hoverResult.contents + ) { + return hoverResult.contents.value as string; + } + return ''; + } + + private validateHoverContent(content: string, patterns: string[]): void { + if (!content || content.length === 0) { + throw new Error('Hover content is empty'); + } + + const lowerContent = content.toLowerCase(); + for (const pattern of patterns) { + if (!lowerContent.includes(pattern.toLowerCase())) { + throw new Error(`Hover content missing expected pattern: ${pattern}`); + } + } + } + + async testAllScenarios(uri: string): Promise { + // Test 1: Hover on resource type using full document update + const s3Template = `AWSTemplateFormatVersion: '2010-09-09' +Resources: + MyResource: + Type: AWS::S3::Bucket + Properties: + BucketName: TestName +`; + + await this.client.updateDocument(uri, 2, s3Template); + + await retryOperationWithPerformance( + () => this.client.hover(uri, 3, 15), + (result: any) => { + if (!result?.contents) { + throw new Error('Hover on resource type returned no content'); + } + + const content = this.extractHoverContent(result); + this.validateHoverContent(content, ['aws::s3::bucket', 'bucket', 's3']); + }, + OperationType.HOVER, + ); + + // Test 2: Hover on property after adding Parameters section using incremental update + const parametersSection = ` +Parameters: + MyParam: + Type: String + Default: TestValue +`; + + await this.client.updateDocument(uri, 3, [ + { + range: { start: { line: 6, character: 0 }, end: { line: 6, character: 0 } }, + text: parametersSection, + }, + ]); + + await retryOperationWithPerformance( + () => this.client.hover(uri, 8, 10), + (result: any) => { + if (!result?.contents) { + throw new Error('Hover on parameter returned no content'); + } + + const content = this.extractHoverContent(result); + this.validateHoverContent(content, ['parameter', 'string']); + }, + OperationType.HOVER, + ); + } +} diff --git a/tools/stability/testers/TesterCommon.ts b/tools/stability/testers/TesterCommon.ts new file mode 100644 index 0000000..3faab00 --- /dev/null +++ b/tools/stability/testers/TesterCommon.ts @@ -0,0 +1,27 @@ +import { recordOperation } from '../Monitoring'; +import { WaitFor } from '../../../tst/utils/Utils'; +import { OperationType, getTesterConfig } from './TesterTypes'; + +const RETRY_INTERVAL_MS = 250; + +export async function retryOperationWithPerformance( + operation: () => Promise, + validate: (result: T) => void, + operationType: OperationType, +): Promise { + const config = getTesterConfig(operationType); + let responseTime: number = 0; + + await WaitFor.waitFor( + async () => { + const startTime = performance.now(); + const result = await operation(); + responseTime = performance.now() - startTime; + validate(result); + }, + config.retryTimeoutMs, + RETRY_INTERVAL_MS, + ); + + recordOperation(responseTime, operationType); +} diff --git a/tools/stability/testers/TesterTypes.ts b/tools/stability/testers/TesterTypes.ts new file mode 100644 index 0000000..ab5d1f1 --- /dev/null +++ b/tools/stability/testers/TesterTypes.ts @@ -0,0 +1,31 @@ +export enum OperationType { + HOVER = 'hover', + COMPLETION = 'completion', +} + +export interface OperationTester { + testAllScenarios(uri: string): Promise; +} + +export interface TesterConfig { + retryTimeoutMs: number; + avgDurationLimitMs: number; + maxDurationLimitMs: number; +} + +export const TESTER_CONFIG: Record = { + [OperationType.HOVER]: { + retryTimeoutMs: 3000, + avgDurationLimitMs: 150, + maxDurationLimitMs: 3000, + }, + [OperationType.COMPLETION]: { + retryTimeoutMs: 5000, + avgDurationLimitMs: 300, + maxDurationLimitMs: 5000, + }, +}; + +export function getTesterConfig(operationType: OperationType): TesterConfig { + return TESTER_CONFIG[operationType]; +}