Skip to content

Commit b91bd34

Browse files
ivchicanoivchicano
committed
Add configuration to make workers exit after test finish
1 parent 0fca877 commit b91bd34

13 files changed

Lines changed: 176 additions & 30 deletions

File tree

‎README.md‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -195,19 +195,20 @@ Configuration for where browsers run. Workers can be manually provided and manag
195195

196196
#### AWS Workers
197197

198-
| Property | Required | Default | Description |
199-
| ------------------ | -------- | ------------ | ---------------------------------------------------------------- |
200-
| `accessKey` | **Yes** | - | AWS access key |
201-
| `secretAccessKey` | **Yes** | - | AWS secret access key |
202-
| `amiId` | **Yes** | - | AMI ID for worker instances |
203-
| `instanceType` | No | `c5.xlarge` | EC2 instance type |
204-
| `keyPairName` | No | - | EC2 key pair name |
205-
| `securityGroupId` | No | - | Security group ID |
206-
| `region` | No | `us-east-1` | AWS region |
207-
| `availabilityZone` | No | `us-east-1f` | AWS availability zone |
208-
| `workersAtStart` | No | `0` | Number of instances to start the test with |
209-
| `rampUpWorkers` | No | `0` | Workers instances to add when the test runs out of existing ones |
210-
| `terminateWorkers` | No | _true_ | Whether to terminate EC2 instances after test completion |
198+
| Property | Required | Default | Description |
199+
| ------------------ | -------- | ------------ | ------------------------------------------------------------------- |
200+
| `accessKey` | **Yes** | - | AWS access key |
201+
| `secretAccessKey` | **Yes** | - | AWS secret access key |
202+
| `amiId` | **Yes** | - | AMI ID for worker instances |
203+
| `instanceType` | No | `c5.xlarge` | EC2 instance type |
204+
| `keyPairName` | No | - | EC2 key pair name |
205+
| `securityGroupId` | No | - | Security group ID |
206+
| `region` | No | `us-east-1` | AWS region |
207+
| `availabilityZone` | No | `us-east-1f` | AWS availability zone |
208+
| `workersAtStart` | No | `0` | Number of instances to start the test with |
209+
| `rampUpWorkers` | No | `0` | Workers instances to add when the test runs out of existing ones |
210+
| `terminateWorkers` | No | _true_ | Whether to terminate EC2 instances after test completion |
211+
| `exitOnEnd` | No | _true_ | Whether to signal workers to cleanup and exit after test completion |
211212

212213
### Distribution
213214

‎browser-emulator/src/controllers/instance.controller.ts‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import type {
1111
} from '../types/initialize.type.ts';
1212
import type { ConfigService } from '../services/config.service.ts';
1313
import type { SeleniumService } from '../services/selenium.service.ts';
14+
import { gracefulExit } from 'exit-hook';
1415

1516
export class InstanceController {
1617
private readonly router: express.Router;
@@ -46,6 +47,7 @@ export class InstanceController {
4647
private setupRoutes(): void {
4748
this.router.get('/ping', this.ping.bind(this));
4849
this.router.post('/initialize', this.initialize.bind(this));
50+
this.router.delete('/shutdown', this.shutdown.bind(this));
4951
}
5052

5153
private ping(_: Request, res: Response): void {
@@ -116,6 +118,23 @@ export class InstanceController {
116118
}
117119
}
118120

121+
private shutdown(_: Request, res: Response) {
122+
try {
123+
console.log('Shutdown signal received, exiting...');
124+
setTimeout(() => {
125+
gracefulExit(0);
126+
}, 1000);
127+
res.status(200).send('Shutdown initiated');
128+
} catch (error) {
129+
console.error('Error during shutdown:', error);
130+
res.status(500).send(error);
131+
}
132+
}
133+
134+
public getRouter(): express.Router {
135+
return this.router;
136+
}
137+
119138
private setupRemotePersistenceService(request: InitializePost) {
120139
let accessKey: string | undefined;
121140
let secretAccessKey: string | undefined;
@@ -144,8 +163,4 @@ export class InstanceController {
144163
);
145164
}
146165
}
147-
148-
public getRouter(): express.Router {
149-
return this.router;
150-
}
151166
}

‎config/config-all.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ workers:
4545
urls: 127.0.0.1
4646
# Set to true if workers don't have HTTPS configured (local testing)
4747
disableHttps: false
48+
# Signal workers to cleanup and exit after test completion
49+
exitOnEnd: true
4850

4951
# ---------- AWS ----------
5052
aws:

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ services:
2525
dockerfile: Dockerfile
2626
image: browser-emulator:local
2727
container_name: browser-emulator
28-
restart: unless-stopped
28+
restart: no
2929
environment:
3030
RUNNING_IN_DOCKER: true
3131
DOCKER_NETWORK_NAME: browseremulator

‎e2e-tests/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ services:
2828
dockerfile: Dockerfile
2929
image: browser-emulator:local
3030
container_name: browser-emulator-e2e
31-
restart: unless-stopped
31+
restart: no
3232
environment:
3333
RUNNING_IN_DOCKER: true
3434
DOCKER_NETWORK_NAME: browseremulator-e2e

‎e2e-tests/scripts/run-smoke-test.sh‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ E2E_TEST_DIR="$(dirname "$(realpath "$0")")/.."
3434
export LOCAL_CONFIG_DIR="$E2E_TEST_DIR/config"
3535
export LOCAL_RESULTS_DIR="$E2E_TEST_DIR/results"
3636
export LOADTEST_CONFIG="/config/smoke-test-config.yaml"
37+
# Clean previous results to avoid false positives
38+
rm -f "$LOCAL_RESULTS_DIR/results.txt" "$LOCAL_RESULTS_DIR/report.html" "$LOCAL_RESULTS_DIR/docker-compose.log"
3739
# Start services
3840
echo "Starting services with docker compose..."
3941
docker compose up --build -d
@@ -66,10 +68,58 @@ if [ $ELAPSED -ge $MAX_WAIT ]; then
6668
docker compose logs
6769
fi
6870

71+
# If results exist, wait for both containers to stop (max 30 seconds)
72+
if [ -f "$LOCAL_RESULTS_DIR/results.txt" ]; then
73+
echo "Waiting for containers to stop..."
74+
TIMEOUT=30
75+
ELAPSED=0
76+
while [ $ELAPSED -lt $TIMEOUT ]; do
77+
RUNNING_CONTAINERS=$(docker compose ps --status running -q 2>/dev/null || true)
78+
if [ -z "$RUNNING_CONTAINERS" ]; then
79+
echo "All containers stopped."
80+
break
81+
fi
82+
sleep 2
83+
ELAPSED=$((ELAPSED + 2))
84+
echo "Waiting for containers to stop... ($ELAPSED/$TIMEOUT seconds)"
85+
done
86+
if [ $ELAPSED -ge $TIMEOUT ]; then
87+
echo "WARNING: Some containers did not stop within $TIMEOUT seconds."
88+
fi
89+
fi
90+
91+
# Check if any containers are still running after shutdown
92+
echo "Checking for leftover containers..."
93+
REMAINING_CONTAINERS=$(docker compose ps --status running -q 2>/dev/null || true)
94+
if [ -n "$REMAINING_CONTAINERS" ]; then
95+
echo "ERROR: Containers still running after shutdown:"
96+
docker compose ps
97+
EXIT_CODE=1
98+
fi
99+
100+
# Check for any Selenium browser containers that might have been left running
101+
echo "Checking for leftover Selenium containers..."
102+
SELENIUM_CONTAINERS=$(docker ps -q --filter "name=selenium" 2>/dev/null || true)
103+
if [ -n "$SELENIUM_CONTAINERS" ]; then
104+
echo "ERROR: Selenium containers still running:"
105+
docker ps --filter "name=selenium"
106+
EXIT_CODE=1
107+
fi
108+
69109
# Stop services
70110
docker compose logs > $LOCAL_RESULTS_DIR/docker-compose.log 2>&1
71111
echo "Stopping services..."
72112
docker compose down
113+
# Stop and remove any remaining selenium containers
114+
SELENIUM_CONTAINERS=$(docker ps -aq --filter "name=selenium" 2>/dev/null || true)
115+
if [ -n "$SELENIUM_CONTAINERS" ]; then
116+
echo "Removing remaining Selenium containers..."
117+
docker rm -f $SELENIUM_CONTAINERS >/dev/null 2>&1 || true
118+
fi
119+
120+
if [ $EXIT_CODE -eq 0 ]; then
121+
echo "✓ All containers stopped successfully"
122+
fi
73123

74124
# Check results
75125
echo "Checking test results..."
@@ -203,7 +253,7 @@ docker compose down
203253
rm -f "$HTML_FILE"
204254
rm -f "$LOCAL_RESULTS_DIR/docker-compose.log"
205255
else
206-
echo "Keeping result files for debugging. Remember to remove them manually before re-running the test."
256+
echo "Keeping result files for debugging."
207257
fi
208258
else
209259
echo "ERROR: Results file or HTML report not found at $LOCAL_RESULTS_DIR/results.txt or report.html"

‎loadtest-controller/src/main/java/io/openvidu/loadtest/config/LoadTestConfig.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ protected LoadTestConfig(Environment env) {
4343
private int workersRumpUp;
4444

4545
private boolean terminateWorkers;
46+
private boolean exitOnEnd;
4647

4748
private String openviduUrl;
4849

@@ -257,6 +258,10 @@ public boolean isTerminateWorkers() {
257258
return terminateWorkers;
258259
}
259260

261+
public boolean isExitOnEnd() {
262+
return exitOnEnd;
263+
}
264+
260265
public boolean isRetryMode() {
261266
return retryMode;
262267
}
@@ -265,8 +270,6 @@ public int getRetryTimes() {
265270
return retryTimes;
266271
}
267272

268-
269-
270273
public List<String> getReportOutput() {
271274
return reportOutput;
272275
}
@@ -407,13 +410,17 @@ private void initWorkerConfig() {
407410
workerMaxLoad = asInt("distribution.maxLoadPercent");
408411
workersRumpUp = asInt("aws.rampUpWorkers");
409412
disableHttps = asBoolean("workers.disableHttps");
413+
// Default exitOnEnd to true
414+
Boolean exitOnEndConfig = yamlConfig.getBooleanOrNull("workers.exitOnEnd");
415+
exitOnEnd = exitOnEndConfig != null ? exitOnEndConfig : true;
410416
}
411417

412418
private void initAwsAndRecordingConfig() {
413419
medianodeLoadForStartRecording = asDouble("recording.mediaNodeLoadThreshold");
414420
recordingSessionGroup = asInt("recording.sessionsGroupSize");
415-
// Default terminateWorkers to true when AWS is configured (no worker URLs), false for local workers
416-
Boolean terminateWorkersConfig = asBoolean("terminateWorkers");
421+
// Default terminateWorkers to true when AWS is configured (no worker URLs),
422+
// false for local workers
423+
Boolean terminateWorkersConfig = yamlConfig.getBooleanOrNull("terminateWorkers");
417424
terminateWorkers = terminateWorkersConfig != null ? terminateWorkersConfig : workerUrlList.isEmpty();
418425
awsSecretAccessKey = asOptionalString("aws.secretAccessKey");
419426
awsAccessKey = asOptionalString("aws.accessKey");

‎loadtest-controller/src/main/java/io/openvidu/loadtest/models/testcase/ResultReport.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,13 @@
88
import java.util.TreeMap;
99
import java.util.concurrent.TimeUnit;
1010

11+
import org.slf4j.Logger;
12+
import org.slf4j.LoggerFactory;
13+
1114
public class ResultReport {
1215

16+
private static Logger log = LoggerFactory.getLogger(ResultReport.class);
17+
1318
private int totalParticipants = 0;
1419
private int numSessionsCompleted = 0;
1520
private int numSessionsCreated = 0;
@@ -210,7 +215,7 @@ public ResultReport setUserStartDelaysPercentiles(double[] percentiles) {
210215
}
211216

212217
public ResultReport setUserSuccessTimestamps(Map<String, Calendar> userSuccessTimestamps) {
213-
System.out.println("DEBUG: setUserSuccessTimestamps called with size " + userSuccessTimestamps.size());
218+
log.debug("setUserSuccessTimestamps called with size {}", userSuccessTimestamps.size());
214219
this.userSuccessTimestamps = userSuccessTimestamps;
215220
return this;
216221
}

‎loadtest-controller/src/main/java/io/openvidu/loadtest/services/BrowserEmulatorClient.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import java.util.concurrent.ExecutorService;
1717
import java.util.concurrent.Executors;
1818
import java.util.concurrent.Future;
19+
import java.util.concurrent.TimeUnit;
20+
import java.util.concurrent.TimeoutException;
1921
import java.util.concurrent.atomic.AtomicBoolean;
2022
import java.util.concurrent.atomic.AtomicInteger;
2123

@@ -676,4 +678,44 @@ public Map<String, Integer> getPerUserRetryCounts() {
676678
}
677679
return counts;
678680
}
681+
682+
public void shutdownWorkers(List<String> workerUrls, boolean waitForResponse) {
683+
if (workerUrls == null || workerUrls.isEmpty()) {
684+
return;
685+
}
686+
687+
ExecutorService executorService = Executors.newFixedThreadPool(Math.min(workerUrls.size(), 10));
688+
List<Future<Void>> futures = new ArrayList<>();
689+
690+
try {
691+
for (String workerUrl : workerUrls) {
692+
Callable<Void> callable = () -> {
693+
try {
694+
httpClient.sendDelete(
695+
httpProtocolPrefix + workerUrl + ":" + WORKER_PORT + "/instance/shutdown",
696+
getHeaders());
697+
} catch (Exception e) {
698+
log.warn("Failed to send shutdown request to worker {}: {}", workerUrl, e.getMessage());
699+
}
700+
return null;
701+
};
702+
futures.add(executorService.submit(callable));
703+
}
704+
705+
if (waitForResponse) {
706+
// Wait for all futures to complete with a timeout
707+
try {
708+
for (Future<Void> future : futures) {
709+
future.get(5, TimeUnit.MINUTES); // 5 minute timeout
710+
}
711+
} catch (TimeoutException e) {
712+
log.warn("Timeout waiting for worker shutdown responses: {}", e.getMessage());
713+
} catch (Exception e) {
714+
log.warn("Error waiting for worker shutdown responses: {}", e.getMessage());
715+
}
716+
}
717+
} finally {
718+
executorService.shutdownNow();
719+
}
720+
}
679721
}

‎loadtest-controller/src/main/java/io/openvidu/loadtest/services/core/LoadTestService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public LoadTestService(BrowserEmulatorClient browserEmulatorClient, LoadTestConf
8686
loadTestConfig);
8787
this.participantOrchestrator = new LoadTestParticipantOrchestrator(this, browserEmulatorClient, esClient,
8888
loadTestConfig, sleeper);
89-
this.topologyOrchestrator = new LoadTestTopologyOrchestrator(this, loadTestConfig, kibanaClient);
89+
this.topologyOrchestrator = new LoadTestTopologyOrchestrator(this, loadTestConfig, kibanaClient, browserEmulatorClient);
9090

9191
prodMode = loadTestConfig.getWorkerUrlList().isEmpty();
9292
devWorkersList = loadTestConfig.getWorkerUrlList();

0 commit comments

Comments
 (0)