Skip to content

Commit 99ab6f5

Browse files
csviricsviri
committed
feat: naive performance test and handling results storing
Stores performance tests as artifacts and compares the results with released version performance tests Signed-off-by: Attila Mészáros <a_meszaros@apple.com>
1 parent f105902 commit 99ab6f5

File tree

10 files changed

+519
-1
lines changed

10 files changed

+519
-1
lines changed

‎.github/workflows/integration-tests.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,10 @@ jobs:
5555
echo "Using profile: ${it_profile}"
5656
./mvnw ${MAVEN_ARGS} -T1C -B install -DskipTests -Pno-apt --file pom.xml
5757
./mvnw ${MAVEN_ARGS} -T1C -B package -P${it_profile} -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml
58+
59+
- name: Upload performance test results
60+
uses: actions/upload-artifact@v4
61+
with:
62+
name: performance-results-java${{ inputs.java-version }}-k8s${{ inputs.kube-version }}-${{ inputs.http-client }}
63+
path: operator-framework/target/performance_test_result.json
64+
if-no-files-found: ignore

‎.github/workflows/pr.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,51 @@ jobs:
3232

3333
build:
3434
uses: ./.github/workflows/build.yml
35+
36+
performance_report:
37+
name: Post Performance Results
38+
runs-on: ubuntu-latest
39+
needs: build
40+
if: always()
41+
permissions:
42+
pull-requests: write
43+
steps:
44+
- uses: actions/checkout@v6
45+
46+
- name: Download all performance artifacts
47+
uses: actions/download-artifact@v4
48+
with:
49+
pattern: performance-results-*
50+
path: performance-results
51+
merge-multiple: true
52+
53+
- name: Check for performance results
54+
id: check_results
55+
run: |
56+
if [ -d "performance-results" ] && [ "$(ls -A performance-results/*.json 2>/dev/null)" ]; then
57+
echo "has_results=true" >> $GITHUB_OUTPUT
58+
else
59+
echo "has_results=false" >> $GITHUB_OUTPUT
60+
fi
61+
62+
- name: Convert performance results to markdown
63+
if: steps.check_results.outputs.has_results == 'true'
64+
id: convert
65+
run: |
66+
echo "# Performance Test Results" > comment.md
67+
echo "" >> comment.md
68+
for file in performance-results/*.json; do
69+
if [ -f "$file" ]; then
70+
echo "Processing $file"
71+
python3 .github/scripts/performance-to-markdown.py "$file" >> comment.md
72+
echo "" >> comment.md
73+
fi
74+
done
75+
76+
- name: Post PR comment
77+
if: steps.check_results.outputs.has_results == 'true' && github.event_name == 'pull_request'
78+
uses: marocchino/sticky-pull-request-comment@v2
79+
with:
80+
path: comment.md
81+
recreate: true
82+

‎operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright Java Operator SDK Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.javaoperatorsdk.operator.baseapi.performance;
17+
18+
import java.util.List;
19+
20+
public class PerformanceTestResult {
21+
22+
private List<PerformanceTestSummary> summaries;
23+
24+
public List<PerformanceTestSummary> getSummaries() {
25+
return summaries;
26+
}
27+
28+
public void setSummaries(List<PerformanceTestSummary> summaries) {
29+
this.summaries = summaries;
30+
}
31+
}

‎operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Copyright Java Operator SDK Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.javaoperatorsdk.operator.baseapi.performance;
17+
18+
public class PerformanceTestSummary {
19+
20+
private String name;
21+
22+
// data about the machine
23+
private int numberOfProcessors;
24+
private long maxMemory;
25+
private long duration;
26+
27+
public String getName() {
28+
return name;
29+
}
30+
31+
public void setName(String name) {
32+
this.name = name;
33+
}
34+
35+
public int getNumberOfProcessors() {
36+
return numberOfProcessors;
37+
}
38+
39+
public void setNumberOfProcessors(int numberOfProcessors) {
40+
this.numberOfProcessors = numberOfProcessors;
41+
}
42+
43+
public long getMaxMemory() {
44+
return maxMemory;
45+
}
46+
47+
public void setMaxMemory(long maxMemory) {
48+
this.maxMemory = maxMemory;
49+
}
50+
51+
public long getDuration() {
52+
return duration;
53+
}
54+
55+
public void setDuration(long duration) {
56+
this.duration = duration;
57+
}
58+
}

‎operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
/*
2+
* Copyright Java Operator SDK Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.javaoperatorsdk.operator.baseapi.performance;
17+
18+
import java.io.File;
19+
import java.io.IOException;
20+
import java.util.ArrayList;
21+
import java.util.List;
22+
import java.util.Map;
23+
import java.util.Set;
24+
import java.util.concurrent.Callable;
25+
import java.util.concurrent.ExecutorService;
26+
import java.util.concurrent.Executors;
27+
28+
import org.junit.jupiter.api.Test;
29+
import org.junit.jupiter.api.extension.RegisterExtension;
30+
import org.slf4j.Logger;
31+
import org.slf4j.LoggerFactory;
32+
33+
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
34+
import io.fabric8.kubernetes.client.informers.ResourceEventHandler;
35+
import io.fabric8.kubernetes.client.informers.SharedIndexInformer;
36+
import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension;
37+
import io.vertx.core.impl.ConcurrentHashSet;
38+
39+
import com.fasterxml.jackson.databind.ObjectMapper;
40+
41+
public class SimplePerformanceTestIT {
42+
43+
private static final Logger log = LoggerFactory.getLogger(SimplePerformanceTestIT.class);
44+
public static final String INITIAL_VALUE = "initialValue";
45+
public static final String RESOURCE_NAME_PREFIX = "resource";
46+
public static final String INDEX = "index";
47+
48+
@RegisterExtension
49+
LocallyRunOperatorExtension extension =
50+
LocallyRunOperatorExtension.builder()
51+
.withReconciler(new SimplePerformanceTestReconciler())
52+
.build();
53+
54+
final int WARM_UP_RESOURCE_NUMBER = 10;
55+
final int TEST_RESOURCE_NUMBER = 150;
56+
57+
ExecutorService executor = Executors.newFixedThreadPool(TEST_RESOURCE_NUMBER);
58+
59+
@Test
60+
void simpleNaivePerformanceTest() {
61+
var processors = Runtime.getRuntime().availableProcessors();
62+
long maxMemory = Runtime.getRuntime().maxMemory();
63+
log.info("Running performance test with memory: {} and processors: {}", maxMemory, processors);
64+
65+
var primaryInformer =
66+
extension
67+
.getKubernetesClient()
68+
.resources(SimplePerformanceTestResource.class)
69+
.inNamespace(extension.getNamespace())
70+
.inform();
71+
72+
var statusChecker =
73+
new StatusChecker(INITIAL_VALUE, 0, WARM_UP_RESOURCE_NUMBER, primaryInformer);
74+
createResources(0, WARM_UP_RESOURCE_NUMBER, INITIAL_VALUE);
75+
statusChecker.waitUntilAllInStatus();
76+
77+
long startTime = System.currentTimeMillis();
78+
statusChecker =
79+
new StatusChecker(
80+
INITIAL_VALUE, WARM_UP_RESOURCE_NUMBER, TEST_RESOURCE_NUMBER, primaryInformer);
81+
createResources(WARM_UP_RESOURCE_NUMBER, TEST_RESOURCE_NUMBER, INITIAL_VALUE);
82+
statusChecker.waitUntilAllInStatus();
83+
var duration = System.currentTimeMillis() - startTime;
84+
85+
log.info("Create duration: {}", duration);
86+
saveResults(duration);
87+
}
88+
89+
private void saveResults(long duration) {
90+
try {
91+
var result = new PerformanceTestResult();
92+
var summary = new PerformanceTestSummary();
93+
result.setSummaries(List.of(summary));
94+
summary.setName("Naive performance test");
95+
summary.setDuration(duration);
96+
summary.setNumberOfProcessors(Runtime.getRuntime().availableProcessors());
97+
summary.setMaxMemory(Runtime.getRuntime().maxMemory());
98+
var objectMapper = new ObjectMapper();
99+
objectMapper.writeValue(new File("target/performance_test_result.json"), result);
100+
} catch (IOException e) {
101+
throw new RuntimeException(e);
102+
}
103+
}
104+
105+
private void createResources(int startIndex, int number, String value) {
106+
try {
107+
List<Callable<Void>> callables = new ArrayList<>(number);
108+
109+
for (int i = startIndex; i < startIndex + number; i++) {
110+
var res = new SimplePerformanceTestResource();
111+
res.setMetadata(
112+
new ObjectMetaBuilder()
113+
.withAnnotations(Map.of(INDEX, "" + i))
114+
.withName(RESOURCE_NAME_PREFIX + i)
115+
.build());
116+
res.setSpec(new SimplePerformanceTestSpec());
117+
res.getSpec().setValue(value);
118+
callables.add(
119+
() -> {
120+
extension.create(res);
121+
return null;
122+
});
123+
}
124+
var futures = executor.invokeAll(callables);
125+
for (var future : futures) {
126+
future.get();
127+
}
128+
} catch (Exception e) {
129+
throw new RuntimeException(e);
130+
}
131+
}
132+
133+
static class StatusChecker {
134+
private final String expectedStatus; // null indicates deleted
135+
private final Set<Integer> remaining = new ConcurrentHashSet<>();
136+
137+
StatusChecker(
138+
String expectedStatus,
139+
int startIndex,
140+
int number,
141+
SharedIndexInformer<SimplePerformanceTestResource> primaryInformer) {
142+
this.expectedStatus = expectedStatus;
143+
for (int i = startIndex; i < startIndex + number; i++) {
144+
remaining.add(i);
145+
}
146+
primaryInformer.addEventHandler(
147+
new ResourceEventHandler<>() {
148+
@Override
149+
public void onAdd(SimplePerformanceTestResource obj) {
150+
checkOnStatus(obj);
151+
}
152+
153+
@Override
154+
public void onUpdate(
155+
SimplePerformanceTestResource oldObj, SimplePerformanceTestResource newObj) {
156+
checkOnStatus(newObj);
157+
}
158+
159+
@Override
160+
public void onDelete(
161+
SimplePerformanceTestResource obj, boolean deletedFinalStateUnknown) {
162+
if (expectedStatus == null) {
163+
synchronized (remaining) {
164+
remaining.remove(Integer.parseInt(obj.getMetadata().getAnnotations().get(INDEX)));
165+
remaining.notifyAll();
166+
}
167+
}
168+
}
169+
});
170+
primaryInformer.getStore().list().forEach(this::checkOnStatus);
171+
}
172+
173+
private void checkOnStatus(SimplePerformanceTestResource res) {
174+
if (expectedStatus != null
175+
&& res.getStatus() != null
176+
&& res.getStatus().getValue().equals(expectedStatus)) {
177+
synchronized (remaining) {
178+
remaining.remove(Integer.parseInt(res.getMetadata().getAnnotations().get(INDEX)));
179+
remaining.notifyAll();
180+
}
181+
}
182+
}
183+
184+
public void waitUntilAllInStatus() {
185+
synchronized (remaining) {
186+
while (!remaining.isEmpty()) {
187+
try {
188+
remaining.wait();
189+
} catch (InterruptedException e) {
190+
Thread.currentThread().interrupt();
191+
throw new RuntimeException(e);
192+
}
193+
}
194+
}
195+
}
196+
}
197+
}

0 commit comments

Comments
 (0)