Skip to content

Commit 9d748f7

Browse files
csviricsviri
committed
wip
Signed-off-by: Attila Mészáros <a_meszaros@apple.com>
1 parent 62fcab1 commit 9d748f7

File tree

3 files changed

+262
-54
lines changed

3 files changed

+262
-54
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ target/
1717
.aider*
1818

1919
docs/content/en/docs/testindex/
20+
21+
# Downloaded Prometheus binary
22+
observability/.prometheus/

‎observability/get-prometheus.sh‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
#
3+
# Copyright Java Operator SDK Authors
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
# Downloads Prometheus locally if not already present.
19+
# Prints the path to the prometheus binary on stdout.
20+
# Usage: ./get-prometheus.sh [install-dir]
21+
22+
set -e
23+
24+
PROMETHEUS_VERSION="3.4.0"
25+
INSTALL_DIR="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.prometheus}"
26+
27+
# If already downloaded, just print the path
28+
if [ -x "$INSTALL_DIR/prometheus" ]; then
29+
echo "$INSTALL_DIR/prometheus"
30+
exit 0
31+
fi
32+
33+
# Check if prometheus is already on PATH
34+
if command -v prometheus &> /dev/null; then
35+
echo "prometheus"
36+
exit 0
37+
fi
38+
39+
# Detect OS and architecture
40+
OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
41+
ARCH="$(uname -m)"
42+
43+
case "$OS" in
44+
linux) PLATFORM="linux" ;;
45+
darwin) PLATFORM="darwin" ;;
46+
*) echo "Unsupported OS: $OS" >&2; exit 1 ;;
47+
esac
48+
49+
case "$ARCH" in
50+
x86_64|amd64) CPU_ARCH="amd64" ;;
51+
aarch64|arm64) CPU_ARCH="arm64" ;;
52+
*) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;;
53+
esac
54+
55+
TAR_NAME="prometheus-${PROMETHEUS_VERSION}.${PLATFORM}-${CPU_ARCH}"
56+
DOWNLOAD_URL="https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${TAR_NAME}.tar.gz"
57+
58+
mkdir -p "$INSTALL_DIR"
59+
TAR_FILE="$INSTALL_DIR/prometheus.tar.gz"
60+
61+
echo "Downloading Prometheus v${PROMETHEUS_VERSION} for ${PLATFORM}/${CPU_ARCH}..." >&2
62+
curl -fsSL -o "$TAR_FILE" "$DOWNLOAD_URL"
63+
64+
echo "Extracting..." >&2
65+
tar xzf "$TAR_FILE" -C "$INSTALL_DIR"
66+
67+
# Move the binary to the install dir root for a clean path
68+
mv "$INSTALL_DIR/$TAR_NAME/prometheus" "$INSTALL_DIR/prometheus"
69+
mv "$INSTALL_DIR/$TAR_NAME/promtool" "$INSTALL_DIR/promtool" 2>/dev/null || true
70+
71+
# Clean up
72+
rm -rf "$INSTALL_DIR/$TAR_NAME" "$TAR_FILE"
73+
74+
echo "Prometheus installed to $INSTALL_DIR/prometheus" >&2
75+
echo "$INSTALL_DIR/prometheus"

‎sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java‎

Lines changed: 184 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import java.net.HttpURLConnection;
2020
import java.net.URL;
2121
import java.nio.charset.StandardCharsets;
22+
import java.nio.file.Files;
23+
import java.nio.file.Path;
2224
import java.time.Duration;
2325
import java.util.ArrayDeque;
2426
import java.util.Deque;
@@ -61,49 +63,195 @@ class MetricsHandlingE2E {
6163
public static final String NAME_LABEL_KEY = "app.kubernetes.io/name";
6264

6365
private LocalPortForward prometheusPortForward;
66+
private Process prometheusProcess;
6467

6568
static final KubernetesClient client = new KubernetesClientBuilder().build();
6669

67-
MetricsHandlingE2E() throws FileNotFoundException {}
68-
69-
@RegisterExtension
70-
AbstractOperatorExtension operator =
71-
isLocal()
72-
? LocallyRunOperatorExtension.builder()
73-
.withReconciler(new MetricsHandlingReconciler1())
74-
.withReconciler(new MetricsHandlingReconciler2())
75-
.withConfigurationService(
76-
c -> {
77-
var registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
78-
try {
79-
MetricsHandlingSampleOperator.startMetricsServer(registry);
80-
} catch (IOException e) {
81-
throw new UncheckedIOException(e);
82-
}
83-
c.withMetrics(MetricsHandlingSampleOperator.initMetrics(registry));
84-
})
85-
.build()
86-
: ClusterDeployedOperatorExtension.builder()
87-
.withOperatorDeployment(
88-
new KubernetesClientBuilder()
89-
.build()
90-
.load(new FileInputStream("k8s/operator.yaml"))
91-
.items())
92-
.build();
70+
MetricsHandlingE2E() {}
71+
72+
static final PrometheusMeterRegistry prometheusRegistry =
73+
isLocal() ? new PrometheusMeterRegistry(PrometheusConfig.DEFAULT) : null;
74+
75+
@RegisterExtension AbstractOperatorExtension operator = createOperatorExtension();
76+
77+
private static AbstractOperatorExtension createOperatorExtension() {
78+
if (!isLocal()) {
79+
try {
80+
return ClusterDeployedOperatorExtension.builder()
81+
.withOperatorDeployment(
82+
new KubernetesClientBuilder()
83+
.build()
84+
.load(new FileInputStream("k8s/operator.yaml"))
85+
.items())
86+
.build();
87+
} catch (FileNotFoundException e) {
88+
throw new UncheckedIOException(e);
89+
}
90+
}
91+
try {
92+
MetricsHandlingSampleOperator.startMetricsServer(prometheusRegistry);
93+
} catch (IOException e) {
94+
throw new UncheckedIOException(e);
95+
}
96+
return LocallyRunOperatorExtension.builder()
97+
.withReconciler(new MetricsHandlingReconciler1())
98+
.withReconciler(new MetricsHandlingReconciler2())
99+
.withConfigurationService(
100+
c -> c.withMetrics(MetricsHandlingSampleOperator.initMetrics(prometheusRegistry)))
101+
.build();
102+
}
93103

94104
@BeforeAll
95-
void setupObservability() {
96-
log.info("Setting up observability stack...");
97-
installObservabilityServices();
98-
createOperatorServiceMonitor();
99-
prometheusPortForward = portForward(NAME_LABEL_KEY, "prometheus", PROMETHEUS_PORT);
105+
void setupObservability() throws Exception {
106+
if (isLocal()) {
107+
log.info("Starting local Prometheus...");
108+
startLocalPrometheus();
109+
} else {
110+
log.info("Setting up observability stack in cluster...");
111+
installObservabilityServices();
112+
createOperatorServiceMonitor();
113+
prometheusPortForward = portForward(NAME_LABEL_KEY, "prometheus", PROMETHEUS_PORT);
114+
}
100115
}
101116

102117
@AfterAll
103118
void cleanup() throws IOException {
119+
if (prometheusProcess != null) {
120+
log.info("Stopping local Prometheus...");
121+
prometheusProcess.destroyForcibly();
122+
}
104123
closePortForward(prometheusPortForward);
105124
}
106125

126+
private void startLocalPrometheus() throws Exception {
127+
// Get the prometheus binary via the download script
128+
String promBinary = getPrometheusBinary();
129+
130+
// Create a temporary prometheus.yml that scrapes the local operator metrics endpoint
131+
Path promDir = Files.createTempDirectory("prometheus-test");
132+
Path promConfig = promDir.resolve("prometheus.yml");
133+
Files.writeString(
134+
promConfig,
135+
"""
136+
global:
137+
scrape_interval: 5s
138+
evaluation_interval: 5s
139+
scrape_configs:
140+
- job_name: 'operator'
141+
static_configs:
142+
- targets: ['localhost:%d']
143+
"""
144+
.formatted(MetricsHandlingSampleOperator.METRICS_PORT));
145+
log.info("Prometheus config written to {}", promConfig);
146+
147+
Path dataDir = promDir.resolve("data");
148+
Files.createDirectories(dataDir);
149+
150+
ProcessBuilder pb =
151+
new ProcessBuilder(
152+
promBinary,
153+
"--config.file=" + promConfig,
154+
"--storage.tsdb.path=" + dataDir,
155+
"--web.listen-address=0.0.0.0:" + PROMETHEUS_PORT,
156+
"--log.level=warn");
157+
pb.redirectErrorStream(true);
158+
prometheusProcess = pb.start();
159+
160+
// Log Prometheus output in a background daemon thread
161+
var outputReader = prometheusProcess.getInputStream();
162+
Thread logThread =
163+
new Thread(
164+
() -> {
165+
try (var reader = new BufferedReader(new InputStreamReader(outputReader))) {
166+
String line;
167+
while ((line = reader.readLine()) != null) {
168+
log.info("Prometheus: {}", line);
169+
}
170+
} catch (IOException e) {
171+
// process terminated
172+
}
173+
},
174+
"prometheus-output");
175+
logThread.setDaemon(true);
176+
logThread.start();
177+
178+
// Wait for Prometheus to be ready
179+
await()
180+
.atMost(Duration.ofSeconds(30))
181+
.pollInterval(Duration.ofSeconds(1))
182+
.untilAsserted(
183+
() -> {
184+
var conn =
185+
(HttpURLConnection)
186+
new URL("http://localhost:" + PROMETHEUS_PORT + "/-/ready").openConnection();
187+
conn.setConnectTimeout(1000);
188+
conn.setReadTimeout(1000);
189+
assertThat(conn.getResponseCode()).isEqualTo(200);
190+
});
191+
log.info("Local Prometheus is ready on port {}", PROMETHEUS_PORT);
192+
}
193+
194+
private String getPrometheusBinary() throws Exception {
195+
File scriptFile = findObservabilityFile("get-prometheus.sh");
196+
log.info("Running get-prometheus.sh to obtain Prometheus binary...");
197+
198+
ProcessBuilder pb = new ProcessBuilder("/bin/sh", scriptFile.getAbsolutePath());
199+
pb.redirectErrorStream(false);
200+
Process process = pb.start();
201+
202+
// Read stderr for progress messages
203+
Thread stderrThread =
204+
new Thread(
205+
() -> {
206+
try (var reader =
207+
new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
208+
String line;
209+
while ((line = reader.readLine()) != null) {
210+
log.info("get-prometheus: {}", line);
211+
}
212+
} catch (IOException e) {
213+
// done
214+
}
215+
},
216+
"get-prometheus-stderr");
217+
stderrThread.setDaemon(true);
218+
stderrThread.start();
219+
220+
// Read stdout — last line is the binary path
221+
String binaryPath;
222+
try (var reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
223+
binaryPath = reader.lines().reduce((first, second) -> second).orElse(null);
224+
}
225+
226+
int exitCode = process.waitFor();
227+
if (exitCode != 0 || binaryPath == null || binaryPath.isBlank()) {
228+
throw new IllegalStateException(
229+
"get-prometheus.sh failed (exit code " + exitCode + "), output: " + binaryPath);
230+
}
231+
232+
log.info("Prometheus binary: {}", binaryPath);
233+
return binaryPath;
234+
}
235+
236+
private File findObservabilityFile(String fileName) {
237+
try {
238+
File projectRoot = new File(".").getCanonicalFile();
239+
while (projectRoot != null && !new File(projectRoot, "observability").exists()) {
240+
projectRoot = projectRoot.getParentFile();
241+
}
242+
if (projectRoot == null) {
243+
throw new IllegalStateException("Could not find observability directory");
244+
}
245+
File file = new File(projectRoot, "observability/" + fileName);
246+
if (!file.exists()) {
247+
throw new IllegalStateException("File not found: " + file.getAbsolutePath());
248+
}
249+
return file;
250+
} catch (IOException e) {
251+
throw new UncheckedIOException(e);
252+
}
253+
}
254+
107255
private LocalPortForward portForward(String labelKey, String labelValue, int port) {
108256
log.info("Waiting for pod with label {}={} to be ready...", labelKey, labelValue);
109257
AtomicReference<Pod> portForwardPod = new AtomicReference<>();
@@ -163,7 +311,7 @@ void testPropagatedMetrics() throws Exception {
163311
operator.create(createResource(MetricsHandlingCustomResource1.class, "test-fail-1", 1));
164312
operator.create(createResource(MetricsHandlingCustomResource2.class, "test-fail-2", 1));
165313

166-
// Continuously trigger reconciliations for ~50 seconds by alternating between
314+
// Continuously trigger reconciliations for ~60 seconds by alternating between
167315
// creating new resources, updating specs of existing ones, and deleting older dynamic ones
168316
long deadline = System.currentTimeMillis() + TEST_DURATION.toMillis();
169317
int counter = 0;
@@ -226,9 +374,7 @@ private void verifyPrometheusMetrics() {
226374
assertMetricPresent(prometheusUrl, "reconciliations_success_total", Duration.ofSeconds(30));
227375
assertMetricPresent(prometheusUrl, "reconciliations_failure_total", Duration.ofSeconds(30));
228376
assertMetricPresent(
229-
prometheusUrl,
230-
"reconciliations_execution_duration_milliseconds_count",
231-
Duration.ofSeconds(30));
377+
prometheusUrl, "reconciliations_execution_duration_seconds_count", Duration.ofSeconds(30));
232378

233379
log.info("All metrics verified successfully in Prometheus");
234380
}
@@ -286,8 +432,8 @@ private String queryPrometheus(String prometheusUrl, String query) throws IOExce
286432
}
287433

288434
/**
289-
* Creates a ServiceMonitor so Prometheus scrapes the operator's /metrics endpoint. For local
290-
* runs, Prometheus scrapes localhost:8080; for remote, it scrapes the operator service.
435+
* Creates a ServiceMonitor so Prometheus scrapes the operator's /metrics endpoint (remote mode
436+
* only).
291437
*/
292438
private void createOperatorServiceMonitor() {
293439
String namespace = operator.getNamespace();
@@ -333,27 +479,11 @@ private void createOperatorServiceMonitor() {
333479

334480
private void installObservabilityServices() {
335481
try {
336-
// Find the observability script relative to project root
337-
File projectRoot = new File(".").getCanonicalFile();
338-
while (projectRoot != null && !new File(projectRoot, "observability").exists()) {
339-
projectRoot = projectRoot.getParentFile();
340-
}
341-
342-
if (projectRoot == null) {
343-
throw new IllegalStateException("Could not find observability directory");
344-
}
345-
346-
File scriptFile = new File(projectRoot, "observability/install-observability.sh");
347-
if (!scriptFile.exists()) {
348-
throw new IllegalStateException(
349-
"Observability script not found at: " + scriptFile.getAbsolutePath());
350-
}
482+
File scriptFile = findObservabilityFile("install-observability.sh");
351483
log.info("Running observability setup script: {}", scriptFile.getAbsolutePath());
352484

353-
// Run the install-observability.sh script
354485
ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", scriptFile.getAbsolutePath());
355486
processBuilder.redirectErrorStream(true);
356-
357487
processBuilder.environment().putAll(System.getenv());
358488
Process process = processBuilder.start();
359489
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));

0 commit comments

Comments
 (0)