Catch exception when creating metrics datastreams

data-node/src/main/java/org/graylog/datanode/metrics/ConfigureMetricsIndexSettings.java

@@ -88,7 +88,14 @@ public void transition(OpensearchEvent trigger, OpensearchState source, Opensear
                 LOGGER.debug("Opensearch client available, tracer instance: {}, client: {}", this, client);
                 if (datastreamCreated.compareAndSet(false, true)) {
                     LOGGER.info("Initial creation of metric templates and datastreams, tracer instance: {}, client: {}", this, client);
-                    createDatastream(client);
+                    try {
+                        createDatastream(client);
+                    } catch (Exception e) {
+                        // Reset so the next AVAILABLE health-check reentry (every 10s) retries. Don't rethrow -
+                        // a metrics datastream failure shouldn't be treated as an opensearch process health failure.
+                        datastreamCreated.set(false);
+                        LOGGER.error("Failed to create metrics datastream, will retry, tracer instance: {}", this, e);
+                    }
                 }
             });
         }

data-node/src/main/java/org/graylog/datanode/periodicals/MetricsCollector.java

@@ -228,7 +228,7 @@ private static void indexDocument(OfficialOpensearchClient client, IndexRequest<
             CompletableFuture<IndexResponse> response = client.asyncWithoutErrorMapping().index(indexRequest);
             response.whenComplete((indexResponse, throwable) -> {
                 if (Objects.nonNull(throwable)) {
-                    LOG.error("Error indexing metrics");
+                    LOG.error("Error indexing metrics into datastream {}", indexRequest.index(), throwable);
                 }
             });
         } catch (IOException e) {