Improve logging messages for Icinga client

It will now warn if an endpoint is not reachable and
received data is logged if it could not be JSON unmarshalled

Author: martialblog

README.md

@@ -69,7 +69,7 @@ Flags:
 --plugin-output-annotations=message,...    List of Annotation names to be used to set the plugin output for the Icinga service ($ALERTMANAGER_ICINGA_BRIDGE_PLUGINOUTPUT_ANNOTATIONS)
 --checks-interval=12h                      Interval (in seconds) to be used for Icinga check_interval and retry_interval ($ALERTMANAGER_ICINGA_BRIDGE_SERVICE_CHECKS_INTERVAL)
 --keep-for=168h                            How long to keep created alerts around after they have been resolved ($ALERTMANAGER_ICINGA_BRIDGE_KEEP_FOR)
---static-service-vars=KEY=VALUE;...        Custom variable to be set for craeted Icinga services (variable=value, can be repeated) ($ALERTMANAGER_ICINGA_BRIDGE_STATIC_SERVICE_VAR)
+--static-service-vars=KEY=VALUE;...        Custom variable to be set for created Icinga services (variable=value, can be repeated) ($ALERTMANAGER_ICINGA_BRIDGE_STATIC_SERVICE_VAR)
 ```
 
 Most flags can be set with environment variables, refer to the help to see which flags.

internal/icinga2/icinga.go

@@ -14,6 +14,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/NETWAYS/alertmanager-icinga-bridge/internal/config"
@@ -71,7 +72,7 @@ func NewClient(config *config.Config, logger *slog.Logger) *Client {
 
 // Do is a small wrapper that tries the given request against all given Icinga API endpoints
 func (c *Client) Do(req *http.Request, path string) (*http.Response, error) {
-	lastErr := ErrNoEndpointReachable
+	var endpointErrors strings.Builder
 
 	for _, base := range c.IcingaURL {
 		req.URL, _ = req.URL.Parse(base + path)
@@ -87,23 +88,24 @@ func (c *Client) Do(req *http.Request, path string) (*http.Response, error) {
 		resp, err := c.httpClient.Do(req)
 
 		if err != nil {
-			// Got an error while trying to reach the endpoint, trying the next
-			lastErr = err
+			c.logger.Warn(fmt.Sprintf("Icinga API not reachable at %s. Trying next endpoint", req.URL), "component", "icinga", "error", err.Error())
+			fmt.Fprintf(&endpointErrors, " error: '%s' from '%s'", req.URL, err.Error())
 
+			// Trying the next endpoint
 			continue
 		}
 
 		if resp.StatusCode >= 500 {
 			resp.Body.Close()
-			lastErr = fmt.Errorf("error %d from %s", resp.StatusCode, req.URL)
+			fmt.Fprintf(&endpointErrors, " error: '%d' from '%s'", resp.StatusCode, req.URL)
 
 			continue
 		}
 
 		return resp, nil
 	}
 
-	return nil, fmt.Errorf("all endpoints failed, last error: %w", lastErr)
+	return nil, fmt.Errorf("failed to call one of the configured endpoints. %s", endpointErrors.String())
 }
 
 // ProcessCheckResult handles a process-check-result for a given service
@@ -173,6 +175,7 @@ func (c *Client) GetHost(ctx context.Context, name string) (Host, error) {
 	errDecode := json.Unmarshal(bodyBytes, &result)
 
 	if errDecode != nil {
+		c.logger.Debug("Response body: " + string(bodyBytes))
 		return Host{}, fmt.Errorf("could not unmarshal JSON: %w", errDecode)
 	}
 
@@ -216,6 +219,7 @@ func (c *Client) GetService(ctx context.Context, name string) (Service, error) {
 	errDecode := json.Unmarshal(bodyBytes, &result)
 
 	if errDecode != nil {
+		c.logger.Debug("Response body: " + string(bodyBytes))
 		return Service{}, fmt.Errorf("could not unmarshal JSON: %w", errDecode)
 	}
 
@@ -265,6 +269,7 @@ func (c *Client) GetServices(ctx context.Context, filter QueryFilter) ([]Service
 	errDecode := json.Unmarshal(bodyBytes, &result)
 
 	if errDecode != nil {
+		c.logger.Debug("Response body: " + string(bodyBytes))
 		return []Service{}, fmt.Errorf("could not unmarshal JSON: %w", errDecode)
 	}
 

internal/icinga2/icinga_test.go

@@ -3,6 +3,7 @@
 package icinga2
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"io"
@@ -11,6 +12,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/NETWAYS/alertmanager-icinga-bridge/internal/config"
@@ -58,6 +60,33 @@ func TestServiceFullName(t *testing.T) {
 	}
 }
 
+func TestGetHost_WithInvalidReponse(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`<invalid JSON>`))
+	}))
+
+	defer server.Close()
+
+	var buf bytes.Buffer
+	logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
+
+	client := NewClient(testConfig(server.URL), logger)
+
+	_, err := client.GetHost(context.Background(), "unittest")
+
+	if err == nil {
+		t.Fatalf("expected error, got nil")
+	}
+
+	actual := buf.String()
+	expected := "Response body: <invalid JSON>"
+
+	if !strings.Contains(actual, expected) {
+		t.Fatalf("expected %v, got %v", expected, actual)
+	}
+}
+
 func TestGetHost(t *testing.T) {
 	datasetHost := "testdata/host.json"