retry network tracer init (#67)

* retry network tracer init

* retry tracer tests + configuration

* retry tracer tests + configuration

Author: zandrewitte

checks/net_common.go

@@ -3,9 +3,13 @@ package checks
 import (
 	"fmt"
 	"github.com/StackVista/stackstate-process-agent/model"
+	"github.com/StackVista/tcptracer-bpf/pkg/tracer"
 	"github.com/StackVista/tcptracer-bpf/pkg/tracer/common"
+	tracerConfig "github.com/StackVista/tcptracer-bpf/pkg/tracer/config"
+	log "github.com/cihub/seelog"
 	"net"
 	"strings"
+	"time"
 )
 
 type ip struct {
@@ -186,3 +190,32 @@ func calculateDirection(d common.Direction) model.ConnectionDirection {
 		return model.ConnectionDirection_none
 	}
 }
+
+// retryTracerInit tries to create a network tracer with a given retry duration and retry amount
+func retryTracerInit(retryDuration time.Duration, retryAmount int, config *tracerConfig.Config,
+	makeTracer func(*tracerConfig.Config) (tracer.Tracer, error)) (tracer.Tracer, error) {
+
+	retryTicker := time.NewTicker(retryDuration)
+	retriesLeft := retryAmount
+
+	var t tracer.Tracer
+	var err error
+
+retry:
+	for {
+		select {
+		case <-retryTicker.C:
+			t, err = makeTracer(config)
+			if err == nil {
+				break retry
+			}
+			log.Debugf("failed to create network tracer: %s. Retrying..", err)
+			retriesLeft = retriesLeft - 1
+			if retriesLeft == 0 {
+				break retry
+			}
+		}
+	}
+
+	return t, err
+}

checks/net_linux.go

@@ -4,14 +4,13 @@ package checks
 
 import (
 	"bytes"
-	"github.com/patrickmn/go-cache"
-
 	"github.com/StackVista/stackstate-process-agent/config"
 	"github.com/StackVista/stackstate-process-agent/model"
 	"github.com/StackVista/stackstate-process-agent/net"
 	"github.com/StackVista/tcptracer-bpf/pkg/tracer"
 	tracerConfig "github.com/StackVista/tcptracer-bpf/pkg/tracer/config"
 	log "github.com/cihub/seelog"
+	"github.com/patrickmn/go-cache"
 	"os"
 )
 
@@ -39,7 +38,7 @@ func (c *ConnectionsCheck) Init(cfg *config.AgentConfig, sysInfo *model.SystemIn
 		conf.MaxConnections = cfg.MaxPerMessage
 		conf.BackfillFromProc = cfg.NetworkInitialConnectionsFromProc
 
-		t, err := tracer.NewTracer(conf)
+		t, err := retryTracerInit(cfg.NetworkTracerInitRetryDuration, cfg.NetworkTracerInitRetryAmount, conf, tracer.NewTracer)
 		if err != nil {
 			log.Errorf("failed to create network tracer: %s.  Set the environment STS_NETWORK_TRACING_ENABLED to false to disable network connections reporting", err)
 			return

checks/net_linux_test.go

@@ -0,0 +1,76 @@
+package checks
+
+import (
+	"errors"
+	"github.com/StackVista/tcptracer-bpf/pkg/tracer"
+	tracerConfig "github.com/StackVista/tcptracer-bpf/pkg/tracer/config"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestConnectionsCheck_retryTracerInit(t *testing.T) {
+	retryDuration := 40 * time.Millisecond
+	retryAmount := 3
+	testRetry := 0
+
+	for _, tc := range []struct {
+		name               string
+		mockMakeTracerFunc func(config *tracerConfig.Config) (tracer.Tracer, error)
+		expectedTracer     tracer.Tracer
+		expectedError      string
+	}{
+		{
+			name: "Returns the tracer when the make tracer function returns one",
+			mockMakeTracerFunc: func(config *tracerConfig.Config) (tracer.Tracer, error) {
+				return &tracer.LinuxTracer{}, nil
+			},
+			expectedTracer: &tracer.LinuxTracer{},
+			expectedError:  "",
+		},
+		{
+			name: "Returns the tracer when the amount of retries are below the configured amount. ie retrying makes it work.",
+			mockMakeTracerFunc: func(config *tracerConfig.Config) (tracer.Tracer, error) {
+				if testRetry < retryAmount-1 {
+					testRetry = testRetry + 1
+					return nil, errors.New("failed to create tracer")
+				}
+				return &tracer.LinuxTracer{}, nil
+			},
+			expectedTracer: &tracer.LinuxTracer{},
+			expectedError:  "",
+		},
+		{
+			name: "Returns an error when max retries are reached.",
+			mockMakeTracerFunc: func(config *tracerConfig.Config) (tracer.Tracer, error) {
+				if testRetry <= retryAmount-1 {
+					testRetry = testRetry + 1
+					return nil, errors.New("failed to create tracer")
+				}
+				return &tracer.LinuxTracer{}, nil
+			},
+			expectedTracer: nil,
+			expectedError:  "failed to create tracer",
+		},
+		{
+			name: "Return an error when the make tracer function returns an error",
+			mockMakeTracerFunc: func(config *tracerConfig.Config) (tracer.Tracer, error) {
+				return nil, errors.New("failed to create tracer")
+			},
+			expectedError: "failed to create tracer",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			tr, err := retryTracerInit(retryDuration, retryAmount, tracerConfig.DefaultConfig, tc.mockMakeTracerFunc)
+			if tc.expectedError != "" {
+				assert.EqualError(t, err, tc.expectedError)
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.EqualValues(t, tc.expectedTracer, tr)
+
+			// reset the test retries
+			testRetry = 0
+		})
+	}
+}

config/config.go

@@ -118,6 +118,8 @@ type AgentConfig struct {
 	NetworkInitialConnectionsFromProc bool
 	NetworkTracerSocketPath           string
 	NetworkTracerLogFile              string
+	NetworkTracerInitRetryDuration    time.Duration
+	NetworkTracerInitRetryAmount      int
 
 	// Check config
 	EnabledChecks  []string
@@ -236,6 +238,8 @@ func NewDefaultAgentConfig() *AgentConfig {
 		NetworkInitialConnectionsFromProc: true,
 		NetworkTracerSocketPath:           defaultNetworkTracerSocketPath,
 		NetworkTracerLogFile:              defaultNetworkLogFilePath,
+		NetworkTracerInitRetryDuration:    5 * time.Second,
+		NetworkTracerInitRetryAmount:      3,
 
 		// Check config
 		EnabledChecks: containerChecks,
@@ -693,6 +697,15 @@ func mergeEnvironmentVariables(c *AgentConfig) *AgentConfig {
 		c.NetworkRelationCacheDurationMin = time.Duration(durationS) * time.Minute
 	}
 
+	if v := os.Getenv("STS_NETWORK_TRACER_INIT_RETRY_DURATION_SEC"); v != "" {
+		durationS, _ := strconv.Atoi(v)
+		c.NetworkTracerInitRetryDuration = time.Duration(durationS) * time.Second
+	}
+
+	if v, err := strconv.Atoi(os.Getenv("STS_NETWORK_TRACER_INIT_RETRY_AMOUNT")); err == nil {
+		c.NetworkTracerInitRetryAmount = v
+	}
+
 	if v, err := strconv.Atoi(os.Getenv("STS_PROCESS_FILTER_SHORT_LIVED_QUALIFIER_SECS")); err == nil {
 		setProcessFilters(c, true, v)
 	}

config/config_test.go

@@ -341,6 +341,19 @@ func TestSetBlacklistFromEnv(t *testing.T) {
 	os.Unsetenv("STS_PROCESS_BLACKLIST_INCLUSIONS_MEM_THRESHOLD")
 }
 
+func TestSetNetworkTracerInitRetryFromEnv(t *testing.T) {
+	os.Setenv("STS_NETWORK_TRACER_INIT_RETRY_DURATION_SEC", "30")
+	os.Setenv("STS_NETWORK_TRACER_INIT_RETRY_AMOUNT", "4")
+
+	agentConfig, _ := NewAgentConfig(nil, nil, nil)
+
+	assert.Equal(t, 30*time.Second, agentConfig.NetworkTracerInitRetryDuration)
+	assert.Equal(t, 4, agentConfig.NetworkTracerInitRetryAmount)
+
+	os.Unsetenv("STS_NETWORK_TRACER_INIT_RETRY_DURATION_SEC")
+	os.Unsetenv("STS_NETWORK_TRACER_INIT_RETRY_AMOUNT")
+}
+
 func TestOnlyEnvConfig(t *testing.T) {
 	// setting an API Key should be enough to generate valid config
 	os.Setenv("DD_API_KEY", "apikey_from_env")
@@ -1280,3 +1293,19 @@ func TestStackStatePreferPROCESS_AGENT_URLOverYamlsts_sts_url(t *testing.T) {
 	assert.Equal("apikey_30", ep.APIKey)
 	assert.Equal("process-endpoint.test.stackstate.com", ep.Endpoint.Hostname())
 }
+
+func TestNetworkTracerInitRetry_FromYaml(t *testing.T) {
+	var ddy YamlAgentConfig
+	err := yaml.Unmarshal([]byte(strings.Join([]string{
+		"network_tracer_config:",
+		"  network_tracer_retry_init_duration_sec: 50",
+		"  network_tracer_retry_init_amount: 10",
+	}, "\n")), &ddy)
+	assert.NoError(t, err)
+
+	agentConfig, err := NewAgentConfig(nil, &ddy, nil)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 10, agentConfig.NetworkTracerInitRetryAmount)
+	assert.Equal(t, 50*time.Second, agentConfig.NetworkTracerInitRetryDuration)
+}

config/yaml_config.go

@@ -121,6 +121,10 @@ type YamlAgentConfig struct {
 		UnixSocketPath string `yaml:"nettracer_socket"`
 		// The full path to the file where network-tracer logs will be written.
 		LogFile string `yaml:"log_file"`
+		// A integer indicating the amount of seconds for the retry interval for initializing the network tracer.
+		NetworkTracerInitRetryDuration int `yaml:"network_tracer_retry_init_duration_sec"`
+		// A integer indicating the amount of retries to use for initializing the network tracer.
+		NetworkTracerInitRetryAmount int `yaml:"network_tracer_retry_init_amount"`
 	} `yaml:"network_tracer_config"`
 }
 
@@ -247,6 +251,14 @@ func mergeYamlConfig(agentConf *AgentConfig, yc *YamlAgentConfig) (*AgentConfig,
 		agentConf.NetworkRelationCacheDurationMin = time.Duration(yc.Process.NetworkRelationCacheDurationMin) * time.Minute
 	}
 
+	if yc.Network.NetworkTracerInitRetryDuration > 0 {
+		agentConf.NetworkTracerInitRetryDuration = time.Duration(yc.Network.NetworkTracerInitRetryDuration) * time.Second
+	}
+
+	if yc.Network.NetworkTracerInitRetryAmount > 0 {
+		agentConf.NetworkTracerInitRetryAmount = yc.Network.NetworkTracerInitRetryAmount
+	}
+
 	// DataScrubber
 	if yc.Process.ScrubArgs != nil {
 		agentConf.Scrubber.Enabled = *yc.Process.ScrubArgs