fix(netfault): replace root qdisc instead of add to support pre-existing qdiscs

joshiste · joshiste · commit 9ad77544895f · 2026-05-28T23:58:21.000+02:00
Network attacks using tc (delay, loss, corruption, bandwidth) failed on
hosts where the kernel had already attached a root qdisc to the target
interface (e.g. `mq` on GKE COS, EKS, AKS). `tc qdisc add ... root`
returned `NLM_F_REPLACE needed to override` and the attack could not
start.

Switch the root qdisc command to `tc qdisc replace ... root` on apply.
On revert we still `qdisc del root`; the kernel then re-attaches its
default qdisc (`mq` on multi-queue devices, `noqueue` on veth, otherwise
the configured `net.core.default_qdisc`), so common cloud node setups
are restored to their pre-attack state.

Add a preflight inspection that runs `tc qdisc show` on each affected
interface and emits a warning if the root qdisc is not one the kernel
auto-restores (anything other than mq, noqueue, pfifo_fast, fq_codel,
fq). Callers receive the warnings via the new `Apply` return value.

Breaking change: `netfault.Apply` now returns `([]string, error)`.
diff --git a/go/action_kit_commons/CHANGELOG.md b/go/action_kit_commons/CHANGELOG.md
@@ -1,5 +1,17 @@
 # Changelog
 
+## 1.8.0
+
+- netfault: use `tc qdisc replace` (instead of `add`) for the root qdisc in
+  delay/loss/corruption/bandwidth attacks so they no longer fail on hosts
+  with a pre-existing root qdisc (e.g. `mq` on GKE COS / EKS / AKS).
+- netfault: add preflight check that warns when an interface has a
+  user-installed root qdisc (anything other than `mq`, `noqueue`,
+  `pfifo_fast`, `fq_codel`, `fq`); the kernel default will be restored
+  after revert in that case.
+- **Breaking:** `netfault.Apply` now returns `([]string, error)` — the
+  string slice contains preflight warnings to surface to the user.
+
 ## 1.6.1
 
 - Add UseMangleChain to TcpResetOpts to enable tcp reset on istio
diff --git a/go/action_kit_commons/network/netfault/bandwidth.go b/go/action_kit_commons/network/netfault/bandwidth.go
@@ -50,6 +50,10 @@ func (o *LimitBandwidthOpts) ipCommands(_ family, _ mode) ([]string, error) {
 	return nil, nil
 }
 
+func (o *LimitBandwidthOpts) affectedInterfaces() []string {
+	return o.Interfaces
+}
+
 func (o *LimitBandwidthOpts) tcCommands(mode mode) ([]string, error) {
 	var cmds []string
 
@@ -63,7 +67,7 @@ func (o *LimitBandwidthOpts) tcCommands(mode mode) ([]string, error) {
 
 	filter := optimizeFilter(o.Filter)
 	for _, ifc := range o.Interfaces {
-		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s root handle 1: htb default 30", mode, ifc))
+		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s root handle 1: htb default 30", rootQdiscVerb(mode), ifc))
 		cmds = append(cmds, fmt.Sprintf("class %s dev %s parent 1: classid %s htb rate %s", mode, ifc, handleInclude, o.Bandwidth))
 
 		filterCmds, err := tcCommandsForFilter(mode, filter, ifc)
diff --git a/go/action_kit_commons/network/netfault/bandwidth_test.go b/go/action_kit_commons/network/netfault/bandwidth_test.go
@@ -61,7 +61,7 @@ func TestLimitBandwidthOpts_TcCommands(t *testing.T) {
 				Bandwidth:  "100mbit",
 				Interfaces: []string{"eth0"},
 			},
-			wantAdd: []byte(`qdisc add dev eth0 root handle 1: htb default 30
+			wantAdd: []byte(`qdisc replace dev eth0 root handle 1: htb default 30
 class add dev eth0 parent 1: classid 1:3 htb rate 100mbit
 filter add dev eth0 protocol ip parent 1: prio 1 u32 match ip src 192.168.2.1/32 match ip sport 80 0xffff flowid 1:1
 filter add dev eth0 protocol ip parent 1: prio 2 u32 match ip dst 192.168.2.1/32 match ip dport 80 0xffff flowid 1:1
diff --git a/go/action_kit_commons/network/netfault/blackhole.go b/go/action_kit_commons/network/netfault/blackhole.go
@@ -87,6 +87,10 @@ func (o *BlackholeOpts) tcCommands(_ mode) ([]string, error) {
 	return nil, nil
 }
 
+func (o *BlackholeOpts) affectedInterfaces() []string {
+	return nil
+}
+
 func (o *BlackholeOpts) String() string {
 	var sb strings.Builder
 	sb.WriteString("blocking traffic ")
diff --git a/go/action_kit_commons/network/netfault/delay.go b/go/action_kit_commons/network/netfault/delay.go
@@ -63,6 +63,10 @@ func (o *DelayOpts) ipCommands(_ family, _ mode) ([]string, error) {
 	return nil, nil
 }
 
+func (o *DelayOpts) affectedInterfaces() []string {
+	return o.Interfaces
+}
+
 const steadybitDelayFwMark uint32 = 0x1
 
 func (o *DelayOpts) iptablesScripts(mode mode) ([]string, []string, error) {
@@ -174,7 +178,7 @@ func (o *DelayOpts) tcCommands(mode mode) ([]string, error) {
 
 	filter := optimizeFilter(o.Filter)
 	for _, ifc := range o.Interfaces {
-		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", mode, ifc))
+		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", rootQdiscVerb(mode), ifc))
 		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s parent %s handle 30: netem delay %dms %dms", mode, ifc, handleInclude, o.Delay.Milliseconds(), o.Jitter.Milliseconds()))
 
 		if o.TcpPshOnly {
diff --git a/go/action_kit_commons/network/netfault/delay_test.go b/go/action_kit_commons/network/netfault/delay_test.go
@@ -55,7 +55,7 @@ func TestDelayOpts_TcCommands(t *testing.T) {
 				Jitter:     10 * time.Millisecond,
 				Interfaces: []string{"eth0"},
 			},
-			wantAdd: []byte(`qdisc add dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+			wantAdd: []byte(`qdisc replace dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 qdisc add dev eth0 parent 1:3 handle 30: netem delay 100ms 10ms
 filter add dev eth0 protocol ip parent 1: prio 1 u32 match ip src 192.168.2.1/32 match ip sport 80 0xffff flowid 1:1
 filter add dev eth0 protocol ip parent 1: prio 2 u32 match ip dst 192.168.2.1/32 match ip dport 80 0xffff flowid 1:1
@@ -112,7 +112,7 @@ qdisc del dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 				Interfaces: []string{"eth0"},
 				TcpPshOnly: true,
 			},
-			wantAdd: []byte(`qdisc add dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+			wantAdd: []byte(`qdisc replace dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 qdisc add dev eth0 parent 1:3 handle 30: netem delay 100ms 10ms
 filter add dev eth0 protocol ip parent 1: prio 1 handle 0x1 fw flowid 1:3
 filter add dev eth0 protocol ipv6 parent 1: prio 2 handle 0x1 fw flowid 1:3
@@ -141,7 +141,7 @@ qdisc del dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 				Jitter:     10 * time.Millisecond,
 				Interfaces: []string{"eth0"},
 			},
-			wantAdd: []byte(`qdisc add dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+			wantAdd: []byte(`qdisc replace dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 qdisc add dev eth0 parent 1:3 handle 30: netem delay 100ms 10ms
 filter add dev eth0 protocol ip parent 1: prio 1 u32 match ip src 192.168.2.1/32 match ip sport 80 0xffff flowid 1:1
 filter add dev eth0 protocol ip parent 1: prio 2 u32 match ip dst 192.168.2.1/32 match ip dport 80 0xffff flowid 1:1
diff --git a/go/action_kit_commons/network/netfault/netfault.go b/go/action_kit_commons/network/netfault/netfault.go
@@ -42,10 +42,22 @@ type CommandRunner interface {
 	id() string
 }
 
-func Apply(ctx context.Context, runner CommandRunner, opts Opts) error {
-	return generateAndRunCommands(ctx, runner, opts, modeAdd)
+// Apply installs the attack described by opts. It returns warnings about
+// pre-existing root qdiscs that will be overwritten and not fully restored
+// on revert; callers should surface these to the user. An error indicates
+// that one or more apply steps failed.
+func Apply(ctx context.Context, runner CommandRunner, opts Opts) ([]string, error) {
+	warnings := preflightWarnings(ctx, runner, opts.affectedInterfaces())
+	if err := generateAndRunCommands(ctx, runner, opts, modeAdd); err != nil {
+		return warnings, err
+	}
+	return warnings, nil
 }
 
+// Revert removes the attack described by opts. After `qdisc del root` the
+// kernel auto-attaches the device's default qdisc (mq/noqueue/fq_codel/...).
+// Anything not in safeRootQdiscKinds will not be restored to its original
+// configuration; the user was warned during Apply.
 func Revert(ctx context.Context, runner CommandRunner, opts Opts) error {
 	return generateAndRunCommands(ctx, runner, opts, modeDelete)
 }
diff --git a/go/action_kit_commons/network/netfault/netfault_test.go b/go/action_kit_commons/network/netfault/netfault_test.go
@@ -45,7 +45,7 @@ func TestApply_Order_IptablesBeforeTcWhenTcpPshOnly(t *testing.T) {
 	}
 
 	r := &fakeRunner{}
-	err := Apply(context.Background(), r, opts)
+	_, err := Apply(context.Background(), r, opts)
 	assert.NoError(t, err)
 
 	iptablesIdx := -1
@@ -54,7 +54,7 @@ func TestApply_Order_IptablesBeforeTcWhenTcpPshOnly(t *testing.T) {
 		if len(c.args) > 0 && c.args[0] == "iptables-restore" {
 			iptablesIdx = i
 		}
-		if len(c.args) > 0 && c.args[0] == "tc" && len(c.cmds) > 0 && strings.HasPrefix(c.cmds[0], "qdisc add") {
+		if len(c.args) > 0 && c.args[0] == "tc" && len(c.cmds) > 0 && strings.HasPrefix(c.cmds[0], "qdisc replace") {
 			tcBatchIdx = i
 		}
 	}
diff --git a/go/action_kit_commons/network/netfault/packageCorruption.go b/go/action_kit_commons/network/netfault/packageCorruption.go
@@ -49,12 +49,16 @@ func (o *CorruptPackagesOpts) ipCommands(_ family, _ mode) ([]string, error) {
 	return nil, nil
 }
 
+func (o *CorruptPackagesOpts) affectedInterfaces() []string {
+	return o.Interfaces
+}
+
 func (o *CorruptPackagesOpts) tcCommands(mode mode) ([]string, error) {
 	var cmds []string
 
 	filter := optimizeFilter(o.Filter)
 	for _, ifc := range o.Interfaces {
-		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", mode, ifc))
+		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", rootQdiscVerb(mode), ifc))
 		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s parent %s handle 30: netem corrupt %d%%", mode, ifc, handleInclude, o.Corruption))
 
 		filterCmds, err := tcCommandsForFilter(mode, filter, ifc)
diff --git a/go/action_kit_commons/network/netfault/packageCorruption_test.go b/go/action_kit_commons/network/netfault/packageCorruption_test.go
@@ -38,7 +38,7 @@ func TestCorruptPackagesOpts_TcCommands(t *testing.T) {
 				Corruption: 90,
 				Interfaces: []string{"eth0"},
 			},
-			wantAdd: []byte(`qdisc add dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+			wantAdd: []byte(`qdisc replace dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 qdisc add dev eth0 parent 1:3 handle 30: netem corrupt 90%
 filter add dev eth0 protocol ip parent 1: prio 1 u32 match ip src 192.168.2.1/32 match ip sport 80 0xffff flowid 1:1
 filter add dev eth0 protocol ip parent 1: prio 2 u32 match ip dst 192.168.2.1/32 match ip dport 80 0xffff flowid 1:1
diff --git a/go/action_kit_commons/network/netfault/packageLoss.go b/go/action_kit_commons/network/netfault/packageLoss.go
@@ -48,12 +48,16 @@ func (o *PackageLossOpts) ipCommands(_ family, _ mode) ([]string, error) {
 	return nil, nil
 }
 
+func (o *PackageLossOpts) affectedInterfaces() []string {
+	return o.Interfaces
+}
+
 func (o *PackageLossOpts) tcCommands(mode mode) ([]string, error) {
 	var cmds []string
 
 	filter := optimizeFilter(o.Filter)
 	for _, ifc := range o.Interfaces {
-		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", mode, ifc))
+		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", rootQdiscVerb(mode), ifc))
 		cmds = append(cmds, fmt.Sprintf("qdisc %s dev %s parent %s handle 30: netem loss random %d%%", mode, ifc, handleInclude, o.Loss))
 
 		filterCmds, err := tcCommandsForFilter(mode, filter, ifc)
diff --git a/go/action_kit_commons/network/netfault/packageLoss_test.go b/go/action_kit_commons/network/netfault/packageLoss_test.go
@@ -38,7 +38,7 @@ func TestPackageLossOpts_TcCommands(t *testing.T) {
 				Loss:       90,
 				Interfaces: []string{"eth0"},
 			},
-			wantAdd: []byte(`qdisc add dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+			wantAdd: []byte(`qdisc replace dev eth0 root handle 1: prio priomap 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 qdisc add dev eth0 parent 1:3 handle 30: netem loss random 90%
 filter add dev eth0 protocol ip parent 1: prio 1 u32 match ip src 192.168.2.1/32 match ip sport 80 0xffff flowid 1:1
 filter add dev eth0 protocol ip parent 1: prio 2 u32 match ip dst 192.168.2.1/32 match ip dport 80 0xffff flowid 1:1
diff --git a/go/action_kit_commons/network/netfault/preflight.go b/go/action_kit_commons/network/netfault/preflight.go
@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: 2026 Steadybit GmbH
+//go:build !windows
+
+package netfault
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+)
+
+// safeRootQdiscKinds lists qdisc kinds that the Linux kernel re-attaches
+// automatically after `tc qdisc del dev <ifc> root`. After we revert an
+// attack, the kernel will restore one of these without losing user
+// configuration:
+//
+//   - mq:          auto-attached on multi-queue devices (e.g. GKE COS eth0);
+//     not really user-installed, kernel re-creates it from the
+//     device's TX queue count.
+//   - noqueue:     attached on devices with IFF_NO_QUEUE (veth, lo, dummy,
+//     bridge, most pod eth0).
+//   - pfifo_fast:  kernel built-in fallback when default_qdisc is unset.
+//   - fq_codel:    `net.core.default_qdisc` on every modern mainstream distro
+//     (Ubuntu, RHEL, Debian, AL2/2023, Azure Linux, RHCOS).
+//   - fq:          occasionally the default on BBR-tuned hosts.
+//
+// Anything else (htb, hfsc, cake, tbf, prio, sfq, ...) indicates the
+// interface was deliberately configured by a sysadmin or CNI bandwidth
+// plugin. After our attack reverts, the kernel default will be installed
+// instead — the original configuration is lost. We emit a warning in that
+// case so the user knows.
+var safeRootQdiscKinds = map[string]struct{}{
+	"mq":         {},
+	"noqueue":    {},
+	"pfifo_fast": {},
+	"fq_codel":   {},
+	"fq":         {},
+}
+
+// preflightWarnings inspects the root qdisc on each given interface and
+// returns a human-readable warning for every interface that has a
+// non-kernel-default root qdisc. The attack will still be applied
+// (`tc qdisc replace` overwrites whatever is there) but the user-installed
+// qdisc will not be restored on revert.
+//
+// Inspection failures (e.g. tc not available, parse error) are logged at
+// warn level and do not cause the apply to fail; the attack should still
+// proceed.
+func preflightWarnings(ctx context.Context, runner CommandRunner, interfaces []string) []string {
+	if len(interfaces) == 0 {
+		return nil
+	}
+
+	var warnings []string
+	for _, ifc := range interfaces {
+		kind, err := inspectRootQdisc(ctx, runner, ifc)
+		if err != nil {
+			log.Warn().Err(err).Str("interface", ifc).Msg("failed to inspect root qdisc; skipping preflight check")
+			continue
+		}
+		if kind == "" {
+			continue
+		}
+		if _, safe := safeRootQdiscKinds[kind]; safe {
+			continue
+		}
+		warnings = append(warnings, fmt.Sprintf(
+			"Pre-existing qdisc %q on interface %q will be replaced during the attack. After the attack ends the kernel will restore its default qdisc, which may differ from the original configuration.",
+			kind, ifc,
+		))
+	}
+	return warnings
+}
+
+// inspectRootQdisc runs `tc qdisc show dev <ifc>` inside the target network
+// namespace and returns the kind of the root qdisc (e.g. "mq", "fq_codel",
+// "htb"). Returns an empty kind with no error if no root qdisc is reported
+// (e.g. on interfaces the runner cannot see).
+//
+// We deliberately parse the human-readable text output rather than
+// `tc -json qdisc show` so the check works on older iproute2 versions that
+// don't support -json. Only the qdisc kind is required; we don't need
+// options here.
+func inspectRootQdisc(ctx context.Context, runner CommandRunner, ifc string) (string, error) {
+	out, err := runner.run(ctx, []string{"tc", "qdisc", "show", "dev", ifc}, nil)
+	if err != nil {
+		return "", fmt.Errorf("tc qdisc show dev %s failed: %w", ifc, err)
+	}
+	return parseRootQdiscKind(out), nil
+}
+
+// parseRootQdiscKind returns the qdisc kind from the first line in `tc qdisc
+// show` output that has `root` as the parent. Lines without a parent (root
+// qdiscs) are formatted as:
+//
+//	qdisc <kind> <handle> dev <ifc> root [refcnt N] [...]
+//
+// `ingress` and `clsact` qdiscs use a `parent ffff:fff1` field and are
+// skipped — they are attached separately from the root and unaffected by
+// our attacks.
+func parseRootQdiscKind(out string) string {
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		fields := strings.Fields(line)
+		// Expected layout: ["qdisc", "<kind>", "<handle>", "dev", "<ifc>", "root", ...]
+		if len(fields) < 6 || fields[0] != "qdisc" {
+			continue
+		}
+		if !containsToken(fields[3:], "root") {
+			continue
+		}
+		return fields[1]
+	}
+	return ""
+}
+
+func containsToken(tokens []string, t string) bool {
+	for _, s := range tokens {
+		if s == t {
+			return true
+		}
+	}
+	return false
+}
diff --git a/go/action_kit_commons/network/netfault/preflight_test.go b/go/action_kit_commons/network/netfault/preflight_test.go
diff --git a/go/action_kit_commons/network/netfault/runner_runc_test.go b/go/action_kit_commons/network/netfault/runner_runc_test.go
diff --git a/go/action_kit_commons/network/netfault/tcp_reset.go b/go/action_kit_commons/network/netfault/tcp_reset.go
diff --git a/go/action_kit_commons/network/netfault/types.go b/go/action_kit_commons/network/netfault/types.go
diff --git a/go/action_kit_commons/network/netfault/utils.go b/go/action_kit_commons/network/netfault/utils.go

Original file line number	Diff line number	Diff line change
`@@ -45,7 +45,7 @@ func TestApply_Order_IptablesBeforeTcWhenTcpPshOnly(t *testing.T) {`
`45`	`45`	`}`
`46`	`46`
`47`	`47`	`r := &fakeRunner{}`
`48`		`- err := Apply(context.Background(), r, opts)`
	`48`	`+ _, err := Apply(context.Background(), r, opts)`
`49`	`49`	`assert.NoError(t, err)`
`50`	`50`
`51`	`51`	`iptablesIdx := -1`
`@@ -54,7 +54,7 @@ func TestApply_Order_IptablesBeforeTcWhenTcpPshOnly(t *testing.T) {`
`54`	`54`	`if len(c.args) > 0 && c.args[0] == "iptables-restore" {`
`55`	`55`	`iptablesIdx = i`
`56`	`56`	`}`
`57`		`- if len(c.args) > 0 && c.args[0] == "tc" && len(c.cmds) > 0 && strings.HasPrefix(c.cmds[0], "qdisc add") {`
	`57`	`+ if len(c.args) > 0 && c.args[0] == "tc" && len(c.cmds) > 0 && strings.HasPrefix(c.cmds[0], "qdisc replace") {`
`58`	`58`	`tcBatchIdx = i`
`59`	`59`	`}`
`60`	`60`	`}`