feat(modules): add nsq and pulsar exposure modules (#297)

* feat(modules): add nsq and pulsar exposure modules

* chore(modules): trim redundant module header comments

drop the leading comment restating id/name on nsq-stats-exposure and pulsar-metrics-exposure

Author: TBX3D

internal/modules/nsq_stats_exposure_test.go

@@ -0,0 +1,95 @@
+package modules_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/vmfunc/sif/internal/modules"
+)
+
+func runNSQModule(t *testing.T, file string, status int, body string) *modules.Result {
+	t.Helper()
+	def, err := modules.ParseYAMLModule(file)
+	if err != nil {
+		t.Fatalf("parse %s: %v", file, err)
+	}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+	defer srv.Close()
+
+	res, err := modules.ExecuteHTTPModule(context.Background(), srv.URL, def, modules.Options{
+		Timeout: 5 * time.Second,
+		Threads: 2,
+	})
+	if err != nil {
+		t.Fatalf("execute %s: %v", file, err)
+	}
+	return res
+}
+
+func nsqExtract(res *modules.Result, key string) string {
+	for _, f := range res.Findings {
+		if v := f.Extracted[key]; v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+func TestNSQStatsExposureModule(t *testing.T) {
+	const nsq = "../../modules/recon/nsq-stats-exposure.yaml"
+
+	t.Run("an nsqd stats response is flagged with its version", func(t *testing.T) {
+		body := `{"version":"1.3.0","health":"OK","start_time":1717000000,"topics":[` +
+			`{"topic_name":"orders","channels":[{"channel_name":"billing","depth":0,` +
+			`"backend_depth":0,"in_flight_count":0,"message_count":1204,"clients":[` +
+			`{"client_id":"worker-1","remote_address":"10.0.4.12:51322"}]}],` +
+			`"depth":0,"message_count":1204}],"producers":[],` +
+			`"memory":{"heap_objects":1024}}`
+		res := runNSQModule(t, nsq, 200, body)
+		if len(res.Findings) == 0 {
+			t.Fatal("expected an nsq finding")
+		}
+		if v := nsqExtract(res, "nsq_version"); v != "1.3.0" {
+			t.Errorf("nsq_version=%q, want 1.3.0", v)
+		}
+	})
+
+	t.Run("a version and topics without health and start_time is not flagged", func(t *testing.T) {
+		body := `{"version":"1.3.0","topics":[{"topic_name":"orders"}]}`
+		if res := runNSQModule(t, nsq, 200, body); len(res.Findings) > 0 {
+			t.Errorf("a partial body should not match nsq, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("an unhealthy status value is not flagged", func(t *testing.T) {
+		body := `{"version":"1.3.0","health":"NOK: no topics","start_time":1717000000,"topics":[]}`
+		if res := runNSQModule(t, nsq, 200, body); len(res.Findings) > 0 {
+			t.Errorf("a non-OK health value should not match, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("a generic monitoring body is not a leak", func(t *testing.T) {
+		body := `{"status":"healthy","version":"3.1.0","uptime_seconds":1234}`
+		if res := runNSQModule(t, nsq, 200, body); len(res.Findings) > 0 {
+			t.Errorf("a generic health body should not match nsq, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("a 401 login page is not a leak", func(t *testing.T) {
+		if res := runNSQModule(t, nsq, 401, "Unauthorized"); len(res.Findings) > 0 {
+			t.Errorf("a 401 should not match, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("a 404 is not a leak", func(t *testing.T) {
+		if res := runNSQModule(t, nsq, 404, "not found"); len(res.Findings) > 0 {
+			t.Errorf("a 404 should not match, got %d findings", len(res.Findings))
+		}
+	})
+}

internal/modules/pulsar_metrics_exposure_test.go

@@ -0,0 +1,87 @@
+package modules_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/vmfunc/sif/internal/modules"
+)
+
+func runPulsarModule(t *testing.T, file string, status int, body string) *modules.Result {
+	t.Helper()
+	def, err := modules.ParseYAMLModule(file)
+	if err != nil {
+		t.Fatalf("parse %s: %v", file, err)
+	}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+	defer srv.Close()
+
+	res, err := modules.ExecuteHTTPModule(context.Background(), srv.URL, def, modules.Options{
+		Timeout: 5 * time.Second,
+		Threads: 2,
+	})
+	if err != nil {
+		t.Fatalf("execute %s: %v", file, err)
+	}
+	return res
+}
+
+func TestPulsarMetricsExposureModule(t *testing.T) {
+	const pulsar = "../../modules/recon/pulsar-metrics-exposure.yaml"
+
+	t.Run("a pulsar broker metrics page is flagged", func(t *testing.T) {
+		body := "# HELP pulsar_topics_count number of topics owned by this broker\n" +
+			"# TYPE pulsar_topics_count gauge\n" +
+			"pulsar_topics_count{cluster=\"standalone\"} 12\n" +
+			"# HELP pulsar_subscriptions_count number of pulsar subscriptions\n" +
+			"# TYPE pulsar_subscriptions_count gauge\n" +
+			"pulsar_subscriptions_count{cluster=\"standalone\"} 4\n"
+		res := runPulsarModule(t, pulsar, 200, body)
+		if len(res.Findings) == 0 {
+			t.Fatal("expected a pulsar finding")
+		}
+	})
+
+	t.Run("topics count without a TYPE line is not flagged", func(t *testing.T) {
+		body := "pulsar_topics_count{cluster=\"standalone\"} 12\n" +
+			"pulsar_subscriptions_count{cluster=\"standalone\"} 4\n"
+		if res := runPulsarModule(t, pulsar, 200, body); len(res.Findings) > 0 {
+			t.Errorf("a body without a prometheus TYPE line should not match, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("an unrelated prometheus exporter is not flagged", func(t *testing.T) {
+		body := "# HELP node_cpu_seconds_total seconds the cpus spent in each mode\n" +
+			"# TYPE node_cpu_seconds_total counter\n" +
+			"node_cpu_seconds_total{cpu=\"0\",mode=\"idle\"} 12345\n"
+		if res := runPulsarModule(t, pulsar, 200, body); len(res.Findings) > 0 {
+			t.Errorf("node_exporter output should not match pulsar, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("only subscriptions count without topics count is not flagged", func(t *testing.T) {
+		body := "# TYPE pulsar_subscriptions_count gauge\n" +
+			"pulsar_subscriptions_count{cluster=\"standalone\"} 4\n"
+		if res := runPulsarModule(t, pulsar, 200, body); len(res.Findings) > 0 {
+			t.Errorf("a partial metric set should not match pulsar, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("a 401 is not a leak", func(t *testing.T) {
+		if res := runPulsarModule(t, pulsar, 401, "Unauthorized"); len(res.Findings) > 0 {
+			t.Errorf("a 401 should not match, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("a 404 is not a leak", func(t *testing.T) {
+		if res := runPulsarModule(t, pulsar, 404, "not found"); len(res.Findings) > 0 {
+			t.Errorf("a 404 should not match, got %d findings", len(res.Findings))
+		}
+	})
+}

modules/recon/nsq-stats-exposure.yaml

@@ -0,0 +1,47 @@
+id: nsq-stats-exposure
+info:
+  name: NSQ Stats API Exposure
+  author: sif
+  severity: high
+  description: Detects an exposed nsqd HTTP API through its unauthenticated stats endpoint. nsqd ships without any authentication mechanism, so the same reachable API also exposes topic/channel pause, delete and publish endpoints
+  tags: [nsq, nsqd, messaging, streaming, api, exposure, unauth, recon]
+
+type: http
+
+http:
+  method: GET
+  paths:
+    - "{{BaseURL}}/stats?format=json"
+
+  matchers:
+    - type: status
+      status:
+        - 200
+
+    - type: word
+      part: body
+      words:
+        - "\"topics\""
+
+    - type: regex
+      part: body
+      regex:
+        - '"health"\s*:\s*"OK"'
+
+    - type: word
+      part: body
+      words:
+        - "\"start_time\""
+
+    - type: word
+      part: body
+      words:
+        - "\"version\""
+
+  extractors:
+    - type: regex
+      name: nsq_version
+      part: body
+      regex:
+        - '"version"\s*:\s*"([^"]+)"'
+      group: 1

modules/recon/pulsar-metrics-exposure.yaml

@@ -0,0 +1,34 @@
+id: pulsar-metrics-exposure
+info:
+  name: Apache Pulsar Broker Metrics Exposure
+  author: sif
+  severity: medium
+  description: Detects an exposed Apache Pulsar broker Prometheus metrics endpoint that leaks per-topic and per-subscription throughput and backlog figures without authentication
+  tags: [pulsar, apache, messaging, streaming, metrics, prometheus, exposure, unauth, recon]
+
+type: http
+
+http:
+  method: GET
+  paths:
+    - "{{BaseURL}}/metrics"
+
+  matchers:
+    - type: status
+      status:
+        - 200
+
+    - type: word
+      part: body
+      words:
+        - "pulsar_topics_count"
+
+    - type: word
+      part: body
+      words:
+        - "pulsar_subscriptions_count"
+
+    - type: regex
+      part: body
+      regex:
+        - '(?m)^# TYPE pulsar_'