Fix process and file descriptor leaks in Salt Master

.github/workflows/nightly-stress-test.yml

@@ -0,0 +1,90 @@
+name: Nightly Stress Test
+
+on:
+  schedule:
+    - cron: '0 2 * * *' # 2 AM UTC
+  workflow_dispatch:
+    inputs:
+      duration:
+        description: 'Duration of the stress test (e.g., 30m, 1h)'
+        required: true
+        default: '30m'
+
+jobs:
+  stress-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Build and Start Environment
+        run: |
+          cd tests/monitoring
+          docker compose build
+          docker compose up -d
+          sleep 30 # Wait for initialization
+
+      - name: Verify Connections
+        run: |
+          docker exec salt-master salt '*' test.ping
+
+      - name: Run Aggressive Stress Test
+        run: |
+          cd tests/monitoring
+          chmod +x stress_test.sh stress_api.sh
+          # Run in background and wait for defined duration
+          ./stress_test.sh &
+          STRESS_PID=$!
+
+          # Default to 30m if not workflow_dispatch
+          DURATION="${{ github.event.inputs.duration || '30m' }}"
+          echo "Running stress test for $DURATION..."
+
+          # Use sleep with suffix support (m, h)
+          sleep $DURATION
+
+          echo "Stopping stress test..."
+          pkill -P $STRESS_PID || true
+          kill $STRESS_PID || true
+
+      - name: Analyze Results
+        run: |
+          cd tests/monitoring
+          # Give Prometheus a moment to finish scraping the final points
+          sleep 30
+          python3 analyze_stats.py
+
+      - name: Snapshot Metrics
+        if: always()
+        run: |
+          # Stop containers to ensure data is flushed to disk
+          cd tests/monitoring
+          docker compose stop prometheus
+          sudo tar -czf ../../prometheus-data.tar.gz ./prometheus_data
+
+      - name: Collect Logs on Failure
+        if: failure()
+        run: |
+          mkdir -p artifacts
+          docker logs salt-master > artifacts/salt-master.log
+          docker logs salt-minion-1 > artifacts/salt-minion-1.log
+          cp monitoring/event_log.txt artifacts/ || true
+
+      - name: Upload Artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: stress-test-results
+          path: |
+            artifacts/
+            prometheus-data.tar.gz

requirements/static/ci/py3.10/darwin-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=macos --python-version=3.10 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.10/darwin-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.10/freebsd-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --universal --python-version=3.10 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.10/freebsd-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.10/linux-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=linux --python-version=3.10 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.10/linux-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.10/windows-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=windows --python-version=3.10 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.10/windows-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.11/darwin-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=macos --python-version=3.11 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.11/darwin-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.11/freebsd-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --universal --python-version=3.11 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.11/freebsd-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.11/linux-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=linux --python-version=3.11 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.11/linux-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.11/windows-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=windows --python-version=3.11 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.11/windows-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.12/darwin-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=macos --python-version=3.12 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.12/darwin-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.12/freebsd-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --universal --python-version=3.12 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.12/freebsd-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.12/linux-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=linux --python-version=3.12 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.12/linux-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.12/windows-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=windows --python-version=3.12 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.12/windows-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.13/darwin-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=macos --python-version=3.13 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.13/darwin-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.13/freebsd-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --universal --python-version=3.13 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.13/freebsd-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.13/linux-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=linux --python-version=3.13 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.13/linux-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.13/windows-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=windows --python-version=3.13 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.13/windows-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.9/darwin-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=macos --python-version=3.9 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.9/darwin-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.9/freebsd-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --universal --python-version=3.9 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.9/freebsd-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.9/linux-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=linux --python-version=3.9 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.9/linux-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

requirements/static/ci/py3.9/windows-crypto.in

@@ -0,0 +1,8 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements/static/ci/crypto.in --python-platform=windows --python-version=3.9 --constraint requirements/constraints.txt --no-emit-index-url -o=requirements/static/ci/py3.9/windows-crypto.in
+m2crypto==0.48.0
+    # via -r requirements/static/ci/crypto.in
+packaging==26.2
+    # via m2crypto
+pycryptodome==3.23.0
+    # via -r requirements/static/ci/crypto.in

salt/auth/__init__.py

@@ -62,6 +62,31 @@ def __init__(self, opts, ckminions=None):
         self.tokens = salt.loader.eauth_tokens(opts)
         self.ckminions = ckminions or salt.utils.minions.CkMinions(opts)
 
+    def destroy(self):
+        """
+        Clean up resources
+        """
+        if hasattr(self, "auth") and self.auth is not None:
+            if hasattr(self.auth, "destroy"):
+                self.auth.destroy()
+            self.auth = {}
+        if hasattr(self, "tokens") and self.tokens is not None:
+            if hasattr(self.tokens, "destroy"):
+                self.tokens.destroy()
+            self.tokens = {}
+        if hasattr(self, "ckminions") and self.ckminions is not None:
+            if hasattr(self.ckminions, "cache") and self.ckminions.cache is not None:
+                if hasattr(self.ckminions.cache, "destroy"):
+                    self.ckminions.cache.destroy()
+                self.ckminions.cache = None
+            self.ckminions = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.destroy()
+
     def load_name(self, load):
         """
         Return the primary name associate with the load, if an empty string

salt/cache/__init__.py

@@ -81,6 +81,12 @@ def modules(self):
             self.__lazy_init()
         return self._modules
 
+    def destroy(self):
+        if hasattr(self, "_modules") and self._modules is not None:
+            if hasattr(self._modules, "destroy"):
+                self._modules.destroy()
+            self._modules = None
+
     def cache(self, bank, key, fun, loop_fun=None, **kwargs):
         """
         Check cache for the data. If it is there, check to see if it needs to

salt/channel/server.py

@@ -871,6 +871,12 @@ def close(self):
         self.transport.close()
         if self.event is not None:
             self.event.destroy()
+        if hasattr(self, "ckminions") and self.ckminions is not None:
+            if hasattr(self.ckminions, "cache") and self.ckminions.cache is not None:
+                if hasattr(self.ckminions.cache, "destroy"):
+                    self.ckminions.cache.destroy()
+                self.ckminions.cache = None
+            self.ckminions = None
 
 
 class PubServerChannel:
@@ -928,6 +934,12 @@ def close(self):
         if self.aes_funcs is not None:
             self.aes_funcs.destroy()
             self.aes_funcs = None
+        if hasattr(self, "ckminions") and self.ckminions is not None:
+            if hasattr(self.ckminions, "cache") and self.ckminions.cache is not None:
+                if hasattr(self.ckminions.cache, "destroy"):
+                    self.ckminions.cache.destroy()
+                self.ckminions.cache = None
+            self.ckminions = None
 
     def pre_fork(self, process_manager, kwargs=None):
         """

salt/client/__init__.py

@@ -2077,6 +2077,18 @@ def destroy(self):
         if self.event is not None:
             self.event.destroy()
             self.event = None
+        if hasattr(self, "returners") and self.returners is not None:
+            if hasattr(self.returners, "destroy"):
+                self.returners.destroy()
+            self.returners = {}
+        if hasattr(self, "functions") and self.functions is not None:
+            if hasattr(self.functions, "destroy"):
+                self.functions.destroy()
+            self.functions = {}
+        if hasattr(self, "utils") and self.utils is not None:
+            if hasattr(self.utils, "destroy"):
+                self.utils.destroy()
+            self.utils = {}
 
     def __enter__(self):
         return self

salt/config/__init__.py

@@ -2379,6 +2379,8 @@ def mminion_config(path, overrides, ignore_config_errors=True):
     apply_sdb(opts)
 
     _validate_opts(opts)
+    if "grains" in opts and hasattr(opts["grains"], "destroy"):
+        opts["grains"].destroy()
     opts["grains"] = salt.loader.grains(opts)
     opts["pillar"] = {}
     salt.features.setup_features(opts)