Merge pull request #4082 from splunk/escu_6

ESCU 6 YAML Porting and Updates

Author: ljstella

.github/workflows/appinspect.yml

@@ -1,45 +1,60 @@
+# This workflow performs a simple build action.
+# It is intentionally separate from the build
+# workflow to provide granular feedback and insight
+# into when a build passes but an appinspect fails.
 name: appinspect
+
 on:
   pull_request:
     types: [opened, reopened, synchronize]
+
 jobs:
   appinspect:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out repository (PR)
-        if: ${{ github.event_name == 'pull_request' }}
+      - name: Check out the repository code
         uses: actions/checkout@v6
-        with:
-          ref: refs/pull/${{ github.event.pull_request.number }}/merge
 
-      - uses: actions/setup-python@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          python-version: '3.11'
-          architecture: 'x64'
+          python-version: 3.14
+          architecture: x64
 
-      - name: Install Python Dependencies and ContentCTL and Atomic Red Team
+      - name: Install contentctl-ng
+        shell: bash
         run: |
-          echo "- Contentctl version - $(cat requirements.txt)"
+          echo "- Build Tool Version - $(cat requirements.txt)"
           pip install -r requirements.txt
-          git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git external_repos/atomic-red-team
-          git clone --depth=1 --single-branch --branch="master" https://github.com/mitre-attack/attack-stix-data external_repos/cti
+      
+      - name: Run a contentctl-ng build
+        run: |
+          contentctl-ng build
 
-      - name: Running appinspect with enrichments
+      - name: Run appinspect with enrichments
         env:
           APPINSPECTUSERNAME: "${{ secrets.APPINSPECTUSERNAME }}"
           APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}"
         run: |
-          echo $APPINSPECTUSERNAME
-          contentctl inspect --splunk-api-username "$APPINSPECTUSERNAME" --splunk-api-password "$APPINSPECTPASSWORD" --enrichments --enable-metadata-validation --suppress-missing-content-exceptions --enforce-deprecation-mapping-requirement
-          echo "done appinspect"
+          # Download the most recent Release in order to enforce metadata validation
+          # --location flag required to follow redirects from the "latest" URL
+          curl --location -o  DA-ESS-ContentUpdate-latest.tar.gz https://github.com/splunk/security_content/releases/latest/download/DA-ESS-ContentUpdate-latest.tar.gz
+          
+          # Inspect, using the release we downloaded above
+          # Also, we intentionally ignore missing content exceptions - private content is 
+          # not expected to be present when running this check in GitHub
+          contentctl-ng inspect --splunkbase-username "$APPINSPECTUSERNAME" --splunkbase-password "$APPINSPECTPASSWORD" --old-app-path DA-ESS-ContentUpdate-latest.tar.gz --ignore-missing-content-exceptions
           mkdir -p artifacts/app_inspect_report
           cp -r dist/*.html artifacts/app_inspect_report
           cp -r dist/*.tar.gz artifacts/
 
+      # Store inspect artifacts
       - name: store_artifacts
+        if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: content-latest
+          name: appinspect_results
           path: |
-            artifacts/DA-ESS-ContentUpdate-latest.tar.gz
-            artifacts/app_inspect_report
+            dist/*.html
+            dist/*.tar.gz
+            

.github/workflows/build.yml

@@ -1,39 +1,46 @@
+# This workflow performs a simple build action.
+# It is intentionally separate from the appinspect
+# workflow to provide granular feedback and insight
+# into when a build passes but an appinspect fails.
 name: build
+
 on:
   pull_request:
     types: [opened, reopened, synchronize]
   push:
     branches:
       - develop
+
+
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository code
         uses: actions/checkout@v6
 
-      - uses: actions/setup-python@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          python-version: '3.11'
-          architecture: 'x64'
+          python-version: 3.14
+          architecture: x64
 
-      - name: Install Python Dependencies and ContentCTL and Atomic Red Team
+      - name: Install contentctl-ng
+        shell: bash
         run: |
-          echo "- Contentctl version - $(cat requirements.txt)"
+          echo "- Build Tool Version - $(cat requirements.txt)"
           pip install -r requirements.txt
-          git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git external_repos/atomic-red-team
-          git clone --depth=1 --single-branch --branch="master" https://github.com/mitre-attack/attack-stix-data external_repos/cti
 
-      - name: Running build with enrichments
-        run: |
-          contentctl build --enrichments --enforce_deprecation_mapping_requirement
-          mkdir artifacts
-          mv dist/DA-ESS-ContentUpdate-latest.tar.gz artifacts/
 
+      - name: Run a contentctl-ng build
+        run: |
+          contentctl-ng build
+      
+      # Store build artifact
       - name: store_artifacts
+        if: always()
         uses: actions/upload-artifact@v7
         with:
           name: content-latest
           path: |
-            artifacts/DA-ESS-ContentUpdate-latest.tar.gz
-            dist/api
+            dist/*.tar.gz

.github/workflows/unit-testing.yml

@@ -4,7 +4,7 @@ on:
     types: [opened, reopened, synchronize]
 jobs:
   unit-testing:
-    runs-on: ubuntu-latest
+    runs-on: large-ubuntu-22.04-32core
     if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job
     steps:
         #For fork PRs, always check out security_content and the PR target in security content!
@@ -13,25 +13,32 @@ jobs:
           with:
             repository: 'splunk/security_content' #this should be the TARGET repo of the PR. we hardcode it for now
             ref: ${{ github.base_ref }}
-          
-
-        - uses: actions/setup-python@v6
+        
+        - name: Print out information abour PR target
+          run: |
+            echo "The PR target branch is: ${{ github.base_ref }}"
+            echo "The PR head branch is: ${{ github.head_ref }}"
+            echo "My current branch is"
+            git branch --show-current
+            git rev-parse HEAD
+        
+            
+        - name: Set up Python
+          uses: actions/setup-python@v6
           with:
-            python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
-            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
+            python-version: 3.14
+            architecture: x64
 
-        - name: Install Python Dependencies and ContentCTL
+        - name: Install contentctl-ng
+          shell: bash
           run: |
-            python -m pip install --upgrade pip
-            echo "- Contentctl version - $(cat requirements.txt)"
+            echo "- Build Tool Version - $(cat requirements.txt)"
             pip install -r requirements.txt
-            
-           
-        # Running contentctl test with a few arguments, before running the command make sure you checkout into the current branch of the pull request. This step only performs unit testing on all the changes against the target-branch. In most cases this target branch will be develop
-        # Make sure we check out the PR, even if it actually lives in a fork
+    
+        # Check out the PR, even if it lives in a fork.
         # Instructions for pulling a PR were taken from: 
         # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally
-        - name: Run ContentCTL test for changes against target branch
+        - name: Checkout the PR branch and the target to calculate changed files for testing
           run: |
             
             echo "Current Branch (Head Ref): ${{ github.head_ref }}"
@@ -41,7 +48,27 @@ jobs:
             git fetch origin pull/${{ github.event.pull_request.number }}/head:new_branch_for_testing
             #We must specifically get the PR's target branch from security_content, not the one that resides in the fork PR's forked repo           
             git switch new_branch_for_testing
-            contentctl test --verbose --disable-tqdm --no-enable-integration-testing --container-settings.num-containers 1 --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }}
+        
+        - name: Run a contentctl-ng build command to create a package that will be tested.
+          run: |
+            contentctl-ng build
+        
+        - name: Start the test environment
+          run: |
+            docker run -d --platform linux/amd64 -p 8088:8088 -p 8089:8089 -p 8000:8000 -e 'SPLUNK_START_ARGS=--accept-license' -e 'SPLUNK_GENERAL_TERMS=--accept-sgt-current-at-splunk-com' -e 'SPLUNK_PASSWORD=Chang3d!' --name splunk splunk/splunk:latest
+            # Wait some time for this environment to be ready
+            sleep 180
+            
+        - name: Run a contentctl-ng install to configure the testing environment
+          env:
+            APPINSPECTUSERNAME: "${{ secrets.APPINSPECTUSERNAME }}"
+            APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}"  
+          run : |
+            contentctl-ng install --splunkbase-username "$APPINSPECTUSERNAME" --splunkbase-password "$APPINSPECTPASSWORD" 
+        
+        - name: Test content which has changed between this branch and the target branch
+          run: |
+            contentctl-ng test --verbose --post-test-behavior NEVER_PAUSE --mode CHANGED --git-ref ${{ github.base_ref }}
             echo "contentctl test - COMPLETED"
 
         # Store test_results/summary.yml and dist/DA-ESS-ContentUpdate-latest.tar.gz to job artifact-test_summary_results.zip
@@ -52,7 +79,7 @@ jobs:
             name: test_summary_results
             path: |
               test_results/summary.yml
-              dist/DA-ESS-ContentUpdate-latest.tar.gz
+              dist/*.tar.gz
 
         # Print entire result summary so that the users can view it in the Github Actions logs 
         - name: Print entire test_results/summary.yml

.vscode/extensions.json

@@ -0,0 +1,6 @@
+{
+    "recommendations": [
+        "redhat.vscode-yaml"
+    ]
+}
+

.vscode/settings.json

@@ -6,5 +6,18 @@
     "python.testing.pytestEnabled": true,
     "python.terminal.activateEnvironment": true,
     "python.envFile": "${workspaceFolder}/.env",
-    "python.testing.cwd": "${workspaceFolder}"
+    "python.testing.cwd": "${workspaceFolder}",
+    "yaml.schemas": {
+        "./schemas/RemovedContent.schema.json": "removed/*/*.yml",
+        "./schemas/Baseline.schema.json": ["baselines/*.yml", "!removed/baselines/*.yml"],
+        "./schemas/CSVLookup.schema.json": "lookups/csv/*.yml",
+        "./schemas/Dashboard.schema.json": "dashboards/*.yml",
+        "./schemas/DataSource.schema.json": "data_sources/*.yml",
+        "./schemas/EventBasedDetection.schema.json": ["detections/**/*.yml", "!removed/detections/*.yml"],
+        "./schemas/KVStoreLookup.schema.json": "lookups/kvstore/*.yml",
+        "./schemas/FilebackedMacro.schema.json": "macros/*.yml",
+        "./schemas/FilebackedSchedule.schema.json": "schedules/*.yml",
+        "./schemas/Playbook.schema.json": "playbooks/*.yml",
+        "./schemas/Story.schema.json": ["stories/*.yml", "!removed/stories/*.yml"]
+    }
 }

app_template/default/data/ui/nav/default.xml

@@ -1,9 +1,10 @@
 <nav search_view="search" color="#65A637">
-  <view name="escu_summary" default="true"/>
-  <view name="feedback"/>
+  <view name="landing" default="true"/>
+  <view name="onboarding"/>
   <view name="search"/>
   <collection label="Dashboards">
     <view source="unclassified" match="__"/>
   </collection>
   <a href="https://help.splunk.com/en/splunk-enterprise-security-8/security-content-update">Docs</a>
+  <a href="/app/SplunkEnterpriseSecuritySuite/ess_use_case_library">Explore in ES</a>
 </nav>