Merge pull request #206 from mitre-attack/detection_strategies_fix

Author: jondricek

examples/.env.example

@@ -12,5 +12,5 @@
 #
 # the default download directory from the above command is "attack-releases"
 
-STIX_BASE_DIR=attack-releases/stix-2.0/v17.1
-STIX_BUNDLE=attack-releases/stix-2.0/v17.1/enterprise-attack.json
+STIX_BASE_DIR=attack-releases/stix-2.0/v18.0
+STIX_BUNDLE=attack-releases/stix-2.0/v18.0/enterprise-attack.json

examples/README.md

@@ -38,7 +38,7 @@ Setting up these tools is out of scope for this README.
 
 ### Downloading ATT&CK STIX Bundles
 
-Many example scripts require ATT&CK STIX bundles, which must be downloaded and placed in the directory specified in your `.env` file (e.g., `attack-releases/stix-2.0/v17.1`).
+Many example scripts require ATT&CK STIX bundles, which must be downloaded and placed in the directory specified in your `.env` file (e.g., `attack-releases/stix-2.0/v18.0`).
 You can download these bundles using the provided CLI command if you have mitreattack-python installed:
 
 ```sh

examples/generate_excel_files.py

@@ -8,7 +8,7 @@ def main():
     output_dir = "output/"
 
     # Path to the STIX bundles for each domain (assumes STIX files are downloaded)
-    stix_base_dir = os.environ.get("STIX_BASE_DIR", "attack-releases/stix-2.0/v17.1")
+    stix_base_dir = os.environ.get("STIX_BASE_DIR", "attack-releases/stix-2.0/v18.0")
     stix_files = {
         "enterprise-attack": os.path.join(stix_base_dir, "enterprise-attack.json"),
         "mobile-attack": os.path.join(stix_base_dir, "mobile-attack.json"),

mitreattack/diffStix/changelog_helper.py

@@ -372,15 +372,16 @@ def load_data(self):
 
                         # Description changes
                         #####################
-                        old_lines = old_stix_obj["description"].replace("\n", " ").splitlines()
-                        new_lines = new_stix_obj["description"].replace("\n", " ").splitlines()
-                        old_lines_unique = [line for line in old_lines if line not in new_lines]
-                        new_lines_unique = [line for line in new_lines if line not in old_lines]
-                        if old_lines_unique or new_lines_unique:
-                            html_diff = difflib.HtmlDiff(wrapcolumn=60)
-                            html_diff._legend = ""  # type: ignore[attr-defined]
-                            delta = html_diff.make_table(old_lines, new_lines, "Old Description", "New Description")
-                            new_stix_obj["description_change_table"] = delta
+                        if "description" in old_stix_obj and "description" in new_stix_obj:
+                            old_lines = old_stix_obj["description"].replace("\n", " ").splitlines()
+                            new_lines = new_stix_obj["description"].replace("\n", " ").splitlines()
+                            old_lines_unique = [line for line in old_lines if line not in new_lines]
+                            new_lines_unique = [line for line in new_lines if line not in old_lines]
+                            if old_lines_unique or new_lines_unique:
+                                html_diff = difflib.HtmlDiff(wrapcolumn=60)
+                                html_diff._legend = ""  # type: ignore[attr-defined]
+                                delta = html_diff.make_table(old_lines, new_lines, "Old Description", "New Description")
+                                new_stix_obj["description_change_table"] = delta
 
                         # Relationship changes
                         ######################
@@ -1631,16 +1632,17 @@ def is_patch_change(old_stix_obj: dict, new_stix_obj: dict) -> bool:
             return True
 
     # description changed, even though modified date didn't
-    old_lines = old_stix_obj["description"].replace("\n", " ").splitlines()
-    new_lines = new_stix_obj["description"].replace("\n", " ").splitlines()
-    old_lines_unique = [line for line in old_lines if line not in new_lines]
-    new_lines_unique = [line for line in new_lines if line not in old_lines]
-    if old_lines_unique or new_lines_unique:
-        logger.warning(
-            f"{stix_id} - {attack_id} has a description change "
-            "without the version being incremented or the last modified date changing"
-        )
-        return True
+    if "description" in old_stix_obj and "description" in new_stix_obj:
+        old_lines = old_stix_obj["description"].replace("\n", " ").splitlines()
+        new_lines = new_stix_obj["description"].replace("\n", " ").splitlines()
+        old_lines_unique = [line for line in old_lines if line not in new_lines]
+        new_lines_unique = [line for line in new_lines if line not in old_lines]
+        if old_lines_unique or new_lines_unique:
+            logger.warning(
+                f"{stix_id} - {attack_id} has a description change "
+                "without the version being incremented or the last modified date changing"
+            )
+            return True
 
     # doesn't meet the definintion of a patch change
     return False

mitreattack/navlayers/generators/gen_helpers.py

@@ -51,6 +51,8 @@ def build_data_strings(data_sources, data_components):
     """
     out = dict()
     for component in data_components:
+        if "x_mitre_data_source_ref" not in component:
+            continue
         ref = component["x_mitre_data_source_ref"]
         try:
             source = [x for x in data_sources if x["id"] == ref][0]

mitreattack/navlayers/generators/overview_generator.py

@@ -64,6 +64,7 @@ def __init__(self, source, domain="enterprise", resource=None):
             "x-mitre-data-component": dict(),
             "campaign": dict(),
             "asset": dict(),
+            "x-mitre-detection-strategy": dict(),
         }
 
         # Scan through all relationships to identify ones that target attack techniques (attack-pattern). Then, sort

mitreattack/navlayers/generators/usage_generator.py

@@ -155,9 +155,8 @@ def generate_layer(self, match):
         )
         raw_layer["techniques"] = processed_listing
         output_layer = Layer(raw_layer)
-        if matched_obj["type"] != "x-mitre-data-component":
-            name = matched_obj["name"]
-        else:
+        name = matched_obj["name"]
+        if matched_obj["type"] == "x-mitre-data-component" and matched_obj["id"] in self.source_mapping:
             name = self.source_mapping[matched_obj["id"]]
         output_layer.description = (
             f"{self.domain.capitalize() if len(self.domain) > 3 else self.domain.upper()} "

tests/test_cli.py

@@ -19,7 +19,6 @@
 from mitreattack.navlayers.layerGenerator_cli import main as LGC_main
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_export_svg(tmp_path: Path, layer_v43: Layer, stix_file_enterprise_latest: str):
     """Test SVG Export capabilities from CLI."""
     demo_file = tmp_path / "demo_file.json"
@@ -43,7 +42,6 @@ def test_export_svg(tmp_path: Path, layer_v43: Layer, stix_file_enterprise_lates
     assert test_export_svg_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_export_excel(tmp_path: Path, layer_v43: Layer, stix_file_enterprise_latest: str):
     """Test excel export capabilities from CLI."""
     demo_file = tmp_path / "demo_file.json"
@@ -67,7 +65,6 @@ def test_export_excel(tmp_path: Path, layer_v43: Layer, stix_file_enterprise_lat
     assert test_export_xlsx_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_overview_group(tmp_path: Path, stix_file_mobile_latest: str):
     """Test CLI group overview generation."""
     output_layer_file = tmp_path / "test_overview_group.json"
@@ -88,7 +85,6 @@ def test_generate_overview_group(tmp_path: Path, stix_file_mobile_latest: str):
     assert output_layer_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_overview_software(tmp_path: Path, stix_file_mobile_latest: str):
     """Test CLI software overview generation."""
     output_layer_file = tmp_path / "test_overview_software.json"
@@ -109,7 +105,6 @@ def test_generate_overview_software(tmp_path: Path, stix_file_mobile_latest: str
     assert output_layer_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_overview_mitigation(tmp_path: Path, stix_file_enterprise_latest: str):
     """Test CLI mitigation overview generation."""
     output_layer_file = tmp_path / "test_overview_mitigation.json"
@@ -130,7 +125,6 @@ def test_generate_overview_mitigation(tmp_path: Path, stix_file_enterprise_lates
     assert output_layer_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_overview_datasource(tmp_path: Path, stix_file_enterprise_latest: str):
     """Test CLI datasource overview generation."""
     output_layer_file = tmp_path / "test_overview_datasource.json"
@@ -151,7 +145,6 @@ def test_generate_overview_datasource(tmp_path: Path, stix_file_enterprise_lates
     assert output_layer_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_mapped_group(tmp_path: Path, stix_file_enterprise_latest: str):
     """Test CLI group mapped generation (APT1)."""
     output_layer_file = tmp_path / "test_mapped_group.json"
@@ -172,7 +165,6 @@ def test_generate_mapped_group(tmp_path: Path, stix_file_enterprise_latest: str)
     assert output_layer_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_mapped_software(tmp_path: Path, stix_file_enterprise_latest: str):
     """Test CLI software mapped generation (S0202)."""
     output_layer_file = tmp_path / "test_mapped_software.json"
@@ -193,7 +185,6 @@ def test_generate_mapped_software(tmp_path: Path, stix_file_enterprise_latest: s
     assert output_layer_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_mapped_mitigation(tmp_path: Path, stix_file_mobile_latest: str):
     """Test CLI mitigation mapped generation (M1013)."""
     output_layer_file = tmp_path / "test_mapped_mitigation.json"
@@ -214,7 +205,6 @@ def test_generate_mapped_mitigation(tmp_path: Path, stix_file_mobile_latest: str
     assert output_layer_file.exists()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_mapped_datasource(tmp_path: Path, stix_file_enterprise_latest: str):
     """Test CLI datasource mapped generation."""
     output_layer_file = tmp_path / "test_mapped_datasource.json"
@@ -277,7 +267,6 @@ def test_generate_batch_software(tmp_path: Path, stix_file_ics_latest: str):
     assert output_layers_dir.is_dir()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_batch_mitigation(tmp_path: Path, stix_file_enterprise_latest: str):
     """Test CLI mitigation batch generation."""
     output_layers_dir = tmp_path / "test_batch_mitigation"
@@ -298,7 +287,6 @@ def test_generate_batch_mitigation(tmp_path: Path, stix_file_enterprise_latest:
     assert output_layers_dir.is_dir()
 
 
-@pytest.mark.skip("Unsupported functionality of mitreattack-python. keeping the unit test for legacy awareness")
 def test_generate_batch_datasource(tmp_path: Path, stix_file_enterprise_latest: str):
     """Test CLI datasource batch generation."""
     output_layers_dir = tmp_path / "test_batch_datasource"

tests/test_mitreattackdata.py

@@ -160,11 +160,6 @@ def test_all_campaigns_using_techniques(self, mitre_attack_data_enterprise: Mitr
         campaigns = mitre_attack_data_enterprise.get_all_campaigns_using_all_techniques()
         assert campaigns
 
-    # def test_all_datacomponents_detecting_all_techniques(self, mitre_attack_data_enterprise: MitreAttackData):
-    #     """Test that all datacomponents detecting all techniques can be retrieved."""
-    #     datacomponents = mitre_attack_data_enterprise.get_all_datacomponents_detecting_all_techniques()
-    #     assert datacomponents
-
     def test_all_groups_attributing_to_all_campaigns(self, mitre_attack_data_enterprise: MitreAttackData):
         """Test that all groups attributing to all campaigns can be retrieved."""
         groups = mitre_attack_data_enterprise.get_all_groups_attributing_to_all_campaigns()
@@ -210,11 +205,6 @@ def test_all_subtechniques_of_all_techniques(self, mitre_attack_data_enterprise:
         subtechniques = mitre_attack_data_enterprise.get_all_subtechniques_of_all_techniques()
         assert subtechniques
 
-    # def test_all_techniques_detected_by_all_datacomponents(self, mitre_attack_data_enterprise: MitreAttackData):
-    #     """Test that all techniques detected by all datacomponents can be retrieved."""
-    #     techniques = mitre_attack_data_enterprise.get_all_techniques_detected_by_all_datacomponents()
-    #     assert techniques
-
     def test_all_techniques_mitigated_by_all_mitigations(self, mitre_attack_data_enterprise: MitreAttackData):
         """Test that all techniques mitigated by all mitigations can be retrieved."""
         techniques = mitre_attack_data_enterprise.get_all_techniques_mitigated_by_all_mitigations()

tests/test_to_excel.py

@@ -49,9 +49,8 @@ def check_excel_files_exist(excel_folder: Path, domain: str):
     assert (excel_folder / f"{domain}-software.xlsx").exists()
     assert (excel_folder / f"{domain}-tactics.xlsx").exists()
     assert (excel_folder / f"{domain}-techniques.xlsx").exists()
-    # TODO: add in check for analytics/detection strategies after ATT&CK v18 is released
-    # assert (excel_folder / f"{domain}-analytics.xlsx").exists()
-    # assert (excel_folder / f"{domain}-detectionstrategies.xlsx").exists()
+    assert (excel_folder / f"{domain}-analytics.xlsx").exists()
+    assert (excel_folder / f"{domain}-detectionstrategies.xlsx").exists()
 
 
 def test_enterprise_latest(tmp_path: Path, memstore_enterprise_latest: stix2.MemoryStore):