Merge remote-tracking branch 'origin/main' into codex-prepare-xdist-setup

Author: scotluns

docs/CONTRIBUTING.md

@@ -63,6 +63,21 @@ preparing bundles directly. Parallel test runs warm the shared STIX cache before
 new xdist-backed test needs an additional ATT&CK release, update the cache warmup list in
 `tests/conftest.py`.
 
+To run STIX-backed tests against specific local bundles, pass the bundle paths to pytest:
+
+```bash
+uv run pytest \
+  --stix-enterprise /path/to/enterprise-attack.json \
+  --stix-mobile /path/to/mobile-attack.json \
+  --stix-ics /path/to/ics-attack.json
+```
+
+To have pytest download a specific ATT&CK release instead, use:
+
+```bash
+uv run pytest --attack-version 16.1 --stix-version 2.1
+```
+
 ### Pull Requests
 
 When making a pull request, please make sure to:

examples/generate_multiple_attack_diffs.py

@@ -1,36 +1,75 @@
+"""Generate ATT&CK changelog outputs for multiple release pairs."""
+
+import argparse
+
 from mitreattack.diffStix.changelog_helper import get_new_changelog_md
 
+DOMAINS = ["enterprise-attack", "mobile-attack", "ics-attack"]
+VERSION_PAIRS = [
+    ("17.1", "18.0"),
+    ("18.0", "18.1"),
+]
+
+
+def get_release_output_folder(old_version: str, new_version: str) -> str:
+    """Return the output folder for a release comparison."""
+    return f"output/v{old_version}-v{new_version}"
+
+
+def get_artifact_link_prefix(old_version: str, new_version: str, *, attack_website_links: bool = False) -> str:
+    """Return the link prefix for generated layers and changelog JSON."""
+    if not attack_website_links:
+        return ""
+    return f"/docs/changelogs/v{old_version}-v{new_version}"
+
+
+def get_parsed_args():
+    """Parse command line arguments for the example script."""
+    parser = argparse.ArgumentParser(description="Generate ATT&CK changelog outputs for multiple release pairs.")
+    parser.add_argument(
+        "-w",
+        "--attack-website-links",
+        action="store_true",
+        help="Use ATT&CK website paths for links to generated layers and changelog JSON.",
+    )
+    return parser.parse_args()
+
+
+def generate_diff(old_version: str, new_version: str, *, attack_website_links: bool = False):
+    """Generate changelog outputs for a single ATT&CK release pair."""
+    output_folder = get_release_output_folder(old_version, new_version)
+    print(f"Generating ATT&CK Diffs between {old_version}-{new_version}: {output_folder}")
+
+    get_new_changelog_md(
+        domains=DOMAINS,
+        layers=[
+            f"{output_folder}/layer-enterprise.json",
+            f"{output_folder}/layer-mobile.json",
+            f"{output_folder}/layer-ics.json",
+        ],
+        old=f"attack-releases/stix-2.0/v{old_version}",
+        new=f"attack-releases/stix-2.0/v{new_version}",
+        show_key=True,
+        # site_prefix: str = "",
+        verbose=True,
+        include_contributors=True,
+        markdown_file=f"{output_folder}/changelog.md",
+        html_file=f"{output_folder}/index.html",
+        html_file_detailed=f"{output_folder}/changelog-detailed.html",
+        additional_formats_prefix=get_artifact_link_prefix(
+            old_version,
+            new_version,
+            attack_website_links=attack_website_links,
+        ),
+        json_file=f"{output_folder}/changelog.json",
+    )
+
 
 def main():
-    version_pairs = [
-        ("17.1", "18.0"),
-        ("18.0", "18.1"),
-    ]
-    for version_pair in version_pairs:
-        old_version = version_pair[0]
-        new_version = version_pair[1]
-
-        output_folder = f"output/v{old_version}-v{new_version}"
-        print(f"Generating ATT&CK Diffs between {old_version}-{new_version}: {output_folder}")
-
-        get_new_changelog_md(
-            domains=["enterprise-attack", "mobile-attack", "ics-attack"],
-            layers=[
-                f"{output_folder}/layer-enterprise.json",
-                f"{output_folder}/layer-mobile.json",
-                f"{output_folder}/layer-ics.json",
-            ],
-            old=f"attack-releases/stix-2.0/v{old_version}",
-            new=f"attack-releases/stix-2.0/v{new_version}",
-            show_key=True,
-            # site_prefix: str = "",
-            verbose=True,
-            include_contributors=True,
-            markdown_file=f"{output_folder}/changelog.md",
-            html_file=f"{output_folder}/index.html",
-            html_file_detailed=f"{output_folder}/changelog-detailed.html",
-            json_file=f"{output_folder}/changelog.json",
-        )
+    """Generate changelog outputs for all configured ATT&CK release pairs."""
+    args = get_parsed_args()
+    for old_version, new_version in VERSION_PAIRS:
+        generate_diff(old_version, new_version, attack_website_links=args.attack_website_links)
 
 
 if __name__ == "__main__":

mitreattack/attackToExcel/stixToDf.py

@@ -753,9 +753,40 @@ def assetsToDf(src):
                 related_assets_descriptions = []
 
                 for related_asset in asset["x_mitre_related_assets"]:
-                    related_assets.append(related_asset["name"])
-                    related_assets_sectors.append(", ".join(related_asset["related_asset_sectors"]))
-                    related_assets_descriptions.append(related_asset["description"])
+                    asset_name = row.get("name", "<unnamed asset>")
+                    asset_id = row.get("ID", "<missing ATT&CK ID>")
+                    asset_stix_id = row.get("STIX ID", "<missing STIX ID>")
+                    related_asset_name = related_asset.get("name", "<unnamed related asset>")
+
+                    if "name" in related_asset:
+                        related_assets.append(related_asset["name"])
+                    else:
+                        logger.error(
+                            f"Missing name for related asset '{related_asset_name}' on asset '{asset_name}' "
+                            f"({asset_id}, {asset_stix_id}). Leaving related assets blank for this related asset "
+                            "and continuing export."
+                        )
+                        related_assets.append("")
+
+                    if "related_asset_sectors" in related_asset:
+                        related_assets_sectors.append(", ".join(related_asset["related_asset_sectors"]))
+                    else:
+                        logger.error(
+                            f"Missing related_asset_sectors for related asset '{related_asset_name}' on asset "
+                            f"'{asset_name}' ({asset_id}, {asset_stix_id}). Leaving related assets sectors blank "
+                            "for this related asset and continuing export."
+                        )
+                        related_assets_sectors.append("")
+
+                    if "description" in related_asset:
+                        related_assets_descriptions.append(related_asset["description"])
+                    else:
+                        logger.error(
+                            f"Missing description for related asset '{related_asset_name}' on asset '{asset_name}' "
+                            f"({asset_id}, {asset_stix_id}). Leaving related assets description blank for this "
+                            "related asset and continuing export."
+                        )
+                        related_assets_descriptions.append("")
 
                 row["related assets"] = "; ".join(related_assets)
                 row["related assets sectors"] = "; ".join(related_assets_sectors)

mitreattack/diffStix/README.md

@@ -14,7 +14,7 @@ Print full usage instructions:
 # You must run `pip install mitreattack-python` in order to access the diff_stix command
 diff_stix --help
 usage: diff_stix [-h] [--old OLD] [--new NEW] [--domains {enterprise-attack,mobile-attack,ics-attack} [{enterprise-attack,mobile-attack,ics-attack} ...]] [--markdown-file MARKDOWN_FILE] [--html-file HTML_FILE] [--html-file-detailed HTML_FILE_DETAILED]
-                 [--json-file JSON_FILE] [--layers [LAYERS ...]] [--site_prefix SITE_PREFIX] [--unchanged] [--use-mitre-cti] [--show-key] [--contributors] [--no-contributors] [-v]
+                 [--json-file JSON_FILE] [--layers [LAYERS ...]] [--site_prefix SITE_PREFIX] [--additional-formats-prefix ADDITIONAL_FORMATS_PREFIX] [--unchanged] [--use-mitre-cti] [--show-key] [--contributors] [--no-contributors] [-v]
 
 Create changelog reports on the differences between two versions of the ATT&CK content. Takes STIX bundles as input. For default operation, put enterprise-attack.json, mobile-attack.json, and ics-attack.json bundles in 'old' and 'new' folders for the script to compare.
 
@@ -37,6 +37,8 @@ options:
                         output/January_2023_Updates_Mobile.json, output/January_2023_Updates_ICS.json, output/January_2023_Updates_Pre.json
   --site_prefix SITE_PREFIX
                         Prefix links in markdown output, e.g. [prefix]/techniques/T1484
+  --additional-formats-prefix ADDITIONAL_FORMATS_PREFIX
+                        Prefix detailed HTML links to generated layers and changelog JSON.
   --unchanged           Show objects without changes in the markdown output
   --use-mitre-cti       Use content from the MITRE CTI repo for the -old data
   --show-key            Add a key explaining the change types to the markdown

mitreattack/diffStix/changelog_helper.py

@@ -1871,7 +1871,14 @@ def layers_dict_to_files(outfiles, layers):
         json.dump(layers["ics-attack"], open(ics_attack_layer_file, "w"), indent=4)
 
 
-def write_detailed_html(html_file_detailed: str, diffStix: DiffStix):
+def _get_additional_format_href(filename: str, additional_formats_prefix: str = "") -> str:
+    """Return a link to an additional changelog output file."""
+    if not additional_formats_prefix:
+        return filename
+    return f"{additional_formats_prefix.rstrip('/')}/{filename}"
+
+
+def write_detailed_html(html_file_detailed: str, diffStix: DiffStix, additional_formats_prefix: str = ""):
     """Write a detailed HTML report of changes between ATT&CK versions.
 
     Parameters
@@ -1880,6 +1887,8 @@ def write_detailed_html(html_file_detailed: str, diffStix: DiffStix):
         File to write HTML for the detailed changelog.
     diffStix : DiffStix
         An instance of a DiffStix object.
+    additional_formats_prefix : str, optional
+        Prefix for links to generated layer and JSON files, by default "".
     """
     old_version = diffStix.data["old"]["enterprise-attack"]["attack_release_version"]
     new_version = diffStix.data["new"]["enterprise-attack"]["attack_release_version"]
@@ -1889,6 +1898,11 @@ def write_detailed_html(html_file_detailed: str, diffStix: DiffStix):
     else:
         header = f"<h1>ATT&CK Changes Between v{old_version} and new content</h1>"
 
+    enterprise_layer_href = _get_additional_format_href("layer-enterprise.json", additional_formats_prefix)
+    mobile_layer_href = _get_additional_format_href("layer-mobile.json", additional_formats_prefix)
+    ics_layer_href = _get_additional_format_href("layer-ics.json", additional_formats_prefix)
+    changelog_json_href = _get_additional_format_href("changelog.json", additional_formats_prefix)
+
     frontmatter = [
         textwrap.dedent(
             """\
@@ -1913,7 +1927,7 @@ def write_detailed_html(html_file_detailed: str, diffStix: DiffStix):
         header,
         markdown.markdown(diffStix.get_md_key()),
         textwrap.dedent(
-            """\
+            f"""\
         <table class=diff summary=Legends>
             <tr>
                 <td>
@@ -1929,11 +1943,11 @@ def write_detailed_html(html_file_detailed: str, diffStix: DiffStix):
         <h2>Additional formats</h2>
         <p>These ATT&CK Navigator layer files can be uploaded to ATT&CK Navigator manually.</p>
         <ul>
-            <li><a href="layer-enterprise.json">Enterprise changes</a></li>
-            <li><a href="layer-mobile.json">Mobile changes</a></li>
-            <li><a href="layer-ics.json">ICS changes</a></li>
+            <li><a href="{enterprise_layer_href}">Enterprise changes</a></li>
+            <li><a href="{mobile_layer_href}">Mobile changes</a></li>
+            <li><a href="{ics_layer_href}">ICS changes</a></li>
         </ul>
-        <p>This JSON file contains the machine readble output used to create this page: <a href="changelog.json">changelog.json</a></p>
+        <p>This JSON file contains the machine readble output used to create this page: <a href="{changelog_json_href}">changelog.json</a></p>
         """
         ),
     ]
@@ -2256,6 +2270,13 @@ def get_parsed_args():
         help="Prefix links in markdown output, e.g. [prefix]/techniques/T1484",
     )
 
+    parser.add_argument(
+        "--additional-formats-prefix",
+        type=str,
+        default="",
+        help="Prefix detailed HTML links to generated layers and changelog JSON.",
+    )
+
     parser.add_argument(
         "--unchanged",
         action="store_true",
@@ -2331,6 +2352,7 @@ def get_new_changelog_md(
     markdown_file: Optional[str] = None,
     html_file: Optional[str] = None,
     html_file_detailed: Optional[str] = None,
+    additional_formats_prefix: str = "",
     json_file: Optional[str] = None,
 ) -> str:
     """Get a Markdown string representation of differences between two ATT&CK versions.
@@ -2365,6 +2387,8 @@ def get_new_changelog_md(
         If set, writes an HTML file from the parsed markdown, by default None
     html_file_detailed : str, optional
         If set, writes a more detailed HTML page, by default None
+    additional_formats_prefix : str, optional
+        Prefix for detailed HTML links to generated layer and JSON files, by default "".
     json_file : str, optional
         If set, writes JSON file of the changes, by default None
 
@@ -2414,7 +2438,11 @@ def get_new_changelog_md(
     if html_file_detailed:
         Path(html_file_detailed).parent.mkdir(parents=True, exist_ok=True)
         logger.info("Writing detailed updates to file")
-        write_detailed_html(html_file_detailed=html_file_detailed, diffStix=diffStix)
+        write_detailed_html(
+            html_file_detailed=html_file_detailed,
+            diffStix=diffStix,
+            additional_formats_prefix=additional_formats_prefix,
+        )
 
     if layers:
         if len(layers) == 0:
@@ -2457,6 +2485,7 @@ def main():
         markdown_file=args.markdown_file,
         html_file=args.html_file,
         html_file_detailed=args.html_file_detailed,
+        additional_formats_prefix=args.additional_formats_prefix,
         json_file=args.json_file,
     )
 

mitreattack/release_info.py

@@ -8,7 +8,7 @@
 # This file contains SHA256 hashes for officially released ATT&CK versions
 # download_string = f"https://raw.githubusercontent.com/mitre/cti/ATT%26CK-v{release}/{domain}-attack/{domain}-attack.json"
 
-LATEST_VERSION = "18.1"
+LATEST_VERSION = "19.0"
 
 STIX20 = {
     "enterprise": {
@@ -48,6 +48,7 @@
         "17.1": "9537a22166367a5b3c1434f5b17b27361cb9c88b34926e655344768fdbda3e85",
         "18.0": "6ecc9655954a4a0eeada8ba6f18f1d053fbcacce0e5411f677729d1dafce5673",
         "18.1": "628c4fc3c01b9ef37e1cd84ca3c421e1d43950a43464a14aabd1a7089601dc45",
+        "19.0": "987d0cfddb1e65797457cf2b045df9161140ede71fd627f15dfdd73c2a2c72ab",
     },
     "mobile": {
         "3.0": "1385d94348054c1c1f7cdc652f0719db353b60c923949b10cbf8a2e815a86eb3",
@@ -89,6 +90,7 @@
         "17.1": "736078773f05ee943c0aa71bf71b935b04315c134809e8b678bd45c89cb1ab49",
         "18.0": "18ab338f8663200bdc62b982d5821ec255d9b420947b68e6d024000a44620404",
         "18.1": "62ecc7e3cabacc2430de0d65078a3726b6d8f7b0eff9493fd6403b514f66518d",
+        "19.0": "1f3f050d6d4cb0a75a07a0c0dd0834ce2de176a0096a3ddd7ffc2ee2feeff592",
     },
     "ics": {
         "8.0": "2e9e9d0d9f0e5d14f64cf2788f46a1a4403bc88ab6ddd419cfcdfe617b0c920d",
@@ -115,6 +117,7 @@
         "17.1": "f0bd44fa2e167f2e9e94700f9081571dfedc49bebd856ea0d7ec24cf896d298b",
         "18.0": "e19597196d96ef07e7d1b0dc3a1e67f792a27f61d615a3242c694169fe81011c",
         "18.1": "76655cd7c363ca9a7474a95e9d60522a0c3211eaf2efad5b5e9cd7f9e0365b51",
+        "19.0": "17f519e62fa85aded4ef6035dff348dd790dd177672757ceaded8e85f8caa950",
     },
     "pre": {
         "3.0": "bc59c1b1398a133cf0adb98e4e28396fdb6a5a2e2353cecb1783c425f066fc94",
@@ -172,6 +175,7 @@
         "17.1": "0d1c347a4d584cf7e11ef46556c33b7689341443bf86299188d46c307274323b",
         "18.0": "ff94838b09edfe7d59eec1cd7af0a1e229c4bc0ae0bdfa98ad170aeec9c3e272",
         "18.1": "f857d8f78f2f0c0b7db321a711a39fba98546c1e3076a657684850c83d0962fb",
+        "19.0": "df520ea0775a57db7bff760145b02fed89290802913e056b7ed5970b02f3626a",
     },
     "mobile": {
         "1.0": "7da1903596bb69ef75a3c2a6c79e80328657bfed9226b2ed400ca18c88e0c1ea",
@@ -212,6 +216,7 @@
         "17.1": "33968697b94a5ff5568016a28bbcc93f7869dc2f2b2653ead833758867ab5bc9",
         "18.0": "f5f7f21c8daa59cc83f94432f0d77743be14d717f61f0464465b663508ef6d4f",
         "18.1": "c6dd56996586b2d1484e6555f9f5307f379dff24e7632e6af23097ef25656ea9",
+        "19.0": "e6c65d1c5b22ad9eb52811c9a0b66f31537d4a4ea71f40da9dfee96412a42315",
     },
     "ics": {
         "8.0": "f3b53ff8d7f0f21f3e48c651edf68353aeb3e07727c32c3e47ef882e3bca10ab",
@@ -238,6 +243,7 @@
         "17.1": "cb207f963ca270994d9dabefe52237d46cf25056f154057f4b961f1c0803a8f3",
         "18.0": "e0c64def90415d548131009ba2ba4d8a4a725ca2293861a4cc2f9e8712625531",
         "18.1": "a7c0106492843485340710be9e841c1584d8fc6da8950e7097db0e7b5bc9f164",
+        "19.0": "4a986f4a440aa0c36bd352e9b320de82160c456a680f7700fe4585d90b4a2522",
     },
 }
 

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mitreattack-python"
 description = "MITRE ATT&CK python library"
-version = "5.4.6"
+version = "5.5.0"
 authors = [{ name = "MITRE ATT&CK", email = "attack@mitre.org" }]
 license = { text = "Apache-2.0" }
 readme = "README.md"
@@ -80,6 +80,9 @@ package = true
 module-name = "mitreattack"
 module-root = ""
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
 [tool.ruff]
 line-length = 120
 extend-exclude = ["tests/resources/", "examples/", "__init__.py"]