fix(license): canonicalize LicenseRef-scancode SPDX keys to mirror license keys (#1223)

Upstream ScanCode enforces an arbitrary "spdx_license_key must be 50
characters or less" lint that squashed or truncated dozens of
LicenseRef-scancode-* SPDX keys (e.g.
LicenseRef-scancode-openssl-exception-lgpl3.0plus instead of the canonical
...lgpl-3.0-plus; see aboutcode-org/scancode-toolkit#5221). For the
LicenseRef-scancode namespace the identifier is by definition the license
key with the namespace prefix, so any deviation is a distortion, not a
semantic choice.

Provenant applies no length limit, so restore the canonical form at
index-build time and keep the previous value in other_spdx_license_keys
for backward compatibility. An audit of all 2733 licenses found 50 such
deviations: 49 are canonicalized by the general rule; 1
(tgc-spec-license-v2) carries an upstream typo in the license *key* while
its SPDX key is already correct (tcg), so it is exempted from the pass
(mirroring the key would regress the correct SPDX key, and renaming the
key would drop a ScanCode-compatible identifier with no alias mechanism).

Signed-off-by: Maxim Stykow <maxim.stykow@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mstykow

docs/LICENSE_DETECTION_ARCHITECTURE.md

@@ -75,6 +75,19 @@ instead of minting a tool-specific variant. This applies across detection output
 fallbacks such as `unknown-spdx`, and parser-side declared-license normalization for keys such as
 `public-domain`, `proprietary-license`, and `unknown-license-reference`.
 
+Because a `LicenseRef-scancode-<key>` identifier is by definition the license `key` with the
+namespace prefix, the identifier must mirror the key. Upstream ScanCode enforces an arbitrary
+"spdx_license_key must be 50 characters or less" lint, which forced dozens of keys to be squashed
+or truncated (for example `LicenseRef-scancode-openssl-exception-lgpl3.0plus` instead of the
+canonical `LicenseRef-scancode-openssl-exception-lgpl-3.0-plus`; see ScanCode PR
+[#5221](https://github.com/aboutcode-org/scancode-toolkit/pull/5221)). Provenant applies no such
+length limit, so `spdx_key_canonicalization` restores the canonical form at index-build time and
+keeps the previous value in `other_spdx_license_keys` for backward compatibility. The rare case
+where the license `key` itself is misspelled (the SPDX key already being correct) is exempted by an
+explicit, justified entry: mirroring the key would regress the correct SPDX key, so the entry is
+left exactly as upstream has it (the license key stays ScanCode-compatible and its SPDX key stays
+right).
+
 Parser-side declared-license normalization also maps bare, informal license names that are not
 valid SPDX but resolve to a license by strong convention (for example `Apache`, `PSF`, `Python`).
 These mappings live in a curated config, `resources/license_detection/declared_license_aliases.toml`,
@@ -141,6 +154,7 @@ The loading process is split into two distinct stages:
 - Apply the checked-in license-index build policy
 - Apply any checked-in downstream overlay files from `resources/license_detection/overlay/`
 - Fail fast if an ignore id no longer exists upstream, an overlay-reason entry is stale or missing, or an overlay file becomes identical to upstream data
+- Canonicalize `LicenseRef-scancode-*` SPDX keys so each mirrors its license key (see the `LicenseRef-*` namespace convention above)
 - Sort embedded rules and licenses deterministically
 - Serialize the embedded loader snapshot with MessagePack
 - Compress the serialized bytes with zstd

resources/license_detection/license_index.zst

@@ -29,6 +29,7 @@ pub mod models;
 pub mod query;
 pub mod rules;
 pub mod seq_match;
+pub mod spdx_key_canonicalization;
 pub mod spdx_lid;
 pub mod spdx_mapping;
 #[cfg(test)]

src/license_detection/mod.rs

@@ -23,7 +23,7 @@ use serde::{Deserialize, Serialize};
 /// This struct contains parsed and normalized data from a .LICENSE file.
 /// It is serialized at build time and deserialized at runtime, then converted
 /// to a runtime `License` during the build stage.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
 pub struct LoadedLicense {
     pub key: String,
     pub short_name: Option<String>,

src/license_detection/models/loaded_license.rs

@@ -0,0 +1,219 @@
+// SPDX-FileCopyrightText: Provenant contributors
+// SPDX-License-Identifier: Apache-2.0
+
+//! Build-time canonicalization of `LicenseRef-scancode-*` SPDX keys.
+//!
+//! Upstream ScanCode enforces an (arbitrary) "spdx_license_key must be 50
+//! characters or less" lint in `licensedcode/models.py`. For licenses in the
+//! ScanCode `LicenseRef-scancode-<key>` namespace this forced dozens of keys to
+//! be squashed or truncated (for example
+//! `LicenseRef-scancode-openssl-exception-lgpl3.0plus` instead of the canonical
+//! `LicenseRef-scancode-openssl-exception-lgpl-3.0-plus`), see ScanCode PR
+//! aboutcode-org/scancode-toolkit#5221.
+//!
+//! For the `LicenseRef-scancode-` namespace the SPDX key is by definition the
+//! license `key` with the namespace prefix, so any deviation is a distortion,
+//! not a semantic choice. Provenant applies no length limit, so this pass
+//! restores the canonical form at index-build time and keeps the previous value
+//! in `other_spdx_license_keys` for backward compatibility.
+//!
+//! A small set of licenses carry a typo in the license `key` itself while their
+//! `spdx_license_key` is already correct. There, mirroring the key would
+//! *regress* the correct SPDX key, so those keys are exempted from this pass.
+//! The license key is left exactly as upstream has it (ScanCode parity), which
+//! is harmless because renaming a license key is a foreign-identity change with
+//! no backward-compatible alias mechanism, and the SPDX key users consume is
+//! already right.
+
+use crate::license_detection::models::LoadedLicense;
+
+/// The ScanCode SPDX LicenseRef namespace prefix.
+const SCANCODE_LICENSEREF_PREFIX: &str = "LicenseRef-scancode-";
+
+/// License keys exempted from canonicalization because the upstream typo is in
+/// the license `key`, not the SPDX key. Mirroring the key would regress an
+/// already-correct `spdx_license_key`, so the entry is left untouched.
+///
+/// Each entry must be justified.
+const SPDX_CANONICALIZATION_EXEMPT_KEYS: &[&str] = &[
+    // "TCG" = Trusted Computing Group (see the license owner/holders). The key
+    // misspells it as "tgc" while spdx_license_key already uses the correct
+    // "tcg" (LicenseRef-scancode-tcg-spec-license-v2). Canonicalizing would
+    // rewrite that correct SPDX key into the typo, so leave it as upstream has
+    // it; the license key stays ScanCode-compatible and the SPDX key stays right.
+    "tgc-spec-license-v2",
+];
+
+/// A single SPDX key that was canonicalized to mirror the license `key`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SpdxKeyCanonicalization {
+    pub license_key: String,
+    pub previous_spdx_license_key: String,
+    pub canonical_spdx_license_key: String,
+}
+
+/// Summary of the changes applied by [`canonicalize_license_spdx_keys`].
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct SpdxKeyCanonicalizationReport {
+    pub canonicalized_spdx_keys: Vec<SpdxKeyCanonicalization>,
+}
+
+impl SpdxKeyCanonicalizationReport {
+    pub fn is_empty(&self) -> bool {
+        self.canonicalized_spdx_keys.is_empty()
+    }
+}
+
+/// Canonicalize `LicenseRef-scancode-*` SPDX keys in place so the suffix mirrors
+/// each license `key`.
+///
+/// This is a build-time curation step: it runs when the embedded license index
+/// artifact is generated, so the corrected values are baked into the artifact
+/// and flow through the SPDX mapping, license references, and SPDX output.
+pub fn canonicalize_license_spdx_keys(
+    licenses: &mut [LoadedLicense],
+) -> SpdxKeyCanonicalizationReport {
+    let mut report = SpdxKeyCanonicalizationReport::default();
+
+    for license in licenses.iter_mut() {
+        if SPDX_CANONICALIZATION_EXEMPT_KEYS.contains(&license.key.as_str()) {
+            continue;
+        }
+
+        let Some(spdx) = license.spdx_license_key.as_deref() else {
+            continue;
+        };
+        if !spdx.starts_with(SCANCODE_LICENSEREF_PREFIX) {
+            continue;
+        }
+
+        let canonical = format!("{SCANCODE_LICENSEREF_PREFIX}{}", license.key);
+        if spdx == canonical {
+            continue;
+        }
+
+        let previous = spdx.to_string();
+
+        // Drop the canonical form from the alias list to avoid duplicating the
+        // new primary, and preserve the previous primary as a backward-compatible
+        // alias.
+        license
+            .other_spdx_license_keys
+            .retain(|k| k != &canonical && k != &previous);
+        license.other_spdx_license_keys.push(previous.clone());
+
+        license.spdx_license_key = Some(canonical.clone());
+        report
+            .canonicalized_spdx_keys
+            .push(SpdxKeyCanonicalization {
+                license_key: license.key.clone(),
+                previous_spdx_license_key: previous,
+                canonical_spdx_license_key: canonical,
+            });
+    }
+
+    report
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn license(key: &str, spdx: Option<&str>, other: &[&str]) -> LoadedLicense {
+        LoadedLicense {
+            key: key.to_string(),
+            spdx_license_key: spdx.map(str::to_string),
+            other_spdx_license_keys: other.iter().map(|s| s.to_string()).collect(),
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn promotes_canonical_form_and_demotes_squashed_primary() {
+        // The openssl-exception-lgpl-3.0-plus case from ScanCode PR #5221.
+        let mut licenses = vec![license(
+            "openssl-exception-lgpl-3.0-plus",
+            Some("LicenseRef-scancode-openssl-exception-lgpl3.0plus"),
+            &["LicenseRef-scancode-openssl-exception-lgpl-3.0-plus"],
+        )];
+
+        let report = canonicalize_license_spdx_keys(&mut licenses);
+
+        assert_eq!(
+            licenses[0].spdx_license_key.as_deref(),
+            Some("LicenseRef-scancode-openssl-exception-lgpl-3.0-plus")
+        );
+        assert_eq!(
+            licenses[0].other_spdx_license_keys,
+            vec!["LicenseRef-scancode-openssl-exception-lgpl3.0plus".to_string()]
+        );
+        assert_eq!(report.canonicalized_spdx_keys.len(), 1);
+    }
+
+    #[test]
+    fn adds_previous_primary_when_canonical_not_already_present() {
+        // Truncation case (gradle-enterprise-sla-2022-11-08): canonical is not in
+        // the alias list yet.
+        let mut licenses = vec![license(
+            "gradle-enterprise-sla-2022-11-08",
+            Some("LicenseRef-scancode-gradle-enterprise-sla-2022-11-"),
+            &[],
+        )];
+
+        canonicalize_license_spdx_keys(&mut licenses);
+
+        assert_eq!(
+            licenses[0].spdx_license_key.as_deref(),
+            Some("LicenseRef-scancode-gradle-enterprise-sla-2022-11-08")
+        );
+        assert_eq!(
+            licenses[0].other_spdx_license_keys,
+            vec!["LicenseRef-scancode-gradle-enterprise-sla-2022-11-".to_string()]
+        );
+    }
+
+    #[test]
+    fn exempts_key_typo_so_correct_spdx_key_is_not_regressed() {
+        // tgc-spec-license-v2: the key is the typo, the SPDX key is already
+        // correct. The pass must leave both fields untouched (no rename, no SPDX
+        // regression).
+        let mut licenses = vec![license(
+            "tgc-spec-license-v2",
+            Some("LicenseRef-scancode-tcg-spec-license-v2"),
+            &[],
+        )];
+
+        let report = canonicalize_license_spdx_keys(&mut licenses);
+
+        assert_eq!(licenses[0].key, "tgc-spec-license-v2");
+        assert_eq!(
+            licenses[0].spdx_license_key.as_deref(),
+            Some("LicenseRef-scancode-tcg-spec-license-v2"),
+            "correct SPDX key must be preserved, not regressed to the typo"
+        );
+        assert!(licenses[0].other_spdx_license_keys.is_empty());
+        assert!(report.is_empty());
+    }
+
+    #[test]
+    fn leaves_canonical_and_real_spdx_keys_untouched() {
+        let mut licenses = vec![
+            license("mit", Some("MIT"), &[]),
+            license(
+                "some-ref",
+                Some("LicenseRef-scancode-some-ref"),
+                &["LicenseRef-scancode-old-alias"],
+            ),
+            license("no-spdx", None, &[]),
+        ];
+
+        let report = canonicalize_license_spdx_keys(&mut licenses);
+
+        assert!(report.is_empty());
+        assert_eq!(licenses[0].spdx_license_key.as_deref(), Some("MIT"));
+        assert_eq!(
+            licenses[1].other_spdx_license_keys,
+            vec!["LicenseRef-scancode-old-alias".to_string()]
+        );
+    }
+}

src/license_detection/spdx_key_canonicalization.rs

@@ -428,6 +428,37 @@ fn test_engine_spdx_mapping() {
     );
 }
 
+#[test]
+fn test_embedded_licenseref_spdx_keys_are_canonicalized() {
+    let engine = get_engine();
+    let mapping = engine.spdx_mapping();
+
+    // The squashed 50-char-limit form is restored to the canonical dashed form
+    // in the embedded artifact (ScanCode PR #5221 and the wider audit).
+    assert_eq!(
+        mapping
+            .scancode_to_spdx("openssl-exception-lgpl-3.0-plus")
+            .as_deref(),
+        Some("LicenseRef-scancode-openssl-exception-lgpl-3.0-plus"),
+    );
+    assert_eq!(
+        mapping.scancode_to_spdx("bash-exception-gpl").as_deref(),
+        Some("LicenseRef-scancode-bash-exception-gpl"),
+    );
+
+    // The `tgc-spec-license-v2` key carries an upstream typo, but its SPDX key is
+    // already correct (`tcg`). It is exempted from canonicalization so the key
+    // stays ScanCode-compatible and the correct SPDX key is not regressed.
+    assert_eq!(
+        mapping.scancode_to_spdx("tgc-spec-license-v2").as_deref(),
+        Some("LicenseRef-scancode-tcg-spec-license-v2"),
+    );
+    assert!(
+        mapping.scancode_to_spdx("tcg-spec-license-v2").is_none(),
+        "the license key is left as upstream has it; no renamed key is introduced",
+    );
+}
+
 #[test]
 fn test_engine_detect_no_license() {
     let engine = get_engine();

src/license_detection/tests.rs

@@ -17,6 +17,7 @@ use provenant::license_detection::embedded::schema::{EmbeddedLoaderSnapshot, SCH
 use provenant::license_detection::rules::{
     load_loaded_licenses_from_directory, load_loaded_rules_from_directory,
 };
+use provenant::license_detection::spdx_key_canonicalization::canonicalize_license_spdx_keys;
 
 #[derive(Parser, Debug)]
 #[command(
@@ -62,6 +63,17 @@ fn main() -> Result<()> {
         apply_default_index_build_policy(loaded_rules, loaded_licenses)?;
     loaded_rules = filtered_rules;
     loaded_licenses = filtered_licenses;
+
+    // Restore canonical LicenseRef-scancode-<key> SPDX keys that upstream squashed
+    // to satisfy its 50-character lint (see spdx_key_canonicalization docs).
+    let spdx_canonicalization = canonicalize_license_spdx_keys(&mut loaded_licenses);
+    if !spdx_canonicalization.is_empty() {
+        println!(
+            "Canonicalized {} LicenseRef-scancode SPDX keys",
+            spdx_canonicalization.canonicalized_spdx_keys.len()
+        );
+    }
+
     let dataset_fingerprint = compute_dataset_fingerprint_string(&loaded_rules, &loaded_licenses)?;
     let license_index_provenance = policy_report
         .to_license_index_provenance(EMBEDDED_LICENSE_INDEX_SOURCE, dataset_fingerprint);