pathfinder: localize CTK coherence and driver policy

Require exact CTK matching only for authored same-component or companion relationships, so independent artifacts can coexist across minors. Add a Linux-only driver-compatibility override for forward-compatibility deployments without relaxing CTK-coherence checks.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: rwgk

cuda_pathfinder/cuda/pathfinder/_compatibility_guard_rails.py

@@ -59,7 +59,7 @@
 ItemKind: TypeAlias = str
 PackagedWith: TypeAlias = str
 CtkVersionConstraintArg: TypeAlias = str | SpecifierSet | None
-PairwiseItemRelation: TypeAlias = str
+PairwiseItemRelationKind: TypeAlias = str
 
 _CTK_VERSION_RE = re.compile(r"^(?P<major>\d+)\.(?P<minor>\d+)")
 _CTK_VERSION_CONSTRAINT_ERROR = (
@@ -69,6 +69,13 @@
 _PAIRWISE_ITEM_RELATION_NONE = "none"
 _PAIRWISE_ITEM_RELATION_EXACT_CTK_MATCH_REQUIRED = "exact-ctk-match-required"
 
+
+@dataclass(frozen=True, slots=True)
+class PairwiseItemRelation:
+    kind: PairwiseItemRelationKind
+    reason: str | None = None
+
+
 _STATIC_LIBS_PACKAGED_WITH: dict[str, PackagedWith] = {
     "cudadevrt": "ctk",
 }
@@ -87,6 +94,10 @@ class CompatibilityInsufficientMetadataError(CompatibilityCheckError):
     """Raised when v1 compatibility checks cannot reach a definitive answer."""
 
 
+class DriverCtkCompatibilityError(CompatibilityCheckError):
+    """Raised when driver-vs-CTK policy rejects a resolved item."""
+
+
 @dataclass(frozen=True, slots=True)
 class CtkMetadata:
     ctk_version: CtkVersion
@@ -140,13 +151,14 @@ def describe(self) -> str:
 class CompatibilityResult:
     status: str
     message: str
+    error_type: type[CompatibilityCheckError] = CompatibilityCheckError
 
     def require_compatible(self) -> None:
         if self.status == "compatible":
             return
         if self.status == "insufficient_metadata":
             raise CompatibilityInsufficientMetadataError(self.message)
-        raise CompatibilityCheckError(self.message)
+        raise self.error_type(self.message)
 
 
 def _parse_ctk_version(cuda_version: str) -> CtkVersion | None:
@@ -434,13 +446,19 @@ def _ctk_constraint_failure_message(item: ResolvedItem, constraint: CtkVersionCo
     return f"{item.describe()} resolves to CTK {item.ctk_version}, which does not satisfy ctk_version{constraint}."
 
 
-def _ctk_pair_mismatch_message(item1: ResolvedItem, item2: ResolvedItem) -> str:
+def _ctk_pair_mismatch_message(
+    item1: ResolvedItem,
+    item2: ResolvedItem,
+    relation: PairwiseItemRelation,
+) -> str:
     assert item1.ctk_version is not None
     assert item2.ctk_version is not None
+    assert relation.reason is not None
+    requirement_reason = relation.reason[:1].upper() + relation.reason[1:]
     return (
         f"{item1.describe()} resolves to CTK {item1.ctk_version}, while "
         f"{item2.describe()} resolves to CTK {item2.ctk_version}. "
-        "v1 requires an exact CTK major.minor match."
+        f"{requirement_reason}, so v1 requires an exact CTK major.minor match."
     )
 
 
@@ -453,11 +471,26 @@ def _driver_major_mismatch_message(driver_cuda_version: DriverCudaVersion, item:
     )
 
 
-def _compatible_pair_message(driver_cuda_version: DriverCudaVersion, item1: ResolvedItem, item2: ResolvedItem) -> str:
+def _compatible_pair_message(
+    driver_cuda_version: DriverCudaVersion,
+    item1: ResolvedItem,
+    item2: ResolvedItem,
+    relation: PairwiseItemRelation,
+) -> str:
     assert item1.ctk_version is not None
+    assert item2.ctk_version is not None
+    if relation.kind == _PAIRWISE_ITEM_RELATION_NONE:
+        return (
+            f"{item1.describe()} resolves to CTK {item1.ctk_version}, "
+            f"{item2.describe()} resolves to CTK {item2.ctk_version}, "
+            "and v1 does not require exact CTK lockstep for this pair. "
+            f"Driver version {driver_cuda_version.encoded} satisfies the v1 driver guard rail."
+        )
+    assert relation.reason is not None
     return (
-        f"{item1.describe()} and {item2.describe()} both resolve to CTK {item1.ctk_version}, "
-        f"and driver version {driver_cuda_version.encoded} satisfies the v1 driver guard rail."
+        f"{item1.describe()} and {item2.describe()} both resolve to CTK {item1.ctk_version}. "
+        f"{relation.reason[:1].upper() + relation.reason[1:]}, and driver version "
+        f"{driver_cuda_version.encoded} satisfies the v1 driver guard rail."
     )
 
 
@@ -473,18 +506,40 @@ def _ctk_metadata_result(item: ResolvedItem) -> CompatibilityResult | None:
     return CompatibilityResult(status="insufficient_metadata", message=_missing_ctk_metadata_message(item))
 
 
-def _classify_pairwise_item_relation(item1: ResolvedItem, item2: ResolvedItem) -> PairwiseItemRelation:
-    if item1.packaged_with == "driver" or item2.packaged_with == "driver":
-        return _PAIRWISE_ITEM_RELATION_NONE
-    return _PAIRWISE_ITEM_RELATION_EXACT_CTK_MATCH_REQUIRED
+def _shared_ctk_companion_tags(item1: ResolvedItem, item2: ResolvedItem) -> tuple[str, ...]:
+    return tuple(sorted(set(item1.ctk_companion_tags).intersection(item2.ctk_companion_tags)))
 
 
-def _ctk_coherence_result(item1: ResolvedItem, item2: ResolvedItem) -> CompatibilityResult | None:
+def _classify_pairwise_item_relation(item1: ResolvedItem, item2: ResolvedItem) -> PairwiseItemRelation:
+    if item1.packaged_with == "driver" or item2.packaged_with == "driver":
+        return PairwiseItemRelation(_PAIRWISE_ITEM_RELATION_NONE)
+    if item1.dynamic_link_component is not None and item1.dynamic_link_component == item2.dynamic_link_component:
+        return PairwiseItemRelation(
+            _PAIRWISE_ITEM_RELATION_EXACT_CTK_MATCH_REQUIRED,
+            reason=f"they are in the same authored dynamic-link component {item1.dynamic_link_component!r}",
+        )
+    shared_companion_tags = _shared_ctk_companion_tags(item1, item2)
+    if shared_companion_tags:
+        if len(shared_companion_tags) == 1:
+            tag_description = repr(shared_companion_tags[0])
+            reason = f"they share the authored companion tag {tag_description}"
+        else:
+            tags_description = ", ".join(repr(tag) for tag in shared_companion_tags)
+            reason = f"they share the authored companion tags {tags_description}"
+        return PairwiseItemRelation(_PAIRWISE_ITEM_RELATION_EXACT_CTK_MATCH_REQUIRED, reason=reason)
+    return PairwiseItemRelation(_PAIRWISE_ITEM_RELATION_NONE)
+
+
+def _ctk_coherence_result(
+    item1: ResolvedItem,
+    item2: ResolvedItem,
+    relation: PairwiseItemRelation,
+) -> CompatibilityResult | None:
     assert item1.ctk_version is not None
     assert item2.ctk_version is not None
     if item1.ctk_version == item2.ctk_version:
         return None
-    return CompatibilityResult(status="incompatible", message=_ctk_pair_mismatch_message(item1, item2))
+    return CompatibilityResult(status="incompatible", message=_ctk_pair_mismatch_message(item1, item2, relation))
 
 
 def _pipeline_compatibility_result(_item1: ResolvedItem, _item2: ResolvedItem) -> CompatibilityResult | None:
@@ -493,16 +548,21 @@ def _pipeline_compatibility_result(_item1: ResolvedItem, _item2: ResolvedItem) -
     return None
 
 
-def _pairwise_policy_result(item1: ResolvedItem, item2: ResolvedItem) -> CompatibilityResult | None:
-    relation = _classify_pairwise_item_relation(item1, item2)
-    if relation == _PAIRWISE_ITEM_RELATION_NONE:
+def _pairwise_policy_result(
+    item1: ResolvedItem,
+    item2: ResolvedItem,
+    relation: PairwiseItemRelation | None = None,
+) -> CompatibilityResult | None:
+    if relation is None:
+        relation = _classify_pairwise_item_relation(item1, item2)
+    if relation.kind == _PAIRWISE_ITEM_RELATION_NONE:
         return None
-    if relation == _PAIRWISE_ITEM_RELATION_EXACT_CTK_MATCH_REQUIRED:
-        result = _ctk_coherence_result(item1, item2)
+    if relation.kind == _PAIRWISE_ITEM_RELATION_EXACT_CTK_MATCH_REQUIRED:
+        result = _ctk_coherence_result(item1, item2, relation)
         if result is not None:
             return result
         return _pipeline_compatibility_result(item1, item2)
-    raise AssertionError(f"Unhandled pairwise item relation: {relation!r}")
+    raise AssertionError(f"Unhandled pairwise item relation: {relation.kind!r}")
 
 
 def _driver_compatibility_result(
@@ -514,6 +574,7 @@ def _driver_compatibility_result(
     return CompatibilityResult(
         status="incompatible",
         message=_driver_major_mismatch_message(driver_cuda_version, item),
+        error_type=DriverCtkCompatibilityError,
     )
 
 
@@ -528,7 +589,8 @@ def compatibility_check(
         if result is not None:
             return result
 
-    result = _pairwise_policy_result(item1, item2)
+    relation = _classify_pairwise_item_relation(item1, item2)
+    result = _pairwise_policy_result(item1, item2, relation)
     if result is not None:
         return result
 
@@ -538,7 +600,7 @@ def compatibility_check(
 
     return CompatibilityResult(
         status="compatible",
-        message=_compatible_pair_message(driver_cuda_version, item1, item2),
+        message=_compatible_pair_message(driver_cuda_version, item1, item2, relation),
     )
 
 
@@ -606,8 +668,8 @@ def _reset_for_testing(self) -> None:
 
     def _register_and_check(self, item: ResolvedItem) -> None:
         # Driver libraries come from the installed display driver rather than a
-        # CUDA Toolkit line, so they do not need CTK metadata and must not lock
-        # the process-wide CTK anchor.
+        # CUDA Toolkit line, so they do not need CTK metadata and must not
+        # create CTK coherence relations by themselves.
         if item.packaged_with == "driver":
             self._remember(item)
             return

cuda_pathfinder/cuda/pathfinder/_process_wide_compatibility_guard_rails.py

@@ -14,6 +14,7 @@
 from cuda.pathfinder._compatibility_guard_rails import (
     CompatibilityGuardRails,
     CompatibilityInsufficientMetadataError,
+    DriverCtkCompatibilityError,
 )
 from cuda.pathfinder._dynamic_libs.load_dl_common import LoadedDL
 from cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib import (
@@ -52,6 +53,10 @@
 _COMPATIBILITY_GUARD_RAILS_MODES = ("off", "best_effort", "strict")
 _COMPATIBILITY_GUARD_RAILS_DEFAULT_MODE = "strict"
 assert _COMPATIBILITY_GUARD_RAILS_DEFAULT_MODE in _COMPATIBILITY_GUARD_RAILS_MODES
+_DRIVER_COMPATIBILITY_ENV_VAR = "CUDA_PATHFINDER_DRIVER_COMPATIBILITY"
+_DRIVER_COMPATIBILITY_MODES = ("default", "assume_forward_compatibility")
+_DRIVER_COMPATIBILITY_DEFAULT_MODE = "default"
+assert _DRIVER_COMPATIBILITY_DEFAULT_MODE in _DRIVER_COMPATIBILITY_MODES
 
 
 class _ProcessWideGuardRailsApi(Protocol):
@@ -93,6 +98,37 @@ def _compatibility_guard_rails_mode() -> str:
     )
 
 
+def _driver_compatibility_mode() -> str:
+    value = os.environ.get(_DRIVER_COMPATIBILITY_ENV_VAR)
+    if not value:
+        return _DRIVER_COMPATIBILITY_DEFAULT_MODE
+    if value not in _DRIVER_COMPATIBILITY_MODES:
+        allowed_values = ", ".join(repr(mode) for mode in _DRIVER_COMPATIBILITY_MODES)
+        raise RuntimeError(
+            f"Invalid {_DRIVER_COMPATIBILITY_ENV_VAR}={value!r}. "
+            f"Allowed values: {allowed_values}. "
+            f"Unset or empty defaults to {_DRIVER_COMPATIBILITY_DEFAULT_MODE!r}."
+        )
+    if value == "assume_forward_compatibility" and not sys.platform.startswith("linux"):
+        raise RuntimeError(f"{_DRIVER_COMPATIBILITY_ENV_VAR}={value!r} is only supported on Linux.")
+    return value
+
+
+def _driver_compatibility_override_hint() -> str:
+    return (
+        "On supported Linux systems that intentionally rely on NVIDIA forward compatibility "
+        f"(`cuda-compat-*`), set {_DRIVER_COMPATIBILITY_ENV_VAR}=assume_forward_compatibility "
+        "to bypass this driver-vs-CTK check. This does not relax CTK-coherence checks "
+        "between headers, libraries, and compiler/JIT components."
+    )
+
+
+def _with_driver_compatibility_hint(message: str) -> str:
+    if _DRIVER_COMPATIBILITY_ENV_VAR in message:
+        return message
+    return f"{message} {_driver_compatibility_override_hint()}"
+
+
 def _public_module() -> _PublicPathfinderModule | None:
     public_module = sys.modules.get("cuda.pathfinder")
     if public_module is None:
@@ -121,6 +157,7 @@ def _reset_process_wide_compatibility_guard_rails() -> None:
 
 
 def _try_process_wide_guard_rails_then_fallback(guard_rails_call: Callable[[], _T], raw_call: Callable[[], _T]) -> _T:
+    driver_compatibility_mode = _driver_compatibility_mode()
     mode = _compatibility_guard_rails_mode()
     if mode == "off":
         return raw_call()
@@ -130,6 +167,12 @@ def _try_process_wide_guard_rails_then_fallback(guard_rails_call: Callable[[], _
         if mode == "best_effort":
             return raw_call()
         raise
+    except DriverCtkCompatibilityError as exc:
+        if driver_compatibility_mode == "assume_forward_compatibility":
+            return raw_call()
+        if sys.platform.startswith("linux"):
+            raise DriverCtkCompatibilityError(_with_driver_compatibility_hint(str(exc))) from exc
+        raise
 
 
 def _cache_clear_with_process_state_reset(cache_clear: Callable[[], object]) -> Callable[[], None]: