feat(gpu): refresh OCSP cache before expiry

Author: kvinwang

dstack/gpu-attest-proxy/README.md

@@ -47,10 +47,16 @@ response is cached no later than its signed `nextUpdate` and no longer than
 hour from `thisUpdate`, matching the pinned NVIDIA SDK. RIM documents default
 to a 30-day TTL.
 
+When an OCSP response has less than `--ocsp-refresh-before` validity remaining
+(five minutes by default), the next request refreshes it synchronously from
+NVIDIA. Concurrent refreshes are coalesced. If the refresh fails, the proxy
+continues serving the old response only until its existing signed expiry; it
+never extends or serves an expired response.
+
 Expired entries are never served. Therefore a warm cache removes the NVIDIA
 service from the boot path only for the signed validity period; it does not
 turn revocation checking into an indefinite fail-open. Response headers expose
-`X-Dstack-Cache: HIT|MISS`, `Age`, and `X-Dstack-Cache-Expires` for operations.
+`X-Dstack-Cache: HIT|MISS|REFRESH`, `Age`, and `X-Dstack-Cache-Expires` for operations.
 Each cache kind is capped at 10,000 entries by default; the oldest entry is
 evicted when the limit is reached. Use `--max-cache-entries-per-kind` to tune
 the bound for a deployment.

dstack/gpu-attest-proxy/src/lib.rs

@@ -47,6 +47,7 @@ pub struct ProxyConfig {
     pub connect_timeout: Duration,
     pub ocsp_max_ttl: Duration,
     pub ocsp_default_ttl: Duration,
+    pub ocsp_refresh_before: Duration,
     pub rim_ttl: Duration,
     pub max_cache_entries_per_kind: usize,
 }
@@ -60,6 +61,7 @@ impl ProxyConfig {
             ("connect timeout", self.connect_timeout),
             ("OCSP maximum TTL", self.ocsp_max_ttl),
             ("OCSP default TTL", self.ocsp_default_ttl),
+            ("OCSP refresh window", self.ocsp_refresh_before),
             ("RIM TTL", self.rim_ttl),
         ] {
             if value.is_zero() {
@@ -205,22 +207,37 @@ impl Proxy {
             }
         };
         if let Some(entry) = self.cache_get("ocsp", &key).await {
-            return cached_response(entry, "HIT");
+            if !self.ocsp_needs_refresh(&entry) {
+                return cached_response(entry, "HIT");
+            }
         }
 
         let fill = self.fill_lock("ocsp", &key);
         let _fill = fill.lock().await;
-        if let Some(entry) = self.cache_get("ocsp", &key).await {
-            return cached_response(entry, "HIT");
+        let cached = self.cache_get("ocsp", &key).await;
+        if let Some(entry) = &cached {
+            if !self.ocsp_needs_refresh(entry) {
+                return cached_response(entry.clone(), "HIT");
+            }
         }
 
         let upstream = match self.fetch_ocsp(body).await {
             Ok(response) => response,
             Err(error) => {
                 warn!(%error, "OCSP upstream request failed");
+                if let Some(entry) = cached {
+                    return cached_response(entry, "HIT");
+                }
                 return text_response(StatusCode::BAD_GATEWAY, "OCSP upstream unavailable\n");
             }
         };
+        if upstream.status != StatusCode::OK {
+            if let Some(entry) = &cached {
+                warn!(status = %upstream.status, "OCSP refresh returned an error; using valid cached response");
+                return cached_response(entry.clone(), "HIT");
+            }
+        }
+        let cache_state = if cached.is_some() { "REFRESH" } else { "MISS" };
         if upstream.status == StatusCode::OK {
             let now = now_epoch();
             match ocsp::response_cache_expiry(
@@ -250,7 +267,7 @@ impl Proxy {
                 }
             }
         }
-        upstream_response(upstream, "MISS")
+        upstream_response(upstream, cache_state)
     }
 
     async fn handle_rim(&self, rim_id: &str) -> Response<Full<Bytes>> {
@@ -345,6 +362,11 @@ impl Proxy {
         }
     }
 
+    fn ocsp_needs_refresh(&self, entry: &CacheEntry) -> bool {
+        entry.expires_at.saturating_sub(now_epoch())
+            <= self.config.ocsp_refresh_before.as_secs() as i64
+    }
+
     fn fill_lock(&self, namespace: &str, key: &str) -> Arc<Mutex<()>> {
         self.fills
             .entry(format!("{namespace}:{key}"))
@@ -498,6 +520,10 @@ mod tests {
     }
 
     fn ocsp_response(now: i64) -> Bytes {
+        ocsp_response_with_ttl(now, 3600)
+    }
+
+    fn ocsp_response_with_ttl(now: i64, ttl: i64) -> Bytes {
         let format = |timestamp| {
             Utc.timestamp_opt(timestamp, 0)
                 .unwrap()
@@ -509,7 +535,7 @@ mod tests {
             cert_id,
             der(0x80, &[]),
             der(0x18, format(now - 60).as_bytes()),
-            der(0xa0, &der(0x18, format(now + 3600).as_bytes())),
+            der(0xa0, &der(0x18, format(now + ttl).as_bytes())),
         ]);
         let response_data = sequence(&[
             der(0x82, &[7; 20]),
@@ -590,6 +616,7 @@ mod tests {
             connect_timeout: Duration::from_secs(1),
             ocsp_max_ttl: Duration::from_secs(86_400),
             ocsp_default_ttl: Duration::from_secs(3600),
+            ocsp_refresh_before: Duration::from_secs(300),
             rim_ttl: Duration::from_secs(86_400),
             max_cache_entries_per_kind: 10,
         };
@@ -616,6 +643,45 @@ mod tests {
         upstream.abort();
     }
 
+    #[tokio::test]
+    async fn ocsp_cache_refreshes_before_expiry() {
+        let now = now_epoch();
+        let response = ocsp_response_with_ttl(now, 60);
+        let (upstream_url, requests, upstream) = mock_upstream(response, OCSP_CONTENT_TYPE).await;
+        let cache_dir = tempfile::tempdir().unwrap();
+        let proxy = Proxy::new(ProxyConfig {
+            listen_addr: "127.0.0.1:0".parse().unwrap(),
+            cache_dir: cache_dir.path().to_path_buf(),
+            ocsp_url: upstream_url,
+            rim_url: Url::parse("https://rim.attestation.nvidia.com").unwrap(),
+            service_key: None,
+            request_timeout: Duration::from_secs(5),
+            connect_timeout: Duration::from_secs(1),
+            ocsp_max_ttl: Duration::from_secs(86_400),
+            ocsp_default_ttl: Duration::from_secs(3600),
+            ocsp_refresh_before: Duration::from_secs(300),
+            rim_ttl: Duration::from_secs(86_400),
+            max_cache_entries_per_kind: 10,
+        })
+        .await
+        .unwrap();
+
+        let nonce = std::process::id() as u8;
+        assert_eq!(
+            proxy.handle_ocsp(ocsp_request(nonce)).await.headers()["x-dstack-cache"],
+            "MISS"
+        );
+        assert_eq!(
+            proxy
+                .handle_ocsp(ocsp_request(nonce.wrapping_add(1)))
+                .await
+                .headers()["x-dstack-cache"],
+            "REFRESH"
+        );
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        upstream.abort();
+    }
+
     #[tokio::test]
     async fn rim_cache_uses_versioned_id() {
         let response = Bytes::from_static(br#"{"id":"NV_GPU_DRIVER_GH100_580.1","rim":"test"}"#);
@@ -632,6 +698,7 @@ mod tests {
             connect_timeout: Duration::from_secs(1),
             ocsp_max_ttl: Duration::from_secs(86_400),
             ocsp_default_ttl: Duration::from_secs(3600),
+            ocsp_refresh_before: Duration::from_secs(300),
             rim_ttl: Duration::from_secs(86_400),
             max_cache_entries_per_kind: 10,
         })

dstack/gpu-attest-proxy/src/main.rs

@@ -59,6 +59,10 @@ struct Args {
     #[arg(long, env = "GPU_ATTEST_PROXY_OCSP_DEFAULT_TTL", default_value = "1h", value_parser = parse_duration)]
     ocsp_default_ttl: Duration,
 
+    /// Refresh an OCSP response when less than this much validity remains.
+    #[arg(long, env = "GPU_ATTEST_PROXY_OCSP_REFRESH_BEFORE", default_value = "5m", value_parser = parse_duration)]
+    ocsp_refresh_before: Duration,
+
     /// Cache lifetime for version-addressed RIM documents.
     #[arg(long, env = "GPU_ATTEST_PROXY_RIM_TTL", default_value = "30d", value_parser = parse_duration)]
     rim_ttl: Duration,
@@ -94,6 +98,7 @@ async fn main() -> Result<()> {
         connect_timeout: args.connect_timeout,
         ocsp_max_ttl: args.ocsp_max_ttl,
         ocsp_default_ttl: args.ocsp_default_ttl,
+        ocsp_refresh_before: args.ocsp_refresh_before,
         rim_ttl: args.rim_ttl,
         max_cache_entries_per_kind: args.max_cache_entries_per_kind,
     })

dstack/gpu-attest-proxy/src/ocsp.rs

@@ -59,9 +59,9 @@ pub(crate) fn response_cache_expiry(
     let max_ttl = i64::try_from(max_ttl_seconds).context("maximum OCSP TTL is too large")?;
     let mut expires_at = now.checked_add(max_ttl).context("OCSP TTL overflow")?;
     for item in validity {
-        // Do not cache a response which claims to have been produced far in
-        // the future. The guest performs the authoritative freshness check.
-        if item.this_update > now.saturating_add(300) {
+        // Do not cache a response which claims to have been produced in the
+        // future. The guest performs the authoritative freshness check.
+        if item.this_update > now {
             bail!("OCSP response thisUpdate is in the future");
         }
         let signed_expiry = item

os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/0001-validate-ocsp-response-freshness.patch

@@ -9,44 +9,43 @@ nonce prevents replay. Trust Outpost-compatible caches deliberately return a
 response with an older nonce, so the verifier must independently reject stale
 signed responses.
 
-Use OpenSSL's OCSP_check_validity with five minutes of clock skew. Preserve the
-SDK's existing one-hour lifetime for responses without nextUpdate and enforce
-that synthesized deadline as well.
+Use OpenSSL's OCSP_check_validity without accepting expired responses through
+a clock-skew grace period. Preserve the SDK's existing one-hour lifetime for
+responses without nextUpdate and enforce that synthesized deadline as well.
 
 Upstream-Status: Pending
 
 ---
- nv-attestation-sdk-cpp/src/nv_ocsp.cpp | 18 ++++++++++++++++++
- 1 file changed, 18 insertions(+)
+ nv-attestation-sdk-cpp/src/nv_ocsp.cpp | 17 +++++++++++++++++
+ 1 file changed, 17 insertions(+)
 diff --git a/nv-attestation-sdk-cpp/src/nv_ocsp.cpp b/nv-attestation-sdk-cpp/src/nv_ocsp.cpp
 index 8054971..a4a29c7 100644
 --- a/nv-attestation-sdk-cpp/src/nv_ocsp.cpp
 +++ b/nv-attestation-sdk-cpp/src/nv_ocsp.cpp
-@@ -291,6 +291,15 @@ Error NvHttpOcspClient::get_ocsp_status(
+@@ -291,6 +291,14 @@ Error NvHttpOcspClient::get_ocsp_status(
              return Error::OcspInvalidResponse;
          }
 
 +        // OCSP_basic_verify checks signature and trust, but not response time.
 +        // Cached responses intentionally cannot satisfy the request nonce, so
 +        // the signed validity window is the remaining replay boundary.
-+        constexpr long OCSP_VALIDITY_SKEW_SECONDS = 300;
-+        if (OCSP_check_validity(thisupd, nextupd, OCSP_VALIDITY_SKEW_SECONDS, -1) != 1) {
++        if (OCSP_check_validity(thisupd, nextupd, 0, -1) != 1) {
 +            LOG_ERROR("OCSP response is outside its validity window: " << get_openssl_error());
 +            return Error::OcspInvalidResponse;
 +        }
 +
          // note: timegm only works on linux.
          struct tm this_update_tm{};
          if (ASN1_TIME_to_tm((const ASN1_TIME *)thisupd, &this_update_tm) != 1) {
-@@ -318,6 +327,15 @@ Error NvHttpOcspClient::get_ocsp_status(
+@@ -318,6 +326,15 @@ Error NvHttpOcspClient::get_ocsp_status(
              LOG_DEBUG("ASN1_TIME_to_tm successful for next update time: " << next_update_time);
              out_ocsp_response.nextupd = next_update_time;
          }
 +        // When nextUpdate is absent OCSP_check_validity has no upper bound to
 +        // evaluate. The SDK already synthesizes one hour from thisUpdate;
 +        // enforce it before exposing a successful claim.
 +        time_t now = time(nullptr);
-+        if (out_ocsp_response.nextupd < now - OCSP_VALIDITY_SKEW_SECONDS) {
++        if (out_ocsp_response.nextupd < now) {
 +            LOG_ERROR("OCSP response has expired (nextUpdate: "
 +                      << out_ocsp_response.nextupd << ", now: " << now << ")");
 +            return Error::OcspInvalidResponse;