Merge pull request #14076 from Turbo87/cache-tags-backfill

Backfill `cache-tags` metadata onto existing S3 objects

Author: Turbo87

Cargo.lock

@@ -69,6 +69,7 @@ aws-credential-types = { version = "=1.2.14", features = ["hardcoded-credentials
 # `rustls-webpki` v0.101.x. Using only `default-https-client` avoids this by
 # using the modern rustls 0.23 + hyper 1.x stack instead.
 aws-sdk-cloudfront = { version = "=1.123.0", default-features = false, features = ["default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "=1.137.0", default-features = false, features = ["default-https-client", "rt-tokio"] }
 aws-sdk-sqs = { version = "=1.102.0", default-features = false, features = ["default-https-client", "rt-tokio"] }
 axum = { version = "=0.8.9", features = ["macros", "matched-path"] }
 axum-extra = { version = "=0.12.6", features = ["erased-json", "middleware", "query", "typed-header"] }

Cargo.toml

@@ -0,0 +1,14 @@
+use crate::schema::cache_tags_backfills;
+
+/// Struct used to `INSERT` a completed cache-tags backfill record into the
+/// `cache_tags_backfills` table.
+///
+/// A row records that every S3 object for the crate has been re-copied with
+/// `cache-tags` metadata. `crate_id` is `None` only if the crate was deleted
+/// after the backfill completed but before the record was written.
+#[derive(Debug, diesel::Insertable, bon::Builder)]
+#[diesel(table_name = cache_tags_backfills, check_for_backend(diesel::pg::Pg))]
+pub struct NewCacheTagsBackfillRow<'a> {
+    pub crate_id: Option<i32>,
+    pub crate_name: &'a str,
+}

crates/crates_io_database/src/models/cache_tags_backfill.rs

@@ -1,4 +1,5 @@
 pub use self::action::{NewVersionOwnerAction, VersionAction, VersionOwnerAction};
+pub use self::cache_tags_backfill::NewCacheTagsBackfillRow;
 pub use self::category::{Category, CrateCategory, NewCategory};
 pub use self::cloudfront_invalidation_queue::{
     CloudFrontDistribution, CloudFrontInvalidationQueueItem,
@@ -24,6 +25,7 @@ pub use self::version::{NewVersion, TopVersions, Version};
 pub mod helpers;
 
 mod action;
+mod cache_tags_backfill;
 pub mod category;
 mod cloudfront_invalidation_queue;
 pub mod crate_owner_invitation;

crates/crates_io_database/src/models/mod.rs

@@ -130,6 +130,23 @@ diesel::table! {
     }
 }
 
+diesel::table! {
+    use diesel::sql_types::*;
+    use diesel_full_text_search::Tsvector;
+
+    /// Records which crates have had their S3 objects backfilled with `cache-tags` metadata. A row exists only for crates whose backfill completed.
+    cache_tags_backfills (id) {
+        /// Timestamp when the backfill for this crate completed.
+        completed_at -> Timestamptz,
+        /// The backfilled crate, or `NULL` if the crate was deleted after the backfill completed.
+        crate_id -> Nullable<Int4>,
+        /// The crate name at backfill time.
+        crate_name -> Text,
+        /// Unique identifier for each backfill record.
+        id -> Int8,
+    }
+}
+
 diesel::table! {
     use diesel::sql_types::*;
     use diesel_full_text_search::Tsvector;
@@ -1277,6 +1294,7 @@ diesel::table! {
 }
 
 diesel::joinable!(api_tokens -> users (user_id));
+diesel::joinable!(cache_tags_backfills -> crates (crate_id));
 diesel::joinable!(crate_downloads -> crates (crate_id));
 diesel::joinable!(crate_owner_invitations -> crates (crate_id));
 diesel::joinable!(crate_owners -> crates (crate_id));
@@ -1313,6 +1331,7 @@ diesel::joinable!(versions_published_by -> versions (version_id));
 diesel::allow_tables_to_appear_in_same_query!(
     api_tokens,
     background_jobs,
+    cache_tags_backfills,
     categories,
     cloudfront_invalidation_queue,
     crate_downloads,

crates/crates_io_database/src/schema.rs

@@ -40,6 +40,12 @@ last_retry = "private"
 created_at = "private"
 priority = "private"
 
+[cache_tags_backfills.columns]
+id = "private"
+crate_id = "private"
+crate_name = "private"
+completed_at = "private"
+
 [categories.columns]
 id = "public"
 category = "public"

crates/crates_io_database_dump/src/dump-db.toml

@@ -0,0 +1 @@
+DROP TABLE cache_tags_backfills;

migrations/2026-06-26-110013-0000_create_cache_tags_backfills/down.sql

@@ -0,0 +1,24 @@
+CREATE TABLE IF NOT EXISTS cache_tags_backfills (
+    id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    crate_id INTEGER REFERENCES crates(id) ON DELETE SET NULL,
+    crate_name TEXT NOT NULL,
+    completed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+COMMENT ON TABLE cache_tags_backfills IS
+    'Records which crates have had their S3 objects backfilled with `cache-tags` metadata. A row exists only for crates whose backfill completed.';
+COMMENT ON COLUMN cache_tags_backfills.id IS
+    'Unique identifier for each backfill record.';
+COMMENT ON COLUMN cache_tags_backfills.crate_id IS
+    'The backfilled crate, or `NULL` if the crate was deleted after the backfill completed.';
+COMMENT ON COLUMN cache_tags_backfills.crate_name IS
+    'The crate name at backfill time.';
+COMMENT ON COLUMN cache_tags_backfills.completed_at IS
+    'Timestamp when the backfill for this crate completed.';
+
+-- safety-assured:start
+-- Suppresses the "ADD INDEX without CONCURRENTLY" diesel-guard check.
+-- The table is brand new and empty, so the unique index builds instantly with
+-- no meaningful SHARE lock contention.
+CREATE UNIQUE INDEX idx_cache_tags_backfills_crate_id ON cache_tags_backfills (crate_id);
+-- safety-assured:end