PM-33527 Database SQL scripts

src/Core/Dirt/Entities/OrganizationEventCleanup.cs

@@ -0,0 +1,18 @@
+using Bit.Core.Entities;
+using Bit.Core.Utilities;
+
+namespace Bit.Core.Dirt.Entities;
+
+public class OrganizationEventCleanup : ITableObject<Guid>
+{
+    public Guid Id { get; set; }
+    public Guid OrganizationId { get; set; }
+    public DateTime CreationDate { get; set; } = DateTime.UtcNow;
+    public DateTime? RevisionDate { get; set; }
+    public DateTime? StartDate { get; set; }
+    public DateTime? CompletedDate { get; set; }
+    public long EventsDeletedCount { get; set; }
+    public int FailureCount { get; set; }
+    public string? LastError { get; set; }
+    public void SetNewId() => Id = CoreHelpers.GenerateComb();
+}

src/Core/Dirt/Repositories/IOrganizationEventCleanupRepository.cs

@@ -0,0 +1,12 @@
+using Bit.Core.Dirt.Entities;
+
+namespace Bit.Core.Dirt.Repositories;
+
+public interface IOrganizationEventCleanupRepository
+{
+    Task CreateAsync(OrganizationEventCleanup cleanup);
+    Task<OrganizationEventCleanup?> ClaimNextPendingAsync();
+    Task UpdateProgressAsync(Guid id, long delta);
+    Task UpdateErrorAsync(Guid id, string message);
+    Task UpdateCompletedAsync(Guid id);
+}

src/Infrastructure.Dapper/DapperServiceCollectionExtensions.cs

@@ -82,6 +82,7 @@ public static void AddDapperRepositories(this IServiceCollection services, bool
         services.AddSingleton<IUserSignatureKeyPairRepository, UserSignatureKeyPairRepository>();
         services.AddSingleton<IOrganizationReportRepository, OrganizationReportRepository>();
         services.AddSingleton<IOrganizationApplicationRepository, OrganizationApplicationRepository>();
+        services.AddSingleton<IOrganizationEventCleanupRepository, OrganizationEventCleanupRepository>();
         services.AddSingleton<IOrganizationMemberBaseDetailRepository, OrganizationMemberBaseDetailRepository>();
 
         if (selfHosted)

src/Infrastructure.Dapper/Dirt/Repositories/OrganizationEventCleanupRepository.cs

@@ -0,0 +1,66 @@
+using System.Data;
+using Bit.Core.Dirt.Entities;
+using Bit.Core.Dirt.Repositories;
+using Bit.Core.Settings;
+using Bit.Infrastructure.Dapper.Repositories;
+using Dapper;
+using Microsoft.Data.SqlClient;
+
+namespace Bit.Infrastructure.Dapper.Dirt.Repositories;
+
+public class OrganizationEventCleanupRepository : BaseRepository, IOrganizationEventCleanupRepository
+{
+    private const int LeaseDurationMinutes = 10;
+    private const int MaxFailureCount = 5;
+
+    public OrganizationEventCleanupRepository(GlobalSettings globalSettings)
+        : base(globalSettings.SqlServer.ConnectionString, globalSettings.SqlServer.ReadOnlyConnectionString)
+    { }
+
+    public async Task CreateAsync(OrganizationEventCleanup cleanup)
+    {
+        cleanup.SetNewId();
+        using var connection = new SqlConnection(ConnectionString);
+        await connection.ExecuteAsync(
+            "[dbo].[OrganizationEventCleanup_Create]",
+            new { cleanup.Id, cleanup.OrganizationId, cleanup.CreationDate },
+            commandType: CommandType.StoredProcedure);
+    }
+
+    public async Task<OrganizationEventCleanup?> ClaimNextPendingAsync()
+    {
+        var now = DateTime.UtcNow;
+        using var connection = new SqlConnection(ConnectionString);
+        return await connection.QuerySingleOrDefaultAsync<OrganizationEventCleanup>(
+            "[dbo].[OrganizationEventCleanup_ClaimNextPending]",
+            new { Now = now, StaleLeaseThreshold = now.AddMinutes(-LeaseDurationMinutes), MaxFailureCount },
+            commandType: CommandType.StoredProcedure);
+    }
+
+    public async Task UpdateProgressAsync(Guid id, long delta)
+    {
+        using var connection = new SqlConnection(ConnectionString);
+        await connection.ExecuteAsync(
+            "[dbo].[OrganizationEventCleanup_UpdateProgress]",
+            new { Id = id, Delta = delta, Now = DateTime.UtcNow },
+            commandType: CommandType.StoredProcedure);
+    }
+
+    public async Task UpdateErrorAsync(Guid id, string message)
+    {
+        using var connection = new SqlConnection(ConnectionString);
+        await connection.ExecuteAsync(
+            "[dbo].[OrganizationEventCleanup_UpdateError]",
+            new { Id = id, Message = message, Now = DateTime.UtcNow },
+            commandType: CommandType.StoredProcedure);
+    }
+
+    public async Task UpdateCompletedAsync(Guid id)
+    {
+        using var connection = new SqlConnection(ConnectionString);
+        await connection.ExecuteAsync(
+            "[dbo].[OrganizationEventCleanup_UpdateCompleted]",
+            new { Id = id, Now = DateTime.UtcNow },
+            commandType: CommandType.StoredProcedure);
+    }
+}

src/Sql/dbo/Dirt/Stored Procedures/OrganizationEventCleanup_ClaimNextPending.sql

@@ -0,0 +1,43 @@
+CREATE PROCEDURE [dbo].[OrganizationEventCleanup_ClaimNextPending]
+    @Now DATETIME2(7),
+    @StaleLeaseThreshold DATETIME2(7),
+    @MaxFailureCount INT
+AS
+BEGIN
+    SET NOCOUNT ON
+
+    ;WITH [Pending] AS (
+        SELECT TOP 1
+            [Id],
+            [OrganizationId],
+            [CreationDate],
+            [RevisionDate],
+            [StartDate],
+            [CompletedDate],
+            [EventsDeletedCount],
+            [FailureCount],
+            [LastError]
+        FROM
+            [dbo].[OrganizationEventCleanup] WITH (UPDLOCK, READPAST)
+        WHERE
+            [CompletedDate] IS NULL
+            AND ([StartDate] IS NULL OR [RevisionDate] < @StaleLeaseThreshold)
+            AND [FailureCount] < @MaxFailureCount
+        ORDER BY
+            [CreationDate] ASC
+    )
+    UPDATE [Pending]
+    SET
+        [StartDate] = COALESCE([StartDate], @Now),
+        [RevisionDate] = @Now
+    OUTPUT
+        inserted.[Id],
+        inserted.[OrganizationId],
+        inserted.[CreationDate],
+        inserted.[RevisionDate],
+        inserted.[StartDate],
+        inserted.[CompletedDate],
+        inserted.[EventsDeletedCount],
+        inserted.[FailureCount],
+        inserted.[LastError]
+END

src/Sql/dbo/Dirt/Stored Procedures/OrganizationEventCleanup_Create.sql

@@ -0,0 +1,21 @@
+CREATE PROCEDURE [dbo].[OrganizationEventCleanup_Create]
+    @Id UNIQUEIDENTIFIER,
+    @OrganizationId UNIQUEIDENTIFIER,
+    @CreationDate DATETIME2(7)
+AS
+BEGIN
+    SET NOCOUNT ON
+
+    INSERT INTO [dbo].[OrganizationEventCleanup]
+    (
+        [Id],
+        [OrganizationId],
+        [CreationDate]
+    )
+    VALUES
+    (
+        @Id,
+        @OrganizationId,
+        @CreationDate
+    )
+END

src/Sql/dbo/Dirt/Stored Procedures/OrganizationEventCleanup_UpdateCompleted.sql

@@ -0,0 +1,15 @@
+CREATE PROCEDURE [dbo].[OrganizationEventCleanup_UpdateCompleted]
+    @Id UNIQUEIDENTIFIER,
+    @Now DATETIME2(7)
+AS
+BEGIN
+    SET NOCOUNT ON
+
+    UPDATE
+        [dbo].[OrganizationEventCleanup]
+    SET
+        [CompletedDate] = @Now,
+        [RevisionDate] = @Now
+    WHERE
+        [Id] = @Id
+END

src/Sql/dbo/Dirt/Stored Procedures/OrganizationEventCleanup_UpdateError.sql

@@ -0,0 +1,17 @@
+CREATE PROCEDURE [dbo].[OrganizationEventCleanup_UpdateError]
+    @Id UNIQUEIDENTIFIER,
+    @Message NVARCHAR(MAX),
+    @Now DATETIME2(7)
+AS
+BEGIN
+    SET NOCOUNT ON
+
+    UPDATE
+        [dbo].[OrganizationEventCleanup]
+    SET
+        [FailureCount] = [FailureCount] + 1,
+        [LastError] = @Message,
+        [RevisionDate] = @Now
+    WHERE
+        [Id] = @Id
+END

src/Sql/dbo/Dirt/Stored Procedures/OrganizationEventCleanup_UpdateProgress.sql

@@ -0,0 +1,16 @@
+CREATE PROCEDURE [dbo].[OrganizationEventCleanup_UpdateProgress]
+    @Id UNIQUEIDENTIFIER,
+    @Delta BIGINT,
+    @Now DATETIME2(7)
+AS
+BEGIN
+    SET NOCOUNT ON
+
+    UPDATE
+        [dbo].[OrganizationEventCleanup]
+    SET
+        [EventsDeletedCount] = [EventsDeletedCount] + @Delta,
+        [RevisionDate] = @Now
+    WHERE
+        [Id] = @Id
+END

src/Sql/dbo/Dirt/Tables/OrganizationEventCleanup.sql

@@ -0,0 +1,17 @@
+CREATE TABLE [dbo].[OrganizationEventCleanup] (
+    [Id]                 UNIQUEIDENTIFIER NOT NULL,
+    [OrganizationId]     UNIQUEIDENTIFIER NOT NULL,
+    [CreationDate]       DATETIME2 (7)    NOT NULL,
+    [RevisionDate]       DATETIME2 (7)    NULL,
+    [StartDate]          DATETIME2 (7)    NULL,
+    [CompletedDate]      DATETIME2 (7)    NULL,
+    [EventsDeletedCount] BIGINT           NOT NULL CONSTRAINT [DF_OrganizationEventCleanup_EventsDeletedCount] DEFAULT (0),
+    [FailureCount]       INT              NOT NULL CONSTRAINT [DF_OrganizationEventCleanup_FailureCount] DEFAULT (0),
+    [LastError]          NVARCHAR(MAX)    NULL,
+    CONSTRAINT [PK_OrganizationEventCleanup] PRIMARY KEY CLUSTERED ([Id] ASC)
+);
+GO
+
+CREATE NONCLUSTERED INDEX [IX_OrganizationEventCleanup_CompletedDate_CreationDate]
+    ON [dbo].[OrganizationEventCleanup]([CompletedDate] ASC, [CreationDate] ASC);
+GO

test/Infrastructure.IntegrationTest/DatabaseDataAttribute.cs

@@ -29,6 +29,7 @@ private static IConfiguration GetConfiguration()
 
     public bool SelfHosted { get; set; }
     public bool UseFakeTimeProvider { get; set; }
+    public SupportedDatabaseProviders[] OnlyOn { get; set; } = [];
 
     public override ValueTask<IReadOnlyCollection<ITheoryDataRow>> GetData(MethodInfo testMethod, DisposalTracker disposalTracker)
     {
@@ -48,6 +49,16 @@ public override ValueTask<IReadOnlyCollection<ITheoryDataRow>> GetData(MethodInf
         {
             unconfiguredDatabases.Remove(database.Type);
 
+            if (OnlyOn.Length > 0 && !OnlyOn.Contains(database.Type))
+            {
+                var theory = new TheoryDataRow()
+                    .WithSkip($"Provider {database.Type} not in OnlyOn")
+                    .WithTrait("Database", database.Type.ToString());
+                theory.Label = database.Type.ToString();
+                theories.Add(theory);
+                continue;
+            }
+
             if (!database.Enabled)
             {
                 var theory = new TheoryDataRow()
@@ -85,6 +96,11 @@ public override ValueTask<IReadOnlyCollection<ITheoryDataRow>> GetData(MethodInf
 
         foreach (var unconfiguredDatabase in unconfiguredDatabases)
         {
+            if (OnlyOn.Length > 0 && !OnlyOn.Contains(unconfiguredDatabase))
+            {
+                continue;
+            }
+
             var theory = new TheoryDataRow()
                 .WithSkip("Unconfigured")
                 .WithTrait("Database", unconfiguredDatabase.ToString());