Automate tenant provisioning and lifecycle management

Introduced a robust workflow for automating tenant provisioning, activation, and health verification. Added `TenantProvisioning` and `TenantProvisioningStep` entities to track provisioning status and steps. Integrated Hangfire for background job processing with inline fallback for development.

Enhanced multitenancy support with `Finbuckle.MultiTenant`, updated `AppTenantInfo` for immutability, and added validation for subscription backdating. Added endpoints for retrieving and retrying provisioning status.

Improved health checks, logging, and auditing with per-tenant context. Refactored `TenantService` to delegate provisioning logic to `TenantProvisioningService`. Updated database migrations and configurations for streamlined provisioning.

Upgraded dependencies, cleaned up outdated migrations, and added development-friendly features like auto-provisioning on startup.

Author: iammukeshm

docs/stories/tenant-lifecycle-automation.md

@@ -79,3 +79,10 @@ Goal: automate tenant provisioning, activation, and health verification so new t
 - Double-submit provisioning for same tenant does not run concurrent workflows (dedupe/lock).
 - Partial seeds are safe to re-run (no duplicate roles/users; admin user upsert).
 - Health check reports degraded for tenants with failed provisioning; improves after successful retry.
+
+## Progress Update (Current State)
+- Provisioning workflow implemented with persisted status/steps and 202 responses on tenant creation; retry endpoint available.
+- Background provisioning via Hangfire, with inline fallback when Hangfire/storage is unavailable (dev-friendly).
+- Startup hosted services: tenant catalog migrate/seed (root tenant) and optional auto-provision enqueue.
+- Provider-aware TenantDbContextFactory to select PostgreSQL via appsettings.
+- Audit pipeline fixed to stamp tenant/user on events; audit sink writes per-tenant batches.

src/BuildingBlocks/Jobs/Extensions.cs

@@ -13,6 +13,9 @@ public static class Extensions
 {
     public static IServiceCollection AddHeroJobs(this IServiceCollection services)
     {
+        services.AddOptions<HangfireOptions>()
+            .BindConfiguration(nameof(HangfireOptions));
+
         services.AddHangfireServer(options =>
         {
             options.HeartbeatInterval = TimeSpan.FromSeconds(30);
@@ -23,10 +26,9 @@ public static IServiceCollection AddHeroJobs(this IServiceCollection services)
 
         services.AddHangfire((provider, config) =>
         {
-            var dbOptions = provider
-                .GetRequiredService<IConfiguration>()
-                .GetSection(nameof(DatabaseOptions))
-                .Get<DatabaseOptions>() ?? throw new CustomException("Database options not found");
+            var configuration = provider.GetRequiredService<IConfiguration>();
+            var dbOptions = configuration.GetSection(nameof(DatabaseOptions)).Get<DatabaseOptions>()
+                ?? throw new CustomException("Database options not found");
 
             switch (dbOptions.Provider.ToUpperInvariant())
             {

src/BuildingBlocks/Jobs/FshJobActivator.cs

@@ -38,10 +38,7 @@ private void ReceiveParameters()
             if (tenantInfo is not null)
             {
                 _scope.ServiceProvider.GetRequiredService<IMultiTenantContextSetter>()
-                    .MultiTenantContext = new MultiTenantContext<AppTenantInfo>
-                    {
-                        TenantInfo = tenantInfo
-                    };
+                    .MultiTenantContext = new MultiTenantContext<AppTenantInfo>(tenantInfo);
             }
 
             string userId = _context.GetJobParameter<string>(QueryStringKeys.UserId);
@@ -60,4 +57,4 @@ public override object Resolve(Type type) =>
                 ? _context
                 : _scope.ServiceProvider.GetService(serviceType);
     }
-}
+}

src/BuildingBlocks/Jobs/HangfireOptions.cs

@@ -1,8 +1,8 @@
-namespace FSH.Framework.Jobs;
+namespace FSH.Framework.Jobs;
 
 public class HangfireOptions
 {
     public string UserName { get; set; } = "admin";
     public string Password { get; set; } = "Secure1234!Me";
     public string Route { get; set; } = "/jobs";
-}
+}

src/BuildingBlocks/Jobs/LogJobFilter.cs

@@ -19,7 +19,7 @@ public void OnCreating(CreatingContext context)
         var job = context.Job;
         var jobName = GetJobName(job);
 
-        Logger.InfoFormat(
+        Logger.DebugFormat(
             "Creating job for {0}.", jobName);
     }
 
@@ -30,7 +30,7 @@ public void OnCreated(CreatedContext context)
         var jobId = context.BackgroundJob?.Id ?? "<unknown>";
         var recurringJobId = context.Parameters.TryGetValue("RecurringJobId", out var r) ? r : null;
 
-        Logger.InfoFormat(
+        Logger.DebugFormat(
             "Job created: Id={0}, Name={1}, RecurringJobId={2}",
             jobId,
             jobName,
@@ -45,7 +45,7 @@ public void OnPerforming(PerformingContext context)
         var recurringJobId = context.GetJobParameter<string>("RecurringJobId") ?? "<none>";
         var args = FormatArguments(job.Args);
 
-        Logger.InfoFormat(
+        Logger.DebugFormat(
             "Starting job: Id={0}, Name={1}, RecurringJobId={2}, Queue={3}, Args={4}",
             backgroundJob.Id,
             jobName,
@@ -60,7 +60,7 @@ public void OnPerformed(PerformedContext context)
         var job = backgroundJob.Job;
         var jobName = GetJobName(job);
 
-        Logger.InfoFormat(
+        Logger.DebugFormat(
             "Job completed: Id={0}, Name={1}, Succeeded={2}",
             backgroundJob.Id,
             jobName,
@@ -81,7 +81,7 @@ public void OnStateElection(ElectStateContext context)
 
     public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
     {
-        Logger.InfoFormat(
+        Logger.DebugFormat(
             "Job state changed: Id={0}, Name={1}, OldState={2}, NewState={3}",
             context.BackgroundJob.Id,
             GetJobName(context.BackgroundJob.Job),
@@ -91,7 +91,7 @@ public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction tran
 
     public void OnStateUnapplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
     {
-        Logger.InfoFormat(
+        Logger.DebugFormat(
             "Job state unapplied: Id={0}, Name={1}, OldState={2}",
             context.BackgroundJob.Id,
             GetJobName(context.BackgroundJob.Job),

src/BuildingBlocks/Shared/Multitenancy/AppTenantInfo.cs

@@ -1,18 +1,18 @@
-using Finbuckle.MultiTenant.Abstractions;
+using Finbuckle.MultiTenant.Abstractions;
 
 namespace FSH.Framework.Shared.Multitenancy;
 
-public class AppTenantInfo : ITenantInfo, IAppTenantInfo
+public record AppTenantInfo(string Id, string Identifier, string? Name = null)
+    : TenantInfo(Id, Identifier, Name), IAppTenantInfo
 {
-    public AppTenantInfo()
+    // Parameterless constructor for tooling/EF.
+    public AppTenantInfo() : this(string.Empty, string.Empty)
     {
     }
 
     public AppTenantInfo(string id, string name, string? connectionString, string adminEmail, string? issuer = null)
+        : this(id, id, name)
     {
-        Id = id;
-        Identifier = id;
-        Name = name;
         ConnectionString = connectionString ?? string.Empty;
         AdminEmail = adminEmail;
         IsActive = true;
@@ -21,12 +21,8 @@ public AppTenantInfo(string id, string name, string? connectionString, string ad
         // Add Default 1 Month Validity for all new tenants. Something like a DEMO period for tenants.
         ValidUpto = DateTime.UtcNow.AddMonths(1);
     }
-    public string Id { get; set; } = default!;
-    public string Identifier { get; set; } = default!;
-
-    public string Name { get; set; } = default!;
-    public string ConnectionString { get; set; } = default!;
 
+    public string ConnectionString { get; set; } = string.Empty;
     public string AdminEmail { get; set; } = default!;
     public bool IsActive { get; set; }
     public DateTime ValidUpto { get; set; }
@@ -35,10 +31,13 @@ public AppTenantInfo(string id, string name, string? connectionString, string ad
     public void AddValidity(int months) =>
         ValidUpto = ValidUpto.AddMonths(months);
 
-    public void SetValidity(in DateTime validTill) =>
-        ValidUpto = ValidUpto < validTill
-            ? validTill
+    public void SetValidity(in DateTime validTill)
+    {
+        var normalized = validTill;
+        ValidUpto = ValidUpto < normalized
+            ? normalized
             : throw new InvalidOperationException("Subscription cannot be backdated.");
+    }
 
     public void Activate()
     {
@@ -59,8 +58,10 @@ public void Deactivate()
 
         IsActive = false;
     }
-    string? ITenantInfo.Id { get => Id; set => Id = value ?? throw new InvalidOperationException("Id can't be null."); }
-    string? ITenantInfo.Identifier { get => Identifier; set => Identifier = value ?? throw new InvalidOperationException("Identifier can't be null."); }
-    string? ITenantInfo.Name { get => Name; set => Name = value ?? throw new InvalidOperationException("Name can't be null."); }
-    string? IAppTenantInfo.ConnectionString { get => ConnectionString; set => ConnectionString = value ?? throw new InvalidOperationException("ConnectionString can't be null."); }
-}
+
+    string? IAppTenantInfo.ConnectionString
+    {
+        get => ConnectionString;
+        set => ConnectionString = value ?? throw new InvalidOperationException("ConnectionString can't be null.");
+    }
+}

src/BuildingBlocks/Shared/Multitenancy/MultitenancyConstants.cs

@@ -13,6 +13,7 @@ public static class Root
 
     public const string DefaultPassword = "123Pa$$word!";
     public const string Identifier = "tenant";
+    public const string Schema = "tenant";
 
     public static class Permissions
     {

src/BuildingBlocks/Shared/Shared.csproj

@@ -5,9 +5,10 @@
 		<AssemblyName>FSH.Framework.Shared</AssemblyName>
 	</PropertyGroup>
 	<ItemGroup>
-		<FrameworkReference Include="Microsoft.AspNetCore.App" />
+	  <FrameworkReference Include="Microsoft.AspNetCore.App" />
 	</ItemGroup>
 	<ItemGroup>
+	  <PackageReference Include="Finbuckle.MultiTenant.Abstractions" />
 	  <PackageReference Include="Finbuckle.MultiTenant" />
 	</ItemGroup>
 

src/Directory.Packages.props

@@ -9,15 +9,15 @@
     <IsPackable>true</IsPackable>
   </PropertyGroup>
   <ItemGroup Label="Aspire">
-    <PackageVersion Include="Aspire.Hosting.PostgreSQL" Version="13.0.0" />
-    <PackageVersion Include="Aspire.Hosting.Redis" Version="13.0.0" />
-    <PackageVersion Include="FluentValidation.DependencyInjectionExtensions" Version="12.1.0" />
+    <PackageVersion Include="Aspire.Hosting.PostgreSQL" Version="13.0.1" />
+    <PackageVersion Include="Aspire.Hosting.Redis" Version="13.0.1" />
+    <PackageVersion Include="FluentValidation.DependencyInjectionExtensions" Version="12.1.1" />
     <PackageVersion Include="Microsoft.AspNetCore.Components.Authorization" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="10.0.0" />
-    <PackageVersion Include="MudBlazor" Version="8.14.0" />
+    <PackageVersion Include="MudBlazor" Version="8.15.0" />
     <PackageVersion Include="MudBlazor.ThemeManager" Version="3.0.0" />
-    <PackageVersion Include="Npgsql.OpenTelemetry" Version="10.0.0-rc.1" />
+    <PackageVersion Include="Npgsql.OpenTelemetry" Version="10.0.0" />
     <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.14.0" />
     <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.14.0" />
     <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.14.0" />
@@ -36,10 +36,12 @@
     <PackageVersion Include="Asp.Versioning.Http" Version="8.1.0" />
     <PackageVersion Include="Asp.Versioning.Mvc" Version="8.1.0" />
     <PackageVersion Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
-    <PackageVersion Include="Finbuckle.MultiTenant" Version="9.4.2" />
-    <PackageVersion Include="Finbuckle.MultiTenant.AspNetCore" Version="9.4.2" />
-    <PackageVersion Include="Finbuckle.MultiTenant.EntityFrameworkCore" Version="9.4.2" />
-    <PackageVersion Include="FluentValidation" Version="12.1.0" />
+    <PackageVersion Include="Finbuckle.MultiTenant" Version="10.0.0" />
+    <PackageVersion Include="Finbuckle.MultiTenant.AspNetCore" Version="10.0.0" />
+    <PackageVersion Include="Finbuckle.MultiTenant.EntityFrameworkCore" Version="10.0.0" />
+    <PackageVersion Include="Finbuckle.MultiTenant.Abstractions" Version="10.0.0" />
+    <PackageVersion Include="Finbuckle.MultiTenant.Identity.EntityFrameworkCore" Version="10.0.0" />
+    <PackageVersion Include="FluentValidation" Version="12.1.1" />
     <PackageVersion Include="Hangfire" Version="1.8.22" />
     <PackageVersion Include="Hangfire.Core" Version="1.8.22" />
     <PackageVersion Include="Hangfire.InMemory" Version="1.0.0" />
@@ -74,12 +76,12 @@
     <PackageVersion Include="MimeKit" Version="4.14.0" />
     <PackageVersion Include="NetArchTest.Rules" Version="1.3.2" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
-    <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.0-rc.2" />
-    <PackageVersion Include="Scalar.AspNetCore" Version="2.10.3" />
+    <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.0" />
+    <PackageVersion Include="Scalar.AspNetCore" Version="2.11.0" />
     <PackageVersion Include="Serilog" Version="4.3.1-dev-02395" />
-    <PackageVersion Include="Serilog.AspNetCore" Version="9.0.0" />
+    <PackageVersion Include="Serilog.AspNetCore" Version="10.0.0" />
     <PackageVersion Include="Serilog.Enrichers.CorrelationId" Version="3.0.1" />
-    <PackageVersion Include="SonarAnalyzer.CSharp" Version="10.15.0.120848" />
+    <PackageVersion Include="SonarAnalyzer.CSharp" Version="10.16.1.129956" />
     <PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="10.0.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
     <PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.14.0" />

src/Modules/Auditing/Modules.Auditing/AuditingModule.cs

@@ -40,7 +40,7 @@ public void ConfigureServices(IHostApplicationBuilder builder)
         // Enrichers used by Audit.Configure (scoped, run on request thread)
         builder.Services.AddScoped<IAuditMaskingService, JsonMaskingService>();
         builder.Services.AddHostedService<AuditingConfigurator>();
-        builder.Services.AddSingleton<IAuditScope, HttpAuditScope>();
+        builder.Services.AddScoped<IAuditScope, HttpAuditScope>();
 
         builder.Services.AddSingleton<ChannelAuditPublisher>();
         builder.Services.AddSingleton<IAuditPublisher>(sp => sp.GetRequiredService<ChannelAuditPublisher>());