Skip to content

Commit 2f66be6

Browse files
johnsimonsjohnsimons
authored
Merge pull request #5625 from Particular/john/ingestion_part2
Implement Known Endpoints persistence with EF Core insert-only tables
2 parents 5939ee8 + ffadf36 commit 2f66be6

44 files changed

Lines changed: 1531 additions & 79 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/Directory.Packages.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<PackageVersion Include="ByteSize" Version="2.1.2" />
1313
<PackageVersion Include="Caliburn.Micro" Version="5.0.258" />
1414
<PackageVersion Include="DnsClient" Version="1.8.0" />
15+
<PackageVersion Include="EFCore.NamingConventions" Version="10.0.1" />
1516
<PackageVersion Include="FluentValidation" Version="12.1.1" />
1617
<PackageVersion Include="Fody" Version="6.9.3" />
1718
<PackageVersion Include="GitHubActionsTestLogger" Version="3.0.4" />

‎src/ServiceControl.Persistence.EFCore.PostgreSql/Infrastructure/KnownEndpointsReconciler.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
namespace ServiceControl.Persistence.EFCore.PostgreSql.Infrastructure;
2+
3+
using Microsoft.EntityFrameworkCore;
4+
using Microsoft.Extensions.DependencyInjection;
5+
using Microsoft.Extensions.Logging;
6+
using ServiceControl.Persistence.EFCore.DbContexts;
7+
using ServiceControl.Persistence.EFCore.Infrastructure;
8+
9+
class KnownEndpointsReconciler(
10+
ILogger<KnownEndpointsReconciler> logger,
11+
TimeProvider timeProvider,
12+
IServiceScopeFactory serviceScopeFactory)
13+
: InsertOnlyTableReconciler(
14+
logger, timeProvider, serviceScopeFactory, nameof(KnownEndpointsReconciler))
15+
{
16+
protected override Task<int> ReconcileBatch(ServiceControlDbContext dbContext, CancellationToken stoppingToken) =>
17+
ReconcileBatch(dbContext, BatchSize, stoppingToken);
18+
19+
// Static so tests can execute a batch deterministically without the background service's timer loop.
20+
// Must be called within an active transaction because of the pg_try_advisory_xact_lock.
21+
internal static async Task<int> ReconcileBatch(ServiceControlDbContext dbContext, int batchSize, CancellationToken cancellationToken)
22+
{
23+
var sql = """
24+
WITH lock_check AS (
25+
SELECT pg_try_advisory_xact_lock(hashtext('known_endpoints_sync')) AS acquired
26+
),
27+
batch AS (
28+
SELECT ctid FROM "known_endpoints_insert_only"
29+
WHERE (SELECT acquired FROM lock_check)
30+
LIMIT @batchSize
31+
),
32+
ins AS (
33+
INSERT INTO "known_endpoints" ("id", "name", "host_id", "host", "monitored")
34+
SELECT DISTINCT ON ("known_endpoint_id") "known_endpoint_id", "name", "host_id", "host", FALSE
35+
FROM "known_endpoints_insert_only"
36+
WHERE ctid IN (SELECT ctid FROM batch)
37+
ON CONFLICT ("id") DO NOTHING
38+
)
39+
DELETE FROM "known_endpoints_insert_only"
40+
WHERE ctid IN (SELECT ctid FROM batch);
41+
""";
42+
43+
var rowsAffected = await dbContext.Database.ExecuteSqlRawAsync(sql, [new Npgsql.NpgsqlParameter("@batchSize", batchSize)], cancellationToken);
44+
return rowsAffected;
45+
}
46+
}

‎src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.Designer.cs‎

Lines changed: 93 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.cs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
using System;
2+
using Microsoft.EntityFrameworkCore.Migrations;
3+
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
4+
5+
#nullable disable
6+
7+
namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations
8+
{
9+
/// <inheritdoc />
10+
public partial class Initial : Migration
11+
{
12+
/// <inheritdoc />
13+
protected override void Up(MigrationBuilder migrationBuilder)
14+
{
15+
migrationBuilder.CreateTable(
16+
name: "known_endpoints",
17+
columns: table => new
18+
{
19+
id = table.Column<Guid>(type: "uuid", nullable: false),
20+
name = table.Column<string>(type: "text", nullable: false),
21+
host_id = table.Column<Guid>(type: "uuid", nullable: false),
22+
host = table.Column<string>(type: "text", nullable: false),
23+
monitored = table.Column<bool>(type: "boolean", nullable: false)
24+
},
25+
constraints: table =>
26+
{
27+
table.PrimaryKey("PK_known_endpoints", x => x.id);
28+
});
29+
30+
migrationBuilder.CreateTable(
31+
name: "known_endpoints_insert_only",
32+
columns: table => new
33+
{
34+
id = table.Column<long>(type: "bigint", nullable: false)
35+
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
36+
known_endpoint_id = table.Column<Guid>(type: "uuid", nullable: false),
37+
name = table.Column<string>(type: "text", nullable: false),
38+
host_id = table.Column<Guid>(type: "uuid", nullable: false),
39+
host = table.Column<string>(type: "text", nullable: false)
40+
},
41+
constraints: table =>
42+
{
43+
table.PrimaryKey("PK_known_endpoints_insert_only", x => x.id);
44+
});
45+
46+
migrationBuilder.CreateIndex(
47+
name: "IX_known_endpoints_insert_only_known_endpoint_id",
48+
table: "known_endpoints_insert_only",
49+
column: "known_endpoint_id");
50+
}
51+
52+
/// <inheritdoc />
53+
protected override void Down(MigrationBuilder migrationBuilder)
54+
{
55+
migrationBuilder.DropTable(
56+
name: "known_endpoints");
57+
58+
migrationBuilder.DropTable(
59+
name: "known_endpoints_insert_only");
60+
}
61+
}
62+
}

‎src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
// <auto-generated />
2+
using System;
3+
using Microsoft.EntityFrameworkCore;
4+
using Microsoft.EntityFrameworkCore.Infrastructure;
5+
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
6+
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
7+
using ServiceControl.Persistence.EFCore.PostgreSql;
8+
9+
#nullable disable
10+
11+
namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations
12+
{
13+
[DbContext(typeof(PostgreSqlServiceControlDbContext))]
14+
partial class PostgreSqlServiceControlDbContextModelSnapshot : ModelSnapshot
15+
{
16+
protected override void BuildModel(ModelBuilder modelBuilder)
17+
{
18+
#pragma warning disable 612, 618
19+
modelBuilder
20+
.HasAnnotation("ProductVersion", "10.0.9")
21+
.HasAnnotation("Relational:MaxIdentifierLength", 63);
22+
23+
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
24+
25+
modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b =>
26+
{
27+
b.Property<Guid>("Id")
28+
.HasColumnType("uuid")
29+
.HasColumnName("id");
30+
31+
b.Property<string>("Host")
32+
.IsRequired()
33+
.HasColumnType("text")
34+
.HasColumnName("host");
35+
36+
b.Property<Guid>("HostId")
37+
.HasColumnType("uuid")
38+
.HasColumnName("host_id");
39+
40+
b.Property<bool>("Monitored")
41+
.HasColumnType("boolean")
42+
.HasColumnName("monitored");
43+
44+
b.Property<string>("Name")
45+
.IsRequired()
46+
.HasColumnType("text")
47+
.HasColumnName("name");
48+
49+
b.HasKey("Id");
50+
51+
b.ToTable("known_endpoints", (string)null);
52+
});
53+
54+
modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointInsertOnlyEntity", b =>
55+
{
56+
b.Property<long>("Id")
57+
.ValueGeneratedOnAdd()
58+
.HasColumnType("bigint")
59+
.HasColumnName("id");
60+
61+
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
62+
63+
b.Property<string>("Host")
64+
.IsRequired()
65+
.HasColumnType("text")
66+
.HasColumnName("host");
67+
68+
b.Property<Guid>("HostId")
69+
.HasColumnType("uuid")
70+
.HasColumnName("host_id");
71+
72+
b.Property<Guid>("KnownEndpointId")
73+
.HasColumnType("uuid")
74+
.HasColumnName("known_endpoint_id");
75+
76+
b.Property<string>("Name")
77+
.IsRequired()
78+
.HasColumnType("text")
79+
.HasColumnName("name");
80+
81+
b.HasKey("Id");
82+
83+
b.HasIndex("KnownEndpointId");
84+
85+
b.ToTable("known_endpoints_insert_only", (string)null);
86+
});
87+
#pragma warning restore 612, 618
88+
}
89+
}
90+
}

‎src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql;
44
using Microsoft.Extensions.DependencyInjection;
55
using ServiceControl.Persistence.EFCore.Abstractions;
66
using ServiceControl.Persistence.EFCore.DbContexts;
7+
using ServiceControl.Persistence.EFCore.PostgreSql.Infrastructure;
78

89
class PostgreSqlPersistence(PostgreSqlPersisterSettings settings) : BasePersistence, IPersistence
910
{
@@ -12,6 +13,8 @@ public void AddPersistence(IServiceCollection services)
1213
RegisterSettings(services);
1314
ConfigureDbContext(services);
1415
RegisterDataStores(services);
16+
17+
services.AddHostedService<KnownEndpointsReconciler>();
1518
}
1619

1720
public void AddInstaller(IServiceCollection services)

‎src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlServiceControlDbContext.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,11 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql;
55

66
public class PostgreSqlServiceControlDbContext(DbContextOptions<PostgreSqlServiceControlDbContext> options) : ServiceControlDbContext(options)
77
{
8+
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
9+
{
10+
base.OnConfiguring(optionsBuilder);
11+
12+
// Use snake_case naming convention for PostgreSQL
13+
optionsBuilder.UseSnakeCaseNamingConvention();
14+
}
815
}

‎src/ServiceControl.Persistence.EFCore.PostgreSql/ServiceControl.Persistence.EFCore.PostgreSql.csproj‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,29 @@
88
<DisableTransitiveProjectReferences>true</DisableTransitiveProjectReferences>
99
</PropertyGroup>
1010

11-
<ItemGroup>
12-
<!-- Private=false & ExcludeAssets=runtime prevent repeatedly including binary dependencies of ServiceControl.Persistence and its dependencies in each persister directory -->
11+
<ItemGroup Condition="'$(Configuration)' == 'Release'">
12+
<!-- Private=false & ExcludeAssets=runtime prevent repeatedly including binary dependencies of ServiceControl.Persistence
13+
and its dependencies in each persister directory. Release-only: the host loads these via PluginAssemblyLoadContext,
14+
which falls back to the host's own copy. Debug needs them copied locally for dotnet-ef design-time tooling. -->
1315
<ProjectReference Include="../ServiceControl.Configuration/ServiceControl.Configuration.csproj" Private="false" ExcludeAssets="runtime" />
1416
<ProjectReference Include="../ServiceControl.Infrastructure/ServiceControl.Infrastructure.csproj" Private="false" ExcludeAssets="runtime" />
1517
<ProjectReference Include="../ServiceControl.Persistence/ServiceControl.Persistence.csproj" Private="false" ExcludeAssets="runtime" />
1618
</ItemGroup>
1719

20+
<ItemGroup Condition="'$(Configuration)' != 'Release'">
21+
<ProjectReference Include="../ServiceControl.Configuration/ServiceControl.Configuration.csproj" />
22+
<ProjectReference Include="../ServiceControl.Infrastructure/ServiceControl.Infrastructure.csproj" />
23+
<ProjectReference Include="../ServiceControl.Persistence/ServiceControl.Persistence.csproj" />
24+
</ItemGroup>
25+
1826
<ItemGroup>
1927
<!-- Not Private=false or ExcludeAssets because the shared EF Core assembly ships inside this persister's folder -->
2028
<ProjectReference Include="../ServiceControl.Persistence.EFCore/ServiceControl.Persistence.EFCore.csproj" />
2129
</ItemGroup>
2230

2331
<ItemGroup>
2432
<!-- Design-time only, for dotnet-ef tooling. Debug-only so it (and its Roslyn dependency tail) stays out of the shipped Release artifact -->
33+
<PackageReference Include="EFCore.NamingConventions" />
2534
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" PrivateAssets="all" Condition="'$(Configuration)' == 'Debug'" />
2635
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" />
2736
</ItemGroup>

0 commit comments

Comments
 (0)