Skip to content

Commit 7501e7f

Browse files
SockworkOrangeSockworkOrange
authored
[release] [patch] fix: support Postgres JSON data types without cast and in batch insert (#272)
1 parent f37ede1 commit 7501e7f

28 files changed

Lines changed: 880 additions & 209 deletions

‎Drivers/DbDriver.cs‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -57,23 +57,6 @@ public abstract class DbDriver
5757

5858
protected abstract Dictionary<string, ColumnMapping> ColumnMappings { get; }
5959

60-
protected const string JsonElementTypeHandler = """
61-
private class JsonElementTypeHandler : SqlMapper.TypeHandler<JsonElement>
62-
{
63-
public override JsonElement Parse(object value)
64-
{
65-
if (value is string s)
66-
return JsonDocument.Parse(s).RootElement;
67-
throw new DataException($"Cannot convert {value?.GetType()} to JsonElement");
68-
}
69-
70-
public override void SetValue(IDbDataParameter parameter, JsonElement value)
71-
{
72-
parameter.Value = value.GetRawText();
73-
}
74-
}
75-
""";
76-
7760
protected const string TransformQueryForSliceArgsImpl = """
7861
public static string TransformQueryForSliceArgs(string originalSql, int sliceSize, string paramName)
7962
{

‎Drivers/MySqlConnectorDriver.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,23 @@ public sealed partial class MySqlConnectorDriver(
156156

157157
public override string TransactionClassName => "MySqlTransaction";
158158

159+
private const string JsonElementTypeHandler = """
160+
private class JsonElementTypeHandler : SqlMapper.TypeHandler<JsonElement>
161+
{
162+
public override JsonElement Parse(object value)
163+
{
164+
if (value is string s)
165+
return JsonDocument.Parse(s).RootElement;
166+
throw new DataException($"Cannot convert {value?.GetType()} to JsonElement");
167+
}
168+
169+
public override void SetValue(IDbDataParameter parameter, JsonElement value)
170+
{
171+
parameter.Value = value.GetRawText();
172+
}
173+
}
174+
""";
175+
159176
public MemberDeclarationSyntax OneDeclare(string queryTextConstant, string argInterface,
160177
string returnInterface, Query query)
161178
{

‎Drivers/NpgsqlDriver.cs‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,16 +143,16 @@ public NpgsqlDriver(
143143
["JsonElement"] = new(
144144
new()
145145
{
146-
{ "json", new() },
147-
{ "jsonb", new() }
146+
{ "json", new(NpgsqlTypeOverride: "NpgsqlDbType.Json") },
147+
{ "jsonb", new(NpgsqlTypeOverride: "NpgsqlDbType.Jsonb") }
148148
},
149149
readerFn: ordinal => $"JsonSerializer.Deserialize<JsonElement>(reader.GetString({ordinal}))",
150150
writerFn: (el, notNull, isDapper) =>
151151
{
152152
if (notNull)
153-
return $"{el}.GetRawText()";
153+
return $"{el}";
154154
var nullValue = isDapper ? "null" : "(object)DBNull.Value";
155-
return $"{el}.HasValue ? {el}.Value.GetRawText() : {nullValue}";
155+
return $"{el}.HasValue ? (object) {el}.Value : {nullValue}";
156156
},
157157
usingDirective: "System.Text.Json",
158158
sqlMapper: "SqlMapper.AddTypeHandler(typeof(JsonElement), new JsonElementTypeHandler());",
@@ -345,6 +345,23 @@ public NpgsqlDriver(
345345

346346
public override string TransactionClassName => "NpgsqlTransaction";
347347

348+
private const string JsonElementTypeHandler = """
349+
private class JsonElementTypeHandler : SqlMapper.TypeHandler<JsonElement>
350+
{
351+
public override JsonElement Parse(object value)
352+
{
353+
if (value is string s)
354+
return JsonDocument.Parse(s).RootElement;
355+
throw new DataException($"Cannot convert {value?.GetType()} to JsonElement");
356+
}
357+
358+
public override void SetValue(IDbDataParameter parameter, JsonElement value)
359+
{
360+
parameter.Value = value;
361+
}
362+
}
363+
""";
364+
348365
private const string XmlDocumentTypeHandler =
349366
"""
350367
private class XmlDocumentTypeHandler : SqlMapper.TypeHandler<XmlDocument>

‎docs/04_Postgres.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ we consider support for the different data types separately for batch inserts an
6262
| tsvector | ✅ | ❌ |
6363
| tsquery | ✅ | ❌ |
6464
| uuid | ✅ | ✅ |
65-
| json | ✅ | ⚠️ |
66-
| jsonb | ✅ | ⚠️ |
65+
| json | ✅ | ✅ |
66+
| jsonb | ✅ | ✅ |
6767
| jsonpath | ✅ | ⚠️ |
6868
| xml | ✅ | ⚠️ |
6969
| enum | ✅ | ⚠️ |

‎end2end/EndToEndScaffold/Config.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ public enum KnownTestType
6363
PostgresFloatingPointCopyFrom,
6464
PostgresDateTimeCopyFrom,
6565
PostgresGuidCopyFrom,
66+
PostgresJsonCopyFrom,
6667
PostgresNetworkCopyFrom,
6768
PostgresArrayCopyFrom,
6869
PostgresGeoCopyFrom,
@@ -179,6 +180,7 @@ internal static class Config
179180
KnownTestType.PostgresFloatingPointCopyFrom,
180181
KnownTestType.PostgresDateTimeCopyFrom,
181182
KnownTestType.PostgresGuidCopyFrom,
183+
KnownTestType.PostgresJsonCopyFrom,
182184
KnownTestType.PostgresArrayCopyFrom,
183185
KnownTestType.PostgresNetworkCopyFrom
184186
];

‎end2end/EndToEndScaffold/Templates/PostgresTests.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,53 @@ void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesRow x, QuerySql.GetPos
509509
}
510510
"""
511511
},
512+
[KnownTestType.PostgresJsonCopyFrom] = new TestImpl
513+
{
514+
Impl = $$"""
515+
[Test]
516+
[TestCase(100, "{\"song\": \"Pinball Wizard\", \"album\": \"Tommy\", \"artist\": \"The Who\"}")]
517+
[TestCase(10, null)]
518+
public async Task TestPostgresJsonCopyFrom(int batchSize, string cJson)
519+
{
520+
JsonElement? cParsedJson = null;
521+
if (cJson != null)
522+
cParsedJson = JsonDocument.Parse(cJson).RootElement;
523+
524+
var batchArgs = Enumerable.Range(0, batchSize)
525+
.Select(_ => new QuerySql.InsertPostgresSpecialTypesBatchArgs
526+
{
527+
CJson = cParsedJson,
528+
CJsonb = cParsedJson
529+
})
530+
.ToList();
531+
await QuerySql.InsertPostgresSpecialTypesBatch(batchArgs);
532+
533+
var expected = new QuerySql.GetPostgresSpecialTypesCntRow
534+
{
535+
Cnt = batchSize,
536+
CJson = cParsedJson,
537+
CJsonb = cParsedJson
538+
};
539+
var actual = await QuerySql.GetPostgresSpecialTypesCnt();
540+
AssertSingularEquals(expected, actual{{Consts.UnknownRecordValuePlaceholder}});
541+
542+
void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.GetPostgresSpecialTypesCntRow y)
543+
{
544+
var options = new JsonSerializerOptions
545+
{
546+
WriteIndented = false
547+
};
548+
Assert.That(y.Cnt, Is.EqualTo(x.Cnt));
549+
Assert.That(y.CJson.HasValue, Is.EqualTo(x.CJson.HasValue));
550+
if (y.CJson.HasValue)
551+
Assert.That(JsonSerializer.Serialize(y.CJson.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJson.Value, options)));
552+
Assert.That(y.CJsonb.HasValue, Is.EqualTo(x.CJsonb.HasValue));
553+
if (y.CJsonb.HasValue)
554+
Assert.That(JsonSerializer.Serialize(y.CJsonb.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJsonb.Value, options)));
555+
}
556+
}
557+
"""
558+
},
512559
[KnownTestType.PostgresInvalidJson] = new TestImpl
513560
{
514561
Impl = $$"""

‎end2end/EndToEndTests/NpgsqlDapperTester.generated.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,40 @@ void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.Get
892892
}
893893
}
894894

895+
[Test]
896+
[TestCase(100, "{\"song\": \"Pinball Wizard\", \"album\": \"Tommy\", \"artist\": \"The Who\"}")]
897+
[TestCase(10, null)]
898+
public async Task TestPostgresJsonCopyFrom(int batchSize, string cJson)
899+
{
900+
JsonElement? cParsedJson = null;
901+
if (cJson != null)
902+
cParsedJson = JsonDocument.Parse(cJson).RootElement;
903+
var batchArgs = Enumerable.Range(0, batchSize).Select(_ => new QuerySql.InsertPostgresSpecialTypesBatchArgs { CJson = cParsedJson, CJsonb = cParsedJson }).ToList();
904+
await QuerySql.InsertPostgresSpecialTypesBatch(batchArgs);
905+
var expected = new QuerySql.GetPostgresSpecialTypesCntRow
906+
{
907+
Cnt = batchSize,
908+
CJson = cParsedJson,
909+
CJsonb = cParsedJson
910+
};
911+
var actual = await QuerySql.GetPostgresSpecialTypesCnt();
912+
AssertSingularEquals(expected, actual);
913+
void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.GetPostgresSpecialTypesCntRow y)
914+
{
915+
var options = new JsonSerializerOptions
916+
{
917+
WriteIndented = false
918+
};
919+
Assert.That(y.Cnt, Is.EqualTo(x.Cnt));
920+
Assert.That(y.CJson.HasValue, Is.EqualTo(x.CJson.HasValue));
921+
if (y.CJson.HasValue)
922+
Assert.That(JsonSerializer.Serialize(y.CJson.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJson.Value, options)));
923+
Assert.That(y.CJsonb.HasValue, Is.EqualTo(x.CJsonb.HasValue));
924+
if (y.CJsonb.HasValue)
925+
Assert.That(JsonSerializer.Serialize(y.CJsonb.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJsonb.Value, options)));
926+
}
927+
}
928+
895929
private static IEnumerable<TestCaseData> PostgresNetworkCopyFromTestCases
896930
{
897931
get

‎end2end/EndToEndTests/NpgsqlTester.generated.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,40 @@ void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.Get
892892
}
893893
}
894894

895+
[Test]
896+
[TestCase(100, "{\"song\": \"Pinball Wizard\", \"album\": \"Tommy\", \"artist\": \"The Who\"}")]
897+
[TestCase(10, null)]
898+
public async Task TestPostgresJsonCopyFrom(int batchSize, string cJson)
899+
{
900+
JsonElement? cParsedJson = null;
901+
if (cJson != null)
902+
cParsedJson = JsonDocument.Parse(cJson).RootElement;
903+
var batchArgs = Enumerable.Range(0, batchSize).Select(_ => new QuerySql.InsertPostgresSpecialTypesBatchArgs { CJson = cParsedJson, CJsonb = cParsedJson }).ToList();
904+
await QuerySql.InsertPostgresSpecialTypesBatch(batchArgs);
905+
var expected = new QuerySql.GetPostgresSpecialTypesCntRow
906+
{
907+
Cnt = batchSize,
908+
CJson = cParsedJson,
909+
CJsonb = cParsedJson
910+
};
911+
var actual = await QuerySql.GetPostgresSpecialTypesCnt();
912+
AssertSingularEquals(expected, actual.Value);
913+
void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.GetPostgresSpecialTypesCntRow y)
914+
{
915+
var options = new JsonSerializerOptions
916+
{
917+
WriteIndented = false
918+
};
919+
Assert.That(y.Cnt, Is.EqualTo(x.Cnt));
920+
Assert.That(y.CJson.HasValue, Is.EqualTo(x.CJson.HasValue));
921+
if (y.CJson.HasValue)
922+
Assert.That(JsonSerializer.Serialize(y.CJson.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJson.Value, options)));
923+
Assert.That(y.CJsonb.HasValue, Is.EqualTo(x.CJsonb.HasValue));
924+
if (y.CJsonb.HasValue)
925+
Assert.That(JsonSerializer.Serialize(y.CJsonb.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJsonb.Value, options)));
926+
}
927+
}
928+
895929
private static IEnumerable<TestCaseData> PostgresNetworkCopyFromTestCases
896930
{
897931
get

‎end2end/EndToEndTestsLegacy/NpgsqlDapperTester.generated.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,40 @@ void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.Get
892892
}
893893
}
894894

895+
[Test]
896+
[TestCase(100, "{\"song\": \"Pinball Wizard\", \"album\": \"Tommy\", \"artist\": \"The Who\"}")]
897+
[TestCase(10, null)]
898+
public async Task TestPostgresJsonCopyFrom(int batchSize, string cJson)
899+
{
900+
JsonElement? cParsedJson = null;
901+
if (cJson != null)
902+
cParsedJson = JsonDocument.Parse(cJson).RootElement;
903+
var batchArgs = Enumerable.Range(0, batchSize).Select(_ => new QuerySql.InsertPostgresSpecialTypesBatchArgs { CJson = cParsedJson, CJsonb = cParsedJson }).ToList();
904+
await QuerySql.InsertPostgresSpecialTypesBatch(batchArgs);
905+
var expected = new QuerySql.GetPostgresSpecialTypesCntRow
906+
{
907+
Cnt = batchSize,
908+
CJson = cParsedJson,
909+
CJsonb = cParsedJson
910+
};
911+
var actual = await QuerySql.GetPostgresSpecialTypesCnt();
912+
AssertSingularEquals(expected, actual);
913+
void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.GetPostgresSpecialTypesCntRow y)
914+
{
915+
var options = new JsonSerializerOptions
916+
{
917+
WriteIndented = false
918+
};
919+
Assert.That(y.Cnt, Is.EqualTo(x.Cnt));
920+
Assert.That(y.CJson.HasValue, Is.EqualTo(x.CJson.HasValue));
921+
if (y.CJson.HasValue)
922+
Assert.That(JsonSerializer.Serialize(y.CJson.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJson.Value, options)));
923+
Assert.That(y.CJsonb.HasValue, Is.EqualTo(x.CJsonb.HasValue));
924+
if (y.CJsonb.HasValue)
925+
Assert.That(JsonSerializer.Serialize(y.CJsonb.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJsonb.Value, options)));
926+
}
927+
}
928+
895929
private static IEnumerable<TestCaseData> PostgresNetworkCopyFromTestCases
896930
{
897931
get

‎end2end/EndToEndTestsLegacy/NpgsqlTester.generated.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,40 @@ void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.Get
892892
}
893893
}
894894

895+
[Test]
896+
[TestCase(100, "{\"song\": \"Pinball Wizard\", \"album\": \"Tommy\", \"artist\": \"The Who\"}")]
897+
[TestCase(10, null)]
898+
public async Task TestPostgresJsonCopyFrom(int batchSize, string cJson)
899+
{
900+
JsonElement? cParsedJson = null;
901+
if (cJson != null)
902+
cParsedJson = JsonDocument.Parse(cJson).RootElement;
903+
var batchArgs = Enumerable.Range(0, batchSize).Select(_ => new QuerySql.InsertPostgresSpecialTypesBatchArgs { CJson = cParsedJson, CJsonb = cParsedJson }).ToList();
904+
await QuerySql.InsertPostgresSpecialTypesBatch(batchArgs);
905+
var expected = new QuerySql.GetPostgresSpecialTypesCntRow
906+
{
907+
Cnt = batchSize,
908+
CJson = cParsedJson,
909+
CJsonb = cParsedJson
910+
};
911+
var actual = await QuerySql.GetPostgresSpecialTypesCnt();
912+
AssertSingularEquals(expected, actual);
913+
void AssertSingularEquals(QuerySql.GetPostgresSpecialTypesCntRow x, QuerySql.GetPostgresSpecialTypesCntRow y)
914+
{
915+
var options = new JsonSerializerOptions
916+
{
917+
WriteIndented = false
918+
};
919+
Assert.That(y.Cnt, Is.EqualTo(x.Cnt));
920+
Assert.That(y.CJson.HasValue, Is.EqualTo(x.CJson.HasValue));
921+
if (y.CJson.HasValue)
922+
Assert.That(JsonSerializer.Serialize(y.CJson.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJson.Value, options)));
923+
Assert.That(y.CJsonb.HasValue, Is.EqualTo(x.CJsonb.HasValue));
924+
if (y.CJsonb.HasValue)
925+
Assert.That(JsonSerializer.Serialize(y.CJsonb.Value, options), Is.EqualTo(JsonSerializer.Serialize(x.CJsonb.Value, options)));
926+
}
927+
}
928+
895929
private static IEnumerable<TestCaseData> PostgresNetworkCopyFromTestCases
896930
{
897931
get

0 commit comments

Comments
 (0)