|
| 1 | +# AppenderMap-based Type-Safe Appender |
| 2 | + |
| 3 | +This implementation provides a type-safe way to append data to DuckDB tables using AppenderMap-based mappings with automatic type validation. |
| 4 | + |
| 5 | +## Problem Solved |
| 6 | + |
| 7 | +The original issue was that users could accidentally append values with mismatched types (e.g., `decimal` to `REAL` column), causing silent data corruption. The AppenderMap approach validates types against actual column types from the database. |
| 8 | + |
| 9 | +## How It Works |
| 10 | + |
| 11 | +### 1. Define an AppenderMap |
| 12 | + |
| 13 | +Create an AppenderMap that defines property mappings in column order: |
| 14 | + |
| 15 | +```csharp |
| 16 | +public class PersonMap : DuckDBAppenderMap<Person> |
| 17 | +{ |
| 18 | + public PersonMap() |
| 19 | + { |
| 20 | + Map(p => p.Id); // Column 0: INTEGER |
| 21 | + Map(p => p.Name); // Column 1: VARCHAR |
| 22 | + Map(p => p.Height); // Column 2: REAL |
| 23 | + Map(p => p.BirthDate); // Column 3: TIMESTAMP |
| 24 | + } |
| 25 | +} |
| 26 | +``` |
| 27 | + |
| 28 | +### 2. Use Type-Safe Appender |
| 29 | + |
| 30 | +```csharp |
| 31 | +// Create table |
| 32 | +connection.ExecuteNonQuery( |
| 33 | + "CREATE TABLE person(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP)"); |
| 34 | + |
| 35 | +// Create data |
| 36 | +var people = new[] |
| 37 | +{ |
| 38 | + new Person { Id = 1, Name = "Alice", Height = 1.65f, BirthDate = new DateTime(1990, 1, 15) }, |
| 39 | + new Person { Id = 2, Name = "Bob", Height = 1.80f, BirthDate = new DateTime(1985, 5, 20) }, |
| 40 | +}; |
| 41 | + |
| 42 | +// Use mapped appender - type validation happens at creation |
| 43 | +using (var appender = connection.CreateAppender<Person, PersonMap>("person")) |
| 44 | +{ |
| 45 | + appender.AppendRecords(people); |
| 46 | +} |
| 47 | +``` |
| 48 | + |
| 49 | +## Benefits |
| 50 | + |
| 51 | +### 1. **Type Validation Against Database Schema** |
| 52 | +The mapped appender retrieves actual column types from the database and validates that your .NET types match: |
| 53 | +- `int` → `INTEGER` ✅ |
| 54 | +- `float` → `REAL` ✅ |
| 55 | +- `decimal` → `REAL` ❌ Throws exception at creation! |
| 56 | + |
| 57 | +### 2. **No Performance Overhead** |
| 58 | +- Type validation happens once when creating the appender |
| 59 | +- Uses the same fast data chunk API as the low-level appender |
| 60 | +- No per-value type checks during append operations |
| 61 | + |
| 62 | +### 3. **Support for Default and Null Values** |
| 63 | +```csharp |
| 64 | +public class MyMap : DuckDBAppenderMap<MyData> |
| 65 | +{ |
| 66 | + public MyMap() |
| 67 | + { |
| 68 | + Map(d => d.Id); |
| 69 | + Map(d => d.Name); |
| 70 | + DefaultValue(); // Use column's default value |
| 71 | + NullValue(); // Insert NULL |
| 72 | + } |
| 73 | +} |
| 74 | +``` |
| 75 | + |
| 76 | +### 4. **Backward Compatible** |
| 77 | +The original fast, low-level `CreateAppender()` API remains unchanged: |
| 78 | +```csharp |
| 79 | +// Still available for maximum performance |
| 80 | +using var appender = connection.CreateAppender("myTable"); |
| 81 | +appender.CreateRow() |
| 82 | + .AppendValue((float?)1.5) |
| 83 | + .EndRow(); |
| 84 | +``` |
| 85 | + |
| 86 | +## Example: Preventing the Original Issue |
| 87 | + |
| 88 | +### ❌ Before (Silent Corruption) |
| 89 | +```csharp |
| 90 | +public class MyData |
| 91 | +{ |
| 92 | + public decimal Value { get; set; } // Oops! decimal is 16 bytes |
| 93 | +} |
| 94 | + |
| 95 | +// This would silently corrupt data |
| 96 | +using var appender = connection.CreateAppender("myTable"); // REAL column |
| 97 | +appender.CreateRow() |
| 98 | + .AppendValue(data.Value) // decimal to REAL - CORRUPTION! |
| 99 | + .EndRow(); |
| 100 | +``` |
| 101 | + |
| 102 | +### ✅ After (Type Safety with Validation) |
| 103 | +```csharp |
| 104 | +public class MyData |
| 105 | +{ |
| 106 | + public float Value { get; set; } // Correct type! |
| 107 | +} |
| 108 | + |
| 109 | +public class MyDataMap : DuckDBClassMap<MyData> |
| 110 | +{ |
| 111 | + public MyDataMap() |
| 112 | + { |
| 113 | + Map(x => x.Value); // Validated: float → REAL ✅ |
| 114 | + } |
| 115 | +} |
| 116 | + |
| 117 | +// Type mismatch detected at appender creation |
| 118 | +using var appender = connection.CreateAppender<MyData, MyDataMap>("myTable"); |
| 119 | +appender.AppendRecords(dataList); // Safe! |
| 120 | +``` |
| 121 | + |
| 122 | +If you tried to use a `decimal` property with a `REAL` column: |
| 123 | +```csharp |
| 124 | +public class WrongMap : DuckDBClassMap<MyData> |
| 125 | +{ |
| 126 | + public WrongMap() |
| 127 | + { |
| 128 | + Map(x => x.DecimalValue); // decimal property |
| 129 | + } |
| 130 | +} |
| 131 | + |
| 132 | +// Throws: "Type mismatch for property 'DecimalValue': |
| 133 | +// Property type is Decimal (maps to Decimal) but column 0 is Float" |
| 134 | +var appender = connection.CreateAppender<MyData, WrongMap>("myTable"); |
| 135 | +``` |
| 136 | + |
| 137 | +## API Overview |
| 138 | + |
| 139 | +### Creating Mapped Appenders |
| 140 | + |
| 141 | +```csharp |
| 142 | +// Simple table name |
| 143 | +var appender = connection.CreateAppender<T, TMap>("tableName"); |
| 144 | + |
| 145 | +// With schema |
| 146 | +var appender = connection.CreateAppender<T, TMap>("schemaName", "tableName"); |
| 147 | + |
| 148 | +// With catalog and schema |
| 149 | +var appender = connection.CreateAppender<T, TMap>("catalog", "schema", "table"); |
| 150 | +``` |
| 151 | + |
| 152 | +### Appending Data |
| 153 | + |
| 154 | +```csharp |
| 155 | +// Multiple records |
| 156 | +appender.AppendRecords(recordList); |
| 157 | + |
| 158 | +// Close and flush |
| 159 | +appender.Close(); |
| 160 | +``` |
| 161 | + |
| 162 | +### Mapping Options |
| 163 | + |
| 164 | +```csharp |
| 165 | +public class MyMap : DuckDBClassMap<MyData> |
| 166 | +{ |
| 167 | + public MyMap() |
| 168 | + { |
| 169 | + Map(x => x.Property1); // Map to column in sequence |
| 170 | + Map(x => x.Property2); |
| 171 | + DefaultValue(); // Use column default |
| 172 | + NullValue(); // Insert NULL |
| 173 | + } |
| 174 | +} |
| 175 | +``` |
| 176 | + |
| 177 | +### Type Mappings |
| 178 | + |
| 179 | +The mapper validates .NET types against DuckDB column types: |
| 180 | + |
| 181 | +| .NET Type | DuckDB Type | |
| 182 | +|-----------|-------------| |
| 183 | +| `bool` | Boolean | |
| 184 | +| `sbyte` | TinyInt | |
| 185 | +| `short` | SmallInt | |
| 186 | +| `int` | Integer | |
| 187 | +| `long` | BigInt | |
| 188 | +| `byte` | UnsignedTinyInt | |
| 189 | +| `ushort` | UnsignedSmallInt | |
| 190 | +| `uint` | UnsignedInteger | |
| 191 | +| `ulong` | UnsignedBigInt | |
| 192 | +| `float` | Float | |
| 193 | +| `double` | Double | |
| 194 | +| `decimal` | Decimal | |
| 195 | +| `string` | Varchar | |
| 196 | +| `DateTime` | Timestamp | |
| 197 | +| `DateTimeOffset` | TimestampTz | |
| 198 | +| `TimeSpan` | Interval | |
| 199 | +| `Guid` | Uuid | |
| 200 | +| `DateOnly` | Date | |
| 201 | +| `TimeOnly` | Time | |
| 202 | + |
| 203 | +## Performance |
| 204 | + |
| 205 | +- **No runtime overhead**: Type mapping validated once at appender creation |
| 206 | +- **Fast value extraction**: Uses compiled expression getters |
| 207 | +- **Same underlying performance**: Uses the same fast data chunk API as the low-level appender |
| 208 | +- **Type safety without cost**: Validation at creation, not per-value |
0 commit comments