Skip to content

Commit baf30a0

Browse files
gqcnCopilot
andauthored
feat(contrib/drivers/dm): add Replace/InsertIgnore support and field type/length enhancements for dm database (gogf#4541)
This pull request introduces significant improvements to the DM database driver, especially around insert operations, and refines documentation and tests to reflect these changes. The main focus is enabling support for "replace" and "insert ignore" operations using DM's `MERGE` statement, improving type reporting for table fields, and updating documentation for clarity and accuracy. ### DM Driver Insert Operations * Added support for `Replace` and `InsertIgnore` operations in the DM driver by internally mapping them to DM's `MERGE` statement. This enables upsert and insert-ignore behavior for DM databases, improving compatibility with other drivers. * Implemented helper methods (`doMergeInsert`, `doInsertIgnore`, and `getPrimaryKeys`) to generate correct `MERGE` SQL statements and automatically detect primary keys when needed. [[1]](diffhunk://#diff-f51b30e3f0b0f1284b905385a89992efd0de2fe9ff8c5a4062344dfab17d428eL31-R94) [[2]](diffhunk://#diff-f51b30e3f0b0f1284b905385a89992efd0de2fe9ff8c5a4062344dfab17d428eL115-R212) * Updated the logic for building update values and SQL generation to ensure correct behavior for both upsert and insert-ignore cases. [[1]](diffhunk://#diff-f51b30e3f0b0f1284b905385a89992efd0de2fe9ff8c5a4062344dfab17d428eL61-R109) [[2]](diffhunk://#diff-f51b30e3f0b0f1284b905385a89992efd0de2fe9ff8c5a4062344dfab17d428eL89-R132) [[3]](diffhunk://#diff-f51b30e3f0b0f1284b905385a89992efd0de2fe9ff8c5a4062344dfab17d428eL100-R144) [[4]](diffhunk://#diff-f51b30e3f0b0f1284b905385a89992efd0de2fe9ff8c5a4062344dfab17d428eL115-R212) ### Table Field Type Reporting * Improved the DM driver's `TableFields` method to report column types with length/precision (e.g., `VARCHAR(128)` instead of just `VARCHAR`), aligning with expectations and other drivers. [[1]](diffhunk://#diff-40a365112421ae1967bd960f8acefcc91ddb8180865b78bc49cd090fbf4883daL26-R26) [[2]](diffhunk://#diff-40a365112421ae1967bd960f8acefcc91ddb8180865b78bc49cd090fbf4883daR88-R105) * Updated related unit tests to expect the new type format for DM table fields. ### Documentation Updates * Removed outdated or redundant documentation in both English and Chinese driver README files, and clarified supported features and limitations for DM and other drivers. [[1]](diffhunk://#diff-d49f5bc3a34b11a6ccb82cc54675b06a7dea5f0a943ae91c4ca0d28bd5003299L1) [[2]](diffhunk://#diff-d49f5bc3a34b11a6ccb82cc54675b06a7dea5f0a943ae91c4ca0d28bd5003299L47-R46) [[3]](diffhunk://#diff-d49f5bc3a34b11a6ccb82cc54675b06a7dea5f0a943ae91c4ca0d28bd5003299L119-L122) [[4]](diffhunk://#diff-05411a14e9c7ca235f7f436bfde732853aa93b364361fe80d65ac768f4e4d613L1-L126) ### Test Suite Enhancements * Refactored and restored unit tests for DM driver insert operations, including tests for `Save`, `Insert`, and the new `InsertIgnore` functionality to ensure correct behavior and compatibility. [[1]](diffhunk://#diff-2b1a59b8b2adaa1ca3074629374ab122929e4d4fbb4cc794b8e1db60ebf8d4c2L143-L245) [[2]](diffhunk://#diff-2b1a59b8b2adaa1ca3074629374ab122929e4d4fbb4cc794b8e1db60ebf8d4c2R512-R632) * Minor adjustments to DM test initialization for improved clarity. ### Core Insert Logic Minor Refactoring * Minor variable renaming for clarity in the core insert logic (`gdb_core.go`), improving code readability. [[1]](diffhunk://#diff-b1bbe5e3995261813e4e0ac6ffee8a37c236eaa2759f2bd82e211711695a70bcL449-R452) [[2]](diffhunk://#diff-b1bbe5e3995261813e4e0ac6ffee8a37c236eaa2759f2bd82e211711695a70bcL466-R474) --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 6e0ba55 commit baf30a0

10 files changed

Lines changed: 1659 additions & 293 deletions

contrib/drivers/README.MD

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
English | [简体中文](README.zh_CN.MD)
21

32
# Database drivers
43

@@ -44,7 +43,7 @@ func main() {
4443

4544
## Supported Drivers
4645

47-
### MySQL/MariaDB/TiDB
46+
### MySQL/MariaDB/TiDB/OceanBase
4847

4948
```go
5049
import _ "github.com/gogf/gf/contrib/drivers/mysql/v2"
@@ -116,10 +115,6 @@ Note:
116115
import _ "github.com/gogf/gf/contrib/drivers/dm/v2"
117116
```
118117

119-
Note:
120-
121-
- It does not support `Replace` features.
122-
123118
## Custom Drivers
124119

125120
It's quick and easy, please refer to current driver source.

contrib/drivers/README.zh_CN.MD

Lines changed: 0 additions & 126 deletions
This file was deleted.

contrib/drivers/dm/dm.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/gogf/gf/v2/frame/g"
1515
)
1616

17+
// Driver is the driver for dm database.
1718
type Driver struct {
1819
*gdb.Core
1920
}

contrib/drivers/dm/dm_do_insert.go

Lines changed: 103 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -28,28 +28,70 @@ func (d *Driver) DoInsert(
2828
return d.doSave(ctx, link, table, list, option)
2929

3030
case gdb.InsertOptionReplace:
31-
// TODO:: Should be Supported
32-
return nil, gerror.NewCode(
33-
gcode.CodeNotSupported, `Replace operation is not supported by dm driver`,
34-
)
35-
}
31+
// dm does not support REPLACE INTO syntax, use SAVE instead.
32+
return d.doSave(ctx, link, table, list, option)
33+
34+
case gdb.InsertOptionIgnore:
35+
// dm does not support INSERT IGNORE syntax, use MERGE instead.
36+
return d.doInsertIgnore(ctx, link, table, list, option)
3637

37-
return d.Core.DoInsert(ctx, link, table, list, option)
38+
default:
39+
return d.Core.DoInsert(ctx, link, table, list, option)
40+
}
3841
}
3942

4043
// doSave support upsert for dm
4144
func (d *Driver) doSave(ctx context.Context,
4245
link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption,
4346
) (result sql.Result, err error) {
44-
if len(option.OnConflict) == 0 {
45-
return nil, gerror.NewCode(
46-
gcode.CodeMissingParameter, `Please specify conflict columns`,
47-
)
47+
return d.doMergeInsert(ctx, link, table, list, option, true)
48+
}
49+
50+
// doInsertIgnore implements INSERT IGNORE operation using MERGE statement for DM database.
51+
// It only inserts records when there's no conflict on primary/unique keys.
52+
func (d *Driver) doInsertIgnore(ctx context.Context,
53+
link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption,
54+
) (result sql.Result, err error) {
55+
return d.doMergeInsert(ctx, link, table, list, option, false)
56+
}
57+
58+
// doMergeInsert implements MERGE-based insert operations for DM database.
59+
// When withUpdate is true, it performs upsert (insert or update).
60+
// When withUpdate is false, it performs insert ignore (insert only when no conflict).
61+
func (d *Driver) doMergeInsert(
62+
ctx context.Context,
63+
link gdb.Link,
64+
table string,
65+
list gdb.List,
66+
option gdb.DoInsertOption,
67+
withUpdate bool,
68+
) (result sql.Result, err error) {
69+
// If OnConflict is not specified, automatically get the primary key of the table
70+
conflictKeys := option.OnConflict
71+
if len(conflictKeys) == 0 {
72+
conflictKeys, err = d.getPrimaryKeys(ctx, table)
73+
if err != nil {
74+
return nil, gerror.WrapCode(
75+
gcode.CodeInternalError,
76+
err,
77+
`failed to get primary keys for table`,
78+
)
79+
}
80+
if len(conflictKeys) == 0 {
81+
return nil, gerror.NewCode(
82+
gcode.CodeMissingParameter,
83+
`Please specify conflict columns or ensure the table has a primary key`,
84+
)
85+
}
4886
}
4987

5088
if len(list) == 0 {
51-
return nil, gerror.NewCode(
52-
gcode.CodeInvalidRequest, `Save operation list is empty by oracle driver`,
89+
opName := "Save"
90+
if !withUpdate {
91+
opName = "InsertIgnore"
92+
}
93+
return nil, gerror.NewCodef(
94+
gcode.CodeInvalidRequest, `%s operation list is empty by dm driver`, opName,
5395
)
5496
}
5597

@@ -58,14 +100,13 @@ func (d *Driver) doSave(ctx context.Context,
58100
oneLen = len(one)
59101
charL, charR = d.GetChars()
60102

61-
conflictKeys = option.OnConflict
62103
conflictKeySet = gset.New(false)
63104

64-
// queryHolders: Handle data with Holder that need to be upsert
65-
// queryValues: Handle data that need to be upsert
105+
// queryHolders: Handle data with Holder that need to be merged
106+
// queryValues: Handle data that need to be merged
66107
// insertKeys: Handle valid keys that need to be inserted
67108
// insertValues: Handle values that need to be inserted
68-
// updateValues: Handle values that need to be updated
109+
// updateValues: Handle values that need to be updated (only when withUpdate=true)
69110
queryHolders = make([]string, oneLen)
70111
queryValues = make([]any, oneLen)
71112
insertKeys = make([]string, oneLen)
@@ -86,9 +127,9 @@ func (d *Driver) doSave(ctx context.Context,
86127
insertKeys[index] = keyWithChar
87128
insertValues[index] = fmt.Sprintf("T2.%s", keyWithChar)
88129

89-
// filter conflict keys in updateValues.
90-
// And the key is not a soft created field.
91-
if !(conflictKeySet.Contains(key) || d.Core.IsSoftCreatedFieldName(key)) {
130+
// Build updateValues only when withUpdate is true
131+
// Filter conflict keys and soft created fields from updateValues
132+
if withUpdate && !(conflictKeySet.Contains(key) || d.Core.IsSoftCreatedFieldName(key)) {
92133
updateValues = append(
93134
updateValues,
94135
fmt.Sprintf(`T1.%s = T2.%s`, keyWithChar, keyWithChar),
@@ -97,8 +138,10 @@ func (d *Driver) doSave(ctx context.Context,
97138
index++
98139
}
99140

100-
batchResult := new(gdb.SqlResult)
101-
sqlStr := parseSqlForUpsert(table, queryHolders, insertKeys, insertValues, updateValues, conflictKeys)
141+
var (
142+
batchResult = new(gdb.SqlResult)
143+
sqlStr = parseSqlForMerge(table, queryHolders, insertKeys, insertValues, updateValues, conflictKeys)
144+
)
102145
r, err := d.DoExec(ctx, link, sqlStr, queryValues...)
103146
if err != nil {
104147
return r, err
@@ -112,40 +155,58 @@ func (d *Driver) doSave(ctx context.Context,
112155
return batchResult, nil
113156
}
114157

115-
// parseSqlForUpsert
116-
// MERGE INTO {{table}} T1
117-
// USING ( SELECT {{queryHolders}} FROM DUAL T2
118-
// ON (T1.{{duplicateKey}} = T2.{{duplicateKey}} AND ...)
119-
// WHEN NOT MATCHED THEN
120-
// INSERT {{insertKeys}} VALUES {{insertValues}}
121-
// WHEN MATCHED THEN
122-
// UPDATE SET {{updateValues}}
123-
func parseSqlForUpsert(table string,
158+
// getPrimaryKeys retrieves the primary key field names of the table as a slice of strings.
159+
// This method extracts primary key information from TableFields.
160+
func (d *Driver) getPrimaryKeys(ctx context.Context, table string) ([]string, error) {
161+
tableFields, err := d.TableFields(ctx, table)
162+
if err != nil {
163+
return nil, err
164+
}
165+
166+
var primaryKeys []string
167+
for _, field := range tableFields {
168+
if field.Key == "PRI" {
169+
primaryKeys = append(primaryKeys, field.Name)
170+
}
171+
}
172+
173+
return primaryKeys, nil
174+
}
175+
176+
// parseSqlForMerge generates MERGE statement for DM database.
177+
// When updateValues is empty, it only inserts (INSERT IGNORE behavior).
178+
// When updateValues is provided, it performs upsert (INSERT or UPDATE).
179+
// Examples:
180+
// - INSERT IGNORE: MERGE INTO table T1 USING (...) T2 ON (...) WHEN NOT MATCHED THEN INSERT(...) VALUES (...)
181+
// - UPSERT: MERGE INTO table T1 USING (...) T2 ON (...) WHEN NOT MATCHED THEN INSERT(...) VALUES (...) WHEN MATCHED THEN UPDATE SET ...
182+
func parseSqlForMerge(table string,
124183
queryHolders, insertKeys, insertValues, updateValues, duplicateKey []string,
125184
) (sqlStr string) {
126185
var (
127186
queryHolderStr = strings.Join(queryHolders, ",")
128187
insertKeyStr = strings.Join(insertKeys, ",")
129188
insertValueStr = strings.Join(insertValues, ",")
130-
updateValueStr = strings.Join(updateValues, ",")
131189
duplicateKeyStr string
132-
pattern = gstr.Trim(`MERGE INTO %s T1 USING (SELECT %s FROM DUAL) T2 ON (%s) WHEN NOT MATCHED THEN INSERT(%s) VALUES (%s) WHEN MATCHED THEN UPDATE SET %s;`)
133190
)
134191

192+
// Build ON condition
135193
for index, keys := range duplicateKey {
136194
if index != 0 {
137195
duplicateKeyStr += " AND "
138196
}
139-
duplicateTmp := fmt.Sprintf("T1.%s = T2.%s", keys, keys)
140-
duplicateKeyStr += duplicateTmp
197+
duplicateKeyStr += fmt.Sprintf("T1.%s = T2.%s", keys, keys)
141198
}
142199

143-
return fmt.Sprintf(pattern,
144-
table,
145-
queryHolderStr,
146-
duplicateKeyStr,
147-
insertKeyStr,
148-
insertValueStr,
149-
updateValueStr,
150-
)
200+
// Build SQL based on whether UPDATE is needed
201+
pattern := gstr.Trim(`MERGE INTO %s T1 USING (SELECT %s FROM DUAL) T2 ON (%s) WHEN NOT MATCHED THEN INSERT(%s) VALUES (%s)`)
202+
if len(updateValues) > 0 {
203+
// Upsert: INSERT or UPDATE
204+
pattern += gstr.Trim(`WHEN MATCHED THEN UPDATE SET %s`)
205+
return fmt.Sprintf(
206+
pattern, table, queryHolderStr, duplicateKeyStr, insertKeyStr, insertValueStr,
207+
strings.Join(updateValues, ","),
208+
)
209+
}
210+
// Insert Ignore: INSERT only
211+
return fmt.Sprintf(pattern, table, queryHolderStr, duplicateKeyStr, insertKeyStr, insertValueStr)
151212
}

0 commit comments

Comments
 (0)