support just chunk based on maxplaceholders

Signed-off-by: hfuss <hayden.fuss@kaleido.io>

Author: onelapahead

pkg/dbsql/crud.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
-	"time"
 
 	sq "github.com/Masterminds/squirrel"
 	"github.com/hyperledger/firefly-common/pkg/ffapi"
@@ -497,11 +496,7 @@ func (c *CrudBase[T]) InsertMany(ctx context.Context, instances []T, allowPartia
 		return err
 	}
 	defer c.DB.RollbackTx(ctx, tx, autoCommit)
-	if c.DB.Features().ArrayInsertBuilder != nil {
-		if err := c.insertManyArray(ctx, tx, instances); err != nil {
-			return err
-		}
-	} else if c.DB.Features().MultiRowInsert {
+	if c.DB.Features().MultiRowInsert {
 		if err := c.insertManyMultiRow(ctx, tx, instances, allowPartialSuccess); err != nil {
 			return err
 		}
@@ -568,54 +563,6 @@ func (c *CrudBase[T]) insertManyMultiRow(ctx context.Context, tx *TXWrapper, ins
 	return nil
 }
 
-func (c *CrudBase[T]) insertManyArray(ctx context.Context, tx *TXWrapper, instances []T) error {
-	l := log.L(ctx)
-	builder := c.DB.Features().ArrayInsertBuilder
-
-	columnValues := make([][]interface{}, len(c.Columns))
-	for i := range c.Columns {
-		columnValues[i] = make([]interface{}, len(instances))
-	}
-
-	for rowIdx, inst := range instances {
-		c.setInsertTimestamps(inst, fftypes.Now())
-		for colIdx, col := range c.Columns {
-			columnValues[colIdx][rowIdx] = c.getFieldValue(inst, col)
-		}
-	}
-
-	sqlQuery, args, err := builder(ctx, c.Table, c.Columns, columnValues, c.DB.SequenceColumn())
-	if err != nil {
-		return i18n.WrapError(ctx, err, i18n.MsgDBQueryBuildFailed)
-	}
-
-	before := time.Now()
-	l.Tracef(`SQL-> array insert: %s (args: %d arrays)`, sqlQuery, len(args))
-	rows, err := tx.sqlTX.QueryContext(ctx, sqlQuery, args...)
-	if err != nil {
-		l.Errorf(`SQL array insert failed: %s sql=[ %s ]: %s`, err, sqlQuery, err)
-		return i18n.WrapError(ctx, err, i18n.MsgDBInsertFailed)
-	}
-	defer rows.Close()
-
-	for i := 0; i < len(instances) && rows.Next(); i++ {
-		var seq int64
-		if err := rows.Scan(&seq); err != nil {
-			return i18n.WrapError(ctx, err, i18n.MsgDBReadErr, c.Table)
-		}
-		c.attemptSetSequence(instances[i], seq)
-	}
-
-	for _, inst := range instances {
-		if c.EventHandler != nil {
-			c.EventHandler(inst.GetID(), Created)
-		}
-	}
-
-	l.Debugf(`SQL<- array insert %s rows=%d (%.2fms)`, c.Table, len(instances), floatMillisSince(before))
-	return nil
-}
-
 func (c *CrudBase[T]) Insert(ctx context.Context, inst T, hooks ...PostCompletionHook) (err error) {
 	ctx, tx, autoCommit, err := c.DB.BeginOrUseTx(ctx)
 	if err != nil {

pkg/dbsql/crud_test.go

@@ -1087,83 +1087,6 @@ func TestInsertManyMultiRowNoChunkingWhenUnderLimit(t *testing.T) {
 	assert.NoError(t, mock.ExpectationsWereMet())
 }
 
-func TestInsertManyArrayInsertOK(t *testing.T) {
-	mp := NewMockProvider()
-	mp.FakePSQLArrayInsert = true
-	db, mock := mp.UTInit()
-	tc := newCRUDCollection(&db.Database, "ns1")
-	mock.ExpectBegin()
-	mock.ExpectQuery(`INSERT INTO crudables .* SELECT UNNEST.*RETURNING seq`).WillReturnRows(
-		sqlmock.NewRows([]string{db.sequenceColumn}).
-			AddRow(301).
-			AddRow(302),
-	)
-	mock.ExpectCommit()
-	err := tc.InsertMany(context.Background(), []*TestCRUDable{
-		{ResourceBase: ResourceBase{ID: fftypes.NewUUID()}},
-		{ResourceBase: ResourceBase{ID: fftypes.NewUUID()}},
-	}, false)
-	assert.NoError(t, err)
-	assert.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestInsertManyArrayInsertFail(t *testing.T) {
-	mp := NewMockProvider()
-	mp.FakePSQLArrayInsert = true
-	db, mock := mp.UTInit()
-	tc := newCRUDCollection(&db.Database, "ns1")
-	mock.ExpectBegin()
-	mock.ExpectQuery(`INSERT INTO crudables .* SELECT UNNEST.*`).WillReturnError(fmt.Errorf("pop"))
-	err := tc.InsertMany(context.Background(), []*TestCRUDable{
-		{ResourceBase: ResourceBase{ID: fftypes.NewUUID()}},
-	}, false)
-	assert.Regexp(t, "FF00177", err)
-	assert.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestInsertManyArrayInsertPrefersOverMultiRow(t *testing.T) {
-	mp := NewMockProvider()
-	mp.FakePSQLArrayInsert = true
-	mp.MultiRowInsert = true
-	db, mock := mp.UTInit()
-	tc := newCRUDCollection(&db.Database, "ns1")
-	mock.ExpectBegin()
-	// Should use UNNEST, not multi-row VALUES
-	mock.ExpectQuery(`INSERT INTO crudables .* SELECT UNNEST.*RETURNING seq`).WillReturnRows(
-		sqlmock.NewRows([]string{db.sequenceColumn}).
-			AddRow(401),
-	)
-	mock.ExpectCommit()
-	err := tc.InsertMany(context.Background(), []*TestCRUDable{
-		{ResourceBase: ResourceBase{ID: fftypes.NewUUID()}},
-	}, false)
-	assert.NoError(t, err)
-	assert.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestInsertManyArrayInsertWithEvents(t *testing.T) {
-	mp := NewMockProvider()
-	mp.FakePSQLArrayInsert = true
-	db, mock := mp.UTInit()
-	tc := newCRUDCollection(&db.Database, "ns1")
-	mock.ExpectBegin()
-	mock.ExpectQuery(`INSERT INTO crudables .* SELECT UNNEST.*RETURNING seq`).WillReturnRows(
-		sqlmock.NewRows([]string{db.sequenceColumn}).
-			AddRow(501).
-			AddRow(502),
-	)
-	mock.ExpectCommit()
-	err := tc.InsertMany(context.Background(), []*TestCRUDable{
-		{ResourceBase: ResourceBase{ID: fftypes.NewUUID()}},
-		{ResourceBase: ResourceBase{ID: fftypes.NewUUID()}},
-	}, false)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, len(tc.events))
-	assert.Equal(t, Created, tc.events[0])
-	assert.Equal(t, Created, tc.events[1])
-	assert.NoError(t, mock.ExpectationsWereMet())
-}
-
 func TestInsertManyFallbackSingleRowFail(t *testing.T) {
 	db, mock := NewMockProvider().UTInit()
 	tc := newCRUDCollection(&db.Database, "ns1")

pkg/dbsql/mock_provider.go

@@ -52,7 +52,6 @@ type MockProviderConfig struct {
 	MultiRowInsert             bool
 	MaxPlaceholders            int
 	FakePSQLUpsertOptimization bool
-	FakePSQLArrayInsert        bool
 }
 
 func NewMockProvider() *MockProvider {
@@ -95,9 +94,6 @@ func (mp *MockProvider) Features() SQLFeatures {
 	}
 	features.MultiRowInsert = mp.MultiRowInsert
 	features.MaxPlaceholders = mp.MaxPlaceholders
-	if mp.FakePSQLArrayInsert {
-		features.ArrayInsertBuilder = BuildPostgreSQLArrayInsert
-	}
 	if mp.FakePSQLUpsertOptimization {
 		features.DBOptimizedUpsertBuilder = BuildPostgreSQLOptimizedUpsert
 	}

pkg/dbsql/postgres_helpers.go

@@ -45,41 +45,3 @@ func BuildPostgreSQLOptimizedUpsert(ctx context.Context, table string, idColumn
 	return insert.Suffix(fmt.Sprintf("ON CONFLICT (%s) DO UPDATE", idColumn)).SuffixExpr(sq.Expr(updateSQL, updateValues...)).Suffix("RETURNING " + returnCol), nil
 }
 
-// PostgreSQLArrayInsertBuilder returns an ArrayInsertBuilder for PostgreSQL using UNNEST.
-// wrapArray must convert a []interface{} into a driver-compatible array value
-// (e.g. pass pq.Array from lib/pq). For unit tests, pass a no-op wrapper.
-func PostgreSQLArrayInsertBuilder(wrapArray func([]interface{}) interface{}) func(ctx context.Context, table string, columns []string, columnValues [][]interface{}, sequenceColumn string) (string, []interface{}, error) {
-	return func(_ context.Context, table string, columns []string, columnValues [][]interface{}, sequenceColumn string) (string, []interface{}, error) {
-		unnests := make([]string, len(columns))
-		args := make([]interface{}, len(columns))
-		for i := range columns {
-			unnests[i] = fmt.Sprintf("UNNEST($%d)", i+1)
-			args[i] = wrapArray(columnValues[i])
-		}
-		sql := fmt.Sprintf(
-			"INSERT INTO %s (%s) SELECT %s RETURNING %s",
-			table,
-			strings.Join(columns, ", "),
-			strings.Join(unnests, ", "),
-			sequenceColumn,
-		)
-		return sql, args, nil
-	}
-}
-
-// mockArrayValue wraps a slice so database/sql treats it as a single driver.Value.
-type mockArrayValue struct {
-	values []interface{}
-}
-
-func (m mockArrayValue) Value() (driver.Value, error) {
-	return fmt.Sprintf("{%d values}", len(m.values)), nil
-}
-
-// BuildPostgreSQLArrayInsert is a convenience for tests that don't need real pq.Array wrapping.
-// It uses a mock driver.Valuer so sqlmock can accept the args.
-func BuildPostgreSQLArrayInsert(ctx context.Context, table string, columns []string, columnValues [][]interface{}, sequenceColumn string) (string, []interface{}, error) {
-	return PostgreSQLArrayInsertBuilder(func(vals []interface{}) interface{} {
-		return mockArrayValue{values: vals}
-	})(ctx, table, columns, columnValues, sequenceColumn)
-}

pkg/dbsql/postgres_helpers_test.go

@@ -61,46 +61,3 @@ func TestBuildPostgreSQLOptimizedUpsertFail(t *testing.T) {
 
 }
 
-func TestBuildPostgreSQLArrayInsert(t *testing.T) {
-	sqlStr, args, err := BuildPostgreSQLArrayInsert(
-		context.Background(),
-		"transfers",
-		[]string{"id", "sender", "amount"},
-		[][]interface{}{
-			{"id1", "id2"},
-			{"0xaaa", "0xbbb"},
-			{100, 200},
-		},
-		"seq",
-	)
-	assert.NoError(t, err)
-	assert.Equal(t, "INSERT INTO transfers (id, sender, amount) SELECT UNNEST($1), UNNEST($2), UNNEST($3) RETURNING seq", sqlStr)
-	assert.Len(t, args, 3)
-	// Args are mockArrayValue wrappers for database/sql compatibility
-	for _, arg := range args {
-		_, ok := arg.(mockArrayValue)
-		assert.True(t, ok, "expected mockArrayValue, got %T", arg)
-	}
-}
-
-func TestPostgreSQLArrayInsertBuilderWithCustomWrapper(t *testing.T) {
-	wrapCalled := 0
-	builder := PostgreSQLArrayInsertBuilder(func(vals []interface{}) interface{} {
-		wrapCalled++
-		return vals
-	})
-	sql, args, err := builder(
-		context.Background(),
-		"things",
-		[]string{"a", "b"},
-		[][]interface{}{
-			{1, 2, 3},
-			{"x", "y", "z"},
-		},
-		"seq",
-	)
-	assert.NoError(t, err)
-	assert.Equal(t, "INSERT INTO things (a, b) SELECT UNNEST($1), UNNEST($2) RETURNING seq", sql)
-	assert.Len(t, args, 2)
-	assert.Equal(t, 2, wrapCalled)
-}

pkg/dbsql/provider.go

@@ -38,11 +38,6 @@ type SQLFeatures struct {
 	// When >0 and MultiRowInsert is true, InsertMany will chunk rows so that
 	// rows*columns never exceeds this limit. PostgreSQL's wire protocol limit is 65535.
 	MaxPlaceholders int
-	// ArrayInsertBuilder enables column-wise array-based bulk inserts (e.g. PostgreSQL UNNEST).
-	// When set, InsertMany prefers this over multi-row VALUES inserts.
-	// The builder receives column names and per-column value slices, and returns raw SQL + args.
-	// The returned SQL must include a RETURNING clause for the sequence column if sequences are needed.
-	ArrayInsertBuilder func(ctx context.Context, table string, columns []string, columnValues [][]interface{}, sequenceColumn string) (sql string, args []interface{}, err error)
 }
 
 func DefaultSQLProviderFeatures() SQLFeatures {