Skip to content

Commit c79be00

Browse files
benwebberbenwebber
committed
sql: Add ClickHouse tests for numberic conversion methods; document number behaviour
1 parent 60afaae commit c79be00

3 files changed

Lines changed: 126 additions & 0 deletions

File tree

‎docs/modules/components/pages/processors/sql_raw.adoc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,9 @@ The `pgx` driver is an alternative to the standard `postgres` (pq) driver and co
230230
231231
The `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `<snowflake_user>@<snowflake_account>/<db_name>/<schema_name>?warehouse=<warehouse>&role=<role>&authenticator=snowflake_jwt&privateKey=<base64_url_encoded_private_key>`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` instead of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.
232232
233+
The `clickhouse` driver automatically converts Bloblang numbers to `Int64` or `Float64` values depending on whether they have a fractional component.
234+
Use the xref:guides:bloblang/methods.adoc[number manipulation methods] to convert numbers to different types if necessary.
235+
233236
The https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.
234237
235238

‎docs/modules/components/pages/processors/sql_select.adoc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,9 @@ The `pgx` driver is an alternative to the standard `postgres` (pq) driver and co
191191
192192
The `snowflake` driver supports multiple DSN formats. Please consult https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String[the docs^] for more details. For https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication[key pair authentication^], the DSN has the following format: `<snowflake_user>@<snowflake_account>/<db_name>/<schema_name>?warehouse=<warehouse>&role=<role>&authenticator=snowflake_jwt&privateKey=<base64_url_encoded_private_key>`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` instead of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.
193193
194+
The `clickhouse` driver automatically converts Bloblang numbers to `Int64` or `Float64` values depending on whether they have a fractional component.
195+
Use the xref:guides:bloblang/methods.adoc[number manipulation methods] to convert numbers to different types if necessary.
196+
194197
The https://pkg.go.dev/github.com/microsoft/gocosmos[`gocosmos`^] driver is still experimental, but it has support for https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys[hierarchical partition keys^] as well as https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query[cross-partition queries^]. Please refer to the https://github.com/microsoft/gocosmos/blob/main/SQL.md[SQL notes^] for details.
195198
196199

‎internal/impl/sql/integration_test.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -763,6 +763,126 @@ args_mapping: 'root = [ this.v ]'
763763
})
764764
})
765765

766+
t.Run("sql_select bloblang number methods", func(t *testing.T) {
767+
tableName, err := gonanoid.Generate("abcdefghijklmnopqrstuvwxyz", 12)
768+
require.NoError(t, err)
769+
770+
_, err = db.ExecContext(t.Context(), fmt.Sprintf(`
771+
CREATE TABLE %s (
772+
i8 Int8, i16 Int16, i32 Int32, i64 Int64,
773+
u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64,
774+
f32 Float32, f64 Float64
775+
) ENGINE=MergeTree() ORDER BY i64`, tableName))
776+
require.NoError(t, err)
777+
_, err = db.ExecContext(t.Context(), fmt.Sprintf(
778+
`INSERT INTO %s VALUES (42, 1000, 70000, 1774903303, 200, 50000, 3000000000, 9000000000, 1.5, 3.14)`,
779+
tableName))
780+
require.NoError(t, err)
781+
782+
cases := []struct {
783+
col string
784+
method string
785+
value string
786+
expected string
787+
}{
788+
{"i8", "int8", "42", "WHERE i8 = 42"},
789+
{"i16", "int16", "1000", "WHERE i16 = 1000"},
790+
{"i32", "int32", "70000", "WHERE i32 = 70000"},
791+
{"i64", "int64", "1774903303", "WHERE i64 = 1774903303"},
792+
{"u8", "uint8", "200", "WHERE u8 = 200"},
793+
{"u16", "uint16", "50000", "WHERE u16 = 50000"},
794+
{"u32", "uint32", "3000000000", "WHERE u32 = 3000000000"},
795+
{"u64", "uint64", "9000000000", "WHERE u64 = 9000000000"},
796+
{"f32", "float32", "1.5", "WHERE f32 = 1.5"},
797+
{"f64", "float64", "3.14", "WHERE f64 = 3.14"},
798+
}
799+
800+
for _, tc := range cases {
801+
t.Run(tc.method, func(t *testing.T) {
802+
conf := fmt.Sprintf(`
803+
driver: clickhouse
804+
dsn: %s
805+
table: %s
806+
columns: [ "%s" ]
807+
where: '%s = ?'
808+
args_mapping: 'root = [ this.n.%s() ]'
809+
`, dsn, tableName, tc.col, tc.col, tc.method)
810+
env := service.NewEnvironment()
811+
selectConfig, err := isql.SelectProcessorConfig().ParseYAML(conf, env)
812+
require.NoError(t, err)
813+
selectProc, err := isql.NewSQLSelectProcessorFromConfig(selectConfig, service.MockResources())
814+
require.NoError(t, err)
815+
t.Cleanup(func() { selectProc.Close(t.Context()) })
816+
817+
msg := service.NewMessage(fmt.Appendf(nil, `{"n":%s}`, tc.value))
818+
_, err = selectProc.ProcessBatch(t.Context(), service.MessageBatch{msg})
819+
require.NoError(t, err)
820+
821+
require.Contains(t, lastQuery(t, tableName), tc.expected)
822+
})
823+
}
824+
})
825+
826+
t.Run("sql_raw bloblang number methods", func(t *testing.T) {
827+
// Verify that explicitly cast Bloblang values (int8(), uint32(), float32(), …)
828+
// arrive at ClickHouse as native numeric literals, not string literals.
829+
tableName, err := gonanoid.Generate("abcdefghijklmnopqrstuvwxyz", 12)
830+
require.NoError(t, err)
831+
832+
_, err = db.ExecContext(t.Context(), fmt.Sprintf(`
833+
CREATE TABLE %s (
834+
i8 Int8, i16 Int16, i32 Int32, i64 Int64,
835+
u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64,
836+
f32 Float32, f64 Float64
837+
) ENGINE=MergeTree() ORDER BY i64`, tableName))
838+
require.NoError(t, err)
839+
_, err = db.ExecContext(t.Context(), fmt.Sprintf(
840+
`INSERT INTO %s VALUES (42, 1000, 70000, 1774903303, 200, 50000, 3000000000, 9000000000, 1.5, 3.14)`,
841+
tableName))
842+
require.NoError(t, err)
843+
844+
cases := []struct {
845+
col string
846+
method string
847+
value string
848+
expected string
849+
}{
850+
{"i8", "int8", "42", "WHERE i8 = 42"},
851+
{"i16", "int16", "1000", "WHERE i16 = 1000"},
852+
{"i32", "int32", "70000", "WHERE i32 = 70000"},
853+
{"i64", "int64", "1774903303", "WHERE i64 = 1774903303"},
854+
{"u8", "uint8", "200", "WHERE u8 = 200"},
855+
{"u16", "uint16", "50000", "WHERE u16 = 50000"},
856+
{"u32", "uint32", "3000000000", "WHERE u32 = 3000000000"},
857+
{"u64", "uint64", "9000000000", "WHERE u64 = 9000000000"},
858+
{"f32", "float32", "1.5", "WHERE f32 = 1.5"},
859+
{"f64", "float64", "3.14", "WHERE f64 = 3.14"},
860+
}
861+
862+
for _, tc := range cases {
863+
t.Run(tc.method, func(t *testing.T) {
864+
conf := fmt.Sprintf(`
865+
driver: clickhouse
866+
dsn: %s
867+
query: 'SELECT %s FROM %s WHERE %s = $1'
868+
args_mapping: 'root = [ this.n.%s() ]'
869+
`, dsn, tc.col, tableName, tc.col, tc.method)
870+
env := service.NewEnvironment()
871+
rawConfig, err := isql.RawProcessorConfig().ParseYAML(conf, env)
872+
require.NoError(t, err)
873+
rawProc, err := isql.NewSQLRawProcessorFromConfig(rawConfig, service.MockResources())
874+
require.NoError(t, err)
875+
t.Cleanup(func() { rawProc.Close(t.Context()) })
876+
877+
msg := service.NewMessage(fmt.Appendf(nil, `{"n":%s}`, tc.value))
878+
_, err = rawProc.ProcessBatch(t.Context(), service.MessageBatch{msg})
879+
require.NoError(t, err)
880+
881+
require.Contains(t, lastQuery(t, tableName), tc.expected)
882+
})
883+
}
884+
})
885+
766886
t.Run("sql_raw query log", func(t *testing.T) {
767887
tableName, err := gonanoid.Generate("abcdefghijklmnopqrstuvwxyz", 12)
768888
require.NoError(t, err)

0 commit comments

Comments
 (0)