From 6dd13a3dc6c9e9c79cbc8c8143a53183e8188236 Mon Sep 17 00:00:00 2001 From: Qifan Zhang Date: Thu, 20 Mar 2025 16:46:43 +0800 Subject: [PATCH 1/8] core code --- .../Drivers/BigQuery/BigQueryConnection.cs | 4 +- .../Drivers/BigQuery/BigQueryParameters.cs | 2 + .../src/Drivers/BigQuery/BigQueryStatement.cs | 54 +++++++++++-------- csharp/src/Drivers/BigQuery/readme.md | 6 +++ .../BigQuery/BigQueryTestConfiguration.cs | 6 +++ .../Drivers/BigQuery/BigQueryTestingUtils.cs | 10 ++++ csharp/test/Drivers/BigQuery/readme.md | 3 +- 7 files changed, 60 insertions(+), 25 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 447f288f38..aec5efed52 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -1058,7 +1058,9 @@ private IReadOnlyDictionary ParseOptions() BigQueryParameters.LargeDecimalsAsString, BigQueryParameters.LargeResultsDestinationTable, BigQueryParameters.GetQueryResultsOptionsTimeout, - BigQueryParameters.MaxFetchConcurrency + BigQueryParameters.MaxFetchConcurrency, + BigQueryParameters.StatementType, + BigQueryParameters.StatementIndex }; foreach (string key in statementOptions) diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs index abfb7c1029..f3dced0148 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs @@ -39,6 +39,8 @@ public class BigQueryParameters public const string GetQueryResultsOptionsTimeout = "adbc.bigquery.get_query_results_options.timeout"; public const string MaxFetchConcurrency = "adbc.bigquery.max_fetch_concurrency"; public const string IncludePublicProjectId = "adbc.bigquery.include_public_project_id"; + public const string StatementType = "adbc.bigquery.multiple_statement.statement_type"; + public const string StatementIndex = "adbc.bigquery.multiple_statement.statement_index"; } /// diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index ddf34104f9..def4215e21 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; +using System.Data; using System.IO; using System.Linq; using System.Threading; @@ -51,52 +52,63 @@ public BigQueryStatement(BigQueryClient client, GoogleCredential credential) public override QueryResult ExecuteQuery() { + // Create job QueryOptions queryOptions = ValidateOptions(); - BigQueryJob job = this.client.CreateQueryJob(SqlQuery, null, queryOptions); + // Get results GetQueryResultsOptions getQueryResultsOptions = new GetQueryResultsOptions(); - if (this.Options?.TryGetValue(BigQueryParameters.GetQueryResultsOptionsTimeout, out string? timeoutSeconds) == true && int.TryParse(timeoutSeconds, out int seconds) && seconds >= 0) { getQueryResultsOptions.Timeout = TimeSpan.FromSeconds(seconds); } - BigQueryResults results = job.GetQueryResults(getQueryResultsOptions); - BigQueryReadClientBuilder readClientBuilder = new BigQueryReadClientBuilder(); - readClientBuilder.Credential = this.credential; - BigQueryReadClient readClient = readClientBuilder.Build(); - + // For multi-statement queries, the results.TableReference is null if (results.TableReference == null) { + string statementType = string.Empty; + if (this.Options?.TryGetValue(BigQueryParameters.StatementType, out string? statementTypeString) == true) + { + statementType = statementTypeString; + } + int statementIndex = 1; + if (this.Options?.TryGetValue(BigQueryParameters.StatementIndex, out string? statementIndexString) == true && + int.TryParse(statementIndexString, out int statementIndexInt) && + statementIndexInt > 0) + { + statementIndex = statementIndexInt; + } + // To get the results of all statements in a multi-statement query, enumerate the child jobs and call jobs.getQueryResults on each of them. // Related public docs: https://cloud.google.com/bigquery/docs/multi-statement-queries#get_all_executed_statements ListJobsOptions listJobsOptions = new ListJobsOptions(); listJobsOptions.ParentJobId = results.JobReference.JobId; - PagedEnumerable joblist = client.ListJobs(listJobsOptions); - BigQueryJob firstQueryJob = new BigQueryJob(client, job.Resource); - foreach (BigQueryJob childJob in joblist) + var joblist = client.ListJobs(listJobsOptions) + .Select(job => client.GetJob(job.Reference)) + .Where(job => string.IsNullOrEmpty(statementType) || job.Statistics.Query.StatementType.Equals(statementType,StringComparison.OrdinalIgnoreCase)) + .OrderBy(job => job.Resource.Statistics.CreationTime) + .ToList(); + + if(statementIndex < 1 || statementIndex > joblist.Count) { - var tempJob = client.GetJob(childJob.Reference); - var query = tempJob.Resource?.Configuration?.Query; - if (query != null && query.DestinationTable != null && query.DestinationTable.ProjectId != null && query.DestinationTable.DatasetId != null && query.DestinationTable.TableId != null) - { - firstQueryJob = tempJob; - } + throw new ArgumentOutOfRangeException($"The specified index {statementIndex} is out of range. There are {joblist.Count} jobs available."); } - results = firstQueryJob.GetQueryResults(); - } + results = joblist[statementIndex - 1].GetQueryResults(getQueryResultsOptions); + } if (results.TableReference == null) { throw new AdbcException("There is no query statement"); } + // BigQuery Read Client for streaming + BigQueryReadClientBuilder readClientBuilder = new BigQueryReadClientBuilder(); + readClientBuilder.Credential = this.credential; + BigQueryReadClient readClient = readClientBuilder.Build(); string table = $"projects/{results.TableReference.ProjectId}/datasets/{results.TableReference.DatasetId}/tables/{results.TableReference.TableId}"; - int maxStreamCount = 1; if (this.Options?.TryGetValue(BigQueryParameters.MaxFetchConcurrency, out string? maxStreamCountString) == true) { @@ -110,16 +122,12 @@ public override QueryResult ExecuteQuery() } ReadSession rs = new ReadSession { Table = table, DataFormat = DataFormat.Arrow }; ReadSession rrs = readClient.CreateReadSession("projects/" + results.TableReference.ProjectId, rs, maxStreamCount); - long totalRows = results.TotalRows == null ? -1L : (long)results.TotalRows.Value; - var readers = rrs.Streams .Select(s => ReadChunk(readClient, s.Name)) .Where(chunk => chunk != null) .Cast(); - IArrowArrayStream stream = new MultiArrowReader(TranslateSchema(results.Schema), readers); - return new QueryResult(totalRows, stream); } diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md index 823a066fa8..f3f4f49af0 100644 --- a/csharp/src/Drivers/BigQuery/readme.md +++ b/csharp/src/Drivers/BigQuery/readme.md @@ -63,6 +63,12 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G **adbc.bigquery.max_fetch_concurrency**
    Optional. Sets the [maxStreamCount](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.Storage.V1/latest/Google.Cloud.BigQuery.Storage.V1.BigQueryReadClient#Google_Cloud_BigQuery_Storage_V1_BigQueryReadClient_CreateReadSession_System_String_Google_Cloud_BigQuery_Storage_V1_ReadSession_System_Int32_Google_Api_Gax_Grpc_CallSettings_) for the CreateReadSession method. If not set, defaults to 1. +**adbc.bigquery.multiple_statement.statement_type**
+    Optional. When executing multiple statements, limit the type of statement returned. If not set, all types of statements are returned. + +**adbc.bigquery.multiple_statement.statement_index**
+    Optional. When executing multiple statements, specify the result of the statement to be returned. If not set, the result of the first statement is returned. + **adbc.bigquery.include_constraints_getobjects**
    Optional. Some callers do not need the constraint details when they get the table information and can improve the speed of obtaining the results. Setting this value to `"false"` will not include the constraint details. The default value is `"true"`. diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs index 9be82bd5c1..efb7c46513 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs @@ -97,6 +97,12 @@ public BigQueryTestEnvironment() [JsonPropertyName("maxStreamCount")] public int? MaxStreamCount { get; set; } + [JsonPropertyName("statementType")] + public string StatementType { get; set; } = string.Empty; + + [JsonPropertyName("statementIndex")] + public int? StatementIndex { get; set; } + /// /// How structs should be handled by the ADO.NET client for this environment. /// diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs index 5768b7e0c9..2bf1249f64 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs @@ -115,6 +115,16 @@ internal static Dictionary GetBigQueryParameters(BigQueryTestEnv parameters.Add(BigQueryParameters.MaxFetchConcurrency, testEnvironment.MaxStreamCount.Value.ToString()); } + if (!string.IsNullOrEmpty(testEnvironment.StatementType)) + { + parameters.Add(BigQueryParameters.StatementType, testEnvironment.StatementType!); + } + + if (testEnvironment.StatementIndex.HasValue) + { + parameters.Add(BigQueryParameters.StatementIndex, testEnvironment.StatementIndex.Value.ToString()); + } + return parameters; } diff --git a/csharp/test/Drivers/BigQuery/readme.md b/csharp/test/Drivers/BigQuery/readme.md index 5d6db3d166..c88fbad72c 100644 --- a/csharp/test/Drivers/BigQuery/readme.md +++ b/csharp/test/Drivers/BigQuery/readme.md @@ -46,7 +46,8 @@ The following values can be setup in the configuration - **includePublicProjectId** - True/False to indicate if the public projects should be included in the result set. - **scopes** - Comma separated list (string) of scopes applied during the test. - **queryTimeout** - The timeout (in seconds) for a query. Similar to a CommandTimeout. - - **maxStreamCount** - The max stream count. + - **statementType** - When executing multiple statements, limit the type of statement returned. + - **statementIndex** - When executing multiple statements, specify the result of the statement to be returned. - **includeTableConstraints** - Whether to include table constraints in the GetObjects query. - **largeResultsDestinationTable** - Sets the [DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable) value of the QueryOptions if configured. Expects the format to be `{projectId}.{datasetId}.{tableId}` to set the corresponding values in the [TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76faf97/Src/Generated/Google.Apis.Bigquery.v2/Google.Apis.Bigquery.v2.cs#L9348) class. - **allowLargeResults** - Whether to allow large results . From a6b47a690a6901f3c04cff960cc90b40d89e2813 Mon Sep 17 00:00:00 2001 From: Qifan Zhang Date: Fri, 11 Apr 2025 15:48:44 +0800 Subject: [PATCH 2/8] filter by EvaluationKind --- .../Drivers/BigQuery/BigQueryConnection.cs | 2 +- .../Drivers/BigQuery/BigQueryParameters.cs | 2 +- .../src/Drivers/BigQuery/BigQueryStatement.cs | 22 ++++++++----------- csharp/src/Drivers/BigQuery/readme.md | 4 ++-- .../BigQuery/BigQueryTestConfiguration.cs | 2 +- .../Drivers/BigQuery/BigQueryTestingUtils.cs | 6 ++--- csharp/test/Drivers/BigQuery/readme.md | 2 +- 7 files changed, 18 insertions(+), 22 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index aec5efed52..0062f2d6ec 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -1060,7 +1060,7 @@ private IReadOnlyDictionary ParseOptions() BigQueryParameters.GetQueryResultsOptionsTimeout, BigQueryParameters.MaxFetchConcurrency, BigQueryParameters.StatementType, - BigQueryParameters.StatementIndex + BigQueryParameters.EvaluationKind }; foreach (string key in statementOptions) diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs index f3dced0148..2fb64da7fa 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs @@ -40,7 +40,7 @@ public class BigQueryParameters public const string MaxFetchConcurrency = "adbc.bigquery.max_fetch_concurrency"; public const string IncludePublicProjectId = "adbc.bigquery.include_public_project_id"; public const string StatementType = "adbc.bigquery.multiple_statement.statement_type"; - public const string StatementIndex = "adbc.bigquery.multiple_statement.statement_index"; + public const string EvaluationKind = "adbc.bigquery.multiple_statement.evaluation_kind"; } /// diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index def4215e21..5f96a30994 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -17,7 +17,6 @@ using System; using System.Collections.Generic; -using System.Data; using System.IO; using System.Linq; using System.Threading; @@ -74,30 +73,27 @@ public override QueryResult ExecuteQuery() { statementType = statementTypeString; } - int statementIndex = 1; - if (this.Options?.TryGetValue(BigQueryParameters.StatementIndex, out string? statementIndexString) == true && - int.TryParse(statementIndexString, out int statementIndexInt) && - statementIndexInt > 0) + string evaluationKind = string.Empty; + if (this.Options?.TryGetValue(BigQueryParameters.EvaluationKind, out string? evaluationKindString) == true) { - statementIndex = statementIndexInt; + evaluationKind = evaluationKindString; } - - // To get the results of all statements in a multi-statement query, enumerate the child jobs and call jobs.getQueryResults on each of them. - // Related public docs: https://cloud.google.com/bigquery/docs/multi-statement-queries#get_all_executed_statements + + // To get the results of all statements in a multi-statement query, enumerate the child jobs. Related public docs: https://cloud.google.com/bigquery/docs/multi-statement-queries#get_all_executed_statements. + // Can filter by StatementType and EvaluationKind. Related public docs: https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#jobstatistics2, https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#evaluationkind ListJobsOptions listJobsOptions = new ListJobsOptions(); listJobsOptions.ParentJobId = results.JobReference.JobId; var joblist = client.ListJobs(listJobsOptions) .Select(job => client.GetJob(job.Reference)) + .Where(job => string.IsNullOrEmpty(evaluationKind) || job.Statistics.ScriptStatistics.EvaluationKind.Equals(evaluationKind, StringComparison.OrdinalIgnoreCase)) .Where(job => string.IsNullOrEmpty(statementType) || job.Statistics.Query.StatementType.Equals(statementType,StringComparison.OrdinalIgnoreCase)) .OrderBy(job => job.Resource.Statistics.CreationTime) .ToList(); - if(statementIndex < 1 || statementIndex > joblist.Count) + if (joblist.Count > 0) { - throw new ArgumentOutOfRangeException($"The specified index {statementIndex} is out of range. There are {joblist.Count} jobs available."); + results = joblist[0].GetQueryResults(getQueryResultsOptions); } - - results = joblist[statementIndex - 1].GetQueryResults(getQueryResultsOptions); } if (results.TableReference == null) { diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md index f3f4f49af0..94c93090c1 100644 --- a/csharp/src/Drivers/BigQuery/readme.md +++ b/csharp/src/Drivers/BigQuery/readme.md @@ -66,8 +66,8 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G **adbc.bigquery.multiple_statement.statement_type**
    Optional. When executing multiple statements, limit the type of statement returned. If not set, all types of statements are returned. -**adbc.bigquery.multiple_statement.statement_index**
-    Optional. When executing multiple statements, specify the result of the statement to be returned. If not set, the result of the first statement is returned. +**adbc.bigquery.multiple_statement.evaluation_kind**
+    Optional. When executing multiple statements, limit the evaluation kind returned. If not set, all evaluation kinds are returned. **adbc.bigquery.include_constraints_getobjects**
    Optional. Some callers do not need the constraint details when they get the table information and can improve the speed of obtaining the results. Setting this value to `"false"` will not include the constraint details. The default value is `"true"`. diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs index efb7c46513..9069504c4c 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs @@ -101,7 +101,7 @@ public BigQueryTestEnvironment() public string StatementType { get; set; } = string.Empty; [JsonPropertyName("statementIndex")] - public int? StatementIndex { get; set; } + public string EvaluationKind { get; set; } = string.Empty; /// /// How structs should be handled by the ADO.NET client for this environment. diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs index 2bf1249f64..da44f35869 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs @@ -117,12 +117,12 @@ internal static Dictionary GetBigQueryParameters(BigQueryTestEnv if (!string.IsNullOrEmpty(testEnvironment.StatementType)) { - parameters.Add(BigQueryParameters.StatementType, testEnvironment.StatementType!); + parameters.Add(BigQueryParameters.StatementType, testEnvironment.StatementType); } - if (testEnvironment.StatementIndex.HasValue) + if (!string.IsNullOrEmpty(testEnvironment.EvaluationKind)) { - parameters.Add(BigQueryParameters.StatementIndex, testEnvironment.StatementIndex.Value.ToString()); + parameters.Add(BigQueryParameters.EvaluationKind, testEnvironment.EvaluationKind); } return parameters; diff --git a/csharp/test/Drivers/BigQuery/readme.md b/csharp/test/Drivers/BigQuery/readme.md index c88fbad72c..1503a01377 100644 --- a/csharp/test/Drivers/BigQuery/readme.md +++ b/csharp/test/Drivers/BigQuery/readme.md @@ -47,7 +47,7 @@ The following values can be setup in the configuration - **scopes** - Comma separated list (string) of scopes applied during the test. - **queryTimeout** - The timeout (in seconds) for a query. Similar to a CommandTimeout. - **statementType** - When executing multiple statements, limit the type of statement returned. - - **statementIndex** - When executing multiple statements, specify the result of the statement to be returned. + - **evaluationKind** - When executing multiple statements, limit the evaluation kind returned. - **includeTableConstraints** - Whether to include table constraints in the GetObjects query. - **largeResultsDestinationTable** - Sets the [DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable) value of the QueryOptions if configured. Expects the format to be `{projectId}.{datasetId}.{tableId}` to set the corresponding values in the [TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76faf97/Src/Generated/Google.Apis.Bigquery.v2/Google.Apis.Bigquery.v2.cs#L9348) class. - **allowLargeResults** - Whether to allow large results . From f18dbb76eb4bb5a1cea1bcd625e9cb9e100b76f9 Mon Sep 17 00:00:00 2001 From: Qifan Zhang Date: Mon, 14 Apr 2025 16:42:38 +0800 Subject: [PATCH 3/8] add test --- .../BigQuery/BigQueryTestConfiguration.cs | 2 +- csharp/test/Drivers/BigQuery/DriverTests.cs | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs index 9069504c4c..fce8cd6061 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs @@ -100,7 +100,7 @@ public BigQueryTestEnvironment() [JsonPropertyName("statementType")] public string StatementType { get; set; } = string.Empty; - [JsonPropertyName("statementIndex")] + [JsonPropertyName("evaluationKind")] public string EvaluationKind { get; set; } = string.Empty; /// diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 12f061a6ac..184bf0fc88 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -279,6 +279,26 @@ public void CanGetTableTypes() } } + /// + /// Validates if the driver can connect to a live server and + /// parse the results of multi-statements. + /// + [SkippableFact, Order(6)] + public void CanExecuteMultiStatements() + { + foreach (BigQueryTestEnvironment environment in _environments) + { + AdbcConnection adbcConnection = GetAdbcConnection(environment.Name); + + AdbcStatement statement = adbcConnection.CreateStatement(); + statement.SqlQuery = "select 1;select 2;"; + + QueryResult queryResult = statement.ExecuteQuery(); + + Tests.DriverTests.CanExecuteQuery(queryResult, environment.ExpectedResultsCount, environment.Name); + } + } + /// /// Validates if the driver can connect to a live server and /// parse the results. From 8cac9e66598ff12c1ed2abaff91e2dda21f15d50 Mon Sep 17 00:00:00 2001 From: Qifan Zhang Date: Mon, 14 Apr 2025 17:02:15 +0800 Subject: [PATCH 4/8] fix the test --- csharp/test/Drivers/BigQuery/DriverTests.cs | 40 ++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 184bf0fc88..1acb7d6d4e 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -279,26 +279,6 @@ public void CanGetTableTypes() } } - /// - /// Validates if the driver can connect to a live server and - /// parse the results of multi-statements. - /// - [SkippableFact, Order(6)] - public void CanExecuteMultiStatements() - { - foreach (BigQueryTestEnvironment environment in _environments) - { - AdbcConnection adbcConnection = GetAdbcConnection(environment.Name); - - AdbcStatement statement = adbcConnection.CreateStatement(); - statement.SqlQuery = "select 1;select 2;"; - - QueryResult queryResult = statement.ExecuteQuery(); - - Tests.DriverTests.CanExecuteQuery(queryResult, environment.ExpectedResultsCount, environment.Name); - } - } - /// /// Validates if the driver can connect to a live server and /// parse the results. @@ -366,5 +346,25 @@ public void QueryTimeoutTest() } } } + + /// + /// Validates if the driver can connect to a live server and + /// parse the results of multi-statements. + /// + [SkippableFact, Order(9)] + public void CanExecuteMultiStatements() + { + foreach (BigQueryTestEnvironment environment in _environments) + { + AdbcConnection adbcConnection = GetAdbcConnection(environment.Name); + + AdbcStatement statement = adbcConnection.CreateStatement(); + statement.SqlQuery = "select 1;select 2;"; + + QueryResult queryResult = statement.ExecuteQuery(); + + Tests.DriverTests.CanExecuteQuery(queryResult, 1, environment.Name); + } + } } } From 26844727c245452fa97e441efa6354f7bca08849 Mon Sep 17 00:00:00 2001 From: Qifan Zhang Date: Tue, 15 Apr 2025 17:02:55 +0800 Subject: [PATCH 5/8] add test --- csharp/test/Drivers/BigQuery/DriverTests.cs | 28 +++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 1acb7d6d4e..03d31df4b9 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -352,17 +352,35 @@ public void QueryTimeoutTest() /// parse the results of multi-statements. /// [SkippableFact, Order(9)] - public void CanExecuteMultiStatements() + public void CanExecuteMultiStatementQuery() { foreach (BigQueryTestEnvironment environment in _environments) { AdbcConnection adbcConnection = GetAdbcConnection(environment.Name); - AdbcStatement statement = adbcConnection.CreateStatement(); - statement.SqlQuery = "select 1;select 2;"; - + string query1 = "SELECT " + + "CAST(1 as INT64) as id, " + + "CAST(1.23 as FLOAT64) as number, " + + "PARSE_NUMERIC(\"4.56\") as decimal, " + + "PARSE_BIGNUMERIC(\"7.89000000000000000000000000000000000000\") as big_decimal, " + + "CAST(True as BOOL) as is_active, " + + "'John Doe' as name, " + + "FROM_BASE64('YWJjMTIz') as data, " + + "CAST('2023-09-08' as DATE) as date, " + + "CAST('12:34:56' as TIME) as time, " + + "CAST('2023-09-08 12:34:56' as DATETIME) as datetime, " + + "CAST('2023-09-08 12:34:56+00:00' as TIMESTAMP) as timestamp, " + + "ST_GEOGPOINT(1, 2) as point, " + + "ARRAY[1, 2, 3] as numbers, " + + "STRUCT('John Doe' as name, 30 as age) as person," + + "PARSE_JSON('{\"name\":\"Jane Doe\",\"age\":29}') as json"; + string query2 = "SELECT " + + "CAST(1.7976931348623157e+308 as FLOAT64) as number, " + + "PARSE_NUMERIC(\"9.99999999999999999999999999999999E+28\") as decimal, " + + "PARSE_BIGNUMERIC(\"5.7896044618658097711785492504343953926634992332820282019728792003956564819968E+37\") as big_decimal"; + string combinedQuery = query1 + ";" + query2 + ";"; + statement.SqlQuery = combinedQuery; QueryResult queryResult = statement.ExecuteQuery(); - Tests.DriverTests.CanExecuteQuery(queryResult, 1, environment.Name); } } From 1bdcafc862105b86a2a88361f481b0c38f10d799 Mon Sep 17 00:00:00 2001 From: Qifan Zhang Date: Fri, 18 Apr 2025 16:27:34 +0800 Subject: [PATCH 6/8] Add statementIndex --- csharp/src/Drivers/BigQuery/BigQueryConnection.cs | 1 + csharp/src/Drivers/BigQuery/BigQueryParameters.cs | 1 + csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 13 ++++++++++++- csharp/src/Drivers/BigQuery/readme.md | 3 +++ .../Drivers/BigQuery/BigQueryTestConfiguration.cs | 3 +++ .../test/Drivers/BigQuery/BigQueryTestingUtils.cs | 5 +++++ csharp/test/Drivers/BigQuery/readme.md | 1 + 7 files changed, 26 insertions(+), 1 deletion(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 0062f2d6ec..0e84e69598 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -1060,6 +1060,7 @@ private IReadOnlyDictionary ParseOptions() BigQueryParameters.GetQueryResultsOptionsTimeout, BigQueryParameters.MaxFetchConcurrency, BigQueryParameters.StatementType, + BigQueryParameters.StatementIndex, BigQueryParameters.EvaluationKind }; diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs index 2fb64da7fa..02873c2f1e 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs @@ -40,6 +40,7 @@ public class BigQueryParameters public const string MaxFetchConcurrency = "adbc.bigquery.max_fetch_concurrency"; public const string IncludePublicProjectId = "adbc.bigquery.include_public_project_id"; public const string StatementType = "adbc.bigquery.multiple_statement.statement_type"; + public const string StatementIndex = "adbc.bigquery.multiple_statement.statement_index"; public const string EvaluationKind = "adbc.bigquery.multiple_statement.evaluation_kind"; } diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 5f96a30994..578f2eef74 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -73,6 +73,13 @@ public override QueryResult ExecuteQuery() { statementType = statementTypeString; } + int statementIndex = 1; + if (this.Options?.TryGetValue(BigQueryParameters.StatementIndex, out string? statementIndexString) == true && + int.TryParse(statementIndexString, out int statementIndexInt) && + statementIndexInt > 0) + { + statementIndex = statementIndexInt; + } string evaluationKind = string.Empty; if (this.Options?.TryGetValue(BigQueryParameters.EvaluationKind, out string? evaluationKindString) == true) { @@ -92,7 +99,11 @@ public override QueryResult ExecuteQuery() if (joblist.Count > 0) { - results = joblist[0].GetQueryResults(getQueryResultsOptions); + if (statementIndex < 1 || statementIndex > joblist.Count) + { + throw new ArgumentOutOfRangeException($"The specified index {statementIndex} is out of range. There are {joblist.Count} jobs available."); + } + results = joblist[statementIndex - 1].GetQueryResults(getQueryResultsOptions); } } if (results.TableReference == null) diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md index 94c93090c1..6b733ca2fd 100644 --- a/csharp/src/Drivers/BigQuery/readme.md +++ b/csharp/src/Drivers/BigQuery/readme.md @@ -66,6 +66,9 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G **adbc.bigquery.multiple_statement.statement_type**
    Optional. When executing multiple statements, limit the type of statement returned. If not set, all types of statements are returned. +**adbc.bigquery.multiple_statement.statement_index**
+    Optional. When executing multiple statements, specify the result of the statement to be returned (Minimum value is 1). If not set, the result of the first statement is returned. + **adbc.bigquery.multiple_statement.evaluation_kind**
    Optional. When executing multiple statements, limit the evaluation kind returned. If not set, all evaluation kinds are returned. diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs index fce8cd6061..8b36e1db81 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs @@ -100,6 +100,9 @@ public BigQueryTestEnvironment() [JsonPropertyName("statementType")] public string StatementType { get; set; } = string.Empty; + [JsonPropertyName("statementIndex")] + public int? StatementIndex { get; set; } + [JsonPropertyName("evaluationKind")] public string EvaluationKind { get; set; } = string.Empty; diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs index da44f35869..07e1876b25 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs @@ -120,6 +120,11 @@ internal static Dictionary GetBigQueryParameters(BigQueryTestEnv parameters.Add(BigQueryParameters.StatementType, testEnvironment.StatementType); } + if (testEnvironment.StatementIndex.HasValue) + { + parameters.Add(BigQueryParameters.StatementIndex, testEnvironment.StatementIndex.Value.ToString()); + } + if (!string.IsNullOrEmpty(testEnvironment.EvaluationKind)) { parameters.Add(BigQueryParameters.EvaluationKind, testEnvironment.EvaluationKind); diff --git a/csharp/test/Drivers/BigQuery/readme.md b/csharp/test/Drivers/BigQuery/readme.md index 1503a01377..346b0313df 100644 --- a/csharp/test/Drivers/BigQuery/readme.md +++ b/csharp/test/Drivers/BigQuery/readme.md @@ -47,6 +47,7 @@ The following values can be setup in the configuration - **scopes** - Comma separated list (string) of scopes applied during the test. - **queryTimeout** - The timeout (in seconds) for a query. Similar to a CommandTimeout. - **statementType** - When executing multiple statements, limit the type of statement returned. + - **statementIndex** - When executing multiple statements, specify the result of the statement to be returned. - **evaluationKind** - When executing multiple statements, limit the evaluation kind returned. - **includeTableConstraints** - Whether to include table constraints in the GetObjects query. - **largeResultsDestinationTable** - Sets the [DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable) value of the QueryOptions if configured. Expects the format to be `{projectId}.{datasetId}.{tableId}` to set the corresponding values in the [TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76faf97/Src/Generated/Google.Apis.Bigquery.v2/Google.Apis.Bigquery.v2.cs#L9348) class. From 56af64701dd87e7dc5612a370a13aa09db746b0c Mon Sep 17 00:00:00 2001 From: Qifan Zhang Date: Fri, 18 Apr 2025 16:56:16 +0800 Subject: [PATCH 7/8] change the test --- csharp/test/Drivers/BigQuery/DriverTests.cs | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 03d31df4b9..15dafc0ec8 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -358,22 +358,7 @@ public void CanExecuteMultiStatementQuery() { AdbcConnection adbcConnection = GetAdbcConnection(environment.Name); AdbcStatement statement = adbcConnection.CreateStatement(); - string query1 = "SELECT " + - "CAST(1 as INT64) as id, " + - "CAST(1.23 as FLOAT64) as number, " + - "PARSE_NUMERIC(\"4.56\") as decimal, " + - "PARSE_BIGNUMERIC(\"7.89000000000000000000000000000000000000\") as big_decimal, " + - "CAST(True as BOOL) as is_active, " + - "'John Doe' as name, " + - "FROM_BASE64('YWJjMTIz') as data, " + - "CAST('2023-09-08' as DATE) as date, " + - "CAST('12:34:56' as TIME) as time, " + - "CAST('2023-09-08 12:34:56' as DATETIME) as datetime, " + - "CAST('2023-09-08 12:34:56+00:00' as TIMESTAMP) as timestamp, " + - "ST_GEOGPOINT(1, 2) as point, " + - "ARRAY[1, 2, 3] as numbers, " + - "STRUCT('John Doe' as name, 30 as age) as person," + - "PARSE_JSON('{\"name\":\"Jane Doe\",\"age\":29}') as json"; + string query1 = "SELECT * FROM bigquery-public-data.covid19_ecdc.covid_19_geographic_distribution_worldwide"; string query2 = "SELECT " + "CAST(1.7976931348623157e+308 as FLOAT64) as number, " + "PARSE_NUMERIC(\"9.99999999999999999999999999999999E+28\") as decimal, " + @@ -381,7 +366,7 @@ public void CanExecuteMultiStatementQuery() string combinedQuery = query1 + ";" + query2 + ";"; statement.SqlQuery = combinedQuery; QueryResult queryResult = statement.ExecuteQuery(); - Tests.DriverTests.CanExecuteQuery(queryResult, 1, environment.Name); + Tests.DriverTests.CanExecuteQuery(queryResult, 61900, environment.Name); } } } From d6bc961e61f61f126c515a65b17883af1dd2737b Mon Sep 17 00:00:00 2001 From: Qifan Zhang Date: Fri, 18 Apr 2025 17:01:16 +0800 Subject: [PATCH 8/8] fix --- csharp/test/Drivers/BigQuery/readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/csharp/test/Drivers/BigQuery/readme.md b/csharp/test/Drivers/BigQuery/readme.md index 346b0313df..fdd1e18ac0 100644 --- a/csharp/test/Drivers/BigQuery/readme.md +++ b/csharp/test/Drivers/BigQuery/readme.md @@ -46,6 +46,7 @@ The following values can be setup in the configuration - **includePublicProjectId** - True/False to indicate if the public projects should be included in the result set. - **scopes** - Comma separated list (string) of scopes applied during the test. - **queryTimeout** - The timeout (in seconds) for a query. Similar to a CommandTimeout. + - **maxStreamCount** - The max stream count. - **statementType** - When executing multiple statements, limit the type of statement returned. - **statementIndex** - When executing multiple statements, specify the result of the statement to be returned. - **evaluationKind** - When executing multiple statements, limit the evaluation kind returned.