Skip to content

Commit 805b68c

Browse files
AbacnAbacn
authored
Merge pull request #35718 from Abacn/use-vendor-calcite
Use vendored calcite 1.40
2 parents 0331f20 + 377dff5 commit 805b68c

192 files changed

Lines changed: 1351 additions & 1372 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/trigger_files/beam_PostCommit_SQL.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run ",
3-
"modification": 2
3+
"modification": 3
44
}

‎.github/trigger_files/beam_PostCommit_XVR_Direct.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
{
2-
"https://github.com/apache/beam/pull/32648": "testing Flink 1.19 support",
3-
"modification": 3
2+
"modification": 4
43
}

‎CHANGES.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,11 @@
7777
## Breaking Changes
7878

7979
* X behavior was changed ([#X](https://github.com/apache/beam/issues/X)).
80-
* Previously deprecated Beam ZetaSQL component has been removed from new releases ([#34423](https://github.com/apache/beam/issues/34423)).
80+
* Previously deprecated Beam ZetaSQL component has been removed ([#34423](https://github.com/apache/beam/issues/34423)).
81+
ZetaSQL users could migrate to Calcite SQL with BigQuery dialect enabled.
82+
* Upgraded Beam vendored Calcite to 1.40.0 for Beam SQL ([#35483](https://github.com/apache/beam/issues/35483)), which
83+
improves support for BigQuery and other SQL dialects. Note: Minor behavior changes are observed such as output
84+
significant digits related to casting.
8185

8286
## Deprecations
8387

‎buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -907,7 +907,7 @@ class BeamModulePlugin implements Plugin<Project> {
907907
threetenbp : "org.threeten:threetenbp:1.6.8",
908908
vendored_grpc_1_69_0 : "org.apache.beam:beam-vendor-grpc-1_69_0:0.1",
909909
vendored_guava_32_1_2_jre : "org.apache.beam:beam-vendor-guava-32_1_2-jre:0.1",
910-
vendored_calcite_1_28_0 : "org.apache.beam:beam-vendor-calcite-1_28_0:0.2",
910+
vendored_calcite_1_40_0 : "org.apache.beam:beam-vendor-calcite-1_40_0:0.1",
911911
woodstox_core_asl : "org.codehaus.woodstox:woodstox-core-asl:4.4.1",
912912
zstd_jni : "com.github.luben:zstd-jni:1.5.6-3",
913913
quickcheck_core : "com.pholser:junit-quickcheck-core:$quickcheck_version",
@@ -1832,21 +1832,20 @@ class BeamModulePlugin implements Plugin<Project> {
18321832
project.ext.includeInJavaBom = configuration.publish
18331833
project.ext.exportJavadoc = configuration.exportJavadoc
18341834

1835-
boolean publishEnabledByCommand = isRelease(project) || project.hasProperty('publishing')
18361835
if (forkJavaVersion == '') {
18371836
// project needs newer version and not served.
18381837
// If not publishing ,disable the project. Otherwise, fail the build
18391838
def msg = "project ${project.name} needs newer Java version to compile. Consider set -Pjava${project.javaVersion}Home"
1840-
if (publishEnabledByCommand) {
1841-
throw new GradleException("Publish enabled but " + msg + ".")
1839+
if (isRelease(project)) {
1840+
throw new GradleException("Release enabled but " + msg + ".")
18421841
} else {
18431842
logger.config(msg + " if needed.")
18441843
project.tasks.each {
18451844
it.enabled = false
18461845
}
18471846
}
18481847
}
1849-
if (publishEnabledByCommand && configuration.publish) {
1848+
if ((isRelease(project) || project.hasProperty('publishing')) && configuration.publish) {
18501849
project.apply plugin: "maven-publish"
18511850

18521851
// plugin to support repository authentication via ~/.m2/settings.xml

‎sdks/java/extensions/sql/build.gradle‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ dependencies {
7272
javacc "net.java.dev.javacc:javacc:4.0"
7373
fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0"
7474
fmppTask "org.freemarker:freemarker:2.3.31"
75-
fmppTemplates library.java.vendored_calcite_1_28_0
75+
fmppTemplates library.java.vendored_calcite_1_40_0
7676
implementation project(path: ":sdks:java:core", configuration: "shadow")
7777
implementation project(":sdks:java:managed")
7878
implementation project(":sdks:java:io:iceberg")
@@ -87,7 +87,7 @@ dependencies {
8787
implementation library.java.commons_csv
8888
implementation library.java.jackson_databind
8989
implementation library.java.joda_time
90-
implementation library.java.vendored_calcite_1_28_0
90+
implementation library.java.vendored_calcite_1_40_0
9191
implementation "org.codehaus.janino:janino:3.0.11"
9292
implementation "org.codehaus.janino:commons-compiler:3.0.11"
9393
implementation library.java.jackson_core
@@ -117,7 +117,7 @@ dependencies {
117117

118118
testImplementation "org.apache.iceberg:iceberg-api:1.6.1"
119119
testImplementation "org.apache.iceberg:iceberg-core:1.6.1"
120-
testImplementation library.java.vendored_calcite_1_28_0
120+
testImplementation library.java.vendored_calcite_1_40_0
121121
testImplementation library.java.vendored_guava_32_1_2_jre
122122
testImplementation library.java.junit
123123
testImplementation library.java.quickcheck_core
@@ -164,11 +164,15 @@ task copyFmppTemplatesFromCalciteCore(type: Copy) {
164164
into "${project.buildDir}/templates-fmpp"
165165
filter{
166166
line ->
167-
line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.')
167+
line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.')
168168
}
169169
filter{
170170
line ->
171-
line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.')
171+
line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.')
172+
}
173+
filter{
174+
line ->
175+
line.replace('import com.google.common.', 'import org.apache.beam.vendor.calcite.v1_40_0.com.google.common.')
172176
}
173177
}
174178

‎sdks/java/extensions/sql/jdbc/build.gradle‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ dependencies {
3838
implementation "jline:jline:2.14.6"
3939
permitUnusedDeclared "jline:jline:2.14.6" // BEAM-11761
4040
implementation "sqlline:sqlline:1.4.0"
41-
implementation library.java.vendored_calcite_1_28_0
42-
permitUnusedDeclared library.java.vendored_calcite_1_28_0
41+
implementation library.java.vendored_calcite_1_40_0
42+
permitUnusedDeclared library.java.vendored_calcite_1_40_0
4343
testImplementation project(path: ":sdks:java:core", configuration: "shadow")
4444
testImplementation project(path: ":sdks:java:io:google-cloud-platform")
4545
testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core")

‎sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineIT.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -129,9 +129,9 @@ public void testSelectFromPubsub() throws Exception {
129129

130130
assertThat(
131131
Arrays.asList(
132-
Arrays.asList("2018-07-01 21:25:20", "enroute", "40.702", "-74.001"),
133-
Arrays.asList("2018-07-01 21:26:06", "enroute", "40.703", "-74.002"),
134-
Arrays.asList("2018-07-02 13:26:06", "enroute", "30.0", "-72.32324")),
132+
Arrays.asList("2018-07-01 21:25:20.000000", "enroute", "40.702", "-74.001"),
133+
Arrays.asList("2018-07-01 21:26:06.000000", "enroute", "40.703", "-74.002"),
134+
Arrays.asList("2018-07-02 13:26:06.000000", "enroute", "30.0", "-72.32324")),
135135
everyItem(IsIn.isOneOf(expectedResult.get(30, TimeUnit.SECONDS).toArray())));
136136
}
137137

@@ -170,8 +170,8 @@ public void testFilterForSouthManhattan() throws Exception {
170170

171171
assertThat(
172172
Arrays.asList(
173-
Arrays.asList("2018-07-01 21:25:20", "enroute", "40.701", "-74.001"),
174-
Arrays.asList("2018-07-01 21:26:06", "enroute", "40.702", "-74.002")),
173+
Arrays.asList("2018-07-01 21:25:20.000000", "enroute", "40.701", "-74.001"),
174+
Arrays.asList("2018-07-01 21:26:06.000000", "enroute", "40.702", "-74.002")),
175175
everyItem(IsIn.isOneOf(expectedResult.get(30, TimeUnit.SECONDS).toArray())));
176176
}
177177

‎sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTest.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,8 @@ public void testSqlLine_fixedWindow() throws Exception {
168168
List<List<String>> lines = toLines(byteArrayOutputStream);
169169
assertThat(
170170
Arrays.asList(
171-
Arrays.asList("2018-07-01 21:26:06", "1"), Arrays.asList("2018-07-01 21:26:07", "1")),
171+
Arrays.asList("2018-07-01 21:26:06.000000", "1"),
172+
Arrays.asList("2018-07-01 21:26:07.000000", "1")),
172173
everyItem(is(oneOf(lines.toArray()))));
173174
}
174175

@@ -190,11 +191,11 @@ public void testSqlLine_slidingWindow() throws Exception {
190191
List<List<String>> lines = toLines(byteArrayOutputStream);
191192
assertThat(
192193
Arrays.asList(
193-
Arrays.asList("2018-07-01 21:26:07", "1"),
194-
Arrays.asList("2018-07-01 21:26:08", "2"),
195-
Arrays.asList("2018-07-01 21:26:09", "2"),
196-
Arrays.asList("2018-07-01 21:26:10", "2"),
197-
Arrays.asList("2018-07-01 21:26:11", "1")),
194+
Arrays.asList("2018-07-01 21:26:07.000000", "1"),
195+
Arrays.asList("2018-07-01 21:26:08.000000", "2"),
196+
Arrays.asList("2018-07-01 21:26:09.000000", "2"),
197+
Arrays.asList("2018-07-01 21:26:10.000000", "2"),
198+
Arrays.asList("2018-07-01 21:26:11.000000", "1")),
198199
everyItem(is(oneOf(lines.toArray()))));
199200
}
200201
}

‎sdks/java/extensions/sql/perf-tests/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryIOPushDownIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@
4545
import org.apache.beam.sdk.testutils.metrics.TimeMonitor;
4646
import org.apache.beam.sdk.testutils.publishing.InfluxDBSettings;
4747
import org.apache.beam.sdk.transforms.ParDo;
48-
import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptRule;
49-
import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet;
50-
import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets;
48+
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.plan.RelOptRule;
49+
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.tools.RuleSet;
50+
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.tools.RuleSets;
5151
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
5252
import org.junit.Before;
5353
import org.junit.BeforeClass;

‎sdks/java/extensions/sql/src/main/codegen/config.fmpp‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,11 @@ data: {
2121

2222
# List of import statements.
2323
imports: [
24-
"org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.ColumnStrategy"
25-
"org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlCreate"
26-
"org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlDrop"
27-
"org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.type.SqlTypeName"
24+
"org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.schema.ColumnStrategy"
25+
"org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.sql.SqlCreate"
26+
"org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.sql.SqlDrop"
27+
"org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.sql.type.SqlTypeName"
28+
"org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.util.NlsString"
2829
"org.apache.beam.sdk.extensions.sql.impl.parser.SqlCreateCatalog"
2930
"org.apache.beam.sdk.extensions.sql.impl.parser.SqlCreateDatabase"
3031
"org.apache.beam.sdk.extensions.sql.impl.parser.SqlCreateExternalTable"
@@ -64,6 +65,8 @@ data: {
6465
"AFTER"
6566
"ALWAYS"
6667
"APPLY"
68+
"ARRAY_AGG"
69+
"ARRAY_CONCAT_AGG"
6770
"ASC"
6871
"ASSERTION"
6972
"ASSIGNMENT"
@@ -102,12 +105,19 @@ data: {
102105
"CONSTRAINTS"
103106
"CONSTRAINT_SCHEMA"
104107
"CONSTRUCTOR"
108+
"CONTAINS_SUBSTR"
105109
"CONTINUE"
106110
"CURSOR_NAME"
107111
"DATA"
108112
"DATABASE"
113+
"DATE_DIFF"
114+
"DATE_TRUNC"
115+
"DATETIME_DIFF"
109116
"DATETIME_INTERVAL_CODE"
110117
"DATETIME_INTERVAL_PRECISION"
118+
"DATETIME_TRUNC"
119+
"DAYOFWEEK"
120+
"DAYOFYEAR"
111121
"DAYS"
112122
"DECADE"
113123
"DEFAULTS"
@@ -126,6 +136,7 @@ data: {
126136
"DOMAIN"
127137
"DOW"
128138
"DOY"
139+
"DOT_FORMAT"
129140
"DYNAMIC_FUNCTION"
130141
"DYNAMIC_FUNCTION_CODE"
131142
"ENCODING"
@@ -148,13 +159,16 @@ data: {
148159
"GO"
149160
"GOTO"
150161
"GRANTED"
162+
"GROUP_CONCAT"
151163
"HIERARCHY"
152164
"HOP"
153165
"HOURS"
154166
"IGNORE"
167+
"ILIKE"
155168
"IMMEDIATE"
156169
"IMMEDIATELY"
157170
"IMPLEMENTATION"
171+
"INCLUDE"
158172
"INCLUDING"
159173
"INCREMENT"
160174
"INITIALLY"
@@ -222,6 +236,7 @@ data: {
222236
"PASSTHROUGH"
223237
"PAST"
224238
"PATH"
239+
"PIVOT"
225240
"PLACING"
226241
"PLAN"
227242
"PLI"
@@ -231,6 +246,7 @@ data: {
231246
"PRIVILEGES"
232247
"PUBLIC"
233248
"QUARTER"
249+
"QUARTERS"
234250
"READ"
235251
"RELATIVE"
236252
"REPEATABLE"
@@ -243,6 +259,7 @@ data: {
243259
"RETURNED_OCTET_LENGTH"
244260
"RETURNED_SQLSTATE"
245261
"RETURNING"
262+
"RLIKE"
246263
"ROLE"
247264
"ROUTINE"
248265
"ROUTINE_CATALOG"
@@ -260,6 +277,7 @@ data: {
260277
"SECTION"
261278
"SECURITY"
262279
"SELF"
280+
"SEPARATOR"
263281
"SEQUENCE"
264282
"SERIALIZABLE"
265283
"SERVER"
@@ -322,15 +340,20 @@ data: {
322340
"SQL_VARCHAR"
323341
"STATE"
324342
"STATEMENT"
343+
"STRING_AGG"
325344
"STRUCTURE"
326345
"STYLE"
327346
"SUBCLASS_ORIGIN"
328347
"SUBSTITUTE"
329348
"TABLE_NAME"
330349
"TEMPORARY"
331350
"TIES"
351+
"TIME_DIFF"
352+
"TIME_TRUNC"
332353
"TIMESTAMPADD"
333354
"TIMESTAMPDIFF"
355+
"TIMESTAMP_DIFF"
356+
"TIMESTAMP_TRUNC"
334357
"TOP_LEVEL_COUNT"
335358
"TRANSACTION"
336359
"TRANSACTIONS_ACTIVE"
@@ -347,6 +370,7 @@ data: {
347370
"UNCOMMITTED"
348371
"UNCONDITIONAL"
349372
"UNDER"
373+
"UNPIVOT"
350374
"UNNAMED"
351375
"USAGE"
352376
"USER_DEFINED_TYPE_CATALOG"
@@ -359,6 +383,7 @@ data: {
359383
"VERSION"
360384
"VIEW"
361385
"WEEK"
386+
"WEEKS"
362387
"WORK"
363388
"WRAPPER"
364389
"WRITE"
@@ -445,6 +470,12 @@ data: {
445470
"SqlDropCatalog"
446471
]
447472

473+
# List of methods for parsing extensions to "TRUNCATE" calls.
474+
# Each must accept arguments "(SqlParserPos pos)".
475+
# Example: "SqlTruncate".
476+
truncateStatementParserMethods: [
477+
]
478+
448479
# Binary operators tokens
449480
binaryOperatorsTokens: [
450481
]
@@ -461,11 +492,12 @@ data: {
461492
"parserImpls.ftl"
462493
]
463494

495+
setOptionParserMethod: "SqlSetOption"
464496
includePosixOperators: false
465497
includeCompoundIdentifier: true
466498
includeBraces: true
467499
includeAdditionalDeclarations: false
468-
500+
includeParsingStringLiteralAsArrayLiteral: false
469501
}
470502
}
471503

0 commit comments

Comments
 (0)