From a7b3681eaf7e4226ef5af4dead7cfbe7d8bec2f0 Mon Sep 17 00:00:00 2001 From: rockyyin Date: Sat, 30 May 2026 13:25:55 +0800 Subject: [PATCH] feat: introduce multi-version Flink module structure (1.18/1.19/1.20) Convert single-module project to multi-module Maven project, supporting 3 actively-supported Flink versions (1.18.1 / 1.19.1 / 1.20.0) sharing one source tree under root src/. Module structure: - Parent pom uses for version harmonization - Each lance-flink-1.x submodule references root src/ via build-helper-maven-plugin (add-source / add-test-source / add-resource / add-test-resource) - All modules compile with Java 8 target Plugin versions kept aligned with main (PR #39): - maven-compiler-plugin 3.15.0 - maven-shade-plugin 3.6.2 - maven-jar-plugin 3.5.0 - maven-source-plugin 3.4.0 - jacoco-maven-plugin 0.8.14 Why 1.17 is excluded: - Flink 1.17 reached end-of-life on 2024-10 - Flink 1.17.x ships an internal shaded Calcite that has known bugs on JDK 8 (MetadataDef assertion + safeArgList IndexOutOfBounds in CacheGeneratorUtil), causing TableEnvironment creation to fail. These are Flink-side issues, unrelated to this connector - Supporting an EOL version with upstream-broken planner is not worth the maintenance cost Notes: - Sliced from feature/flink-multi-version-modules but excludes the 4 PR #14 namespace catalog commits (those will land via their own PR) - Removes unused TableSchema import in LanceDynamicTableSource - Verified locally on JDK 8 (Tencent Kona): mvn validate / compile / test-compile / package -> BUILD SUCCESS mvn test -> 178 run, 1 failure, 0 errors, 17 skipped (per module) The single failure (LanceReadOptimizationsTest$FilterPushDown Tests.testInPredicatePushDown) is a pre-existing main bug that reproduces identically across 1.18 / 1.19 / 1.20 Closes #17 in favor of this clean slice. --- lance-flink-1.18/pom.xml | 234 ++++++ .../src/test/resources/log4j2-test.xml | 15 + lance-flink-1.19/pom.xml | 233 ++++++ .../src/test/resources/log4j2-test.xml | 15 + lance-flink-1.20/pom.xml | 233 ++++++ .../src/test/resources/log4j2-test.xml | 15 + pom.xml | 714 ++++++++++-------- .../lance/table/LanceDynamicTableSource.java | 1 - 8 files changed, 1148 insertions(+), 312 deletions(-) create mode 100644 lance-flink-1.18/pom.xml create mode 100644 lance-flink-1.18/src/test/resources/log4j2-test.xml create mode 100644 lance-flink-1.19/pom.xml create mode 100644 lance-flink-1.19/src/test/resources/log4j2-test.xml create mode 100644 lance-flink-1.20/pom.xml create mode 100644 lance-flink-1.20/src/test/resources/log4j2-test.xml diff --git a/lance-flink-1.18/pom.xml b/lance-flink-1.18/pom.xml new file mode 100644 index 0000000..d0c48bc --- /dev/null +++ b/lance-flink-1.18/pom.xml @@ -0,0 +1,234 @@ + + + 4.0.0 + + + org.apache.flink + lance-flink-root + 1.0.0-SNAPSHOT + ../pom.xml + + + lance-flink-1.18 + Lance Flink Connector - Flink 1.18 + Lance Flink Connector for Apache Flink 1.18 + jar + + + ${flink118.version} + + + + + + com.lancedb + lance-core + + + + + org.apache.arrow + arrow-vector + + + + org.apache.arrow + arrow-memory-netty + + + + org.apache.arrow + arrow-c-data + + + + + org.apache.flink + flink-streaming-java + + + + org.apache.flink + flink-table-api-java + + + + org.apache.flink + flink-table-api-java-bridge + + + + org.apache.flink + flink-table-common + + + + org.apache.flink + flink-connector-base + + + + + org.slf4j + slf4j-api + + + + + org.junit.jupiter + junit-jupiter-api + + + + org.junit.jupiter + junit-jupiter-engine + + + + org.junit.jupiter + junit-jupiter-params + + + + org.apache.flink + flink-test-utils + + + + org.apache.flink + flink-runtime + test-jar + + + + org.apache.flink + flink-streaming-java + test-jar + + + + org.apache.flink + flink-table-planner_2.12 + + + + org.mockito + mockito-core + + + + org.mockito + mockito-junit-jupiter + + + + org.assertj + assertj-core + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.flink + flink-clients + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + ${project.basedir}/../src/main/java + + + + + add-test-source + generate-test-sources + + add-test-source + + + + ${project.basedir}/../src/test/java + + + + + add-resource + generate-resources + + add-resource + + + + + ${project.basedir}/../src/main/resources + + + + + + add-test-resource + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/../src/test/resources + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + org.jacoco + jacoco-maven-plugin + + + + diff --git a/lance-flink-1.18/src/test/resources/log4j2-test.xml b/lance-flink-1.18/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000..c73119e --- /dev/null +++ b/lance-flink-1.18/src/test/resources/log4j2-test.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/lance-flink-1.19/pom.xml b/lance-flink-1.19/pom.xml new file mode 100644 index 0000000..ec5e146 --- /dev/null +++ b/lance-flink-1.19/pom.xml @@ -0,0 +1,233 @@ + + + 4.0.0 + + + org.apache.flink + lance-flink-root + 1.0.0-SNAPSHOT + ../pom.xml + + + lance-flink-1.19 + Lance Flink Connector - Flink 1.19 + Lance Flink Connector for Apache Flink 1.19 + jar + + + ${flink119.version} + + + + + + com.lancedb + lance-core + + + + + org.apache.arrow + arrow-vector + + + + org.apache.arrow + arrow-memory-netty + + + + org.apache.arrow + arrow-c-data + + + + + org.apache.flink + flink-streaming-java + + + + org.apache.flink + flink-table-api-java + + + + org.apache.flink + flink-table-api-java-bridge + + + + org.apache.flink + flink-table-common + + + + org.apache.flink + flink-connector-base + + + + + org.slf4j + slf4j-api + + + + + org.junit.jupiter + junit-jupiter-api + + + + org.junit.jupiter + junit-jupiter-engine + + + + org.junit.jupiter + junit-jupiter-params + + + + org.apache.flink + flink-test-utils + + + + org.apache.flink + flink-runtime + test-jar + + + + org.apache.flink + flink-streaming-java + test-jar + + + + org.apache.flink + flink-table-planner_2.12 + + + + org.mockito + mockito-core + + + + org.mockito + mockito-junit-jupiter + + + + org.assertj + assertj-core + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.flink + flink-clients + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + ${project.basedir}/../src/main/java + + + + + add-test-source + generate-test-sources + + add-test-source + + + + ${project.basedir}/../src/test/java + + + + + add-resource + generate-resources + + add-resource + + + + + ${project.basedir}/../src/main/resources + + + + + + add-test-resource + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/../src/test/resources + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + org.jacoco + jacoco-maven-plugin + + + + diff --git a/lance-flink-1.19/src/test/resources/log4j2-test.xml b/lance-flink-1.19/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000..c73119e --- /dev/null +++ b/lance-flink-1.19/src/test/resources/log4j2-test.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/lance-flink-1.20/pom.xml b/lance-flink-1.20/pom.xml new file mode 100644 index 0000000..cc98ef2 --- /dev/null +++ b/lance-flink-1.20/pom.xml @@ -0,0 +1,233 @@ + + + 4.0.0 + + + org.apache.flink + lance-flink-root + 1.0.0-SNAPSHOT + ../pom.xml + + + lance-flink-1.20 + Lance Flink Connector - Flink 1.20 + Lance Flink Connector for Apache Flink 1.20 + jar + + + ${flink120.version} + + + + + + com.lancedb + lance-core + + + + + org.apache.arrow + arrow-vector + + + + org.apache.arrow + arrow-memory-netty + + + + org.apache.arrow + arrow-c-data + + + + + org.apache.flink + flink-streaming-java + + + + org.apache.flink + flink-table-api-java + + + + org.apache.flink + flink-table-api-java-bridge + + + + org.apache.flink + flink-table-common + + + + org.apache.flink + flink-connector-base + + + + + org.slf4j + slf4j-api + + + + + org.junit.jupiter + junit-jupiter-api + + + + org.junit.jupiter + junit-jupiter-engine + + + + org.junit.jupiter + junit-jupiter-params + + + + org.apache.flink + flink-test-utils + + + + org.apache.flink + flink-runtime + test-jar + + + + org.apache.flink + flink-streaming-java + test-jar + + + + org.apache.flink + flink-table-planner_2.12 + + + + org.mockito + mockito-core + + + + org.mockito + mockito-junit-jupiter + + + + org.assertj + assertj-core + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.flink + flink-clients + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + ${project.basedir}/../src/main/java + + + + + add-test-source + generate-test-sources + + add-test-source + + + + ${project.basedir}/../src/test/java + + + + + add-resource + generate-resources + + add-resource + + + + + ${project.basedir}/../src/main/resources + + + + + + add-test-resource + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/../src/test/resources + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + org.jacoco + jacoco-maven-plugin + + + + diff --git a/lance-flink-1.20/src/test/resources/log4j2-test.xml b/lance-flink-1.20/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000..c73119e --- /dev/null +++ b/lance-flink-1.20/src/test/resources/log4j2-test.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 5547152..abc0867 100644 --- a/pom.xml +++ b/pom.xml @@ -4,24 +4,38 @@ 4.0.0 org.apache.flink - flink-connector-lance + lance-flink-root 1.0.0-SNAPSHOT - jar + pom - Flink Connector Lance + Lance Flink Connector Apache Flink Connector for Lance vector data format - https://github.com/apache/flink-connector-lance + https://github.com/lance-format/lance-flink UTF-8 UTF-8 1.8 1.8 - - - 1.16.1 + + + 1.0.0-SNAPSHOT + + 0.23.3 + + 14.0.0 + + + 1.18.1 + 1.19.1 + 1.20.0 + + ${flink118.version} + + 5.9.3 1.7.36 2.20.0 @@ -29,332 +43,409 @@ 3.24.2 - - - - - org.apache.flink - flink-streaming-java - ${flink.version} - provided - - - - - org.apache.flink - flink-table-api-java - ${flink.version} - provided - - - - - org.apache.flink - flink-table-api-java-bridge - ${flink.version} - provided - - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - - org.apache.flink - flink-connector-base - ${flink.version} - provided - - - - - com.lancedb - lance-core - ${lance.version} - - - - - org.apache.arrow - arrow-vector - ${arrow.version} - - - - org.apache.arrow - arrow-memory-netty - ${arrow.version} - - - - org.apache.arrow - arrow-c-data - ${arrow.version} - - - - - org.slf4j - slf4j-api - ${slf4j.version} - - - - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - - org.junit.jupiter - junit-jupiter-params - ${junit.version} - test - - - - - org.apache.flink - flink-test-utils - ${flink.version} - test - - - - org.apache.flink - flink-runtime - ${flink.version} - test-jar - test - - - - org.apache.flink - flink-streaming-java - ${flink.version} - test-jar - test - - - - - org.apache.flink - flink-table-planner_2.12 - ${flink.version} - test - - - - - org.mockito - mockito-core - ${mockito.version} - test - - - - org.mockito - mockito-junit-jupiter - ${mockito.version} - test - - - - - org.assertj - assertj-core - ${assertj.version} - test - - - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - - org.apache.logging.log4j - log4j-api - ${log4j.version} - test - - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - - - org.apache.flink - flink-clients - ${flink.version} - test - - + + + + + com.lancedb + lance-core + ${lance.version} + + + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + + org.apache.arrow + arrow-c-data + ${arrow.version} + + + + + org.apache.flink + flink-streaming-java + ${flink.version} + provided + + + + org.apache.flink + flink-table-api-java + ${flink.version} + provided + + + + org.apache.flink + flink-table-api-java-bridge + ${flink.version} + provided + + + + org.apache.flink + flink-table-common + ${flink.version} + provided + + + + org.apache.flink + flink-connector-base + ${flink.version} + provided + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-params + ${junit.version} + test + + + + org.apache.flink + flink-test-utils + ${flink.version} + test + + + + org.apache.flink + flink-runtime + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-streaming-java + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-table-planner_2.12 + ${flink.version} + test + + + + org.mockito + mockito-core + ${mockito.version} + test + + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + + + + org.assertj + assertj-core + ${assertj.version} + test + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + test + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + test + + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + test + + + + org.apache.flink + flink-clients + ${flink.version} + test + + + - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.15.0 - - ${maven.compiler.source} - ${maven.compiler.target} - - -Xlint:all - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.1.2 - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.6.2 - - - package - - shade - - - - - org.apache.flink:flink-streaming-java - org.apache.flink:flink-table-api-java - org.apache.flink:flink-table-api-java-bridge - org.apache.flink:flink-table-common - org.apache.flink:flink-connector-base - org.slf4j:* - - - - - *:* + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.15.0 + + ${maven.compiler.source} + ${maven.compiler.target} + + -Xlint:all + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.1.2 + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.2 + + + package + + shade + + + - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA + org.apache.flink:flink-streaming-java + org.apache.flink:flink-table-api-java + org.apache.flink:flink-table-api-java-bridge + org.apache.flink:flink-table-common + org.apache.flink:flink-connector-base + org.slf4j:* - - - - - - - - - org.apache.arrow - org.apache.flink.connector.lance.shaded.arrow - - - - - - + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + org.apache.arrow + org.apache.flink.connector.lance.shaded.arrow + + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.5.0 + + + + true + true + + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.4.0 + + + attach-sources + + jar-no-fork + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.5.0 + + none + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.14 + + + + prepare-agent + + + + report + test + + report + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.3.0 + + + enforce-maven + + enforce + + + + + 3.6.3 + + + [1.8,) + + + + + + + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.40.0 + + + + 1.7 + + + + + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + + + - + org.apache.maven.plugins - maven-jar-plugin - 3.5.0 - - - - true - true - - - + maven-compiler-plugin - - org.apache.maven.plugins - maven-source-plugin - 3.4.0 - - - attach-sources - - jar-no-fork - - - + maven-surefire-plugin - - org.apache.maven.plugins - maven-javadoc-plugin - 3.5.0 - - none - - - - - - org.jacoco - jacoco-maven-plugin - 0.8.14 - - - - prepare-agent - - - - report - test - - report - - - + maven-enforcer-plugin + + lance-flink-1.18 + lance-flink-1.19 + lance-flink-1.20 + + - central https://repo.maven.apache.org/maven2 - apache-snapshots https://repository.apache.org/content/repositories/snapshots/ @@ -366,4 +457,5 @@ + \ No newline at end of file diff --git a/src/main/java/org/apache/flink/connector/lance/table/LanceDynamicTableSource.java b/src/main/java/org/apache/flink/connector/lance/table/LanceDynamicTableSource.java index dfcf186..5ec7634 100644 --- a/src/main/java/org/apache/flink/connector/lance/table/LanceDynamicTableSource.java +++ b/src/main/java/org/apache/flink/connector/lance/table/LanceDynamicTableSource.java @@ -25,7 +25,6 @@ import org.apache.flink.connector.lance.config.LanceOptions; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.connector.source.DataStreamScanProvider; import org.apache.flink.table.connector.source.DynamicTableSource;