diff --git a/lance-flink-1.18/pom.xml b/lance-flink-1.18/pom.xml
new file mode 100644
index 0000000..d0c48bc
--- /dev/null
+++ b/lance-flink-1.18/pom.xml
@@ -0,0 +1,234 @@
+
+
+ 4.0.0
+
+
+ org.apache.flink
+ lance-flink-root
+ 1.0.0-SNAPSHOT
+ ../pom.xml
+
+
+ lance-flink-1.18
+ Lance Flink Connector - Flink 1.18
+ Lance Flink Connector for Apache Flink 1.18
+ jar
+
+
+ ${flink118.version}
+
+
+
+
+
+ com.lancedb
+ lance-core
+
+
+
+
+ org.apache.arrow
+ arrow-vector
+
+
+
+ org.apache.arrow
+ arrow-memory-netty
+
+
+
+ org.apache.arrow
+ arrow-c-data
+
+
+
+
+ org.apache.flink
+ flink-streaming-java
+
+
+
+ org.apache.flink
+ flink-table-api-java
+
+
+
+ org.apache.flink
+ flink-table-api-java-bridge
+
+
+
+ org.apache.flink
+ flink-table-common
+
+
+
+ org.apache.flink
+ flink-connector-base
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+
+
+
+ org.apache.flink
+ flink-test-utils
+
+
+
+ org.apache.flink
+ flink-runtime
+ test-jar
+
+
+
+ org.apache.flink
+ flink-streaming-java
+ test-jar
+
+
+
+ org.apache.flink
+ flink-table-planner_2.12
+
+
+
+ org.mockito
+ mockito-core
+
+
+
+ org.mockito
+ mockito-junit-jupiter
+
+
+
+ org.assertj
+ assertj-core
+
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+
+
+
+ org.apache.logging.log4j
+ log4j-core
+
+
+
+ org.apache.flink
+ flink-clients
+
+
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+ add-source
+ generate-sources
+
+ add-source
+
+
+
+ ${project.basedir}/../src/main/java
+
+
+
+
+ add-test-source
+ generate-test-sources
+
+ add-test-source
+
+
+
+ ${project.basedir}/../src/test/java
+
+
+
+
+ add-resource
+ generate-resources
+
+ add-resource
+
+
+
+
+ ${project.basedir}/../src/main/resources
+
+
+
+
+
+ add-test-resource
+ generate-test-resources
+
+ add-test-resource
+
+
+
+
+ ${project.basedir}/../src/test/resources
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+
+
+
+
diff --git a/lance-flink-1.18/src/test/resources/log4j2-test.xml b/lance-flink-1.18/src/test/resources/log4j2-test.xml
new file mode 100644
index 0000000..c73119e
--- /dev/null
+++ b/lance-flink-1.18/src/test/resources/log4j2-test.xml
@@ -0,0 +1,15 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lance-flink-1.19/pom.xml b/lance-flink-1.19/pom.xml
new file mode 100644
index 0000000..ec5e146
--- /dev/null
+++ b/lance-flink-1.19/pom.xml
@@ -0,0 +1,233 @@
+
+
+ 4.0.0
+
+
+ org.apache.flink
+ lance-flink-root
+ 1.0.0-SNAPSHOT
+ ../pom.xml
+
+
+ lance-flink-1.19
+ Lance Flink Connector - Flink 1.19
+ Lance Flink Connector for Apache Flink 1.19
+ jar
+
+
+ ${flink119.version}
+
+
+
+
+
+ com.lancedb
+ lance-core
+
+
+
+
+ org.apache.arrow
+ arrow-vector
+
+
+
+ org.apache.arrow
+ arrow-memory-netty
+
+
+
+ org.apache.arrow
+ arrow-c-data
+
+
+
+
+ org.apache.flink
+ flink-streaming-java
+
+
+
+ org.apache.flink
+ flink-table-api-java
+
+
+
+ org.apache.flink
+ flink-table-api-java-bridge
+
+
+
+ org.apache.flink
+ flink-table-common
+
+
+
+ org.apache.flink
+ flink-connector-base
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+
+
+
+ org.apache.flink
+ flink-test-utils
+
+
+
+ org.apache.flink
+ flink-runtime
+ test-jar
+
+
+
+ org.apache.flink
+ flink-streaming-java
+ test-jar
+
+
+
+ org.apache.flink
+ flink-table-planner_2.12
+
+
+
+ org.mockito
+ mockito-core
+
+
+
+ org.mockito
+ mockito-junit-jupiter
+
+
+
+ org.assertj
+ assertj-core
+
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+
+
+
+ org.apache.logging.log4j
+ log4j-core
+
+
+
+ org.apache.flink
+ flink-clients
+
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+ add-source
+ generate-sources
+
+ add-source
+
+
+
+ ${project.basedir}/../src/main/java
+
+
+
+
+ add-test-source
+ generate-test-sources
+
+ add-test-source
+
+
+
+ ${project.basedir}/../src/test/java
+
+
+
+
+ add-resource
+ generate-resources
+
+ add-resource
+
+
+
+
+ ${project.basedir}/../src/main/resources
+
+
+
+
+
+ add-test-resource
+ generate-test-resources
+
+ add-test-resource
+
+
+
+
+ ${project.basedir}/../src/test/resources
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+
+
+
+
diff --git a/lance-flink-1.19/src/test/resources/log4j2-test.xml b/lance-flink-1.19/src/test/resources/log4j2-test.xml
new file mode 100644
index 0000000..c73119e
--- /dev/null
+++ b/lance-flink-1.19/src/test/resources/log4j2-test.xml
@@ -0,0 +1,15 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lance-flink-1.20/pom.xml b/lance-flink-1.20/pom.xml
new file mode 100644
index 0000000..cc98ef2
--- /dev/null
+++ b/lance-flink-1.20/pom.xml
@@ -0,0 +1,233 @@
+
+
+ 4.0.0
+
+
+ org.apache.flink
+ lance-flink-root
+ 1.0.0-SNAPSHOT
+ ../pom.xml
+
+
+ lance-flink-1.20
+ Lance Flink Connector - Flink 1.20
+ Lance Flink Connector for Apache Flink 1.20
+ jar
+
+
+ ${flink120.version}
+
+
+
+
+
+ com.lancedb
+ lance-core
+
+
+
+
+ org.apache.arrow
+ arrow-vector
+
+
+
+ org.apache.arrow
+ arrow-memory-netty
+
+
+
+ org.apache.arrow
+ arrow-c-data
+
+
+
+
+ org.apache.flink
+ flink-streaming-java
+
+
+
+ org.apache.flink
+ flink-table-api-java
+
+
+
+ org.apache.flink
+ flink-table-api-java-bridge
+
+
+
+ org.apache.flink
+ flink-table-common
+
+
+
+ org.apache.flink
+ flink-connector-base
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+
+
+
+ org.apache.flink
+ flink-test-utils
+
+
+
+ org.apache.flink
+ flink-runtime
+ test-jar
+
+
+
+ org.apache.flink
+ flink-streaming-java
+ test-jar
+
+
+
+ org.apache.flink
+ flink-table-planner_2.12
+
+
+
+ org.mockito
+ mockito-core
+
+
+
+ org.mockito
+ mockito-junit-jupiter
+
+
+
+ org.assertj
+ assertj-core
+
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+
+
+
+ org.apache.logging.log4j
+ log4j-core
+
+
+
+ org.apache.flink
+ flink-clients
+
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+ add-source
+ generate-sources
+
+ add-source
+
+
+
+ ${project.basedir}/../src/main/java
+
+
+
+
+ add-test-source
+ generate-test-sources
+
+ add-test-source
+
+
+
+ ${project.basedir}/../src/test/java
+
+
+
+
+ add-resource
+ generate-resources
+
+ add-resource
+
+
+
+
+ ${project.basedir}/../src/main/resources
+
+
+
+
+
+ add-test-resource
+ generate-test-resources
+
+ add-test-resource
+
+
+
+
+ ${project.basedir}/../src/test/resources
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+
+
+
+
diff --git a/lance-flink-1.20/src/test/resources/log4j2-test.xml b/lance-flink-1.20/src/test/resources/log4j2-test.xml
new file mode 100644
index 0000000..c73119e
--- /dev/null
+++ b/lance-flink-1.20/src/test/resources/log4j2-test.xml
@@ -0,0 +1,15 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/pom.xml b/pom.xml
index 5547152..abc0867 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,24 +4,38 @@
4.0.0
org.apache.flink
- flink-connector-lance
+ lance-flink-root
1.0.0-SNAPSHOT
- jar
+ pom
- Flink Connector Lance
+ Lance Flink Connector
Apache Flink Connector for Lance vector data format
- https://github.com/apache/flink-connector-lance
+ https://github.com/lance-format/lance-flink
UTF-8
UTF-8
1.8
1.8
-
-
- 1.16.1
+
+
+ 1.0.0-SNAPSHOT
+
+
0.23.3
+
+
14.0.0
+
+
+ 1.18.1
+ 1.19.1
+ 1.20.0
+
+ ${flink118.version}
+
+
5.9.3
1.7.36
2.20.0
@@ -29,332 +43,409 @@
3.24.2
-
-
-
-
- org.apache.flink
- flink-streaming-java
- ${flink.version}
- provided
-
-
-
-
- org.apache.flink
- flink-table-api-java
- ${flink.version}
- provided
-
-
-
-
- org.apache.flink
- flink-table-api-java-bridge
- ${flink.version}
- provided
-
-
-
-
- org.apache.flink
- flink-table-common
- ${flink.version}
- provided
-
-
-
-
- org.apache.flink
- flink-connector-base
- ${flink.version}
- provided
-
-
-
-
- com.lancedb
- lance-core
- ${lance.version}
-
-
-
-
- org.apache.arrow
- arrow-vector
- ${arrow.version}
-
-
-
- org.apache.arrow
- arrow-memory-netty
- ${arrow.version}
-
-
-
- org.apache.arrow
- arrow-c-data
- ${arrow.version}
-
-
-
-
- org.slf4j
- slf4j-api
- ${slf4j.version}
-
-
-
-
-
- org.junit.jupiter
- junit-jupiter-api
- ${junit.version}
- test
-
-
-
- org.junit.jupiter
- junit-jupiter-engine
- ${junit.version}
- test
-
-
-
- org.junit.jupiter
- junit-jupiter-params
- ${junit.version}
- test
-
-
-
-
- org.apache.flink
- flink-test-utils
- ${flink.version}
- test
-
-
-
- org.apache.flink
- flink-runtime
- ${flink.version}
- test-jar
- test
-
-
-
- org.apache.flink
- flink-streaming-java
- ${flink.version}
- test-jar
- test
-
-
-
-
- org.apache.flink
- flink-table-planner_2.12
- ${flink.version}
- test
-
-
-
-
- org.mockito
- mockito-core
- ${mockito.version}
- test
-
-
-
- org.mockito
- mockito-junit-jupiter
- ${mockito.version}
- test
-
-
-
-
- org.assertj
- assertj-core
- ${assertj.version}
- test
-
-
-
-
- org.apache.logging.log4j
- log4j-slf4j-impl
- ${log4j.version}
- test
-
-
-
- org.apache.logging.log4j
- log4j-api
- ${log4j.version}
- test
-
-
-
- org.apache.logging.log4j
- log4j-core
- ${log4j.version}
- test
-
-
-
-
- org.apache.flink
- flink-clients
- ${flink.version}
- test
-
-
+
+
+
+
+ com.lancedb
+ lance-core
+ ${lance.version}
+
+
+
+
+ org.apache.arrow
+ arrow-vector
+ ${arrow.version}
+
+
+
+ org.apache.arrow
+ arrow-memory-netty
+ ${arrow.version}
+
+
+
+ org.apache.arrow
+ arrow-c-data
+ ${arrow.version}
+
+
+
+
+ org.apache.flink
+ flink-streaming-java
+ ${flink.version}
+ provided
+
+
+
+ org.apache.flink
+ flink-table-api-java
+ ${flink.version}
+ provided
+
+
+
+ org.apache.flink
+ flink-table-api-java-bridge
+ ${flink.version}
+ provided
+
+
+
+ org.apache.flink
+ flink-table-common
+ ${flink.version}
+ provided
+
+
+
+ org.apache.flink
+ flink-connector-base
+ ${flink.version}
+ provided
+
+
+
+
+ org.slf4j
+ slf4j-api
+ ${slf4j.version}
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ ${junit.version}
+ test
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ ${junit.version}
+ test
+
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+ ${junit.version}
+ test
+
+
+
+ org.apache.flink
+ flink-test-utils
+ ${flink.version}
+ test
+
+
+
+ org.apache.flink
+ flink-runtime
+ ${flink.version}
+ test-jar
+ test
+
+
+
+ org.apache.flink
+ flink-streaming-java
+ ${flink.version}
+ test-jar
+ test
+
+
+
+ org.apache.flink
+ flink-table-planner_2.12
+ ${flink.version}
+ test
+
+
+
+ org.mockito
+ mockito-core
+ ${mockito.version}
+ test
+
+
+
+ org.mockito
+ mockito-junit-jupiter
+ ${mockito.version}
+ test
+
+
+
+ org.assertj
+ assertj-core
+ ${assertj.version}
+ test
+
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+ ${log4j.version}
+ test
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+ ${log4j.version}
+ test
+
+
+
+ org.apache.logging.log4j
+ log4j-core
+ ${log4j.version}
+ test
+
+
+
+ org.apache.flink
+ flink-clients
+ ${flink.version}
+ test
+
+
+
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- 3.15.0
-
- ${maven.compiler.source}
- ${maven.compiler.target}
-
- -Xlint:all
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
- 3.1.2
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
- 3.6.2
-
-
- package
-
- shade
-
-
-
-
- org.apache.flink:flink-streaming-java
- org.apache.flink:flink-table-api-java
- org.apache.flink:flink-table-api-java-bridge
- org.apache.flink:flink-table-common
- org.apache.flink:flink-connector-base
- org.slf4j:*
-
-
-
-
- *:*
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.15.0
+
+ ${maven.compiler.source}
+ ${maven.compiler.target}
+
+ -Xlint:all
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 3.1.2
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+ 3.4.0
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.6.2
+
+
+ package
+
+ shade
+
+
+
- META-INF/*.SF
- META-INF/*.DSA
- META-INF/*.RSA
+ org.apache.flink:flink-streaming-java
+ org.apache.flink:flink-table-api-java
+ org.apache.flink:flink-table-api-java-bridge
+ org.apache.flink:flink-table-common
+ org.apache.flink:flink-connector-base
+ org.slf4j:*
-
-
-
-
-
-
-
-
- org.apache.arrow
- org.apache.flink.connector.lance.shaded.arrow
-
-
-
-
-
-
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
+
+
+
+ org.apache.arrow
+ org.apache.flink.connector.lance.shaded.arrow
+
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.5.0
+
+
+
+ true
+ true
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 3.4.0
+
+
+ attach-sources
+
+ jar-no-fork
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 3.5.0
+
+ none
+
+
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+ 0.8.14
+
+
+
+ prepare-agent
+
+
+
+ report
+ test
+
+ report
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-enforcer-plugin
+ 3.3.0
+
+
+ enforce-maven
+
+ enforce
+
+
+
+
+ 3.6.3
+
+
+ [1.8,)
+
+
+
+
+
+
+
+
+
+
+ com.diffplug.spotless
+ spotless-maven-plugin
+ 2.40.0
+
+
+
+ 1.7
+
+
+
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+
+
+
+
-
+
org.apache.maven.plugins
- maven-jar-plugin
- 3.5.0
-
-
-
- true
- true
-
-
-
+ maven-compiler-plugin
-
-
org.apache.maven.plugins
- maven-source-plugin
- 3.4.0
-
-
- attach-sources
-
- jar-no-fork
-
-
-
+ maven-surefire-plugin
-
-
org.apache.maven.plugins
- maven-javadoc-plugin
- 3.5.0
-
- none
-
-
-
-
-
- org.jacoco
- jacoco-maven-plugin
- 0.8.14
-
-
-
- prepare-agent
-
-
-
- report
- test
-
- report
-
-
-
+ maven-enforcer-plugin
+
+ lance-flink-1.18
+ lance-flink-1.19
+ lance-flink-1.20
+
+
-
central
https://repo.maven.apache.org/maven2
-
apache-snapshots
https://repository.apache.org/content/repositories/snapshots/
@@ -366,4 +457,5 @@
+
\ No newline at end of file
diff --git a/src/main/java/org/apache/flink/connector/lance/table/LanceDynamicTableSource.java b/src/main/java/org/apache/flink/connector/lance/table/LanceDynamicTableSource.java
index dfcf186..5ec7634 100644
--- a/src/main/java/org/apache/flink/connector/lance/table/LanceDynamicTableSource.java
+++ b/src/main/java/org/apache/flink/connector/lance/table/LanceDynamicTableSource.java
@@ -25,7 +25,6 @@
import org.apache.flink.connector.lance.config.LanceOptions;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.ChangelogMode;
import org.apache.flink.table.connector.source.DataStreamScanProvider;
import org.apache.flink.table.connector.source.DynamicTableSource;