Skip to content

Commit 5dae9a8

Browse files
andygroveandygrove
committed
Merge remote-tracking branch 'apache/main' into remove-iceberg-compat
2 parents ee1e499 + 64b5ac3 commit 5dae9a8

91 files changed

Lines changed: 150 additions & 332 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/setup-spark-builder/action.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ runs:
5151
path: |
5252
~/.m2/repository
5353
/root/.m2/repository
54-
key: ${{ runner.os }}-spark-sql-${{ hashFiles('spark/**/pom.xml', 'common/**/pom.xml') }}
54+
key: ${{ runner.os }}-spark-sql-${{ hashFiles('common/**/pom.xml', 'spark/**/pom.xml') }}
5555
restore-keys: |
5656
${{ runner.os }}-spark-sql-
5757

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ on:
3333
- "native/core/benches/**"
3434
- "native/spark-expr/benches/**"
3535
- "spark/src/test/**"
36-
- "common/src/test/**"
3736
- "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
3837
- "spark-integration/**"
3938
pull_request:
@@ -45,7 +44,6 @@ on:
4544
- "native/core/benches/**"
4645
- "native/spark-expr/benches/**"
4746
- "spark/src/test/**"
48-
- "common/src/test/**"
4947
- "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
5048
- "spark-integration/**"
5149
# manual trigger

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ jobs:
189189
uses: ./.github/actions/setup-builder
190190
with:
191191
rust-version: ${{ env.RUST_VERSION }}
192-
jdk-version: 17 # JDK only needed for common module proto generation
192+
jdk-version: 17 # JDK only needed for JVM module proto generation
193193

194194
- name: Restore Cargo cache
195195
uses: actions/cache/restore@v5

‎Makefile‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -68,28 +68,28 @@ endif
6868
core-amd64:
6969
rustup target add x86_64-apple-darwin
7070
cd native && RUSTFLAGS="-Ctarget-cpu=skylake" CC=o64-clang CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release $(FEATURES_ARG)
71-
mkdir -p common/target/classes/org/apache/comet/darwin/x86_64
72-
cp native/target/x86_64-apple-darwin/release/libcomet.dylib common/target/classes/org/apache/comet/darwin/x86_64
71+
mkdir -p spark/target/classes/org/apache/comet/darwin/x86_64
72+
cp native/target/x86_64-apple-darwin/release/libcomet.dylib spark/target/classes/org/apache/comet/darwin/x86_64
7373
cd native && RUSTFLAGS="-Ctarget-cpu=x86-64-v3" cargo build --release $(FEATURES_ARG)
74-
mkdir -p common/target/classes/org/apache/comet/linux/amd64
75-
cp native/target/release/libcomet.so common/target/classes/org/apache/comet/linux/amd64
76-
jar -cf common/target/comet-native-x86_64.jar \
77-
-C common/target/classes/org/apache/comet darwin \
78-
-C common/target/classes/org/apache/comet linux
79-
./dev/deploy-file common/target/comet-native-x86_64.jar comet-native-x86_64${COMET_CLASSIFIER} jar
74+
mkdir -p spark/target/classes/org/apache/comet/linux/amd64
75+
cp native/target/release/libcomet.so spark/target/classes/org/apache/comet/linux/amd64
76+
jar -cf spark/target/comet-native-x86_64.jar \
77+
-C spark/target/classes/org/apache/comet darwin \
78+
-C spark/target/classes/org/apache/comet linux
79+
./dev/deploy-file spark/target/comet-native-x86_64.jar comet-native-x86_64${COMET_CLASSIFIER} jar
8080

8181
core-arm64:
8282
rustup target add aarch64-apple-darwin
8383
cd native && RUSTFLAGS="-Ctarget-cpu=apple-m1" CC=arm64-apple-darwin21.4-clang CXX=arm64-apple-darwin21.4-clang++ CARGO_FEATURE_NEON=1 cargo build --target aarch64-apple-darwin --release $(FEATURES_ARG)
84-
mkdir -p common/target/classes/org/apache/comet/darwin/aarch64
85-
cp native/target/aarch64-apple-darwin/release/libcomet.dylib common/target/classes/org/apache/comet/darwin/aarch64
84+
mkdir -p spark/target/classes/org/apache/comet/darwin/aarch64
85+
cp native/target/aarch64-apple-darwin/release/libcomet.dylib spark/target/classes/org/apache/comet/darwin/aarch64
8686
cd native && RUSTFLAGS="-Ctarget-cpu=neoverse-n1" cargo build --release $(FEATURES_ARG)
87-
mkdir -p common/target/classes/org/apache/comet/linux/aarch64
88-
cp native/target/release/libcomet.so common/target/classes/org/apache/comet/linux/aarch64
89-
jar -cf common/target/comet-native-aarch64.jar \
90-
-C common/target/classes/org/apache/comet darwin \
91-
-C common/target/classes/org/apache/comet linux
92-
./dev/deploy-file common/target/comet-native-aarch64.jar comet-native-aarch64${COMET_CLASSIFIER} jar
87+
mkdir -p spark/target/classes/org/apache/comet/linux/aarch64
88+
cp native/target/release/libcomet.so spark/target/classes/org/apache/comet/linux/aarch64
89+
jar -cf spark/target/comet-native-aarch64.jar \
90+
-C spark/target/classes/org/apache/comet darwin \
91+
-C spark/target/classes/org/apache/comet linux
92+
./dev/deploy-file spark/target/comet-native-aarch64.jar comet-native-aarch64${COMET_CLASSIFIER} jar
9393

9494
release-linux: clean
9595
rustup target add aarch64-apple-darwin x86_64-apple-darwin

‎common/pom.xml‎

Lines changed: 18 additions & 176 deletions
Original file line numberDiff line numberDiff line change
@@ -33,196 +33,38 @@ under the License.
3333
<artifactId>comet-common-spark${spark.version.short}_${scala.binary.version}</artifactId>
3434
<name>comet-common</name>
3535

36+
<!--
37+
Minimal bootstrap module. Contains only the exception classes that the
38+
Rust JNI bridge needs to load via JNI (CometNativeException and friends).
39+
Kept small on purpose so `mvn compile -pl common` is a cheap prerequisite
40+
for `cargo test` in the Rust workspace. All other JVM code lives in
41+
comet-spark.
42+
-->
43+
3644
<properties>
3745
<!-- Reverse default (skip installation), and then enable only for child modules -->
3846
<maven.deploy.skip>false</maven.deploy.skip>
3947
</properties>
4048

41-
<dependencies>
42-
<dependency>
43-
<groupId>org.apache.spark</groupId>
44-
<artifactId>spark-sql_${scala.binary.version}</artifactId>
45-
</dependency>
46-
<dependency>
47-
<groupId>org.apache.parquet</groupId>
48-
<artifactId>parquet-column</artifactId>
49-
</dependency>
50-
<dependency>
51-
<groupId>org.apache.parquet</groupId>
52-
<artifactId>parquet-hadoop</artifactId>
53-
</dependency>
54-
<dependency>
55-
<groupId>org.apache.parquet</groupId>
56-
<artifactId>parquet-format-structures</artifactId>
57-
</dependency>
58-
<dependency>
59-
<groupId>org.apache.arrow</groupId>
60-
<artifactId>arrow-vector</artifactId>
61-
</dependency>
62-
<dependency>
63-
<groupId>org.apache.arrow</groupId>
64-
<artifactId>arrow-memory-unsafe</artifactId>
65-
</dependency>
66-
<dependency>
67-
<groupId>org.apache.arrow</groupId>
68-
<artifactId>arrow-c-data</artifactId>
69-
</dependency>
70-
<dependency>
71-
<groupId>org.scala-lang.modules</groupId>
72-
<artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
73-
</dependency>
74-
<dependency>
75-
<groupId>junit</groupId>
76-
<artifactId>junit</artifactId>
77-
<scope>test</scope>
78-
</dependency>
79-
<dependency>
80-
<groupId>org.assertj</groupId>
81-
<artifactId>assertj-core</artifactId>
82-
<scope>test</scope>
83-
</dependency>
84-
</dependencies>
85-
8649
<build>
8750
<plugins>
88-
<plugin>
89-
<groupId>io.github.git-commit-id</groupId>
90-
<artifactId>git-commit-id-maven-plugin</artifactId>
91-
<version>${git-commit-id-maven-plugin.version}</version>
92-
<executions>
93-
<execution>
94-
<id>get-the-git-infos</id>
95-
<goals>
96-
<goal>revision</goal>
97-
</goals>
98-
<phase>initialize</phase>
99-
</execution>
100-
</executions>
101-
<configuration>
102-
<generateGitPropertiesFile>true</generateGitPropertiesFile>
103-
<generateGitPropertiesFilename>${project.build.outputDirectory}/comet-git-info.properties</generateGitPropertiesFilename>
104-
<commitIdGenerationMode>full</commitIdGenerationMode>
105-
<includeOnlyProperties>
106-
<includeOnlyProperty>^git.branch$</includeOnlyProperty>
107-
<includeOnlyProperty>^git.build.*$</includeOnlyProperty>
108-
<includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
109-
<includeOnlyProperty>^git.remote.*$</includeOnlyProperty>
110-
</includeOnlyProperties>
111-
</configuration>
112-
</plugin>
113-
<plugin>
114-
<groupId>org.apache.maven.plugins</groupId>
115-
<artifactId>maven-shade-plugin</artifactId>
116-
<executions>
117-
<execution>
118-
<phase>package</phase>
119-
<goals>
120-
<goal>shade</goal>
121-
</goals>
122-
<configuration>
123-
<createSourcesJar>true</createSourcesJar>
124-
<shadeSourcesContent>true</shadeSourcesContent>
125-
<shadedArtifactAttached>false</shadedArtifactAttached>
126-
<createDependencyReducedPom>true</createDependencyReducedPom>
127-
<artifactSet>
128-
<includes>
129-
<!-- We shade & relocation most of the Arrow classes, to prevent them from
130-
conflicting with those in Spark -->
131-
<include>org.apache.arrow:*</include>
132-
</includes>
133-
</artifactSet>
134-
<filters>
135-
<filter>
136-
<artifact>*:*</artifact>
137-
<excludes>
138-
<exclude>**/*.thrift</exclude>
139-
<exclude>git.properties</exclude>
140-
<exclude>log4j.properties</exclude>
141-
<exclude>log4j2.properties</exclude>
142-
<exclude>arrow-git.properties</exclude>
143-
</excludes>
144-
</filter>
145-
<filter>
146-
<artifact>org.apache.arrow:arrow-vector</artifact>
147-
<excludes>
148-
<!-- Comet doesn't need codegen templates on Arrow -->
149-
<exclude>codegen/**</exclude>
150-
</excludes>
151-
</filter>
152-
</filters>
153-
<relocations>
154-
<relocation>
155-
<pattern>org.apache.arrow</pattern>
156-
<shadedPattern>${comet.shade.packageName}.arrow</shadedPattern>
157-
<excludes>
158-
<!-- We can't allocate Jni classes. These classes has no extra dependencies
159-
so it should be OK to exclude -->
160-
<exclude>org/apache/arrow/c/jni/JniWrapper</exclude>
161-
<exclude>org/apache/arrow/c/jni/PrivateData</exclude>
162-
<exclude>org/apache/arrow/c/jni/CDataJniException</exclude>
163-
<!-- Also used by JNI: https://github.com/apache/arrow/blob/apache-arrow-11.0.0/java/c/src/main/cpp/jni_wrapper.cc#L341
164-
Note this class is not used by us, but required when loading the native lib -->
165-
<exclude>org/apache/arrow/c/ArrayStreamExporter$ExportedArrayStreamPrivateData
166-
</exclude>
167-
</excludes>
168-
</relocation>
169-
</relocations>
170-
</configuration>
171-
</execution>
172-
</executions>
173-
</plugin>
51+
<!-- scala-maven-plugin handles javac too: the parent pom's
52+
maven-compiler-plugin is configured with skipMain=true. -->
17453
<plugin>
17554
<groupId>net.alchim31.maven</groupId>
17655
<artifactId>scala-maven-plugin</artifactId>
17756
</plugin>
57+
<!-- This module has no Scala sources; skip scalafix so the lint job
58+
does not fail with "No files to fix". -->
17859
<plugin>
179-
<groupId>org.codehaus.mojo</groupId>
180-
<artifactId>build-helper-maven-plugin</artifactId>
181-
<executions>
182-
<execution>
183-
<id>add-shim-source</id>
184-
<phase>generate-sources</phase>
185-
<goals>
186-
<goal>add-source</goal>
187-
</goals>
188-
<configuration>
189-
<sources>
190-
<source>src/main/${shims.majorVerSrc}</source>
191-
<source>src/main/${shims.minorVerSrc}</source>
192-
</sources>
193-
</configuration>
194-
</execution>
195-
</executions>
60+
<groupId>io.github.evis</groupId>
61+
<artifactId>scalafix-maven-plugin_${scala.binary.version}</artifactId>
62+
<version>${scalafix-maven-plugin.version}</version>
63+
<configuration>
64+
<skip>true</skip>
65+
</configuration>
19666
</plugin>
19767
</plugins>
198-
<resources>
199-
<resource>
200-
<directory>${project.basedir}/src/main/resources</directory>
201-
</resource>
202-
<resource>
203-
<directory>${project.basedir}/../native/target/x86_64-apple-darwin/release</directory>
204-
<includes>
205-
<include>libcomet.dylib</include>
206-
</includes>
207-
<targetPath>org/apache/comet/darwin/x86_64</targetPath>
208-
</resource>
209-
<resource>
210-
<directory>${project.basedir}/../native/target/aarch64-apple-darwin/release</directory>
211-
<includes>
212-
<include>libcomet.dylib</include>
213-
</includes>
214-
<targetPath>org/apache/comet/darwin/aarch64</targetPath>
215-
</resource>
216-
<resource>
217-
<directory>${jni.dir}</directory>
218-
<includes>
219-
<include>libcomet.dylib</include>
220-
<include>libcomet.so</include>
221-
<include>comet.dll</include>
222-
</includes>
223-
<targetPath>org/apache/comet/${platform}/${arch}</targetPath>
224-
</resource>
225-
</resources>
22668
</build>
22769

22870
</project>

‎common/src/main/resources/log4j2.properties‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎common/src/test/resources/log4j.properties‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

0 commit comments

Comments
 (0)