Skip to content

Commit 82b70aa

Browse files
authored
[INFRA] Use hadoop 2.10.2 for uniffle test (#12111)
* [INFRA] Use hadoop 2.10.2 for uniffle test * fix CI * remove unused uniffle tarball in centos9 Dockerfile
1 parent 0cc376a commit 82b70aa

3 files changed

Lines changed: 4 additions & 6 deletions

File tree

.github/workflows/velox_backend_x86.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -554,7 +554,7 @@ jobs:
554554
matrix:
555555
spark: [ "spark-3.3" ]
556556
uniffle: [ "0.10.0" ]
557-
hadoop: [ "2.8.5" ]
557+
hadoop: [ "2.10.2" ]
558558
runs-on: ubuntu-22.04
559559
container: apache/gluten:centos-8-jdk8
560560
steps:
@@ -573,10 +573,10 @@ jobs:
573573
run: |
574574
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
575575
if [ ! -e "/opt/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz" ]; then
576-
${WGET_CMD} https://archive.apache.org/dist/uniffle/${{ matrix.uniffle }}/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz -P /opt
576+
${WGET_CMD} https://www.apache.org/dyn/closer.lua/uniffle/${{ matrix.uniffle }}/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz?action=download -O /opt/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz
577577
fi
578578
if [ ! -e "/opt/hadoop-${{ matrix.hadoop }}.tar.gz" ]; then
579-
${WGET_CMD} https://archive.apache.org/dist/hadoop/common/hadoop-${{ matrix.hadoop }}/hadoop-${{ matrix.hadoop }}.tar.gz -P /opt
579+
${WGET_CMD} https://www.apache.org/dyn/closer.lua/hadoop/common/hadoop-${{ matrix.hadoop }}/hadoop-${{ matrix.hadoop }}.tar.gz?action=download -O /opt/hadoop-${{ matrix.hadoop }}.tar.gz
580580
fi
581581
cd /opt && rm -rf shims && \
582582
mkdir /opt/uniffle && tar xzf apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz -C /opt/uniffle --strip-components=1 && \

dev/docker/Dockerfile.centos8-dynamic-build

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ RUN set -ex; \
3737
wget -nv ${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download -O /opt/apache-celeborn-0.5.4-bin.tgz; \
3838
wget -nv ${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download -O /opt/apache-celeborn-0.6.3-bin.tgz; \
3939
wget -nv ${mirror_host}/uniffle/0.10.0/apache-uniffle-0.10.0-bin.tar.gz?action=download -O /opt/apache-uniffle-0.10.0-bin.tar.gz; \
40-
wget -nv ${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz?action=download -O /opt/hadoop-2.8.5.tar.gz; \
40+
wget -nv ${mirror_host}/hadoop/common/hadoop-2.10.2/hadoop-2.10.2.tar.gz?action=download -O /opt/hadoop-2.10.2.tar.gz; \
4141
git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \
4242
cd /opt/gluten/.github/workflows/util/; \
4343
./install-spark-resources.sh 3.3; \

dev/docker/Dockerfile.centos9-dynamic-build

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@ RUN set -ex; \
3434
mirror_host="https://www.apache.org/dyn/closer.lua"; \
3535
wget -nv ${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download -O /opt/apache-celeborn-0.5.4-bin.tgz; \
3636
wget -nv ${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download -O /opt/apache-celeborn-0.6.3-bin.tgz; \
37-
wget -nv ${mirror_host}/uniffle/0.10.0/apache-uniffle-0.10.0-bin.tar.gz?action=download -O /opt/apache-uniffle-0.10.0-bin.tar.gz; \
38-
wget -nv ${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz?action=download -O /opt/hadoop-2.8.5.tar.gz; \
3937
git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \
4038
cd /opt/gluten/.github/workflows/util/; \
4139
./install-spark-resources.sh 3.3; \

0 commit comments

Comments
 (0)