[SYSTEMDS-3877] Docker cleanup

This commit fixes three bugs in the new testing image.

1. The seal library install path, to be included in the library path
2. The locale specification to US English UTF-8, to allow string
comparison of Hadoop read files, without casting from 16 to 8.

Furthermore, the commit does a few maintainable cleanups in the
testing to reduce verbose logging of our test cases, and allow
non-buffered writing of cloud running tests, to enable easy live
tracking of GitHub action tests.

Closes #2335

Author: Baunsgaard

.gitignore

@@ -154,3 +154,6 @@ scripts/resource/output
 # docker tests
 docker/mountFolder/*.bin
 docker/mountFolder/*.bin.mtd
+
+SEAL-*/
+

docker/README.md

@@ -138,6 +138,7 @@ Test your testing image locally by running the following command:
 ```bash
 docker run \
   -v $(pwd):/github/workspace \
+  -v $HOME/.m2/repository:/root/.m2/repository \
   apache/systemds:testing-latest \
   org.apache.sysds.test.component.**
 ```

docker/entrypoint.sh

@@ -29,23 +29,22 @@ cd /github/workspace
 export MAVEN_OPTS="-Xmx512m"
 
 log="/tmp/sysdstest.log"
-mvn -ntp -B test-compile 2>&1 | grep -E "BUILD|Total time:|---|Building SystemDS"
+mvn -ntp -B test-compile 2>&1 | stdbuf -oL grep -E "BUILD|Total time:|---|Building SystemDS"
 mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=true -D test=$1 2>&1 \
-	| grep -v "already exists in destination." \
-	| grep -v 'WARNING: Using incubator modules' | tee $log
+	| stdbuf -oL grep -Ev "already exists in destination.|Using incubator" \
+	| tee $log
 
-# Merge Federated test runs.
-[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec
-mvn -ntp -B jacoco:merge
-
-# Merge Federated test runs.
-[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec
-mvn -ntp -B jacoco:merge
 
 grep_args="SUCCESS"
 grepvals="$( tail -n 100 $log | grep $grep_args)"
 
 if [[ $grepvals == *"SUCCESS"* ]]; then
+	# Merge Federated test runs.
+	# if merged jacoco exist temporarily rename to not overwrite.
+	[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec
+	# merge jacoco files.
+	mvn -ntp -B jacoco:merge 2>&1 | stdbuf -oL grep -E "jacoco"
+
 	exit 0
 else
 	exit 1

docker/testsysds.Dockerfile

@@ -19,7 +19,7 @@
 #
 #-------------------------------------------------------------
 # Stage 1: Build SEAL
-FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 AS seal-build
+FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS seal-build
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
@@ -40,16 +40,24 @@ RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz
     && cmake --install build --prefix /seal-install
 
 # Stage 2: Final image with R, JDK, Maven, SEAL
-FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973
+FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 
 
 WORKDIR /usr/src/
 ENV MAVEN_VERSION=3.9.9
 ENV MAVEN_HOME=/usr/lib/mvn
 
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+ENV LANG=en_US.UTF-8
 ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6
 ENV PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y locales \
+    && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
+    && locale-gen \
+    && update-locale LANG=en_US.UTF-8
+
+RUN apt-get install -y --no-install-recommends \
     r-base \
 	wget \
     cmake \
@@ -78,8 +86,26 @@ COPY ./src/test/scripts/installDependencies.R installDependencies.R
 RUN Rscript installDependencies.R \
     && rm -f installDependencies.R
 
+ENV HADOOP_VERSION=3.3.6
+ENV HADOOP_HOME=/opt/hadoop
+ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native
+ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
+
+RUN mkdir -p $HADOOP_HOME/lib/native \
+	&& wget -q https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \
+    tar --strip-components=2 -xzf hadoop-${HADOOP_VERSION}.tar.gz \
+        hadoop-${HADOOP_VERSION}/lib/native && \
+    mv native/libhadoop.so.1.0.0 /opt/hadoop/lib/native && \
+	mv native/libhadoop.so /opt/hadoop/lib/native && \
+    rm hadoop-${HADOOP_VERSION}.tar.gz && \
+	rm -rf native
+
 # Copy SEAL
-COPY --from=seal-build /seal-install /usr/local
+COPY --from=seal-build /seal-install/lib/ /usr/local/lib/
+COPY --from=seal-build /seal-install/include/ /usr/local/include/
+
+ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native;/usr/local/lib/
+
 
 # Finally copy the entrypoint script
 # This is last to enable quick updates to the script after initial local build.

src/main/cpp/build_HE.sh

@@ -20,7 +20,7 @@
 #
 #-------------------------------------------------------------
 
-echo "Build Homomorphic Encryption Liberary: "
+echo "Build Homomorphic Encryption Library: "
 
 # compile HE
 cmake he/ -B HE -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++
@@ -29,4 +29,4 @@ rm -R HE
 
 echo ""
 
-echo "Sucessfull install of Homomorphic Encryption Liberary SEAL"
+echo "Successful install of Homomorphic Encryption Library SEAL"

src/test/java/org/apache/sysds/test/TestUtils.java

@@ -1760,15 +1760,25 @@ public static HashMap<CellIndex, Double> convert2DDoubleArrayToHashMap(double[][
 	}
 
 	public static double[][] convertHashMapToDoubleArray(HashMap <CellIndex, Double> matrix) {
+		if(matrix.isEmpty()){
+			LOG.error("converting empty hashMap");
+			return new double[0][0];
+		}
 		int max_rows = -1, max_cols= -1;
+
 		for(CellIndex ix : matrix.keySet()) {
 			max_rows = Math.max(max_rows, ix.row);
 			max_cols = Math.max(max_cols, ix.column);
 		}
+		
 		return convertHashMapToDoubleArray(matrix, max_rows, max_cols);
 	}
 
 	public static double[][] convertHashMapToDoubleArray(HashMap<CellIndex, Double> matrix, int rows, int cols) {
+		if(rows <= -1 || cols <= -1){
+			LOG.error("converting negative size hashmap rows: " + rows + " cols: " + cols);
+			return new double[0][0];
+		}
 		double [][] ret_arr = new double[rows][cols];
 		for(Entry<CellIndex, Double> e : matrix.entrySet()) {
 			int i = e.getKey().row-1;

src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java

@@ -148,16 +148,16 @@ private void runTransformTest( ExecMode rt, String ofmt, boolean subset, boolean
 			String[] fromDisk = DataConverter.toString(fb2).split("\n");
 			String[] printed = stdOut.split("\n");
 			boolean equal = true;
-			String err = "";
+			StringBuilder err = new StringBuilder();
 			for(int i = 0; i < fromDisk.length; i++){
-				if(! fromDisk[i].equals(printed[i])){
-					err += "\n not equal: \n"+ (fromDisk[i] + "\n" + printed[i]);
+				if(! fromDisk[i].strip().equals(printed[i].strip())){
+					err.append("\n not equal: \n'"+ (fromDisk[i] + "'\n'" + printed[i] + "'"));
 					equal = false;
 				}
 
 			}
 			if(!equal)
-				fail(err);
+				fail(err.toString());
 
 		}
 		catch(Exception ex) {