Fix CI tests - macOS, Ubuntu, SYCL (#7087)

* Cache storage change:  gz->xz, 5->4GB
* Use github OS default xcode versions. Each macOS version now comes with a single xcode version.
* Add SYCL fallback CPU device as last SYCL device to allow distinguishing CPU and GPU devices.
* Hide symbols from core and visualization to reduce Windows DLL size.
* Skip last SYCL device (CPU fallback) in testing
* Disable MSVC std compliance due to PoissonRecon header error

Author: ssheorey

.github/workflows/macos.yml

@@ -32,9 +32,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # macos-12 is Intel runner, macos-14 is Apple Silicon
+        # macos-13 is Intel runner, macos-14 is Apple Silicon
         # https://github.com/actions/runner-images
-        os: [macos-12, macos-14]
+        os: [macos-13, macos-14]
         CONFIG: [ON, OFF]
     env:
       BUILD_SHARED_LIBS: ${{ matrix.CONFIG }}
@@ -68,23 +68,16 @@ jobs:
         run: |
           brew install ccache pkg-config
 
-          if [[ ${{ runner.arch}} == "ARM64"  ]]; then
+          if [[ ${{ runner.arch }} == "ARM64"  ]]; then
             # Fix gfortran not found issue
             ln -s $(which gfortran-13) /usr/local/bin/gfortran
-
-            # Default macos-14 image Xcode (version 15.0.1) linker causes build issues.
-            # Newer Xcode versions work, but embree recommends Apple clang <= 14 on
-            # arm64 to avoid possible "EXEC_BAD_INSTRUCTION" runtime exceptions:
-            # https://github.com/embree/embree/releases/tag/v4.3.1
-            sudo xcode-select -switch /Applications/Xcode_14.3.1.app
           fi
 
           # Install libomp 11.1.0 from old brew bottle for x64 catalina (10.15)
           # / arm64 big sur (11.0). Directly installing the Ruby formula will
           # install for the current OS.
           # https://github.com/microsoft/LightGBM/issues/4229
           if [[ ${{ runner.arch}} == "X64"  ]]; then
-            brew unlink libomp
             # x64 catalina (10.15) bottle
             export LIBOMP_BOTTLE_HASH=45a5aa653bd45bd5ff5858580b1a4670c4b5a51ea29d68d45a53f72f56010e05
           else # ARM64
@@ -142,7 +135,7 @@ jobs:
     permissions:
       contents: write  # Release upload
     name: Fuse x64 and ARM64 viewer app
-    runs-on: [macos-12]
+    runs-on: [macos-13]
     needs: [MacOS]
     steps:
       - name: Checkout source code  # for gh release upload
@@ -191,9 +184,9 @@ jobs:
       fail-fast: false
       # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6
       matrix:
-        # macos-12 is Intel runner, macos-14 is Apple Silicon
+        # macos-13 is Intel runner, macos-14 is Apple Silicon
         # https://github.com/actions/runner-images
-        os: [macos-12, macos-14]
+        os: [macos-13, macos-14]
         python_version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         is_main:
           - ${{ github.ref == 'refs/heads/main' }}
@@ -260,12 +253,10 @@ jobs:
           # Fix macos-14 arm64 runner image issues, see comments in MacOS job.
           if [[ ${{ runner.arch}} == "ARM64"  ]]; then
             ln -s $(which gfortran-13) /usr/local/bin/gfortran
-            sudo xcode-select -switch /Applications/Xcode_14.3.1.app
           fi
 
           # Install libomp 11.1.0. See comment above.
           if [[ ${{ runner.arch}} == "X64"  ]]; then
-            brew unlink libomp
             # x64 catalina (10.15) bottle
             export LIBOMP_BOTTLE_HASH=45a5aa653bd45bd5ff5858580b1a4670c4b5a51ea29d68d45a53f72f56010e05
           else # ARM64
@@ -308,7 +299,7 @@ jobs:
     name: Fuse universal2 wheel
     permissions:
       contents: write  # Release upload
-    runs-on: [macos-12]
+    runs-on: [macos-13]
     needs: [build-wheel]
     strategy:
       fail-fast: false
@@ -379,7 +370,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, macos-14]
+        os: [macos-13, macos-14]
         python_version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         is_main:
           - ${{ github.ref == 'refs/heads/main' }}

.github/workflows/ubuntu-cuda.yml

@@ -180,7 +180,7 @@ jobs:
           gcloud compute ssh "${INSTANCE_NAME}" \
             --zone="${GCE_ZONE}" \
             --command="ls -alh \
-                    && gsutil cp ${CCACHE_TAR_NAME}.tar.gz gs://open3d-ci-cache/"
+                    && gsutil cp ${CCACHE_TAR_NAME}.tar.xz gs://open3d-ci-cache/"
 
       - name: VM delete
         if: always()

CMakeLists.txt

@@ -476,6 +476,8 @@ if(WIN32)
     #       Then, we could use -fvisibility=hidden for Linux as well
     SET(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
     if(MSVC)
+        # MSVC standards-compliant mode: Error in PoissonRecon 3rd party header
+        # add_compile_options($<$<COMPILE_LANGUAGE:CXX>:"/permissive-">)   
         # Make sure we don't hit the 65535 object member limit with MSVC
         #
         # /bigobj allows object files with more than 65535 members

cpp/open3d/CMakeLists.txt

@@ -89,6 +89,7 @@ target_sources(Open3D PRIVATE
 open3d_ispc_target_sources_TARGET_OBJECTS(Open3D PRIVATE
     camera
     core
+    core_impl
     data
     geometry
     tgeometry
@@ -101,6 +102,7 @@ open3d_ispc_target_sources_TARGET_OBJECTS(Open3D PRIVATE
     tpipelines_kernel
     utility
     visualization
+    visualization_impl
 )
 
 if (BUILD_GUI)

cpp/open3d/core/CMakeLists.txt

@@ -41,22 +41,7 @@ target_sources(core PRIVATE
     hashmap/HashBackendBuffer.cpp
     hashmap/HashMap.cpp
     hashmap/HashSet.cpp
-    kernel/Arange.cpp
-    kernel/ArangeCPU.cpp
-    kernel/BinaryEW.cpp
-    kernel/BinaryEWCPU.cpp
-    kernel/IndexGetSet.cpp
-    kernel/IndexGetSetCPU.cpp
-    kernel/IndexReduction.cpp
-    kernel/IndexReductionCPU.cpp
     kernel/Kernel.cpp
-    kernel/NonZero.cpp
-    kernel/NonZeroCPU.cpp
-    kernel/Reduction.cpp
-    kernel/ReductionCPU.cpp
-    kernel/UnaryEW.cpp
-    kernel/UnaryEWCPU.cpp
-    kernel/UnaryEWSYCL.cpp
     linalg/AddMM.cpp
     linalg/AddMMCPU.cpp
     linalg/Det.cpp
@@ -82,11 +67,33 @@ target_sources(core PRIVATE
     nns/NNSIndex.cpp
 )
 
+# core_impl contains the implementation of core functions that are not exposed in the public API
+open3d_ispc_add_library(core_impl OBJECT)
+set_target_properties(core_impl PROPERTIES CXX_VISIBILITY_PRESET "hidden")
+
+target_sources(core_impl PRIVATE
+    kernel/Arange.cpp
+    kernel/ArangeCPU.cpp
+    kernel/BinaryEW.cpp
+    kernel/BinaryEWCPU.cpp
+    kernel/IndexGetSet.cpp
+    kernel/IndexGetSetCPU.cpp
+    kernel/IndexReduction.cpp
+    kernel/IndexReductionCPU.cpp
+    kernel/NonZero.cpp
+    kernel/NonZeroCPU.cpp
+    kernel/Reduction.cpp
+    kernel/ReductionCPU.cpp
+    kernel/UnaryEW.cpp
+    kernel/UnaryEWCPU.cpp
+    kernel/UnaryEWSYCL.cpp
+)
+
 if (BUILD_CUDA_MODULE)
     target_sources(core PRIVATE
         MemoryManagerCUDA.cpp
     )
-    target_sources(core PRIVATE
+    target_sources(core_impl PRIVATE
         hashmap/CUDA/CreateCUDAHashBackend.cu
         hashmap/CUDA/CUDAHashBackendBuffer.cu
         hashmap/CUDA/SlabNodeManager.cu
@@ -116,6 +123,8 @@ endif()
 if (BUILD_ISPC_MODULE)
     target_sources(core PRIVATE
         Indexer.ispc
+    )
+    target_sources(core_impl PRIVATE
         kernel/BinaryEWCPU.ispc
         kernel/UnaryEWCPU.ispc
     )
@@ -126,6 +135,12 @@ open3d_set_global_properties(core)
 open3d_set_open3d_lib_properties(core)
 open3d_link_3rdparty_libraries(core)
 
+open3d_show_and_abort_on_warning(core_impl)
+open3d_set_global_properties(core_impl)
+open3d_set_open3d_lib_properties(core_impl)
+open3d_link_3rdparty_libraries(core_impl)
+
 if(BUILD_CUDA_MODULE)
     target_include_directories(core SYSTEM PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    target_include_directories(core_impl SYSTEM PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
 endif()

cpp/open3d/core/SYCLContext.cpp

@@ -54,22 +54,22 @@ SYCLContext::SYCLContext() {
     } catch (const sycl::exception &e) {
     }
 
-    if (devices_.size() == 0) {
-        // SYCL CPU fallback.
-        // This could happen if the Intel GPGPU driver is not installed or if
-        // your CPU does not have integrated GPU.
-        try {
-            const sycl::device &sycl_device =
-                    sycl::device(sycl::cpu_selector_v);
-            const Device open3d_device = Device("SYCL:0");
+    // SYCL CPU fallback.
+    // This could happen if the Intel GPGPU driver is not installed or if
+    // your CPU does not have integrated GPU.
+    try {
+        if (devices_.size() == 0) {
             utility::LogWarning(
                     "SYCL GPU device is not available, falling back to SYCL "
                     "host device.");
-            devices_.push_back(open3d_device);
-            device_to_sycl_device_[open3d_device] = sycl_device;
-            device_to_default_queue_[open3d_device] = sycl::queue(sycl_device);
-        } catch (const sycl::exception &e) {
         }
+        const sycl::device &sycl_device = sycl::device(sycl::cpu_selector_v);
+        const Device open3d_device =
+                Device("SYCL:" + std::to_string(devices_.size()));
+        devices_.push_back(open3d_device);
+        device_to_sycl_device_[open3d_device] = sycl_device;
+        device_to_default_queue_[open3d_device] = sycl::queue(sycl_device);
+    } catch (const sycl::exception &e) {
     }
 
     if (devices_.size() == 0) {

cpp/open3d/core/SYCLUtils.cpp

@@ -118,14 +118,32 @@ static std::string SYCLDeviceToString(const sycl::device &device) {
 
 void PrintSYCLDevices(bool print_all) {
 #ifdef BUILD_SYCL_MODULE
-    const char *filter = std::getenv("SYCL_DEVICE_FILTER");
+    const char *filter = std::getenv("ONEAPI_DEVICE_SELECTOR");
     if (filter) {
         utility::LogWarning(
-                "SYCL_DEVICE_FILTER environment variable is set to {}. To see "
-                "the correct device id, please unset SYCL_DEVICE_FILTER.",
+                "ONEAPI_DEVICE_SELECTOR environment variable is set to {}. To "
+                "see the correct device id, please unset "
+                "ONEAPI_DEVICE_SELECTOR.",
                 filter);
     }
 
+    int nd = 0;
+    utility::LogInfo("# Open3D SYCL device");
+    try {
+        utility::LogInfo(
+                "- Device(\"SYCL:{}\"): {}", nd,
+                SYCLDeviceToString(sycl::device(sycl::gpu_selector_v)));
+        ++nd;
+    } catch (const sycl::exception &e) {
+    }
+    try {
+        utility::LogInfo("# Open3D SYCL device (CPU fallback)");
+        utility::LogInfo(
+                "- Device(\"SYCL:{}\"): {}", nd,
+                SYCLDeviceToString(sycl::device(sycl::cpu_selector_v)));
+    } catch (const sycl::exception &e) {
+        if (nd == 0) utility::LogInfo("- Device(\"SYCL:0\"): N/A");
+    }
     if (print_all) {
         utility::LogInfo("# All SYCL devices");
         const std::vector<sycl::platform> &platforms =
@@ -168,24 +186,6 @@ void PrintSYCLDevices(bool print_all) {
         } catch (const sycl::exception &e) {
             utility::LogInfo("- sycl::accelerator_selector_v: N/A");
         }
-
-        utility::LogInfo("# Open3D SYCL device");
-        try {
-            const sycl::device &device = sycl::device(sycl::gpu_selector_v);
-            utility::LogInfo("- Device(\"SYCL:0\"): {}",
-                             SYCLDeviceToString(device));
-        } catch (const sycl::exception &e) {
-            utility::LogInfo("- Device(\"SYCL:0\"): N/A");
-        }
-    } else {
-        utility::LogInfo("# Open3D SYCL device");
-        try {
-            const sycl::device &device = sycl::device(sycl::gpu_selector_v);
-            utility::LogInfo("- Device(\"SYCL:0\"): {}",
-                             SYCLDeviceToString(device));
-        } catch (const sycl::exception &e) {
-            utility::LogInfo("- Device(\"SYCL:0\"): N/A");
-        }
     }
 
 #else

cpp/open3d/geometry/SurfaceReconstructionBallPivoting.cpp

@@ -17,6 +17,7 @@
 
 namespace open3d {
 namespace geometry {
+namespace {
 
 class BallPivotingVertex;
 class BallPivotingEdge;
@@ -733,6 +734,7 @@ class BallPivoting {
     std::vector<BallPivotingVertexPtr> vertices;
     std::shared_ptr<TriangleMesh> mesh_;
 };
+}  // namespace
 
 std::shared_ptr<TriangleMesh> TriangleMesh::CreateFromPointCloudBallPivoting(
         const PointCloud& pcd, const std::vector<double>& radii) {

cpp/open3d/geometry/SurfaceReconstructionPoisson.cpp

@@ -39,7 +39,7 @@
 
 namespace open3d {
 namespace geometry {
-namespace poisson {
+namespace {
 
 // The order of the B-Spline used to splat in data for color interpolation
 static const int DATA_DEGREE = 0;
@@ -713,7 +713,7 @@ void Execute(const open3d::geometry::PointCloud& pcd,
                       Time() - startTime, FEMTree<Dim, Real>::MaxMemoryUsage());
 }
 
-}  // namespace poisson
+}  // namespace
 
 std::tuple<std::shared_ptr<TriangleMesh>, std::vector<double>>
 TriangleMesh::CreateFromPointCloudPoisson(const PointCloud& pcd,
@@ -722,10 +722,9 @@ TriangleMesh::CreateFromPointCloudPoisson(const PointCloud& pcd,
                                           float scale,
                                           bool linear_fit,
                                           int n_threads) {
-    static const BoundaryType BType = poisson::DEFAULT_FEM_BOUNDARY;
+    static const BoundaryType BType = DEFAULT_FEM_BOUNDARY;
     typedef IsotropicUIntPack<
-            poisson::DIMENSION,
-            FEMDegreeAndBType</* Degree */ 1, BType>::Signature>
+            DIMENSION, FEMDegreeAndBType</* Degree */ 1, BType>::Signature>
             FEMSigs;
 
     if (!pcd.HasNormals()) {
@@ -746,8 +745,8 @@ TriangleMesh::CreateFromPointCloudPoisson(const PointCloud& pcd,
 
     auto mesh = std::make_shared<TriangleMesh>();
     std::vector<double> densities;
-    poisson::Execute<float>(pcd, mesh, densities, static_cast<int>(depth),
-                            width, scale, linear_fit, FEMSigs());
+    Execute<float>(pcd, mesh, densities, static_cast<int>(depth), width, scale,
+                   linear_fit, FEMSigs());
 
     ThreadPool::Terminate();