CMake: properly fix build on macOS with Ninja (related to response files)

On macOS several build-related programs (`cc`, `ar`) will not accept
the linker line if too many files have been added to it in the
DYNAMIC_ARCH case. (This now occurs on all platforms where macOS
runs / used to run, as there are enough ARM variants that are
build with DYNAMIC_ARCH that the number of files exceeds the
intrinsic limit on macOS.)

The workaround for this is to use response files that contain the
file list themselves, and pass these to the build system. For
dynamic libraries this works fine, but for static libraries it
doesn't because `ar` doesn't accept response files on macOS. For this
reason there was previously a workaround in CMakeLists.txt to handle
this, but the workaround didn't take into consideration that when
building only a shared library it is not actually necessary, and
when building static libraries the response file CMake generates for
Ninja is at a different location than the response files CMake
generates for Unix Makefiles.

This commit cleans this all up by setting the necessary options for
the response files to be properly generated, and adjusts the additional
workaround to only be applicable in the case a static library is
built. The workaround is also adjusted to handle the Ninja case.

Fixes GitHub issue #5775

Author: Christian Seiler

CMakeLists.txt

@@ -243,6 +243,30 @@ if (${DYNAMIC_ARCH})
   endforeach()
 endif ()
 
+# Work around Apple issue with "argument list too long" when linking.
+# Seen for DYNAMIC_ARCH when there are a _LOT_ of variants build, previously
+# mostly with older versions on POWERPC or Intel CPUs, but with newer ARM
+# CPUs and newer compilers this will also occur on ARM.
+#
+# This must happen before the add_library() calls, otherwise this will have
+# no effect.
+#
+# Unfortunately, while this workaround will work for shared libraries (with
+# either Ninja oder Unix Makefiles), but not for static libraries, as ar
+# does not accept response files. So for the case where we (also) build
+# static libraries we need an additional workaround, see below. (The
+# generation of the response files by CMake must still be enabled, so this
+# is required regardless.)
+if(APPLE)
+  set(CMAKE_C_USE_RESPONSE_FILE_FOR_OBJECTS ON)
+  set(CMAKE_C_USE_RESPONSE_FILE_FOR_LIBRARIES ON)
+  if(NOT NOFORTRAN)
+    set(CMAKE_Fortran_USE_RESPONSE_FILE_FOR_OBJECTS ON)
+    set(CMAKE_Fortran_USE_RESPONSE_FILE_FOR_LIBRARIES ON)
+  endif()
+  set(CMAKE_NINJA_FORCE_RESPONSE_FILE ON)
+endif()
+
 # add objects to the openblas lib
 if(NOT NO_LAPACK)
   add_library(LAPACK_OVERRIDES OBJECT ${LA_SOURCES})
@@ -308,20 +332,22 @@ if (USE_OPENMP)
   endif()
 endif()
 
-# Fix "Argument list too long" for macOS - mostly seen with older OS versions on POWERPC or Intel CPUs 
-if(APPLE AND "${CMAKE_GENERATOR}" MATCHES ".*Makefiles")
-  # Use response files to get around the ARG_MAX limit, unless using the Ninja generator
-  set(CMAKE_C_USE_RESPONSE_FILE_FOR_OBJECTS 1)
+# Second part of "Argument list too long" fix when static libraries are
+# built. See above for details.
+if(APPLE AND BUILD_STATIC_LIBS)
   # Always build static library first
-  if(BUILD_STATIC_LIBS)
-    set(STATIC_PATH "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/lib${OpenBLAS_LIBNAME}.a")
+  set(STATIC_PATH "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/lib${OpenBLAS_LIBNAME}.a")
+  # Handle differences in how many response files are generated and/or where
+  # they are stored for Ninja / Unix Makefiles.
+  if("${CMAKE_GENERATOR}" MATCHES "Ninja")
+    set(CREATE_STATIC_LIBRARY_COMMAND
+      "sh -c 'cat ${CMAKE_BINARY_DIR}/CMakeFiles/openblas_static.rsp | xargs -n 1024 ${CMAKE_AR} -ru ${STATIC_PATH} && exit 0' "
+      "sh -c '${CMAKE_AR} -rs ${STATIC_PATH} ${CMAKE_BINARY_DIR}/driver/others/CMakeFiles/driver_others.dir/xerbla.c.o && exit 0' ")
   else()
-    add_library(${OpenBLAS_LIBNAME}_static STATIC ${TARGET_OBJS} ${OpenBLAS_DEF_FILE})
-    set(STATIC_PATH "lib${OpenBLAS_LIBNAME}.a")
+    set(CREATE_STATIC_LIBRARY_COMMAND
+      "sh -c 'cat ${CMAKE_BINARY_DIR}/CMakeFiles/${OpenBLAS_LIBNAME}_static.dir/objects*.rsp | xargs -n 1024 ${CMAKE_AR} -ru ${STATIC_PATH} && exit 0' "
+      "sh -c '${CMAKE_AR} -rs ${STATIC_PATH} ${CMAKE_BINARY_DIR}/driver/others/CMakeFiles/driver_others.dir/xerbla.c.o && exit 0' ")
   endif()
-  set(CREATE_STATIC_LIBRARY_COMMAND
-    "sh -c 'cat ${CMAKE_BINARY_DIR}/CMakeFiles/${OpenBLAS_LIBNAME}_static.dir/objects*.rsp | xargs -n 1024 ${CMAKE_AR} -ru ${STATIC_PATH} && exit 0' "
-    "sh -c '${CMAKE_AR} -rs ${STATIC_PATH} ${CMAKE_BINARY_DIR}/driver/others/CMakeFiles/driver_others.dir/xerbla.c.o && exit 0' ")
   if(BUILD_SHARED_LIBS)
     add_dependencies(${OpenBLAS_LIBNAME}_shared ${OpenBLAS_LIBNAME}_static)
     set(SHARED_PATH "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/libopenblas.${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.dylib")
@@ -332,8 +358,7 @@ if(APPLE AND "${CMAKE_GENERATOR}" MATCHES ".*Makefiles")
     set(OMP_LIB "")
   endif()
   if(NOT NOFORTRAN)
-    set(CMAKE_Fortran_USE_RESPONSE_FILE_FOR_OBJECTS 1)
-	set(CMAKE_Fortran_CREATE_STATIC_LIBRARY ${CREATE_STATIC_LIBRARY_COMMAND})
+    set(CMAKE_Fortran_CREATE_STATIC_LIBRARY ${CREATE_STATIC_LIBRARY_COMMAND})
     if(BUILD_SHARED_LIBS)
       set(CMAKE_Fortran_CREATE_SHARED_LIBRARY
 	"sh -c 'echo \"\" | ${CMAKE_Fortran_COMPILER} -o dummy.o -c -x f95-cpp-input - '"