Merge pull request #3428 from jimklimov/issue-3108

CI: Optimize use of autoconf caching

Author: jimklimov

Makefile.am

@@ -136,22 +136,22 @@ install-fanout-cleanup: install-recursive @dotMAKE@
 # Called from generated-sources-with-a-touch:
 touch-include-all-nut_version-generated.timestamp:
 	@[ -s include/nut_version.h ]
-	@touch -r include/nut_version.h -d '-10 seconds' include/.all.nut_version-generated.timestamp && exit ; \
+	@touch -r include/nut_version.h -d '-10 seconds' include/.all.nut_version-generated.timestamp 2>/dev/null && exit ; \
 	 touch -d '1970-01-01T00:00:00' include/.all.nut_version-generated.timestamp && exit ; \
 	 touch include/.all.nut_version-generated.timestamp
 
 touch-clients-all-libupsclient_version-generated.timestamp:
 	@[ -s clients/libupsclient-version.h ]
-	@touch -r clients/libupsclient-version.h -d '-10 seconds' clients/.all.libupsclient_version-generated.timestamp && exit ; \
+	@touch -r clients/libupsclient-version.h -d '-10 seconds' clients/.all.libupsclient_version-generated.timestamp 2>/dev/null && exit ; \
 	 touch -d '1970-01-01T00:00:00' clients/.all.libupsclient_version-generated.timestamp && exit ; \
 	 touch clients/.all.libupsclient_version-generated.timestamp
 
 touch-docs-man-all-linkman-generated.timestamp:
 	@[ -s docs/man/linkman-driver-names.txt ] && [ -s docs/man/linkman-drivertool-names.txt ]
 	@if test -n "`find docs/man/linkman-driver-names.txt -newer docs/man/linkman-drivertool-names.txt`" ; then \
-	    touch -r docs/man/linkman-drivertool-names.txt -d '-10 seconds' docs/man/.all.nut_linkman-generated.timestamp && exit ; \
+	    touch -r docs/man/linkman-drivertool-names.txt -d '-10 seconds' docs/man/.all.nut_linkman-generated.timestamp 2>/dev/null && exit ; \
 	 else \
-	    touch -r docs/man/linkman-driver-names.txt -d '-10 seconds' docs/man/.all.nut_linkman-generated.timestamp && exit ; \
+	    touch -r docs/man/linkman-driver-names.txt -d '-10 seconds' docs/man/.all.nut_linkman-generated.timestamp 2>/dev/null && exit ; \
 	 fi ; \
 	 touch -d '1970-01-01T00:00:00' docs/man/.all.nut_linkman-generated.timestamp && exit ; \
 	 touch docs/man/.all.nut_linkman-generated.timestamp

ci_build.adoc

@@ -106,10 +106,24 @@ It can be stored outside of the build area(s), under `${CI_CACHE_NUT_BASEDIR}`
 by caller (e.g. via `~/.profile` of a developer) or guessed from environment
 variables typical for CI runs.
 
+Note there is also a separate `DO_USE_NUTCI_CACHE_DEBUG=yes` (or `...=first`)
+toggle, which aims to keep a copy of not only the `config.cache`, but also of
+`config.log` and `config.h` files, as well as a copy of the help text from
+`configure` script, from a first run with new cache hash for troubleshooting.
+
+A further `DO_USE_NUTCI_CACHE_DEBUG=each` keeps such files time-stamped, from
+every run (primarily to help determine if re-runs do modify some detection
+results due to mis-interpretation of cached data -- i.e. to help fix the `m4`
+scripts and/or `configure.ac` source file).
+
+WARNING: It is up to the caller (developer, CI) to ensure that third-party
+dependencies and tools (and their flags delivered by `pkg-config`, or module
+paths for interpreted languages, etc.) remain the same between cached runs.
+
 If something acts up, or the build environment changes invisibly to the
 script, an `export DO_CLEAN_NUTCI_CACHE_BEFORE=yes` (or a more targeted
-`DO_CLEAN_AUTOCONF_CACHE_BEFORE`, since NIT uses similar logic and general
-toggle to retain generated test certificates) should fix that.
+`export DO_CLEAN_AUTOCONF_CACHE_BEFORE=yes`, since NIT uses similar logic
+and general toggle to retain generated test certificates) should fix that.
 
 You can also always nuke the `${CI_CACHE_NUT_BASEDIR}` directory, or the
 `AUTOCONF_*` subdirectories and stale lock files under it.

ci_build.sh

@@ -1242,6 +1242,13 @@ get_CI_CACHE_NUT_HASHDIR_CFG_OPT() {
     fi
 }
 
+get_CI_CACHE_NUT_HASHDIR_CFG_OPT_WITH_ENV() {
+    # Hash environment variables which we commonly use and that can impact
+    # the decisions of configure script as something that requires separate
+    # cache files, just in case.
+    get_CI_CACHE_NUT_HASHDIR_CFG_OPT "$* CC='$CC' CXX='$CXX' CPP='$CPP' MAKE='$MAKE' SHELL='$SHELL' CONFIG_SHELL='$CONFIG_SHELL' PATH='$PATH' LD_LIBRARY_PATH='$LD_LIBRARY_PATH' PKG_CONFIG_PATH='$PKG_CONFIG_PATH' ARCH='$ARCH' BITS='$BITS' CFLAGS='$CFLAGS' CXXFLAGS='$CXXFLAGS' CPPFLAGS='$CPPFLAGS' LDFLAGS='$LDFLAGS'"
+}
+
 configure_CI_BUILDDIR() {
     autogen_get_CONFIGURE_SCRIPT
 
@@ -1269,7 +1276,7 @@ configure_nut() {
     # Help copy-pasting build setups from CI logs to terminal:
     local CONFIG_OPTS_STR="`END=' \'; NUM=0; for F in \"${CONFIG_OPTS[@]}\" ; do NUM=$(($NUM + 1)); [ x\"$NUM\" = x\"${#CONFIG_OPTS[@]}\" ] && END=''; printf \"'%s'%s\n\" \"$F\" \"$END\" ; done`"
 
-    get_CI_CACHE_NUT_HASHDIR_CFG_OPT "${CONFIG_OPTS_STR} CC='$CC' CXX='$CXX' CPP='$CPP'"
+    get_CI_CACHE_NUT_HASHDIR_CFG_OPT_WITH_ENV "${CONFIG_OPTS_STR}"
 
     CI_CACHE_NUT_RETRIED=false
     while : ; do # Note the CI_SHELL_IS_FLAKY=true support below
@@ -1292,14 +1299,39 @@ configure_nut() {
           "${CONFIG_OPTS[@]}" \
       && echo "$0: configure phase complete (0)" >&2 \
       && {
-        if [ x"${DO_USE_AUTOCONF_CACHE}" = xyes ] && [ x"${DO_USE_AUTOCONF_CACHE_DEBUG}" = xyes ] && [ -n "${CI_CACHE_NUT_HASHDIR_CFG_OPT}" ] && [ -s "${CI_CACHE_NUT_HASHDIR_CFG}/config.cache" ] ; then
-            if [ x = x"`cat \"${CI_CACHE_NUT_HASHDIR_CFG}/config.log\" \"${CI_CACHE_NUT_HASHDIR_CFG}/config.h\"`" ] ; then
-                # Stash a copy to track evolution:
-                cp -pf "${CI_CACHE_NUT_HASHDIR_CFG}/config.cache" "${CI_CACHE_NUT_HASHDIR_CFG}/config.cache.orig"
-                # Populate on first run (may cost 1-2Mb):
-                cp -pf config.log "${CI_CACHE_NUT_HASHDIR_CFG}/"
-                cp -pf include/config.h "${CI_CACHE_NUT_HASHDIR_CFG}/"
-            fi
+        if [ x"${DO_USE_AUTOCONF_CACHE}" = xyes ] \
+        && [ -n "${CI_CACHE_NUT_HASHDIR_CFG_OPT}" ] \
+        && [ -s "${CI_CACHE_NUT_HASHDIR_CFG}/config.cache" ] \
+        ; then
+            case x"${DO_USE_AUTOCONF_CACHE_DEBUG}" in
+                xyes|xfirst)
+                    if [ x = x"`cat \"${CI_CACHE_NUT_HASHDIR_CFG}/config.log\" \"${CI_CACHE_NUT_HASHDIR_CFG}/config.h\"`" ] ; then
+                        # Stash a copy to track evolution:
+                        cp -pf "${CI_CACHE_NUT_HASHDIR_CFG}/config.cache" "${CI_CACHE_NUT_HASHDIR_CFG}/config.cache.orig"
+                        # Populate on first run (may cost 1-2Mb):
+                        cp -pf config.log "${CI_CACHE_NUT_HASHDIR_CFG}/"
+                        cp -pf include/config.h "${CI_CACHE_NUT_HASHDIR_CFG}/"
+                    fi
+
+                    # Make sure all args are declared outside cached logic:
+                    $CONFIGURE_SCRIPT --help > "${CI_CACHE_NUT_HASHDIR_CFG}/config.help" 2>&1
+                    ;;
+                xeach)
+                    TS="`date '+%s'`" && [ -n "$TS" ] && [ "$TS" -gt 0 ] \
+                    || { TS="`date | tr -d ':' | tr -d ' '`" && [ -n "$TS" ] ; } \
+                    || TS=$$
+
+                    for F in \
+                        "${CI_CACHE_NUT_HASHDIR_CFG}/config.cache" \
+                        config.log include/config.h \
+                    ; do
+                        cp -pf "$F" "${CI_CACHE_NUT_HASHDIR_CFG}/`basename \"$F\"`.$TS"
+                    done
+
+                    # Make sure all args are declared outside cached logic:
+                    $CONFIGURE_SCRIPT --help > "${CI_CACHE_NUT_HASHDIR_CFG}/config.help.$TS" 2>&1
+                    ;;
+            esac
         fi
       } && return 0 \
       || { RES_CFG=$?
@@ -1573,7 +1605,7 @@ optional_maintainer_clean_check() {
 
         # If this exports CI_CACHE_NUT_HASHDIR_CFG_OPT, the `make distcheck`
         # handlers in the stack of calls via Makefile.am should hear it
-        get_CI_CACHE_NUT_HASHDIR_CFG_OPT "${DISTCHECK_FLAGS} CC='$CC' CXX='$CXX' CPP='$CPP' DISTCHECK_TGT='maintainer-clean'"
+        get_CI_CACHE_NUT_HASHDIR_CFG_OPT_WITH_ENV "${DISTCHECK_FLAGS} DISTCHECK_TGT='maintainer-clean'"
 
         # Note: currently Makefile.am has just a dummy "distcleancheck" rule
         MAKE_RES=0
@@ -1617,7 +1649,7 @@ optional_dist_clean_check() {
 
         # If this exports CI_CACHE_NUT_HASHDIR_CFG_OPT, the `make distcheck`
         # handlers in the stack of calls via Makefile.am should hear it
-        get_CI_CACHE_NUT_HASHDIR_CFG_OPT "${DISTCHECK_FLAGS} CC='$CC' CXX='$CXX' CPP='$CPP' DISTCHECK_TGT='distclean'"
+        get_CI_CACHE_NUT_HASHDIR_CFG_OPT_WITH_ENV "${DISTCHECK_FLAGS} DISTCHECK_TGT='distclean'"
 
         # Note: currently Makefile.am has just a dummy "distcleancheck" rule
         MAKE_RES=0
@@ -2087,6 +2119,22 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-al
         CONFIG_OPTS+=("CC=${CC}")
         CONFIG_OPTS+=("CXX=${CXX}")
         CONFIG_OPTS+=("CPP=${CPP}")
+
+        # Do not let autoconf-cached re-runs (re-running the configure script
+        # due to a `make` with changed *.m4, *.am or configure.ac sources)
+        # complain that CCACHE_* vars were not previously set. This hassle
+        # comes with use of AC_ARG_VAR to mark "precious" arguments:
+        CONFIG_OPTS+=("CCACHE_NAMESPACE=${CCACHE_NAMESPACE}")
+        CONFIG_OPTS+=("CCACHE_BASEDIR=${CCACHE_BASEDIR}")
+        CONFIG_OPTS+=("CCACHE_DIR=${CCACHE_DIR}")
+        CONFIG_OPTS+=("CCACHE_PATH=${CCACHE_PATH}")
+    else
+        # Still have them declared; the configure script will probably parse
+        # them as empty/undefined and re-evaluate if situation warrants that:
+        CONFIG_OPTS+=("CCACHE_NAMESPACE=")
+        CONFIG_OPTS+=("CCACHE_BASEDIR=")
+        CONFIG_OPTS+=("CCACHE_DIR=")
+        CONFIG_OPTS+=("CCACHE_PATH=")
     fi
 
     # Build and check this project; note that zprojects always have an autogen.sh
@@ -2198,7 +2246,7 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-al
 
             # If this exports CI_CACHE_NUT_HASHDIR_CFG_OPT, the `make distcheck`
             # handlers in the stack of calls via Makefile.am should hear it
-            get_CI_CACHE_NUT_HASHDIR_CFG_OPT "${DISTCHECK_FLAGS} CC='$CC' CXX='$CXX' CPP='$CPP' DISTCHECK_TGT='$BUILD_TGT'"
+            get_CI_CACHE_NUT_HASHDIR_CFG_OPT_WITH_ENV "${DISTCHECK_FLAGS} DISTCHECK_TGT='$BUILD_TGT'"
 
             # Tell the sub-makes (likely distcheck*) to hush down
             # NOTE: Parameter pass-through was tested with:
@@ -2940,7 +2988,7 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-al
 
         # If this exports CI_CACHE_NUT_HASHDIR_CFG_OPT, the `make distcheck`
         # handlers in the stack of calls via Makefile.am should hear it
-        get_CI_CACHE_NUT_HASHDIR_CFG_OPT "${DISTCHECK_FLAGS} CC='$CC' CXX='$CXX' CPP='$CPP' DISTCHECK_TGT='$DISTCHECK_TGT'"
+        get_CI_CACHE_NUT_HASHDIR_CFG_OPT_WITH_ENV "${DISTCHECK_FLAGS} DISTCHECK_TGT='$DISTCHECK_TGT'"
 
         # Tell the sub-makes (distcheck) to hush down
         MAKEFLAGS="${MAKEFLAGS-} $MAKE_FLAGS_QUIET" \
@@ -3111,6 +3159,22 @@ bindings)
         CONFIG_OPTS+=("CC=${CC}")
         CONFIG_OPTS+=("CXX=${CXX}")
         CONFIG_OPTS+=("CPP=${CPP}")
+
+        # Do not let autoconf-cached re-runs (re-running the configure script
+        # due to a `make` with changed *.m4, *.am or configure.ac sources)
+        # complain that CCACHE_* vars were not previously set. This hassle
+        # comes with use of AC_ARG_VAR to mark "precious" arguments:
+        CONFIG_OPTS+=("CCACHE_NAMESPACE=${CCACHE_NAMESPACE}")
+        CONFIG_OPTS+=("CCACHE_BASEDIR=${CCACHE_BASEDIR}")
+        CONFIG_OPTS+=("CCACHE_DIR=${CCACHE_DIR}")
+        CONFIG_OPTS+=("CCACHE_PATH=${CCACHE_PATH}")
+    else
+        # Still have them declared; the configure script will probably parse
+        # them as empty/undefined and re-evaluate if situation warrants that:
+        CONFIG_OPTS+=("CCACHE_NAMESPACE=")
+        CONFIG_OPTS+=("CCACHE_BASEDIR=")
+        CONFIG_OPTS+=("CCACHE_DIR=")
+        CONFIG_OPTS+=("CCACHE_PATH=")
     fi
 
     # If detect_platform_PKG_CONFIG_PATH_and_FLAGS() customized anything here,

common/common.c

@@ -3183,12 +3183,17 @@ int upsnotify(upsnotify_state_t state, const char *fmt, ...)
 	 * a reload action for Type=notify-reload; for more details see
 	 * https://github.com/systemd/systemd/blob/main/src/core/service.c#L2618
 	 */
-	struct timespec monoclock_ts;
-	int got_monoclock = clock_gettime(CLOCK_MONOTONIC, &monoclock_ts);
+	struct timespec	monoclock_ts;
+	int	got_monoclock = clock_gettime(CLOCK_MONOTONIC, &monoclock_ts);
 #  endif	/* HAVE_CLOCK_GETTIME && HAVE_CLOCK_MONOTONIC */
 # endif	/* HAVE_SD_NOTIFY */
 #endif	/* WITH_LIBSYSTEMD */
 
+	/* Some code paths (build configurations/goals) do not involve these,
+	 * but easier to make-believe that we do than pepper code with ifdefs */
+	NUT_UNUSED_VARIABLE(buf);
+	NUT_UNUSED_VARIABLE(msglen);
+
 	/* Were we asked to be quiet on the console? */
 	if (upsnotify_report_verbosity < 0) {
 		char *quiet_init = getenv("NUT_QUIET_INIT_UPSNOTIFY");
@@ -3258,8 +3263,6 @@ int upsnotify(upsnotify_state_t state, const char *fmt, ...)
 
 #if defined(WITH_LIBSYSTEMD) && (WITH_LIBSYSTEMD)
 # if defined(WITHOUT_LIBSYSTEMD) && (WITHOUT_LIBSYSTEMD)
-	NUT_UNUSED_VARIABLE(buf);
-	NUT_UNUSED_VARIABLE(msglen);
 	if (!upsnotify_reported_disabled_systemd) {
 		upsdebugx(upsnotify_report_verbosity,
 			"%s: notify about state %s with libsystemd: "

configure.ac

@@ -2854,6 +2854,33 @@ NUT_ARG_WITH([nss], [enable SSL support using Mozilla NSS], [auto])
 NUT_ARG_WITH([openssl], [enable SSL support using OpenSSL], [auto])
 NUT_ARG_WITH([ssl-client-validation], [enable SSL client certificate validation ability (currently requires NSS, beware issue #3329)], [auto])
 
+dnl Optionally reset/initialize the cache values shared by several backends
+dnl if the check methods below never ran for all active code paths (one or more):
+AS_CASE([x${nut_cv_checked_libopenssl}${nut_cv_checked_libnss}x],
+    [xyesx], [],
+    [xyesyesx], [],
+        [
+        dnl Magic-format string to hold chosen SSL library version and its config-source
+        nut_cv_ssl_lib=""
+
+        nut_cv_LIBSSL_CFLAGS=""
+        nut_cv_LIBSSL_LIBS=""
+        nut_cv_LIBSSL_REQUIRES=""
+        nut_cv_LIBSSL_LDFLAGS_RPATH=""
+        nut_cv_LIBSSL_REQUIRES_SOURCE=""
+        nut_cv_LIBSSL_CFLAGS_SOURCE=""
+        nut_cv_LIBSSL_LIBS_SOURCE=""
+
+        dnl Start with what the build args said:
+        nut_cv_with_ssl="${nut_with_ssl}"
+
+        dnl Ensure re-evaluation if need be:
+        AS_UNSET([nut_cv_checked_libopenssl])
+        AS_UNSET([nut_noncv_checked_libopenssl_now])
+        AS_UNSET([nut_cv_checked_libnss])
+        AS_UNSET([nut_noncv_checked_libnss_now])
+    ])
+
 dnl ${nut_with_ssl}: any value except "yes" or "no" is treated as "auto".
 if test "${nut_with_ssl}" != "no"; then
    dnl check if either NSS or OpenSSL was explicitly requested
@@ -2889,6 +2916,17 @@ if test "${nut_with_ssl}" != "no"; then
    fi
 fi
 
+dnl Summary for re-runs:
+AS_IF([test "${nut_noncv_checked_libopenssl_now}" = no || test "${nut_noncv_checked_libnss_now}" = no], [
+    AC_MSG_NOTICE([Some LIBSSL (cached): ${nut_with_ssl} (${nut_ssl_lib})])
+    AC_MSG_NOTICE([Some LIBSSL (cached): cflags_source=${LIBSSL_CFLAGS_SOURCE} libs_source=${LIBSSL_LIBS_SOURCE}])
+    AC_MSG_NOTICE([Some LIBSSL (cached): LIBSSL_CFLAGS=${LIBSSL_CFLAGS}])
+    AC_MSG_NOTICE([Some LIBSSL (cached): LIBSSL_LIBS=${LIBSSL_LIBS}])
+    AC_MSG_NOTICE([Some LIBSSL (cached): LIBSSL_LDFLAGS_RPATH=${LIBSSL_LDFLAGS_RPATH}])
+    AC_MSG_NOTICE([Some LIBSSL (cached): requires_source=${LIBSSL_REQUIRES_SOURCE} LIBSSL_REQUIRES=${LIBSSL_REQUIRES}])
+])
+
+
 AM_CONDITIONAL(WITH_NSS, test "${nut_with_nss}" = "yes")
 AM_CONDITIONAL(WITH_OPENSSL, test "${nut_with_openssl}" = "yes")
 
@@ -6331,6 +6369,10 @@ AC_SUBST(LIBSSL_LDFLAGS_RPATH)
 AC_SUBST(LIBSSL_REQUIRES)
 AC_SUBST(LIBGD_CFLAGS)
 AC_SUBST(LIBGD_LDFLAGS)
+AC_SUBST(LIBGLIB_CFLAGS)
+AC_SUBST(LIBGLIB_LIBS)
+AC_SUBST(LIBGIO_CFLAGS)
+AC_SUBST(LIBGIO_LIBS)
 AC_SUBST(LIBNETSNMP_CFLAGS)
 AC_SUBST(LIBNETSNMP_LIBS)
 AC_SUBST(LIBREGEX_LIBS)