Make a JSON manifest the source of truth for help

Viewer/ecflowUI/src/CommandDesignerWidget.cpp

@@ -15,6 +15,7 @@
 #include "CustomCommandHandler.hpp"
 #include "NodeExpression.hpp"
 #include "NodeQueryResult.hpp"
+#include "ecflow/base/HelpCatalog.hpp"
 #include "ecflow/core/Child.hpp"
 #include "ecflow/core/Str.hpp"
 
@@ -228,16 +229,14 @@ void CommandDesignerWidget::showCommandHelp(QListWidgetItem* item, bool showFull
         );
 
     if (od) {
-        // get the description, but only take the first line
-        std::vector<std::string> lines;
-        ecf::algorithm::split_at(lines, od->description(), "\n");
-        if (!lines.empty()) {
-            QString text = qCommand + QString(": ");
-            commandHelpLabel_->setText(text + QString::fromStdString(lines[0]));
-        }
+        std::string summary = ecf::HelpCatalog::summary_for(od->long_name()).value_or(ecf::HelpCatalog::not_provided);
+        QString text        = qCommand + QString(": ");
+        commandHelpLabel_->setText(text + QString::fromStdString(summary));
 
         if (showFullHelp) {
-            commandManPage_->setText(qCommand + "\n\n" + QString::fromStdString(od->description()));
+            std::string description =
+                ecf::HelpCatalog::description_for(od->long_name()).value_or(ecf::HelpCatalog::not_provided);
+            commandManPage_->setText(qCommand + "\n\n" + QString::fromStdString(description));
         }
     }
     else {

cmake/CompilerOptions.cmake

@@ -83,24 +83,28 @@ if (HAVE_WARNINGS)
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:GNU>,$<VERSION_GREATER_EQUAL:$<CXX_COMPILER_VERSION>,9.0.0>>:-Wno-deprecated-copy> # silence warnings in Qt5 related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:GNU>>:-Wno-unused-result>
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:GNU>>:-Wno-unused-parameter>
+    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:GNU>>:-Wno-overlength-strings> # silence warning for the embedded CLI help manifest string literal (generated_client_help.hpp)
     ## Clang (MacOS Homebrew, AMD Clang-base)
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-deprecated-copy-with-user-provided-copy> # silence warnings in Qt5 related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-missing-field-initializers> # silence warning in Boost.Python related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-unused-parameter>
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>,$<VERSION_GREATER_EQUAL:$<CXX_COMPILER_VERSION>,14.0.0>>:-Wno-c++20-attribute-extensions> # silence warning in Qt6 related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>,$<VERSION_GREATER_EQUAL:$<CXX_COMPILER_VERSION>,21.0.0>>:-Wno-character-conversion> # silence warning in Qt6 related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>,$<VERSION_GREATER_EQUAL:$<CXX_COMPILER_VERSION>,22.0.0>>:-Wno-c2y-extensions> # silence warning in Boost related headers (__COUNTER__)
+    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-overlength-strings> # silence warning for the embedded CLI help manifest string literal (generated_client_help.hpp)
     ## Clang (MacOS AppleClang)
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:AppleClang>>:-Wno-deprecated-copy-with-user-provided-copy> # silence warnings in Qt5 related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:AppleClang>>:-Wno-missing-field-initializers> # silence warning in Boost.Python related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:AppleClang>>:-Wno-unused-parameter>
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:AppleClang>,$<VERSION_GREATER_EQUAL:$<CXX_COMPILER_VERSION>,14.0.0>>:-Wno-c++20-attribute-extensions> # silence warning in Qt6 related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:AppleClang>,$<VERSION_GREATER_EQUAL:$<CXX_COMPILER_VERSION>,21.0.0>>:-Wno-character-conversion> # silence warning in Qt6 related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:AppleClang>,$<VERSION_GREATER_EQUAL:$<CXX_COMPILER_VERSION>,22.0.0>>:-Wno-c2y-extensions> # silence warning in Boost related headers (__COUNTER__)
+    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:AppleClang>>:-Wno-overlength-strings> # silence warning for the embedded CLI help manifest string literal (generated_client_help.hpp)
     ## Clang (Intel Clang-based)
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:IntelLLVM>>:-Wno-deprecated-copy-with-user-provided-copy> # silence warnings in Qt5 related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:IntelLLVM>>:-Wno-missing-field-initializers> # silence warning in Boost.Python related headers
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:IntelLLVM>>:-Wno-unused-parameter>
+    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:IntelLLVM>>:-Wno-overlength-strings> # silence warning for the embedded CLI help manifest string literal (generated_client_help.hpp)
   )
 
 endif ()

cmake/GenerateClientHelp.cmake

@@ -0,0 +1,75 @@
+# Copyright 2009- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+#
+# Generates a C++ header embedding docs/client_api/help.json as a raw string
+# literal, so ecflow_client can carry its own CLI help data without reading an
+# external file at runtime.
+#
+# Invoked as a build-time custom command (not configure_file), so the header
+# regenerates whenever the manifest changes, without requiring a full CMake
+# reconfigure.
+#
+# Expected variables (passed via -D on the command line):
+#   MANIFEST   path to docs/client_api/help.json
+#   OUTPUT     path to the generated header to write
+#
+
+if(NOT DEFINED MANIFEST)
+  message(FATAL_ERROR "GenerateClientHelp.cmake: MANIFEST is not set")
+endif()
+if(NOT DEFINED OUTPUT)
+  message(FATAL_ERROR "GenerateClientHelp.cmake: OUTPUT is not set")
+endif()
+
+file(READ "${MANIFEST}" manifest_content)
+
+if(manifest_content MATCHES "\\)ecflow_help\"")
+  message(FATAL_ERROR
+    "GenerateClientHelp.cmake: ${MANIFEST} contains the raw-string delimiter "
+    ")ecflow_help\", which would break the generated header. Pick a different "
+    "delimiter here and in generated_client_help.hpp's consumers.")
+endif()
+
+file(WRITE "${OUTPUT}" "/*
+ * Copyright 2009- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation
+ * nor does it submit to any jurisdiction.
+ */
+
+/*
+ * Generated from docs/client_api/help.json -- see cmake/GenerateClientHelp.cmake.
+ *
+ * DO NOT EDIT DIRECTLY.
+ */
+
+#ifndef ecflow_base_generated_client_help_HPP
+#define ecflow_base_generated_client_help_HPP
+
+#include <string_view>
+
+/*
+ * Workaround for EDG-based compilers (e.g. NVidia) that complain about
+ * constexpr value exceeding the maximum allowed size.
+ */
+#if defined(__EDG__)
+#define ECFLOW_HELP_CONSTEXPR const
+#else
+#define ECFLOW_HELP_CONSTEXPR constexpr
+#endif
+
+inline ECFLOW_HELP_CONSTEXPR std::string_view client_help_json = R\"ecflow_help(
+${manifest_content})ecflow_help\";
+
+#endif /* ecflow_base_generated_client_help_HPP */
+")

docs/CMakeLists.txt

@@ -22,6 +22,28 @@ else()
   ecbuild_fatal_error("Sphinx executable not present at ${SPHINX_EXECUTABLE}")
 endif()
 
+#
+# Validate the ecFlow CLI help manifest
+#
+# This is the single source of ecflow_client CLI help metadata, eventually consumed
+# both by the C++ client (embedded at build time) and by this documentation build.
+# Validation fails fast on an invalid manifest, before Sphinx or the (future)
+# manifest-driven build.py consume it.
+#
+
+add_custom_command(
+  OUTPUT validate_ecflow_client_help_manifest
+  COMMAND
+    ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/client_api/validate_help_manifest.py
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/client_api
+  USES_TERMINAL
+  DEPENDS
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/help.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/help.schema.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/validate_help_manifest.py
+)
+add_custom_target(ecflow_client_help_validate DEPENDS validate_ecflow_client_help_manifest)
+
 #
 # (Re-)generate ecFlow CLI documentation
 #
@@ -32,8 +54,9 @@ add_custom_command(
   COMMAND
     ${CMAKE_COMMAND} -E rm -rf ${CMAKE_CURRENT_SOURCE_DIR}/client_api/api/*
   COMMAND
-    ${CMAKE_COMMAND} -E env "PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH}"
-        ${CMAKE_CURRENT_SOURCE_DIR}/client_api/build.py
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/build.py
+  COMMAND
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/update_internal.py
   COMMAND
     ${CMAKE_COMMAND} -E copy_directory
       ${CMAKE_CURRENT_BINARY_DIR}/client_api/api
@@ -52,9 +75,15 @@ add_custom_command(
       ${CMAKE_CURRENT_SOURCE_DIR}/client_api
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/client_api
   USES_TERMINAL
-  DEPENDS ecflow_client
+  DEPENDS
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/build.py
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/update_internal.py
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/command_internals.rst
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/help.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/help.schema.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_api/validate_help_manifest.py
 )
-add_custom_target(ecflow_client_docs DEPENDS generate_ecflow_client_docs)
+add_custom_target(ecflow_client_docs DEPENDS generate_ecflow_client_docs ecflow_client_help_validate)
 
 
 #

docs/client_api/api/abort.rst

@@ -23,17 +23,21 @@ The following help text is generated by :code:`ecflow_client --help=abort`
 
    Mark task as aborted. For use in the '.ecf' script file *only*
    Hence the context is supplied via environment variables
-     arg1 = (optional) string(reason)
-            Optionally provide a reason why the abort was raised
 
-   If this child command is a zombie, then the default action will be to *block*.
+   Argument(s):
+
+       reason: (optional) string
+           # Optionally provide a reason why the abort was raised
+
+   If this task command is a zombie, then the default action will be to *block*.
    The default can be overridden by using zombie attributes.
    Otherwise the blocking period is defined by ECF_TIMEOUT.
 
    Usage:
-     ecflow_client --abort=reasonX
 
-   The client considers, for both user and child commands, the following environment variables:
+       --abort=<some-reason>
+
+   The client considers, for both user and task commands, the following environment variables:
 
      ECF_HOST <string> [mandatory*]
        The main server hostname; default value is 'localhost'
@@ -50,20 +54,20 @@ The following help text is generated by :code:`ecflow_client --help=abort`
    with the server, either by setting the environment variables or by specifying the
    command line options.
 
-   The following environment variables are used specifically by child commands:
+   The following environment variables are used specifically by task commands:
 
      ECF_NAME <string> [mandatory]
        Full path name to the task
      ECF_PASS <string> [mandatory]
        The job password (defined by the server, and used to authenticate client requests)
      ECF_TRYNO <int> [mandatory]
-       The run number of the job (defined by the server, and used in job/output file name generation.
-     ECF_RID <string> [mandatory]
+       The run number of the job (defined by the server, and used in job/output file name generation).
+     ECF_RID <string> [mandatory*]
        The process identifier. Supports identifying zombies and automated killing of running jobs
      ECF_TIMEOUT <int> [optional]
        Maximum time in *seconds* for client to deliver message to main server; default is 24 hours
      ECF_DENIED <any> [optional]
-       Allows task to exit with an error, upon connection failure, thus avoids ECF_TIMEOUTs wait.
+       Allows task to exit with an error, upon connection failure, thus avoids ECF_TIMEOUT's wait.
      NO_ECF <any> [optional]
        If set, ecflow_client exits immediately with success; useful to test the scripts without a server
 

docs/client_api/api/alter.rst

@@ -114,48 +114,47 @@ The following help text is generated by :code:`ecflow_client --help=alter`
 
    Usage:
 
-      ecflow_client --alter=add variable <variable-name> "value" /             # add server variable
-      ecflow_client --alter=add variable <variable-name> "value" /path/to/node # add node variable
+       --alter=add variable <variable-name> "value" /             # add server variable
+       --alter=add variable <variable-name> "value" /path/to/node # add node variable
 
-      ecflow_client --alter=add time "+00:20" /path/to/node
+       --alter=add time "+00:20" /path/to/node
 
-      ecflow_client --alter=add date "01.*.*" /path/to/node
+       --alter=add date "01.*.*" /path/to/node
 
-      ecflow_client --alter=add day "sunday"  /path/to/node
+       --alter=add day "sunday"  /path/to/node
 
-      ecflow_client --alter=add label <label-name> "label_value" /path/to/node
+       --alter=add label <label-name> "label_value" /path/to/node
 
-      ecflow_client --alter=add event <event-name> "set"|"clear" /path/to/node
+       --alter=add event <event-name> "set"|"clear" /path/to/node
 
-      ecflow_client --alter=add meter <meter-name> "<min>,<max>,value" /path/to/node
+       --alter=add meter <meter-name> "<min>,<max>,value" /path/to/node
 
-      ecflow_client --alter=add late "-s 00:01 -a 14:30 -c +00:01" /path/to/node
+       --alter=add late "-s 00:01 -a 14:30 -c +00:01" /path/to/node
 
-      ecflow_client --alter=add limit mars "100" /path/to/node
+       --alter=add limit mars "100" /path/to/node
 
-      ecflow_client --alter=add inlimit /path/to/node/withlimit:limit_name "10" /path/to/node
+       --alter=add inlimit /path/to/node/withlimit:limit_name "10" /path/to/node
 
-      ecflow_client --alter=add inlimit /path/to/node/withlimit:limit_name "-s 10" /path/to/node
+       --alter=add inlimit /path/to/node/withlimit:limit_name "-s 10" /path/to/node
 
-      ecflow_client --alter=add inlimit /path/to/node/withlimit:limit_name "-n 10" /path/to/node
+       --alter=add inlimit /path/to/node/withlimit:limit_name "-n 10" /path/to/node
 
-      # zombie attributes have the following structure:
-        `zombie_type`:(`client_side_action` | `server_side_action`):`child`:`zombie_life_time`
-         zombie_type        = "user" | "ecf" | "path" | "ecf_pid" | "ecf_passwd" | "ecf_pid_passwd"
-         client_side_action = "fob" | "fail" | "block"
-         server_side_action = "adopt" | "delete" | "kill"
-         child              = "init" | "event" | "meter" | "label" | "wait" | "abort" | "complete" | "queue"
-         zombie_life_time   = unsigned integer default: user(300), ecf(3600), path(900)  minimum is 60
+       # zombie attributes have the following structure:
+         `zombie_type`:(`client_side_action` | `server_side_action`):`child`:`zombie_life_time`
+          zombie_type        = "user" | "ecf" | "path" | "ecf_pid" | "ecf_passwd" | "ecf_pid_passwd"
+          client_side_action = "fob" | "fail" | "block"
+          server_side_action = "adopt" | "delete" | "kill"
+          child              = "init" | "event" | "meter" | "label" | "wait" | "abort" | "complete" | "queue"
+          zombie_life_time   = unsigned integer default: user(300), ecf(3600), path(900)  minimum is 60
 
-      ecflow_client --alter=add zombie "ecf:fail::" /path/to/node     # ask system zombies to fail
-      ecflow_client --alter=add zombie "user:fail::" /path/to/node    # ask user generated zombies to fail
-      ecflow_client --alter=add zombie "path:fail::" /path/to/node    # ask path zombies to fail
+       --alter=add zombie "ecf:fail::" /path/to/node     # ask system zombies to fail
+       --alter=add zombie "user:fail::" /path/to/node    # ask user generated zombies to fail
+       --alter=add zombie "path:fail::" /path/to/node    # ask path zombies to fail
 
-      ecflow_client --alter=delete variable FRED /path/to/node  # delete variable FRED
-      ecflow_client --alter=delete variable      /path/to/node  # delete *ALL* variables on the given snode
+       --alter=delete variable FRED /path/to/node  # delete variable FRED
+       --alter=delete variable      /path/to/node  # delete *ALL* variables on the given snode
 
-
-   The client considers, for both user and child commands, the following environment variables:
+   The client considers, for both user and task commands, the following environment variables:
 
      ECF_HOST <string> [mandatory*]
        The main server hostname; default value is 'localhost'

docs/client_api/api/archive.rst

@@ -53,12 +53,13 @@ The following help text is generated by :code:`ecflow_client --help=archive`
      endfamily
    endsuite
 
-   Usage::
-      --archive=/s1           # archive suite s1
-      --archive=/s1/f1 /s2    # archive family /s1/f1 and suite /s2
-      --archive=force /s1 /s2 # archive suites /s1,/s2 even if they have active tasks
+   Usage:
 
-   The client considers, for both user and child commands, the following environment variables:
+       --archive=/s1           # archive suite s1
+       --archive=/s1/f1 /s2    # archive family /s1/f1 and suite /s2
+       --archive=force /s1 /s2 # archive suites /s1,/s2 even if they have active tasks
+
+   The client considers, for both user and task commands, the following environment variables:
 
      ECF_HOST <string> [mandatory*]
        The main server hostname; default value is 'localhost'

docs/client_api/api/begin.rst

@@ -23,18 +23,22 @@ The following help text is generated by :code:`ecflow_client --help=begin`
 
    Begin playing the definition in the server.
    Expects zero or a single quoted string.
-     arg1 = suite-name | Nothing | force
-            play the chosen suite, if no arg specified, play all suites, in the definition
-            force means reset the begin status on the suites and bypass checks.
-            This is only required if suite-name is provide as the first argument
-            Using force can cause the creation of zombies
+
+   Argument(s):
+
+       suite-name: (optional)
+           # The name of the selected suite; if not specified, means all suites.
+           # Including `--force` in the value means reset the begin status and bypass checks.
+           # Important: using `--force` might cause the appearance of zombies.
+
    Usage:
-   --begin                     # will begin all suites
-   --begin="--force"         # reset and then begin all suites, bypassing any checks. Note: string must be quoted
-   --begin="mySuite"         # begin playing suite of name 'mySuite'
-   --begin="mySuite --force" # reset and begin playing suite 'mySuite', bypass check
 
-   The client considers, for both user and child commands, the following environment variables:
+       --begin                     # will begin all suites
+       --begin="--force"         # reset and then begin all suites, bypassing any checks. Note: string must be quoted
+       --begin="mySuite"         # begin playing suite of name 'mySuite'
+       --begin="mySuite --force" # reset and begin playing suite 'mySuite', bypass check
+
+   The client considers, for both user and task commands, the following environment variables:
 
      ECF_HOST <string> [mandatory*]
        The main server hostname; default value is 'localhost'