Merge branch 'antalya-26.3' into expand-replicated-partition-exports-columns

Author: arthurpassos

ci/docker/integration/runner/Dockerfile

@@ -80,6 +80,8 @@ org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.1,\
 org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.4.3,\
 org.apache.hadoop:hadoop-aws:3.3.4,\
 com.amazonaws:aws-java-sdk-bundle:1.12.262,\
+org.apache.hadoop:hadoop-azure:3.3.4,\
+com.microsoft.azure:azure-storage:8.6.6,\
 org.apache.spark:spark-avro_2.12:3.5.1"\
     && /spark-3.5.5-bin-hadoop3/bin/spark-shell --packages "$packages" \
     && find /root/.ivy2/ -name '*.jar' -exec ln -sf {} /spark-3.5.5-bin-hadoop3/jars/ \;

docs/en/interfaces/cli.md

@@ -836,6 +836,8 @@ All command-line options can be specified directly on the command line or as def
 | `-d [ --database ] <database>`   | Select the database to default to for this connection.                                                                                                                                                                                                                                                                             | The current database from the server settings (`default` by default)                                             |
 | `-h [ --host ] <host>`           | The hostname of the ClickHouse server to connect to. Can either be a hostname or an IPv4 or IPv6 address. Multiple hosts can be passed via multiple arguments.                                                                                                                                                                    | `localhost`                                                                                                      |
 | `--jwt <value>`                  | Use JSON Web Token (JWT) for authentication. <br/><br/>Server JWT authorization is only available in ClickHouse Cloud.                                                                                                                                                                                                            | -                                                                                                                |
+| `--jwt-command <command>`        | Shell command whose stdout is used as the JWT. Invoked on the first connect, before reconnects when the cached JWT is near expiry, and after the server rejects the cached token. See [`--jwt-command` details](#jwt-command-details) below.                                                                                       | -                                                                                                                |
+| `--jwt-command-timeout <seconds>` | Timeout for `--jwt-command`. Also settable as `<jwt-command-timeout>` in the config file; CLI wins.                                                                                                                                                                                                                                  | `30`                                                                                                            |
 | `--login[=<mode>]`       | Authenticate via OAuth2. Bare `--login` (no `=<mode>`) triggers ClickHouse Cloud automatic login — the provider is inferred from the server. To authenticate against a custom OpenID Connect provider, supply a `mode` and `--oauth-credentials`: `--login=browser` runs the Authorization Code + PKCE flow (opens a browser), `--login=device` runs the Device Authorization flow (prints a URL and short code — no browser needed). | -                                                                                                                |
 | `--oauth-credentials <path>` | Path to an OAuth2 credentials JSON file (Google Cloud Console format). Required when using `--login=browser` or `--login=device` with a custom OpenID Connect provider. See [OAuth credentials file format](#oauth-credentials-file) below. Refresh tokens are cached in `~/.clickhouse-client/oauth_cache.json` (mode `0600`). | `~/.clickhouse-client/oauth_client.json`                                                                        |
 | `--no-warnings`                  | Disable showing warnings from `system.warnings` when the client connects to the server.                                                                                                                                                                                                                                            | -                                                                                                                |
@@ -880,6 +882,18 @@ The default path is `~/.clickhouse-client/oauth_client.json`. Override it with `
 
 After a successful login the obtained refresh token is cached in `~/.clickhouse-client/oauth_cache.json` (file mode `0600`). Subsequent runs reuse the cached token silently and only open the browser or print a device code when the refresh token has expired.
 
+### `--jwt-command` details {#jwt-command-details}
+
+The command is executed via `/bin/sh -c`. Stdout is taken as the JWT (one trailing newline stripped); any human-facing output (prompts, URLs, device codes) must go to stderr — it is forwarded unbuffered to the client's stderr. Stdin is closed.
+
+The command runs on the first connect to obtain the initial token. On subsequent (re)connects the client reuses the cached token; it re-invokes the command only when (a) the cached token parses as a JWT whose `exp` claim is within 30 seconds, or (b) the server rejects the cached token with an authentication failure, in which case the client refetches the token and retries the handshake once. Opaque tokens (anything that does not parse as a JWT) and JWTs without a usable `exp` claim are reused until the server rejects them — caching/refresh in those cases is the script's responsibility.
+
+```bash
+clickhouse-client --jwt-command "curl -sS https://idp.example/token | jq -r .access_token"
+```
+
+Cannot be combined with `--jwt`, `--login`, or a non-default `--user`. Non-zero exit, empty output, or exceeding `--jwt-command-timeout` (default `30`s, overridable via `<jwt-command-timeout>` in `~/.clickhouse-client/config.xml`) fails authentication. On timeout the entire helper subprocess tree is terminated.
+
 ### Query options {#command-line-options-query}
 
 | Option                          | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |

programs/client/Client.cpp

@@ -26,6 +26,7 @@
 #include <Interpreters/Context.h>
 
 #include <Client/JWTProvider.h>
+#include <Client/CommandJWTProvider.h>
 #include <Client/ClientBaseHelpers.h>
 #include <Client/OAuthLogin.h>
 
@@ -374,10 +375,30 @@ try
     }
 
 #if USE_JWT_CPP && USE_SSL
-    if (config().getBool("cloud_oauth_pending", false) && !config().has("jwt"))
+    /// Empty-value check; `config().has(k)` returns true for empty XML elements too.
+    const bool has_jwt_command_value = !config().getString("jwt-command", "").empty();
+    const bool has_jwt_value = !config().getString("jwt", "").empty();
+
+    if (has_jwt_command_value && has_jwt_value)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "jwt-command and jwt cannot both be specified");
+
+    if (has_jwt_command_value)
+    {
+        int timeout = config().getInt("jwt-command-timeout", DEFAULT_JWT_COMMAND_TIMEOUT_SECONDS);
+        if (timeout <= 0)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "jwt-command-timeout must be positive, got {}", timeout);
+
+        jwt_provider = std::make_shared<CommandJWTProvider>(config().getString("jwt-command"), timeout);
+        config().setString("jwt", "");
+    }
+
+    if (config().getBool("cloud_oauth_pending", false) && !has_jwt_value && !has_jwt_command_value)
     {
         login();
     }
+#else
+    if (!config().getString("jwt-command", "").empty() || !config().getString("jwt", "").empty())
+        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is disabled, because ClickHouse is built without JWT or SSL support");
 #endif
 
     bool asked_password = false;
@@ -393,10 +414,16 @@ try
         {
             auto code = e.code();
 
+            /// Don't prompt for a password on a JWT auth failure.
+            const bool jwt_auth_in_use =
+                !config().getString("jwt", "").empty()
+                || !config().getString("jwt-command", "").empty();
+
             bool should_ask_password = !asked_password && is_interactive &&
                 (code == ErrorCodes::AUTHENTICATION_FAILED || code == ErrorCodes::REQUIRED_PASSWORD) &&
                 !config().has("password") && !config().getBool("ask-password", false) &&
-                !config().has("ssh-key-file");
+                !config().has("ssh-key-file") &&
+                !jwt_auth_in_use;
 
             if (should_ask_password)
             {
@@ -742,6 +769,10 @@ void Client::printHelpMessage(const OptionsDescription & options_description)
 
 void Client::addExtraOptions(OptionsDescription & options_description)
 {
+    static const std::string jwt_command_timeout_help =
+        "Timeout in seconds for --jwt-command. Default: " + std::to_string(DEFAULT_JWT_COMMAND_TIMEOUT_SECONDS)
+        + ". Also configurable as <jwt-command-timeout> in the client config file.";
+
     /// Main commandline options related to client functionality and all parameters from Settings.
     options_description.main_description->add_options()
         ("config,c", po::value<std::string>(), "config-file path (another shorthand)")
@@ -756,6 +787,11 @@ void Client::addExtraOptions(OptionsDescription & options_description)
         ("ssh-key-passphrase", po::value<std::string>(), "Passphrase for the SSH private key specified by --ssh-key-file.")
         ("quota_key", po::value<std::string>(), "A string to differentiate quotas when the user have keyed quotas configured on server")
         ("jwt", po::value<std::string>(), "Use JWT for authentication")
+        ("jwt-command", po::value<std::string>(),
+            "Shell command whose stdout is used as the JWT. Invoked on the first connect, "
+            "before reconnects when the cached JWT is near expiry, and after the server "
+            "rejects the cached token with an authentication failure.")
+        ("jwt-command-timeout", po::value<int>(), jwt_command_timeout_help.c_str())
         ("one-time-password", po::value<std::string>(), "Time-based one-time password (TOTP) for two-factor authentication")
         ("login", po::value<std::string>()->implicit_value(""),
             "Authenticate via OAuth2. Optional mode: 'browser' (auth-code + PKCE, opens browser) "
@@ -931,75 +967,108 @@ void Client::processOptions(
         config().setString("jwt", options["jwt"].as<std::string>());
         config().setString("user", "");
     }
+    if (options.contains("jwt-command-timeout"))
+        config().setInt("jwt-command-timeout", options["jwt-command-timeout"].as<int>());
+
+    if (options.contains("jwt-command"))
+    {
+#if USE_JWT_CPP && USE_SSL
+        if (options.contains("jwt"))
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "--jwt-command and --jwt cannot both be specified");
+        if (options.contains("login"))
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "--jwt-command and --login cannot both be specified");
+        if (!options["user"].defaulted())
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "User and JWT flags can't be specified together");
+
+        /// Defer execution to Client::main, after processConfig has loaded the XML config.
+        /// Reading config().getInt("jwt-command-timeout", ...) here would miss the XML value.
+        config().setString("jwt-command", options["jwt-command"].as<std::string>());
+        config().setString("user", "");
+#else
+        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is disabled, because ClickHouse is built without JWT or SSL support");
+#endif
+    }
     if (options.count("oauth-credentials") && !options.count("login"))
         throw Exception(
             ErrorCodes::BAD_ARGUMENTS,
             "--oauth-credentials requires --login=browser or --login=device");
 
     if (options.count("login"))
     {
-        /// Reject mixed JWT + --login from any source. The --login branch below
-        /// ends up calling config().setString("jwt", jwt_provider->getJWT()),
-        /// which would silently overwrite a JWT supplied via --jwt or via the
-        /// XML config file. config().has("jwt") covers both: CLI --jwt was
-        /// already copied into config() above, and a <jwt> element in
-        /// ~/.clickhouse-client/config.xml is loaded into config() at startup.
-        if (config().has("jwt"))
-            throw Exception(
-                ErrorCodes::BAD_ARGUMENTS,
-                "--login cannot be combined with a JWT (provided via --jwt or in the config file)");
-
-        const std::string login_mode = options["login"].as<std::string>();
-        if (!login_mode.empty() && login_mode != "browser" && login_mode != "device")
-            throw Exception(
-                ErrorCodes::BAD_ARGUMENTS,
-                "--login value must be 'browser' or 'device', got '{}'",
-                login_mode);
+        bool defer_to_existing_jwt = false;
 
 #if USE_JWT_CPP && USE_SSL
-        if (!options["user"].defaulted())
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "--user and --login cannot both be specified");
-
-        // Bare --login (empty mode, including auto-added for *.clickhouse.cloud) → cloud path.
-        // Explicit --login=browser or --login=device (or --oauth-credentials) → credentials-file
-        // OIDC path. This prevents the credentials file from hijacking the cloud auto-login.
-        const bool use_credentials_file
-            = !login_mode.empty()
-            || options.count("oauth-credentials");
+        /// --login would overwrite config["jwt"]; reject if a JWT is already configured.
+        /// Auto-added --login (cloud endpoint, no CLI auth) defers silently to it instead.
+        const bool jwt_already_configured
+            = !config().getString("jwt", "").empty()
+            || !config().getString("jwt-command", "").empty();
 
-        if (use_credentials_file)
+        if (jwt_already_configured)
         {
-            const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
-            const std::string default_creds_path = home_path_cstr
-                ? std::string(home_path_cstr) + "/.clickhouse-client/oauth_client.json"
-                : "";
-
-            const std::string creds_path = options.count("oauth-credentials")
-                ? options["oauth-credentials"].as<std::string>()
-                : default_creds_path;
-
-            auto creds = loadOAuthCredentials(creds_path);
-            const auto mode = (login_mode == "device") ? OAuthFlowMode::Device : OAuthFlowMode::AuthCode;
-
-            // createOAuthJWTProvider runs the initial flow (trying the cached
-            // refresh token first) and returns a provider that Connection can
-            // call to refresh the id_token transparently during long sessions.
-            jwt_provider = createOAuthJWTProvider(creds, mode);
-            config().setString("jwt", jwt_provider->getJWT());
-            config().setString("user", "");
+            if (!login_was_auto_added)
+                throw Exception(
+                    ErrorCodes::BAD_ARGUMENTS,
+                    "--login cannot be combined with a JWT (provided via --jwt, --jwt-command, or in the config file)");
+            login_was_auto_added = false;
+            defer_to_existing_jwt = true;
         }
-        else
+#endif
+
+        if (!defer_to_existing_jwt)
         {
-            // Cloud-specific login path — bare --login, including auto-added for
-            // *.clickhouse.cloud endpoints. Use a separate config key so that
-            // argsToConfig() overwriting config["login"] with the raw string value
-            // cannot cause getBool("login") to throw in main().
-            config().setBool("cloud_oauth_pending", true);
-            config().setString("user", "");
-        }
+            const std::string login_mode = options["login"].as<std::string>();
+            if (!login_mode.empty() && login_mode != "browser" && login_mode != "device")
+                throw Exception(
+                    ErrorCodes::BAD_ARGUMENTS,
+                    "--login value must be 'browser' or 'device', got '{}'",
+                    login_mode);
+
+#if USE_JWT_CPP && USE_SSL
+            if (!options["user"].defaulted())
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "--user and --login cannot both be specified");
+
+            // Bare --login (empty mode, including auto-added for *.clickhouse.cloud) → cloud path.
+            // Explicit --login=browser or --login=device (or --oauth-credentials) → credentials-file
+            // OIDC path. This prevents the credentials file from hijacking the cloud auto-login.
+            const bool use_credentials_file
+                = !login_mode.empty()
+                || options.count("oauth-credentials");
+
+            if (use_credentials_file)
+            {
+                const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
+                const std::string default_creds_path = home_path_cstr
+                    ? std::string(home_path_cstr) + "/.clickhouse-client/oauth_client.json"
+                    : "";
+
+                const std::string creds_path = options.count("oauth-credentials")
+                    ? options["oauth-credentials"].as<std::string>()
+                    : default_creds_path;
+
+                auto creds = loadOAuthCredentials(creds_path);
+                const auto mode = (login_mode == "device") ? OAuthFlowMode::Device : OAuthFlowMode::AuthCode;
+
+                // createOAuthJWTProvider runs the initial flow (trying the cached
+                // refresh token first) and returns a provider that Connection can
+                // call to refresh the id_token transparently during long sessions.
+                jwt_provider = createOAuthJWTProvider(creds, mode);
+                config().setString("jwt", jwt_provider->getJWT());
+                config().setString("user", "");
+            }
+            else
+            {
+                // Cloud-specific login path — bare --login, including auto-added for
+                // *.clickhouse.cloud endpoints. Use a separate config key so that
+                // argsToConfig() overwriting config["login"] with the raw string value
+                // cannot cause getBool("login") to throw in main().
+                config().setBool("cloud_oauth_pending", true);
+                config().setString("user", "");
+            }
 #else
-        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "OAuth login requires a build with JWT and SSL support");
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "OAuth login requires a build with JWT and SSL support");
 #endif
+        }
     }
 #if USE_JWT_CPP && USE_SSL
     if (options.contains("oauth-url"))