[improvement](build) Harden Hive docker startup refresh path

### What problem does this PR solve?

Issue Number: None

Related PR: None

Problem Summary: Fix Hive docker startup regressions in refresh mode by making preinstalled HQL state incremental, wiring json-serde into HiveServer2, reducing noisy Hadoop copy logs, and ensuring JuiceFS metadata initialization relies on the docker mysql dependency path.

### Release note

None

### Check List (For Author)

- Test: Manual test
    - Hive thirdparties startup and log inspection
- Behavior changed: Yes (Hive docker startup now auto-starts mysql when the default JuiceFS metadata DSN is used and refresh mode resumes preinstalled HQL incrementally)
- Does this need documentation: No

Author: suxiaogang223

docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl

@@ -65,9 +65,12 @@ services:
     environment:
       HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${IP_HOST}:${PG_PORT}/metastore"
       SERVICE_PRECONDITION: "${IP_HOST}:${HMS_PORT}"
+      HIVE_SITE_CONF_hive_aux_jars_path: "file:///mnt/scripts/auxlib/json-serde-1.3.9-SNAPSHOT-jar-with-dependencies.jar"
     container_name: ${CONTAINER_UID}hive2-server
     expose:
       - "${HS_PORT}"
+    volumes:
+      - ./scripts:/mnt/scripts
     depends_on:
       datanode:
         condition: service_healthy

docker/thirdparties/docker-compose/hive/hive-2x_settings.env

@@ -26,7 +26,7 @@ export HS_PORT=10000 # should be same as hive2ServerPort in regression-conf.groo
 export PG_PORT=5432 # should be same as hive2PgPort in regression-conf.groovy
 
 # JuiceFS metadata endpoint for property `juicefs.cluster.meta`.
-# CI can override this env, e.g.:
+# CI can override this env, e.g. to point at the docker-published mysql_57 port:
 #   export JFS_CLUSTER_META="mysql://user:pwd@(127.0.0.1:3316)/juicefs_meta"
 # default to mysql_57 (3316) because external pipeline always starts mysql, but not redis.
 export JFS_CLUSTER_META="${JFS_CLUSTER_META:-mysql://root:123456@(127.0.0.1:3316)/juicefs_meta}"

docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl

@@ -67,9 +67,12 @@ services:
       HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${IP_HOST}:${PG_PORT}/metastore"
       SERVICE_PRECONDITION: "${IP_HOST}:${HMS_PORT}"
       JVM_OPTS: -Xmx2g
+      HIVE_SITE_CONF_hive_aux_jars_path: "file:///mnt/scripts/auxlib/json-serde-1.3.9-SNAPSHOT-jar-with-dependencies.jar"
     container_name: ${CONTAINER_UID}hive3-server
     expose:
       - "${HS_PORT}"
+    volumes:
+      - ./scripts:/mnt/scripts
     depends_on:
       datanode:
         condition: service_healthy

docker/thirdparties/docker-compose/hive/hive-3x_settings.env

@@ -26,7 +26,7 @@ export HS_PORT=13000 # should be same as hive3ServerPort in regression-conf.groo
 export PG_PORT=5732 # should be same as hive3PgPort in regression-conf.groovy
 
 # JuiceFS metadata endpoint for property `juicefs.cluster.meta`.
-# CI can override this env, e.g.:
+# CI can override this env, e.g. to point at the docker-published mysql_57 port:
 #   export JFS_CLUSTER_META="mysql://user:pwd@(127.0.0.1:3316)/juicefs_meta"
 export JFS_CLUSTER_META="${JFS_CLUSTER_META:-mysql://root:123456@(127.0.0.1:3316)/juicefs_meta}"
 

docker/thirdparties/docker-compose/hive/scripts/bin/hadoop

@@ -44,14 +44,30 @@ resolve_real_hadoop() {
 
 REAL_HADOOP="$(resolve_real_hadoop)"
 
+exec_quiet_hadoop_fs() {
+    HADOOP_ROOT_LOGGER="${HADOOP_ROOT_LOGGER:-WARN,console}" exec "${REAL_HADOOP}" "$@"
+}
+
 if [[ "$#" -ge 2 && "$1" == "fs" && "$2" == "-put" ]]; then
     shift 2
     case "${1:-}" in
     -f|-p|-l)
-        exec "${REAL_HADOOP}" fs -put "$@"
+        exec_quiet_hadoop_fs fs -put "$@"
+        ;;
+    *)
+        exec_quiet_hadoop_fs fs -put -f "$@"
+        ;;
+    esac
+fi
+
+if [[ "$#" -ge 2 && "$1" == "fs" && "$2" == "-copyFromLocal" ]]; then
+    shift 2
+    case "${1:-}" in
+    -f|-p|-l)
+        exec_quiet_hadoop_fs fs -copyFromLocal "$@"
         ;;
     *)
-        exec "${REAL_HADOOP}" fs -put -f "$@"
+        exec_quiet_hadoop_fs fs -copyFromLocal -f "$@"
         ;;
     esac
 fi

docker/thirdparties/docker-compose/hive/scripts/hive-module-lib.sh

@@ -64,6 +64,12 @@ module_state_file() {
     echo "${HIVE_STATE_DIR}/modules/${module}.sha"
 }
 
+preinstalled_hql_state_file() {
+    local relative_path="$1"
+    local safe_name="${relative_path//\//__}"
+    echo "${HIVE_STATE_DIR}/modules/preinstalled_hql__${safe_name}.sha"
+}
+
 baseline_version_file() {
     echo "${HIVE_STATE_DIR}/baseline.version"
 }
@@ -122,10 +128,43 @@ calc_module_sha() {
     hash_files "${files[@]}"
 }
 
+calc_preinstalled_hql_sha() {
+    local hql_path="$1"
+    hash_files "${hql_path}"
+}
+
 module_needs_refresh() {
     local module="$1"
     local current_sha
     local recorded_sha_file
+    local hql_path=""
+    local relative_hql_path=""
+    local current_file_sha=""
+    local recorded_file_sha=""
+
+    if [[ "${module}" == "preinstalled_hql" ]]; then
+        shopt -s nullglob
+        for hql_path in /mnt/scripts/create_preinstalled_scripts/*.hql; do
+            relative_hql_path="${hql_path#/mnt/scripts/}"
+            if ! bootstrap_item_selected "${BOOTSTRAP_GROUPS}" "preinstalled_hql" "${relative_hql_path}"; then
+                continue
+            fi
+
+            current_file_sha="$(calc_preinstalled_hql_sha "${hql_path}")"
+            recorded_sha_file="$(preinstalled_hql_state_file "${relative_hql_path}")"
+            if [[ ! -f "${recorded_sha_file}" ]]; then
+                shopt -u nullglob
+                return 0
+            fi
+            recorded_file_sha="$(cat "${recorded_sha_file}")"
+            if [[ "${recorded_file_sha}" != "${current_file_sha}" ]]; then
+                shopt -u nullglob
+                return 0
+            fi
+        done
+        shopt -u nullglob
+        return 1
+    fi
 
     current_sha="$(calc_module_sha "${module}")"
     recorded_sha_file="$(module_state_file "${module}")"
@@ -189,7 +228,9 @@ refresh_preinstalled_hql_module() {
     local preinstalled_hqls=()
     local hql_path=""
     local relative_hql_path=""
-    local merged_preinstalled_hql="/tmp/merged-preinstalled.hql"
+    local current_sha=""
+    local state_file=""
+    local refreshed=0
 
     shopt -s nullglob
     for hql_path in /mnt/scripts/create_preinstalled_scripts/*.hql; do
@@ -203,8 +244,26 @@ refresh_preinstalled_hql_module() {
     if (( ${#preinstalled_hqls[@]} > 0 )); then
         IFS=$'\n' preinstalled_hqls=($(printf '%s\n' "${preinstalled_hqls[@]}" | sort))
         unset IFS
-        bash /mnt/scripts/merge-preinstalled-hql.sh "${merged_preinstalled_hql}" "${preinstalled_hqls[@]}"
-        run_hive_hql "${merged_preinstalled_hql}" "Merged preinstalled HQLs"
+        for hql_path in "${preinstalled_hqls[@]}"; do
+            relative_hql_path="${hql_path#/mnt/scripts/}"
+            current_sha="$(calc_preinstalled_hql_sha "${hql_path}")"
+            state_file="$(preinstalled_hql_state_file "${relative_hql_path}")"
+
+            if [[ -f "${state_file}" ]] && grep -Fxq "${current_sha}" "${state_file}"; then
+                echo "  [preinstalled_hql] up-to-date ${relative_hql_path}"
+                continue
+            fi
+
+            echo "  [preinstalled_hql] BEGIN ${relative_hql_path}"
+            run_hive_hql "${hql_path}" "${relative_hql_path}"
+            printf '%s\n' "${current_sha}" >"${state_file}"
+            echo "  [preinstalled_hql] END   ${relative_hql_path}"
+            refreshed=1
+        done
+
+        if (( refreshed == 0 )); then
+            echo "  [preinstalled_hql] all selected HQL files are up-to-date"
+        fi
     fi
 }
 
@@ -220,6 +279,7 @@ refresh_module() {
         ;;
     preinstalled_hql)
         refresh_preinstalled_hql_module
+        return 0
         ;;
     view)
         run_hive_hql /mnt/scripts/create_view_scripts/create_view.hql "create_view.hql"

docker/thirdparties/run-thirdparties-docker.sh

@@ -260,6 +260,25 @@ if [[ "${RUN_HIVE3}" -eq 1 ]] && [[ -z "${HIVE3_BOOTSTRAP_GROUPS+x}" ]]; then
     export HIVE3_BOOTSTRAP_GROUPS="common,hive3_only"
 fi
 
+hive_requires_mysql_component() {
+    local hive_version="$1"
+    local jfs_meta=""
+    local settings_env="${ROOT}/docker-compose/hive/hive-${hive_version#hive}x_settings.env"
+
+    # shellcheck disable=SC1090
+    . "${settings_env}"
+    jfs_meta="${JFS_CLUSTER_META:-}"
+    [[ "${jfs_meta}" == mysql://* ]] || return 1
+    [[ "${jfs_meta}" == *"@(127.0.0.1:3316)/"* || "${jfs_meta}" == *"@(localhost:3316)/"* ]]
+}
+
+if [[ "${RUN_HIVE2}" -eq 1 ]] && hive_requires_mysql_component "hive2"; then
+    RUN_MYSQL=1
+fi
+if [[ "${RUN_HIVE3}" -eq 1 ]] && hive_requires_mysql_component "hive3"; then
+    RUN_MYSQL=1
+fi
+
 reserve_ports() {
     if [[ "${NEED_RESERVE_PORTS}" -eq 0 ]]; then
         return
@@ -366,16 +385,18 @@ ensure_juicefs_meta_database() {
     meta_db="${jfs_meta##*/}"
     meta_db="${meta_db%%\?*}"
 
-    if command -v mysql >/dev/null 2>&1; then
-        mysql -h127.0.0.1 -P3316 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS \`${meta_db}\`;"
-        return 0
-    fi
-
     mysql_container=$(sudo docker ps --format '{{.Names}}' | grep -E "(^|-)${CONTAINER_UID}mysql_57(-[0-9]+)?$" | head -n 1 || true)
     if [[ -n "${mysql_container}" ]]; then
-        sudo docker exec "${mysql_container}" \
-            mysql -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS \`${meta_db}\`;"
+        if sudo docker exec "${mysql_container}" \
+            mysql -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS \`${meta_db}\`;" >/dev/null 2>&1; then
+            return 0
+        fi
+        echo "WARN: docker mysql ${mysql_container} is unavailable for JuiceFS metadata init." >&2
+        return 0
     fi
+
+    echo "WARN: docker mysql_57 is not running; skip eager JuiceFS metadata database creation for ${meta_db}." >&2
+    return 0
 }
 
 run_juicefs_cli() {
@@ -423,7 +444,7 @@ prepare_juicefs_meta_for_hive() {
     sudo mkdir -p "${bucket_dir}"
     sudo chmod 777 "${bucket_dir}"
 
-    # For local mysql_57 metadata DSN, ensure metadata database exists.
+    # For the default docker mysql_57 metadata DSN, ensure metadata database exists.
     ensure_juicefs_meta_database "${jfs_meta}"
 
     if run_juicefs_cli status "${jfs_meta}" >/dev/null 2>&1; then