Merge pull request #35 from dhslove/codex/hangctl-migration-protection-design

hangctl 라이브 마이그레이션 보호 로직 개선

Author: dhslove

VERSION

@@ -1,3 +1,3 @@
 # VERSION file
-VERSION=0.9.1
+VERSION=0.9.2
 RELEASE=1

bin/ablestack_vm_hangctl.sh

@@ -208,21 +208,16 @@ hangctl_detect_probe_maybe_act_one_vm() {
   local stuck_sec="${duration_sec}"
 
   # --- [단계 3] 마이그레이션/백업 작업 인식 및 계측 결정 ---
-  local is_migration=0
-  [[ "${domstate_full}" == *"migration"* ]] && is_migration=1
-  
-  # libvirt가 자체적으로 감지한 에러 상태 확인
   local is_disk_error=0
   [[ "${domstate_full}" == *"disk error"* ]] && is_disk_error=1
 
-  # [규칙] 백업/스냅샷 작업 여부 확인 (domjobinfo 이용)
-  local job_out job_type
-  job_out=$(virsh -c qemu:///system domjobinfo "${vm}" 2>/dev/null || true)
-  job_type=$(echo "${job_out}" | grep "Job type:" | awk '{print $3}' || echo "None")
-  
+  local job_out job_err job_rc job_type job_operation
+  job_out=""; job_err=""; job_rc=0
+  hangctl_virsh "${HANGCTL_VIRSH_TIMEOUT_SEC}" job_out job_err job_rc -- -c qemu:///system domjobinfo "${vm}" || true
+
+  local is_migration=0
   local is_backup=0
-  # Job type이 None이거나 Completed가 아니면 작업 중으로 간주
-  [[ "${job_type}" != "None" && "${job_type}" != "Completed" && -n "${job_type}" ]] && is_backup=1
+  hangctl_classify_domjobinfo "${domstate_full}" "${job_out}" job_type job_operation is_migration is_backup
 
   local current_window="${HANGCTL_CONFIRM_WINDOW_SEC}"
   if [[ "${is_migration}" -eq 1 || "${is_backup}" -eq 1 ]]; then
@@ -239,23 +234,26 @@ hangctl_detect_probe_maybe_act_one_vm() {
     decision="suspect"
   fi
 
+  local job_operation_url="${job_operation// /%20}"
   hangctl_log_event "detect" "vm.status_check" "ok" "${vm}" "" "" \
-    "domstate=${domstate_full} duration_sec=${duration_sec} decision=${decision} confirm_window=${current_window} job_type=${job_type} io_stall=${io_stall}"
+    "domstate=${domstate_full} duration_sec=${duration_sec} decision=${decision} confirm_window=${current_window} job_type=${job_type} operation_url=${job_operation_url} is_migration=${is_migration} is_backup=${is_backup} io_stall=${io_stall}"
 
-  # 의심 상황이 아니면 다음 VM으로 넘어감
-  if [[ "${decision}" != "suspect" ]]; then
-    return 0
-  fi
-
-  # --- [단계 5] 추가 검증(마이그레이션 좀비체크) ---
+  local migration_status=""
+  local migration_detail=""
   if [[ "${is_migration}" -eq 1 ]]; then
-    if ! hangctl_probe_migration_zombie_check "${vm}"; then
-      hangctl_log_event "detect" "vm.migration_check" "ok" "${vm}" "" "" \
-        "status=progressing note=protecting_active_migration stuck_sec=${stuck_sec}"
+    hangctl_probe_migration_progress_evaluate "${vm}" "${job_out}" "${duration_sec}" migration_status migration_detail || true
+    hangctl_log_event "detect" "vm.migration_check" "ok" "${vm}" "" "" \
+      "${migration_detail} stuck_sec=${stuck_sec} job_type=${job_type} operation_url=${job_operation_url}"
+    if [[ "${migration_status}" != "zombie_no_progress" ]]; then
       return 0
     fi
   fi
 
+  # Non-migration or confirmed zombie migration continues through normal suspect handling.
+  if [[ "${decision}" != "suspect" ]]; then
+    return 0
+  fi
+
   # --- [단계 6] 최종 결정 로직 ---
   local final_decision="suspect"
   local confirm_reason="domstate_stuck"

docs/hangctl/ablestack_vm_hangctl_migration_protection_design.md

@@ -0,0 +1,239 @@
+# ABLESTACK VM Hangctl Migration Protection Design
+
+## Background
+
+`ablestack_vm_hangctl` currently has a migration protection path, but it is not
+strong enough for long-running live migrations.
+
+The current flow has these weaknesses:
+
+- A VM is treated as migrating only when `virsh domstate --reason` contains
+  `migration`.
+- `virsh domjobinfo` is used for backup/job detection, but active migration jobs
+  are not classified separately before the generic backup path.
+- `HANGCTL_MIGRATION_PROGRESS_CHECK_SEC` is defined but not used for migration
+  zombie decisions.
+- Migration progress is parsed from `Data processed` or `Memory processed`
+  without unit normalization, so values such as `47.235 MiB` are not reliable
+  inputs for shell integer arithmetic.
+- A long-running live migration can therefore reach the confirmed action path
+  and be destroyed even when the migration is still active.
+
+The protection policy must be conservative: an active migration should be
+protected unless hangctl has clear evidence that the migration job is present
+but has made no progress for the configured grace period.
+
+## Goals
+
+- Detect active live migration from both `domstate --reason` and `domjobinfo`.
+- Use `HANGCTL_MIGRATION_PROGRESS_CHECK_SEC` as the no-progress grace window.
+- Keep `HANGCTL_MIGRATION_CONFIRM_WINDOW_SEC` as the wider confirmation window
+  before destructive action is allowed.
+- Treat progress parsing failures as protected active migration when a
+  migration job is visible.
+- Avoid classifying migration jobs as generic backup/block jobs.
+- Add smoke tests that do not require a real libvirt migration.
+
+## Non-Goals
+
+- Do not change the default destructive action behavior for non-migration hangs.
+- Do not replace hangctl's existing evidence, dump, storage guard, or libvirtd
+  health-check behavior.
+- Do not require new binary dependencies.
+
+## Proposed Decision Flow
+
+For each VM scan:
+
+1. Collect QMP status, blockstats, `domstate --reason`, and `domjobinfo`.
+2. Classify `domjobinfo` before choosing the confirmation window.
+3. If active migration is detected:
+   - Parse a progress metric in bytes.
+   - If the metric increased, update `migration_last_progress_ts` and protect.
+   - If the metric cannot be parsed, protect.
+   - If the metric did not increase, compare the current time with
+     `migration_last_progress_ts`.
+   - Confirm `migration_zombie_no_progress` only when both of these are true:
+     - `last_progress_age_sec >= HANGCTL_MIGRATION_PROGRESS_CHECK_SEC`
+     - `duration_sec >= HANGCTL_MIGRATION_CONFIRM_WINDOW_SEC`
+4. If active migration is not detected, continue through the existing hangctl
+   non-migration decision logic.
+
+This keeps short no-progress windows safe while still allowing hangctl to act on
+real zombie migration jobs after the configured confirmation period.
+
+## Migration Classification
+
+Add a helper in `lib/hangctl/detect.sh`:
+
+```bash
+hangctl_classify_domjobinfo <domjobinfo_text> <out_job_type> <out_operation> <out_is_migration> <out_is_backup>
+```
+
+`out_is_migration=1` when any of these are true:
+
+- `domstate --reason` contains `migration`.
+- `domjobinfo` contains an `Operation:` value with `migration`.
+- `domjobinfo` contains migration-specific progress fields such as
+  `Memory processed`, `Memory remaining`, or `Memory total`.
+
+`out_is_backup=1` should be reserved for non-migration active jobs. A non-`None`
+job must not automatically become backup if it is classified as migration.
+
+## Progress Metric
+
+Add a size parser in `lib/hangctl/detect.sh`:
+
+```bash
+hangctl_parse_size_to_bytes <value> <unit>
+```
+
+Supported examples:
+
+- `1024 bytes`
+- `47.235 MiB`
+- `2.1 GiB`
+- `1 TB`
+
+The preferred progress metric order is:
+
+1. `Data processed`
+2. `Memory processed`
+3. `Memory remaining` as an inverse metric, where a decrease means progress
+
+If no metric can be parsed from an active migration job, the result must be
+`protect_unknown_progress`, not `confirmed`.
+
+## State Cache Format
+
+Replace the single numeric `<vm>.state.migrate` payload with key-value fields.
+The existing file path can remain the same to limit blast radius.
+
+```text
+migration_metric_bytes=123456789
+migration_metric_kind=data_processed
+migration_last_progress_ts=1780000000
+migration_last_seen_ts=1780000030
+migration_job_type=Unbounded
+migration_operation=Migration Out
+```
+
+Required helpers in `lib/hangctl/state_cache.sh`:
+
+```bash
+hangctl_state_get_migration_kv <vm> <key>
+hangctl_state_set_migration_kv_all <vm> key=value ...
+hangctl_state_reset_migration <vm>
+```
+
+Backward compatibility:
+
+- If the old file contains only a number, treat it as the previous
+  `migration_metric_bytes`.
+- Rewrite the file in key-value format on the next active migration scan.
+
+## Zombie Evaluation API
+
+Replace the current boolean-only migration zombie function with a richer helper:
+
+```bash
+hangctl_probe_migration_progress_evaluate \
+  <vm> \
+  <domjobinfo_text> \
+  <duration_sec> \
+  <out_status_var> \
+  <out_detail_var>
+```
+
+Status values:
+
+- `progressing`
+- `no_progress_within_grace`
+- `zombie_no_progress`
+- `protect_unknown_progress`
+- `not_migration`
+
+The caller maps these statuses as follows:
+
+- `progressing`, `no_progress_within_grace`, `protect_unknown_progress`:
+  return without action.
+- `zombie_no_progress`: continue to confirmed decision with
+  `migration_zombie_no_progress`.
+- `not_migration`: continue existing non-migration logic.
+
+## Logging
+
+Add structured details to the existing JSONL events.
+
+Progressing:
+
+```text
+event=vm.migration_check
+status=progressing
+metric_kind=data_processed
+metric_bytes=123456789
+delta_bytes=10485760
+last_progress_age_sec=0
+note=protecting_active_migration
+```
+
+No progress but still protected:
+
+```text
+event=vm.migration_check
+status=no_progress_within_grace
+metric_kind=data_processed
+metric_bytes=123456789
+last_progress_age_sec=180
+progress_window_sec=300
+note=protecting_active_migration
+```
+
+Confirmed zombie:
+
+```text
+event=vm.decision
+final=confirmed
+reason=migration_zombie_no_progress
+last_progress_age_sec=3900
+progress_window_sec=300
+confirm_window_sec=3600
+```
+
+Unknown progress:
+
+```text
+event=vm.migration_check
+status=protect_unknown_progress
+reason=metric_parse_failed
+note=protecting_active_migration
+```
+
+## Test Plan
+
+Add a shell smoke test under `tests/`, for example:
+
+```text
+tests/hangctl_migration_protection_smoke.sh
+```
+
+Suggested cases:
+
+- `domstate="paused (in-migration)"` and progress increases: no action.
+- `domstate="running"` with `domjobinfo Operation: Migration Out`: no action.
+- Active migration with unparsable progress: no action.
+- Active migration with no progress for less than
+  `HANGCTL_MIGRATION_PROGRESS_CHECK_SEC`: no action.
+- Active migration with no progress beyond
+  `HANGCTL_MIGRATION_PROGRESS_CHECK_SEC` but before
+  `HANGCTL_MIGRATION_CONFIRM_WINDOW_SEC`: no action.
+- Active migration with no progress beyond both windows:
+  `migration_zombie_no_progress`.
+- Non-migration active backup/block job: keep existing backup policy.
+
+Run:
+
+```bash
+bash -n bin/ablestack_vm_hangctl.sh lib/hangctl/*.sh
+bash tests/hangctl_migration_protection_smoke.sh
+```