install-dynamo.sh

#!/usr/bin/env bash
# shfmt -i 2 -ci -w
set -Eo pipefail

trap exit SIGINT SIGTERM

################################################################################
# NVIDIA Dynamo Installation on AKS
# Installs the Dynamo Platform Helm chart.
# GPU stack (driver, device plugin, DCGM) is fully managed by AKS
# via --enable-managed-gpu=true on the node pool. No GPU Operator needed.
#
# References:
#   https://github.com/ai-dynamo/dynamo/blob/main/examples/deployments/AKS/AKS-deployment.md
#   https://github.com/ai-dynamo/dynamo/blob/main/docs/kubernetes/installation-guide.md
#   https://learn.microsoft.com/azure/aks/aks-managed-gpu-nodes

################################################################################
# Default configuration
RELEASE_VERSION=${RELEASE_VERSION:-1.0.2}
NAMESPACE=${NAMESPACE:-dynamo-system}
HF_TOKEN=${HF_TOKEN:-}
NGC_API_KEY=${NGC_API_KEY:-}
# Azure Monitor managed Prometheus workspace query endpoint.
# Format: https://<workspace-name>.prometheus.monitor.azure.com
# Find it: az monitor account show -n <workspace> -g <rg> --query metrics.prometheusQueryEndpoint -o tsv
PROMETHEUS_ENDPOINT=${PROMETHEUS_ENDPOINT:-}
################################################################################

__usage="
    -x  action to be executed.

Possible verbs are:
    install        Install Dynamo Platform.
    delete         Uninstall Dynamo Platform.
    show           Show Dynamo pod status.
    check-deps     Check required dependencies.

Environment variables (with defaults):
    RELEASE_VERSION=${RELEASE_VERSION}
    NAMESPACE=${NAMESPACE}
    PROMETHEUS_ENDPOINT=${PROMETHEUS_ENDPOINT}   (Azure Monitor workspace query endpoint)
    HF_TOKEN=(required for gated models; optional for public models like Qwen3)
    NGC_API_KEY=(required for nvcr.io image pulls)
"

usage() {
  echo "usage: ${0##*/} [options]"
  echo "${__usage/[[:space:]]/}"
  exit 1
}

print_header() {
  echo ""
  echo "NVIDIA Dynamo Installation — AKS"
  echo "=========================================="
  echo ""
  echo "Release Version:  $RELEASE_VERSION"
  echo "Namespace:        $NAMESPACE"
  echo ""
}

log() {
  echo "[$(date +'%r')] $*"
}

check_dependencies() {
  log "Checking dependencies..."
  local _NEEDED="az kubectl helm"
  local _DEP_FLAG=false

  for i in ${_NEEDED}; do
    if hash "$i" 2>/dev/null; then
      log "  $i: OK"
    else
      log "  $i: NOT FOUND"
      _DEP_FLAG=true
    fi
  done

  if [[ "${_DEP_FLAG}" == "true" ]]; then
    log "Dependencies missing. Please install them before proceeding."
    exit 1
  fi

  log "All dependencies satisfied."
}

create_hf_secret() {
  if [[ -z "${HF_TOKEN}" ]]; then
    log "WARNING: HF_TOKEN is not set — skipping HuggingFace secret creation."
    log "  Open models (e.g., Qwen3) will work without it but may be rate-limited."
    log "  Gated models (e.g., Llama, Mistral) will fail without a token."
    log "  To create the secret later:"
    log "    export HF_TOKEN=<your-hf-token>"
    log "    kubectl create secret generic hf-token-secret --from-literal=HF_TOKEN=\$HF_TOKEN --namespace default"
    return 0
  fi

  log "Creating HuggingFace token secret..."
  for ns in default "${NAMESPACE}"; do
    kubectl create secret generic hf-token-secret \
      --from-literal=HF_TOKEN="${HF_TOKEN}" \
      --namespace "${ns}" \
      --dry-run=client -o yaml | kubectl apply -f -
  done
  log "HuggingFace token secret created."
}

create_namespace() {
  log "Creating namespace '${NAMESPACE}' if not present..."
  kubectl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -
}

create_ngc_secret() {
  if [[ -z "${NGC_API_KEY}" ]]; then
    log "WARNING: NGC_API_KEY is not set — skipping NGC image pull secret creation."
    log "  The Dynamo planner and vllm-runtime images are pulled from nvcr.io."
    log "  Without this secret, image pulls will fail for pods in this namespace."
    log "  Get an API key at: https://ngc.nvidia.com/setup/api-key"
    log "  To create manually:"
    log "    kubectl create secret docker-registry nvcr-imagepullsecret \\"
    log "      --docker-server=nvcr.io \\"
    log "      --docker-username='\$oauthtoken' \\"
    log "      --docker-password=<NGC_API_KEY> \\"
    log "      -n ${NAMESPACE}"
    return 0
  fi

  log "Creating NGC image pull secret..."
  kubectl create secret docker-registry nvcr-imagepullsecret \
    --docker-server=nvcr.io \
    --docker-username='$oauthtoken' \
    --docker-password="${NGC_API_KEY}" \
    --namespace "${NAMESPACE}" \
    --dry-run=client -o yaml | kubectl apply -f -
  log "NGC image pull secret created in namespace ${NAMESPACE}."
}

install_dynamo_crds() {
  log "Adding/updating NVIDIA AI Dynamo Helm repo..."
  helm repo add nvidia-dynamo https://helm.ngc.nvidia.com/nvidia/ai-dynamo --force-update
  helm repo update nvidia-dynamo

  log "Installing Dynamo CRDs v${RELEASE_VERSION}..."
  # dynamo-crds is a separate chart only in older releases; from 1.0.x it is
  # included inside dynamo-platform. Try it first; skip gracefully if not found.
  if helm search repo nvidia-dynamo/dynamo-crds --version "${RELEASE_VERSION}" 2>/dev/null | grep -q dynamo-crds; then
    helm upgrade --install dynamo-crds nvidia-dynamo/dynamo-crds \
      --version "${RELEASE_VERSION}" \
      --namespace default
    log "Dynamo CRDs chart installed."
  else
    log "  dynamo-crds chart not found (bundled in dynamo-platform for v${RELEASE_VERSION}) — skipping."
  fi
}

install_dynamo_platform() {
  log "Installing Dynamo Platform v${RELEASE_VERSION} from Helm repo..."

  local _prometheus_flag=""
  if [[ -n "${PROMETHEUS_ENDPOINT}" ]]; then
    _prometheus_flag="--set prometheusEndpoint=${PROMETHEUS_ENDPOINT}"
    log "Installing Dynamo Platform (prometheusEndpoint: ${PROMETHEUS_ENDPOINT})..."
  else
    log "Installing Dynamo Platform (no prometheusEndpoint set — Planner autoscaling disabled)..."
    log "  Set PROMETHEUS_ENDPOINT to your Azure Monitor workspace query endpoint to enable autoscaling."
  fi

  # To enable Grove + KAI Scheduler for multinode or disaggregated inference,
  # add: --set "global.grove.install=true" --set "global.kai-scheduler.install=true"
  # shellcheck disable=SC2086
  helm upgrade --install dynamo-platform nvidia-dynamo/dynamo-platform \
    --version "${RELEASE_VERSION}" \
    --namespace "${NAMESPACE}" \
    --create-namespace \
    ${_prometheus_flag}

  log "Waiting for Dynamo platform pods..."
  # Wait for the operator (selector: app.kubernetes.io/name=dynamo-operator)
  kubectl wait pod \
    --for=condition=Ready \
    --selector=app.kubernetes.io/name=dynamo-operator \
    --namespace "${NAMESPACE}" \
    --timeout=300s || true

  log "Dynamo Platform installed."
}

validate_install() {
  log "Validating installation..."

  log "  CRDs:"
  kubectl get crd | grep dynamo

  log "  Dynamo pods ($NAMESPACE):"
  kubectl get pods -n "${NAMESPACE}"
}

do_install() {
  check_dependencies
  create_namespace
  create_hf_secret
  create_ngc_secret
  install_dynamo_crds
  install_dynamo_platform
  validate_install
  log ""
  log "Installation complete!"
  log "Deploy a model with: kubectl apply -f manifests/quickstart.yaml -n ${NAMESPACE}"
}

do_delete() {
  log "Uninstalling Dynamo Platform..."
  if helm status dynamo-platform -n "${NAMESPACE}" >/dev/null 2>&1; then
    helm uninstall dynamo-platform -n "${NAMESPACE}"
    log "  Removing Dynamo CRDs..."
    kubectl get crd | grep 'dynamo.*nvidia.com' | awk '{print $1}' | xargs kubectl delete crd
    log "Dynamo Platform removed."
  else
    log "Dynamo Platform not found — skipping."
  fi

  if helm status dynamo-crds -n default >/dev/null 2>&1; then
    helm uninstall dynamo-crds -n default
    log "Dynamo CRDs chart removed."
  fi

}


do_show() {
  log "Dynamo pods ($NAMESPACE):"
  kubectl get pods -n "${NAMESPACE}"

  echo ""
  log "Dynamo CRDs:"
  kubectl get crd | grep dynamo

  echo ""
  log "GPU node status:"
  kubectl get nodes -l agentpool=h100pool -o wide
}

exec_case() {
  local _opt=$1

  case ${_opt} in
  install) do_install ;;
  delete) do_delete ;;
  show) do_show ;;
  check-deps) check_dependencies ;;
  *) usage ;;
  esac
  unset _opt
}

################################################################################
# Entry point
main() {
  while getopts "x:" opt; do
    case $opt in
      x)
        exec_flag=true
        EXEC_OPT="${OPTARG}"
        ;;
      *) usage ;;
    esac
  done
  shift $((OPTIND - 1))

  if [ $OPTIND = 1 ]; then
    print_header
    usage
    exit 0
  fi

  if [[ "${exec_flag}" == "true" ]]; then
    exec_case "${EXEC_OPT}"
  fi
}

main "$@"
exit 0