@@ -539,7 +539,48 @@ migrate_config() {
539539 if [[ -f " $NEW_ENV " ]]; then
540540 patch_env_with_images " $NEW_ENV "
541541 fi
542+
543+ # Extract latest provider default.json from new package image
544+ # This ensures the new version's config is used for both Docker and K8s
545+ local deployment_type=" docker-compose"
546+ if [[ -f " ${NEW_DIR} /.deployment-type" ]]; then
547+ deployment_type=$( cat " ${NEW_DIR} /.deployment-type" )
548+ fi
542549
550+ if [[ " $deployment_type " == " docker-compose" ]]; then
551+ msg_info " Extracting latest provider configuration from new package..."
552+ local docker_config_path=" ${NEW_DIR} /services/cis-provider/config/default.json"
553+
554+ # Source env to get provider image
555+ if [[ -f " $NEW_ENV " ]]; then
556+ source " $NEW_ENV "
557+ local provider_image=" ${CIS_PROVIDER_IMAGE:- } "
558+
559+ if [[ -n " $provider_image " ]]; then
560+ # Pull the image first
561+ if docker pull " $provider_image " >> " $LOG_FILE " 2>&1 ; then
562+ # Create temp container and extract config
563+ if docker create --name temp-provider-config-upgrade " $provider_image " > /dev/null 2>&1 ; then
564+ if docker cp temp-provider-config-upgrade:/opt/bito/xmcp/config/default.json " $docker_config_path " 2>> " $LOG_FILE " ; then
565+ chmod 666 " $docker_config_path " 2> /dev/null || true
566+ msg_success " Provider configuration extracted from new image"
567+ else
568+ msg_warn " Could not extract provider config from image, using packaged config"
569+ fi
570+ docker rm temp-provider-config-upgrade > /dev/null 2>&1 || true
571+ else
572+ msg_warn " Could not create temp container for config extraction"
573+ fi
574+ else
575+ msg_warn " Could not pull provider image for config extraction"
576+ fi
577+ fi
578+ fi
579+ else
580+ # Kubernetes: config is already in helm-bitoarch/services/cis-provider/config/default.json from package
581+ msg_info " Kubernetes deployment - using packaged provider configuration"
582+ fi
583+
543584 msg_success " Configuration migrated"
544585}
545586
@@ -839,8 +880,12 @@ upgrade_kubernetes() {
839880 # Set SCRIPT_DIR for values-generator.sh
840881 export SCRIPT_DIR=" $NEW_DIR "
841882
842- # Execute values-generator with better error capture
843- if ! bash " ${NEW_DIR} /scripts/values-generator.sh" 2>&1 | tee -a " $LOG_FILE " ; then
883+ # Source the values-generator and call with imagePullPolicy=Always
884+ # shellcheck disable=SC1090
885+ source " ${NEW_DIR} /scripts/values-generator.sh"
886+
887+ # Execute values-generator function with Always pull policy for upgrades
888+ if ! generate_k8s_values_from_env " Always" 2>&1 | tee -a " $LOG_FILE " ; then
844889 msg_error " Failed to generate Helm values"
845890 msg_error " Check log file for details: $LOG_FILE "
846891 tail -20 " $LOG_FILE " | grep -i " error" || tail -20 " $LOG_FILE "
@@ -864,6 +909,25 @@ upgrade_kubernetes() {
864909 --wait \
865910 --timeout 10m >> " $LOG_FILE " 2>&1 ; then
866911 msg_success " Helm upgrade completed"
912+
913+ # Force rollout restart to ensure pods pull latest images with imagePullPolicy=Always
914+ # Critical when upgrading to same version but with updated images in registry
915+ kubectl rollout restart deployment -n " $namespace " -l " app.kubernetes.io/name=bitoarch" >> " $LOG_FILE " 2>&1 || true
916+
917+ log_silent " Pod rollout initiated to force image pull"
918+
919+ # CRITICAL: Wait for ALL rollouts to complete before starting port-forwards
920+ # This prevents race condition where port-forward connects to terminating pod
921+ msg_info " Waiting for all deployment rollouts to complete..."
922+ for component in provider manager config tracker mysql; do
923+ msg_info " Waiting for ai-architect-${component} rollout..."
924+ if kubectl rollout status deployment/ai-architect-${component} -n " $namespace " --timeout=180s >> " $LOG_FILE " 2>&1 ; then
925+ log_silent " Rollout complete: ai-architect-${component} "
926+ else
927+ msg_warn " Rollout may not have completed for ai-architect-${component} "
928+ fi
929+ done
930+ msg_success " All deployment rollouts completed"
867931 else
868932 msg_error " Helm upgrade failed"
869933 msg_error " Check log file for details: $LOG_FILE "
@@ -875,6 +939,154 @@ upgrade_kubernetes() {
875939 fi
876940
877941 log_silent " Kubernetes upgrade completed successfully"
942+
943+ # CRITICAL: Wait for pods to be ready after rollout restart
944+ # Inline implementation to work regardless of kubernetes-manager.sh version
945+ msg_info " Waiting for pods to be ready after rollout restart..."
946+ local max_wait=180
947+ local waited=0
948+ local services=(" mysql" " config" " manager" " provider" " tracker" )
949+
950+ while [ $waited -lt $max_wait ]; do
951+ local all_ready=true
952+ for service in " ${services[@]} " ; do
953+ local ready=$( kubectl get pods -n " $namespace " -l " app.kubernetes.io/component=$service " \
954+ -o jsonpath=' {.items[0].status.conditions[?(@.type=="Ready")].status}' 2> /dev/null || echo " False" )
955+ if [ " $ready " != " True" ]; then
956+ all_ready=false
957+ break
958+ fi
959+ done
960+
961+ if [ " $all_ready " = true ]; then
962+ msg_success " All pods are ready"
963+ break
964+ fi
965+
966+ echo -n " ."
967+ sleep 5
968+ waited=$(( waited + 5 ))
969+ done
970+
971+ if [ $waited -ge $max_wait ]; then
972+ msg_warn " Some pods may still be initializing"
973+ fi
974+ echo " "
975+
976+ # Load environment variables from new installation for port configuration
977+ set -a
978+ source " ${NEW_DIR} /.env-bitoarch" 2> /dev/null || true
979+ set +a
980+
981+ # Setup port-forwards for immediate CLI access
982+ # INLINE implementation to work regardless of kubernetes-manager.sh version in old installation
983+ msg_info " Setting up port-forwards to new pods..."
984+
985+ # Kill any existing port-forwards
986+ pkill -f " kubectl.*port-forward.*${namespace} " 2> /dev/null || true
987+ sleep 2
988+
989+ # Read port configuration
990+ local provider_ext=" ${CIS_PROVIDER_EXTERNAL_PORT:- 5001} "
991+ local manager_ext=" ${CIS_MANAGER_EXTERNAL_PORT:- 5002} "
992+ local config_ext=" ${CIS_CONFIG_EXTERNAL_PORT:- 5003} "
993+ local mysql_ext=" ${MYSQL_EXTERNAL_PORT:- 5004} "
994+ local tracker_ext=" ${CIS_TRACKER_EXTERNAL_PORT:- 5005} "
995+
996+ local provider_int=" ${XMCP_HTTP_PORT:- 8080} "
997+ local manager_int=" ${CIS_MANAGER_PORT:- 9090} "
998+ local config_int=" ${CIS_CONFIG_PORT:- 8081} "
999+ local mysql_int=" ${MYSQL_PORT:- 3306} "
1000+ local tracker_int=" ${CIS_TRACKING_PORT:- 9920} "
1001+
1002+ # Launch port-forwards with proper daemonization
1003+ # Use: nohup + stdin from /dev/null + disown to fully detach from shell
1004+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-provider " ${provider_ext} :${provider_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1005+ disown 2> /dev/null || true
1006+ sleep 0.5
1007+
1008+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-manager " ${manager_ext} :${manager_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1009+ disown 2> /dev/null || true
1010+ sleep 0.5
1011+
1012+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-config " ${config_ext} :${config_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1013+ disown 2> /dev/null || true
1014+ sleep 0.5
1015+
1016+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-mysql " ${mysql_ext} :${mysql_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1017+ disown 2> /dev/null || true
1018+ sleep 0.5
1019+
1020+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-tracker " ${tracker_ext} :${tracker_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1021+ disown 2> /dev/null || true
1022+ sleep 2
1023+
1024+ # Verify and retry port-forwards with health check
1025+ local max_verify_attempts=3
1026+ local verify_attempt=1
1027+
1028+ while [ $verify_attempt -le $max_verify_attempts ]; do
1029+ local pf_count=$( ps aux | grep " kubectl port-forward" | grep -E " (-n |--namespace=|-n=)$namespace " | grep -v grep | wc -l | xargs)
1030+
1031+ if [ " $pf_count " -ge 5 ]; then
1032+ msg_success " Port-forwards established (5/5 services)"
1033+ break
1034+ fi
1035+
1036+ msg_warn " Only $pf_count /5 port-forwards running (attempt $verify_attempt )"
1037+
1038+ # Restart missing port-forwards
1039+ if ! ps aux | grep " kubectl port-forward" | grep " ai-architect-provider" | grep -v grep > /dev/null 2>&1 ; then
1040+ log_silent " Restarting provider port-forward"
1041+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-provider " ${provider_ext} :${provider_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1042+ disown 2> /dev/null || true
1043+ fi
1044+
1045+ if ! ps aux | grep " kubectl port-forward" | grep " ai-architect-manager" | grep -v grep > /dev/null 2>&1 ; then
1046+ log_silent " Restarting manager port-forward"
1047+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-manager " ${manager_ext} :${manager_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1048+ disown 2> /dev/null || true
1049+ fi
1050+
1051+ if ! ps aux | grep " kubectl port-forward" | grep " ai-architect-config" | grep -v grep > /dev/null 2>&1 ; then
1052+ log_silent " Restarting config port-forward"
1053+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-config " ${config_ext} :${config_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1054+ disown 2> /dev/null || true
1055+ fi
1056+
1057+ if ! ps aux | grep " kubectl port-forward" | grep " ai-architect-mysql" | grep -v grep > /dev/null 2>&1 ; then
1058+ log_silent " Restarting mysql port-forward"
1059+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-mysql " ${mysql_ext} :${mysql_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1060+ disown 2> /dev/null || true
1061+ fi
1062+
1063+ if ! ps aux | grep " kubectl port-forward" | grep " ai-architect-tracker" | grep -v grep > /dev/null 2>&1 ; then
1064+ log_silent " Restarting tracker port-forward"
1065+ nohup kubectl port-forward --address 0.0.0.0 -n " $namespace " svc/ai-architect-tracker " ${tracker_ext} :${tracker_int} " < /dev/null >> " $LOG_FILE " 2>&1 &
1066+ disown 2> /dev/null || true
1067+ fi
1068+
1069+ sleep 2
1070+ verify_attempt=$(( verify_attempt + 1 ))
1071+ done
1072+
1073+ # Final health check - test actual connectivity
1074+ msg_info " Verifying port-forward connectivity..."
1075+ local health_ok=true
1076+
1077+ for port in $provider_ext $manager_ext $config_ext $tracker_ext ; do
1078+ if ! curl -s --connect-timeout 2 " http://localhost:$port /health" > /dev/null 2>&1 ; then
1079+ health_ok=false
1080+ log_silent " Port $port health check failed"
1081+ fi
1082+ done
1083+
1084+ if [ " $health_ok " = true ]; then
1085+ msg_success " All port-forwards healthy and responding"
1086+ else
1087+ msg_warn " Some port-forwards may not be healthy"
1088+ msg_info " If port-forwards die, restart with: cd ${NEW_DIR} && ./setup.sh --restart"
1089+ fi
8781090}
8791091
8801092# Check Kubernetes deployment status
@@ -927,14 +1139,11 @@ print_kubernetes_success() {
9271139 echo " "
9281140 echo -e " ${BLUE} cd ${NEW_DIR}${NC} "
9291141 echo " "
930- echo -e " ${YELLOW} Verify the deployment:${NC} "
931- echo -e " ${BLUE} kubectl get pods -n bito-ai-architect${NC} "
1142+ echo -e " ${YELLOW} Verify Upgrade:${NC} "
9321143 echo -e " ${BLUE} bitoarch status${NC} "
1144+ echo -e " ${BLUE} bitoarch index-status${NC} "
9331145 echo " "
9341146 echo -e " ${BLUE} ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC} "
935- echo -e " ⚠️ After a successful upgrade, rollback is supported via Helm:"
936- echo -e " ${YELLOW} helm rollback bitoarch -n bito-ai-architect${NC} "
937- echo " "
9381147 echo -e " ${BLUE} 📋 Upgrade Log${NC} "
9391148 echo -e " ${LOG_FILE} "
9401149 echo " "
0 commit comments