Skip to content

Commit 3f2adf5

Browse files
devaladedevalade
committed
Fix deploy hang caused by broken SSH multiplex cleanup trap
Centralized SSH multiplex management in load_config() and fixed trap chain in zero-downtime deployments to include both release_deploy_lock and stop_ssh_multiplex. Refactored all SSH commands to use new remote_exec/remote_copy/remote_rsync helpers for consistency and reduced code duplication.
1 parent 90cadb1 commit 3f2adf5

17 files changed

Lines changed: 147 additions & 80 deletions

‎build-dist.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ GREEN='\033[0;32m'
77
BLUE='\033[0;34m'
88
NC='\033[0m'
99

10-
VERSION="1.1.0"
10+
VERSION="1.1.1"
1111
DIST_DIR="dist"
1212
ARCHIVE_NAME="shipnode-payload.tar.gz"
1313
INSTALLER_NAME="shipnode-installer.sh"
@@ -90,7 +90,7 @@ YELLOW='\033[1;33m'
9090
BLUE='\033[0;34m'
9191
NC='\033[0m'
9292
93-
VERSION="1.1.0"
93+
VERSION="1.1.1"
9494
INSTALL_DIR="$HOME/.shipnode"
9595
9696
echo -e "${BLUE}╔════════════════════════════════════╗${NC}"

‎lib/commands/config.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ load_config() {
5858
error "BACKEND_PORT required for backend apps"
5959
fi
6060
fi
61+
62+
# Start SSH multiplex connection once per session
63+
if [ -z "${_SHIPNODE_MULTIPLEX_STARTED:-}" ]; then
64+
start_ssh_multiplex
65+
_SHIPNODE_MULTIPLEX_STARTED=1
66+
fi
6167
}
6268

6369
# Interactive users.yml generation

‎lib/commands/deploy.sh‎

Lines changed: 51 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,17 @@ cmd_deploy() {
1414
info "Deploying $APP_TYPE to $SSH_USER@$SSH_HOST..."
1515

1616
# Create remote directory
17-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "mkdir -p $REMOTE_PATH"
17+
remote_exec "mkdir -p $REMOTE_PATH"
1818

1919
if [ "$APP_TYPE" = "backend" ]; then
20-
deploy_backend
20+
deploy_backend "$SKIP_BUILD"
2121
else
2222
deploy_frontend "$SKIP_BUILD"
2323
fi
2424
}
2525

2626
deploy_backend() {
27+
local SKIP_BUILD=$1
2728
info "Deploying backend application..."
2829

2930
# Check if package.json exists
@@ -54,37 +55,52 @@ Deployment blocked to prevent port conflict."
5455
fi
5556

5657
if [ "$ZERO_DOWNTIME" = "true" ]; then
57-
deploy_backend_zero_downtime
58+
deploy_backend_zero_downtime "$SKIP_BUILD"
5859
else
59-
deploy_backend_legacy
60+
deploy_backend_legacy "$SKIP_BUILD"
6061
fi
6162
}
6263

6364
deploy_backend_legacy() {
65+
local SKIP_BUILD=$1
6466
info "Using legacy deployment (non-zero-downtime)..."
6567

6668
# Rsync application files
6769
info "Syncing files to server..."
68-
rsync -avz --progress \
70+
remote_rsync -avz --progress \
6971
--exclude 'node_modules' \
7072
--exclude '.env' \
7173
--exclude '.git' \
7274
--exclude '.gitignore' \
7375
--exclude 'shipnode.conf' \
7476
--exclude '*.log' \
75-
-e "ssh -p $SSH_PORT" \
7677
./ "$SSH_USER@$SSH_HOST:$REMOTE_PATH/"
7778

7879
success "Files synced"
7980

80-
# Install dependencies
81+
# Install dependencies and build
8182
info "Installing dependencies..."
82-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
83+
remote_exec bash << ENDSSH
8384
set -e
8485
cd $REMOTE_PATH
8586
$PKG_INSTALL_CMD
87+
88+
# Build if package.json has build script and not skipping
89+
if [ "$SKIP_BUILD" = false ]; then
90+
if jq -e '.scripts.build' package.json >/dev/null 2>&1; then
91+
echo "Building application..."
92+
$PKG_RUN_CMD
93+
fi
94+
fi
95+
96+
# Link .env into build/ for frameworks that resolve from build dir (AdonisJS)
97+
if [ -d build ] && [ -f .env ]; then
98+
ln -sf $REMOTE_PATH/.env build/.env
99+
fi
86100
ENDSSH
87101

102+
success "Dependencies installed and build complete"
103+
88104
# Run pre-deploy hook
89105
if ! run_pre_deploy_hook "$REMOTE_PATH"; then
90106
error "Pre-deploy hook failed, aborting deployment"
@@ -96,9 +112,9 @@ ENDSSH
96112
# Always regenerate ecosystem file to ensure it's up to date
97113
info "Generating PM2 ecosystem config..."
98114
generate_ecosystem_file "$PKG_MANAGER" "$PM2_APP_NAME" "$REMOTE_PATH" \
99-
| ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "cat > $REMOTE_PATH/ecosystem.config.cjs"
115+
| remote_exec "cat > $REMOTE_PATH/ecosystem.config.cjs"
100116

101-
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
117+
remote_exec bash << ENDSSH
102118
set -e
103119
pm2 startOrReload $REMOTE_PATH/ecosystem.config.cjs --update-env
104120
pm2 save
@@ -118,12 +134,13 @@ ENDSSH
118134
}
119135

120136
deploy_backend_zero_downtime() {
137+
local SKIP_BUILD=$1
121138
info "Using zero-downtime deployment..."
122139

123140
# Acquire deployment lock
124141
info "Acquiring deployment lock..."
125142
acquire_deploy_lock
126-
trap release_deploy_lock EXIT
143+
trap 'release_deploy_lock; stop_ssh_multiplex' EXIT
127144
success "Lock acquired"
128145

129146
# Generate release timestamp
@@ -142,21 +159,20 @@ deploy_backend_zero_downtime() {
142159

143160
# Rsync to new release directory
144161
info "Syncing files to release directory..."
145-
rsync -avz --progress \
162+
remote_rsync -avz --progress \
146163
--exclude 'node_modules' \
147164
--exclude '.env' \
148165
--exclude '.git' \
149166
--exclude '.gitignore' \
150167
--exclude 'shipnode.conf' \
151168
--exclude '*.log' \
152-
-e "ssh -p $SSH_PORT" \
153169
./ "$SSH_USER@$SSH_HOST:$release_path/"
154170

155171
success "Files synced to $release_path"
156172

157-
# Link shared resources and install dependencies
173+
# Link shared resources, install dependencies, and build
158174
info "Setting up release environment..."
159-
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
175+
remote_exec bash << ENDSSH
160176
set -e
161177
cd $release_path
162178
@@ -167,6 +183,19 @@ deploy_backend_zero_downtime() {
167183
168184
# Install dependencies
169185
$PKG_INSTALL_CMD
186+
187+
# Build if package.json has build script and not skipping
188+
if [ "$SKIP_BUILD" = false ]; then
189+
if jq -e '.scripts.build' package.json >/dev/null 2>&1; then
190+
echo "Building application..."
191+
$PKG_RUN_CMD
192+
fi
193+
fi
194+
195+
# Link .env into build/ for frameworks that resolve from build dir (AdonisJS)
196+
if [ -d build ] && [ -f .env ]; then
197+
ln -sf $release_path/.env build/.env
198+
fi
170199
ENDSSH
171200

172201
success "Release prepared"
@@ -186,9 +215,9 @@ ENDSSH
186215
# Always regenerate ecosystem file to ensure it's up to date
187216
info "Generating PM2 ecosystem config..."
188217
generate_ecosystem_file "$PKG_MANAGER" "$PM2_APP_NAME" "$REMOTE_PATH/current" \
189-
| ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "cat > $REMOTE_PATH/shared/ecosystem.config.cjs"
218+
| remote_exec "cat > $REMOTE_PATH/shared/ecosystem.config.cjs"
190219

191-
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
220+
remote_exec bash << ENDSSH
192221
set -e
193222
pm2 startOrReload $REMOTE_PATH/shared/ecosystem.config.cjs --update-env
194223
pm2 save
@@ -262,8 +291,7 @@ deploy_frontend_legacy() {
262291

263292
# Rsync build directory
264293
info "Syncing $BUILD_DIR to server..."
265-
rsync -avz --progress --delete \
266-
-e "ssh -p $SSH_PORT" \
294+
remote_rsync -avz --progress --delete \
267295
"$BUILD_DIR/" "$SSH_USER@$SSH_HOST:$REMOTE_PATH/"
268296

269297
success "Frontend deployed"
@@ -291,7 +319,7 @@ deploy_frontend_zero_downtime() {
291319

292320
# Acquire deployment lock
293321
acquire_deploy_lock
294-
trap release_deploy_lock EXIT
322+
trap 'release_deploy_lock; stop_ssh_multiplex' EXIT
295323

296324
# Generate release timestamp
297325
local timestamp=$(generate_release_timestamp)
@@ -304,8 +332,7 @@ deploy_frontend_zero_downtime() {
304332

305333
# Rsync build output to release directory
306334
info "Syncing $BUILD_DIR to release directory..."
307-
rsync -avz --progress --delete \
308-
-e "ssh -p $SSH_PORT" \
335+
remote_rsync -avz --progress --delete \
309336
"$BUILD_DIR/" "$SSH_USER@$SSH_HOST:$release_path/"
310337

311338
success "Files synced to $release_path"
@@ -340,7 +367,7 @@ deploy_frontend_zero_downtime() {
340367
configure_caddy_backend() {
341368
info "Configuring Caddy reverse proxy for $DOMAIN..."
342369

343-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
370+
remote_exec bash << ENDSSH
344371
set -e
345372
346373
# Create conf.d directory for per-app configs
@@ -394,7 +421,7 @@ configure_caddy_frontend() {
394421
# Derive app name from remote path for config file naming
395422
local APP_NAME=$(basename "$REMOTE_PATH")
396423

397-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
424+
remote_exec bash << ENDSSH
398425
set -e
399426
400427
# Create conf.d directory for per-app configs

‎lib/commands/doctor.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ check_ssh_connection() {
158158
local ssh_port="${SSH_PORT:-22}"
159159

160160
# Test connection with 5 second timeout
161-
if ssh -o BatchMode=yes -o ConnectTimeout=5 -p "$ssh_port" "$SSH_USER@$SSH_HOST" "exit" &>/dev/null; then
161+
if remote_exec "exit" &>/dev/null; then
162162
echo " ✓ SSH connection successful ($SSH_USER@$SSH_HOST:$ssh_port)"
163163
return 0
164164
else
@@ -182,7 +182,7 @@ check_remote_environment() {
182182

183183
# Single batched SSH call for all remote checks
184184
local remote_output
185-
remote_output=$(ssh -p "$ssh_port" "$SSH_USER@$SSH_HOST" bash << 'REMOTE_CHECKS'
185+
remote_output=$(remote_exec bash << 'REMOTE_CHECKS'
186186
# Check node
187187
if command -v node &> /dev/null; then
188188
echo "NODE_OK:$(node --version)"

‎lib/commands/env.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,23 +15,23 @@ cmd_env() {
1515
if [ "$ZERO_DOWNTIME" = "true" ]; then
1616
# Upload to shared directory for zero-downtime deployments
1717
TARGET_PATH="$REMOTE_PATH/shared/.env"
18-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "mkdir -p $REMOTE_PATH/shared"
18+
remote_exec "mkdir -p $REMOTE_PATH/shared"
1919
else
2020
# Upload directly to app directory for legacy deployments
2121
TARGET_PATH="$REMOTE_PATH/.env"
22-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "mkdir -p $REMOTE_PATH"
22+
remote_exec "mkdir -p $REMOTE_PATH"
2323
fi
2424

2525
# Upload the environment file
26-
scp -P "$SSH_PORT" "$LOCAL_ENV" "$SSH_USER@$SSH_HOST:$TARGET_PATH"
26+
remote_copy "$LOCAL_ENV" "$SSH_USER@$SSH_HOST:$TARGET_PATH"
2727

2828
success "Uploaded $LOCAL_ENV to $TARGET_PATH"
2929

3030
# Restart backend app if running to reload env vars
3131
if [ "$APP_TYPE" = "backend" ]; then
3232
info "Restarting app to reload environment variables..."
33-
if ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "pm2 describe $PM2_APP_NAME" &> /dev/null; then
34-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "pm2 reload $PM2_APP_NAME"
33+
if remote_exec "pm2 describe $PM2_APP_NAME" &> /dev/null; then
34+
remote_exec "pm2 reload $PM2_APP_NAME"
3535
success "App restarted with new environment variables"
3636
else
3737
warn "App not running. Environment variables will be loaded on next deploy."

‎lib/commands/migrate.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@ cmd_migrate() {
2727
# Generate ecosystem file for backend apps (always regenerate to ensure it's up to date)
2828
if [ "$APP_TYPE" = "backend" ]; then
2929
info "Generating PM2 ecosystem config..."
30-
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "mkdir -p $REMOTE_PATH/shared"
30+
remote_exec "mkdir -p $REMOTE_PATH/shared"
3131
generate_ecosystem_file "$PKG_MANAGER" "$PM2_APP_NAME" "$REMOTE_PATH/current" \
32-
| ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "cat > $REMOTE_PATH/shared/ecosystem.config.cjs"
32+
| remote_exec "cat > $REMOTE_PATH/shared/ecosystem.config.cjs"
3333
fi
3434

35-
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
35+
remote_exec bash << ENDSSH
3636
set -e
3737
cd $REMOTE_PATH
3838

‎lib/commands/rollback.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ cmd_rollback() {
1010
info "Fetching release history..."
1111

1212
# Get target release
13-
local target_release=$(ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
13+
local target_release=$(remote_exec bash << ENDSSH
1414
cd $REMOTE_PATH/.shipnode
1515
cat releases.json | jq -r ".[-$((steps_back + 1))].timestamp // empty"
1616
ENDSSH
@@ -55,7 +55,7 @@ cmd_releases() {
5555

5656
info "Fetching releases..."
5757

58-
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
58+
remote_exec bash << ENDSSH
5959
cd $REMOTE_PATH
6060
6161
# Get current release

‎lib/commands/setup.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ cmd_setup() {
44
info "Setting up server $SSH_USER@$SSH_HOST..."
55

66
# Check SSH connection
7-
if ! ssh -o ConnectTimeout=10 -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "exit"; then
7+
if ! remote_exec "exit"; then
88
error "Cannot connect to $SSH_USER@$SSH_HOST:$SSH_PORT"
99
fi
1010

@@ -34,7 +34,7 @@ cmd_setup() {
3434

3535
info "Node.js version: $node_version"
3636

37-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << 'ENDSSH'
37+
remote_exec bash << 'ENDSSH'
3838
NODE_VERSION="'"$node_version"'"
3939
set -e
4040

‎lib/commands/status.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ cmd_status() {
33

44
if [ "$APP_TYPE" = "backend" ]; then
55
info "Checking PM2 status for $PM2_APP_NAME..."
6-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "pm2 status $PM2_APP_NAME"
6+
remote_exec "pm2 status $PM2_APP_NAME"
77
else
88
info "Checking frontend files..."
9-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "ls -lh $REMOTE_PATH | head -20"
9+
remote_exec "ls -lh $REMOTE_PATH | head -20"
1010
fi
1111
}
1212

@@ -19,7 +19,7 @@ cmd_logs() {
1919
fi
2020

2121
info "Streaming logs for $PM2_APP_NAME (Ctrl+C to exit)..."
22-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "pm2 logs $PM2_APP_NAME"
22+
ssh_cmd -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "pm2 logs $PM2_APP_NAME"
2323
}
2424

2525
# Restart app (backend only)
@@ -31,7 +31,7 @@ cmd_restart() {
3131
fi
3232

3333
info "Restarting $PM2_APP_NAME..."
34-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "pm2 restart $PM2_APP_NAME"
34+
remote_exec "pm2 restart $PM2_APP_NAME"
3535
success "App restarted"
3636
}
3737

@@ -44,7 +44,7 @@ cmd_stop() {
4444
fi
4545

4646
info "Stopping $PM2_APP_NAME..."
47-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "pm2 stop $PM2_APP_NAME"
47+
remote_exec "pm2 stop $PM2_APP_NAME"
4848
success "App stopped"
4949
}
5050

‎lib/commands/unlock.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ cmd_unlock() {
44
info "Checking for deployment lock on $SSH_USER@$SSH_HOST..."
55

66
local lock_info
7-
lock_info=$(ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash -s "$REMOTE_PATH" << 'ENDSSH'
7+
lock_info=$(remote_exec bash -s "$REMOTE_PATH" << 'ENDSSH'
88
REMOTE_PATH="$1"
99
LOCK_FILE="$REMOTE_PATH/.shipnode/deploy.lock"
1010
@@ -33,7 +33,7 @@ ENDSSH
3333
return 0
3434
fi
3535

36-
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "rm -f $REMOTE_PATH/.shipnode/deploy.lock"
36+
remote_exec "rm -f $REMOTE_PATH/.shipnode/deploy.lock"
3737
success "Deployment lock cleared"
3838
}
3939

0 commit comments

Comments
 (0)