Skip to content

Commit d9c42b3

Browse files
devaladedevalade
committed
Fix PM2 cwd bug in zero-downtime deployments
Problem: pm2 reload --update-env does NOT update PM2's stored cwd. After deploying a new release, PM2 kept running from the old resolved release path (e.g. releases/20260131...) even though the 'current' symlink pointed to the new release. Root cause: When PM2 starts a process, it resolves symlinks and stores the absolute path as cwd. The 'pm2 reload' command only restarts the process and updates env vars--it never updates the cwd. So after each deploy, the app ran from the old release directory, causing crashes due to missing dependencies or stale code. Fix: Replace 'pm2 reload' with 'pm2 delete' + 'pm2 start' in: - deploy_backend() (non-zero-downtime) - deploy_backend_zero_downtime() - rollback_to_release() This ensures PM2 always picks up the new release directory after each deployment or rollback. The tradeoff is a brief moment of downtime during the delete→start transition. Tested: Deployed nodejs-boilerplate successfully, confirmed health check passes and PM2 runs from the correct release directory.
1 parent 024013b commit d9c42b3

2 files changed

Lines changed: 103 additions & 70 deletions

File tree

‎lib/commands/deploy.sh‎

Lines changed: 61 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,13 @@ ENDSSH
9898
set -e
9999
cd $REMOTE_PATH
100100
101-
if pm2 describe $PM2_APP_NAME > /dev/null 2>&1; then
102-
pm2 reload $PM2_APP_NAME
101+
# Delete and re-start to ensure PM2 picks up the correct cwd
102+
pm2 delete $PM2_APP_NAME 2>/dev/null || true
103+
104+
if [ -f ecosystem.config.js ]; then
105+
pm2 start ecosystem.config.js
103106
else
104-
if [ -f ecosystem.config.js ]; then
105-
pm2 start ecosystem.config.js
106-
else
107-
$PKG_START_CMD
108-
fi
107+
$PKG_START_CMD
109108
fi
110109
111110
pm2 save
@@ -197,14 +196,13 @@ ENDSSH
197196
set -e
198197
cd $REMOTE_PATH/current
199198
200-
if pm2 describe $PM2_APP_NAME > /dev/null 2>&1; then
201-
pm2 reload $PM2_APP_NAME --update-env
199+
# Delete and re-start to ensure PM2 picks up the new release cwd
200+
pm2 delete $PM2_APP_NAME 2>/dev/null || true
201+
202+
if [ -f ecosystem.config.js ]; then
203+
pm2 start ecosystem.config.js
202204
else
203-
if [ -f ecosystem.config.js ]; then
204-
pm2 start ecosystem.config.js
205-
else
206-
$PKG_START_CMD
207-
fi
205+
$PKG_START_CMD
208206
fi
209207
210208
pm2 save
@@ -356,28 +354,42 @@ deploy_frontend_zero_downtime() {
356354
configure_caddy_backend() {
357355
info "Configuring Caddy reverse proxy for $DOMAIN..."
358356

359-
local CADDY_CONFIG="/etc/caddy/Caddyfile"
360-
361357
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
362358
set -e
363359
364-
# Backup existing Caddyfile
365-
[ -f $CADDY_CONFIG ] && cp $CADDY_CONFIG ${CADDY_CONFIG}.backup
360+
# Create conf.d directory for per-app configs
361+
mkdir -p /etc/caddy/conf.d
362+
363+
# Ensure main Caddyfile imports conf.d configs
364+
if [ ! -f /etc/caddy/Caddyfile ]; then
365+
cat > /etc/caddy/Caddyfile << 'MAIN_EOF'
366+
# ShipNode managed Caddyfile
367+
# Per-app configurations are in /etc/caddy/conf.d/
368+
369+
import /etc/caddy/conf.d/*.caddy
370+
MAIN_EOF
371+
elif ! grep -q "import /etc/caddy/conf.d/\*.caddy" /etc/caddy/Caddyfile 2>/dev/null; then
372+
# Backup existing Caddyfile
373+
cp /etc/caddy/Caddyfile /etc/caddy/Caddyfile.backup
374+
# Add import if missing
375+
echo "" >> /etc/caddy/Caddyfile
376+
echo "import /etc/caddy/conf.d/*.caddy" >> /etc/caddy/Caddyfile
377+
fi
366378
367-
# Create Caddyfile
368-
cat > $CADDY_CONFIG << 'EOF'
379+
# Write per-app configuration
380+
cat > /etc/caddy/conf.d/$PM2_APP_NAME.caddy << 'APP_EOF'
369381
$DOMAIN {
370382
reverse_proxy localhost:$BACKEND_PORT
371383
encode gzip
372384
373385
log {
374-
output file /var/log/caddy/${PM2_APP_NAME}.log
386+
output file /var/log/caddy/$PM2_APP_NAME.log
375387
}
376388
}
377-
EOF
389+
APP_EOF
378390
379391
# Reload Caddy
380-
caddy reload --config $CADDY_CONFIG
392+
caddy reload --config /etc/caddy/Caddyfile
381393
ENDSSH
382394

383395
success "Caddy configured for $DOMAIN → localhost:$BACKEND_PORT"
@@ -386,22 +398,40 @@ ENDSSH
386398
configure_caddy_frontend() {
387399
info "Configuring Caddy static file server for $DOMAIN..."
388400

389-
local CADDY_CONFIG="/etc/caddy/Caddyfile"
390401
local SERVE_PATH="$REMOTE_PATH"
391402

392403
# Use current symlink if zero-downtime is enabled
393404
if [ "$ZERO_DOWNTIME" = "true" ]; then
394405
SERVE_PATH="$REMOTE_PATH/current"
395406
fi
396407

408+
# Derive app name from remote path for config file naming
409+
local APP_NAME=$(basename "$REMOTE_PATH")
410+
397411
ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
398412
set -e
399413
400-
# Backup existing Caddyfile
401-
[ -f $CADDY_CONFIG ] && cp $CADDY_CONFIG ${CADDY_CONFIG}.backup
414+
# Create conf.d directory for per-app configs
415+
mkdir -p /etc/caddy/conf.d
416+
417+
# Ensure main Caddyfile imports conf.d configs
418+
if [ ! -f /etc/caddy/Caddyfile ]; then
419+
cat > /etc/caddy/Caddyfile << 'MAIN_EOF'
420+
# ShipNode managed Caddyfile
421+
# Per-app configurations are in /etc/caddy/conf.d/
422+
423+
import /etc/caddy/conf.d/*.caddy
424+
MAIN_EOF
425+
elif ! grep -q "import /etc/caddy/conf.d/\*.caddy" /etc/caddy/Caddyfile 2>/dev/null; then
426+
# Backup existing Caddyfile
427+
cp /etc/caddy/Caddyfile /etc/caddy/Caddyfile.backup
428+
# Add import if missing
429+
echo "" >> /etc/caddy/Caddyfile
430+
echo "import /etc/caddy/conf.d/*.caddy" >> /etc/caddy/Caddyfile
431+
fi
402432
403-
# Create Caddyfile
404-
cat > $CADDY_CONFIG << 'EOF'
433+
# Write per-app configuration
434+
cat > /etc/caddy/conf.d/$APP_NAME.caddy << 'APP_EOF'
405435
$DOMAIN {
406436
root * $SERVE_PATH
407437
file_server
@@ -410,13 +440,13 @@ $DOMAIN {
410440
try_files {path} /index.html
411441
412442
log {
413-
output file /var/log/caddy/frontend.log
443+
output file /var/log/caddy/$APP_NAME.log
414444
}
415445
}
416-
EOF
446+
APP_EOF
417447
418448
# Reload Caddy
419-
caddy reload --config $CADDY_CONFIG
449+
caddy reload --config /etc/caddy/Caddyfile
420450
ENDSSH
421451

422452
success "Caddy configured for $DOMAIN → $SERVE_PATH"

‎lib/release.sh‎

Lines changed: 42 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,19 @@ rollback_to_release() {
115115
switch_symlink "$release_path"
116116

117117
if [ "$APP_TYPE" = "backend" ]; then
118-
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "pm2 reload $PM2_APP_NAME"
118+
# Delete and re-start to ensure PM2 picks up the rolled-back release cwd
119+
local PKG_START_CMD=$(get_pkg_start_cmd "$PKG_MANAGER" "$PM2_APP_NAME")
120+
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
121+
set -e
122+
cd $REMOTE_PATH/current
123+
pm2 delete $PM2_APP_NAME 2>/dev/null || true
124+
if [ -f ecosystem.config.js ]; then
125+
pm2 start ecosystem.config.js
126+
else
127+
$PKG_START_CMD
128+
fi
129+
pm2 save
130+
ENDSSH
119131
fi
120132

121133
success "Rolled back to $timestamp"
@@ -126,50 +138,46 @@ rollback_to_release() {
126138
run_pre_deploy_hook() {
127139
local release_path=$1
128140
local hook_script=${PRE_DEPLOY_SCRIPT:-".shipnode/pre-deploy.sh"}
129-
141+
130142
# Check if hook script exists locally
131143
if [ ! -f "$hook_script" ]; then
132144
return 0
133145
fi
134-
146+
135147
info "Running pre-deploy hook: $hook_script"
136-
148+
137149
# Copy hook script to release directory
138150
if ! scp -P "$SSH_PORT" "$hook_script" "$SSH_USER@$SSH_HOST:$release_path/.shipnode-pre-deploy.sh" 2>&1; then
139151
error "Failed to copy pre-deploy hook to server"
152+
return 1
140153
fi
141-
142-
# Execute hook on remote server
143-
local result
144-
result=$(ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
154+
155+
# Execute hook on remote server with output streaming (not captured)
156+
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
145157
set -e
146158
cd $release_path
147-
159+
148160
# Export environment variables for hook
149161
export RELEASE_PATH="$release_path"
150162
export REMOTE_PATH="$REMOTE_PATH"
151163
export PM2_APP_NAME="${PM2_APP_NAME:-}"
152164
export BACKEND_PORT="${BACKEND_PORT:-}"
153165
export SHARED_ENV_PATH="$REMOTE_PATH/shared/.env"
154-
166+
155167
# Make hook executable and run it
156168
chmod +x .shipnode-pre-deploy.sh
157-
if ./.shipnode-pre-deploy.sh; then
158-
echo "SUCCESS"
159-
else
160-
echo "FAILED"
161-
fi
162-
169+
./.shipnode-pre-deploy.sh
170+
163171
# Cleanup hook script
164172
rm -f .shipnode-pre-deploy.sh
165173
ENDSSH
166-
)
167-
168-
if [[ "$result" == *"SUCCESS"* ]]; then
174+
175+
local exit_code=$?
176+
if [ $exit_code -eq 0 ]; then
169177
success "Pre-deploy hook completed"
170178
return 0
171179
else
172-
error "Pre-deploy hook failed"
180+
error "Pre-deploy hook failed (exit code: $exit_code)"
173181
return 1
174182
fi
175183
}
@@ -179,51 +187,46 @@ ENDSSH
179187
run_post_deploy_hook() {
180188
local current_path="$REMOTE_PATH/current"
181189
local hook_script=${POST_DEPLOY_SCRIPT:-".shipnode/post-deploy.sh"}
182-
190+
183191
# Check if hook script exists locally
184192
if [ ! -f "$hook_script" ]; then
185193
return 0
186194
fi
187-
195+
188196
info "Running post-deploy hook: $hook_script"
189-
197+
190198
# Copy hook script to current directory
191199
if ! scp -P "$SSH_PORT" "$hook_script" "$SSH_USER@$SSH_HOST:$current_path/.shipnode-post-deploy.sh" 2>&1; then
192200
warn "Failed to copy post-deploy hook to server"
193201
return 1
194202
fi
195-
196-
# Execute hook on remote server
197-
local result
198-
result=$(ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
203+
204+
# Execute hook on remote server with output streaming (not captured)
205+
ssh -T -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" bash << ENDSSH
199206
set -e
200207
cd $current_path
201-
208+
202209
# Export environment variables for hook
203210
export RELEASE_PATH="$current_path"
204211
export REMOTE_PATH="$REMOTE_PATH"
205212
export PM2_APP_NAME="${PM2_APP_NAME:-}"
206213
export BACKEND_PORT="${BACKEND_PORT:-}"
207214
export SHARED_ENV_PATH="$REMOTE_PATH/shared/.env"
208-
215+
209216
# Make hook executable and run it
210217
chmod +x .shipnode-post-deploy.sh
211-
if ./.shipnode-post-deploy.sh; then
212-
echo "SUCCESS"
213-
else
214-
echo "FAILED"
215-
fi
216-
218+
./.shipnode-post-deploy.sh
219+
217220
# Cleanup hook script
218221
rm -f .shipnode-post-deploy.sh
219222
ENDSSH
220-
)
221-
222-
if [[ "$result" == *"SUCCESS"* ]]; then
223+
224+
local exit_code=$?
225+
if [ $exit_code -eq 0 ]; then
223226
success "Post-deploy hook completed"
224227
return 0
225228
else
226-
warn "Post-deploy hook failed (deployment still successful)"
229+
warn "Post-deploy hook failed (deployment still successful, exit code: $exit_code)"
227230
return 1
228231
fi
229232
}

0 commit comments

Comments
 (0)