Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion python/packages/jumpstarter/jumpstarter/exporter/hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,17 @@ async def execute_after_lease_hook(self, lease_scope: "LeaseContext") -> str | N
LogSource.AFTER_LEASE_HOOK,
)

async def _safe_release_lease(
self,
request_lease_release: Callable[[], Awaitable[None]] | None,
) -> None:
"""Call request_lease_release if provided, logging any errors."""
if request_lease_release:
try:
await request_lease_release()
except Exception as e:
logger.error("Failed to request lease release: %s", e, exc_info=True)

async def run_before_lease_hook(
self,
lease_scope: "LeaseContext",
Expand Down Expand Up @@ -656,7 +667,7 @@ async def run_before_lease_hook(

except HookExecutionError as e:
if e.should_shutdown_exporter():
# on_failure='exit' - defer shutdown until client handles the failure
# on_failure='exit' - release the lease, then defer shutdown
logger.error("beforeLease hook failed with on_failure='exit': %s", e)
lease_scope.skip_after_lease_hook = True
await report_status(
Expand All @@ -667,6 +678,12 @@ async def run_before_lease_hook(
ExporterStatus.OFFLINE,
"Exporter shutting down due to beforeLease hook failure",
)
# Release the lease so the controller sends leased=False,
# which unblocks the serve() loop to check _stop_requested.
# Without this, the exporter hangs waiting for a lease end
# that never comes.
await anyio.sleep(1.0)
await self._safe_release_lease(request_lease_release)
# Defer shutdown: sets _stop_requested=True, actual stop after lease cleanup
shutdown(exit_code=1, wait_for_lease_exit=True, should_unregister=True)
Comment on lines +681 to 688

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | 🏗️ Heavy lift

Preserve OFFLINE and set stop intent before releasing the lease.

The runtime callback passed here is Exporter._request_lease_release, which reports ExporterStatus.AVAILABLE with release_lease=True before setting lease_ended; this runs after Lines 677-680 report OFFLINE and before Line 688 sets _stop_requested. That can briefly advertise a shutting-down exporter as available and can let lease cleanup observe the lease end before the deferred stop flag is set.

Use a shutdown-specific release path that keeps the exporter OFFLINE, and set the deferred stop intent before unblocking lease cleanup.

Suggested direction
-                await anyio.sleep(1.0)
-                await self._safe_release_lease(request_lease_release)
                 # Defer shutdown: sets _stop_requested=True, actual stop after lease cleanup
                 shutdown(exit_code=1, wait_for_lease_exit=True, should_unregister=True)
+                await anyio.sleep(1.0)
+                await self._safe_release_lease(request_lease_release)

Also adjust the concrete release callback used for this path so release_lease=True does not report AVAILABLE after the exporter has been marked OFFLINE.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@python/packages/jumpstarter/jumpstarter/exporter/hooks.py` around lines 681 -
688, The shutdown path in Exporter._safe_release_lease /
Exporter._request_lease_release is briefly reporting the exporter as AVAILABLE
after it has already been marked OFFLINE, and _stop_requested is being set too
late. Update the shutdown-specific lease release flow so it preserves OFFLINE
status instead of using the normal release path, and set the deferred stop
intent before unblocking lease cleanup. Make the change in the shutdown callback
path that calls shutdown(..., wait_for_lease_exit=True, should_unregister=True)
so release_lease=True does not advertise AVAILABLE during shutdown.

else:
Expand Down
30 changes: 28 additions & 2 deletions python/packages/jumpstarter/jumpstarter/exporter/hooks_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -484,25 +484,51 @@ async def test_noninteractive_environment(self, lease_scope) -> None:
assert any("DEBIAN_FRONTEND=noninteractive" in call for call in info_calls)
assert any("GIT_TERMINAL_PROMPT=0" in call for call in info_calls)

async def test_before_lease_hook_exit_sets_skip_flag(self, lease_scope) -> None:
"""Test that beforeLease hook failure with on_failure=exit sets skip_after_lease_hook flag."""
async def test_before_lease_hook_exit_sets_skip_flag_and_releases_lease(self, lease_scope) -> None:
"""Test that beforeLease hook failure with on_failure=exit sets skip flag and releases lease."""
hook_config = HookConfigV1Alpha1(
before_lease=HookInstanceConfigV1Alpha1(script="exit 1", timeout=10, on_failure="exit"),
)
executor = HookExecutor(config=hook_config)

mock_report_status = AsyncMock()
mock_shutdown = MagicMock()
mock_request_lease_release = AsyncMock()

assert lease_scope.skip_after_lease_hook is False

await executor.run_before_lease_hook(
lease_scope,
mock_report_status,
mock_shutdown,
mock_request_lease_release,
)

assert lease_scope.skip_after_lease_hook is True
mock_request_lease_release.assert_called_once()
mock_shutdown.assert_called_once_with(exit_code=1, wait_for_lease_exit=True, should_unregister=True)

async def test_before_lease_hook_exit_handles_release_error(self, lease_scope) -> None:
"""Test that beforeLease hook with on_failure=exit handles release errors gracefully."""
hook_config = HookConfigV1Alpha1(
before_lease=HookInstanceConfigV1Alpha1(script="exit 1", timeout=10, on_failure="exit"),
)
executor = HookExecutor(config=hook_config)

mock_report_status = AsyncMock()
mock_shutdown = MagicMock()
mock_request_lease_release = AsyncMock(side_effect=RuntimeError("controller unavailable"))

await executor.run_before_lease_hook(
lease_scope,
mock_report_status,
mock_shutdown,
mock_request_lease_release,
)

assert lease_scope.skip_after_lease_hook is True
mock_request_lease_release.assert_called_once()
# shutdown should still be called even if release fails
mock_shutdown.assert_called_once_with(exit_code=1, wait_for_lease_exit=True, should_unregister=True)

async def test_before_lease_hook_endlease_does_not_set_skip_flag(self, lease_scope) -> None:
Expand Down
Loading