Skip to content

Commit 6b4e74f

Browse files
[DPE-10450] Fix crash restoring non-settable unit status in PG14 (#1791)
* [DPE-10450] Fix crash restoring non-settable unit status in _check_detached_storage The unit.status getter can return statuses the setter rejects (e.g. an "error" status left over from a previously failed hook). When storage is already attached, _check_detached_storage restored the cached status verbatim, raising InvalidStatusError ("status must be in active, blocked, maintenance, waiting, not 'error'") and deadlocking the unit in error. Only restore the cached status when it is a settable status type. Assisted-by: Claude:claude-4.8-opus * [DPE-10450] Guard restore of non-settable status in enable_disable_extensions enable_disable_extensions caches the current unit status and restores it verbatim at the end, exactly like _check_detached_storage did. The status getter can return statuses the setter rejects (e.g. an "error" status left over from a previously failed hook), so this restore hit the same InvalidStatusError crash. Extract the settable-status check into a shared _restore_unit_status helper and route both restore paths through it. Also cover the "unknown" status (not just "error") in both tests. Backport of 66ffc20de7e6bb5ff964de24dcfbc9f29c6ced8a; adapted to this branch, which sets self.unit.status directly (no set_unit_status helper). Assisted-by: Claude:claude-4.8-opus
1 parent 29b0f7e commit 6b4e74f

2 files changed

Lines changed: 55 additions & 2 deletions

File tree

src/charm.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
MaintenanceStatus,
5454
ModelError,
5555
Relation,
56+
StatusBase,
5657
Unit,
5758
WaitingStatus,
5859
)
@@ -1293,7 +1294,7 @@ def enable_disable_extensions(self, database: str | None = None) -> None:
12931294
if original_status.message == EXTENSION_OBJECT_MESSAGE:
12941295
self.unit.status = ActiveStatus()
12951296
return
1296-
self.unit.status = original_status
1297+
self._restore_unit_status(original_status)
12971298

12981299
def _check_extension_dependencies(self, extension: str, enable: bool) -> bool:
12991300
skip = False
@@ -1963,6 +1964,17 @@ def clean_ca_file_from_workload(self, secret_name: str) -> bool:
19631964
logger.exception("CA file failed to clean. Error in config update")
19641965
return False
19651966

1967+
def _restore_unit_status(self, status: StatusBase) -> None:
1968+
"""Restore a previously cached unit status.
1969+
1970+
The status getter can return statuses that the setter rejects (e.g. an
1971+
"error" status left over from a previously failed hook). Skip restoring
1972+
any status juju does not allow us to set, to avoid an InvalidStatusError
1973+
that would deadlock the unit.
1974+
"""
1975+
if isinstance(status, ActiveStatus | BlockedStatus | MaintenanceStatus | WaitingStatus):
1976+
self.unit.status = status
1977+
19661978
def _check_detached_storage(self) -> None:
19671979
"""Wait for storage to become available.
19681980
@@ -1978,7 +1990,7 @@ def _check_detached_storage(self) -> None:
19781990
logger.error("Data directory not attached.")
19791991
self.unit.status = WaitingStatus("Data directory not attached")
19801992
raise StorageUnavailableError()
1981-
self.unit.status = cached_status
1993+
self._restore_unit_status(cached_status)
19821994

19831995
def _restart(self, event: RunWithLock) -> None:
19841996
"""Restart PostgreSQL."""

tests/unit/test_charm.py

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@
2020
from ops.model import (
2121
ActiveStatus,
2222
BlockedStatus,
23+
ErrorStatus,
2324
MaintenanceStatus,
2425
ModelError,
2526
RelationDataTypeError,
27+
UnknownStatus,
2628
WaitingStatus,
2729
)
2830
from ops.testing import Harness
@@ -402,6 +404,33 @@ def test_enable_disable_extensions(harness, caplog):
402404
assert isinstance(harness.charm.unit.status, ActiveStatus)
403405

404406

407+
@pytest.mark.parametrize("unsettable_status", [ErrorStatus(), UnknownStatus()])
408+
def test_enable_disable_extensions_does_not_restore_unsettable_status(harness, unsettable_status):
409+
# enable_disable_extensions caches the current unit status and restores it
410+
# afterwards. The getter can return statuses the backend rejects on set
411+
# (e.g. "error" or "unknown") left over from a previously failed hook;
412+
# restoring such a cached status must not be attempted, otherwise the backend
413+
# raises InvalidStatusError/ModelError and deadlocks the unit.
414+
with (
415+
patch("charm.Patroni.get_primary") as _get_primary,
416+
patch("charm.PostgresqlOperatorCharm._unit_ip"),
417+
patch("charm.PostgresqlOperatorCharm._patroni"),
418+
patch("subprocess.check_output", return_value=b"C"),
419+
patch.object(PostgresqlOperatorCharm, "postgresql", Mock()) as postgresql_mock,
420+
):
421+
_get_primary.return_value = harness.charm.unit
422+
postgresql_mock.enable_disable_extensions.side_effect = None
423+
424+
# Cache an unsettable status that would otherwise be restored verbatim.
425+
harness.charm.unit._status = unsettable_status
426+
427+
# Must not raise when the cached status is restored at the end.
428+
harness.charm.enable_disable_extensions()
429+
430+
# The unsettable cached status was skipped, not restored.
431+
assert not isinstance(harness.charm.unit.status, ErrorStatus | UnknownStatus)
432+
433+
405434
def test_on_start(harness):
406435
with (
407436
patch(
@@ -1023,6 +1052,18 @@ def test_check_detached_storage(harness):
10231052
assert isinstance(harness.charm.unit.status, WaitingStatus)
10241053

10251054

1055+
@pytest.mark.parametrize("unsettable_status", [ErrorStatus(), UnknownStatus()])
1056+
def test_check_detached_storage_does_not_restore_unsettable_status(harness, unsettable_status):
1057+
# The unit.status getter can return statuses that the juju backend rejects
1058+
# on set (e.g. "error" or "unknown"). Restoring such a cached status must not
1059+
# be attempted, otherwise the backend raises InvalidStatusError/ModelError.
1060+
harness.charm.unit._status = unsettable_status
1061+
with patch("charm.PostgresqlOperatorCharm._is_storage_attached", return_value=True):
1062+
# Storage is attached, so the cached status would be restored verbatim;
1063+
# this must not raise.
1064+
harness.charm._check_detached_storage()
1065+
1066+
10261067
def test_restart(harness):
10271068
with (
10281069
patch("charm.Patroni.restart_postgresql") as _restart_postgresql,

0 commit comments

Comments
 (0)