From 72eaae5c20cd1253ace345edd1c79f2c84bd6025 Mon Sep 17 00:00:00 2001 From: Pietro Date: Mon, 18 May 2026 11:04:11 +0000 Subject: [PATCH 1/2] feat(release-controller): add --ignore-proposal-id to force resubmission The reconciler rebuilds its in-memory `ReconcilerState` on every cycle from the public dashboard (`/proposals?limit=50&...`) and the governance canister (`dre proposals filter -t ic-os-version-election`), and treats every matching election proposal as "already submitted" regardless of status. A rejected or failed proposal therefore prevents the reconciler from ever submitting a fresh one for the same version, with no operator escape hatch. Add a repeatable `--ignore-proposal-id=` flag that the reconciler applies as a filter on the result of both proposal-retriever sources before they populate `ReconcilerState`, with a warning log per drop. The affected version goes back to the "no proposal" state and a new proposal is submitted on the next cycle. The IC governance canister still refuses to re-elect an already blessed version, so this only helps when the prior proposal did not result in a blessing (typically REJECTED or FAILED). Add integration tests covering both branches of the filter (with and without ignore IDs configured) using a self-contained retriever fixture that does not depend on the existing `MockDashboard` quirk where `_fake_proposal` always emits `hostos_version_to_elect`. Refresh the README "Operator runbook" with a "Forcing resubmission" section and replace the stale `--skip-preloading-state` dev-mode docs (the flag is not wired into `main`'s argparse). --- release-controller/README.md | 37 +++++- release-controller/reconciler.py | 107 +++++++++++++++- .../tests/test_reconciler_integration.py | 118 ++++++++++++++++++ 3 files changed, 254 insertions(+), 8 deletions(-) diff --git a/release-controller/README.md b/release-controller/README.md index 10abbd5e0..5c40b3017 100644 --- a/release-controller/README.md +++ b/release-controller/README.md @@ -205,6 +205,33 @@ If the proposal was indeed submitted, you don't have to do anything -- the recon rm /state/ ``` +### Forcing resubmission of a previously-submitted proposal + +The reconciler rebuilds its in-memory state on every cycle from the public +dashboard and the governance canister (via `dre proposals filter -t +ic-os-version-election`), and treats every matching election proposal as +"already submitted" regardless of its status. A rejected or failed proposal +will therefore prevent the reconciler from ever submitting a fresh one for +the same version. + +To work around this, pass one or more `--ignore-proposal-id=` flags +when starting the reconciler. Each listed proposal is filtered out of +the dashboard / governance lookups, so the affected version goes back to +the "no proposal" state and the reconciler submits a new proposal on the +next cycle. + +In production, edit `bases/apps/ic-release-controller/controller/controller.yaml` +in the `k8s` repo and append the flag(s) to the `release-controller` container's +`args` list, e.g. + +```yaml +args: [--one-line-logs, --verbose, --ignore-proposal-id=141776] +``` + +Caveat: the IC governance canister independently refuses to re-elect a +version that is already blessed. This flag only helps when the prior +proposal did **not** result in a blessing (typically REJECTED or FAILED). + ## Development Please see the parent folder's `README.md` for virtual environment setup. @@ -243,11 +270,11 @@ bazel run //release-controller:release-controller \ Typing errors preventing you from running it, because you are editing code and testing your changes? Add `--output_groups=-mypy` right after `bazel run`. -The optional argument `--skip-preloading-state` makes it so that the reconciler -will not preload its list of known proposals by version from the governance -canister. It is useful (in conjunction with an empty reconciler state folder) -to make the reconciler do all the work of submitting proposals again. It should -only be used alongside `--dry-run`, to avoid submitting proposals twice. +The optional argument `--ignore-proposal-id` (repeatable, takes an NNS proposal +ID) makes the reconciler skip the matching election proposal when it builds its +list of known proposals by version. The corresponding version is then treated +as not yet proposed and a fresh proposal is submitted on the next cycle. See +*Forcing resubmission of a previously-submitted proposal* above. ### Running the reconciler in the container it ships diff --git a/release-controller/reconciler.py b/release-controller/reconciler.py index 1fff2982b..86f2b8ea8 100644 --- a/release-controller/reconciler.py +++ b/release-controller/reconciler.py @@ -418,8 +418,19 @@ def __init__( dashboard: public_dashboard.DashboardAPI, slack_announcer: slack_announce.SlackAnnouncerProtocol, ignore_releases: list[str] | None = None, + ignore_proposal_ids: typing.Iterable[int] | None = None, ): - """Create a new reconciler.""" + """Create a new reconciler. + + ``ignore_proposal_ids`` is an optional set of NNS proposal IDs that + will be dropped from the result of every election-proposal lookup + before it is loaded into the reconciler state. This allows forcing + a resubmission for a version whose prior proposal is no longer + actionable (e.g. rejected/failed). Note that the IC governance + canister still independently rejects re-election of an already + blessed version, so this only helps when the prior proposal did + not bless the version. + """ self.forum_client = forum_client self.loader = loader self.notes_client = notes_client @@ -429,12 +440,73 @@ def __init__( self.ic_prometheus = active_version_provider self.ic_repo = ic_repo self.ignore_releases = ignore_releases or [] + self.ignore_proposal_ids: set[int] = set(ignore_proposal_ids or ()) self.dre = dre self.dashboard = dashboard self.slack_announcer = slack_announcer self.change_determinator_factory = change_determinator_factory self.local_release_state: dict[str, dict[str, dict[OsKind, VersionState]]] = {} + def _filtered_proposals_retriever( + self, + retriever: typing.Callable[ + [], + tuple[ + dict[str, dre_cli.ElectionProposal], + dict[str, dre_cli.ElectionProposal], + ], + ], + source: str, + ) -> typing.Callable[ + [], + tuple[ + dict[str, dre_cli.ElectionProposal], + dict[str, dre_cli.ElectionProposal], + ], + ]: + """Wrap a proposal retriever so it drops ignored proposal IDs. + + When the reconciler is started with ``--ignore-proposal-id``, the + wrapped retriever omits any matching proposals from the dictionaries + used to populate :class:`reconciler_state.ReconcilerState`, so the + affected versions are treated as if no proposal had been submitted. + """ + + def _drop_ignored( + d: dict[str, dre_cli.ElectionProposal], + os_kind: OsKind, + ) -> dict[str, dre_cli.ElectionProposal]: + out: dict[str, dre_cli.ElectionProposal] = {} + for version, proposal in d.items(): + if proposal["id"] in self.ignore_proposal_ids: + LOGGER.warning( + "Ignoring %s %s election proposal %s for version %s" + " (requested via --ignore-proposal-id); the" + " reconciler will treat this version as not yet" + " proposed.", + source, + os_kind, + proposal["id"], + version, + ) + continue + out[version] = proposal + return out + + def wrapped() -> tuple[ + dict[str, dre_cli.ElectionProposal], + dict[str, dre_cli.ElectionProposal], + ]: + guestos, hostos = retriever() + if not self.ignore_proposal_ids: + return guestos, hostos + return ( + _drop_ignored(guestos, GUESTOS), + _drop_ignored(hostos, HOSTOS), + ) + + return wrapped + def reconcile(self) -> None: """Reconcile the state of the network with the release index.""" logger = LOGGER.getChild("reconciler") @@ -449,7 +521,12 @@ def reconcile(self) -> None: # Fetch latest election proposals and remember their state. try: - self.state.update_state(self.dashboard.get_election_proposals_by_version) + self.state.update_state( + self._filtered_proposals_retriever( + self.dashboard.get_election_proposals_by_version, + source="dashboard", + ) + ) except Exception as e: logger.warning( "Did not succeed in retrieving proposals from" @@ -460,7 +537,12 @@ def reconcile(self) -> None: # from the DRE CLI (coming from governance canister) # after the dashboard, since these are the authoritative proposals # that have the freshest state (dashboard lags). - self.state.update_state(self.dre.get_election_proposals_by_version) + self.state.update_state( + self._filtered_proposals_retriever( + self.dre.get_election_proposals_by_version, + source="dre", + ) + ) # Preload the cache of known successfully processed releases. # We will use this information as an operation plan. @@ -950,6 +1032,24 @@ def main() -> None: default=9467, help="Set the Prometheus telemetry port to listen on. Telemetry is only served if --loop-every is greater than 0.", ) + parser.add_argument( + "--ignore-proposal-id", + action="append", + type=int, + dest="ignore_proposal_ids", + default=[], + metavar="ID", + help=( + "Ignore an NNS election proposal by ID when loading the" + " reconciler state, so the corresponding version is treated as" + " not yet proposed and a fresh proposal is submitted on the" + " next cycle. Repeat the flag to ignore multiple proposals." + " Use this to force resubmission for a version whose prior" + " election proposal was rejected or otherwise failed; the IC" + " governance canister will still reject any proposal that" + " tries to re-elect an already blessed version." + ), + ) parser.add_argument( "dotenv_file", nargs="?", @@ -1055,6 +1155,7 @@ def change_determinator_factory() -> ChangeDeterminatorProtocol: nns_url="https://ic0.app", state=state, ignore_releases=IGNORED_RELEASES, + ignore_proposal_ids=opts.ignore_proposal_ids, ic_repo=ic_repo, active_version_provider=ICPrometheus( url="https://victoria.mainnet.dfinity.network/select/0/prometheus" diff --git a/release-controller/tests/test_reconciler_integration.py b/release-controller/tests/test_reconciler_integration.py index 0e7735d12..bb99f0dac 100644 --- a/release-controller/tests/test_reconciler_integration.py +++ b/release-controller/tests/test_reconciler_integration.py @@ -6,6 +6,7 @@ import git_repo import pydantic_yaml import pytest_mock.plugin +import reconciler_state from dre_cli import ElectionProposal from public_dashboard import DashboardAPI from reconciler import Reconciler @@ -140,6 +141,123 @@ def _cdf(r: git_repo.GitRepo) -> commit_annotation.ChangeDeterminatorProtocol: return commit_annotation.LocalCommitChangeDeterminator(r) +def _ignore_filter_retriever_fixture() -> typing.Callable[ + [], + tuple[dict[str, ElectionProposal], dict[str, ElectionProposal]], +]: + """ + Build a self-contained retriever returning one GuestOS and one HostOS + election proposal with distinct IDs. This bypasses ``MockDashboard`` + (whose ``_fake_proposal`` always emits ``hostos_version_to_elect``) + so the test exercises both branches of the ignore filter. + """ + guestos_commit = "11" * 20 + hostos_commit = "22" * 20 + + def retriever() -> tuple[ + dict[str, ElectionProposal], dict[str, ElectionProposal] + ]: + guestos: dict[str, ElectionProposal] = { + guestos_commit: { + "id": 200001, + "payload": { + "replica_version_to_elect": guestos_commit, + "release_package_sha256_hex": "aa" * 32, + }, + "proposal_timestamp_seconds": 1743789296, + "proposer": 61, + "status": "REJECTED", + "summary": "...stubbed out...", + "title": "Elect new IC/GuestOS revision (test fixture)", + } + } + hostos: dict[str, ElectionProposal] = { + hostos_commit: { + "id": 200002, + "payload": { + "hostos_version_to_elect": hostos_commit, + "release_package_sha256_hex": "bb" * 32, + }, + "proposal_timestamp_seconds": 1743789296, + "proposer": 61, + "status": "REJECTED", + "summary": "...stubbed out...", + "title": "Elect new IC/HostOS revision (test fixture)", + } + } + return guestos, hostos + + return retriever + + +def test_reconciler_filtered_proposals_retriever_drops_ignored_ids( + ic_repo: git_repo.GitRepo, +) -> None: + """ + With ``ignore_proposal_ids`` set, the wrapper around the proposal + retriever must omit any matching proposal so the corresponding + version is treated by ``ReconcilerState`` as not yet proposed. + """ + d, f, n, rs, a, dre, s, rl, p, db = _defaults_for_test() + retriever = _ignore_filter_retriever_fixture() + guestos_commit, hostos_commit = "11" * 20, "22" * 20 + ignored_guestos_id = 200001 + + reconciler = Reconciler( + f, + rl, + n, + p, + "", + rs, + ic_repo, + lambda: _cdf(ic_repo), + a, + dre, + db, + s, + ignore_proposal_ids=[ignored_guestos_id], + ) + + wrapped = reconciler._filtered_proposals_retriever( + retriever, source="dashboard" + ) + guestos, hostos = wrapped() + + assert guestos == {}, guestos + assert list(hostos.keys()) == [hostos_commit], hostos + assert hostos[hostos_commit]["id"] == 200002 + + rs.update_state(wrapped) + assert isinstance( + rs.version_proposal(guestos_commit, const.GUESTOS), + reconciler_state.NoProposal, + ) + assert isinstance( + rs.version_proposal(hostos_commit, const.HOSTOS), + reconciler_state.SubmittedProposal, + ) + + +def test_reconciler_filtered_proposals_retriever_is_noop_without_ids( + ic_repo: git_repo.GitRepo, +) -> None: + """Without ``ignore_proposal_ids``, the wrapper returns the data unchanged.""" + d, f, n, rs, a, dre, s, rl, p, db = _defaults_for_test() + retriever = _ignore_filter_retriever_fixture() + reconciler = Reconciler( + f, rl, n, p, "", rs, ic_repo, lambda: _cdf(ic_repo), a, dre, db, s + ) + + direct_guestos, direct_hostos = retriever() + wrapped_guestos, wrapped_hostos = reconciler._filtered_proposals_retriever( + retriever, source="dashboard" + )() + + assert wrapped_guestos == direct_guestos + assert wrapped_hostos == direct_hostos + + def test_reconciler_reconciles_without_error_already_submitted_proposals( ic_repo: git_repo.GitRepo, mocker: pytest_mock.plugin.MockerFixture, From 09d9b76c6ec1fcb198cd9fe5567bf310f2cc6fd5 Mon Sep 17 00:00:00 2001 From: Pietro Date: Mon, 18 May 2026 11:44:02 +0000 Subject: [PATCH 2/2] fixup: drive ignored_proposals from release-index.yaml instead of CLI flag Move the operator escape hatch from a `--ignore-proposal-id` CLI flag on the release controller binary into a top-level `ignored_proposals: List[int]` field on `release-index.yaml`. The reconciler reads the list per cycle from the freshly-loaded index, so edits to `release-index.yaml` are picked up live without a controller redeploy and the configuration lives versioned alongside the releases it pertains to. - release-index-schema.json: add the new top-level array. - release-controller/release_index.py: add `ignored_proposals: List[int] = []` to `ReleaseIndex` (manual edit, mirroring the generator output). - release-controller/reconciler.py: drop the `--ignore-proposal-id` argparse flag, drop the `Reconciler.ignore_proposal_ids` constructor parameter, turn `_filtered_proposals_retriever` into a static method that takes the ignore set explicitly, and have `reconcile()` pull it from `index.root.ignored_proposals` before invoking the wrappers. - release-index.yaml: pin `ignored_proposals: [141776]` so the rejected GuestOS proposal stops blocking resubmission of its underlying version on the next reconcile cycle. - README.md: rewrite the runbook to point at the release-index.yaml workflow and drop the stale CLI-flag dev-mode section. - tests: switch the two filter wrapper tests to call the static method directly with an explicit ignore set; add a new end-to-end test asserting that an entry in `release-index.yaml`'s `ignored_proposals` propagates through the loader into a fresh proposal submission via the dryrun DRECli. --- release-controller/README.md | 42 ++++--- release-controller/reconciler.py | 63 ++++------- release-controller/release_index.py | 1 + .../tests/test_reconciler_integration.py | 106 ++++++++++++------ release-index-schema.json | 7 ++ release-index.yaml | 9 ++ 6 files changed, 138 insertions(+), 90 deletions(-) diff --git a/release-controller/README.md b/release-controller/README.md index 5c40b3017..1fa9d13ed 100644 --- a/release-controller/README.md +++ b/release-controller/README.md @@ -214,23 +214,32 @@ ic-os-version-election`), and treats every matching election proposal as will therefore prevent the reconciler from ever submitting a fresh one for the same version. -To work around this, pass one or more `--ignore-proposal-id=` flags -when starting the reconciler. Each listed proposal is filtered out of -the dashboard / governance lookups, so the affected version goes back to -the "no proposal" state and the reconciler submits a new proposal on the +To work around this, edit [`release-index.yaml`](../release-index.yaml) +and add a top-level `ignored_proposals` entry listing the NNS proposal IDs +to forget. The reconciler reads this list on every cycle (no redeploy +required) and drops the matching proposals from its dashboard / governance +lookups before they reach `ReconcilerState`, so the affected versions go +back to the "no proposal" state and a fresh proposal is submitted on the next cycle. -In production, edit `bases/apps/ic-release-controller/controller/controller.yaml` -in the `k8s` repo and append the flag(s) to the `release-controller` container's -`args` list, e.g. - ```yaml -args: [--one-line-logs, --verbose, --ignore-proposal-id=141776] +ignored_proposals: + - 141776 # GuestOS election proposal rejected; resubmit the version. +releases: + - rc_name: rc--... + versions: + - ... ``` -Caveat: the IC governance canister independently refuses to re-elect a -version that is already blessed. This flag only helps when the prior -proposal did **not** result in a blessing (typically REJECTED or FAILED). +Caveats: + +- The IC governance canister independently refuses to re-elect a version + that is already blessed. This lever only helps when the prior + proposal did **not** result in a blessing (typically REJECTED or + FAILED). +- Remove the entry from `release-index.yaml` once the replacement + proposal has been submitted. Leaving it in indefinitely silently + swallows any future state for the same proposal id. ## Development @@ -270,11 +279,10 @@ bazel run //release-controller:release-controller \ Typing errors preventing you from running it, because you are editing code and testing your changes? Add `--output_groups=-mypy` right after `bazel run`. -The optional argument `--ignore-proposal-id` (repeatable, takes an NNS proposal -ID) makes the reconciler skip the matching election proposal when it builds its -list of known proposals by version. The corresponding version is then treated -as not yet proposed and a fresh proposal is submitted on the next cycle. See -*Forcing resubmission of a previously-submitted proposal* above. +To force the reconciler to resubmit a proposal whose prior submission was +rejected or otherwise failed, edit `release-index.yaml` and add the +proposal ID to the top-level `ignored_proposals` list. See *Forcing +resubmission of a previously-submitted proposal* above. ### Running the reconciler in the container it ships diff --git a/release-controller/reconciler.py b/release-controller/reconciler.py index 86f2b8ea8..05d11859c 100644 --- a/release-controller/reconciler.py +++ b/release-controller/reconciler.py @@ -418,19 +418,8 @@ def __init__( dashboard: public_dashboard.DashboardAPI, slack_announcer: slack_announce.SlackAnnouncerProtocol, ignore_releases: list[str] | None = None, - ignore_proposal_ids: typing.Iterable[int] | None = None, ): - """Create a new reconciler. - - ``ignore_proposal_ids`` is an optional set of NNS proposal IDs that - will be dropped from the result of every election-proposal lookup - before it is loaded into the reconciler state. This allows forcing - a resubmission for a version whose prior proposal is no longer - actionable (e.g. rejected/failed). Note that the IC governance - canister still independently rejects re-election of an already - blessed version, so this only helps when the prior proposal did - not bless the version. - """ + """Create a new reconciler.""" self.forum_client = forum_client self.loader = loader self.notes_client = notes_client @@ -440,15 +429,14 @@ def __init__( self.ic_prometheus = active_version_provider self.ic_repo = ic_repo self.ignore_releases = ignore_releases or [] - self.ignore_proposal_ids: set[int] = set(ignore_proposal_ids or ()) self.dre = dre self.dashboard = dashboard self.slack_announcer = slack_announcer self.change_determinator_factory = change_determinator_factory self.local_release_state: dict[str, dict[str, dict[OsKind, VersionState]]] = {} + @staticmethod def _filtered_proposals_retriever( - self, retriever: typing.Callable[ [], tuple[ @@ -457,6 +445,7 @@ def _filtered_proposals_retriever( ], ], source: str, + ignore_proposal_ids: typing.Iterable[int], ) -> typing.Callable[ [], tuple[ @@ -466,11 +455,12 @@ def _filtered_proposals_retriever( ]: """Wrap a proposal retriever so it drops ignored proposal IDs. - When the reconciler is started with ``--ignore-proposal-id``, the - wrapped retriever omits any matching proposals from the dictionaries - used to populate :class:`reconciler_state.ReconcilerState`, so the - affected versions are treated as if no proposal had been submitted. + The wrapped retriever omits any proposal whose ``id`` is listed in + the index's top-level ``ignored_proposals`` so the affected versions + are treated by :class:`reconciler_state.ReconcilerState` as if no + proposal had been submitted. """ + ignore_set: set[int] = set(ignore_proposal_ids) def _drop_ignored( d: dict[str, dre_cli.ElectionProposal], @@ -478,11 +468,11 @@ def _drop_ignored( ) -> dict[str, dre_cli.ElectionProposal]: out: dict[str, dre_cli.ElectionProposal] = {} for version, proposal in d.items(): - if proposal["id"] in self.ignore_proposal_ids: + if proposal["id"] in ignore_set: LOGGER.warning( "Ignoring %s %s election proposal %s for version %s" - " (requested via --ignore-proposal-id); the" - " reconciler will treat this version as not yet" + " (listed in release-index.yaml ignored_proposals);" + " the reconciler will treat this version as not yet" " proposed.", source, os_kind, @@ -498,7 +488,7 @@ def wrapped() -> tuple[ dict[str, dre_cli.ElectionProposal], ]: guestos, hostos = retriever() - if not self.ignore_proposal_ids: + if not ignore_set: return guestos, hostos return ( _drop_ignored(guestos, GUESTOS), @@ -511,6 +501,14 @@ def reconcile(self) -> None: """Reconcile the state of the network with the release index.""" logger = LOGGER.getChild("reconciler") index = self.loader.index() + ignore_proposal_ids = list(index.root.ignored_proposals) + if ignore_proposal_ids: + logger.info( + "release-index.yaml requests that proposals %s be ignored" + " when populating reconciler state; the corresponding" + " versions will be treated as not yet proposed.", + ignore_proposal_ids, + ) # As a matter of principle, we will only process the very top # two releases (and all its versions). All else will be @@ -525,6 +523,7 @@ def reconcile(self) -> None: self._filtered_proposals_retriever( self.dashboard.get_election_proposals_by_version, source="dashboard", + ignore_proposal_ids=ignore_proposal_ids, ) ) except Exception as e: @@ -541,6 +540,7 @@ def reconcile(self) -> None: self._filtered_proposals_retriever( self.dre.get_election_proposals_by_version, source="dre", + ignore_proposal_ids=ignore_proposal_ids, ) ) @@ -1032,24 +1032,6 @@ def main() -> None: default=9467, help="Set the Prometheus telemetry port to listen on. Telemetry is only served if --loop-every is greater than 0.", ) - parser.add_argument( - "--ignore-proposal-id", - action="append", - type=int, - dest="ignore_proposal_ids", - default=[], - metavar="ID", - help=( - "Ignore an NNS election proposal by ID when loading the" - " reconciler state, so the corresponding version is treated as" - " not yet proposed and a fresh proposal is submitted on the" - " next cycle. Repeat the flag to ignore multiple proposals." - " Use this to force resubmission for a version whose prior" - " election proposal was rejected or otherwise failed; the IC" - " governance canister will still reject any proposal that" - " tries to re-elect an already blessed version." - ), - ) parser.add_argument( "dotenv_file", nargs="?", @@ -1155,7 +1137,6 @@ def change_determinator_factory() -> ChangeDeterminatorProtocol: nns_url="https://ic0.app", state=state, ignore_releases=IGNORED_RELEASES, - ignore_proposal_ids=opts.ignore_proposal_ids, ic_repo=ic_repo, active_version_provider=ICPrometheus( url="https://victoria.mainnet.dfinity.network/select/0/prometheus" diff --git a/release-controller/release_index.py b/release-controller/release_index.py index d309190ee..6adbb6b08 100644 --- a/release-controller/release_index.py +++ b/release-controller/release_index.py @@ -38,6 +38,7 @@ class Release(BaseModel): class ReleaseIndex(BaseModel): model_config = ConfigDict(extra="forbid") + ignored_proposals: List[int] = [] releases: List[Release] def version(self, rc_name: str, name: str) -> str: diff --git a/release-controller/tests/test_reconciler_integration.py b/release-controller/tests/test_reconciler_integration.py index bb99f0dac..5c2970a4e 100644 --- a/release-controller/tests/test_reconciler_integration.py +++ b/release-controller/tests/test_reconciler_integration.py @@ -190,38 +190,22 @@ def retriever() -> tuple[ return retriever -def test_reconciler_filtered_proposals_retriever_drops_ignored_ids( - ic_repo: git_repo.GitRepo, -) -> None: +def test_reconciler_filtered_proposals_retriever_drops_ignored_ids() -> None: """ - With ``ignore_proposal_ids`` set, the wrapper around the proposal + With ``ignored_proposals`` set, the wrapper around the proposal retriever must omit any matching proposal so the corresponding version is treated by ``ReconcilerState`` as not yet proposed. """ - d, f, n, rs, a, dre, s, rl, p, db = _defaults_for_test() retriever = _ignore_filter_retriever_fixture() guestos_commit, hostos_commit = "11" * 20, "22" * 20 ignored_guestos_id = 200001 + rs = ReconcilerState() - reconciler = Reconciler( - f, - rl, - n, - p, - "", - rs, - ic_repo, - lambda: _cdf(ic_repo), - a, - dre, - db, - s, + wrapped = Reconciler._filtered_proposals_retriever( + retriever, + source="dashboard", ignore_proposal_ids=[ignored_guestos_id], ) - - wrapped = reconciler._filtered_proposals_retriever( - retriever, source="dashboard" - ) guestos, hostos = wrapped() assert guestos == {}, guestos @@ -239,25 +223,83 @@ def test_reconciler_filtered_proposals_retriever_drops_ignored_ids( ) -def test_reconciler_filtered_proposals_retriever_is_noop_without_ids( - ic_repo: git_repo.GitRepo, -) -> None: - """Without ``ignore_proposal_ids``, the wrapper returns the data unchanged.""" - d, f, n, rs, a, dre, s, rl, p, db = _defaults_for_test() +def test_reconciler_filtered_proposals_retriever_is_noop_without_ids() -> None: + """Without ``ignored_proposals``, the wrapper returns the data unchanged.""" retriever = _ignore_filter_retriever_fixture() - reconciler = Reconciler( - f, rl, n, p, "", rs, ic_repo, lambda: _cdf(ic_repo), a, dre, db, s - ) direct_guestos, direct_hostos = retriever() - wrapped_guestos, wrapped_hostos = reconciler._filtered_proposals_retriever( - retriever, source="dashboard" + wrapped_guestos, wrapped_hostos = Reconciler._filtered_proposals_retriever( + retriever, + source="dashboard", + ignore_proposal_ids=[], )() assert wrapped_guestos == direct_guestos assert wrapped_hostos == direct_hostos +def test_reconciler_picks_up_ignored_proposals_from_release_index( + ic_repo: git_repo.GitRepo, + mocker: pytest_mock.plugin.MockerFixture, +) -> None: + """ + ``ignored_proposals`` declared at the top of ``release-index.yaml`` must + propagate through the loader into the per-cycle ignore set, so a + previously-rejected proposal stops blocking resubmission. + """ + with mocker.patch.object(ic_repo, "push_release_tags"): + d, f, n, rs, a, dre, s, rl, p, db = _defaults_for_test() + already_blocked_hostos_id = 138817 + rl = StaticReleaseLoader( + pydantic_yaml.to_yaml_str( + ReleaseIndexModel.model_validate( + { + "ignored_proposals": [already_blocked_hostos_id], + "releases": [ + _release( + "rc--2025-10-02_03-13", + {"base": "45657852c1eca6728ff313808db29b47c862ad13"}, + ), + _release( + "rc--2025-09-25_09-52", + {"base": "206b61a8616bc93d36d6a014e5cc8edf1ba256ae"}, + ), + _release( + "rc--2025-09-19_10-17", + {"base": "bf0d4d1b8cb6c0c19a5afa1454ada014847aa5c6"}, + ), + ], + } + ) + ) + ) + reconciler = Reconciler( + f, rl, n, p, "", rs, ic_repo, lambda: _cdf(ic_repo), a, dre, db, s + ) + + def fake_approved_release_notes(*args): # type: ignore + return f"Fake changelog for {args}" + + rl.proposal_summary = fake_approved_release_notes # type: ignore + reconciler.reconcile() + + # The HostOS dashboard fixture pins id 138817 to both top releases. + # After reconcile, those versions must have been seen as "not yet + # proposed" — i.e. the reconciler issued a fresh proposal for them + # (dryrun.DRECli returns a synthetic, version-derived ID), so the + # state for them no longer points at the ignored id. + for commit in ( + "45657852c1eca6728ff313808db29b47c862ad13", + "206b61a8616bc93d36d6a014e5cc8edf1ba256ae", + ): + prop = rs.version_proposal(commit, const.HOSTOS) + assert isinstance(prop, reconciler_state.SubmittedProposal), prop + assert prop.proposal_id != already_blocked_hostos_id, ( + f"Expected a fresh proposal id for HostOS commit {commit}, " + f"got the previously-blocked id {already_blocked_hostos_id}." + ) + + def test_reconciler_reconciles_without_error_already_submitted_proposals( ic_repo: git_repo.GitRepo, mocker: pytest_mock.plugin.MockerFixture, diff --git a/release-index-schema.json b/release-index-schema.json index bebceb1b7..5a605d115 100644 --- a/release-index-schema.json +++ b/release-index-schema.json @@ -6,6 +6,13 @@ "type": "object", "additionalProperties": false, "properties": { + "ignored_proposals": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "NNS election proposal IDs the reconciler must treat as if they had never been submitted, so it submits a fresh proposal for the underlying version on the next cycle. Intended to recover from rejected/failed proposals." + }, "releases": { "type": "array", "items": { diff --git a/release-index.yaml b/release-index.yaml index 7851f6c16..405c1afbc 100644 --- a/release-index.yaml +++ b/release-index.yaml @@ -7,6 +7,15 @@ # HostOS or GuestOS: # rc_name: # name: +# - Use 'ignored_proposals' (top-level) to force the reconciler to forget +# specific NNS election proposals when it builds its in-memory state, so +# the affected versions go back to the "no proposal" state and a fresh +# proposal is submitted on the next cycle. Intended to recover from +# rejected / failed proposals. Remove the entry once the replacement +# proposal has been submitted; the IC governance canister will still +# refuse to re-elect an already blessed version. +ignored_proposals: + - 141776 # GuestOS election proposal rejected; resubmit the version. releases: - rc_name: rc--2026-05-15_10-22 versions: