-
Notifications
You must be signed in to change notification settings - Fork 87
Skip syncing number_of_replicas when follower has auto_expand_replica… #1662
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -166,6 +166,32 @@ open class IndexReplicationTask(id: Long, type: String, action: String, descript | |
| const val AUTOPAUSED_REASON_PREFIX = "AutoPaused: " | ||
| const val TASK_CANCELLATION_REASON = AUTOPAUSED_REASON_PREFIX + "Index replication task was cancelled by user" | ||
|
|
||
| /** | ||
| * Returns true if the given settings has `index.auto_expand_replicas` set to anything other | ||
| * than "false" (the OpenSearch sentinel that disables auto-expand). | ||
| * | ||
| * When this returns true, CCR must NOT sync `index.number_of_replicas` from the leader to the | ||
| * follower: the follower's local OpenSearch is responsible for computing `number_of_replicas` | ||
| * from its own data-node count. See issue #1661. | ||
| */ | ||
| internal fun isAutoExpandReplicasActive(settings: Settings): Boolean { | ||
| val value = settings.get(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.key) ?: return false | ||
| // OpenSearch stores the disabled sentinel as the literal string "false". | ||
| return !value.equals("false", ignoreCase = true) | ||
| } | ||
|
|
||
| /** | ||
| * Returns a copy of the given settings with `index.number_of_replicas` removed. Used to | ||
| * prevent CCR from syncing `number_of_replicas` to a follower whose auto-expand is active | ||
| * (see issue #1661). | ||
| */ | ||
| internal fun filterOutNumberOfReplicas(settings: Settings): Settings { | ||
| if (settings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS) == null) { | ||
| return settings | ||
| } | ||
| return settings.filter { k: String -> k != IndexMetadata.SETTING_NUMBER_OF_REPLICAS } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| //only for testing | ||
|
|
@@ -523,7 +549,22 @@ open class IndexReplicationTask(id: Long, type: String, action: String, descript | |
| } | ||
| } | ||
| } | ||
| val desiredSettings = desiredSettingsBuilder.build() | ||
| var desiredSettings = desiredSettingsBuilder.build() | ||
|
|
||
| // Issue #1661: When the follower has `index.auto_expand_replicas` active (any value other | ||
| // than "false"), the follower cluster independently manages `index.number_of_replicas` based | ||
| // on its local node count. Syncing `number_of_replicas` from the leader in that case destroys | ||
| // STARTED replica shards and triggers perpetual YELLOW state as `adaptAutoExpandReplicas()` | ||
| // re-creates them only for the next sync to destroy them again. | ||
| // | ||
| // Note: any explicit `number_of_replicas` the user set in the replication metadata overrides | ||
| // is also suppressed here. This is intentional \u2014 `auto_expand_replicas` and a fixed | ||
| // `number_of_replicas` are contradictory settings, and the follower's active auto-expand | ||
| // takes precedence until it is disabled. | ||
| if (isAutoExpandReplicasActive(followerSettings)) { | ||
| desiredSettings = filterOutNumberOfReplicas(desiredSettings) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we add this logic above at line 543 where we are already iterating over the settings ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we can add it in line 535 when the desiredSettingsBuilder is being initialised. have done the same fix here in this pr #1664 |
||
| followerSettings = filterOutNumberOfReplicas(followerSettings) | ||
| } | ||
|
|
||
| val changedSettingsBuilder = Settings.builder() | ||
| for(key in desiredSettings.keySet()) { | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Wouldnt it be simpler to check
value.equals("true)