From a15cabf8f7529fd06061bae990dc0b71252ac58b Mon Sep 17 00:00:00 2001 From: fabrizzio-dotCMS Date: Fri, 19 Jun 2026 13:17:52 -0600 Subject: [PATCH 1/2] fix(opensearch-migration): unify phase-aware OS startup connection gate (#36244) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dotCMS had inconsistent OpenSearch connection gating at startup. The OS readiness gate was not phase-aware and the empty-DB bootstrap path created OS indices with no connection gate at all, so an unreachable/misconfigured OS surfaced as an opaque ConnectionClosedException deep inside createContentIndex ~30s into startup instead of a fast, actionable failure. Root cause found while implementing: OSIndexAPIImpl.getClusterStats() swallows all exceptions and returns a non-null empty result, so the retry loop in waitUtilIndexReady() could never observe a failure — the gate was dead code and always passed. Changes (minimal scope): - B: OSIndexAPIImpl.waitUtilIndexReady() now probes with client.info() (which propagates transport/TLS/auth failures) and its exhaustion outcome is phase-aware — Phase 3 aborts via SystemExitManager.immediateExit with an actionable message (phase + endpoints + cause); Phase 1/2 halts the migration (haltMigration → ES-only fallback) and returns false instead of killing the server. Retry count/sleep remain configurable. - C: ContentletIndexAPIImpl.bootstrapAndPointOS() runs the phase-aware OS gate before creating OS indices. As the single chokepoint for OS index creation, both startup paths (populated-DB via InitServlet and empty-DB via Task00004LoadStarter) now pass through the same gate; a shadow-phase fallback skips OS creation. - A: MainServlet startup readiness wait clarified — getESIndexAPI() already returns the phase-aware router, so it waits on the primary store for the phase; on a shadow-phase fallback to ES it waits again to gate the new primary. Adds OSIndexAPIImplWaitReadyTest covering the Phase 1/2 ES-only fallback (no exit, phase reset to 0). Phase 3 abort is verified by IT/manual QA since SystemExitManager halts the JVM. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../business/ContentletIndexAPIImpl.java | 21 ++++ .../index/opensearch/OSIndexAPIImpl.java | 109 ++++++++++++++++-- .../liferay/portal/servlet/MainServlet.java | 11 +- .../OSIndexAPIImplWaitReadyTest.java | 97 ++++++++++++++++ 4 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 dotCMS/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyTest.java diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java index e890f3a6cad2..6922661383ff 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java @@ -873,6 +873,27 @@ private void bootstrapAndPointES(final String workingName, final String liveName */ private void bootstrapAndPointOS(final String workingName, final String liveName) throws DotDataException { + + // Connection gate (issue #36244): verify OS reachability BEFORE creating OS indices. + // This is the single chokepoint for all OS index creation (fresh-install bootstrap and + // migration catchup), so both startup paths — populated-DB (InitServlet) and empty-DB + // (Task00004LoadStarter) — pass through the same phase-aware gate instead of failing + // late and opaquely with a transport exception deep inside createContentIndex. + // + // operationsOS.indexAPI() is the OS-specific IndexAPI, so the gate always probes OS + // regardless of the current read provider (in Phase 1 the read provider is ES). The + // phase-aware outcome lives in OSIndexAPIImpl.waitUtilIndexReady(): Phase 3 aborts the + // JVM with an actionable message; Phase 1/2 halts the migration (ES-only fallback) and + // returns false — in which case we must NOT create OS indices. + if (!operationsOS.indexAPI().waitUtilIndexReady()) { + Logger.warn(this.getClass(), + "Skipping OpenSearch index bootstrap (working=" + workingName + + ", live=" + liveName + "): OS was unreachable and the migration was halted" + + " (now ES-only). OS indices will be created on a later restart once OS is" + + " reachable and the migration phase is re-enabled."); + return; + } + boolean result; try { // Targeted: executed directly against this provider only. No phase fan-out here. diff --git a/dotCMS/src/main/java/com/dotcms/content/index/opensearch/OSIndexAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/index/opensearch/OSIndexAPIImpl.java index 3e53bf79267d..316ae9d7a7e9 100644 --- a/dotCMS/src/main/java/com/dotcms/content/index/opensearch/OSIndexAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/content/index/opensearch/OSIndexAPIImpl.java @@ -18,6 +18,8 @@ import com.dotmarketing.exception.DotDataException; import com.dotmarketing.util.AdminLogger; import com.dotcms.content.index.IndexConfigHelper; +import com.dotcms.content.index.IndexConfigHelper.MigrationPhase; +import com.dotmarketing.util.Config; import com.dotmarketing.util.DateUtil; import com.dotmarketing.util.Logger; import com.dotmarketing.util.UtilMethods; @@ -508,25 +510,110 @@ public ClusterStats getClusterStats() { } } + /** + * Waits for the OpenSearch cluster to become reachable, retrying up to + * {@code OS_CONNECTION_ATTEMPTS} times (falling back to {@code ES_CONNECTION_ATTEMPTS}, + * default 24) with a {@code OS_CONNECTION_RETRY_SLEEP_SECONDS} (default 5s) pause between + * attempts. Used as the OpenSearch startup connection gate (issue #36244). + * + *

Active probe — not the swallowing {@code getClusterStats()}

+ *

Connectivity is verified with {@code client.info()}, which round-trips to the cluster + * and propagates any transport / TLS / auth failure. This is deliberate: + * {@link #getClusterStats()} catches every exception and returns an empty result, so a retry + * loop built on it can never observe a failure — the gate would always pass and the real + * error would only surface much later, deep inside {@code createContentIndex} (the opaque + * late crash this gate exists to prevent).

+ * + *

Phase-aware outcome on exhaustion

+ * + * + * @return {@code true} when OS is reachable; {@code false} when OS was unreachable in a + * shadow phase and the migration was halted (ES-only fallback). In Phase 3 this method + * never returns {@code false} — it aborts the JVM instead. + */ @Override public boolean waitUtilIndexReady() { - ClusterStats stats = null; final int attempts = IndexConfigHelper.getInt(OSIndexProperty.CONNECTION_ATTEMPTS, 24); + final long sleepMs = + IndexConfigHelper.getInt(OSIndexProperty.CONNECTION_RETRY_SLEEP_SECONDS, 5) * 1000L; + Exception lastError = null; for (int i = 0; i < attempts; i++) { try { - stats = getClusterStats(); - break; + // Active probe: info() round-trips to the cluster and throws on any failure. + clientProvider.getClient().info(); + return true; } catch (Exception e) { - Logger.error(this.getClass(), - "OpenSearch Connection Attempt #" + (i + 1) + ": " + e.getMessage()); + lastError = e; + Logger.error(this.getClass(), "OpenSearch Connection Attempt #" + (i + 1) + + " of " + attempts + ": " + e.getMessage()); } - DateUtil.sleep(IndexConfigHelper.getInt(OSIndexProperty.CONNECTION_RETRY_SLEEP_SECONDS, 5) * 1000L); + DateUtil.sleep(sleepMs); } - if (stats == null) { - Logger.fatal(this.getClass(), "Cannot connect to OpenSearch, giving up."); - com.dotcms.shutdown.SystemExitManager.immediateExit(1, "OpenSearch connection failed"); - } - return true; + return handleConnectionExhausted(attempts, lastError); + } + + /** + * Phase-aware handler invoked once the OS connection retries are exhausted. + * + * @param attempts the number of attempts that were made (for the log message) + * @param lastError the last connection error observed, or {@code null} + * @return {@code false} after halting the migration in a shadow phase; never returns in Phase 3 + * (the JVM is terminated). + */ + private boolean handleConnectionExhausted(final int attempts, final Exception lastError) { + final MigrationPhase phase = MigrationPhase.current(); + final String cause = lastError != null ? lastError.getMessage() : "unknown"; + final String detail = "OpenSearch is not reachable after " + attempts + " attempt(s)." + + " phase=" + phase.name() + + ", endpoints=" + resolveEndpointsForLogging() + + ", cause=" + cause; + + if (phase.isMigrationComplete()) { + // Phase 3: OS is primary and ES is decommissioned — no fallback is possible. + Logger.fatal(this.getClass(), detail + + " — OS is the primary store in " + phase.name() + "; cannot fall back to ES." + + " Verify OS_ENDPOINTS, OS_PROTOCOL/OS_TLS_ENABLED (scheme must match the" + + " server), and credentials, then restart dotCMS."); + com.dotcms.shutdown.SystemExitManager.immediateExit(1, + "OpenSearch connection failed in PHASE_3_OPENSEARCH_ONLY"); + return false; // unreachable — immediateExit terminates the JVM + } + + // Phase 1 / 2 (shadow): ES still holds the authoritative state. Fall back to ES-only + // instead of killing the server. + Logger.error(this.getClass(), detail + + " — OS is a shadow store in " + phase.name() + "; falling back to ES-only" + + " (resetting FEATURE_FLAG_OPEN_SEARCH_PHASE to 0 via haltMigration)." + + " Fix OS connectivity and re-enable the migration phase when ready."); + IndexConfigHelper.haltMigration(); + return false; + } + + /** + * Resolves the configured OpenSearch endpoints for an actionable log message, mirroring + * {@code ConfigurableOpenSearchProvider} resolution: the explicit {@code OS_ENDPOINTS} array + * when set, otherwise a single {@code protocol://host:port} synthesised from the OS connection + * properties (with ES fallback). + */ + private static String resolveEndpointsForLogging() { + final String[] endpoints = Config.getStringArrayProperty("OS_ENDPOINTS", null); + if (endpoints != null && endpoints.length > 0) { + return Arrays.toString(endpoints); + } + final String protocol = IndexConfigHelper.getString(OSIndexProperty.PROTOCOL, "https"); + final String hostname = IndexConfigHelper.getString(OSIndexProperty.HOSTNAME, "localhost"); + final int port = IndexConfigHelper.getInt(OSIndexProperty.PORT, 9200); + return protocol + "://" + hostname + ":" + port; } diff --git a/dotCMS/src/main/java/com/liferay/portal/servlet/MainServlet.java b/dotCMS/src/main/java/com/liferay/portal/servlet/MainServlet.java index df38e7854566..01898fd7e403 100644 --- a/dotCMS/src/main/java/com/liferay/portal/servlet/MainServlet.java +++ b/dotCMS/src/main/java/com/liferay/portal/servlet/MainServlet.java @@ -109,8 +109,15 @@ public void init(ServletConfig config) throws ServletException { Logger.debug(InitServlet.class, "IOException: " + e1.getMessage(), e1); } - // Make sure elasticseach is up - APILocator.getESIndexAPI().waitUtilIndexReady(); + // Make sure the primary search store for the current migration phase is up before running + // the startup tasks (issue #36244). getESIndexAPI() returns the phase-aware router + // (IndexAPIImpl), so waitUtilIndexReady() waits on the primary store for the phase: ES in + // phases 0–1, OS in phases 2–3 — not ES-hardcoded. If OS was the primary and a shadow-phase + // (1/2) fallback to ES occurred, waitUtilIndexReady() returns false after halting the + // migration to Phase 0; wait again so the new primary (ES) is gated too. + if (!APILocator.getESIndexAPI().waitUtilIndexReady()) { + APILocator.getESIndexAPI().waitUtilIndexReady(); + } diff --git a/dotCMS/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyTest.java b/dotCMS/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyTest.java new file mode 100644 index 000000000000..50772d46146e --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyTest.java @@ -0,0 +1,97 @@ +package com.dotcms.content.index.opensearch; + +import static com.dotcms.content.index.IndexConfigHelper.MigrationPhase.FLAG_KEY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import com.dotcms.content.index.IndexConfigHelper.MigrationPhase; +import com.dotmarketing.util.Config; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.opensearch.client.opensearch.OpenSearchClient; + +/** + * Unit tests for the phase-aware OpenSearch startup connection gate + * {@link OSIndexAPIImpl#waitUtilIndexReady()} (issue #36244). + * + *

These tests exercise the retry-exhausted branch with an + * {@link OSClientProvider} whose {@code getClient()} always throws, simulating an + * unreachable / misconfigured OpenSearch cluster. The connection attempts and retry sleep are + * forced to their minimum so the gate exhausts immediately without a real cluster.

+ * + *

What is and isn't covered here

+ * + */ +public class OSIndexAPIImplWaitReadyTest { + + /** {@link OSClientProvider} that always fails to produce a client (unreachable cluster). */ + private static final class FailingClientProvider implements OSClientProvider { + @Override + public OpenSearchClient getClient() { + throw new RuntimeException("simulated OpenSearch connection failure (test)"); + } + } + + @Before + public void fastRetries() { + // Exhaust the gate immediately: a single attempt with no sleep between retries. + Config.setProperty(OSIndexProperty.CONNECTION_ATTEMPTS.osKey, "1"); + Config.setProperty(OSIndexProperty.CONNECTION_RETRY_SLEEP_SECONDS.osKey, "0"); + } + + @After + public void clearProps() { + Config.setProperty(FLAG_KEY, null); + Config.setProperty(OSIndexProperty.CONNECTION_ATTEMPTS.osKey, null); + Config.setProperty(OSIndexProperty.CONNECTION_RETRY_SLEEP_SECONDS.osKey, null); + } + + private static void setPhase(final int ordinal) { + Config.setProperty(FLAG_KEY, String.valueOf(ordinal)); + } + + /** + * Given : Phase 1 (dual-write, ES reads) and an unreachable OpenSearch cluster. + * When : waitUtilIndexReady() exhausts its retries. + * Then : the server is NOT killed — the gate returns {@code false} and the migration is + * halted (FEATURE_FLAG_OPEN_SEARCH_PHASE reset to 0), so dotCMS falls back to ES-only. + */ + @Test + public void test_phase1_osUnreachable_fallsBackToEs_noExit() { + setPhase(1); + final OSIndexAPIImpl api = new OSIndexAPIImpl(new FailingClientProvider()); + + final boolean ready = api.waitUtilIndexReady(); + + assertFalse("Phase 1 must NOT abort: the gate returns false (ES-only fallback)", ready); + assertEquals("Migration phase must be reset to PHASE_0 after the ES fallback", + MigrationPhase.PHASE_0_MIGRATION_NOT_STARTED, MigrationPhase.current()); + } + + /** + * Given : Phase 2 (dual-write, OS reads) and an unreachable OpenSearch cluster. + * When : waitUtilIndexReady() exhausts its retries. + * Then : same shadow-phase behavior as Phase 1 — fall back to ES (halt), return {@code false}, + * never abort. ES still holds the authoritative state in Phase 2. + */ + @Test + public void test_phase2_osUnreachable_fallsBackToEs_noExit() { + setPhase(2); + final OSIndexAPIImpl api = new OSIndexAPIImpl(new FailingClientProvider()); + + final boolean ready = api.waitUtilIndexReady(); + + assertFalse("Phase 2 must NOT abort: the gate returns false (ES-only fallback)", ready); + assertEquals("Migration phase must be reset to PHASE_0 after the ES fallback", + MigrationPhase.PHASE_0_MIGRATION_NOT_STARTED, MigrationPhase.current()); + } +} From 949d49b94431b5be40ef6b0520d0bbaccc4f63ce Mon Sep 17 00:00:00 2001 From: fabrizzio-dotCMS Date: Mon, 22 Jun 2026 14:16:51 -0600 Subject: [PATCH 2/2] test(opensearch): make OS startup gate test an IT in the OpenSearch suite Move OSIndexAPIImplWaitReadyTest (a misplaced unit test under dotCMS/src/test) into the integration module as OSIndexAPIImplWaitReadyIT and register it in OpenSearchUpgradeSuite so CI actually runs it under the opensearch-upgrade profile. Tests for the OS-touching paths belong in dotcms-integration with the IT suffix, not as plain unit tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../com/dotcms/OpenSearchUpgradeSuite.java | 2 ++ .../opensearch/OSIndexAPIImplWaitReadyIT.java | 32 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) rename dotCMS/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyTest.java => dotcms-integration/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyIT.java (80%) diff --git a/dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java b/dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java index 28a7d54051ae..722e82865a7c 100644 --- a/dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java +++ b/dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java @@ -8,6 +8,7 @@ import com.dotcms.content.index.opensearch.OSMappingAPIImplIntegrationTest; import com.dotcms.content.index.VersionedIndicesAPITest; import com.dotcms.content.index.opensearch.OSIndexAPIImplIntegrationTest; +import com.dotcms.content.index.opensearch.OSIndexAPIImplWaitReadyIT; import com.dotcms.content.index.opensearch.OSClientConfigTest; import com.dotcms.content.index.opensearch.OSClientProviderIntegrationTest; import com.dotcms.content.index.opensearch.OSSearchAPIImplIntegrationTest; @@ -36,6 +37,7 @@ @SuiteClasses({ VersionedIndicesAPITest.class, OSIndexAPIImplIntegrationTest.class, + OSIndexAPIImplWaitReadyIT.class, OSMappingAPIImplIntegrationTest.class, ContentletIndexOperationsOSIntegrationTest.class, OSCreateContentIndexIntegrationTest.class, diff --git a/dotCMS/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyTest.java b/dotcms-integration/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyIT.java similarity index 80% rename from dotCMS/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyTest.java rename to dotcms-integration/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyIT.java index 50772d46146e..7ee87c2b1cf6 100644 --- a/dotCMS/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplWaitReadyIT.java @@ -5,14 +5,16 @@ import static org.junit.Assert.assertFalse; import com.dotcms.content.index.IndexConfigHelper.MigrationPhase; +import com.dotcms.util.IntegrationTestInitService; import com.dotmarketing.util.Config; import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.opensearch.client.opensearch.OpenSearchClient; /** - * Unit tests for the phase-aware OpenSearch startup connection gate + * Integration tests for the phase-aware OpenSearch startup connection gate * {@link OSIndexAPIImpl#waitUtilIndexReady()} (issue #36244). * *

These tests exercise the retry-exhausted branch with an @@ -20,18 +22,29 @@ * unreachable / misconfigured OpenSearch cluster. The connection attempts and retry sleep are * forced to their minimum so the gate exhausts immediately without a real cluster.

* + *

Registered in {@link com.dotcms.OpenSearchUpgradeSuite}. Run with: + * 

+ *   ./mvnw verify -pl :dotcms-integration \
+ *       -Dcoreit.test.skip=false \
+ *       -Dopensearch.upgrade.test=true
+ *
+ *

+ * *

What is and isn't covered here

*
    *
  • Phase 1 / 2 (shadow) — covered: the gate must NOT kill the JVM; it halts - * the migration (phase reset to 0) and returns {@code false}.
    • - *
  • Phase 3 (OS primary) — intentionally NOT unit-tested: the gate aborts via + * the migration (phase reset to 0) and returns {@code false}. These two cases use the failing + * client provider, so they do not need the live OpenSearch container.
    • + *
  • Phase 3 (OS primary) — intentionally NOT tested here: the gate aborts via * {@code SystemExitManager.immediateExit}, which calls {@code Runtime.halt()} and would kill - * the test JVM. The Phase 3 abort is verified by integration / manual QA instead.
    • - *
  • Success path — covered by the OpenSearch integration suite (needs a real - * cluster to satisfy {@code client.info()}).
    • + * the test JVM. The Phase 3 abort is verified by manual QA instead. + *
  • Success path — covered by the other OpenSearch integration tests that run + * against the live cluster (a reachable {@code client.info()}).
    • *
+ * + * @author Fabrizzio Araya */ -public class OSIndexAPIImplWaitReadyTest { +public class OSIndexAPIImplWaitReadyIT { /** {@link OSClientProvider} that always fails to produce a client (unreachable cluster). */ private static final class FailingClientProvider implements OSClientProvider { @@ -41,6 +54,11 @@ public OpenSearchClient getClient() { } } + @BeforeClass + public static void prepare() throws Exception { + IntegrationTestInitService.getInstance().init(); + } + @Before public void fastRetries() { // Exhaust the gate immediately: a single attempt with no sleep between retries.