Add support for multiple Elasticsearch hosts (#1052)

Bug: [T416157](https://phabricator.wikimedia.org/T416157)

Author: AndrewKostka

app/Http/Controllers/WikiController.php

@@ -5,7 +5,6 @@
 use App\Helper\DomainHelper;
 use App\Helper\DomainValidator;
 use App\Helper\ProfileValidator;
-use App\Jobs\CirrusSearch\ElasticSearchIndexInit;
 use App\Jobs\ElasticSearchAliasInit;
 use App\Jobs\KubernetesIngressCreate;
 use App\Jobs\MediawikiInit;
@@ -34,14 +33,12 @@ public function __construct(DomainValidator $domainValidator, ProfileValidator $
     }
 
     public function create(Request $request): \Illuminate\Http\Response {
-        $clusterWithoutSharedIndex = Config::get('wbstack.elasticsearch_cluster_without_shared_index');
-        $sharedIndexHost = Config::get('wbstack.elasticsearch_shared_index_host');
         $sharedIndexPrefix = Config::get('wbstack.elasticsearch_shared_index_prefix');
+        $esHosts = Config::get('wbstack.elasticsearch_hosts');
+        $isSearchConfigValid = $esHosts && $sharedIndexPrefix;
 
-        if (Config::get('wbstack.elasticsearch_enabled_by_default')) {
-            if (!$clusterWithoutSharedIndex && !($sharedIndexHost && $sharedIndexPrefix)) {
-                abort(503, 'Search enabled, but its configuration is invalid');
-            }
+        if (Config::get('wbstack.elasticsearch_enabled_by_default') && !$isSearchConfigValid) {
+            abort(503, 'Search enabled, but its configuration is invalid');
         }
         $user = $request->user();
 
@@ -171,12 +168,9 @@ public function create(Request $request): \Illuminate\Http\Response {
         }
 
         // dispatch elasticsearch init job to enable the feature
-        if (Config::get('wbstack.elasticsearch_enabled_by_default')) {
-            if ($clusterWithoutSharedIndex) {
-                dispatch(new ElasticSearchIndexInit($wiki->id, $clusterWithoutSharedIndex));
-            }
-            if ($sharedIndexHost && $sharedIndexPrefix) {
-                dispatch(new ElasticSearchAliasInit($wiki->id));
+        if (Config::get('wbstack.elasticsearch_enabled_by_default') && $isSearchConfigValid) {
+            foreach ($esHosts as $esHost) {
+                dispatch(new ElasticSearchAliasInit($wiki->id, $esHost));
             }
         }
 

app/Jobs/CirrusSearch/ElasticSearchIndexInit.php

@@ -9,15 +9,18 @@
 class ElasticSearchAliasInit extends Job {
     private $wikiId;
 
+    private $esHost;
+
     private $dbName;
 
     private $sharedPrefix;
 
     /**
      * @param  string  $dbName
      */
-    public function __construct(int $wikiId, ?string $sharedPrefix = null) {
+    public function __construct(int $wikiId, string $esHost, ?string $sharedPrefix = null) {
         $this->wikiId = $wikiId;
+        $this->esHost = $esHost;
         $this->sharedPrefix = $sharedPrefix ?? getenv('ELASTICSEARCH_SHARED_INDEX_PREFIX');
     }
 
@@ -68,7 +71,7 @@ public function handle(HttpRequest $request) {
         }
 
         $request->setOptions([
-            CURLOPT_URL => getenv('ELASTICSEARCH_SHARED_INDEX_HOST') . '/_aliases',
+            CURLOPT_URL => $this->esHost . '/_aliases',
             CURLOPT_RETURNTRANSFER => true,
             CURLOPT_TIMEOUT => 60 * 15,
             CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,

app/Jobs/ElasticSearchAliasInit.php

@@ -21,8 +21,6 @@
 
     'elasticsearch_hosts' => array_filter(explode(',', env('ELASTICSEARCH_HOST', ''))),
     'elasticsearch_enabled_by_default' => env('WBSTACK_ELASTICSEARCH_ENABLED_BY_DEFAULT', false),
-    'elasticsearch_cluster_without_shared_index' => env('ELASTICSEARCH_CLUSTER_WITHOUT_SHARED_INDEX', null),
-    'elasticsearch_shared_index_host' => env('ELASTICSEARCH_SHARED_INDEX_HOST', null),
     'elasticsearch_shared_index_prefix' => env('ELASTICSEARCH_SHARED_INDEX_PREFIX', null),
 
     'signup_throttling_limit' => env('WBSTACK_SIGNUP_THROTTLING_LIMIT', ''),

config/wbstack.php

@@ -12,13 +12,16 @@
 class ElasticSearchAliasInitTest extends TestCase {
     private $wikiId;
 
+    private $esHost;
+
     private $prefix;
 
     private $dbName;
 
     protected function setUp(): void {
         parent::setUp();
         $this->wikiId = Wiki::factory()->create()->id;
+        $this->esHost = 'elasticsearch-1.localhost';
         $this->dbName = WikiDb::factory()->create(['wiki_id' => $this->wikiId])->name;
         $this->prefix = 'testing_1';
         putenv('ELASTICSEARCH_SHARED_INDEX_PREFIX');
@@ -48,7 +51,7 @@ private function getMockRequest() {
             ->method('setOptions')
             ->with(
                 [
-                    CURLOPT_URL => getenv('ELASTICSEARCH_SHARED_INDEX_HOST') . '/_aliases',
+                    CURLOPT_URL => $this->esHost . '/_aliases',
                     CURLOPT_RETURNTRANSFER => true,
                     CURLOPT_TIMEOUT => 60 * 15,
                     CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
@@ -80,7 +83,7 @@ public function testSuccess() {
             ->method('fail')
             ->withAnyParameters();
 
-        $job = new ElasticSearchAliasInit($this->wikiId, $this->prefix);
+        $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost, $this->prefix);
         $job->setJob($mockJob);
         $job->handle($request);
     }
@@ -95,7 +98,7 @@ public function testFailure() {
             ->method('fail')
             ->with(new \RuntimeException("Updating Elasticsearch aliases failed for $this->wikiId with {\"acknowledged\":false}"));
 
-        $job = new ElasticSearchAliasInit($this->wikiId, $this->prefix);
+        $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost, $this->prefix);
         $job->setJob($mockJob);
         $job->handle($request);
     }
@@ -109,7 +112,7 @@ public function testMissingDatabaseFailure() {
             ->method('fail')
             ->with(new \RuntimeException("Failed to get database name for $this->wikiId"));
 
-        $job = new ElasticSearchAliasInit($this->wikiId, $this->prefix);
+        $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost, $this->prefix);
         $job->setJob($mockJob);
         $job->handle($request);
     }
@@ -127,7 +130,7 @@ public function testSuccessWithPrefixEnv() {
             ->method('fail')
             ->withAnyParameters();
 
-        $job = new ElasticSearchAliasInit($this->wikiId);
+        $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost);
         $job->setJob($mockJob);
         $job->handle($request);
     }
@@ -140,7 +143,7 @@ public function testMissingPrefixFailure() {
             ->method('fail')
             ->with(new \RuntimeException("Missing shared index prefix for $this->wikiId"));
 
-        $job = new ElasticSearchAliasInit($this->wikiId);
+        $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost);
         $job->setJob($mockJob);
         $job->handle($request);
     }