Use MediaWikiHostResolver in all Jobs and Commands (#988)

* remove obsolete apiUrl assignment

* refactor MediawikiInit Job

* refactor MediawikiSandboxLoadData

* refactor CirrusSearchJob

* refactor SiteStatsUpdateJob

* Revert "remove obsolete apiUrl assignment"

This reverts commit f8635db.

* refactor PageFetcher and its uses

* refactor PollForMediaWikiJobsJob

* refactor WikiEntityImportJob

* refactor UpdateWikiSiteStatsJob

* fix copypasta

* add Wiki::getBackendHost

* use Wiki::getBackendHost in UpdateWikiSiteStatsJob

* refactor CirrusSearchJob

* update SiteStatsUpdateJob

* attempt to fix wikiEntityImportJob

* remove Wiki::getBackendHost()

* typo

* add function keyword

* add ;

* remove ;

* fix MediawikiInitTest

* CirrusSearchJob fix resolver access

* wikiEntityImportJob: fix mwHostResolver access in static method

* refactor class creation

* add mock resoler to ElasticSearchIndexInitTest

* mock resolver in ForceSearchIndexTest

* mock resolver in QueueSearchIndexBatchesTest

* fix PollForMediaWikiJobsJob

* fix SiteStatsUpdateJobTest

* streamline tests

* streamline naming

* fix PageFetcher

* cleanup

* fix UpdateWikiSiteStatsJob

* fix WikiEntityImportJob

* undo refactoring

* fix PlatformStatsSummaryJob

* fix RebuildQueryserviceData

* run pint fix

* remove changes from #994

* linting

* clean up usages of PLATFORM_MW_BACKEND_HOST in tests

* fix rebase

* fix rebase part 2

Author: deer-wmde

app/Console/Commands/RebuildQueryserviceData.php

@@ -5,6 +5,7 @@
 use App\Constants\MediawikiNamespace;
 use App\Jobs\SpawnQueryserviceUpdaterJob;
 use App\QueryserviceNamespace;
+use App\Services\MediaWikiHostResolver;
 use App\Traits;
 use App\Wiki;
 use App\WikiSetting;
@@ -25,11 +26,10 @@ class RebuildQueryserviceData extends Command {
 
     protected string $queueName;
 
-    public function handle() {
+    public function handle(MediaWikiHostResolver $mwHostResolver) {
         $this->chunkSize = intval($this->option('chunkSize'));
         $this->sparqlUrlFormat = $this->option('sparqlUrlFormat');
         $this->queueName = $this->option('queueName');
-        $this->apiUrl = getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php';
 
         $wikiDomains = $this->option('domain');
         $exitCode = 0;
@@ -42,6 +42,8 @@ public function handle() {
         $skippedWikis = 0;
         $processedWikis = 0;
         foreach ($wikis as $wiki) {
+            $this->apiUrl = $mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php'; // used in PageFetcher::fetchPagesInNamespace
+
             try {
                 $entities = $this->getEntitiesForWiki($wiki);
                 $sparqlUrl = $this->getSparqlUrl($wiki);

app/Jobs/CirrusSearch/CirrusSearchJob.php

@@ -4,6 +4,7 @@
 
 use App\Http\Curl\HttpRequest;
 use App\Jobs\Job;
+use App\Services\MediaWikiHostResolver;
 use App\Wiki;
 use App\WikiSetting;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
@@ -41,7 +42,7 @@ public function wikiId(): int {
     /**
      * @return void
      */
-    public function handle(HttpRequest $request) {
+    public function handle(HttpRequest $request, MediaWikiHostResolver $mwHostResolver) {
         $this->wiki = Wiki::whereId($this->wikiId)->with('settings')->with('wikiDb')->first();
 
         // job got triggered but no wiki
@@ -69,7 +70,7 @@ public function handle(HttpRequest $request) {
 
         $request->setOptions(
             [
-                CURLOPT_URL => getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php?action=' . $this->apiModule() . $this->getQueryParams(),
+                CURLOPT_URL => $mwHostResolver->getBackendHostForDomain($this->wiki->domain) . '/w/api.php?action=' . $this->apiModule() . $this->getQueryParams(),
                 CURLOPT_RETURNTRANSFER => true,
                 CURLOPT_ENCODING => '',
                 CURLOPT_TIMEOUT => $this->getRequestTimeout(),

app/Jobs/MediawikiInit.php

@@ -3,6 +3,7 @@
 namespace App\Jobs;
 
 use App\Http\Curl\HttpRequest;
+use App\Services\MediaWikiHostResolver;
 
 class MediawikiInit extends Job {
     private $wikiDomain;
@@ -23,14 +24,14 @@ public function __construct($wikiDomain, $username, $email) {
     /**
      * @return void
      */
-    public function handle(HttpRequest $request) {
+    public function handle(HttpRequest $request, MediaWikiHostResolver $mwHostResolver) {
         $data = [
             'username' => $this->username,
             'email' => $this->email,
         ];
 
         $request->setOptions([
-            CURLOPT_URL => getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php?action=wbstackInit&format=json',
+            CURLOPT_URL => $mwHostResolver->getBackendHostForDomain($this->wikiDomain) . '/w/api.php?action=wbstackInit&format=json',
             CURLOPT_RETURNTRANSFER => true,
             CURLOPT_ENCODING => '',
             CURLOPT_TIMEOUT => 60,

app/Jobs/MediawikiSandboxLoadData.php

@@ -2,6 +2,8 @@
 
 namespace App\Jobs;
 
+use App\Services\MediaWikiHostResolver;
+
 class MediawikiSandboxLoadData extends Job {
     private $wikiDomain;
 
@@ -18,14 +20,14 @@ public function __construct($wikiDomain, $dataSet) {
     /**
      * @return void
      */
-    public function handle() {
+    public function handle(MediaWikiHostResolver $mwHostResolver) {
         $data = [
             'dataSet' => $this->dataSet,
         ];
 
         $curl = curl_init();
         curl_setopt_array($curl, [
-            CURLOPT_URL => getenv('PLATFORM_MW_BACKEND_HOST') . '/w/rest.php/wikibase-exampledata/v0/load',
+            CURLOPT_URL => $mwHostResolver->getBackendHostForDomain($this->wikiDomain) . '/w/rest.php/wikibase-exampledata/v0/load',
             CURLOPT_RETURNTRANSFER => true,
             CURLOPT_ENCODING => '',
             CURLOPT_TIMEOUT => 10 * 60, // TODO Long 10 mins (probably shouldn't keep the request open...)

app/Jobs/PlatformStatsSummaryJob.php

@@ -4,6 +4,7 @@
 
 use App\Constants\MediawikiNamespace;
 use App\Helper\MWTimestampHelper;
+use App\Services\MediaWikiHostResolver;
 use App\Traits;
 use App\User;
 use App\Wiki;
@@ -39,10 +40,11 @@ class PlatformStatsSummaryJob extends Job {
 
     private $platformSummaryStatsVersion = 'v1';
 
+    private MediaWikiHostResolver $mwHostResolver;
+
     public function __construct() {
         $this->inactiveThreshold = Config::get('wbstack.platform_summary_inactive_threshold');
         $this->creationRateRanges = Config::get('wbstack.platform_summary_creation_rate_ranges');
-        $this->apiUrl = getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php';
     }
 
     private function isNullOrEmpty($value): bool {
@@ -77,6 +79,7 @@ public function prepareStats(array $allStats, $wikis): array {
         $currentTime = CarbonImmutable::now();
 
         foreach ($wikis as $wiki) {
+            $this->apiUrl = $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php'; // used in PageFetcher::fetchPagesInNamespace
 
             if (!is_null($wiki->deleted_at)) {
                 $deletedWikis[] = $wiki;
@@ -164,7 +167,8 @@ public function prepareStats(array $allStats, $wikis): array {
         ];
     }
 
-    public function handle(DatabaseManager $manager): void {
+    public function handle(DatabaseManager $manager, MediaWikiHostResolver $mwHostResolver): void {
+        $this->mwHostResolver = $mwHostResolver;
         $wikis = Wiki::withTrashed()->with('wikidb')->get();
 
         $conn = $manager->connection('mysql');

app/Jobs/PollForMediaWikiJobsJob.php

@@ -2,16 +2,20 @@
 
 namespace App\Jobs;
 
+use App\Services\MediaWikiHostResolver;
 use App\Wiki;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Log;
 
 class PollForMediaWikiJobsJob extends Job implements ShouldBeUnique, ShouldQueue {
+    private MediaWikiHostResolver $mwHostResolver;
+
     public $timeout = 1800;
 
-    public function handle(): void {
+    public function handle(MediaWikiHostResolver $mwHostResolver): void {
+        $this->mwHostResolver = $mwHostResolver;
         $allWikiDomains = Wiki::all()->pluck('domain');
         foreach ($allWikiDomains as $wikiDomain) {
             if ($this->hasPendingJobs($wikiDomain)) {
@@ -24,7 +28,7 @@ private function hasPendingJobs(string $wikiDomain): bool {
         $response = Http::withHeaders([
             'host' => $wikiDomain,
         ])->get(
-            getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json'
+            $this->mwHostResolver->getBackendHostForDomain($wikiDomain) . '/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json'
         );
 
         if ($response->failed()) {

app/Jobs/SiteStatsUpdateJob.php

@@ -3,6 +3,7 @@
 namespace App\Jobs;
 
 use App\Http\Curl\HttpRequest;
+use App\Services\MediaWikiHostResolver;
 use App\Wiki;
 use Illuminate\Bus\Batchable;
 use Illuminate\Support\Facades\Log;
@@ -22,18 +23,18 @@ public function __construct($wiki_id) {
         $this->wiki_id = $wiki_id;
     }
 
-    public function handle(HttpRequest $request): void {
+    public function handle(HttpRequest $request, MediaWikiHostResolver $mwHostResolver): void {
         $timeStart = microtime(true);
 
         $wiki = Wiki::where('id', $this->wiki_id)->first();
         if (!$wiki) {
             $this->fail(new \RuntimeException(" Could not find wiki with id: $this->wiki_id"));
         }
 
-        Log::info(__METHOD__ . ": Updating stats for or $wiki->domain");
+        Log::info(__METHOD__ . ": Updating stats for $wiki->domain");
 
         $request->setOptions([
-            CURLOPT_URL => getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php?action=wbstackSiteStatsUpdate&format=json',
+            CURLOPT_URL => $mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php?action=wbstackSiteStatsUpdate&format=json',
             CURLOPT_RETURNTRANSFER => true,
             CURLOPT_ENCODING => '',
             CURLOPT_TIMEOUT => 60 * 5,

app/Jobs/UpdateWikiSiteStatsJob.php

@@ -2,6 +2,7 @@
 
 namespace App\Jobs;
 
+use App\Services\MediaWikiHostResolver;
 use App\Wiki;
 use App\WikiSiteStats;
 use Carbon\Carbon;
@@ -17,7 +18,11 @@ class UpdateWikiSiteStatsJob extends Job implements ShouldBeUnique {
 
     public $timeout = 3600;
 
-    public function handle(): void {
+    private MediaWikiHostResolver $mwHostResolver;
+
+    public function handle(MediaWikiHostResolver $mwHostResolver): void {
+        $this->mwHostResolver = $mwHostResolver;
+
         $allWikis = Wiki::all();
         foreach ($allWikis as $wiki) {
             try {
@@ -54,7 +59,7 @@ private function updateSiteStats(Wiki $wiki): void {
         $response = Http::withHeaders([
             'host' => $wiki->getAttribute('domain'),
         ])->get(
-            getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json'
+            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json'
         );
 
         if ($response->failed()) {
@@ -76,7 +81,7 @@ private function updateSiteStats(Wiki $wiki): void {
 
     private function getFirstEditedDate(Wiki $wiki): ?CarbonInterface {
         $allRevisions = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get(
-            getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php',
+            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php',
             [
                 'action' => 'query',
                 'format' => 'json',
@@ -94,7 +99,7 @@ private function getFirstEditedDate(Wiki $wiki): ?CarbonInterface {
         }
 
         $revisionInfo = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get(
-            getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php',
+            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php',
             [
                 'action' => 'query',
                 'format' => 'json',
@@ -114,7 +119,7 @@ private function getFirstEditedDate(Wiki $wiki): ?CarbonInterface {
 
     private function getLastEditedDate(Wiki $wiki): ?CarbonInterface {
         $allRevisions = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get(
-            getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php',
+            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php',
             [
                 'action' => 'query',
                 'format' => 'json',
@@ -132,7 +137,7 @@ private function getLastEditedDate(Wiki $wiki): ?CarbonInterface {
         }
 
         $revisionInfo = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get(
-            getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php',
+            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php',
             [
                 'action' => 'query',
                 'format' => 'json',

app/Jobs/WikiEntityImportJob.php

@@ -2,6 +2,7 @@
 
 namespace App\Jobs;
 
+use App\Services\MediaWikiHostResolver;
 use App\Wiki;
 use App\WikiEntityImport;
 use App\WikiEntityImportStatus;
@@ -27,20 +28,20 @@ public function __construct(
         public int $wikiId,
         public string $sourceWikiUrl,
         public array $entityIds,
-        public int $importId,
+        public int $importId
     ) {}
 
     private string $targetWikiUrl;
 
     /**
      * Execute the job.
      */
-    public function handle(Client $kubernetesClient): void {
+    public function handle(Client $kubernetesClient, MediaWikiHostResolver $mwHostResolver): void {
         $import = null;
         try {
             $wiki = Wiki::findOrFail($this->wikiId);
             $import = WikiEntityImport::findOrFail($this->importId);
-            $creds = $this->acquireCredentials($wiki->domain);
+            $creds = $this->acquireCredentials($wiki->domain, $mwHostResolver);
 
             $this->targetWikiUrl = $this->domainToOrigin($wiki->domain);
 
@@ -77,9 +78,9 @@ private static function domainToOrigin(string $domain): string {
             : 'https://' . $domain;
     }
 
-    private static function acquireCredentials(string $wikiDomain): OAuthCredentials {
+    private static function acquireCredentials(string $wikiDomain, MediaWikiHostResolver $mwHostResolver): OAuthCredentials {
         $response = Http::withHeaders(['host' => $wikiDomain])->asForm()->post(
-            getenv('PLATFORM_MW_BACKEND_HOST') . '/w/api.php?action=wbstackPlatformOauthGet&format=json',
+            $mwHostResolver->getBackendHostForDomain($wikiDomain) . '/w/api.php?action=wbstackPlatformOauthGet&format=json',
             [
                 'consumerName' => 'WikiEntityImportJob',
                 'ownerOnly' => '1',

phpunit.xml

@@ -9,7 +9,6 @@
     <env name="APP_ENV" value="testing"/>
     <env name="CACHE_DRIVER" value="array"/>
     <env name="QUEUE_CONNECTION" value="sync"/>
-    <env name="PLATFORM_MW_BACKEND_HOST" value="mediawiki-139-app-backend.default.svc.cluster.default"/>
     <env name="WBSTACK_SUMMARY_CREATION_RATE_RANGES" value="PT24H,P30D"/>
   </php>
   <source>