Add table and job for daily tracking and recording of wiki metrics (#880)

* Add WikiDailyMetric table
Add Job for the above table updates daily

Bug:T383421

* import needed classes in test

* Refactoring: created a wiki class in the metric folde and added tests for it

* added comments

* Remove test annotations

* Update CHANGELOG

* Fix typo in CHANGELOG and tests

Author: rosalieper

CHANGELOG.md

@@ -1,5 +1,11 @@
 # api
 
+
+## 10x.18.3 - 10 February 2025
+- Added new table wiki_daily_metrics
+- Added a new Wiki.php class in the metric folder for wiki related metrics
+- Added a Job scheduled to run daily and update metrics in the wiki_daily_metric table.
+
 ## 10x.18.2 - 16 December 2024
 - Update FAQ link in empty wiki notification email
 

app/Console/Kernel.php

@@ -10,11 +10,13 @@
 use App\Jobs\RequeuePendingQsBatchesJob;
 use App\Jobs\SandboxCleanupJob;
 use App\Jobs\PollForMediaWikiJobsJob;
+use App\Jobs\UpdateWikiDailyMetricJob;
 use App\Jobs\UpdateWikiSiteStatsJob;
 use App\Jobs\SendEmptyWikiNotificationsJob;
 use App\Jobs\CreateQueryserviceBatchesJob;
 use App\Jobs\FailStalledEntityImportsJob;
 use App\Jobs\UpdateQueryserviceAllowList;
+use App\Wiki;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
 
@@ -59,7 +61,10 @@ protected function schedule(Schedule $schedule): void
 
         $schedule->job(new SendEmptyWikiNotificationsJob)->dailyAt('21:00');
 
+        $schedule->job(new UpdateWikiDailyMetricJob)->dailyAt('23:00');
+
         $schedule->job(new UpdateQueryserviceAllowList)->weeklyOn(Schedule::MONDAY, '01:00');
+
     }
 
     /**

app/Jobs/UpdateWikiDailyMetricJob.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Wiki;
+use \App\Metrics\App\WikiMetrics;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+
+//This job is for the daily measurements of metrics per wikibases.
+//This is to help in understanding the purpose of active wikis.
+class UpdateWikiDailyMetricJob implements ShouldQueue
+{
+    use Dispatchable;
+    public int $timeout = 3600;
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $wikis= Wiki::withTrashed()->get();
+        foreach ( $wikis as $wiki ) {
+            (new WikiMetrics())->saveMetrics($wiki);
+        }
+    }
+}

app/Metrics/App/WikiMetrics.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Metrics\App;
+
+use App\Wiki;
+use App\WikiDailyMetrics;
+
+class WikiMetrics
+{
+
+    public function saveMetrics(Wiki $wiki): void
+    {
+        $today = now()->format('Y-m-d');
+        $oldRecord = WikiDailyMetrics::where('wiki_id', $wiki->id)->latest('date')->first();
+        $todayPageCount = $wiki->wikiSiteStats()->first()->pages ?? 0;
+        $isDeleted = (bool)$wiki->deleted_at;
+        if ($oldRecord) {
+            if ($oldRecord->is_deleted) {
+                \Log::info("WikiMetrics is deleted, no new record for WikiMetrics ID {$wiki->id}.");
+                return;
+            }
+            if ($oldRecord->pages === $todayPageCount) {
+                \Log::info("Page count unchanged for WikiMetrics ID {$wiki->id}, no new record added.");
+                return;
+            }
+        }
+        WikiDailyMetrics::create([
+            'id' => $wiki->id . '_' . date('Y-m-d'),
+            'pages' => $todayPageCount,
+            'is_deleted' => $isDeleted,
+            'date' => $today,
+            'wiki_id' => $wiki->id,
+        ]);
+        \Log::info("New metric recorded for WikiMetrics ID {$wiki->id}");
+    }
+
+}
+

app/WikiDailyMetrics.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class WikiDailyMetrics extends Model
+{
+    use HasFactory;
+
+    protected $table = 'wiki_daily_metrics';
+
+    protected $primaryKey = 'id';
+
+    public $incrementing = false; // Disable auto-increment
+
+    protected $keyType = 'string';
+
+    protected $fillable = [
+        'id',
+        'wiki_id',
+        'date',
+        'pages',
+        'is_deleted',
+    ];
+
+
+}

database/migrations/2025_01_28_144045_wiki_daily_metric.php

@@ -0,0 +1,32 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::create('wiki_daily_metrics', function (Blueprint $table) {
+            $table->string('id')->primary();
+            $table->string('wiki_id');
+            $table->integer('pages');
+            $table->boolean('is_deleted');
+            $table->date('date');
+            $table->timestamps(); // Created at & Updated at
+            $table->unique(['wiki_id', 'date']);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+            Schema::dropIfExists('wiki_daily_metrics');
+    }
+};

tests/Jobs/UpdateWikiDailyMetricJobTest.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace Tests\Jobs;
+
+use App\Jobs\UpdateWikiDailyMetricJob;
+use App\Wiki;
+use Carbon\Carbon;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Queue;
+use Tests\TestCase;
+
+class UpdateWikiDailyMetricJobTest extends TestCase
+{
+
+    use RefreshDatabase;
+
+
+    public function testDispatchJob()
+    {
+        Queue::fake();
+
+        UpdateWikiDailyMetricJob::dispatch();
+
+        Queue::assertPushed(UpdateWikiDailyMetricJob::class);
+    }
+
+
+    public function testRunJobForAllWikisIncludingDeletedWikis()
+    {
+        $activeWiki = Wiki::factory()->create([
+            'domain' => 'example.wikibase.cloud',
+        ]);
+        $deletedWiki = Wiki::factory()->create([
+            'domain' => 'deletedwiki.wikibase.cloud',
+        ]);
+        $deletedWiki->delete();
+
+        (new UpdateWikiDailyMetricJob())->handle();
+
+        $this->assertDatabaseHas('wiki_daily_metrics', [
+            'wiki_id' => $activeWiki->id,
+            'date' => Carbon::today()->toDateString()
+        ]);
+
+        $this->assertDatabaseHas('wiki_daily_metrics', [
+            'wiki_id' => $deletedWiki->id,
+            'date' => Carbon::today()->toDateString()
+        ]);
+    }
+
+}

tests/Metrics/WikiMetricsTest.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Tests\Metrics;
+
+use App\Metrics\App\WikiMetrics;
+use App\Wiki;
+use App\WikiDailyMetrics;
+use Carbon\Carbon;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Tests\TestCase;
+
+class WikiMetricsTest extends TestCase
+{
+    use RefreshDatabase;
+
+
+    public function testSuccessfullyAddRecords()
+    {
+        $wiki = Wiki::factory()->create([
+            'domain' => 'thisfake.wikibase.cloud'
+        ]);
+
+        (new WikiMetrics())->saveMetrics($wiki);
+        // Assert the metric is updated in the database
+        $this->assertDatabaseHas('wiki_daily_metrics', [
+            'date' => now()->toDateString()
+        ]);
+    }
+
+
+    public function testDoesNotAddDuplicateRecordsWithOnlyDateChange()
+    {
+        $wiki = Wiki::factory()->create([
+            'domain' => 'thisfake.wikibase.cloud'
+        ]);
+        //Insert an old metric value for a wiki
+        WikiDailyMetrics::create([
+            'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(),
+            'wiki_id' => $wiki->id,
+            'date' => Carbon::yesterday()->toDateString(),
+            'pages' => 0,
+            'is_deleted' => 0
+        ]);
+        (new WikiMetrics())->saveMetrics($wiki);
+
+        //Assert No new record was created for today
+        $this->assertDatabaseMissing('wiki_daily_metrics', [
+            'wiki_id' => $wiki->id,
+            'date' => Carbon::today()->toDateString()
+        ]);
+    }
+}
+