From 10a08c7ec7f32981f934b21b2ace7679409b5455 Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Wed, 13 Aug 2025 08:13:26 +0200 Subject: [PATCH 1/9] Add migrations for new columns in the wiiki_daily_metrics table --- ..._item_count_to_wiki_daily_merics_table.php | 28 +++++++++++++++++++ ...rties_count_to_wiki_daily_merics_table.php | 28 +++++++++++++++++++ ...xemes_count_to_wiki_daily_merics_table.php | 28 +++++++++++++++++++ ...hemas_count_to_wiki_daily_merics_table.php | 28 +++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php create mode 100644 database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php create mode 100644 database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php create mode 100644 database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php diff --git a/database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php b/database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php new file mode 100644 index 000000000..2beb8c96b --- /dev/null +++ b/database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php @@ -0,0 +1,28 @@ +integer('item_count')->nullable()->default(null); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metrics', function (Blueprint $table) { + $table->dropColumn('item_count'); + }); + } +}; diff --git a/database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php b/database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php new file mode 100644 index 000000000..3d2fa9848 --- /dev/null +++ b/database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php @@ -0,0 +1,28 @@ +integer('property_count')->nullable()->default(null); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metrics', function (Blueprint $table) { + $table->dropColumn('property_count'); + }); + } +}; diff --git a/database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php b/database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php new file mode 100644 index 000000000..f617454e7 --- /dev/null +++ b/database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php @@ -0,0 +1,28 @@ +integer('lexeme_count')->nullable()->default(null); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metrics', function (Blueprint $table) { + $table->dropColumn('lexeme_count'); + }); + } +}; diff --git a/database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php b/database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php new file mode 100644 index 000000000..7a06e2bb2 --- /dev/null +++ b/database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php @@ -0,0 +1,28 @@ +integer('entity_schema_count')->nullable()->default(null); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metrics', function (Blueprint $table) { + $table->dropColumn('entity_schema_count'); + }); + } +}; From fd74893d2a859650923a56f663efa072d393eaab Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Wed, 13 Aug 2025 09:53:27 +0200 Subject: [PATCH 2/9] added logic for counting and saving entities count in the metrics --- app/Metrics/App/WikiMetrics.php | 33 ++++++++++++++++++++++++++++++++- app/WikiDailyMetrics.php | 8 ++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/app/Metrics/App/WikiMetrics.php b/app/Metrics/App/WikiMetrics.php index 5609782a4..a2d735f9f 100644 --- a/app/Metrics/App/WikiMetrics.php +++ b/app/Metrics/App/WikiMetrics.php @@ -8,6 +8,7 @@ use Illuminate\Support\Arr; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; +use PDO; class WikiMetrics { @@ -32,7 +33,7 @@ public function saveMetrics(Wiki $wiki): void $weeklyActions = $this->getNumberOfActions(self::INTERVAL_WEEKLY); $monthlyActions = $this->getNumberOfActions(self::INTERVAL_MONTHLY); $quarterlyActions = $this->getNumberOfActions(self::INTERVAL_QUARTERLY); - + $numberOfEntities = $this->getNumberOfEntities(); $monthlyNumberOfUsersPerActivityType = $this->getNumberOfUsersPerActivityType(); $dailyMetrics = new WikiDailyMetrics([ @@ -46,6 +47,10 @@ public function saveMetrics(Wiki $wiki): void 'weekly_actions' => $weeklyActions, 'monthly_actions' => $monthlyActions, 'quarterly_actions' => $quarterlyActions, + 'item_count' => $numberOfEntities['120'], + 'property_count' => $numberOfEntities['122'], + 'lexeme_count' => $numberOfEntities['146'], + 'entity_schema_count' => $numberOfEntities['640'], 'monthly_casual_users' => $monthlyNumberOfUsersPerActivityType[0], 'monthly_active_users' => $monthlyNumberOfUsersPerActivityType[1], ]); @@ -171,4 +176,30 @@ private function getNumberOfUsersPerActivityType() : array Arr::get($result, 'monthly_active_users',null) ]; } + + private function getNumberOfEntities() : array + { + $resultArray = [ 120 => 0, 122 => 0, 146 => 0, 640 => 0 ]; + $wikiDb = $this->wiki->wikiDb; + $tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page'; + $query = "SELECT + page_namespace AS namespace, + COUNT(*) AS count + FROM $tablePage + WHERE page_namespace in (120, 146, 122, 640) + AND page_is_redirect=0 -- non-redirects only + GROUP BY page_namespace"; + + $manager = app()->db; + $manager->purge('mw'); + $conn = $manager->connection('mw'); + $pdo = $conn->getPdo(); + $result = $pdo->query($query)->fetchAll(PDO::FETCH_ASSOC); + if (count($result) === 0){ + return $resultArray; + } + $resultArray = array_column($result, 'count', 'namespace'); + + return $resultArray; + } } diff --git a/app/WikiDailyMetrics.php b/app/WikiDailyMetrics.php index d0114842d..60ad57718 100644 --- a/app/WikiDailyMetrics.php +++ b/app/WikiDailyMetrics.php @@ -28,6 +28,10 @@ class WikiDailyMetrics extends Model 'monthly_actions', 'quarterly_actions', 'number_of_triples', + 'item_count', + 'property_count', + 'lexeme_count', + 'entity_schema_count', 'monthly_casual_users', 'monthly_active_users', @@ -42,6 +46,10 @@ class WikiDailyMetrics extends Model 'monthly_actions', 'quarterly_actions', 'number_of_triples', + 'item_count', + 'property_count', + 'lexeme_count', + 'entity_schema_count', 'monthly_casual_users', 'monthly_active_users', ]; From e5325aaadcf4c2964a84f57fc166b0db33f72f8c Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Thu, 14 Aug 2025 09:18:12 +0200 Subject: [PATCH 3/9] Add test for entity count function in metrics --- tests/Metrics/WikiMetricsTest.php | 100 ++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index 19140aeb5..90c3d4773 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -170,5 +170,105 @@ public function testSaveNullForFailedRequestOfTriplesCount() 'number_of_triples' => null ]); } + + public function testSavesEntityCountsCorrectly() + { + $wiki = Wiki::factory()->create([ + 'domain' => 'entitycounttest.wikibase.cloud' + ]); + + $wikiDb = WikiDb::first(); + $wikiDb->update(['wiki_id' => $wiki->id]); + + $tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page'; + + DB::statement("CREATE TABLE IF NOT EXISTS $tablePage ( + page_id INT AUTO_INCREMENT PRIMARY KEY, + page_namespace INT, + page_is_redirect TINYINT(1) DEFAULT 0, + page_title VARCHAR(255), + page_random DOUBLE, + page_touched BINARY(14), + page_latest INT, + page_len INT + + )"); + + // Insert dummy data + DB::table($tablePage)->insert([ + [ + 'page_namespace' => 120, + 'page_is_redirect' => 0, + 'page_title' => 'foo', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2 + ], // item + [ + 'page_namespace' => 120, + 'page_is_redirect' => 0, + 'page_title' => 'bar', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 0, + 'page_len' => 2 + ], // item_rand + [ + 'page_namespace' => 122, + 'page_is_redirect' => 0, + 'page_title' => 'foo', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2] + , // property + [ + 'page_namespace' => 640, + 'page_is_redirect' => 0, + 'page_title' => 'bar', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2 + ], // entity schema + [ + 'page_namespace' => 146, + 'page_is_redirect' => 0, + 'page_title' => 'foo', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2 + ], // lexeme + [ + 'page_namespace' => 640, + 'page_is_redirect' => 1, + 'page_title' => 'foo', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2 + ], // entity schema (redirect, ignored) + ]); + WikiDailyMetrics::create([ + 'id' => $wiki->id . '_' . now()->subDay()->toDateString(), + 'wiki_id' => $wiki->id, + 'date' => now()->subDay()->toDateString(), + 'pages' => 0, + 'is_deleted' => 0 + ]); + + (new WikiMetrics())->saveMetrics($wiki); + + $this->assertDatabaseHas('wiki_daily_metrics', [ + 'wiki_id' => $wiki->id, + 'item_count' => 2, + 'property_count' => 1, + 'lexeme_count' => 1, + 'entity_schema_count' => 1 // the redirect should be ignored + ]); + + } } From b44f03d9c4680a13b90f264858af368e61e2b4b8 Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Thu, 14 Aug 2025 13:59:54 +0200 Subject: [PATCH 4/9] Reduce database transactions conflicts --- tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php b/tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php index 83543e357..21d19f246 100644 --- a/tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php +++ b/tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php @@ -4,6 +4,7 @@ use App\User; use App\WikiManager; use Carbon\CarbonImmutable; +use Illuminate\Foundation\Testing\RefreshDatabase; use Tests\Routes\Traits\OptionsRequestAllowed; use Tests\TestCase; use App\WikiSiteStats; @@ -16,7 +17,7 @@ class DeletedWikiMetricsControllerTest extends TestCase protected string $route = 'deletedWikiMetrics'; use OptionsRequestAllowed; - use DatabaseTransactions; + use RefreshDatabase; public function setUp(): void { From c2e74c47842bafe35e0b6162f6574674f5cd0eab Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Mon, 18 Aug 2025 15:42:58 +0200 Subject: [PATCH 5/9] fix tests --- tests/Metrics/WikiMetricsTest.php | 27 +++++++++++-------- .../Wiki/DeletedWikiMetricsControllerTest.php | 3 +-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index 90c3d4773..8d9cff0f3 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -9,9 +9,11 @@ use App\WikiDailyMetrics; use App\Jobs\ProvisionWikiDbJob; use Carbon\Carbon; +use Illuminate\Database\Schema\Blueprint; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Schema; use Tests\TestCase; class WikiMetricsTest extends TestCase @@ -182,17 +184,17 @@ public function testSavesEntityCountsCorrectly() $tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page'; - DB::statement("CREATE TABLE IF NOT EXISTS $tablePage ( - page_id INT AUTO_INCREMENT PRIMARY KEY, - page_namespace INT, - page_is_redirect TINYINT(1) DEFAULT 0, - page_title VARCHAR(255), - page_random DOUBLE, - page_touched BINARY(14), - page_latest INT, - page_len INT - - )"); + Schema::dropIfExists($tablePage); + Schema::create($tablePage, function (Blueprint $table) { + $table->increments('page_id'); + $table->integer('page_namespace'); + $table->boolean('page_is_redirect')->default(0); + $table->string('page_title', 255); + $table->double('page_random'); + $table->binary('page_touched'); + $table->integer('page_latest'); + $table->integer('page_len'); + }); // Insert dummy data DB::table($tablePage)->insert([ @@ -269,6 +271,9 @@ public function testSavesEntityCountsCorrectly() 'entity_schema_count' => 1 // the redirect should be ignored ]); + //clean up after the test + $wiki->forceDelete(); + Schema::dropIfExists($tablePage); } } diff --git a/tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php b/tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php index 21d19f246..83543e357 100644 --- a/tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php +++ b/tests/Routes/Wiki/DeletedWikiMetricsControllerTest.php @@ -4,7 +4,6 @@ use App\User; use App\WikiManager; use Carbon\CarbonImmutable; -use Illuminate\Foundation\Testing\RefreshDatabase; use Tests\Routes\Traits\OptionsRequestAllowed; use Tests\TestCase; use App\WikiSiteStats; @@ -17,7 +16,7 @@ class DeletedWikiMetricsControllerTest extends TestCase protected string $route = 'deletedWikiMetrics'; use OptionsRequestAllowed; - use RefreshDatabase; + use DatabaseTransactions; public function setUp(): void { From cc5d94e51153da2a211b2cab30e0877ba3eacb6e Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Mon, 18 Aug 2025 15:55:18 +0200 Subject: [PATCH 6/9] remove unuseful comments --- tests/Metrics/WikiMetricsTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index 8d9cff0f3..e3c731baf 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -215,7 +215,7 @@ public function testSavesEntityCountsCorrectly() 'page_touched' => random_bytes(10), 'page_latest' => 0, 'page_len' => 2 - ], // item_rand + ], // item [ 'page_namespace' => 122, 'page_is_redirect' => 0, @@ -251,7 +251,7 @@ public function testSavesEntityCountsCorrectly() 'page_touched' => random_bytes(10), 'page_latest' => 1, 'page_len' => 2 - ], // entity schema (redirect, ignored) + ], // entity schema ]); WikiDailyMetrics::create([ 'id' => $wiki->id . '_' . now()->subDay()->toDateString(), From 3120fec2de4c611ea3f228e19443a7bc5b0086cc Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Mon, 18 Aug 2025 16:16:19 +0200 Subject: [PATCH 7/9] a little refactoring --- app/Metrics/App/WikiMetrics.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/Metrics/App/WikiMetrics.php b/app/Metrics/App/WikiMetrics.php index a2d735f9f..e9f0e80af 100644 --- a/app/Metrics/App/WikiMetrics.php +++ b/app/Metrics/App/WikiMetrics.php @@ -179,7 +179,6 @@ private function getNumberOfUsersPerActivityType() : array private function getNumberOfEntities() : array { - $resultArray = [ 120 => 0, 122 => 0, 146 => 0, 640 => 0 ]; $wikiDb = $this->wiki->wikiDb; $tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page'; $query = "SELECT @@ -196,10 +195,8 @@ private function getNumberOfEntities() : array $pdo = $conn->getPdo(); $result = $pdo->query($query)->fetchAll(PDO::FETCH_ASSOC); if (count($result) === 0){ - return $resultArray; + return [ 120 => 0, 122 => 0, 146 => 0, 640 => 0 ]; } - $resultArray = array_column($result, 'count', 'namespace'); - - return $resultArray; + return array_column($result, 'count', 'namespace'); } } From 1fee556edc7fd1a8a1f7758fc2bf5f7e0bb3d09a Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Wed, 20 Aug 2025 15:27:00 +0200 Subject: [PATCH 8/9] update number of pages of wiki in the test to reflect the number of entities. placed the DB cleanup before assertion in the test --- tests/Metrics/WikiMetricsTest.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index e3c731baf..868498cf8 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -257,12 +257,16 @@ public function testSavesEntityCountsCorrectly() 'id' => $wiki->id . '_' . now()->subDay()->toDateString(), 'wiki_id' => $wiki->id, 'date' => now()->subDay()->toDateString(), - 'pages' => 0, + 'pages' => 6, 'is_deleted' => 0 ]); (new WikiMetrics())->saveMetrics($wiki); + //clean up after the test + $wiki->forceDelete(); + Schema::dropIfExists($tablePage); + $this->assertDatabaseHas('wiki_daily_metrics', [ 'wiki_id' => $wiki->id, 'item_count' => 2, @@ -270,10 +274,6 @@ public function testSavesEntityCountsCorrectly() 'lexeme_count' => 1, 'entity_schema_count' => 1 // the redirect should be ignored ]); - - //clean up after the test - $wiki->forceDelete(); - Schema::dropIfExists($tablePage); } } From 824fa546414503dc9f107721ef9b06e6c623c695 Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Wed, 20 Aug 2025 15:57:45 +0200 Subject: [PATCH 9/9] Update test: UpdateWikiDailyMetricJobTest.php --- tests/Jobs/UpdateWikiDailyMetricJobTest.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php index 90123c2c7..f59cd0a3e 100644 --- a/tests/Jobs/UpdateWikiDailyMetricJobTest.php +++ b/tests/Jobs/UpdateWikiDailyMetricJobTest.php @@ -61,6 +61,10 @@ public function testRunJobForAllWikisIncludingDeletedWikis() 'weekly_actions' => null, 'monthly_actions' => null, 'quarterly_actions' => null, + 'item_count' => 0, + 'property_count' => 0, + 'lexeme_count' => 0, + 'entity_schema_count' => 0, ]); $this->assertDatabaseHas('wiki_daily_metrics', [ @@ -70,6 +74,10 @@ public function testRunJobForAllWikisIncludingDeletedWikis() 'weekly_actions' => null, 'monthly_actions' => null, 'quarterly_actions' => null, + 'item_count' => 0, + 'property_count' => 0, + 'lexeme_count' => 0, + 'entity_schema_count' => 0, ]); }