Skip to content

Commit 584b87f

Browse files
rosalieperdeer-wmde
rosalieper
authored and
deer-wmde
committed
Add metrics: total number of entites (#957)
* Add migrations for new columns in the wiiki_daily_metrics table * added logic for counting and saving entities count in the metrics * Add test for entity count function in metrics * Reduce database transactions conflicts * fix tests * remove unuseful comments * a little refactoring * update number of pages of wiki in the test to reflect the number of entities. placed the DB cleanup before assertion in the test * Update test: UpdateWikiDailyMetricJobTest.php
1 parent a2a44a4 commit 584b87f

8 files changed

Lines changed: 262 additions & 1 deletion

‎app/Metrics/App/WikiMetrics.php‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use Illuminate\Support\Arr;
99
use Illuminate\Support\Facades\Http;
1010
use Illuminate\Support\Facades\Log;
11+
use PDO;
1112

1213
class WikiMetrics {
1314
const INTERVAL_DAILY = 'INTERVAL 1 DAY';
@@ -33,7 +34,7 @@ public function saveMetrics(Wiki $wiki): void {
3334
$weeklyActions = $this->getNumberOfActions(self::INTERVAL_WEEKLY);
3435
$monthlyActions = $this->getNumberOfActions(self::INTERVAL_MONTHLY);
3536
$quarterlyActions = $this->getNumberOfActions(self::INTERVAL_QUARTERLY);
36-
37+
$numberOfEntities = $this->getNumberOfEntities();
3738
$monthlyNumberOfUsersPerActivityType = $this->getNumberOfUsersPerActivityType();
3839

3940
$dailyMetrics = new WikiDailyMetrics([
@@ -47,6 +48,10 @@ public function saveMetrics(Wiki $wiki): void {
4748
'weekly_actions' => $weeklyActions,
4849
'monthly_actions' => $monthlyActions,
4950
'quarterly_actions' => $quarterlyActions,
51+
'item_count' => $numberOfEntities['120'],
52+
'property_count' => $numberOfEntities['122'],
53+
'lexeme_count' => $numberOfEntities['146'],
54+
'entity_schema_count' => $numberOfEntities['640'],
5055
'monthly_casual_users' => $monthlyNumberOfUsersPerActivityType[0],
5156
'monthly_active_users' => $monthlyNumberOfUsersPerActivityType[1],
5257
]);
@@ -177,4 +182,27 @@ private function getNumberOfUsersPerActivityType(): array {
177182
Arr::get($result, 'monthly_active_users', null),
178183
];
179184
}
185+
186+
private function getNumberOfEntities() : array
187+
{
188+
$wikiDb = $this->wiki->wikiDb;
189+
$tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page';
190+
$query = "SELECT
191+
page_namespace AS namespace,
192+
COUNT(*) AS count
193+
FROM $tablePage
194+
WHERE page_namespace in (120, 146, 122, 640)
195+
AND page_is_redirect=0 -- non-redirects only
196+
GROUP BY page_namespace";
197+
198+
$manager = app()->db;
199+
$manager->purge('mw');
200+
$conn = $manager->connection('mw');
201+
$pdo = $conn->getPdo();
202+
$result = $pdo->query($query)->fetchAll(PDO::FETCH_ASSOC);
203+
if (count($result) === 0){
204+
return [ 120 => 0, 122 => 0, 146 => 0, 640 => 0 ];
205+
}
206+
return array_column($result, 'count', 'namespace');
207+
}
180208
}

‎app/WikiDailyMetrics.php‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ class WikiDailyMetrics extends Model {
2727
'monthly_actions',
2828
'quarterly_actions',
2929
'number_of_triples',
30+
'item_count',
31+
'property_count',
32+
'lexeme_count',
33+
'entity_schema_count',
3034
'monthly_casual_users',
3135
'monthly_active_users',
3236

@@ -41,6 +45,10 @@ class WikiDailyMetrics extends Model {
4145
'monthly_actions',
4246
'quarterly_actions',
4347
'number_of_triples',
48+
'item_count',
49+
'property_count',
50+
'lexeme_count',
51+
'entity_schema_count',
4452
'monthly_casual_users',
4553
'monthly_active_users',
4654
];

‎database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
use Illuminate\Database\Migrations\Migration;
4+
use Illuminate\Database\Schema\Blueprint;
5+
use Illuminate\Support\Facades\Schema;
6+
7+
return new class extends Migration
8+
{
9+
/**
10+
* Run the migrations.
11+
*/
12+
public function up(): void
13+
{
14+
Schema::table('wiki_daily_metrics', function (Blueprint $table) {
15+
$table->integer('item_count')->nullable()->default(null);
16+
});
17+
}
18+
19+
/**
20+
* Reverse the migrations.
21+
*/
22+
public function down(): void
23+
{
24+
Schema::table('wiki_daily_metrics', function (Blueprint $table) {
25+
$table->dropColumn('item_count');
26+
});
27+
}
28+
};

‎database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
use Illuminate\Database\Migrations\Migration;
4+
use Illuminate\Database\Schema\Blueprint;
5+
use Illuminate\Support\Facades\Schema;
6+
7+
return new class extends Migration
8+
{
9+
/**
10+
* Run the migrations.
11+
*/
12+
public function up(): void
13+
{
14+
Schema::table('wiki_daily_metrics', function (Blueprint $table) {
15+
$table->integer('property_count')->nullable()->default(null);
16+
});
17+
}
18+
19+
/**
20+
* Reverse the migrations.
21+
*/
22+
public function down(): void
23+
{
24+
Schema::table('wiki_daily_metrics', function (Blueprint $table) {
25+
$table->dropColumn('property_count');
26+
});
27+
}
28+
};

‎database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
use Illuminate\Database\Migrations\Migration;
4+
use Illuminate\Database\Schema\Blueprint;
5+
use Illuminate\Support\Facades\Schema;
6+
7+
return new class extends Migration
8+
{
9+
/**
10+
* Run the migrations.
11+
*/
12+
public function up(): void
13+
{
14+
Schema::table('wiki_daily_metrics', function (Blueprint $table) {
15+
$table->integer('lexeme_count')->nullable()->default(null);
16+
});
17+
}
18+
19+
/**
20+
* Reverse the migrations.
21+
*/
22+
public function down(): void
23+
{
24+
Schema::table('wiki_daily_metrics', function (Blueprint $table) {
25+
$table->dropColumn('lexeme_count');
26+
});
27+
}
28+
};

‎database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
use Illuminate\Database\Migrations\Migration;
4+
use Illuminate\Database\Schema\Blueprint;
5+
use Illuminate\Support\Facades\Schema;
6+
7+
return new class extends Migration
8+
{
9+
/**
10+
* Run the migrations.
11+
*/
12+
public function up(): void
13+
{
14+
Schema::table('wiki_daily_metrics', function (Blueprint $table) {
15+
$table->integer('entity_schema_count')->nullable()->default(null);
16+
});
17+
}
18+
19+
/**
20+
* Reverse the migrations.
21+
*/
22+
public function down(): void
23+
{
24+
Schema::table('wiki_daily_metrics', function (Blueprint $table) {
25+
$table->dropColumn('entity_schema_count');
26+
});
27+
}
28+
};

‎tests/Jobs/UpdateWikiDailyMetricJobTest.php‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ public function testRunJobForAllWikisIncludingDeletedWikis() {
5353
'weekly_actions' => null,
5454
'monthly_actions' => null,
5555
'quarterly_actions' => null,
56+
'item_count' => 0,
57+
'property_count' => 0,
58+
'lexeme_count' => 0,
59+
'entity_schema_count' => 0,
5660
]);
5761

5862
$this->assertDatabaseHas('wiki_daily_metrics', [
@@ -62,6 +66,10 @@ public function testRunJobForAllWikisIncludingDeletedWikis() {
6266
'weekly_actions' => null,
6367
'monthly_actions' => null,
6468
'quarterly_actions' => null,
69+
'item_count' => 0,
70+
'property_count' => 0,
71+
'lexeme_count' => 0,
72+
'entity_schema_count' => 0,
6573
]);
6674
}
6775
}

‎tests/Metrics/WikiMetricsTest.php‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@
99
use App\WikiDailyMetrics;
1010
use App\WikiDb;
1111
use Carbon\Carbon;
12+
use Illuminate\Database\Schema\Blueprint;
1213
use Illuminate\Foundation\Testing\RefreshDatabase;
1314
use Illuminate\Support\Facades\DB;
1415
use Illuminate\Support\Facades\Http;
16+
use Illuminate\Support\Facades\Schema;
1517
use Tests\TestCase;
1618

1719
class WikiMetricsTest extends TestCase {
@@ -165,4 +167,107 @@ public function testSaveNullForFailedRequestOfTriplesCount() {
165167
'number_of_triples' => null,
166168
]);
167169
}
170+
171+
public function testSavesEntityCountsCorrectly()
172+
{
173+
$wiki = Wiki::factory()->create([
174+
'domain' => 'entitycounttest.wikibase.cloud'
175+
]);
176+
177+
$wikiDb = WikiDb::first();
178+
$wikiDb->update(['wiki_id' => $wiki->id]);
179+
180+
$tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page';
181+
182+
Schema::dropIfExists($tablePage);
183+
Schema::create($tablePage, function (Blueprint $table) {
184+
$table->increments('page_id');
185+
$table->integer('page_namespace');
186+
$table->boolean('page_is_redirect')->default(0);
187+
$table->string('page_title', 255);
188+
$table->double('page_random');
189+
$table->binary('page_touched');
190+
$table->integer('page_latest');
191+
$table->integer('page_len');
192+
});
193+
194+
// Insert dummy data
195+
DB::table($tablePage)->insert([
196+
[
197+
'page_namespace' => 120,
198+
'page_is_redirect' => 0,
199+
'page_title' => 'foo',
200+
'page_random' => 0,
201+
'page_touched' => random_bytes(10),
202+
'page_latest' => 1,
203+
'page_len' => 2
204+
], // item
205+
[
206+
'page_namespace' => 120,
207+
'page_is_redirect' => 0,
208+
'page_title' => 'bar',
209+
'page_random' => 0,
210+
'page_touched' => random_bytes(10),
211+
'page_latest' => 0,
212+
'page_len' => 2
213+
], // item
214+
[
215+
'page_namespace' => 122,
216+
'page_is_redirect' => 0,
217+
'page_title' => 'foo',
218+
'page_random' => 0,
219+
'page_touched' => random_bytes(10),
220+
'page_latest' => 1,
221+
'page_len' => 2]
222+
, // property
223+
[
224+
'page_namespace' => 640,
225+
'page_is_redirect' => 0,
226+
'page_title' => 'bar',
227+
'page_random' => 0,
228+
'page_touched' => random_bytes(10),
229+
'page_latest' => 1,
230+
'page_len' => 2
231+
], // entity schema
232+
[
233+
'page_namespace' => 146,
234+
'page_is_redirect' => 0,
235+
'page_title' => 'foo',
236+
'page_random' => 0,
237+
'page_touched' => random_bytes(10),
238+
'page_latest' => 1,
239+
'page_len' => 2
240+
], // lexeme
241+
[
242+
'page_namespace' => 640,
243+
'page_is_redirect' => 1,
244+
'page_title' => 'foo',
245+
'page_random' => 0,
246+
'page_touched' => random_bytes(10),
247+
'page_latest' => 1,
248+
'page_len' => 2
249+
], // entity schema
250+
]);
251+
WikiDailyMetrics::create([
252+
'id' => $wiki->id . '_' . now()->subDay()->toDateString(),
253+
'wiki_id' => $wiki->id,
254+
'date' => now()->subDay()->toDateString(),
255+
'pages' => 6,
256+
'is_deleted' => 0
257+
]);
258+
259+
(new WikiMetrics())->saveMetrics($wiki);
260+
261+
//clean up after the test
262+
$wiki->forceDelete();
263+
Schema::dropIfExists($tablePage);
264+
265+
$this->assertDatabaseHas('wiki_daily_metrics', [
266+
'wiki_id' => $wiki->id,
267+
'item_count' => 2,
268+
'property_count' => 1,
269+
'lexeme_count' => 1,
270+
'entity_schema_count' => 1 // the redirect should be ignored
271+
]);
272+
}
168273
}

0 commit comments

Comments
 (0)