Skip to content

Commit 23907d4

Browse files
m90m90
authored
feat(platform-stats): provide number of recently created wikis and users (#617)
* feat(platform-stats): provide number of recently created wikis * refactor: default to empty ranges * docs: update changelog * refactor: rename wiki creation rates * feat: calculate rates of user created * refactor: collect creation rates from db query * test: undo changes to original test * test: null out wikis pre seeding * test: seed wikis only when needed for test * test: add test for creation stats * test: remove teardown * refactor: remove redundant array_filter callback * docs: update changelog
1 parent ecc032f commit 23907d4

5 files changed

Lines changed: 92 additions & 34 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
# api
22

3+
## 8x.14.0 - 04 July 2023
4+
- Collect signup and wiki creation rate metrics with platform summary job
5+
36
## 8x.13.0 - 22 June 2023
47
- Force lowercase domain names on wiki creation
58

‎app/Jobs/PlatformStatsSummaryJob.php‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,12 @@
33
namespace App\Jobs;
44

55
use App\Wiki;
6+
use App\User;
67
use Illuminate\Database\DatabaseManager;
78
use PDO;
89
use Carbon\Carbon;
910
use Illuminate\Support\Facades\Log;
10-
use Illuminate\Notifications\Notifiable;
11-
use App\Notifications\PlatformStatsSummaryNotification;
1211
use Illuminate\Support\Facades\Config;
13-
use Illuminate\Support\Facades\Notification;
1412
use Illuminate\Support\Facades\App;
1513

1614
/*
@@ -30,24 +28,39 @@
3028
class PlatformStatsSummaryJob extends Job
3129
{
3230
private $inactiveThreshold;
31+
private $creationRateRanges;
3332

3433
private $platformSummaryStatsVersion = "v1";
3534
public function __construct() {
3635
$this->inactiveThreshold = Config::get('wbstack.platform_summary_inactive_threshold');
36+
$this->creationRateRanges = Config::get('wbstack.platform_summary_creation_rate_ranges');
3737
}
3838

3939
private function isNullOrEmpty( $value ): bool {
4040
return is_null($value) || intVal($value) === 0;
4141
}
4242

43-
public function prepareStats( array $allStats, $wikis ): array {
43+
public function getCreationStats(): array {
44+
$result = [];
45+
$now = Carbon::now();
46+
foreach ($this->creationRateRanges as $range) {
47+
$limit = $now->clone()->sub(new \DateInterval($range));
48+
$wikis = Wiki::where('created_at', '>=', $limit)->count();
49+
$result['wikis_created_'.$range] = $wikis;
50+
$users = User::where('created_at', '>=', $limit)->count();
51+
$result['users_created_'.$range] = $users;
52+
}
53+
return $result;
54+
}
4455

45-
$deletedWikis = [];
46-
$activeWikis= [];
47-
$inactive = [];
48-
$emptyWikis = [];
56+
public function prepareStats( array $allStats, $wikis): array {
4957

58+
$deletedWikis = [];
59+
$activeWikis = [];
60+
$inactive = [];
61+
$emptyWikis = [];
5062
$nonDeletedStats = [];
63+
5164
$currentTime = Carbon::now()->timestamp;
5265

5366
foreach( $wikis as $wiki ) {
@@ -60,7 +73,6 @@ public function prepareStats( array $allStats, $wikis ): array {
6073
$wikiDb = $wiki->wikiDb()->first();
6174

6275
if( !$wikiDb ) {
63-
6476
Log::error(__METHOD__ . ": Could not find WikiDB for {$wiki->domain}");
6577
continue;
6678
}
@@ -71,7 +83,7 @@ public function prepareStats( array $allStats, $wikis ): array {
7183
Log::warning(__METHOD__ . ": Could not find stats for {$wiki->domain}");
7284
continue;
7385
}
74-
86+
7587
$stats = $allStats[$found_key];
7688

7789
// is it empty?
@@ -86,7 +98,7 @@ public function prepareStats( array $allStats, $wikis ): array {
8698
if(!is_null($stats['lastEdit'])){
8799
$lastTimestamp = intVal($stats['lastEdit']);
88100
$diff = $currentTime - $lastTimestamp;
89-
101+
90102
if ($diff >= $this->inactiveThreshold) {
91103
$inactive[] = $wiki;
92104
continue;
@@ -95,7 +107,7 @@ public function prepareStats( array $allStats, $wikis ): array {
95107

96108
$activeWikis[] = $wiki;
97109
}
98-
110+
99111
$totalNonDeletedUsers = array_sum(array_column($nonDeletedStats, 'users'));
100112
$totalNonDeletedActiveUsers = array_sum(array_column($nonDeletedStats, 'active_users'));
101113
$totalNonDeletedPages = array_sum(array_column($nonDeletedStats, 'pages'));
@@ -133,11 +145,11 @@ public function handle( DatabaseManager $manager ): void
133145
$mediawikiPdo = $mwConn->getPdo();
134146

135147
$pdo->setAttribute(PDO::ATTR_EMULATE_PREPARES, 1);
136-
148+
137149
// prepare the first query
138150
$statement = $pdo->prepare($this->wikiStatsQuery);
139151
$statement->execute();
140-
152+
141153
// produces the stats query
142154
$result = $statement->fetchAll(PDO::FETCH_ASSOC)[0];
143155
$query = array_values($result)[0];
@@ -146,9 +158,12 @@ public function handle( DatabaseManager $manager ): void
146158
$allStats = $mediawikiPdo->query($query)->fetchAll(PDO::FETCH_ASSOC);
147159
$summary = $this->prepareStats( $allStats, $wikis );
148160

161+
$creationStats = $this->getCreationStats();
162+
$summary = array_merge($summary, $creationStats);
163+
149164
$manager->purge('mw');
150165
$manager->purge('mysql');
151-
166+
152167
// Output to be scraped from logs
153168
if( !App::runningUnitTests() ) {
154169
print( json_encode($summary) . PHP_EOL );
@@ -183,7 +198,7 @@ public function handle( DatabaseManager $manager ): void
183198
"
184199
185200
) SEPARATOR ' UNION ALL ')
186-
201+
187202
FROM apidb.wiki_dbs
188203
LEFT JOIN apidb.wikis ON wiki_dbs.wiki_id = wikis.id;
189204

‎config/wbstack.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
'wiki_max_per_user' => env('WBSTACK_MAX_PER_USER', false),
1313

1414
'platform_summary_inactive_threshold' => env('WBSTACK_SUMMARY_INACTIVE_THRESHOLD', 60 * 60 * 24 * 90),
15+
'platform_summary_creation_rate_ranges' => array_filter(
16+
explode(',', env('WBSTACK_SUMMARY_CREATION_RATE_RANGES', ''))
17+
),
1518

1619
'elasticsearch_host' => env('ELASTICSEARCH_HOST', false),
1720
'elasticsearch_enabled_by_default' => env('WBSTACK_ELASTICSEARCH_ENABLED_BY_DEFAULT', false),

‎phpunit.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,6 @@
1515
<env name="CACHE_DRIVER" value="array"/>
1616
<env name="QUEUE_CONNECTION" value="sync"/>
1717
<env name="PLATFORM_MW_BACKEND_HOST" value="mediawiki-139-app-backend.default.svc.cluster.default"/>
18+
<env name="WBSTACK_SUMMARY_CREATION_RATE_RANGES" value="PT24H,P30D"/>
1819
</php>
1920
</phpunit>

‎tests/Jobs/PlatformStatsSummaryJobTest.php‎

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
namespace Tests\Jobs;
44

5-
use Illuminate\Foundation\Testing\DatabaseTransactions;
5+
use Illuminate\Foundation\Testing\RefreshDatabase;
66
use Tests\TestCase;
77
use App\User;
88
use App\Wiki;
@@ -16,28 +16,29 @@
1616

1717
class PlatformStatsSummaryJobTest extends TestCase
1818
{
19-
use DatabaseTransactions;
19+
use RefreshDatabase;
2020

2121
private $numWikis = 5;
2222
private $wikis = [];
23+
private $users = [];
2324

2425
private $db_prefix = "somecoolprefix";
2526
private $db_name = "some_cool_db_name";
2627

2728
protected function setUp(): void {
2829
parent::setUp();
29-
for($n = 0; $n < $this->numWikis; $n++ ) {
30+
for ($n = 0; $n < $this->numWikis; $n++) {
3031
DB::connection('mysql')->getPdo()->exec("DROP DATABASE IF EXISTS {$this->db_name}{$n};");
3132
}
32-
$this->seedWikis();
33-
$this->manager = $this->app->make('db');
33+
$this->wikis = [];
34+
$this->users = [];
3435
}
3536

3637
protected function tearDown(): void {
37-
foreach($this->wikis as $wiki) {
38-
$wiki['wiki']->wikiDb()->forceDelete();
39-
$wiki['wiki']->forceDelete();
40-
}
38+
Wiki::query()->delete();
39+
User::query()->delete();
40+
WikiManager::query()->delete();
41+
WikiDb::query()->delete();
4142
parent::tearDown();
4243
}
4344

@@ -55,15 +56,14 @@ private function seedWikis() {
5556
$wikiDb = WikiDb::whereName($this->db_name.$n)->first();
5657
$wikiDb->update( ['wiki_id' => $wiki->id] );
5758

58-
$this->wikis[] = [
59-
'user' => $user,
60-
'wiki' => Wiki::whereId($wiki->id)->with('wikidb')->first()
61-
];
59+
$this->wikis[] = $wiki;
60+
$this->users[] = $user;
6261
}
6362

6463
}
6564
public function testQueryGetsStats()
6665
{
66+
$this->seedWikis();
6767
$manager = $this->app->make('db');
6868

6969
$mockJob = $this->createMock(Job::class);
@@ -83,16 +83,15 @@ public function testGroupings()
8383
$job = new PlatformStatsSummaryJob();
8484
$job->setJob($mockJob);
8585

86-
$testWikis = [
86+
$wikis = [
8787
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki1.com' ] ),
8888
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki2.com' ] ),
8989
Wiki::factory()->create( [ 'deleted_at' => Carbon::now()->subDays(90)->timestamp, 'domain' => 'wiki3.com' ] ),
9090
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki4.com' ] )
91-
9291
];
9392

94-
foreach($testWikis as $wiki) {
95-
$wikiDB = WikiDb::create([
93+
foreach($wikis as $wiki) {
94+
WikiDb::create([
9695
'name' => 'mwdb_asdasfasfasf' . $wiki->id,
9796
'user' => 'asdasd',
9897
'password' => 'asdasfasfasf',
@@ -147,7 +146,7 @@ public function testGroupings()
147146
];
148147

149148

150-
$groups = $job->prepareStats($stats, $testWikis);
149+
$groups = $job->prepareStats($stats, $wikis);
151150

152151
$this->assertEquals(
153152
[
@@ -165,6 +164,43 @@ public function testGroupings()
165164
$groups,
166165
);
167166
}
167+
function testCreationStats() {
168+
$mockJob = $this->createMock(Job::class);
169+
$mockJob->expects($this->never())->method('fail');
168170

171+
$job = new PlatformStatsSummaryJob();
172+
$job->setJob($mockJob);
173+
174+
Wiki::factory()->create([
175+
'created_at' => Carbon::now()->subHours(1)
176+
]);
177+
Wiki::factory()->create([
178+
'created_at' => Carbon::now()->subDays(2)
179+
]);
180+
Wiki::factory()->create([
181+
'created_at' => Carbon::now()->subDays(90)
182+
]);
183+
User::factory()->create([
184+
'created_at' => Carbon::now()->subHours(1)
185+
]);
186+
User::factory()->create([
187+
'created_at' => Carbon::now()->subHours(2)
188+
]);
189+
User::factory()->create([
190+
'created_at' => Carbon::now()->subDays(200)
191+
]);
192+
193+
$stats = $job->getCreationStats();
194+
195+
$this->assertEquals(
196+
[
197+
'wikis_created_PT24H' => 1,
198+
'wikis_created_P30D' => 2,
199+
'users_created_PT24H' => 2,
200+
'users_created_P30D' => 2,
201+
],
202+
$stats,
203+
);
169204

205+
}
170206
}

0 commit comments

Comments
 (0)