api/app/Jobs/UpdateWikiSiteStatsJob.php at e4bf0340f4a9531daeb7ae9dd56bc13c01c82625 · wbstack/api · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
<?php

namespace App\Jobs;

use App\Services\MediaWikiHostResolver;
use App\Wiki;
use App\WikiSiteStats;
use Carbon\Carbon;
use Carbon\CarbonInterface;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;

class UpdateWikiSiteStatsJob extends Job implements ShouldBeUnique {
    use Dispatchable;

    public $timeout = 3600;

    private MediaWikiHostResolver $mwHostResolver;

    public function handle(MediaWikiHostResolver $mwHostResolver): void {
        $this->mwHostResolver = $mwHostResolver;

        $allWikis = Wiki::all();
        foreach ($allWikis as $wiki) {
            try {
                $this->updateSiteStats($wiki);
                $this->updateLifecycleEvents($wiki);
            } catch (\Exception $ex) {
                $this->job->markAsFailed();
                Log::error(
                    'Failure polling wiki ' . $wiki->getAttribute('domain') . ' for sitestats: ' . $ex->getMessage()
                );
            }
        }
    }

    private function updateLifecycleEvents(Wiki $wiki): void {
        $update = [];

        $firstEdited = $this->getFirstEditedDate($wiki);
        if ($firstEdited) {
            $update['first_edited'] = $firstEdited;
        }

        $lastEdited = $this->getLastEditedDate($wiki);
        if ($lastEdited) {
            $update['last_edited'] = Carbon::parse($lastEdited);
        }

        DB::transaction(function () use ($wiki, $update) {
            $wiki->wikiLifecycleEvents()->lockForUpdate()->updateOrCreate(['wiki_id' => $wiki->id], $update);
        });
    }

    private function updateSiteStats(Wiki $wiki): void {
        $response = Http::withHeaders([
            'host' => $wiki->getAttribute('domain'),
        ])->get(
            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json'
        );

        if ($response->failed()) {
            throw new \Exception('Request failed with reason ' . $response->body());
        }

        $responseBody = $response->json();
        $update = [];
        foreach (WikiSiteStats::FIELDS as $field) {
            $value = data_get($responseBody, 'query.statistics.' . $field, null);
            if ($value !== null) {
                $update[$field] = $value;
            }
        }
        DB::transaction(function () use ($wiki, $update) {
            $wiki->wikiSiteStats()->lockForUpdate()->updateOrCreate(['wiki_id' => $wiki->id], $update);
        });
    }

    private function getFirstEditedDate(Wiki $wiki): ?CarbonInterface {
        $allRevisions = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get(
            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php',
            [
                'action' => 'query',
                'format' => 'json',
                'list' => 'allrevisions',
                'formatversion' => 2,
                'arvlimit' => 1,
                'arvprop' => 'ids',
                'arvexcludeuser' => 'PlatformReservedUser',
                'arvdir' => 'newer',
            ],
        );
        $firstRevision = data_get($allRevisions->json(), 'query.allrevisions.0.revisions.0.revid');
        if (!$firstRevision) {
            return null;
        }

        $revisionInfo = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get(
            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php',
            [
                'action' => 'query',
                'format' => 'json',
                'prop' => 'revisions',
                'rvprop' => 'timestamp',
                'formatversion' => 2,
                'revids' => $firstRevision,
            ],
        );
        $result = data_get($revisionInfo->json(), 'query.pages.0.revisions.0.timestamp');
        if (!$result) {
            return null;
        }

        return Carbon::parse($result);
    }

    private function getLastEditedDate(Wiki $wiki): ?CarbonInterface {
        $allRevisions = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get(
            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php',
            [
                'action' => 'query',
                'format' => 'json',
                'list' => 'allrevisions',
                'formatversion' => 2,
                'arvlimit' => 1,
                'arvprop' => 'ids',
                'arvexcludeuser' => 'PlatformReservedUser',
                'arvdir' => 'older',
            ],
        );
        $lastRevision = data_get($allRevisions->json(), 'query.allrevisions.0.revisions.0.revid');
        if (!$lastRevision) {
            return null;
        }

        $revisionInfo = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get(
            $this->mwHostResolver->getBackendHostForDomain($wiki->domain) . '/w/api.php',
            [
                'action' => 'query',
                'format' => 'json',
                'prop' => 'revisions',
                'rvprop' => 'timestamp',
                'formatversion' => 2,
                'revids' => $lastRevision,
            ],
        );
        $result = data_get($revisionInfo->json(), 'query.pages.0.revisions.0.timestamp');
        if (!$result) {
            return null;
        }

        return Carbon::parse($result);
    }
}