Skip to content

Commit 0101597

Browse files
m90m90
authored
Add command for rebuilding QueryService data (#704)
* feat(qs): scaffold command for rebuilding queryservice data * feat: add stub methods * feat: fetch entity ids from mediawiki api * chore: log received args in dummy job for testing * refactor: use max page size * fix: use proper command name * fix: directly query namespaces table * fix: do not fail on null lexemes setting * test: add test setup for qs rebuild command * test: add test for full command run * refactor: make sparql url configurable * test: explicitly set config values in test * refactor: source command config from cli args with defaults * test: add test case for command failure * test: empty wikis shall not dispatch jobs * feat: allow passing multiple domains * fix: explicitly request json responses from api * feat(k8s): add job that can spawn one off queryservice-updater (#701) * feat(k8s): add job that can spawn one off queryservice-updater * fix: consider entity payload for job uniqueness * refactor: read env vars through config * chore: dispatch proper job * fix: kubernetes labels cannot contain commas * docs: add changelog * chore: pass exception to extra logging args * chore: add logging to command * fix: entity payload in labels exceeds allowed length
1 parent 3a81750 commit 0101597

9 files changed

Lines changed: 776 additions & 1 deletion

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
## 8x.28.11 - 12 December 2023
44
- Batches that are done shall never be marked failed
55

6+
## 8x.29.0 - 13 December 2023
7+
- Add command for rebuilding Queryservice data
8+
69
## 8x.28.10 - 29 November 2023
710
- Requeueing should not select failed batches
811

‎app/Console/Commands/RebuildQueryserviceData.php‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
<?php
2+
3+
namespace App\Console\Commands;
4+
5+
use Illuminate\Console\Command;
6+
use Illuminate\Support\Facades\Http;
7+
use Illuminate\Support\Facades\Log;
8+
use App\Wiki;
9+
use App\WikiSetting;
10+
use App\QueryserviceNamespace;
11+
use App\Jobs\SpawnQueryserviceUpdaterJob;
12+
13+
class RebuildQueryserviceData extends Command
14+
{
15+
private const NAMESPACE_ITEM = 120;
16+
private const NAMESPACE_PROPERTY = 122;
17+
private const NAMESPACE_LEXEME = 146;
18+
19+
protected $signature = 'wbs-qs:rebuild {--domain=*} {--chunkSize=50} {--sparqlUrlFormat=http://queryservice.default.svc.cluster.local:9999/bigdata/namespace/%s/sparql}';
20+
21+
protected $description = 'Rebuild the queryservice data for a certain wiki or all wikis';
22+
23+
protected int $chunkSize;
24+
protected string $apiUrl;
25+
protected string $sparqlUrlFormat;
26+
27+
public function handle()
28+
{
29+
$this->chunkSize = intval($this->option('chunkSize'));
30+
$this->sparqlUrlFormat = $this->option('sparqlUrlFormat');
31+
$this->apiUrl = getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php';
32+
33+
$wikiDomains = $this->option('domain');
34+
$exitCode = 0;
35+
36+
$wikis = count($wikiDomains) !== 0
37+
? Wiki::whereIn('domain', $wikiDomains)->get()
38+
: Wiki::query()->get();
39+
40+
$jobTotal = 0;
41+
$skippedWikis = 0;
42+
$processedWikis = 0;
43+
foreach ($wikis as $wiki) {
44+
try {
45+
$entities = $this->getEntitiesForWiki($wiki);
46+
$sparqlUrl = $this->getSparqlUrl($wiki);
47+
} catch (\Exception $ex) {
48+
Log::error(
49+
'Failed to get prerequisites for enqueuing wiki '.$wiki->domain.', will not dispatch jobs.',
50+
[$ex],
51+
);
52+
$exitCode = 1;
53+
$skippedWikis += 1;
54+
break;
55+
}
56+
57+
$entityChunks = array_chunk($entities, $this->chunkSize);
58+
foreach ($entityChunks as $entityChunk) {
59+
dispatch(
60+
new SpawnQueryserviceUpdaterJob(
61+
$wiki->domain,
62+
implode(',', $entityChunk),
63+
$sparqlUrl,
64+
)
65+
);
66+
}
67+
$jobTotal += count($entityChunks);
68+
$processedWikis += 1;
69+
Log::info('Dispatched '.count($entityChunks).' job(s) for wiki '.$wiki->domain.'.');
70+
}
71+
72+
Log::info(
73+
'Done. Jobs dispatched: '.$jobTotal.' Wikis processed: '.$processedWikis.' Wikis skipped: '.$skippedWikis
74+
);
75+
return $exitCode;
76+
}
77+
78+
private function getEntitiesForWiki (Wiki $wiki): array
79+
{
80+
$items = $this->fetchPagesInNamespace($wiki->domain, self::NAMESPACE_ITEM);
81+
$properties = $this->fetchPagesInNamespace($wiki->domain, self::NAMESPACE_PROPERTY);
82+
83+
$lexemesSetting = WikiSetting::where(
84+
[
85+
'wiki_id' => $wiki->id,
86+
'name' => WikiSetting::wwExtEnableWikibaseLexeme
87+
],
88+
)->first();
89+
$hasLexemesEnabled = $lexemesSetting !== null && $lexemesSetting->value === '1';
90+
$lexemes = $hasLexemesEnabled
91+
? $this->fetchPagesInNamespace($wiki->domain, self::NAMESPACE_LEXEME)
92+
: [];
93+
94+
$merged = array_merge($items, $properties, $lexemes);
95+
return $this->stripPrefixes($merged);
96+
}
97+
98+
private function getSparqlUrl (Wiki $wiki): string
99+
{
100+
$match = QueryserviceNamespace::where(['wiki_id' => $wiki->id])->first();
101+
if (!$match) {
102+
throw new \Exception(
103+
'Unable to find queryservice namespace record for wiki '.$wiki->domain
104+
);
105+
}
106+
return sprintf($this->sparqlUrlFormat, $match->namespace);
107+
}
108+
109+
private function fetchPagesInNamespace(string $wikiDomain, int $namespace): array
110+
{
111+
$titles = [];
112+
$cursor = '';
113+
while (true) {
114+
$response = Http::withHeaders([
115+
'host' => $wikiDomain
116+
])->get(
117+
$this->apiUrl,
118+
[
119+
'action' => 'query',
120+
'list' => 'allpages',
121+
'apnamespace' => $namespace,
122+
'apcontinue' => $cursor,
123+
'aplimit' => 'max',
124+
'format' => 'json',
125+
],
126+
);
127+
128+
if ($response->failed()) {
129+
throw new \Exception(
130+
'Failed to fetch allpages for wiki '.$wikiDomain
131+
);
132+
}
133+
134+
$jsonResponse = $response->json();
135+
$error = data_get($jsonResponse, 'error');
136+
if ($error !== null) {
137+
throw new \Exception(
138+
'Error response fetching allpages for wiki '.$wikiDomain.': '.$error
139+
);
140+
}
141+
142+
$pages = data_get($jsonResponse, 'query.allpages', []);
143+
$titles = array_merge($titles, array_map(function (array $page) {
144+
return $page['title'];
145+
}, $pages));
146+
147+
$nextCursor = data_get($jsonResponse, 'continue.apcontinue');
148+
if ($nextCursor === null) {
149+
break;
150+
}
151+
$cursor = $nextCursor;
152+
}
153+
154+
return $titles;
155+
}
156+
157+
private static function stripPrefixes (array $items): array
158+
{
159+
return array_map(function (string $item) {
160+
return preg_replace('/^[a-zA-Z]+:/', '', $item);
161+
}, $items);
162+
}
163+
}

‎app/Jobs/ProcessMediaWikiJobsJob.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
use Illuminate\Contracts\Queue\ShouldBeUnique;
88
use Illuminate\Queue\InteractsWithQueue;
99
use Illuminate\Support\Facades\Log;
10+
use Illuminate\Support\Facades\Config;
1011
use Maclof\Kubernetes\Client;
1112
use Maclof\Kubernetes\Models\Job as KubernetesJob;
1213

@@ -20,7 +21,7 @@ class ProcessMediaWikiJobsJob implements ShouldQueue, ShouldBeUnique
2021
public function __construct (string $wikiDomain)
2122
{
2223
$this->wikiDomain = $wikiDomain;
23-
$this->jobsKubernetesNamespace = env('API_JOB_NAMESPACE', 'api-jobs');
24+
$this->jobsKubernetesNamespace = Config::get('wbstack.api_job_namespace');
2425
}
2526

2627
public function uniqueId(): string

‎app/Jobs/SpawnQueryserviceUpdaterJob.php‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
<?php
2+
3+
namespace App\Jobs;
4+
5+
use Illuminate\Bus\Queueable;
6+
use Illuminate\Contracts\Queue\ShouldQueue;
7+
use Illuminate\Contracts\Queue\ShouldBeUnique;
8+
use Illuminate\Queue\InteractsWithQueue;
9+
use Illuminate\Support\Facades\Log;
10+
use Illuminate\Support\Facades\Config;
11+
use Maclof\Kubernetes\Client;
12+
use Maclof\Kubernetes\Models\Job as KubernetesJob;
13+
14+
class SpawnQueryserviceUpdaterJob implements ShouldQueue, ShouldBeUnique
15+
{
16+
use InteractsWithQueue, Queueable;
17+
18+
public string $wikiDomain;
19+
public string $entities;
20+
public string $sparqlUrl;
21+
public string $qsKubernetesNamespace;
22+
23+
public function __construct (string $wikiDomain, string $entities, string $sparqlUrl)
24+
{
25+
$sortedEntities = explode(',', $entities);
26+
asort($sortedEntities);
27+
28+
$this->wikiDomain = $wikiDomain;
29+
$this->entities = implode(',', $sortedEntities);
30+
$this->sparqlUrl = $sparqlUrl;
31+
$this->qsKubernetesNamespace = Config::get('wbstack.qs_job_namespace');
32+
}
33+
34+
public function uniqueId(): string
35+
{
36+
return $this->wikiDomain.$this->entities;
37+
}
38+
39+
public function handle (Client $kubernetesClient): void
40+
{
41+
$kubernetesClient->setNamespace('default');
42+
$qsUpdaterPod = $kubernetesClient->pods()->setFieldSelector([
43+
'status.phase' => 'Running'
44+
])->setLabelSelector([
45+
'app.kubernetes.io/name' => 'queryservice-updater',
46+
])->first();
47+
48+
if ($qsUpdaterPod === null) {
49+
$this->fail(
50+
new \RuntimeException(
51+
'Unable to find a running queryservice-updater pod in the cluster, '.
52+
'cannot continue.'
53+
)
54+
);
55+
return;
56+
}
57+
$qsUpdaterPod = $qsUpdaterPod->toArray();
58+
59+
$kubernetesClient->setNamespace($this->qsKubernetesNamespace);
60+
$jobSpec = new KubernetesJob([
61+
'metadata' => [
62+
'name' => 'run-qs-updater-'.hash('sha1', $this->uniqueId()),
63+
'namespace' => $this->qsKubernetesNamespace,
64+
'labels' => [
65+
'app.kubernetes.io/instance' => $this->wikiDomain,
66+
'app.kubernetes.io/name' => 'run-qs-updater',
67+
]
68+
],
69+
'spec' => [
70+
'ttlSecondsAfterFinished' => 14 * 24 * 60 * 60, // 2 weeks
71+
'template' => [
72+
'metadata' => [
73+
'name' => 'run-qs-updater'
74+
],
75+
'spec' => [
76+
'containers' => [
77+
0 => [
78+
'name' => 'run-qs-updater',
79+
'image' => $qsUpdaterPod['spec']['containers'][0]['image'],
80+
'env' => $qsUpdaterPod['spec']['containers'][0]['env'],
81+
'command' => [
82+
0 => 'bash',
83+
1 => '-c',
84+
2 => <<<CMD
85+
/wdqsup/runUpdateWbStack.sh -- \
86+
--wikibaseHost {$this->wikiDomain} \
87+
--ids {$this->entities} \
88+
--entityNamespaces 120,122,146 \
89+
--sparqlUrl {$this->sparqlUrl} \
90+
--wikibaseScheme http \
91+
--conceptUri https://{$this->wikiDomain}
92+
CMD
93+
],
94+
]
95+
],
96+
'restartPolicy' => 'Never'
97+
]
98+
]
99+
]
100+
]);
101+
102+
$job = $kubernetesClient->jobs()->apply($jobSpec);
103+
$jobName = data_get($job, 'metadata.name');
104+
if (data_get($job, 'status') === 'Failure' || !$jobName) {
105+
// The k8s client does not fail reliably on 4xx responses, so checking the name
106+
// currently serves as poor man's error handling.
107+
$this->fail(
108+
new \RuntimeException('Queryservice Updater creation for wiki "'.$this->wikiDomain.'" failed with message: '.data_get($job, 'message', 'n/a'))
109+
);
110+
return;
111+
}
112+
Log::info(
113+
'Queryservice Updater for wiki "'.$this->wikiDomain.'" and entities "'.$this->entities.'" exists or was created with name "'.$jobName.'".'
114+
);
115+
116+
return;
117+
}
118+
119+
}

‎app/QueryserviceNamespace.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace App;
44

55
use Illuminate\Database\Eloquent\Model;
6+
use Illuminate\Database\Eloquent\Factories\HasFactory;
67

78
/**
89
* App\QueryserviceNamespace.
@@ -27,6 +28,7 @@
2728
*/
2829
class QueryserviceNamespace extends Model
2930
{
31+
use HasFactory;
3032
/**
3133
* The attributes that are mass assignable.
3234
*

‎config/wbstack.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,7 @@
2828
'qs_batch_pending_timeout' => env('WBSTACK_QS_BATCH_PENDING_TIMEOUT', 'PT300S'),
2929
'qs_batch_mark_failed_after' => intval(env('WBSTACK_QS_BATCH_MARK_FAILED_AFTER', '3')),
3030
'qs_batch_entity_limit' => intval(env('WBSTACK_QS_BATCH_ENTITY_LIMIT', '10')),
31+
32+
'api_job_namespace' => env('API_JOB_NAMESPACE') || env('WBSTACK_API_JOB_NAMESPACE', 'api-jobs'),
33+
'qs_job_namespace' => env('WBSTACK_QS_JOB_NAMESPACE', 'qs-jobs'),
3134
];

‎database/factories/QueryserviceNamespaceFactory.php‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<?php
2+
3+
namespace Database\Factories;
4+
5+
use App\QueryserviceNamespace;
6+
use Illuminate\Database\Eloquent\Factories\Factory;
7+
8+
class QueryserviceNamespaceFactory extends Factory
9+
{
10+
/**
11+
* The name of the factory's corresponding model.
12+
*
13+
* @var string
14+
*/
15+
protected $model = QueryserviceNamespace::class;
16+
17+
/**
18+
* Define the model's default state.
19+
*
20+
* @return array
21+
*/
22+
public function definition()
23+
{
24+
return [];
25+
}
26+
}

0 commit comments

Comments
 (0)