Skip to content

Commit 7fa229e

Browse files
calmacxclaude
calmacx
and
claude
committed
fix(GAT-9018): reconstruct GWDM envelope instead of reading raw metadata column
Delta dataset_version rows store an empty metadata column by design (full GWDM lives in snapshots + patches), but IndexElastic and the tool/publication extraction jobs read $latestVersion->metadata directly, causing crashes or silent no-ops on delta rows. They now reconstruct via DatasetService::getReconstructedMetadataEnvelope(), matching the working snapshot-row path. Also adds a GwdmMetadataHandler::supportsElasticIndexing() gate so datasets on a non-indexable GWDM version get deindexed rather than partially indexed, and moves toSearchableArray() field extraction into the handler hierarchy via toSearchableFields(). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1 parent e800c00 commit 7fa229e

8 files changed

Lines changed: 513 additions & 85 deletions

‎app/Http/Traits/IndexElastic.php‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
use App\Models\ToolHasTypeCategory;
3333
use App\Models\TypeCategory;
3434
use App\Services\DatasetService;
35+
use App\Services\Gwdm\GwdmHandlerFactory;
3536
use ElasticClientController as ECC;
3637

3738
trait IndexElastic
@@ -85,6 +86,18 @@ private function reconstructedLatestEnvelope(Dataset $dataset): array
8586
);
8687
}
8788

89+
// Whether this envelope's GWDM version should be indexed into Elasticsearch.
90+
private function isElasticIndexable(array $envelope): bool
91+
{
92+
$version = $envelope['gwdmVersion'] ?? null;
93+
94+
if ($version === null) {
95+
return true; // no version info available — preserve current behaviour
96+
}
97+
98+
return app(GwdmHandlerFactory::class)->resolve($version)->supportsElasticIndexing();
99+
}
100+
88101
/**
89102
* Calls a re-indexing of Elastic search when a dataset is created, updated or added to a collection.
90103
*
@@ -114,6 +127,12 @@ public function reindexElastic(string $datasetId, bool $returnParams = false, bo
114127
}
115128

116129
$metadata = $this->reconstructedLatestEnvelope($datasetMatch);
130+
131+
if (!$this->isElasticIndexable($metadata)) {
132+
$this->deleteDatasetFromElastic($datasetId);
133+
return null;
134+
}
135+
117136
$latestVersion = $datasetMatch->latestVersion();
118137

119138
// inject relationships via Local functions
@@ -333,13 +352,16 @@ public function reindexElasticDataProvider(string $teamId, bool $returnParams =
333352

334353
$latestVersion = $dataset->latestVersion();
335354
if ($latestVersion) {
336-
$datasetVersionIds[] = $latestVersion->id;
337-
$metadata = $latestVersion->metadata;
338-
$datasetTitles[] = $latestVersion->short_title ?? ($metadata['metadata']['summary']['shortTitle'] ?? '');
339-
$types = explode(';,;', $metadata['metadata']['summary']['datasetType']);
340-
foreach ($types as $t) {
341-
if (!in_array($t, $dataTypes)) {
342-
$dataTypes[] = $t;
355+
$metadata = $this->reconstructedLatestEnvelope($dataset);
356+
357+
if ($this->isElasticIndexable($metadata)) {
358+
$datasetVersionIds[] = $latestVersion->id;
359+
$datasetTitles[] = $latestVersion->short_title ?? $this->getValueByPossibleKeys($metadata, ['metadata.summary.shortTitle'], '');
360+
$types = $this->normalizeDelimitedList($this->getValueByPossibleKeys($metadata, ['metadata.summary.datasetType'], ''));
361+
foreach ($types as $t) {
362+
if (!in_array($t, $dataTypes)) {
363+
$dataTypes[] = $t;
364+
}
343365
}
344366
}
345367
}
@@ -505,10 +527,19 @@ public function indexElasticCollections(int $collectionId, bool $returnParams =
505527
$datasetTitles = array();
506528
$datasetAbstracts = array();
507529
foreach ($datasetIds as $d) {
508-
$latestVersion = Dataset::where(['id' => $d])->first()->latestVersion();
509-
$metadata = $latestVersion->metadata;
510-
$datasetTitles[] = $latestVersion->short_title ?? ($metadata['metadata']['summary']['shortTitle'] ?? '');
511-
$datasetAbstracts[] = $metadata['metadata']['summary']['abstract'];
530+
$datasetRow = Dataset::where(['id' => $d])->first();
531+
$latestVersion = $datasetRow?->latestVersion();
532+
if (!$latestVersion) {
533+
continue;
534+
}
535+
536+
$metadata = $this->reconstructedLatestEnvelope($datasetRow);
537+
if (!$this->isElasticIndexable($metadata)) {
538+
continue;
539+
}
540+
541+
$datasetTitles[] = $latestVersion->short_title ?? $this->getValueByPossibleKeys($metadata, ['metadata.summary.shortTitle'], '');
542+
$datasetAbstracts[] = $this->getValueByPossibleKeys($metadata, ['metadata.summary.abstract'], '');
512543
}
513544

514545
$keywords = array();
@@ -1009,7 +1040,10 @@ public function deleteDataProviderCollFromElastic(string $id)
10091040
public function getMaterialTypes(array $metadata): array|null
10101041
{
10111042
$materialTypes = null;
1012-
if (version_compare(Config::get('metadata.GWDM.version'), "2.0", "<")) {
1043+
// Check the reconstructed envelope's own gwdmVersion, not the global config
1044+
// default — a dataset's actual version can diverge from that default.
1045+
$rowGwdmVersion = $metadata['gwdmVersion'] ?? Config::get('metadata.GWDM.version');
1046+
if (version_compare($rowGwdmVersion, "2.0", "<")) {
10131047
$containsTissue = !empty($this->getValueByPossibleKeys($metadata, [
10141048
'metadata.coverage.biologicalsamples',
10151049
'metadata.coverage.physicalSampleAvailability',

‎app/Jobs/ExtractPublicationsFromMetadata.php‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66
use App\Models\Team;
77
use App\Models\User;
88
use App\Models\Dataset;
9+
use App\Models\DatasetVersion;
910
use App\Models\Publication;
11+
use App\Services\DatasetService;
1012
use Illuminate\Support\Arr;
1113
use Illuminate\Bus\Queueable;
1214
use Illuminate\Support\Facades\Http;
@@ -62,17 +64,15 @@ public function publication($datasetVersionId)
6264
$linkagePublicationAboutDataset = 'metadata.linkage.publicationAboutDataset';
6365
$linkagePublicationUsingDataset = 'metadata.linkage.publicationUsingDataset';
6466

65-
$metadata = \DB::table('dataset_versions')
66-
->where('id', $datasetVersionId)
67-
->select('id', 'dataset_id', 'metadata', \DB::raw('JSON_TYPE(metadata) as metadata_type'))
68-
->first();
67+
// Reconstruct via DatasetService — the raw `metadata` column is empty on delta rows.
68+
$dv = DatasetVersion::where('id', $datasetVersionId)->first();
6969

70-
if (is_null($metadata)) {
70+
if (is_null($dv)) {
7171
\Log::warning('ExtractPublicationsFromMetadata :: Metadata not found.', $this->loggingContext);
7272
return;
7373
}
7474

75-
$dataset = Dataset::where('id', $metadata->dataset_id)->select(['id', 'user_id', 'team_id'])->first();
75+
$dataset = Dataset::where('id', $dv->dataset_id)->select(['id', 'user_id', 'team_id'])->first();
7676
if (is_null($dataset)) {
7777
\Log::warning('ExtractPublicationsFromMetadata :: Dataset not found.', $this->loggingContext);
7878
return;
@@ -93,16 +93,9 @@ public function publication($datasetVersionId)
9393
$datasetUserId = (int) $dataset->user_id;
9494
$datasetTeamId = (int) $dataset->team_id;
9595

96-
$data = null;
97-
if ($metadata->metadata_type === 'OBJECT') {
98-
$data = json_decode($metadata->metadata, true);
99-
}
100-
101-
if ($metadata->metadata_type === 'STRING') {
102-
$data = json_decode(json_decode($metadata->metadata), true);
103-
}
96+
$data = app(DatasetService::class)->getReconstructedMetadataEnvelope($dv->dataset_id, $dv->version, false, $dv);
10497

105-
if (count($data ?: []) === 0) {
98+
if (empty($data['metadata'])) {
10699
return;
107100
}
108101

‎app/Jobs/ExtractToolsFromMetadata.php‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
use App\Models\Tool;
77
use App\Models\User;
88
use App\Models\Dataset;
9+
use App\Models\DatasetVersion;
10+
use App\Services\DatasetService;
911
use Illuminate\Support\Arr;
1012
use Illuminate\Bus\Queueable;
1113
use App\Http\Traits\IndexElastic;
@@ -59,17 +61,15 @@ public function tool(int $datasetVersionId)
5961
$linkageToolsDataset = 'metadata.linkage.tools';
6062
$type = 'Used on';
6163

62-
$metadata = \DB::table('dataset_versions')
63-
->where('id', $datasetVersionId)
64-
->select('id', 'dataset_id', 'metadata', \DB::raw('JSON_TYPE(metadata) as metadata_type'))
65-
->first();
64+
// Reconstruct via DatasetService — the raw `metadata` column is empty on delta rows.
65+
$dv = DatasetVersion::where('id', $datasetVersionId)->first();
6666

67-
if (is_null($metadata)) {
67+
if (is_null($dv)) {
6868
\Log::info('ExtractToolsFromMetadata :: Metadata not found.', $this->loggingContext);
6969
return;
7070
}
7171

72-
$dataset = Dataset::where('id', $metadata->dataset_id)->select(['id', 'user_id', 'team_id'])->first();
72+
$dataset = Dataset::where('id', $dv->dataset_id)->select(['id', 'user_id', 'team_id'])->first();
7373
if (is_null($dataset)) {
7474
\Log::info('ExtractToolsFromMetadata :: Dataset not found.', $this->loggingContext);
7575
return;
@@ -92,16 +92,9 @@ public function tool(int $datasetVersionId)
9292

9393
$this->cleanToolDatasetVersion($datasetVersionId, $type, (int)$dataset->id);
9494

95-
$data = null;
96-
if ($metadata->metadata_type === 'OBJECT') {
97-
$data = json_decode($metadata->metadata, true);
98-
}
99-
100-
if ($metadata->metadata_type === 'STRING') {
101-
$data = json_decode(json_decode($metadata->metadata), true);
102-
}
95+
$data = app(DatasetService::class)->getReconstructedMetadataEnvelope($dv->dataset_id, $dv->version, false, $dv);
10396

104-
if (count($data ?: []) === 0) {
97+
if (empty($data['metadata'])) {
10598
return;
10699
}
107100

‎app/Models/DatasetVersion.php‎

Lines changed: 19 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
namespace App\Models;
44

55
use App\Models\Base\BaseTypesenseModel;
6+
use App\Services\DatasetService;
7+
use App\Services\Gwdm\GwdmHandlerFactory;
68
use Illuminate\Database\Eloquent\Model;
79
use App\Observers\DatasetVersionObserver;
810
use Illuminate\Database\Eloquent\Builder;
@@ -283,50 +285,24 @@ public function shouldBeSearchable(): bool
283285

284286
public function toSearchableArray(): array
285287
{
286-
$meta = $this->metadata ?? [];
287-
288-
$keywords = data_get($meta, 'metadata.summary.keywords', '');
289-
$dataType = data_get($meta, 'metadata.summary.datasetType', '');
290-
$conformsTo = data_get($meta, 'metadata.accessibility.formatAndStandards.conformsTo', '');
291-
$structural = data_get($meta, 'metadata.structuralMetadata', []);
292-
if (is_string($structural)) {
293-
$structural = json_decode($structural, true) ?? [];
294-
}
295-
if (!is_array($structural)) {
296-
$structural = [];
297-
}
288+
$envelope = app(DatasetService::class)->getReconstructedMetadataEnvelope(
289+
$this->dataset_id,
290+
$this->version,
291+
false,
292+
$this,
293+
);
298294

299-
return [
300-
'id' => (string) $this->id,
301-
'dataset_id' => (string) $this->dataset_id,
302-
'title' => $this->title ?? '',
303-
'shortTitle' => $this->short_title ?? '',
304-
'abstract' => data_get($meta, 'metadata.summary.abstract', ''),
305-
'description' => data_get($meta, 'metadata.summary.description', ''),
306-
'keywords' => array_values(array_filter(explode(';,;', $keywords))),
307-
'publisherName' => data_get(
308-
$meta,
309-
'metadata.summary.publisher.name',
310-
data_get($meta, 'metadata.summary.publisher.publisherName', '')
311-
),
312-
'dataType' => array_values(array_filter(explode(';,;', $dataType))),
313-
'populationSize' => (int) data_get($meta, 'metadata.summary.populationSize', -1),
314-
'geographicLocation' => $this->spatialCoverage->pluck('region')->all(),
315-
'datasetDOI' => data_get($meta, 'metadata.summary.doiName', ''),
316-
'conformsTo' => array_values(array_filter(explode(';,;', $conformsTo))),
317-
'structuralTableNames' => collect($structural)
318-
->pluck('name')
319-
->filter(fn ($v) => is_string($v) && $v !== '')
320-
->values()->all(),
321-
'structuralColumnNames' => collect($structural)
322-
->flatMap(fn ($t) => collect($t['columns'] ?? [])->pluck('name'))
323-
->filter(fn ($v) => is_string($v) && $v !== '')
324-
->values()->all(),
325-
'structuralColumnDescriptions' => collect($structural)
326-
->flatMap(fn ($t) => collect($t['columns'] ?? [])->pluck('description'))
327-
->filter(fn ($v) => is_string($v) && $v !== '')
328-
->values()->all(),
329-
];
295+
$fields = app(GwdmHandlerFactory::class)
296+
->resolve($envelope['gwdmVersion'] ?? $this->gwdm_version ?? '2.0')
297+
->toSearchableFields($envelope);
298+
299+
return array_merge($fields, [
300+
'id' => (string) $this->id,
301+
'dataset_id' => (string) $this->dataset_id,
302+
'title' => $this->title ?? '',
303+
'shortTitle' => $this->short_title ?? '',
304+
'geographicLocation' => $this->spatialCoverage->pluck('region')->all(),
305+
]);
330306
}
331307

332308
public function makeAllSearchableUsing(Builder $query): Builder

‎app/Services/Gwdm/GwdmMetadataHandler.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,60 @@ public function extractLinkages(DatasetVersion $dv): void
189189
// no-op for versions without linkage extraction support
190190
}
191191

192+
/**
193+
* Whether datasets on this GWDM version should be indexed into Elasticsearch.
194+
*/
195+
public function supportsElasticIndexing(): bool
196+
{
197+
return true;
198+
}
199+
200+
/**
201+
* Extract Typesense-searchable fields from a reconstructed GWDM envelope.
202+
* Override when a version's field shapes diverge from delimited strings.
203+
*/
204+
public function toSearchableFields(array $envelope): array
205+
{
206+
$keywords = data_get($envelope, 'metadata.summary.keywords', '');
207+
$dataType = data_get($envelope, 'metadata.summary.datasetType', '');
208+
$conformsTo = data_get($envelope, 'metadata.accessibility.formatAndStandards.conformsTo', '');
209+
$structural = data_get($envelope, 'metadata.structuralMetadata', []);
210+
211+
if (is_string($structural)) {
212+
$structural = json_decode($structural, true) ?? [];
213+
}
214+
if (!is_array($structural)) {
215+
$structural = [];
216+
}
217+
218+
return [
219+
'abstract' => data_get($envelope, 'metadata.summary.abstract', ''),
220+
'description' => data_get($envelope, 'metadata.summary.description', ''),
221+
'keywords' => array_values(array_filter(explode(';,;', $keywords))),
222+
'publisherName' => data_get(
223+
$envelope,
224+
'metadata.summary.publisher.name',
225+
data_get($envelope, 'metadata.summary.publisher.publisherName', '')
226+
),
227+
'dataType' => array_values(array_filter(explode(';,;', $dataType))),
228+
'populationSize' => (int) data_get($envelope, 'metadata.summary.populationSize', -1),
229+
'datasetDOI' => data_get($envelope, 'metadata.summary.doiName', ''),
230+
'conformsTo' => array_values(array_filter(explode(';,;', $conformsTo))),
231+
'structuralTableNames' => collect($structural)
232+
->pluck('name')
233+
->filter(fn ($v) => is_string($v) && $v !== '')
234+
->values()->all(),
235+
'structuralColumnNames' => collect($structural)
236+
->flatMap(fn ($t) => collect($t['columns'] ?? [])->pluck('name'))
237+
->filter(fn ($v) => is_string($v) && $v !== '')
238+
->values()->all(),
239+
'structuralColumnDescriptions' => collect($structural)
240+
->flatMap(fn ($t) => collect($t['columns'] ?? [])->pluck('description'))
241+
->filter(fn ($v) => is_string($v) && $v !== '')
242+
->values()->all(),
243+
];
244+
}
245+
192246
// ── Onboarding form defaults ──────────────────────────────────────────────
193247

194248
/**

0 commit comments

Comments
 (0)