feat(GAT-9018): free-text linkage fallback + delta-row indexing fixes

Squashed net contribution of feat/GAT-9018-2-2 over dev (which already
carries the GWDM version-context + handler hierarchy from #1699/#1704).

- Free-text linkage fallback: persist raw_url/raw_pid/raw_title/raw_doi for
  unresolved dataset/publication linkages (new add_unresolved_linkage_columns
  migration) and reconstruct them on read. Layered onto dev's resolveDataset/
  PublicationLinkages helpers so resolved-title consistency and partial-update
  handling are preserved; getLinkages now lives on Gwdm2xHandler.
- IndexElastic + ExtractTools/PublicationsFromMetadata: correct delta-row
  handling so reduced (patch) versions reconstruct the full GWDM envelope
  before indexing/extraction instead of reading the stripped metadata column.
- Gate Elasticsearch indexing per GWDM version via handler->supportsElasticIndexing()
  and toSearchableFields().
- DatasetService: add getMetadataOnly() fast path and validateAllVersions();
  spatial-coverage mapping now prunes stale pivot rows on edit-down.
- Tests: linkage free-text roundtrip, delta-row index/extract, GWDM index
  gating, and spatial-coverage pruning.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: calmacx

app/Http/Controllers/Api/V2/DatasetController.php

@@ -187,6 +187,15 @@ public function showActive(GetDataset $request, int $id): JsonResponse|BinaryFil
                 return $this->streamStructuralMetadataExport($dataset, $id);
             }
 
+            if ($request->query('view') === 'metadataOnly') {
+                $metadata = $this->datasetService->getMetadataOnly(
+                    $dataset,
+                    $request->query('schema_model') ?? $this->outputSchemaContext->schemaModel(),
+                    $request->query('schema_version') ?? $this->outputSchemaContext->schemaVersion(),
+                );
+                return response()->json(['message' => 'success', 'data' => $metadata]);
+            }
+
             $dataset = $this->datasetService->prepareForShow(
                 $dataset,
                 $request->query('schema_model') ?? $this->outputSchemaContext->schemaModel(),
@@ -209,7 +218,7 @@ public function showActive(GetDataset $request, int $id): JsonResponse|BinaryFil
 
         } catch (\InvalidArgumentException $e) {
             return response()->json(['message' => $e->getMessage()], 400);
-        } catch (ModelNotFoundException $e) {
+        } catch (\Illuminate\Database\Eloquent\ModelNotFoundException $e) {
             return response()->json(['message' => $e->getMessage()], 404);
         } catch (\RuntimeException $e) {
             return response()->json(['message' => 'failed to translate', 'details' => $e->getMessage()], 400);

app/Http/Traits/IndexElastic.php

@@ -31,6 +31,8 @@
 use App\Models\ToolHasTag;
 use App\Models\ToolHasTypeCategory;
 use App\Models\TypeCategory;
+use App\Services\DatasetService;
+use App\Services\Gwdm\GwdmHandlerFactory;
 use ElasticClientController as ECC;
 
 trait IndexElastic
@@ -43,6 +45,59 @@ trait IndexElastic
     private $publications = [];
     private $collections = [];
 
+    /**
+     * Single named resolution point for DatasetService within this trait.
+     *
+     * Constructor injection isn't viable here: IndexElastic is a trait consumed
+     * by ~20 unrelated controllers/jobs/observers, so there is no single
+     * constructor to inject into without a much larger refactor.
+     */
+    private function datasetService(): DatasetService
+    {
+        return app(DatasetService::class);
+    }
+
+    /**
+     * Reconstruct the full GWDM envelope for a dataset's latest version.
+     *
+     * DatasetVersion->metadata cannot be read directly for indexing: delta rows
+     * store an empty metadata blob, and GWDM 3.0 rows omit the `metadata` key
+     * entirely (their data lives in dedicated SQL tables). Funnelling through
+     * DatasetService::getReconstructedMetadataEnvelope() yields a complete
+     * {gwdmVersion, metadata, original_metadata} envelope for every schema
+     * version and every snapshot/delta storage shape. Validation is skipped —
+     * data is validated at write time.
+     *
+     * @return array The reconstructed envelope, or [] when the dataset has no versions.
+     */
+    private function reconstructedLatestEnvelope(Dataset $dataset): array
+    {
+        $version = $dataset->latestVersion();
+
+        if (!$version) {
+            return [];
+        }
+
+        return $this->datasetService()->getReconstructedMetadataEnvelope(
+            $dataset->id,
+            $version->version,
+            false,
+            $version,
+        );
+    }
+
+    // Whether this envelope's GWDM version should be indexed into Elasticsearch.
+    private function isElasticIndexable(array $envelope): bool
+    {
+        $version = $envelope['gwdmVersion'] ?? null;
+
+        if ($version === null) {
+            return true; // no version info available — preserve current behaviour
+        }
+
+        return app(GwdmHandlerFactory::class)->resolve($version)->supportsElasticIndexing();
+    }
+
     /**
      * Calls a re-indexing of Elastic search when a dataset is created, updated or added to a collection.
      *
@@ -71,7 +126,14 @@ public function reindexElastic(string $datasetId, bool $returnParams = false, bo
                 // throw new \Exception("Error: DatasetVersion is missing for dataset ID=$datasetId.");
             }
 
-            $metadata = $datasetMatch->latestVersion()->metadata;
+            $metadata = $this->reconstructedLatestEnvelope($datasetMatch);
+
+            if (!$this->isElasticIndexable($metadata)) {
+                $this->deleteDatasetFromElastic($datasetId);
+                return null;
+            }
+
+            $latestVersion = $datasetMatch->latestVersion();
 
             // inject relationships via Local functions
             $materialTypes = $this->getMaterialTypes($metadata);
@@ -82,8 +144,10 @@ public function reindexElastic(string $datasetId, bool $returnParams = false, bo
                 'abstract' => $this->getValueByPossibleKeys($metadata, ['metadata.summary.abstract'], ''),
                 'keywords' => $this->normalizeDelimitedList($this->getValueByPossibleKeys($metadata, ['metadata.summary.keywords'], '')),
                 'description' => $this->getValueByPossibleKeys($metadata, ['metadata.summary.description'], ''),
-                'shortTitle' => $this->getValueByPossibleKeys($metadata, ['metadata.summary.shortTitle'], ''),
-                'title' => $this->getValueByPossibleKeys($metadata, ['metadata.summary.title'], ''),
+                // short_title/title are populated columns on every write (see DatasetVersion);
+                // only fall back to the envelope for legacy pre-migration rows where they're null.
+                'shortTitle' => $latestVersion->short_title ?? $this->getValueByPossibleKeys($metadata, ['metadata.summary.shortTitle'], ''),
+                'title' => $latestVersion->title ?? $this->getValueByPossibleKeys($metadata, ['metadata.summary.title'], ''),
                 'populationSize' => $this->getValueByPossibleKeys($metadata, ['metadata.summary.populationSize'], -1),
                 'publisherName' => $this->getValueByPossibleKeys($metadata, ['metadata.summary.publisher.name', 'metadata.summary.publisher.publisherName'], ''),
                 'startDate' => $this->getValueByPossibleKeys($metadata, ['metadata.provenance.temporal.startDate'], null),
@@ -288,13 +352,16 @@ public function reindexElasticDataProvider(string $teamId, bool $returnParams =
 
                 $latestVersion = $dataset->latestVersion();
                 if ($latestVersion) {
-                    $datasetVersionIds[] = $latestVersion->id;
-                    $metadata = $latestVersion->metadata;
-                    $datasetTitles[] = $metadata['metadata']['summary']['shortTitle'];
-                    $types = explode(';,;', $metadata['metadata']['summary']['datasetType']);
-                    foreach ($types as $t) {
-                        if (!in_array($t, $dataTypes)) {
-                            $dataTypes[] = $t;
+                    $metadata = $this->reconstructedLatestEnvelope($dataset);
+
+                    if ($this->isElasticIndexable($metadata)) {
+                        $datasetVersionIds[] = $latestVersion->id;
+                        $datasetTitles[] = $latestVersion->short_title ?? $this->getValueByPossibleKeys($metadata, ['metadata.summary.shortTitle'], '');
+                        $types = $this->normalizeDelimitedList($this->getValueByPossibleKeys($metadata, ['metadata.summary.datasetType'], ''));
+                        foreach ($types as $t) {
+                            if (!in_array($t, $dataTypes)) {
+                                $dataTypes[] = $t;
+                            }
                         }
                     }
                 }
@@ -460,12 +527,19 @@ public function indexElasticCollections(int $collectionId, bool $returnParams =
         $datasetTitles = array();
         $datasetAbstracts = array();
         foreach ($datasetIds as $d) {
-            $metadata = Dataset::where(['id' => $d])
-                ->first()
-                ->latestVersion()
-                ->metadata;
-            $datasetTitles[] = $metadata['metadata']['summary']['shortTitle'];
-            $datasetAbstracts[] = $metadata['metadata']['summary']['abstract'];
+            $datasetRow = Dataset::where(['id' => $d])->first();
+            $latestVersion = $datasetRow?->latestVersion();
+            if (!$latestVersion) {
+                continue;
+            }
+
+            $metadata = $this->reconstructedLatestEnvelope($datasetRow);
+            if (!$this->isElasticIndexable($metadata)) {
+                continue;
+            }
+
+            $datasetTitles[] = $latestVersion->short_title ?? $this->getValueByPossibleKeys($metadata, ['metadata.summary.shortTitle'], '');
+            $datasetAbstracts[] = $this->getValueByPossibleKeys($metadata, ['metadata.summary.abstract'], '');
         }
 
         $keywords = array();
@@ -535,8 +609,7 @@ private function indexElasticDataProviderColl(int $id): void
 
             foreach ($datasets as $dataset) {
                 $dataset->setAttribute('spatialCoverage', $dataset->allSpatialCoverages  ?? []);
-                $metadata = $dataset['versions'][0];
-                $datasetTitles[] = $metadata['metadata']['metadata']['summary']['shortTitle'];
+                $datasetTitles[] = $dataset->latestVersion()?->short_title ?? '';
                 foreach ($dataset['spatialCoverage'] as $loc) {
                     if (!in_array($loc['region'], $locations)) {
                         $locations[] = $loc['region'];
@@ -598,11 +671,7 @@ public function indexElasticDur(string $id, bool $returnParams = false): null|ar
 
             $datasetTitles = array();
             foreach ($datasetIds as $d) {
-                $metadata = Dataset::where(['id' => $d])
-                    ->first()
-                    ->latestVersion()
-                    ->metadata;
-                $datasetTitles[] = $metadata['metadata']['summary']['shortTitle'];
+                $datasetTitles[] = Dataset::where(['id' => $d])->first()->latestVersion()->short_title ?? '';
             }
 
             $keywords = array();
@@ -698,9 +767,10 @@ public function indexElasticPublication(string $id, bool $returnParams = false):
             ];
 
             foreach ($datasets as $dataset) {
-                $metadata = Dataset::where(['id' => $dataset['id']])->first()->latestVersion()->metadata;
-                $latestVersionID = Dataset::where(['id' => $dataset['id']])->first()->latestVersion()->id;
-                $datasetTitles[] = $metadata['metadata']['summary']['shortTitle'];
+                $datasetModel = Dataset::where(['id' => $dataset['id']])->first();
+                $latestVersion = $datasetModel->latestVersion();
+                $latestVersionID = $latestVersion->id;
+                $datasetTitles[] = $latestVersion->short_title ?? '';
 
                 // change from 'UNKNOWN' to 'USING' - temp
                 $linkType = PublicationHasDatasetVersion::where([
@@ -838,8 +908,7 @@ public function indexElasticTools(int $toolId, bool $returnParams = false): null
                 $datasetTitles[] = '_Can be used with any dataset';
             } else {
                 foreach ($datasets as $dataset) {
-                    $dataset_version = $dataset['versions'][0];
-                    $datasetTitles[] = $dataset_version['metadata']['metadata']['summary']['shortTitle'];
+                    $datasetTitles[] = $dataset->latestVersion()?->short_title ?? '';
                 }
                 usort($datasetTitles, 'strcasecmp');
             }
@@ -971,7 +1040,10 @@ public function deleteDataProviderCollFromElastic(string $id)
     public function getMaterialTypes(array $metadata): array|null
     {
         $materialTypes = null;
-        if (version_compare(Config::get('metadata.GWDM.version'), "2.0", "<")) {
+        // Check the reconstructed envelope's own gwdmVersion, not the global config
+        // default — a dataset's actual version can diverge from that default.
+        $rowGwdmVersion = $metadata['gwdmVersion'] ?? Config::get('metadata.GWDM.version');
+        if (version_compare($rowGwdmVersion, "2.0", "<")) {
             $containsTissue = !empty($this->getValueByPossibleKeys($metadata, [
                 'metadata.coverage.biologicalsamples',
                 'metadata.coverage.physicalSampleAvailability',
@@ -1110,16 +1182,12 @@ public function checkingDataset(int $datasetId)
         $publicationIds = array_column($dataset->allActivePublications, 'id') ?? [];
         $toolIds = array_column($dataset->allActiveTools, 'id') ?? [];
 
-        $version = $dataset->latestVersion();
-        $withLinks = DatasetVersion::where('id', $version['id'])
-            ->with(['linkedDatasetVersions'])
-            ->first();
-
-        $dataset->setAttribute('versions', [$withLinks]);
-
-        $metadataSummary = $dataset['versions'][0]['metadata']['metadata']['summary'] ?? [];
+        $metadataSummary = $this->reconstructedLatestEnvelope($dataset)['metadata']['summary'] ?? [];
 
-        $title = $this->getValueByPossibleKeys($metadataSummary, ['title'], '');
+        // title/short_title are populated columns on every write; only fall back
+        // to the reconstructed envelope for legacy pre-migration rows.
+        $title = $dataset->latestVersion()?->title
+            ?? $this->getValueByPossibleKeys($metadataSummary, ['title'], '');
         $populationSize = $this->getValueByPossibleKeys($metadataSummary, ['populationSize'], -1);
         $datasetType = $this->getValueByPossibleKeys($metadataSummary, ['datasetType'], '');
 

app/Jobs/ExtractPublicationsFromMetadata.php

@@ -6,7 +6,9 @@
 use App\Models\Team;
 use App\Models\User;
 use App\Models\Dataset;
+use App\Models\DatasetVersion;
 use App\Models\Publication;
+use App\Services\DatasetService;
 use Illuminate\Support\Arr;
 use Illuminate\Bus\Queueable;
 use Illuminate\Support\Facades\Http;
@@ -62,17 +64,15 @@ public function publication($datasetVersionId)
         $linkagePublicationAboutDataset = 'metadata.linkage.publicationAboutDataset';
         $linkagePublicationUsingDataset = 'metadata.linkage.publicationUsingDataset';
 
-        $metadata = \DB::table('dataset_versions')
-            ->where('id', $datasetVersionId)
-            ->select('id', 'dataset_id', 'metadata', \DB::raw('JSON_TYPE(metadata) as metadata_type'))
-            ->first();
+        // Reconstruct via DatasetService — the raw `metadata` column is empty on delta rows.
+        $dv = DatasetVersion::where('id', $datasetVersionId)->first();
 
-        if (is_null($metadata)) {
+        if (is_null($dv)) {
             \Log::warning('ExtractPublicationsFromMetadata :: Metadata not found.', $this->loggingContext);
             return;
         }
 
-        $dataset = Dataset::where('id', $metadata->dataset_id)->select(['id', 'user_id', 'team_id'])->first();
+        $dataset = Dataset::where('id', $dv->dataset_id)->select(['id', 'user_id', 'team_id'])->first();
         if (is_null($dataset)) {
             \Log::warning('ExtractPublicationsFromMetadata :: Dataset not found.', $this->loggingContext);
             return;
@@ -93,16 +93,9 @@ public function publication($datasetVersionId)
         $datasetUserId = (int) $dataset->user_id;
         $datasetTeamId = (int) $dataset->team_id;
 
-        $data = null;
-        if ($metadata->metadata_type === 'OBJECT') {
-            $data = json_decode($metadata->metadata, true);
-        }
-
-        if ($metadata->metadata_type === 'STRING') {
-            $data = json_decode(json_decode($metadata->metadata), true);
-        }
+        $data = app(DatasetService::class)->getReconstructedMetadataEnvelope($dv->dataset_id, $dv->version, false, $dv);
 
-        if (count($data ?: []) === 0) {
+        if (empty($data['metadata'])) {
             return;
         }
 

app/Jobs/ExtractToolsFromMetadata.php

@@ -6,6 +6,8 @@
 use App\Models\Tool;
 use App\Models\User;
 use App\Models\Dataset;
+use App\Models\DatasetVersion;
+use App\Services\DatasetService;
 use Illuminate\Support\Arr;
 use Illuminate\Bus\Queueable;
 use App\Http\Traits\IndexElastic;
@@ -59,17 +61,15 @@ public function tool(int $datasetVersionId)
         $linkageToolsDataset = 'metadata.linkage.tools';
         $type = 'Used on';
 
-        $metadata = \DB::table('dataset_versions')
-            ->where('id', $datasetVersionId)
-            ->select('id', 'dataset_id', 'metadata', \DB::raw('JSON_TYPE(metadata) as metadata_type'))
-            ->first();
+        // Reconstruct via DatasetService — the raw `metadata` column is empty on delta rows.
+        $dv = DatasetVersion::where('id', $datasetVersionId)->first();
 
-        if (is_null($metadata)) {
+        if (is_null($dv)) {
             \Log::info('ExtractToolsFromMetadata :: Metadata not found.', $this->loggingContext);
             return;
         }
 
-        $dataset = Dataset::where('id', $metadata->dataset_id)->select(['id', 'user_id', 'team_id'])->first();
+        $dataset = Dataset::where('id', $dv->dataset_id)->select(['id', 'user_id', 'team_id'])->first();
         if (is_null($dataset)) {
             \Log::info('ExtractToolsFromMetadata :: Dataset not found.', $this->loggingContext);
             return;
@@ -92,16 +92,9 @@ public function tool(int $datasetVersionId)
 
         $this->cleanToolDatasetVersion($datasetVersionId, $type, (int)$dataset->id);
 
-        $data = null;
-        if ($metadata->metadata_type === 'OBJECT') {
-            $data = json_decode($metadata->metadata, true);
-        }
-
-        if ($metadata->metadata_type === 'STRING') {
-            $data = json_decode(json_decode($metadata->metadata), true);
-        }
+        $data = app(DatasetService::class)->getReconstructedMetadataEnvelope($dv->dataset_id, $dv->version, false, $dv);
 
-        if (count($data ?: []) === 0) {
+        if (empty($data['metadata'])) {
             return;
         }
 

app/MetadataManagementController/MetadataManagementController.php

@@ -171,6 +171,39 @@ public function validateDataModelType(string &$dataset, string $input_schema, st
         }
     }
 
+    /**
+     * Validate a metadata JSON string against the given schema and version, returning
+     * structured error details from TRASER rather than a plain boolean.
+     *
+     * @return array{valid: bool, errors: mixed}
+     */
+    public function validateDataModelTypeWithErrors(string &$dataset, string $input_schema, string $input_version): array
+    {
+        $loggingContext = $this->getLoggingContext(\request());
+        $loggingContext['method_name'] = class_basename($this) . '@' . __FUNCTION__;
+
+        try {
+            $urlString = sprintf(
+                '%s/validate?input_schema=%s&input_version=%s',
+                config('services.traser.url'),
+                $input_schema,
+                $input_version
+            );
+
+            $response = Http::withBody($dataset, 'application/json')->post($urlString);
+
+            $valid = $response->status() === 200;
+
+            return [
+                'valid'  => $valid,
+                'errors' => $valid ? null : $response->json(),
+            ];
+        } catch (Exception $e) {
+            \Log::info($e->getMessage(), $loggingContext);
+            throw new MMCException($e->getMessage());
+        }
+    }
+
     /**
      * Finds the matching data model schema for a given dataset by querying the TRASER service.
      *