diff --git a/app/Http/Controllers/Api/V1/DatasetController.php b/app/Http/Controllers/Api/V1/DatasetController.php index a030c2f16..26e551ccc 100644 --- a/app/Http/Controllers/Api/V1/DatasetController.php +++ b/app/Http/Controllers/Api/V1/DatasetController.php @@ -10,6 +10,7 @@ use App\Models\User; use App\Models\Dataset; use App\Jobs\TermExtraction; +use App\Jobs\DatasetSqlLinkageJob; use Illuminate\Http\Request; use App\Models\DatasetVersion; use App\Http\Traits\CheckAccess; @@ -18,7 +19,6 @@ use Illuminate\Http\JsonResponse; use App\Http\Controllers\Controller; - use App\Http\Traits\MetadataOnboard; use Maatwebsite\Excel\Facades\Excel; use App\Http\Traits\AddMetadataVersion; @@ -43,6 +43,7 @@ class DatasetController extends Controller use MetadataOnboard; use CheckAccess; + /** * @OA\Get( * path="/api/v1/datasets", @@ -418,12 +419,21 @@ public function show(GetDataset $request, int $id): JsonResponse|BinaryFileRespo // - For the FE i just need a tools linkage count so i'm gonna return `count(dataset->allTools)` for now // - Same for collections // - Leaving this as it is as im not 100% sure what any FE knock-on effect would be + + // notes Tom 11th Oct 2024... + // - this is where we could be injecting these values into the "extra" array prior to validation + // - We can restrict this to IDs using the following logic + // - $toolIds = implode(', ', array_column($dataset->allTools, 'id')); + // - however, this could be built into the function + // - getRelationsViaDatasetVersion in the Dataset Model + $dataset->setAttribute('durs_count', $dataset->latestVersion()->durHasDatasetVersions()->count()); $dataset->setAttribute('publications_count', $dataset->latestVersion()->publicationHasDatasetVersions()->count()); $dataset->setAttribute('tools_count', count($dataset->allTools)); $dataset->setAttribute('collections_count', count($dataset->allCollections)); $dataset->setAttribute('spatialCoverage', $dataset->allSpatialCoverages ?? []); $dataset->setAttribute('durs', $dataset->allDurs ?? []); + $dataset->setAttribute('tools', $dataset->allTools ?? []); $dataset->setAttribute('publications', $dataset->allPublications ?? []); $dataset->setAttribute('named_entities', $dataset->allNamedEntities ?? []); $dataset->setAttribute('collections', $dataset->allCollections ?? []); @@ -763,6 +773,16 @@ public function update(UpdateDataset $request, int $id) ); $versionNumber = $currDataset->lastMetadataVersionNumber()->version; + + // Dispatch the dataset to update the SQL linkages tables + // DatasetVersionHasTool + // DatasetVersionHasDatasetVersion + // PublicationHasDatasetVersion + // DatasetVersionHasSpatialCoverage + // This uses the input metadata and new dataset version to create the indexes. + //$this->createSqlLinkageFromDataset($input['metadata'], $currDataset, false); + DatasetSqlLinkageJob::dispatch($input['metadata'], $currDataset, false); + // Dispatch term extraction to a subprocess if the dataset moves from draft to active if($request['status'] === Dataset::STATUS_ACTIVE && Config::get('ted.enabled')) { diff --git a/app/Http/Traits/GetValueByPossibleKeys.php b/app/Http/Traits/GetValueByPossibleKeys.php index fbffc8bd5..b2b9d04a7 100644 --- a/app/Http/Traits/GetValueByPossibleKeys.php +++ b/app/Http/Traits/GetValueByPossibleKeys.php @@ -8,26 +8,117 @@ trait GetValueByPossibleKeys { /** - * Search for a value in an array by trying multiple possible keys in order. + * Search for a value in an array by trying multiple possible keys or paths in order. + * + * Note 15th Oct 2024 - Tom Giles - I have now enhanced this function for full searching: + * The function now supports both simple keys and dot notation paths. for example ['tools'] + * or ['linkages.tools'] or even ['metadata.linkages.tools'] will now return the ['tools'] + * object. Previously the full path in the array was requried. I have done this because + * widening the potential search parathenisis provides more flexible in terms maintaining + * functionality across data model changes. * * @param array $array The array to search. - * @param array $keys The list of possible keys to try, in order. + * @param array $keys The list of possible keys or paths to try, in order. * @param mixed $default The default value to return if none of the keys are found. * @return mixed The value of the first key found, or the default value if none are found. */ public function getValueByPossibleKeys(array $array, array $keys, $default = null) { foreach ($keys as $key) { - $value = Arr::get($array, $key, null); - if (!is_null($value)) { - return $value; + if (strpos($key, '.') !== false) { + // Treat key as a dot notation path + $value = $this->findByPath($array, $key); + if (!is_null($value)) { + return $value; + } + } else { + // Treat key as a simple key and search recursively + $value = $this->recursiveSearch($array, $key); + if (!is_null($value)) { + return $value; + } } } - // Removing, as it doesn't really serve a purpose dumping this to stdout. - // Suggest, if we care about it, then we audit it instead. + + // Optionally log if no keys are found // Log::info('No value found for any of the specified keys', [ // 'keys' => $keys, // ]); + return $default; } -} + + /** + * Recursively search for a key in a multi-dimensional array. + * + * @param array $array The array to search. + * @param string $key The key to search for. + * @return mixed|null The value if found, or null otherwise. + */ + protected function recursiveSearch(array $array, string $key) + { + foreach ($array as $k => $v) { + if ($k === $key) { + return $v; + } + + if (is_array($v)) { + $result = $this->recursiveSearch($v, $key); + if (!is_null($result)) { + return $result; + } + } + } + return null; + } + + /** + * Find a value in a multi-dimensional array based on a dot notation path. + * This function searches through all possible branches to find the path. + * + * @param array $array The array to search. + * @param string $path The dot notation path to search for. + * @return mixed|null The value if found, or null otherwise. + */ + protected function findByPath(array $array, string $path) + { + $segments = explode('.', $path); + + return $this->recursivePathSearch($array, $segments); + } + + /** + * Helper function to perform recursive path search. + * + * @param array $array The current array segment. + * @param array $segments The remaining path segments to search. + * @return mixed|null The value if found, or null otherwise. + */ + protected function recursivePathSearch(array $array, array $segments) + { + $currentSegment = array_shift($segments); + + if (array_key_exists($currentSegment, $array)) { + $value = $array[$currentSegment]; + if (empty($segments)) { + return $value; + } + + if (is_array($value)) { + return $this->recursivePathSearch($value, $segments); + } + } + + // If the current segment is not found, search deeper in the array + foreach ($array as $key => $subArray) { + if (is_array($subArray)) { + $result = $this->recursivePathSearch($subArray, array_merge([$currentSegment], $segments)); + if (!is_null($result)) { + return $result; + } + } + } + + return null; + } +} \ No newline at end of file diff --git a/app/Http/Traits/MetadataOnboard.php b/app/Http/Traits/MetadataOnboard.php index 92870fae0..78a3213ad 100644 --- a/app/Http/Traits/MetadataOnboard.php +++ b/app/Http/Traits/MetadataOnboard.php @@ -6,13 +6,10 @@ use Exception; use App\Models\Dataset; -use App\Models\DatasetVersion; -use App\Models\DatasetVersionHasSpatialCoverage; -use App\Models\SpatialCoverage; +use App\Jobs\DatasetSqlLinkageJob; use App\Jobs\TermExtraction; use Illuminate\Support\Str; - use MetadataManagementController as MMC; trait MetadataOnboard @@ -147,9 +144,15 @@ public function metadataOnboard( 'metadata' => json_encode($input['metadata']), 'version' => 1, ]); - - // map coverage/spatial field to controlled list for filtering - $this->mapCoverage($input['metadata'], $version); + + // Dispatch the dataset to update the SQL linkages tables + // DatasetVersionHasTool + // DatasetVersionHasDatasetVersion + // PublicationHasDatasetVersion + // DatasetVersionHasSpatialCoverage + // This uses the input metadata and new dataset version to create the indexes. + //$this->createSqlLinkageFromDataset($input['metadata'], $dataset, false); + DatasetSqlLinkageJob::dispatch($input['metadata'], $dataset, false); // Dispatch term extraction to a subprocess if the dataset is marked as active if($input['status'] === Dataset::STATUS_ACTIVE && Config::get('ted.enabled')) { @@ -180,45 +183,6 @@ public function metadataOnboard( } - private function mapCoverage(array $metadata, DatasetVersion $version): void - { - if (!isset($metadata['metadata']['coverage']['spatial'])) { - return; - } - - $coverage = strtolower($metadata['metadata']['coverage']['spatial']); - $ukCoverages = SpatialCoverage::whereNot('region', 'Rest of the world')->get(); - $worldId = SpatialCoverage::where('region', 'Rest of the world')->first()->id; - - $matchFound = false; - foreach ($ukCoverages as $c) { - if (str_contains($coverage, strtolower($c['region']))) { - - DatasetVersionHasSpatialCoverage::updateOrCreate([ - 'dataset_version_id' => (int)$version['id'], - 'spatial_coverage_id' => (int)$c['id'], - ]); - $matchFound = true; - } - } - - if (!$matchFound) { - if (str_contains($coverage, 'united kingdom')) { - foreach ($ukCoverages as $c) { - DatasetVersionHasSpatialCoverage::updateOrCreate([ - 'dataset_version_id' => (int)$version['id'], - 'spatial_coverage_id' => (int)$c['id'], - ]); - } - } else { - DatasetVersionHasSpatialCoverage::updateOrCreate([ - 'dataset_version_id' => (int)$version['id'], - 'spatial_coverage_id' => (int)$worldId, - ]); - } - } - } - public function getVersion(int $version) { if($version > 999) { diff --git a/app/Http/Traits/UpdateDatasetLinkages.php b/app/Http/Traits/UpdateDatasetLinkages.php new file mode 100644 index 000000000..4d2a17cc6 --- /dev/null +++ b/app/Http/Traits/UpdateDatasetLinkages.php @@ -0,0 +1,471 @@ +latestVersion(); + + // Retrieve all dataset_version_ids for the given dataset_id + $allDatasetVersionIds = DatasetVersion::where('dataset_id', $dataset->id)->select('id'); + + // Build the search array that will be used for finding matching datasets + $datasetSearchArray = $this->buildDatasetSearchArray(); + + // If delete is set to true, remove all existing linkages for the dataset + if ($delete) { + $this->removeExistingLinkages($allDatasetVersionIds); + } + + // Log the dataset processing + Log::info("Processing Dataset: $dataset->id"); + + // Process linkages for tools, publications, and dataset relationships + $this->processDatasetVersionToToolRelationships($version, 'linkage.tools', 'USING'); + $this->processDatasetVersionToSpatialCoverageRelationships($metadata, $version, 'coverage.spatial',""); + $this->processDatasetVersionToPublicationRelationships($version, 'linkage.publicationUsingDataset', 'USING'); + $this->processDatasetVersionToPublicationRelationships($version, 'linkage.publicationAboutDataset', 'ABOUT'); + + // Process dataset relationships like isDerivedFrom, isPartOf, and linkedDatasets GWDM.2x + $this->processDatasetVersionToDatasetVersionRelationships($version['metadata'], $version, 'linkage.datasetLinkage.isDerivedFrom', 'isDerivedFrom', 'Derived From', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($version['metadata'], $version, 'linkage.datasetLinkage.isPartOf', 'isPartOf', 'Part Of', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($version['metadata'], $version, 'linkage.datasetLinkage.isMemberOf', 'isMemberOf', 'Member Of', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($version['metadata'], $version, 'linkage.datasetLinkage.linkedDatasets', 'linkedDatasets', 'Linked Datasets', $datasetSearchArray); + + // Process dataset relationships like isDerivedFrom, isPartOf, and linkedDatasets HDR.3.0 + $this->processDatasetVersionToDatasetVersionRelationships($metadata, $version, 'enrichmentAndLinkage.derivedFrom', 'isDerivedFrom', 'Derived From', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($metadata, $version, 'enrichmentAndLinkage.isPartOf', 'isPartOf', 'Part Of', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($metadata, $version, 'enrichmentAndLinkage.similarToDatasets', 'isMemberOf', 'Member Of', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($metadata, $version, 'enrichmentAndLinkage.linkedDatasets', 'linkedDatasets', 'Linked Datasets', $datasetSearchArray); + } + + /** + * Process tool linkages for a specific dataset version. + * + * This method searches the dataset metadata for associated tools (by URL) and + * links them to the dataset version. If the tool is found by its URL in the database, + * it creates or updates the linkage. + * + * @param DatasetVersion $version The dataset version for which tools are linked. + * @return void + */ + private function processDatasetVersionToToolRelationships($version, $metadataKey, $linkType): void + { + // Retrieve the 'tools' metadata field from the dataset version + $tools = $this->getValueByPossibleKeys($version['metadata'], [$metadataKey]); + + // Check if any tools are found in the metadata + if ($tools) { + // Ensure the tools variable is treated as an array (in case it's a single item) + $tools = (array) $tools; + + // Loop through each tool URL found in the metadata + foreach ($tools as $tool) { + // Attempt to find a matching tool in the database by its URL + $toolModel = Tool::where('url', $tool)->first(); + + // If a matching tool is found in the database, create or update the linkage + if ($toolModel) { + DatasetVersionHasTool::updateOrCreate([ + 'dataset_version_id' => $version->id, // The current dataset version ID + 'tool_id' => $toolModel->id, // The ID of the matching tool + 'link_type' => $linkType, // Define the type of linkage (URL match) + ]); + + // Log the successful creation of the linkage + Log::info("Link created between datasetVersion: $version->id and tool: $tool: link type: $linkType"); + } else { + // Log if no matching tool is found in the database for the given URL + Log::info("Tool not found: $tool"); + } + } + } else { + // Log if no tools were found in the dataset metadata + Log::info("No Dataset to Tool Linkages found in metadata"); + } + } + + + /** + * Process publication linkages for a specific dataset version. + * + * This method searches the dataset metadata for publications associated with the dataset. + * It removes any "https://doi.org/" or "http://doi.org/" from the publication DOI if present, + * and then searches for the publication in the database by its DOI. + * + * @param DatasetVersion $version The dataset version for which publications are linked. + * @param string $metadataKey The metadata key (e.g., 'publicationUsingDataset', 'publicationAboutDataset'). + * @param string $linkType The type of linkage (e.g., 'USING', 'ABOUT'). + * @return void + */ + private function processDatasetVersionToPublicationRelationships($version, $metadataKey, $linkType): void + { + // Retrieve the list of publications from the metadata using the provided key + $publications = $this->getValueByPossibleKeys($version['metadata'], [$metadataKey]); + + // Check if any publications are found in the metadata + if ($publications) { + // Ensure the publications variable is treated as an array (in case it's a single item) + $publications = (array) $publications; + + // Loop through each publication DOI found in the metadata + foreach ($publications as $publication) { + + // Attempt to find a matching publication in the database by its DOI + $publicationModel = Publication::where('paper_doi', $publication)->first(); + + if(!$publicationModel){ + // Remove "https://doi.org/" or "http://doi.org/" prefix from the DOI if present + $publication = preg_replace('/^https?:\/\/doi\.org\//', '', $publication); + + // Try again to find a matching publication in the database by its DOI + $publicationModel = Publication::where('paper_doi', $publication)->first(); + } + + // If a matching publication is found, create or update the linkage + if ($publicationModel) { + PublicationHasDatasetVersion::updateOrCreate([ + 'dataset_version_id' => $version->id, // The current dataset version ID + 'publication_id' => $publicationModel->id, // The ID of the matching publication + 'link_type' => $linkType, // The type of linkage being created (e.g., 'USING', 'ABOUT') + ]); + + // Log the successful creation of the linkage + Log::info("Link created between datasetVersion: $version->id and publication: $publication"); + } else { + // Log if no matching publication is found in the database for the given DOI + Log::info("Publication not found for DOI: $publication"); + } + } + } else { + // Log if no publications were found in the dataset metadata for the given key + Log::info("No publications found for key: $metadataKey"); + } + } + + /** + * Process spatial coverage linkages for a specific dataset version. + * + * This method searches the dataset metadata for spatial coverage and links it to the dataset version. + * It matches against known regions in the database and, if no match is found, links it to "Rest of the world." + * + * @param array $metadata The metadata array containing the spatial coverage information. + * @param DatasetVersion $version The dataset version for which spatial coverage is linked. + * @param string $metadataKey The key in the metadata array to search for spatial coverage. + * @param string $linkType The type of linkage (e.g., 'spatial' or 'isDerivedFrom'). + * @return void + */ + private function processDatasetVersionToSpatialCoverageRelationships(array $metadata, DatasetVersion $version, string $metadataKey, string $linkType): void + { + // Retrieve the spatial coverage from the metadata using the provided metadata key + $spatialCoverages = $this->getValueByPossibleKeys($metadata, [$metadataKey]); + + // Check if any spatial coverage is found in the metadata + if ($spatialCoverages) { + // Ensure the spatial coverage variable is treated as an array (in case it's a single item) + $spatialCoverages = (array) $spatialCoverages; + + // Retrieve known UK and World spatial coverage records from the database + $ukCoverages = SpatialCoverage::whereNot('region', 'Rest of the world')->get(); + $worldId = SpatialCoverage::where('region', 'Rest of the world')->first()->id; + + // Loop through each spatial coverage found in the metadata + foreach ($spatialCoverages as $coverage) { + $coverage = strtolower($coverage); // Convert coverage to lowercase for matching + $matchFound = false; + + // Loop through the UK coverages to find a matching region in the coverage metadata + foreach ($ukCoverages as $c) { + if (str_contains($coverage, strtolower($c['region']))) { + // If a match is found, create or update the linkage + DatasetVersionHasSpatialCoverage::updateOrCreate([ + 'dataset_version_id' => $version->id, + 'spatial_coverage_id' => $c->id + //'link_type' => $linkType, // placemaker to Include the link type + ]); + + // Log the successful linkage + Log::info("Spatial coverage linked: DatasetVersion {$version->id} to region {$c['region']} with link type: {$linkType}"); + $matchFound = true; + break; // Exit the loop if a match is found + } + } + + // If no specific match found, check for "united kingdom" or default to "Rest of the world" + if (!$matchFound) { + if (str_contains($coverage, 'united kingdom')) { + foreach ($ukCoverages as $c) { + DatasetVersionHasSpatialCoverage::updateOrCreate([ + 'dataset_version_id' => $version->id, + 'spatial_coverage_id' => $c->id + //'link_type' => $linkType, // placemaker to Include the link type + ]); + + // Log the successful linkage for the entire UK + Log::info("Spatial coverage linked: DatasetVersion {$version->id} to region {$c['region']} (United Kingdom) with link type: {$linkType}"); + } + } else { + DatasetVersionHasSpatialCoverage::updateOrCreate([ + 'dataset_version_id' => $version->id, + 'spatial_coverage_id' => $worldId + //'link_type' => $linkType, // placemaker to Include the link type + ]); + + // Log the default linkage to "Rest of the world" + Log::info("Spatial coverage linked: DatasetVersion {$version->id} to Rest of the world with link type: {$linkType}"); + } + } + } + } else { + // Log if no spatial coverage was found in the dataset metadata + Log::info("No spatial coverage found in metadata for DatasetVersion {$version->id}"); + } + } + + + /** + * Process dataset-to-dataset version relationships for a specific dataset version. + * + * This method processes the dataset linkages from the provided metadata. It handles cases where the + * metadata entries are either structured (e.g., with pid, title, or URL) or are simple arrays of values. + * + * @param array $metadata The metadata containing dataset relationships. + * @param DatasetVersion $version The dataset version being processed. + * @param string $metadataKey The metadata key for the linkage type (e.g., 'isDerivedFrom', 'linkedDatasets'). + * @param string $linkageType The linkage type to be created. + * @param string $linkageDescription Description of the linkage for logging purposes. + * @param array $datasetSearchArray The search array for matching datasets by text. + * @return void + */ + private function processDatasetVersionToDatasetVersionRelationships(array $metadata, DatasetVersion $version, $metadataKey, $linkageType, $linkageDescription, $datasetSearchArray): void + { + // Retrieve linkage data from the metadata using the provided key + $datasetLinkages = $this->getValueByPossibleKeys($metadata, [$metadataKey]); + + // Check if any linkages are found in the metadata + if ($datasetLinkages) { + $datasetLinkages = (array) $datasetLinkages; // Ensure it's treated as an array + + // Loop through each linkage found in the metadata + foreach ($datasetLinkages as $datasetLinkage) { + + // Determine if the datasetLinkage is an object with pid, title, or URL fields or just a simple value + if (is_array($datasetLinkage) || is_object($datasetLinkage)) { + // If it is a structured dataset descriptor object, handle based on pid, title, or url + $pid = $datasetLinkage['pid'] ?? null; + $url = $datasetLinkage['url'] ?? null; + $title = $datasetLinkage['title'] ?? null; + } else { + // If it is a simple value, set all possible fields to the datasetLinkage value + $pid = $datasetLinkage; + $url = $datasetLinkage; + $title = $datasetLinkage; + } + + // Try to find the dataset directly by PID or ID + $datasetModel = Dataset::where('id', $pid) + ->orWhere('pid', $pid) + ->first(); + + // If that doesnt work, see if the linkage is a URL, process based on URL basename + if (!$datasetModel){ + if ($url && preg_match('/^https?:\/\//', $url)) { + $datasetLinkageBasename = basename($url); // Extract the basename from the URL + // Try to find a dataset by ID or PID using the basename + $datasetModel = Dataset::where('id', $datasetLinkageBasename) + ->orWhere('pid', $datasetLinkageBasename) + ->first(); + } + } + $textMatches = []; + + if (!$datasetModel){ + // If no PID or URL is present, attempt a text-based search using title + $textMatches = $this->searchInDataset($datasetSearchArray, $title); + } + + // If a dataset model is found, create or update the linkage + if (isset($datasetModel) && $datasetModel) { + + // Retrieve the latest version ID of the target dataset + $datasetVersionTargetID = $datasetModel->latestVersion(['id'])->id; + $datasetVersionSourceID = $version->id; + + if ($datasetVersionSourceID != $datasetVersionTargetID) { + // Create or update the linkage between the source and target dataset versions + DatasetVersionHasDatasetVersion::updateOrCreate([ + 'dataset_version_source_id' => $datasetVersionSourceID, // Source dataset version ID + 'dataset_version_target_id' => $datasetVersionTargetID, // Target dataset version ID + 'linkage_type' => $linkageType, // The type of linkage (e.g., 'isDerivedFrom') + 'direct_linkage' => 1, // Mark the linkage as direct + 'description' => "Linked on Dataset PID", // Description for logging purposes + ]); + + // Log the successful creation of the linkage + Log::info("Link created between datasetVersion: {$version->id} and datasetVersion: $datasetVersionTargetID of type: $linkageType"); + } else { + // Log that self-loop cannot be created + Log::info("Prevented creation of self loop between datasetVersion: {$version->id} and datasetVersion: {$datasetVersionTargetID} of type: $linkageType"); + } + } elseif (isset($textMatches) && $textMatches) { + // If no dataset model was found but text matches were found, create linkages based on the text matches + foreach ($textMatches as $textMatch) { + $datasetVersionTargetID = $textMatch['dataset_version_id']; + $datasetVersionSourceID = $version->id; + $matchingField = $textMatch['field']; + + if ($datasetVersionSourceID != $datasetVersionTargetID) { + DatasetVersionHasDatasetVersion::updateOrCreate([ + 'dataset_version_source_id' => $datasetVersionSourceID, + 'dataset_version_target_id' => $datasetVersionTargetID, + 'linkage_type' => $linkageType, + 'direct_linkage' => 1, + 'description' => "Linked by text matching on Dataset {$matchingField}", + ]); + + // Log the successful creation of the linkage based on the text match + Log::info("Link created between datasetVersion: {$datasetVersionSourceID} and datasetVersion: {$datasetVersionTargetID} of type: $linkageType, match={$matchingField}"); + } else { + // Log that self-loop cannot be created + Log::info("Prevented creation of self loop between datasetVersion: {$datasetVersionSourceID} and datasetVersion: {$datasetVersionTargetID} of type: $linkageType, match={$matchingField}"); + } + } + } else { + // Log if no matching dataset or text match was found + Log::info("DatasetVersion linkage not found for key: $datasetLinkage ($linkageDescription)"); + } + } + } else { + // Log if no linkages were found in the metadata + Log::info("No $linkageDescription Dataset Linkages found in metadata"); + } + } + + + /** + * Build an array for searching dataset versions by specific fields. + * + * This method gathers all dataset versions and extracts specific fields + * such as 'title', 'shortTitle', and 'doiName' into a searchable array. + * This allows for efficient dataset searching when trying to match relationships. + * + * @return array Array containing searchable dataset fields. + */ + private function buildDatasetSearchArray(): array + { + // Retrieve all datasets from the database + $allDatasets = Dataset::where('status',Dataset::STATUS_ACTIVE)->get(); + + // Initialize an empty array to hold the dataset search data + $datasetSearchArray = []; + + // Loop through each dataset to access its latest version + foreach ($allDatasets as $singleDataset) { + // Retrieve the latest version of the dataset + $latestVersion = $singleDataset->latestVersion(); + + // Only proceed if a latest version exists + if ($latestVersion) { + // Add the relevant metadata fields to the search array for this version + $datasetSearchArray[] = [ + 'dataset_version_id' => $latestVersion->id, // The ID of the dataset version + 'title' => $this->getValueByPossibleKeys($latestVersion['metadata'], ['metadata.summary.title']), // The title of the dataset version + 'short_title' => $this->getValueByPossibleKeys($latestVersion['metadata'], ['metadata.summary.shortTitle']), // The short title, if available + 'doi_name' => $this->getValueByPossibleKeys($latestVersion['metadata'], ['metadata.summary.doiName']), // The DOI name associated with the dataset version + ]; + } + } + + // Return the fully built dataset search array + return $datasetSearchArray; + } + + /** + * Remove existing linkages for all versions of a dataset. + * + * This method clears existing linkages for the dataset versions provided in the + * $allDatasetVersionIds array. It deletes entries from multiple tables such as + * DatasetVersionHasTool, PublicationHasDatasetVersion, and DatasetVersionHasDatasetVersion. + * + * @param array $allDatasetVersionIds Array of dataset version IDs whose linkages will be deleted. + * @return void + */ + private function removeExistingLinkages($allDatasetVersionIds): void + { + DatasetVersionHasTool::whereIn('dataset_version_id', $allDatasetVersionIds)->delete(); + PublicationHasDatasetVersion::whereIn('dataset_version_id', $allDatasetVersionIds)->delete(); + DatasetVersionHasDatasetVersion::whereIn('dataset_version_source_id', $allDatasetVersionIds)->delete(); + } + + /** + * Search for a dataset linkage by any field (except version_id). + * + * This method searches across multiple fields for a matching string, excluding + * the version_id field. It returns the version_id and the field where the match occurred. + * + * @param array $datasetSearchArray The array of datasets to search in. + * @param string $searchTerm The term to search for. + * @return array An array of matching dataset version_id and field name. + */ + private function searchInDataset(array $datasetSearchArray, string $searchTerm): array + { + // Initialize an empty array to store matching results + $matches = []; + + // Loop through each dataset version in the search array + foreach ($datasetSearchArray as $version) { + // Loop through each field and value within the dataset version + foreach ($version as $field => $value) { + // Exclude 'dataset_version_id' from the search and ensure the value is a string + if ($field !== 'dataset_version_id' && is_string($value)) { + // Check if the search term exists within the field's value + if (strpos($value, $searchTerm) !== false) { + // If a match is found, add the version ID and matching field to the results array + $matches[] = [ + 'dataset_version_id' => $version['dataset_version_id'], // The ID of the matching dataset version + 'field' => $field, // The field in which the match was found + 'matching_value' => $value, // The actual value that matched the search term + ]; + } + } + } + } + + // Return the array of matches (if any) + return $matches; + } +} \ No newline at end of file diff --git a/app/Jobs/DatasetSqlLinkageJob.php b/app/Jobs/DatasetSqlLinkageJob.php new file mode 100644 index 000000000..e34c8ad14 --- /dev/null +++ b/app/Jobs/DatasetSqlLinkageJob.php @@ -0,0 +1,471 @@ +timeout = config('jobs.default_timeout', 600); // Timeout for the job + $this->tries = config('jobs.ntries', 2); // Number of attempts + $this->metadata = $metadata; + $this->dataset = $dataset; + $this->deleteExistingLinkages = $deleteExistingLinkages; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle(): void + { + try { + // Call the linkage creation method + $this->createSqlLinkageFromDataset($this->metadata, $this->dataset, $this->deleteExistingLinkages); + + } catch (Exception $e) { + // Log the exception and rethrow + Auditor::log([ + 'action_type' => 'EXCEPTION', + 'action_name' => class_basename($this) . '@' . __FUNCTION__, + 'description' => $e->getMessage(), + ]); + + throw new Exception($e->getMessage()); + } + } + + + /** + * Create SQL linkages for tools, publications, and dataset relationships. + * + * This method processes and creates SQL linkages for tools, publications, + * and dataset relationships (such as isDerivedFrom, isPartOf, linkedDatasets, etc.) + * based on the latest metadata version of a dataset. + * + * If the $delete flag is set to true, existing linkages for all dataset versions + * will be deleted prior to creating new ones. + * + * @param Dataset $dataset The dataset for which linkages are created. + * @param bool $delete Optional flag to delete existing linkages. Defaults to false. + * @return void + */ + private function createSqlLinkageFromDataset(array $metadata, Dataset $dataset, bool $delete = false): void + { + // Retrieve the latest dataset_version for the given dataset_id + $version = $dataset->latestVersion(); + + // Retrieve all dataset_version_ids for the given dataset_id + $allDatasetVersionIds = DatasetVersion::where('dataset_id', $dataset->id)->select('id'); + + // Build the search array that will be used for finding matching datasets + $datasetSearchArray = $this->buildDatasetSearchArray(); + + // If delete is set to true, remove all existing linkages for the dataset + if ($delete) { + $this->removeExistingLinkages($allDatasetVersionIds); + } + + // Process linkages for tools, publications, and dataset relationships + $this->processDatasetVersionToToolRelationships($version, 'linkage.tools', 'USING'); + $this->processDatasetVersionToSpatialCoverageRelationships($metadata, $version, 'coverage.spatial',""); + $this->processDatasetVersionToPublicationRelationships($version, 'linkage.publicationUsingDataset', 'USING'); + $this->processDatasetVersionToPublicationRelationships($version, 'linkage.publicationAboutDataset', 'ABOUT'); + + // Process dataset relationships like isDerivedFrom, isPartOf, and linkedDatasets GWDM.2x + $this->processDatasetVersionToDatasetVersionRelationships($version['metadata'], $version, 'linkage.datasetLinkage.isDerivedFrom', 'isDerivedFrom', 'Derived From', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($version['metadata'], $version, 'linkage.datasetLinkage.isPartOf', 'isPartOf', 'Part Of', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($version['metadata'], $version, 'linkage.datasetLinkage.isMemberOf', 'isMemberOf', 'Member Of', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($version['metadata'], $version, 'linkage.datasetLinkage.linkedDatasets', 'linkedDatasets', 'Linked Datasets', $datasetSearchArray); + + // Process dataset relationships like isDerivedFrom, isPartOf, and linkedDatasets HDR.3.0 + $this->processDatasetVersionToDatasetVersionRelationships($metadata, $version, 'enrichmentAndLinkage.derivedFrom', 'isDerivedFrom', 'Derived From', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($metadata, $version, 'enrichmentAndLinkage.isPartOf', 'isPartOf', 'Part Of', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($metadata, $version, 'enrichmentAndLinkage.similarToDatasets', 'isMemberOf', 'Member Of', $datasetSearchArray); + $this->processDatasetVersionToDatasetVersionRelationships($metadata, $version, 'enrichmentAndLinkage.linkedDatasets', 'linkedDatasets', 'Linked Datasets', $datasetSearchArray); + } + + /** + * Process tool linkages for a specific dataset version. + * + * This method searches the dataset metadata for associated tools (by URL) and + * links them to the dataset version. If the tool is found by its URL in the database, + * it creates or updates the linkage. + * + * @param DatasetVersion $version The dataset version for which tools are linked. + * @return void + */ + private function processDatasetVersionToToolRelationships($version, $metadataKey, $linkType): void + { + // Retrieve the 'tools' metadata field from the dataset version + $tools = $this->getValueByPossibleKeys($version['metadata'], [$metadataKey]); + + // Check if any tools are found in the metadata + if ($tools) { + // Ensure the tools variable is treated as an array (in case it's a single item) + $tools = (array) $tools; + + // Loop through each tool URL found in the metadata + foreach ($tools as $tool) { + // Attempt to find a matching tool in the database by its URL + $toolModel = Tool::where('url', $tool)->first(); + + // If a matching tool is found in the database, create or update the linkage + if ($toolModel) { + DatasetVersionHasTool::updateOrCreate([ + 'dataset_version_id' => $version->id, // The current dataset version ID + 'tool_id' => $toolModel->id, // The ID of the matching tool + 'link_type' => $linkType, // Define the type of linkage (URL match) + ]); + } + } + } + } + + + /** + * Process publication linkages for a specific dataset version. + * + * This method searches the dataset metadata for publications associated with the dataset. + * It removes any "https://doi.org/" or "http://doi.org/" from the publication DOI if present, + * and then searches for the publication in the database by its DOI. + * + * @param DatasetVersion $version The dataset version for which publications are linked. + * @param string $metadataKey The metadata key (e.g., 'publicationUsingDataset', 'publicationAboutDataset'). + * @param string $linkType The type of linkage (e.g., 'USING', 'ABOUT'). + * @return void + */ + private function processDatasetVersionToPublicationRelationships($version, $metadataKey, $linkType): void + { + // Retrieve the list of publications from the metadata using the provided key + $publications = $this->getValueByPossibleKeys($version['metadata'], [$metadataKey]); + + // Check if any publications are found in the metadata + if ($publications) { + // Ensure the publications variable is treated as an array (in case it's a single item) + $publications = (array) $publications; + + // Loop through each publication DOI found in the metadata + foreach ($publications as $publication) { + + // Attempt to find a matching publication in the database by its DOI + $publicationModel = Publication::where('paper_doi', $publication)->first(); + + if(!$publicationModel){ + // Remove "https://doi.org/" or "http://doi.org/" prefix from the DOI if present + $publication = preg_replace('/^https?:\/\/doi\.org\//', '', $publication); + + // Try again to find a matching publication in the database by its DOI + $publicationModel = Publication::where('paper_doi', $publication)->first(); + } + + // If a matching publication is found, create or update the linkage + if ($publicationModel) { + PublicationHasDatasetVersion::updateOrCreate([ + 'dataset_version_id' => $version->id, // The current dataset version ID + 'publication_id' => $publicationModel->id, // The ID of the matching publication + 'link_type' => $linkType, // The type of linkage being created (e.g., 'USING', 'ABOUT') + ]); + } + } + } + } + + /** + * Process spatial coverage linkages for a specific dataset version. + * + * This method searches the dataset metadata for spatial coverage and links it to the dataset version. + * It matches against known regions in the database and, if no match is found, links it to "Rest of the world." + * + * @param array $metadata The metadata array containing the spatial coverage information. + * @param DatasetVersion $version The dataset version for which spatial coverage is linked. + * @param string $metadataKey The key in the metadata array to search for spatial coverage. + * @param string $linkType The type of linkage (e.g., 'spatial' or 'isDerivedFrom'). + * @return void + */ + private function processDatasetVersionToSpatialCoverageRelationships(array $metadata, DatasetVersion $version, string $metadataKey, string $linkType): void + { + // Retrieve the spatial coverage from the metadata using the provided metadata key + $spatialCoverages = $this->getValueByPossibleKeys($metadata, [$metadataKey]); + + // Check if any spatial coverage is found in the metadata + if ($spatialCoverages) { + // Ensure the spatial coverage variable is treated as an array (in case it's a single item) + $spatialCoverages = (array) $spatialCoverages; + + // Retrieve known UK and World spatial coverage records from the database + $ukCoverages = SpatialCoverage::whereNot('region', 'Rest of the world')->get(); + $worldId = SpatialCoverage::where('region', 'Rest of the world')->first()->id; + + // Loop through each spatial coverage found in the metadata + foreach ($spatialCoverages as $coverage) { + $coverage = strtolower($coverage); // Convert coverage to lowercase for matching + $matchFound = false; + + // Loop through the UK coverages to find a matching region in the coverage metadata + foreach ($ukCoverages as $c) { + if (str_contains($coverage, strtolower($c['region']))) { + // If a match is found, create or update the linkage + DatasetVersionHasSpatialCoverage::updateOrCreate([ + 'dataset_version_id' => $version->id, + 'spatial_coverage_id' => $c->id + //'link_type' => $linkType, // placemaker to Include the link type + ]); + $matchFound = true; + break; // Exit the loop if a match is found + } + } + + // If no specific match found, check for "united kingdom" or default to "Rest of the world" + if (!$matchFound) { + if (str_contains($coverage, 'united kingdom')) { + foreach ($ukCoverages as $c) { + DatasetVersionHasSpatialCoverage::updateOrCreate([ + 'dataset_version_id' => $version->id, + 'spatial_coverage_id' => $c->id + //'link_type' => $linkType, // placemaker to Include the link type + ]); + } + } else { + DatasetVersionHasSpatialCoverage::updateOrCreate([ + 'dataset_version_id' => $version->id, + 'spatial_coverage_id' => $worldId + //'link_type' => $linkType, // placemaker to Include the link type + ]); + } + } + } + } + } + + + /** + * Process dataset-to-dataset version relationships for a specific dataset version. + * + * This method processes the dataset linkages from the provided metadata. It handles cases where the + * metadata entries are either structured (e.g., with pid, title, or URL) or are simple arrays of values. + * + * @param array $metadata The metadata containing dataset relationships. + * @param DatasetVersion $version The dataset version being processed. + * @param string $metadataKey The metadata key for the linkage type (e.g., 'isDerivedFrom', 'linkedDatasets'). + * @param string $linkageType The linkage type to be created. + * @param string $linkageDescription Description of the linkage for logging purposes. + * @param array $datasetSearchArray The search array for matching datasets by text. + * @return void + */ + private function processDatasetVersionToDatasetVersionRelationships(array $metadata, DatasetVersion $version, $metadataKey, $linkageType, $linkageDescription, $datasetSearchArray): void + { + // Retrieve linkage data from the metadata using the provided key + $datasetLinkages = $this->getValueByPossibleKeys($metadata, [$metadataKey]); + + // Check if any linkages are found in the metadata + if ($datasetLinkages) { + $datasetLinkages = (array) $datasetLinkages; // Ensure it's treated as an array + + // Loop through each linkage found in the metadata + foreach ($datasetLinkages as $datasetLinkage) { + + // Determine if the datasetLinkage is an object with pid, title, or URL fields or just a simple value + if (is_array($datasetLinkage) || is_object($datasetLinkage)) { + // If it is a structured dataset descriptor object, handle based on pid, title, or url + $pid = $datasetLinkage['pid'] ?? null; + $url = $datasetLinkage['url'] ?? null; + $title = $datasetLinkage['title'] ?? null; + } else { + // If it is a simple value, set all possible fields to the datasetLinkage value + $pid = $datasetLinkage; + $url = $datasetLinkage; + $title = $datasetLinkage; + } + + // Try to find the dataset directly by PID or ID + $datasetModel = Dataset::where('id', $pid) + ->orWhere('pid', $pid) + ->first(); + + // If that doesnt work, see if the linkage is a URL, process based on URL basename + if (!$datasetModel){ + if ($url && preg_match('/^https?:\/\//', $url)) { + $datasetLinkageBasename = basename($url); // Extract the basename from the URL + // Try to find a dataset by ID or PID using the basename + $datasetModel = Dataset::where('id', $datasetLinkageBasename) + ->orWhere('pid', $datasetLinkageBasename) + ->first(); + } + } + + // If a dataset model is found, create or update the linkage + if ($datasetModel) { + + // Retrieve the latest version ID of the target dataset + $datasetVersionTargetID = $datasetModel->latestVersion(['id'])->id; + $datasetVersionSourceID = $version->id; + + if ($datasetVersionSourceID != $datasetVersionTargetID) { + // Create or update the linkage between the source and target dataset versions + DatasetVersionHasDatasetVersion::updateOrCreate([ + 'dataset_version_source_id' => $datasetVersionSourceID, // Source dataset version ID + 'dataset_version_target_id' => $datasetVersionTargetID, // Target dataset version ID + 'linkage_type' => $linkageType, // The type of linkage (e.g., 'isDerivedFrom') + 'direct_linkage' => 1, // Mark the linkage as direct + 'description' => "Linked on Dataset PID", // Description for logging purposes + ]); + } + } + + if (!$datasetModel){ + + $textMatches = []; + + // If no PID or URL is present, attempt a text-based search using title + $textMatches = $this->searchInDataset($datasetSearchArray, $title); + + // If no dataset model was found but text matches were found, create linkages based on the text matches + foreach ($textMatches as $textMatch) { + $datasetVersionTargetID = $textMatch['dataset_version_id']; + $datasetVersionSourceID = $version->id; + $matchingField = $textMatch['field']; + + if ($datasetVersionSourceID != $datasetVersionTargetID) { + DatasetVersionHasDatasetVersion::updateOrCreate([ + 'dataset_version_source_id' => $datasetVersionSourceID, + 'dataset_version_target_id' => $datasetVersionTargetID, + 'linkage_type' => $linkageType, + 'direct_linkage' => 1, + 'description' => "Linked by text matching on Dataset {$matchingField}", + ]); + } + } + } + } + } + } + + /** + * Remove existing linkages for all versions of a dataset. + * + * This method clears existing linkages for the dataset versions provided in the + * $allDatasetVersionIds array. It deletes entries from multiple tables such as + * DatasetVersionHasTool, PublicationHasDatasetVersion, and DatasetVersionHasDatasetVersion. + * + * @param array $allDatasetVersionIds Array of dataset version IDs whose linkages will be deleted. + * @return void + */ + private function removeExistingLinkages($allDatasetVersionIds): void + { + DatasetVersionHasTool::whereIn('dataset_version_id', $allDatasetVersionIds)->delete(); + PublicationHasDatasetVersion::whereIn('dataset_version_id', $allDatasetVersionIds)->delete(); + DatasetVersionHasDatasetVersion::whereIn('dataset_version_source_id', $allDatasetVersionIds)->delete(); + } + + /** + * Build an array for searching dataset versions by specific fields. + * + * This method gathers all dataset versions and extracts specific fields + * such as 'title', 'shortTitle', and 'doiName' into a searchable array. + * This allows for efficient dataset searching when trying to match relationships. + * + * @return array Array containing searchable dataset fields. + */ + private function buildDatasetSearchArray(): array + { + // Retrieve all datasets from the database + $allDatasets = Dataset::where('status',Dataset::STATUS_ACTIVE)->get(); + + // Initialize an empty array to hold the dataset search data + $datasetSearchArray = []; + + // Loop through each dataset to access its latest version + foreach ($allDatasets as $singleDataset) { + // Retrieve the latest version of the dataset + $latestVersion = $singleDataset->latestVersion(); + + // Only proceed if a latest version exists + if ($latestVersion) { + // Add the relevant metadata fields to the search array for this version + $datasetSearchArray[] = [ + 'dataset_version_id' => $latestVersion->id, // The ID of the dataset version + 'title' => $this->getValueByPossibleKeys($latestVersion['metadata'], ['metadata.summary.title']), // The title of the dataset version + 'short_title' => $this->getValueByPossibleKeys($latestVersion['metadata'], ['metadata.summary.shortTitle']), // The short title, if available + 'doi_name' => $this->getValueByPossibleKeys($latestVersion['metadata'], ['metadata.summary.doiName']), // The DOI name associated with the dataset version + ]; + } + } + + // Return the fully built dataset search array + return $datasetSearchArray; + } + + /** + * Search for a dataset linkage by any field (except version_id). + * + * This method searches across multiple fields for a matching string, excluding + * the version_id field. It returns the version_id and the field where the match occurred. + * + * @param array $datasetSearchArray The array of datasets to search in. + * @param string $searchTerm The term to search for. + * @return array An array of matching dataset version_id and field name. + */ + private function searchInDataset(array $datasetSearchArray, string $searchTerm): array + { + // Initialize an empty array to store matching results + $matches = []; + + // Loop through each dataset version in the search array + foreach ($datasetSearchArray as $version) { + // Loop through each field and value within the dataset version + foreach ($version as $field => $value) { + // Exclude 'dataset_version_id' from the search and ensure the value is a string + if ($field !== 'dataset_version_id' && is_string($value)) { + // Check if the search term exists within the field's value + if (strpos($value, $searchTerm) !== false) { + // If a match is found, add the version ID and matching field to the results array + $matches[] = [ + 'dataset_version_id' => $version['dataset_version_id'], // The ID of the matching dataset version + 'field' => $field, // The field in which the match was found + 'matching_value' => $value, // The actual value that matched the search term + ]; + } + } + } + } + + // Return the array of matches (if any) + return $matches; + } +} diff --git a/tests/Feature/DatasetTest.php b/tests/Feature/DatasetTest.php index 7e42a47a7..446c00180 100644 --- a/tests/Feature/DatasetTest.php +++ b/tests/Feature/DatasetTest.php @@ -5,13 +5,23 @@ use Config; use Tests\TestCase; use App\Models\Dataset; +use App\Models\License; use App\Models\NamedEntities; +use App\Models\Publication; +use App\Models\DatasetVersionHasDatasetVersion; +use App\Models\DatasetVersionHasTool; use Illuminate\Support\Carbon; use Tests\Traits\Authorization; use App\Http\Enums\TeamMemberOf; +use App\Models\DatasetVersionHasSpatialCoverage; +use App\Models\PublicationHasDatasetVersion; use Tests\Traits\MockExternalApis; use Illuminate\Support\Facades\Storage; use Database\Seeders\SpatialCoverageSeeder; +use Database\Seeders\CategorySeeder; +use Database\Seeders\TypeCategorySeeder; + +use Database\Seeders\LicenseSeeder; use Illuminate\Foundation\Testing\RefreshDatabase; class DatasetTest extends TestCase @@ -26,6 +36,8 @@ class DatasetTest extends TestCase public const TEST_URL_TEAM = '/api/v1/teams'; public const TEST_URL_NOTIFICATION = '/api/v1/notifications'; public const TEST_URL_USER = '/api/v1/users'; + public const TEST_URL_TOOL = '/api/v1/tools'; + public const TEST_URL_PUBLICATION = '/api/v1/publications'; protected $metadata; protected $metadataAlt; @@ -36,11 +48,15 @@ public function setUp(): void $this->seed([ SpatialCoverageSeeder::class, + LicenseSeeder::class, + CategorySeeder::class, + TypeCategorySeeder::class, ]); $this->metadata = $this->getMetadata(); $this->metadataAlt = $this->metadata; $this->metadataAlt['metadata']['summary']['title'] = 'ABC title'; + $this->metadataAlt['metadata']['required']['gatewayPid'] = '12345'; } /** @@ -1021,6 +1037,391 @@ public function test_create_update_delete_dataset_with_success(): void $responseDeleteUser->assertStatus(200); } + /** + * Create Dataset linkage with success + * + * @return void + */ + public function test_dataset_linkage__with_success(): void + { + // create team + // First create a notification to be used by the new team + $responseNotification = $this->json( + 'POST', + self::TEST_URL_NOTIFICATION, + [ + 'notification_type' => 'applicationSubmitted', + 'message' => 'Some message here', + 'email' => 'some@email.com', + 'opt_in' => 1, + 'enabled' => 1, + ], + $this->header, + ); + $contentNotification = $responseNotification->decodeResponseJson(); + $notificationID = $contentNotification['data']; + + // Create the new team + $responseCreateTeam = $this->json( + 'POST', + self::TEST_URL_TEAM, + [ + 'name' => 'Team Test ' . fake()->regexify('[A-Z]{5}[0-4]{1}'), + 'enabled' => 1, + 'allows_messaging' => 1, + 'workflow_enabled' => 1, + 'access_requests_management' => 1, + 'uses_5_safes' => 1, + 'is_admin' => 1, + 'member_of' => fake()->randomElement([ + TeamMemberOf::ALLIANCE, + TeamMemberOf::HUB, + TeamMemberOf::OTHER, + TeamMemberOf::NCS, + ]), + 'contact_point' => 'dinos345@mail.com', + 'application_form_updated_by' => 'Someone Somewhere', + 'application_form_updated_on' => '2023-04-06 15:44:41', + 'notifications' => [$notificationID], + 'users' => [], + ], + $this->header, + ); + + $responseCreateTeam->assertStatus(Config::get('statuscodes.STATUS_OK.code')) + ->assertJsonStructure([ + 'message', + 'data', + ]); + + $contentCreateTeam = $responseCreateTeam->decodeResponseJson(); + $teamId = $contentCreateTeam['data']; + + // create user + $responseCreateUser = $this->json( + 'POST', + self::TEST_URL_USER, + [ + 'firstname' => 'Firstname', + 'lastname' => 'Lastname', + 'email' => 'firstname.lastname.123456789@test.com', + 'password' => 'Passw@rd1!', + 'sector_id' => 1, + 'organisation' => 'Test Organisation', + 'bio' => 'Test Biography', + 'domain' => 'https://testdomain.com', + 'link' => 'https://testlink.com/link', + 'orcid' => "https://orcid.org/75697342", + 'contact_feedback' => 1, + 'contact_news' => 1, + 'mongo_id' => 1234566, + 'mongo_object_id' => "12345abcde", + ], + $this->header, + ); + $responseCreateUser->assertStatus(201); + $contentCreateUser = $responseCreateUser->decodeResponseJson(); + $userId = $contentCreateUser['data']; + + // create a tool + $licenseId = License::where('valid_until', null)->get()->random()->id; + $responseCreateTool = $this->json( + 'POST', + self::TEST_URL_TOOL, + [ + "mongo_object_id" => "5ece82082abda8b3a06f1941", + "name" => "Similique sapiente est vero eum.", + "url" => "https://github.com/HDRUK/papers", + "description" => "Quod maiores id qui iusto. Aut qui velit qui aut nisi et officia. Ab inventore dolores ut quia quo. Quae veritatis fugiat ad vel.", + 'results_insights' => "asfhiasfh aoshfa ", + "license" => $licenseId, + "tech_stack" => "Cumque molestias excepturi quam at.", + "category_id" => 1, + "user_id" => 1, + "tag" => [], + "dataset" => [], + "programming_language" => [], + "programming_package" => [], + "type_category" => [1, 2], + "enabled" => 1, + "publications" => [], + "durs" => [], + "collections" => [], + "any_dataset" => false, + "status" => "ACTIVE" + ], + $this->header, + ); + + $responseCreateTool->assertStatus(201); + $contentCreateTool = $responseCreateTool->decodeResponseJson(); + $toolId = $contentCreateTool['data']; + + // create a Publication (about the dataset) + $responseCreatePublicationAbout = $this->json( + 'POST', + self::TEST_URL_PUBLICATION, + [ + 'paper_title' => 'Not A Test Paper Title', + 'authors' => 'Einstein, Albert, Yankovich, Al', + 'year_of_publication' => '2022', + 'paper_doi' => '10.1000/182', + 'publication_type' => 'Paper and such', + 'journal_name' => 'Something Journal-y here', + 'abstract' => 'Some blurb about this made up paper written by people who should never meet.', + 'url' => 'http://paper_example1.html', + 'datasets' => [], + 'tools' => [], + 'status' => 'ACTIVE' + ], + $this->header, + ); + + $responseCreatePublicationAbout->assertStatus(201); + $contentCreatePublicationAbout = $responseCreatePublicationAbout->decodeResponseJson(); + $PublicationAboutId = $contentCreatePublicationAbout['data']; + + // create a Publication (using the dataset) + $responseCreatePublicationUsing = $this->json( + 'POST', + self::TEST_URL_PUBLICATION, + [ + 'paper_title' => 'Not A Test Paper Title', + 'authors' => 'Einstein, Albert, Yankovich, Al', + 'year_of_publication' => '2022', + 'paper_doi' => '10.1300/182', + 'publication_type' => 'Paper and such', + 'journal_name' => 'Something Journal-y here', + 'abstract' => 'Some blurb about this made up paper written by people who should never meet.', + 'url' => 'http://paper_example2.html', + 'datasets' => [], + 'tools' => [], + 'status' => 'ACTIVE' + ], + $this->header, + ); + + $responseCreatePublicationUsing->assertStatus(201); + $contentCreatePublicationUsing = $responseCreatePublicationUsing->decodeResponseJson(); + $PublicationUsingId = $contentCreatePublicationUsing['data']; + + // inject the publication information into the metadata + $this->metadata['metadata']['linkage']['publicationAboutDataset']= "10.1000/182"; + $this->metadata['metadata']['linkage']['publicationUsingDataset']= "10.1300/182"; + + // create dataset1 + $responseCreateDataset1 = $this->json( + 'POST', + self::TEST_URL_DATASET, + [ + 'team_id' => $teamId, + 'user_id' => $userId, + 'metadata' => $this->metadata, + 'create_origin' => Dataset::ORIGIN_MANUAL, + 'status' => Dataset::STATUS_DRAFT, + ], + $this->header, + ); + $responseCreateDataset1->assertStatus(201); + $contentCreateDataset1 = $responseCreateDataset1->decodeResponseJson(); + $datasetId1 = $contentCreateDataset1['data']; + + // update dataset1 + $responseUpdateDataset1 = $this->json( + 'PUT', + self::TEST_URL_DATASET . '/' . $datasetId1, + [ + 'team_id' => $teamId, + 'user_id' => $userId, + 'metadata' => $this->metadata, + 'create_origin' => Dataset::ORIGIN_MANUAL, + 'status' => Dataset::STATUS_ACTIVE, + ], + $this->header, + ); + + $responseUpdateDataset1->assertStatus(200); + + + // create dataset2 + $responseCreateDataset2 = $this->json( + 'POST', + self::TEST_URL_DATASET, + [ + 'team_id' => $teamId, + 'user_id' => $userId, + 'metadata' => $this->metadataAlt, + 'create_origin' => Dataset::ORIGIN_MANUAL, + 'status' => Dataset::STATUS_DRAFT, + ], + $this->header, + ); + $responseCreateDataset2->assertStatus(201); + $contentCreateDataset2 = $responseCreateDataset2->decodeResponseJson(); + + $datasetId2 = $contentCreateDataset2['data']; + + // Get the Dataset 1PID and inject it, and a Title based linkage into the alt metadata + $pid=Dataset::where('id',$datasetId1)->first()->pid; + $this->metadataAlt['metadata']['linkage']['datasetLinkage']['isPartOf']= $pid; + $this->metadataAlt['metadata']['linkage']['datasetLinkage']['isMemberOf']= 'HDR UK Papers & Preprints'; + + + // update dataset2 + $responseUpdateDataset2 = $this->json( + 'PUT', + self::TEST_URL_DATASET . '/' . $datasetId2, + [ + 'team_id' => $teamId, + 'user_id' => $userId, + 'metadata' => $this->metadataAlt, + 'create_origin' => Dataset::ORIGIN_MANUAL, + 'status' => Dataset::STATUS_ACTIVE, + ], + $this->header, + ); + + $responseUpdateDataset2->assertStatus(200); + + + + // TESTING + # Now we get testing: lets check out the linkage for Dataset 1: + # lets get the latest version IDs: + + $latestVersionId1 = Dataset::where('id',$datasetId1)->first()->latestVersion()->id; + $latestVersionId2 = Dataset::where('id',$datasetId2)->first()->latestVersion()->id; + + # lets check if our DatasetVersion-to-Tool relationship was created + $linkedToolsCount=DatasetVersionHasTool::where('dataset_version_id',$latestVersionId1) + ->where('tool_id',$toolId) + ->get() + ->count(); + $this->assertEquals($linkedToolsCount, 1); + + # lets check if our DatasetVersion-about-Publication relationship was created + $linkedPublicationAboutCount=PublicationHasDatasetVersion::where('dataset_version_id',$latestVersionId1) + ->where('publication_id',$PublicationAboutId) + ->get() + ->count(); + $this->assertEquals($linkedPublicationAboutCount, 1); + + # lets check if our DatasetVersion-using-Publication relationship was created + $linkedPublicationUsingCount=PublicationHasDatasetVersion::where('dataset_version_id',$latestVersionId1) + ->where('publication_id',$PublicationUsingId) + ->get() + ->count(); + $this->assertEquals($linkedPublicationUsingCount, 1); + + # lets check if our DatasetVersion-to-SpatialCoverage relationship was created + $linkedPublicationUsingCount=DatasetVersionHasSpatialCoverage::where('dataset_version_id',$latestVersionId1) + ->get() + ->count(); + + $this->assertEquals($linkedPublicationUsingCount, 1); + + # lets check if our DatasetVersion-to-DatasetVersion relationship(s) were created (one using text searching) + $linkedDatasetVersions=DatasetVersionHasDatasetVersion::where('dataset_version_target_id',$latestVersionId1) + ->where('dataset_version_source_id',$latestVersionId2) + ->get() + ->count(); + + $this->assertEquals($linkedDatasetVersions, 2); + + # lets check that a DatasetVersion-DatasetVersion self loop was not created for dataset2 + $selfLinkedDatasetVersions=DatasetVersionHasDatasetVersion::where('dataset_version_target_id',$latestVersionId2) + ->where('dataset_version_source_id',$latestVersionId2) + ->get() + ->count(); + + $this->assertEquals(0, $selfLinkedDatasetVersions); + + // DELETE STUFF + // permanent delete dataset1 + $responseDeleteDataset = $this->json( + 'DELETE', + self::TEST_URL_DATASET . '/' . $datasetId1 . '?deletePermanently=true', + [], + $this->header + ); + $responseDeleteDataset->assertJsonStructure([ + 'message' + ]); + $responseDeleteDataset->assertStatus(200); + + // permanent delete dataset2 + $responseDeleteDataset2 = $this->json( + 'DELETE', + self::TEST_URL_DATASET . '/' . $datasetId2 . '?deletePermanently=true', + [], + $this->header + ); + $responseDeleteDataset2->assertJsonStructure([ + 'message' + ]); + $responseDeleteDataset2->assertStatus(200); + + // delete Tool + $responseDeleteTool = $this->json( + 'DELETE', + self::TEST_URL_TOOL . '/' . $toolId, + [], + $this->header + ); + $responseDeleteTool->assertJsonStructure([ + 'message' + ]); + $responseDeleteTool->assertStatus(200); + + // delete Publication about + $responseDeletePublicationAbout = $this->json( + 'DELETE', + self::TEST_URL_PUBLICATION . '/' . $PublicationAboutId, + [], + $this->header + ); + $responseDeletePublicationAbout->assertJsonStructure([ + 'message' + ]); + $responseDeletePublicationAbout->assertStatus(200); + + // delete Publication + $responseDeletePublicationUsing = $this->json( + 'DELETE', + self::TEST_URL_PUBLICATION . '/' . $PublicationUsingId, + [], + $this->header + ); + $responseDeletePublicationUsing->assertJsonStructure([ + 'message' + ]); + $responseDeletePublicationUsing->assertStatus(200); + + // delete team + $responseDeleteTeam = $this->json( + 'DELETE', + self::TEST_URL_TEAM . '/' . $teamId . '?deletePermanently=true', + [], + $this->header + ); + $responseDeleteTeam->assertJsonStructure([ + 'message' + ]); + $responseDeleteTeam->assertStatus(200); + + // delete user + $responseDeleteUser = $this->json( + 'DELETE', + self::TEST_URL_USER . '/' . $userId, + [], + $this->header + ); + $responseDeleteUser->assertJsonStructure([ + 'message' + ]); + $responseDeleteUser->assertStatus(200); + } + public function test_create_dataset_fails_with_invalid_origin(): void { // create team