Merged in task/dspace-cris-2023_02_x/DSC-2734 (pull request DSpace#5398)

[DSC-2734] added CORE API to LiveImportFramework

Approved-by: Mykhaylo Boychuk

Author: Tnnienn

dspace-api/src/main/java/org/dspace/importer/external/core/CoreFieldMapping.java

@@ -0,0 +1,38 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.importer.external.core;
+
+import java.util.Map;
+import javax.annotation.Resource;
+
+import org.dspace.importer.external.metadatamapping.AbstractMetadataFieldMapping;
+
+/**
+ * An implementation of {@link AbstractMetadataFieldMapping}
+ * Responsible for defining the mapping of the CORE metadatum fields on the DSpace metadatum fields
+ *
+ * @author Antonio Fasanella (antonio.fasanella@4science.com)
+ */
+@SuppressWarnings("rawtypes")
+public class CoreFieldMapping extends AbstractMetadataFieldMapping {
+
+    /**
+     * Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it
+     * only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over
+     * what metadatafield is generated.
+     *
+     * @param metadataFieldMap The map containing the link between retrieve metadata and metadata that will be set to
+     *                         the item.
+     */
+    @Override
+    @SuppressWarnings("unchecked")
+    @Resource(name = "coreMetadataFieldMap")
+    public void setMetadataFieldMap(Map metadataFieldMap) {
+        super.setMetadataFieldMap(metadataFieldMap);
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/core/CoreImportMetadataSourceServiceImpl.java

@@ -0,0 +1,351 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.importer.external.core;
+
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import javax.el.MethodNotFoundException;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.dspace.content.Item;
+import org.dspace.importer.external.datamodel.ImportRecord;
+import org.dspace.importer.external.datamodel.Query;
+import org.dspace.importer.external.exception.MetadataSourceException;
+import org.dspace.importer.external.liveimportclient.service.LiveImportClient;
+import org.dspace.importer.external.service.AbstractImportMetadataSourceService;
+import org.dspace.importer.external.service.DoiCheck;
+import org.dspace.importer.external.service.components.QuerySource;
+import org.dspace.services.ConfigurationService;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Live import provider for the CORE aggregator (https://core.ac.uk).
+ * Uses CORE API v3 to search and retrieve scholarly works.
+ */
+public class CoreImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService<String>
+        implements QuerySource {
+
+    private static final Logger log = LogManager.getLogger(CoreImportMetadataSourceServiceImpl.class);
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    @Autowired
+    private LiveImportClient liveImportClient;
+
+    @Autowired
+    private ConfigurationService configurationService;
+
+    @Override
+    public String getImportSource() {
+        return "core";
+    }
+
+    @Override
+    public void init() throws Exception {
+        // no-op
+    }
+
+    @Override
+    public Collection<ImportRecord> getRecords(String query, int start, int count) throws MetadataSourceException {
+        return retry(new SearchByQueryCallable(query, start, count));
+    }
+
+    @Override
+    public Collection<ImportRecord> getRecords(Query query) throws MetadataSourceException {
+        return retry(new SearchByQueryCallable(query));
+    }
+
+    @Override
+    public ImportRecord getRecord(String recordId) throws MetadataSourceException {
+        return retry(new GetByIdCallable(recordId));
+    }
+
+    @Override
+    public ImportRecord getRecord(Query query) throws MetadataSourceException {
+        return retry(new GetByIdCallable(query));
+    }
+
+    @Override
+    public int getRecordsCount(String query) throws MetadataSourceException {
+        return retry(new CountByQueryCallable(query));
+    }
+
+    @Override
+    public int getRecordsCount(Query query) throws MetadataSourceException {
+        return retry(new CountByQueryCallable(query));
+    }
+
+    @Override
+    public Collection<ImportRecord> findMatchingRecords(Query query) throws MetadataSourceException {
+        return retry(new SearchByQueryCallable(query));
+    }
+
+    @Override
+    public Collection<ImportRecord> findMatchingRecords(Item item) throws MetadataSourceException {
+        throw new MethodNotFoundException("This method is not implemented for CORE");
+    }
+
+    /**
+     * If the query is a DOI, converts it to CORE search format: doi:"10.1234/example".
+     * Returns empty string if not a DOI.
+     *
+     * @param query the search query
+     * @return DOI in CORE search format, or empty string
+     */
+    private String getID(String query) {
+        if (StringUtils.isBlank(query)) {
+            return StringUtils.EMPTY;
+        }
+        String q = query;
+        // Handle double URL-encoded slash: %252F -> /
+        if (q.contains("%252F")) {
+            q = q.replace("%252F", "/");
+        }
+        if (DoiCheck.isDoi(q)) {
+            return "doi:\"" + q + "\"";
+        }
+        return StringUtils.EMPTY;
+    }
+
+    private int getTimeoutMs() {
+        return configurationService.getIntProperty("core.timeout", 30000);
+    }
+
+    private int getDefaultPageSize() {
+        return configurationService.getIntProperty("core.pageSize", 10);
+    }
+
+    private String getBaseUrl() {
+        return configurationService.getProperty("core.api.url", "https://api.core.ac.uk/v3");
+    }
+
+    private String getApiKey() {
+        return StringUtils.trimToNull(configurationService.getProperty("core.apiKey"));
+    }
+
+    /**
+     * Builds the request map expected by LiveImportClient.
+     * - uriParameters: querystring params
+     * - headers: HTTP headers (Authorization)
+     */
+    private Map<String, Map<String, String>> buildParams(Map<String, String> uriParameters) {
+        Map<String, Map<String, String>> params = new HashMap<>();
+
+        params.put("uriParameters", uriParameters != null ? uriParameters : new HashMap<>());
+
+        Map<String, String> headers = new HashMap<>();
+        String apiKey = getApiKey();
+        if (apiKey != null) {
+            headers.put("Authorization", "Bearer " + apiKey);
+        } else {
+            log.warn("CORE api key is not configured (property core.apiKey). Requests may fail with 401/403.");
+        }
+        params.put("headers", headers);
+
+        return params;
+    }
+
+    private String buildWorksSearchUrl() {
+        return getBaseUrl() + "/search/works";
+    }
+
+    private String buildWorkByIdUrl(String identifier) {
+        return getBaseUrl() + "/works/" + encodePathSegment(identifier);
+    }
+
+    private String encodePathSegment(String segment) {
+        try {
+            return URLEncoder.encode(segment, StandardCharsets.UTF_8).replace("+", "%20");
+        } catch (Exception e) {
+            return segment;
+        }
+    }
+
+    private class SearchByQueryCallable implements Callable<List<ImportRecord>> {
+
+        private final Query query;
+
+        private SearchByQueryCallable(String queryString, Integer start, Integer count) {
+            Query q = new Query();
+            q.addParameter("query", StringUtils.trimToNull(queryString));
+            q.addParameter("start", start);
+            q.addParameter("count", count);
+            this.query = q;
+        }
+
+        private SearchByQueryCallable(Query query) {
+            this.query = query != null ? query : new Query();
+        }
+
+        @Override
+        public List<ImportRecord> call() {
+            List<ImportRecord> records = new ArrayList<>();
+
+            String raw = query.getParameterAsClass("query", String.class);
+            String q = getID(raw);
+            if (StringUtils.isBlank(q)) {
+                q = StringUtils.trimToNull(raw);
+            }
+            if (q == null) {
+                return records;
+            }
+
+            Integer start = query.getParameterAsClass("start", Integer.class);
+            Integer count = query.getParameterAsClass("count", Integer.class);
+
+            int offset = (start != null) ? Math.max(start, 0) : 0;
+            int limit = (count != null && count > 0) ? count : getDefaultPageSize();
+
+            Map<String, String> uriParameters = new HashMap<>();
+            uriParameters.put("q", q);
+            uriParameters.put("limit", Integer.toString(limit));
+            uriParameters.put("offset", Integer.toString(offset));
+
+            String url = buildWorksSearchUrl();
+            String response = liveImportClient.executeHttpGetRequest(getTimeoutMs(), url, buildParams(uriParameters));
+
+            JsonNode jsonNode = convertStringJsonToJsonNode(response);
+            if (jsonNode == null) {
+                log.warn("CORE returned invalid JSON");
+                return records;
+            }
+
+            JsonNode resultsNode = jsonNode.at("/results");
+            if (resultsNode != null && resultsNode.isArray()) {
+                for (JsonNode workNode : resultsNode) {
+                    if (workNode == null || workNode.isMissingNode() || workNode.isNull()) {
+                        continue;
+                    }
+                    records.add(transformSourceRecords(workNode.toString()));
+                }
+            } else {
+                log.debug("CORE: missing /results array in response");
+            }
+
+            return records;
+        }
+    }
+
+    private class GetByIdCallable implements Callable<ImportRecord> {
+
+        private final Query query;
+
+        private GetByIdCallable(String recordId) {
+            Query q = new Query();
+            q.addParameter("id", StringUtils.trimToNull(recordId));
+            this.query = q;
+        }
+
+        private GetByIdCallable(Query query) {
+            this.query = query != null ? query : new Query();
+        }
+
+        @Override
+        public ImportRecord call() throws Exception {
+
+            String id = query.getParameterAsClass("id", String.class);
+            id = StringUtils.trimToNull(id);
+            if (id == null) {
+                return null;
+            }
+
+            String url = buildWorkByIdUrl(id);
+
+            String response;
+            try {
+                response = liveImportClient.executeHttpGetRequest(getTimeoutMs(), url, buildParams(new HashMap<>()));
+            } catch (RuntimeException e) {
+                log.error("CORE getRecord failed for identifier={}", id, e);
+                throw new MetadataSourceException("CORE getRecord failed for identifier=" + id, e);
+            }
+
+            JsonNode jsonNode = convertStringJsonToJsonNode(response);
+            if (jsonNode == null || jsonNode.isMissingNode() || jsonNode.isNull()) {
+                log.warn("CORE /works/{id} returned invalid JSON for identifier={}", id);
+                return null;
+            }
+
+            return transformSourceRecords(jsonNode.toString());
+        }
+    }
+
+    private class CountByQueryCallable implements Callable<Integer> {
+
+        private final Query query;
+
+        private CountByQueryCallable(String queryString) {
+            Query q = new Query();
+            q.addParameter("query", StringUtils.trimToNull(queryString));
+            this.query = q;
+        }
+
+        private CountByQueryCallable(Query query) {
+            this.query = query != null ? query : new Query();
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            String raw = query.getParameterAsClass("query", String.class);
+            String q = getID(raw);
+            if (StringUtils.isBlank(q)) {
+                q = StringUtils.trimToNull(raw);
+            }
+            if (q == null) {
+                return 0;
+            }
+
+            Map<String, String> uriParameters = new HashMap<>();
+            uriParameters.put("q", q);
+            uriParameters.put("limit", "1");
+            uriParameters.put("offset", "0");
+
+            String url = buildWorksSearchUrl();
+            String responseString = liveImportClient.executeHttpGetRequest(getTimeoutMs(), url,
+                    buildParams(uriParameters));
+
+            JsonNode jsonNode = convertStringJsonToJsonNode(responseString);
+            if (jsonNode == null) {
+                log.warn("CORE returned invalid JSON");
+                throw new MetadataSourceException("Could not read CORE source");
+            }
+
+            JsonNode totalNode = jsonNode.at("/totalHits");
+            if (totalNode != null && totalNode.isNumber()) {
+                return totalNode.asInt();
+            }
+            if (totalNode != null && totalNode.isTextual()) {
+                try {
+                    return Integer.parseInt(totalNode.asText());
+                } catch (Exception e) {
+                    log.debug("Could not parse totalHits: {}", totalNode.asText(), e);
+                }
+            }
+            return 0;
+        }
+    }
+
+    private JsonNode convertStringJsonToJsonNode(String json) {
+        try {
+            return objectMapper.readTree(json);
+        } catch (JsonProcessingException e) {
+            log.error("Unable to process json response.", e);
+        }
+        return null;
+    }
+}