Merge pull request #67 from IQSS/develop

Update from IQSS

Author: lubitchv

.github/workflows/container_base_push.yml

@@ -0,0 +1,85 @@
+---
+name: Container Base Module
+
+on:
+    push:
+        branches:
+            - 'develop'
+            - 'master'
+        paths:
+            - 'modules/container-base/**'
+            - 'modules/dataverse-parent/pom.xml'
+            - '.github/workflows/container_base_push.yml'
+    pull_request:
+        branches:
+            - 'develop'
+            - 'master'
+        paths:
+            - 'modules/container-base/**'
+            - 'modules/dataverse-parent/pom.xml'
+            - '.github/workflows/container_base_push.yml'
+
+env:
+    IMAGE_TAG: unstable
+
+jobs:
+    build:
+        name: Build image
+        runs-on: ubuntu-latest
+        permissions:
+            contents: read
+            packages: read
+        strategy:
+            matrix:
+                jdk: [ '11' ]
+        # Only run in upstream repo - avoid unnecessary runs in forks
+        if: ${{ github.repository_owner == 'IQSS' }}
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v3
+
+            - name: Set up JDK ${{ matrix.jdk }}
+              uses: actions/setup-java@v3
+              with:
+                  java-version: ${{ matrix.jdk }}
+                  distribution: 'adopt'
+            - name: Cache Maven packages
+              uses: actions/cache@v3
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            - name: Build base container image with local architecture
+              run: mvn -f modules/container-base -Pct package
+
+            # Run anything below only if this is not a pull request.
+            # Accessing, pushing tags etc. to DockerHub will only succeed in upstream because secrets.
+
+            - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }}
+              name: Push description to DockerHub
+              uses: peter-evans/dockerhub-description@v3
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+                  repository: gdcc/base
+                  short-description: "Dataverse Base Container image providing Payara application server and optimized configuration"
+                  readme-filepath: ./modules/container-base/README.md
+
+            - if: ${{ github.event_name != 'pull_request' }}
+              name: Log in to the Container registry
+              uses: docker/login-action@v2
+              with:
+                  registry: ${{ env.REGISTRY }}
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - if: ${{ github.event_name != 'pull_request' }}
+              name: Set up QEMU for multi-arch builds
+              uses: docker/setup-qemu-action@v2
+            - name: Re-set image tag based on branch
+              if: ${{ github.ref_name == 'master' }}
+              run: echo "IMAGE_TAG=stable"
+            - if: ${{ github.event_name != 'pull_request' }}
+              name: Deploy multi-arch base container image to Docker Hub
+              run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }}

.github/workflows/shellcheck.yml

@@ -0,0 +1,24 @@
+name: "Shellcheck"
+on:
+    push:
+        paths:
+            - conf/solr/**
+            - modules/container-base/**
+    pull_request:
+        paths:
+            - conf/solr/**
+            - modules/container-base/**
+jobs:
+    shellcheck:
+        name: Shellcheck
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+            - name: shellcheck
+              uses: reviewdog/action-shellcheck@v1
+              with:
+                  github_token: ${{ secrets.github_token }}
+                  reporter: github-pr-review # Change reporter.
+                  fail_on_error: true
+                  # Container base image uses dumb-init shebang, so nail to using bash
+                  shellcheck_flags: "--shell=bash --external-sources"

.github/workflows/shellspec.yml

@@ -13,18 +13,6 @@ on:
 env:
     SHELLSPEC_VERSION: 0.28.1
 jobs:
-    shellcheck:
-        name: Shellcheck
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v2
-            - name: shellcheck
-              uses: reviewdog/action-shellcheck@v1
-              with:
-                  github_token: ${{ secrets.github_token }}
-                  reporter: github-pr-review # Change reporter.
-                  fail_on_error: true
-                  exclude: "./tests/shell/*"
     shellspec-ubuntu:
         name: "Ubuntu"
         runs-on: ubuntu-latest

conf/docker-aio/0prep_deps.sh

@@ -4,9 +4,9 @@ if [ ! -d dv/deps ]; then
 fi
 wdir=`pwd`
 
-if [ ! -e dv/deps/payara-5.2021.6.zip ]; then
+if [ ! -e dv/deps/payara-5.2022.3.zip ]; then
 	echo "payara dependency prep"
-	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.6/payara-5.2021.6.zip  -O dv/deps/payara-5.2021.6.zip
+	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2022.3/payara-5.2022.3.zip  -O dv/deps/payara-5.2022.3.zip
 fi
 
 if [ ! -e dv/deps/solr-8.11.1dv.tgz ]; then

conf/docker-aio/c8.dockerfile

@@ -24,7 +24,7 @@ COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
 RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.11.1dv.tgz
-RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.6.zip ; ln -s /opt/payara5 /opt/glassfish4
+RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2022.3.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
 # this dies under Java 11, do we keep it?

conf/solr/8.11.1/schema.xml

@@ -228,6 +228,11 @@
 
     <field name="dsPersistentId" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="filePersistentId" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <!-- Dataverse geospatial search -->
+    <!-- https://solr.apache.org/guide/8_11/spatial-search.html#rpt -->
+    <field name="geolocation" type="location_rpt" multiValued="true" stored="true" indexed="true"/>
+    <!-- https://solr.apache.org/guide/8_11/spatial-search.html#bboxfield -->
+    <field name="boundingBox" type="bbox" multiValued="true" stored="true" indexed="true"/>
 
     <!--
         METADATA SCHEMA FIELDS
@@ -261,6 +266,9 @@
     <field name="cleaningOperations" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="collectionMode" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="collectorTraining" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="workflowType" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="workflowCodeRepository" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="workflowDocumentation" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="contributor" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="contributorName" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="contributorType" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -498,6 +506,9 @@
     <copyField source="cleaningOperations" dest="_text_" maxChars="3000"/>
     <copyField source="collectionMode" dest="_text_" maxChars="3000"/>
     <copyField source="collectorTraining" dest="_text_" maxChars="3000"/>
+    <copyField source="workflowType" dest="_text_" maxChars="3000"/>
+    <copyField source="workflowCodeRepository" dest="_text_" maxChars="3000"/>
+    <copyField source="workflowDocumentation" dest="_text_" maxChars="3000"/>
     <copyField source="contributor" dest="_text_" maxChars="3000"/>
     <copyField source="contributorName" dest="_text_" maxChars="3000"/>
     <copyField source="contributorType" dest="_text_" maxChars="3000"/>
@@ -1098,6 +1109,9 @@
     -->
     <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
                geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
+    <!-- Dataverse - per GeoBlacklight, adding field type for bboxField that enables, among other things, overlap ratio calculations -->
+    <fieldType name="bbox" class="solr.BBoxField"
+           geo="true" distanceUnits="kilometers" numberType="pdouble" />
 
     <!-- Payloaded field types -->
     <fieldType name="delimited_payloads_float" stored="false" indexed="true" class="solr.TextField">

doc/JAVADOC_GUIDE.md

@@ -88,7 +88,7 @@ Here's a better approach:
     /** The dataverse we move the dataset <em>from</em> */
     private Dataverse sourceDataverse;
 
-    /** The dataverse we movet the dataset <em>to</em> */
+    /** The dataverse we move the dataset <em>to</em> */
     private Dataverse destinationDataverse;
 
 

doc/mergeParty/readme.md

@@ -73,10 +73,10 @@ Note that before we were asking `isGuest` and now we ask `isAuthenticated`, so t
 ## Other Added Things
 
 ### Settings bean
-Settings (in `edu.harvard.iq.dataverse.settings`) are where the application stores its more complex, admin-editable configuration. Technically, its a persistent `Map<String,String>`, that can be accessed via API (`edu.harvard.iq.dataverse.api.Admin`, on path `{server}/api/s/settings`). Currenly used for the signup mechanism.
+Settings (in `edu.harvard.iq.dataverse.settings`) are where the application stores its more complex, admin-editable configuration. Technically, its a persistent `Map<String,String>`, that can be accessed via API (`edu.harvard.iq.dataverse.api.Admin`, on path `{server}/api/s/settings`). Currently used for the signup mechanism.
 
 ### Admin API
-Accessible under url `{server}/api/s/`, API calls to this bean should be editing confugurations, allowing full indexing and more. The idea behing putting all of them under the `/s/` path is that we can later block these calls using a filter. This way, we could, say, allow access from localhost only. Or, we could block this completely based on some environemnt variable.
+Accessible under url `{server}/api/s/`, API calls to this bean should be editing configurations, allowing full indexing and more. The idea behind putting all of them under the `/s/` path is that we can later block these calls using a filter. This way, we could, say, allow access from localhost only. Or, we could block this completely based on some environment variable.
 
 ### `setup-all.sh` script
 A new script that sets up the users and the dataverses, sets the system up for built-in signup, and then indexes the dataverses using solr. Requires the [jq utility](http://stedolan.github.io/jq/). On Macs with [homebrew](http://brew.sh) installed, getting this utility is a `brew install jq` command away.

doc/release-notes/5.10-release-notes.md

@@ -140,7 +140,7 @@ or
 To find datasets with a without a CC0 license and with empty terms:
 
 ```
-select CONCAT('doi:', dvo.authority, '/', dvo.identifier), v.alias as dataverse_alias, case when versionstate='RELEASED' then concat(dv.versionnumber, '.', dv.minorversionnumber) else versionstate END  as version, dv.id as datasetversion_id, t.id as termsofuseandaccess_id, t.termsofuse, t.confidentialitydeclaration, t.specialpermissions, t.restrictions, t.citationrequirements, t.depositorrequirements, t.conditions, t.disclaimer from dvobject dvo, termsofuseandaccess t, datasetversion dv, dataverse v where dv.dataset_id=dvo.id and dv.termsofuseandaccess_id=t.id and dvo.owner_id=v.id and t.license='NONE' and t.termsofuse is null;
+select CONCAT('doi:', dvo.authority, '/', dvo.identifier), v.alias as dataverse_alias, case when versionstate='RELEASED' then concat(dv.versionnumber, '.', dv.minorversionnumber) else versionstate END  as version, dv.id as datasetversion_id, t.id as termsofuseandaccess_id, t.termsofuse, t.confidentialitydeclaration, t.specialpermissions, t.restrictions, t.citationrequirements, t.depositorrequirements, t.conditions, t.disclaimer from dvobject dvo, termsofuseandaccess t, datasetversion dv, dataverse v where dv.dataset_id=dvo.id and dv.termsofuseandaccess_id=t.id and dvo.owner_id=v.id and (t.license='NONE' or t.license is null) and t.termsofuse is null;
 ```
 
 As before, there are a couple options.