Merged back develop

Author: janvanmansum

conf/docker-aio/c8.dockerfile

@@ -2,9 +2,14 @@ FROM centos:8
 # OS dependencies
 # PG 10 is the default in centos8; keep the repo comment for when we bump to 11+
 #RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
-RUN yum install -y java-11-openjdk-devel postgresql-server sudo epel-release unzip curl httpd
+
+RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf
+RUN yum install -y java-11-openjdk-devel postgresql-server sudo epel-release unzip curl httpd python2 diffutils
 RUN yum install -y jq lsof awscli
 
+# for older search scripts
+RUN ln -s /usr/bin/python2 /usr/bin/python
+
 # copy and unpack dependencies (solr, payara)
 COPY dv /tmp/dv
 COPY testdata/schema*.xml /tmp/dv/

conf/docker-aio/configure_doi.bash

@@ -1,24 +1,24 @@
 #!/usr/bin/env bash
 
-cd /usr/local/glassfish4
+cd /opt/payara5
 
 # if appropriate; reconfigure PID provider on the basis of environmental variables.
 if [ ! -z "${DoiProvider}" ]; then
         curl -X PUT -d ${DoiProvider} http://localhost:8080/api/admin/settings/:DoiProvider
 fi
 if [ ! -z "${doi_username}" ]; then
-        bin/asadmin create-jvm-options "-Ddoi.username=${doi_password}"
+        bin/asadmin create-jvm-options "-Ddoi.username=${doi_username}"
 fi
 if [ ! -z "${doi_password}" ]; then
         bin/asadmin create-jvm-options "-Ddoi.password=${doi_password}"
 fi
 if [ ! -z "${doi_baseurl}" ]; then
         bin/asadmin delete-jvm-options "-Ddoi.baseurlstring=https\://mds.test.datacite.org"
-        doi_baseurl_esc=`echo ${doi_baseurl} | sed -e 's/:/\\:/'`
-        bin/asadmin create-jvm-options "\"-Ddoi.baseurlstring=${doi_baseurl_esc}\""
+        doi_baseurl_esc=`echo ${doi_baseurl} | sed -e 's/:/\\\:/'`
+        bin/asadmin create-jvm-options "-Ddoi.baseurlstring=${doi_baseurl_esc}"
 fi
 if [ ! -z "${doi_dataciterestapiurl}" ]; then
         bin/asadmin delete-jvm-options "-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"
-        doi_dataciterestapiurl_esc=`echo ${doi_dataciterestapiurl} | sed -e 's/:/\\:/'`
-        bin/asadmin create-jvm-options "\"-Ddoi.dataciterestapiurlstring=${doi_dataciterestapiurl_esc}\""
+        doi_dataciterestapiurl_esc=`echo ${doi_dataciterestapiurl} | sed -e 's/:/\\\:/'`
+        bin/asadmin create-jvm-options "-Ddoi.dataciterestapiurlstring=${doi_dataciterestapiurl_esc}"
 fi

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT -Ddataverse.test.baseurl=$dvurl
+source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT -Ddataverse.test.baseurl=$dvurl

conf/docker-aio/testscripts/install

@@ -1,7 +1,7 @@
 #!/bin/sh
 export HOST_ADDRESS=localhost
-export GLASSFISH_ROOT=/usr/local/glassfish4
-export FILES_DIR=/usr/local/glassfish4/glassfish/domains/domain1/files
+export GLASSFISH_ROOT=/opt/payara5
+export FILES_DIR=/opt/payara5/glassfish/domains/domain1/files
 export DB_NAME=dvndb
 export DB_PORT=5432
 export DB_HOST=localhost

conf/docker-aio/testscripts/post

@@ -2,7 +2,6 @@
 cd scripts/api
 ./setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out
 cd ../..
-psql -U dvnapp dvndb -f scripts/database/reference_data.sql
 psql -U dvnapp dvndb -f doc/sphinx-guides/source/_static/util/createsequence.sql
 scripts/search/tests/publish-dataverse-root
 #git checkout scripts/api/data/dv-root.json

doc/release-notes/7400-opendp-download.md

@@ -0,0 +1,12 @@
+Auxiliary Files can now be downloaded from the web interface.
+
+- Aux files uploaded as type=DP appear under "Differentially Private Statistics" under file level download. The rest appear under "Other Auxiliary Files".
+
+In addition, related changes were made, including the following:
+
+- New tooltip over the lock indicating if you have been granted access to a restricted file or not.
+- When downloading individual files, you will see "Restricted with Access Granted" or just "Restricted" (followed by "Users may not request access to files.") as appropriate.
+- When downloading individual files, instead of "Download" you should expect to see the file type such as "JPEG Image" or "Original File Format" if the type is unknown.
+- Downloaded aux files now have a file extension if it can be determined.
+
+Please note that the auxiliary files feature is experimental and if you don't need it, its API endpoints can be blocked.

doc/sphinx-guides/source/conf.py

@@ -71,7 +71,7 @@
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:

doc/sphinx-guides/source/developers/aux-file-support.rst

@@ -1,11 +1,11 @@
 Auxiliary File Support
 ======================
 
-Auxiliary file support is experimental. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (OpenDP.io). In future versions, this approach may become more broadly used and supported. 
+Auxiliary file support is experimental and as such, related APIs may be added, changed or removed without standard backward compatibility. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (opendp.org). In future versions, this approach will likely become more broadly used and supported.
 
 Adding an Auxiliary File to a Datafile
 --------------------------------------
-To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and the formatTag and formatVersion (if applicable) associated with the auxiliary file. There are two form parameters. "Origin" specifies the application/entity that created the auxiliary file, an "isPublic" controls access to downloading the file. If "isPublic" is true, any user can download the file, else, access authorization is based on the access rules as defined for the DataFile itself.
+To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and the formatTag and formatVersion (if applicable) associated with the auxiliary file. There are multiple form parameters. "Origin" specifies the application/entity that created the auxiliary file, and "isPublic" controls access to downloading the file. If "isPublic" is true, any user can download the file if the dataset has been published, else, access authorization is based on the access rules as defined for the DataFile itself. The "type" parameter is used to group similar auxiliary files in the UI. Currently, auxiliary files with type "DP" appear under "Differentially Private Statistics", while all other auxiliary files appear under "Other Auxiliary Files".
 
 .. code-block:: bash
 
@@ -14,9 +14,10 @@ To add an auxiliary file, specify the primary key of the datafile (FILE_ID), and
   export FILE_ID='12345'
   export FORMAT_TAG='dpJson'
   export FORMAT_VERSION='v1'
+  export TYPE='DP'
   export SERVER_URL=https://demo.dataverse.org
  
-  curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'origin=myApp' -F 'isPublic=true' "$SERVER_URL/api/access/datafile/$FILE_ID/metadata/$FORMAT_TAG/$FORMAT_VERSION"
+  curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'origin=myApp' -F 'isPublic=true' -F "type=$TYPE" "$SERVER_URL/api/access/datafile/$FILE_ID/metadata/$FORMAT_TAG/$FORMAT_VERSION"
 
 You should expect a 200 ("OK") response and JSON with information about your newly uploaded auxiliary file.
 
@@ -33,4 +34,4 @@ formatTag and formatVersion (if applicable) associated with the auxiliary file:
   export FORMAT_TAG='dpJson'
   export FORMAT_VERSION='v1'
 
-  curl "$SERVER_URL/api/access/datafile/$FILE_ID/$FORMAT_TAG/$FORMAT_VERSION"
+  curl "$SERVER_URL/api/access/datafile/$FILE_ID/metadata/$FORMAT_TAG/$FORMAT_VERSION"

doc/sphinx-guides/source/index.rst

@@ -21,7 +21,7 @@ These documentation guides are for the |version| version of Dataverse. To find g
   style/index
 
 How the Guides Are Organized
-============================
+----------------------------
 
 The guides are documentation that explain how to use Dataverse,
 which are divided into the following sections: User Guide,
@@ -35,7 +35,7 @@ project or who want to modify the code to suit their own needs. Finally, the API
 Developers that work on other applications and are interested in connecting with Dataverse through our APIs.
 
 Other Resources
-===============
+---------------
 
 **Dataverse Project Site**
 
@@ -71,7 +71,7 @@ If you have a **security issue** to report, please email `security@dataverse.org
 
 
 Indices and Tables
-==================
+------------------
 
 * :ref:`genindex`
 * :ref:`modindex`

doc/sphinx-guides/source/user/dataset-management.rst

@@ -179,6 +179,8 @@ Additional download options available for tabular data (found in the same drop-d
 - Data File Citation (currently in either RIS, EndNote XML, or BibTeX format); 
 - All of the above, as a zipped bundle. 
 
+Differentially Private (DP) Metadata can also be accessed for restricted tabular files if the data depositor has created a DP Metadata Release. See :ref:`dp-release-create` for more information.
+
 Astronomy (FITS)
 ----------------
 
@@ -210,6 +212,8 @@ Restricted Files
 
 When you restrict a file it cannot be downloaded unless permission has been granted.
 
+Differentially Private (DP) Metadata can be accessed for restricted tabular files if the data depositor has created a DP Metadata Release. See :ref:`dp-release-create` for more information.
+
 See also :ref:`terms-of-access` and :ref:`permissions`.
 
 Edit Files
@@ -302,6 +306,23 @@ If you restrict any files in your dataset, you will be prompted by a pop-up to e
 
 See also :ref:`restricted-files`.
 
+.. _dp-release-create:
+
+Creating and Depositing Differentially Private Metadata (Experimental)
+----------------------------------------------------------------------
+
+Through an integration with tools from the OpenDP Project (opendp.org), the Dataverse Software offers an experimental workflow that allows a data depositor to create and deposit Differentially Private (DP) Metadata files, which can then be used for exploratory data analysis. This workflow allows researchers to view the DP metadata for a tabular file, determine whether or not the file contains useful information, and then make an informed decision about whether or not to request access to the original file.
+
+If this integration has been enabled in your Dataverse installation, you can follow these steps to create a DP Metadata Release and make it available to researchers, while still keeping the files themselves restricted and able to be accessed after a successful access request.
+
+- Deposit a tabular file and let the ingest process complete
+- Restrict the File
+- In the kebab next to the file on the dataset page, or from the "Edit Files" dropdown on the file page, click "OpenDP Tool"
+- Go through the process to create a DP Metadata Release in the OpenDP tool, and at the end of the process deposit the DP Metadata Release back to the Dataverse installation
+- Publish the Dataset
+
+Once the dataset is published, users will be able to request access using the normal process, but will also have the option to download DP Statistics in order to get more information about the file. 
+
 Guestbook
 ---------