Merge branch 'develop'

jeanetteclark · jeanetteclark · commit 189540ef5ef1 · 2025-12-09T14:05:52.000-07:00
diff --git a/Docker/metadig-worker/requirements.txt b/Docker/metadig-worker/requirements.txt
@@ -10,5 +10,7 @@ meson-python
 ninja
 yara-python
 xarray[io]
+rasterio
 cftime
-metadig @ git+https://github.com/NCEAS/metadig-py.git
+geopandas
+metadig @ git+https://github.com/NCEAS/metadig-py@develop
diff --git a/helm/metadig-controller/Chart.yaml b/helm/metadig-controller/Chart.yaml
@@ -21,4 +21,4 @@ version: 1.0.8
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.2.1"
+appVersion: "v.3.2.2"
diff --git a/helm/metadig-controller/config.dev/metadig.properties b/helm/metadig-controller/config.dev/metadig.properties
@@ -4,6 +4,8 @@
 DataONE.authToken = not-set in config
 cnStage.subjectId = CN=urn:node:cnStage,DC=dataone,DC=org
 cnStage.serviceUrl = https://cn-stage.test.dataone.org/cn 
+cnSandbox.subjectId = CN=urn:node:cnSandbox,DC=dataone,DC=org
+cnSandbox.serviceUrl = https://cn-sandbox.test.dataone.org/cn 
 CN.subjectId = CN=urn:node:cnStage,DC=dataone,DC=org
 CN.serviceUrl = https://cn-stage.test.dataone.org/cn 
 mnTestARCTIC.subjectId = CN=urn:node:mnTestARCTIC,DC=dataone,DC=org
@@ -13,11 +15,9 @@ mnTestKNB.serviceUrl = https://dev.nceas.ucsb.edu/knb/d1/mn
 RabbitMQ.host = metadig-rabbitmq.metadig.svc.cluster.local
 RabbitMQ.port = 5672
 RabbitMQ.username = metadig
-RabbitMQ.password = quality
 solr.location = http://metadig-solr.metadig.svc.cluster.local:8983/solr
 jdbc.url = jdbc:postgresql://pooler-metadig-pg.metadig.svc.cluster.local:5432/metadig?sslmode=disable
 postgres.user = metadig
-postgres.passwd = metadig
 task.file = /opt/local/metadig/taskList.csv
 quality.serviceUrl = http://metadig-controller.metadig.svc:8080/quality
 metadig.base.directory = /opt/local/metadig
diff --git a/helm/metadig-controller/config/metadig.properties b/helm/metadig-controller/config/metadig.properties
@@ -31,12 +31,10 @@ mnStageUCSB2.serviceUrl = https://mn-stage-ucsb-2.test.dataone.org/metacat/d1/mn
 RabbitMQ.host = metadig-rabbitmq.metadig.svc.cluster.local
 RabbitMQ.port = 5672
 RabbitMQ.username = metadig
-RabbitMQ.password = quality
 #solr.location = http://k8s-ctrl-1.dataone.org:8983/solr
 solr.location = http://metadig-solr.metadig.svc.cluster.local:8983/solr
 # PostgreSQL connection information
 postgres.user = metadig
-postgres.passwd = metadig
 jdbc.url = jdbc:postgresql://metadig-pg-pooler.metadig.svc.cluster.local:5432/metadig?sslmode=disable
 # The metadig-scheduler task file
 task.file = /opt/local/metadig/taskList.csv
diff --git a/helm/metadig-controller/templates/deployment.yaml b/helm/metadig-controller/templates/deployment.yaml
@@ -49,6 +49,16 @@ spec:
                 # If 'false", the secret must exist before the pod can start
                 # App versions before v2.3.4 will read this value from a config file, not the env var.
                 optional: true
+          - name: RABBITMQ_AUTH
+            valueFrom:
+              secretKeyRef:
+                name: metadig-rmq-auth
+                key: rabbitmq-password
+          - name: PG_AUTH
+            valueFrom:
+              secretKeyRef:
+                name: metadig-pg
+                key: password          
           volumeMounts:
           - name: metadig-pv
             mountPath: {{ .Values.persistence.mountPath }}
diff --git a/helm/metadig-controller/values.yaml b/helm/metadig-controller/values.yaml
@@ -39,8 +39,10 @@ podAnnotations: {}
 
 podSecurityContext:
   runAsUser: 59990
-  runAsGroup: 59990
+  runAsGroup: 59997
   fsGroup: 59990
+  supplementalGroups:
+    - 1004
 
 securityContext: {}
   # capabilities:
diff --git a/helm/metadig-rabbitmq/rabbitmq-values.yaml b/helm/metadig-rabbitmq/rabbitmq-values.yaml
@@ -3,8 +3,8 @@ image:
   repository: bitnamilegacy/rabbitmq
 
 auth:
-  username: metadig
-  password: quality
+  existingPasswordSecret: metadig-rmq-auth
+  username: metadig 
 
 replicaCount: 6
 
diff --git a/helm/metadig-scheduler/Chart.yaml b/helm/metadig-scheduler/Chart.yaml
@@ -21,4 +21,4 @@ version: 1.0.7
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.2.1"
+appVersion: "v.3.2.2"
diff --git a/helm/metadig-scheduler/config.dev/taskList.csv b/helm/metadig-scheduler/config.dev/taskList.csv
@@ -19,10 +19,10 @@ task-type,task-name,task-group,cron-schedule,params
 #   - requestType: for score tasks, determine type of portal processing ("portal" or "node")
 #
 # Dataset quality scoring tasks
-#quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite-1.2.0;urn:node:mnTestARCTIC;2024-05-01T00:00:00.00Z;1;1000"
-quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*|^science-on-schema.org.*;FAIR-suite-0.5.0;urn:node:mnTestARCTIC;2025-09-01T00:00:00.00Z;1;1000"
-quality,quality-test-data-suite,metadig,15 0/1 * * * ?,"^eml.*|^http.*eml.*;data-suite-0.1.0;urn:node:mnTestARCTIC;2024-11-01T00:00:00.00Z;1;1000"
-#quality,quality-test-essdive,metadig,20 0/1 * * * ?,"^eml.*|^http.*eml.*;ess-dive.data.center.suite-1.3.0;urn:node:mnTestARCTIC;2025-10-01T00:00:00.00Z;1;1000"
+quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite-1.2.0;urn:node:mnTestARCTIC;2024-05-01T00:00:00.00Z;1;1000"
+quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*|^science-on-schema.org.*;FAIR-suite-0.5.0;urn:node:cnSandbox;2025-09-01T00:00:00.00Z;1;1000"
+quality,quality-test-data-suite,metadig,15 0/1 * * * ?,"^eml.*|^http.*eml.*;data-suite-0.2.0;urn:node:mnTestARCTIC;2024colnam-11-01T00:00:00.00Z;1;1000"
+
 
 #
 # Portal scoring tasks
diff --git a/helm/metadig-scheduler/templates/deployment.yaml b/helm/metadig-scheduler/templates/deployment.yaml
@@ -45,6 +45,11 @@ spec:
                 # If 'false", the secret must exist before the pod can start
                 # App versions before v2.3.4 will read this value from a config file, not the env var.
                 optional: true
+          - name: PG_AUTH
+            valueFrom:
+              secretKeyRef:
+                name: metadig-pg
+                key: password   
           volumeMounts:
           - name: metadig-pv
             mountPath: {{ .Values.persistence.mountPath }}
diff --git a/helm/metadig-scorer/Chart.yaml b/helm/metadig-scorer/Chart.yaml
@@ -21,4 +21,4 @@ version: 1.0.7
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.2.1"
+appVersion: "v.3.2.2"
diff --git a/helm/metadig-scorer/templates/deployment.yaml b/helm/metadig-scorer/templates/deployment.yaml
@@ -44,6 +44,16 @@ spec:
                 # If 'false", the secret must exist before the pod can start
                 # App versions before v2.3.4 will read this value from a config file, not the env var.
                 optional: true
+          - name: RABBITMQ_AUTH
+            valueFrom:
+              secretKeyRef:
+                name: metadig-rmq-auth
+                key: rabbitmq-password
+          - name: PG_AUTH
+            valueFrom:
+              secretKeyRef:
+                name: metadig-pg
+                key: password  
           volumeMounts:
           - name: metadig-pv
             mountPath: {{ .Values.persistence.mountPath }}
diff --git a/helm/metadig-worker/Chart.yaml b/helm/metadig-worker/Chart.yaml
@@ -21,4 +21,4 @@ version: 1.0.8
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.2.1"
+appVersion: "v.3.2.2"
diff --git a/helm/metadig-worker/templates/deployment.yaml b/helm/metadig-worker/templates/deployment.yaml
@@ -43,6 +43,16 @@ spec:
               secretKeyRef:
                 name: dataone-token
                 key: DataONEauthToken
+          - name: RABBITMQ_AUTH
+            valueFrom:
+              secretKeyRef:
+                name: metadig-rmq-auth
+                key: rabbitmq-password
+          - name: PG_AUTH
+            valueFrom:
+              secretKeyRef:
+                name: metadig-pg
+                key: password  
           - name: JMX_OPTS
             value: {{ join " " .Values.java.extraOpts }}
                     {{- if .Values.java.jmxEnabled }}
diff --git a/pom.xml b/pom.xml
@@ -5,7 +5,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>edu.ucsb.nceas</groupId>
   <artifactId>metadig-engine</artifactId>
-  <version>3.2.1</version>
+  <version>3.2.2</version>
   <packaging>jar</packaging>
   <name>metadig-engine</name>
   <description>MetaDIG library for running metadata quality tests</description>
@@ -80,13 +80,13 @@
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-api</artifactId>
-      <version>5.8.2</version>
+      <version>6.0.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-engine</artifactId>
-      <version>5.8.2</version>
+      <version>6.0.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -121,7 +121,7 @@
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
-      <version>2.17.0</version>
+      <version>2.20.0</version>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -137,7 +137,7 @@
     <dependency>
       <groupId>commons-beanutils</groupId>
       <artifactId>commons-beanutils</artifactId>
-      <version>1.9.4</version>
+      <version>1.11.0</version>
     </dependency>
     <!-- http://mvnrepository.com/artifact/org.eclipse.persistence/org.eclipse.persistence.moxy -->
     <dependency>
@@ -576,7 +576,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>3.2.5</version>
+        <version>3.5.4</version>
         <configuration>
           <excludes>
             <exclude>${modules.test.excludes}</exclude>
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/Controller.java b/src/main/java/edu/ucsb/nceas/mdqengine/Controller.java
@@ -290,8 +290,7 @@ public void initTests(int cnt) {
      */
     public void readConfig() throws ConfigurationException, IOException {
         MDQconfig cfg = new MDQconfig();
-
-        RabbitMQpassword = cfg.getString("RabbitMQ.password");
+        RabbitMQpassword = System.getenv("RABBITMQ_AUTH");
         RabbitMQusername = cfg.getString("RabbitMQ.username");
         RabbitMQhost = cfg.getString("RabbitMQ.host");
         RabbitMQport = cfg.getInt("RabbitMQ.port");
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/Worker.java b/src/main/java/edu/ucsb/nceas/mdqengine/Worker.java
@@ -90,7 +90,7 @@ public static void main(String[] argv) throws Exception {
         Dispatcher.setupJep();
 
         try {
-            RabbitMQpassword = cfg.getString("RabbitMQ.password");
+            RabbitMQpassword = System.getenv("RABBITMQ_AUTH");
             RabbitMQusername = cfg.getString("RabbitMQ.username");
             RabbitMQhost = cfg.getString("RabbitMQ.host");
             RabbitMQport = cfg.getInt("RabbitMQ.port");
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/dispatch/Dispatcher.java b/src/main/java/edu/ucsb/nceas/mdqengine/dispatch/Dispatcher.java
@@ -156,6 +156,9 @@ public Result dispatch(Map<String, Object> variables, String code) throws Script
                     log.error(e.getStackTrace());
                     dr.setOutput(new Output("ERROR: " + e.getMessage())); // catch the python stack trace
                     dr.setStatus(Status.valueOf("ERROR"));
+                    // if it errors, just exit
+                    bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE);
+                    return dr;
                 }
                 // try to get the global output variable from python
                 try {
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/filestore/FilestoreDB.java b/src/main/java/edu/ucsb/nceas/mdqengine/filestore/FilestoreDB.java
@@ -14,9 +14,6 @@
 import java.sql.*;
 import java.util.Properties;
 
-import java.sql.Connection;
-import java.sql.DriverManager;
-
 public class FilestoreDB {
 
     private static String dbUrl = null;
@@ -37,7 +34,10 @@ private void init() throws MetadigFilestoreException {
             MDQconfig cfg = new MDQconfig();
             dbUrl = cfg.getString("jdbc.url");
             dbUser = cfg.getString("postgres.user");
-            dbPasswd = cfg.getString("postgres.passwd");
+            dbPasswd = System.getenv("PG_AUTH");
+            if (dbPasswd == null){
+                dbPasswd = System.getProperty("PG_AUTH");
+            }
 
         } catch (ConfigurationException | IOException ex) {
             log.error(ex.getMessage());
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/scheduler/JobScheduler.java b/src/main/java/edu/ucsb/nceas/mdqengine/scheduler/JobScheduler.java
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/scheduler/RequestReportJob.java b/src/main/java/edu/ucsb/nceas/mdqengine/scheduler/RequestReportJob.java
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/scorer/Scorer.java b/src/main/java/edu/ucsb/nceas/mdqengine/scorer/Scorer.java
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/store/DatabaseStore.java b/src/main/java/edu/ucsb/nceas/mdqengine/store/DatabaseStore.java
diff --git a/src/test/java/edu/ucsb/nceas/mdqengine/filestore/FilestoreTestIT.java b/src/test/java/edu/ucsb/nceas/mdqengine/filestore/FilestoreTestIT.java

Original file line number	Diff line number	Diff line change
`@@ -19,10 +19,10 @@ task-type,task-name,task-group,cron-schedule,params`
`19`	`19`	`# - requestType: for score tasks, determine type of portal processing ("portal" or "node")`
`20`	`20`	`#`
`21`	`21`	`# Dataset quality scoring tasks`
`22`		`-#quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.\|^http.eml.*;arctic.data.center.suite-1.2.0;urn:node:mnTestARCTIC;2024-05-01T00:00:00.00Z;1;1000"`
`23`		`-quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.\|^http.eml.\|.www.isotc211.org.\|^science-on-schema.org.;FAIR-suite-0.5.0;urn:node:mnTestARCTIC;2025-09-01T00:00:00.00Z;1;1000"`
`24`		`-quality,quality-test-data-suite,metadig,15 0/1 * * * ?,"^eml.\|^http.eml.*;data-suite-0.1.0;urn:node:mnTestARCTIC;2024-11-01T00:00:00.00Z;1;1000"`
`25`		`-#quality,quality-test-essdive,metadig,20 0/1 * * * ?,"^eml.\|^http.eml.*;ess-dive.data.center.suite-1.3.0;urn:node:mnTestARCTIC;2025-10-01T00:00:00.00Z;1;1000"`
	`22`	`+quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.\|^http.eml.*;arctic.data.center.suite-1.2.0;urn:node:mnTestARCTIC;2024-05-01T00:00:00.00Z;1;1000"`
	`23`	`+quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.\|^http.eml.\|.www.isotc211.org.\|^science-on-schema.org.;FAIR-suite-0.5.0;urn:node:cnSandbox;2025-09-01T00:00:00.00Z;1;1000"`
	`24`	`+quality,quality-test-data-suite,metadig,15 0/1 * * * ?,"^eml.\|^http.eml.*;data-suite-0.2.0;urn:node:mnTestARCTIC;2024colnam-11-01T00:00:00.00Z;1;1000"`
	`25`	`+`
`26`	`26`
`27`	`27`	`#`
`28`	`28`	`# Portal scoring tasks`