Merge branch 'master' into fix-userrole-jsonpatch-lazy-members

Author: jason-p-pickering

.github/workflows/run-auth-idp-tests.yml

@@ -0,0 +1,103 @@
+name: Run auth IdP tests
+
+# External-IdP authentication e2e suite (Keycloak OIDC, LDAP): runs the dhis-test-e2e
+# "auth-idp" tagged tests against the docker-compose.e2e-auth.yml stack.
+# Runs on every PR (reusing the dhis2/core-pr image published by "Run api tests"
+# when possible), nightly on master, and manually.
+
+env:
+  MAVEN_OPTS: -Xmx1024m -Xms1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
+
+on:
+  pull_request:
+  schedule:
+    # Nightly on the default branch
+    - cron: "30 3 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  auth-idp-test:
+    runs-on: ubuntu-latest
+    env:
+      # Mirrors DOCKERHUB_PUSH in run-api-tests.yml: for same-repo PRs the api tests
+      # workflow pushes dhis2/core-pr:<number> to Docker Hub, so we pull it instead of
+      # rebuilding. Fork/dependabot PRs, nightly and dispatch runs build the image here.
+      REUSE_PR_IMAGE: ${{ github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
+      CORE_IMAGE_NAME: ${{ (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]') && format('dhis2/core-pr:{0}', github.event.number) || 'dhis2/core-dev:local' }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up JDK 17
+        if: env.REUSE_PR_IMAGE != 'true'
+        uses: actions/setup-java@v5
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: maven
+
+      - name: Wait for PR image from api tests build
+        if: env.REUSE_PR_IMAGE == 'true'
+        run: |
+          expected="${{ github.event.pull_request.head.sha }}"
+          echo "Waiting for $CORE_IMAGE_NAME with revision $expected"
+          for i in $(seq 1 60); do
+            if docker pull --quiet "$CORE_IMAGE_NAME" > /dev/null 2>&1; then
+              revision=$(docker inspect --format '{{ index .Config.Labels "DHIS2_BUILD_REVISION" }}' "$CORE_IMAGE_NAME")
+              if [ "$revision" = "$expected" ]; then
+                echo "Found $CORE_IMAGE_NAME for revision $revision"
+                exit 0
+              fi
+              echo "Image revision '$revision' is stale, retrying..."
+            fi
+            sleep 20
+          done
+          echo "Timed out waiting for $CORE_IMAGE_NAME with revision $expected (did the api tests build fail?)" >&2
+          exit 1
+
+      - name: Build container image
+        if: env.REUSE_PR_IMAGE != 'true'
+        run: |
+          mvn clean verify --threads 2C --batch-mode --no-transfer-progress \
+            -DskipTests -Dpackaging.type=jar -Dmdep.analyze.skip --update-snapshots --file dhis-2/pom.xml \
+            --projects dhis-web-server --also-make --activate-profiles embedded,jibDockerBuild \
+            -Djib.to.image=$CORE_IMAGE_NAME
+
+      - name: Run auth IdP tests
+        run: |
+          cd dhis-2/dhis-test-e2e
+          DHIS2_E2E_TEST_ARGS="-Pauth-idp" SELENIUM_IMAGE=selenium/standalone-chrome:latest DHIS2_IMAGE=$CORE_IMAGE_NAME \
+            docker compose -f docker-compose.yml -f docker-compose.e2e.yml -f docker-compose.e2e-auth.yml \
+            up --remove-orphans --exit-code-from test
+
+      - name: Upload logs
+        if: failure()
+        run: |
+          cd dhis-2/dhis-test-e2e
+          docker compose -f docker-compose.yml -f docker-compose.e2e.yml -f docker-compose.e2e-auth.yml logs web > ~/web-logs.txt
+          docker compose -f docker-compose.yml -f docker-compose.e2e.yml -f docker-compose.e2e-auth.yml logs keycloak > ~/keycloak-logs.txt
+
+      - uses: actions/upload-artifact@v7
+        if: failure()
+        with:
+          name: "auth_idp_logs"
+          path: |
+            ~/web-logs.txt
+            ~/keycloak-logs.txt
+
+  send-slack-message:
+    runs-on: ubuntu-latest
+    if: |
+      always() &&
+      contains(needs.*.result, 'failure') &&
+      github.event_name == 'schedule'
+    needs: [auth-idp-test]
+    steps:
+      - uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_BACKEND_WEBHOOK }}
+          SLACK_CHANNEL: "team-backend"
+          SLACK_MESSAGE: "Nightly auth IdP (Keycloak/LDAP) e2e run failed and needs investigation :detective-duck:."
+          SLACK_COLOR: "#ff0000"

dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryOptionComboStore.java

@@ -44,6 +44,18 @@ public interface CategoryOptionComboStore extends IdentifiableObjectStore<Catego
   CategoryOptionCombo getCategoryOptionCombo(
       CategoryCombo categoryCombo, Set<CategoryOption> categoryOptions);
 
+  /**
+   * Returns the {@link CategoryOptionCombo}s of the given {@link CategoryCombo}s with their {@link
+   * CategoryOptionCombo#getCategoryOptions()} eagerly fetched in a single query. Used to prime the
+   * session before serialising category option combos, avoiding the per-combo N+1 select on the
+   * {@code categoryoptioncombos_categoryoptions} join table.
+   *
+   * @param categoryCombos the category combos whose option combos to load.
+   * @return the option combos with their category options initialised.
+   */
+  List<CategoryOptionCombo> getCategoryOptionCombosWithCategoryOptions(
+      Collection<CategoryCombo> categoryCombos);
+
   void updateNames();
 
   void deleteNoRollBack(CategoryOptionCombo categoryOptionCombo);

dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryService.java

@@ -412,6 +412,18 @@ CategoryOptionCombo getCategoryOptionCombo(
    */
   List<CategoryOptionCombo> getAllCategoryOptionCombos();
 
+  /**
+   * Returns the {@link CategoryOptionCombo}s of the given {@link CategoryCombo}s with their
+   * category options eagerly fetched in a single query. Used to prime the session before
+   * serialising category option combos, avoiding the per-combo N+1 select on the {@code
+   * categoryoptioncombos_categoryoptions} join table.
+   *
+   * @param categoryCombos the category combos whose option combos to load.
+   * @return the option combos with their category options initialised.
+   */
+  List<CategoryOptionCombo> getCategoryOptionCombosWithCategoryOptions(
+      Collection<CategoryCombo> categoryCombos);
+
   /**
    * Generates and persists a default Category, CategoryOption, CategoryCombo and
    * CategoryOptionCombo.

dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/auth/OAuth2ClientCredentialsAuthScheme.java

@@ -30,6 +30,7 @@
 package org.hisp.dhis.common.auth;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.function.UnaryOperator;
@@ -44,6 +45,7 @@
 import org.springframework.context.ApplicationContext;
 import org.springframework.security.authentication.AbstractAuthenticationToken;
 import org.springframework.security.core.Authentication;
+import org.springframework.security.core.GrantedAuthority;
 import org.springframework.security.oauth2.client.OAuth2AuthorizationContext;
 import org.springframework.security.oauth2.client.OAuth2AuthorizeRequest;
 import org.springframework.security.oauth2.client.OAuth2AuthorizedClient;
@@ -64,7 +66,7 @@ public class OAuth2ClientCredentialsAuthScheme implements AuthScheme {
   public static final String OAUTH2_CLIENT_CREDENTIALS_TYPE = "oauth2-client-credentials";
 
   public static final Authentication ANONYMOUS_AUTHENTICATION =
-      new AbstractAuthenticationToken(null) {
+      new AbstractAuthenticationToken((Collection<? extends GrantedAuthority>) null) {
         @Override
         public Object getCredentials() {
           return "";

dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/DataSetService.java

@@ -29,9 +29,11 @@
  */
 package org.hisp.dhis.dataset;
 
+import java.util.Collection;
 import java.util.Date;
 import java.util.List;
 import org.apache.commons.collections4.SetValuedMap;
+import org.hisp.dhis.dataelement.DataElement;
 import org.hisp.dhis.dataentryform.DataEntryForm;
 import org.hisp.dhis.organisationunit.OrganisationUnit;
 import org.hisp.dhis.period.Period;
@@ -83,6 +85,19 @@ public interface DataSetService extends DataSetDataIntegrityProvider {
    */
   DataSet getDataSet(String uid);
 
+  /**
+   * Returns the distinct {@link DataElement}s that are members of the given data sets, loaded in a
+   * single query (with each data element's {@code dataSetElements} and {@code categoryCombo}
+   * eagerly fetched) to avoid N+1 selects when iterating {@link DataSet#getDataElements()} and
+   * {@link DataElement#getCategoryCombos()} across many data sets. Does not apply sharing/ACL
+   * predicates on the data elements; callers are responsible for authorizing access to the data
+   * sets.
+   *
+   * @param dataSets the data sets.
+   * @return the distinct data elements of the given data sets.
+   */
+  List<DataElement> getDataElementsByDataSet(Collection<DataSet> dataSets);
+
   /**
    * Returns the DataSet with the given UID. Bypasses the ACL system.
    *

dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/DataSetStore.java

@@ -53,6 +53,22 @@ public interface DataSetStore
 
   List<DataSetElement> getDataSetElementsByDataElement(Collection<DataElement> dataElements);
 
+  /**
+   * Returns the distinct {@link DataElement}s that are members of the given data sets (via {@link
+   * DataSetElement}), loaded in a single query. Each data element's {@link
+   * DataElement#getDataSetElements()} collection and its {@link DataElement#getCategoryCombo()} are
+   * eagerly fetched as well, so that {@link DataSet#getDataElements()} and {@link
+   * DataElement#getCategoryCombos()} can be evaluated without triggering per-element (or
+   * per-category-combo) N+1 selects.
+   *
+   * <p>This method does <b>not</b> apply sharing/ACL predicates on the data elements. Callers are
+   * responsible for authorizing access to the data sets.
+   *
+   * @param dataSets the data sets.
+   * @return the distinct data elements of the given data sets.
+   */
+  List<DataElement> getDataElementsByDataSet(Collection<DataSet> dataSets);
+
   /** Get all DatSets that have any of the CategoryCombos. */
   List<DataSet> getByCategoryCombo(List<CategoryCombo> sources);
 

dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/Program.java

@@ -185,12 +185,21 @@ public class Program extends BaseMetadataObject
   @Column(name = "relationshiplabel", columnDefinition = "text")
   private String relationshipLabel;
 
+  @Column(name = "relationshipslabel", columnDefinition = "text")
+  private String relationshipsLabel;
+
   @Column(name = "notelabel", columnDefinition = "text")
   private String noteLabel;
 
+  @Column(name = "noteslabel", columnDefinition = "text")
+  private String notesLabel;
+
   @Column(name = "trackedentityattributelabel", columnDefinition = "text")
   private String trackedEntityAttributeLabel;
 
+  @Column(name = "trackedentityattributeslabel", columnDefinition = "text")
+  private String trackedEntityAttributesLabel;
+
   @Column(name = "programstagelabel", columnDefinition = "text")
   private String programStageLabel;
 
@@ -969,6 +978,24 @@ public void setRelationshipLabel(String relationshipLabel) {
     this.relationshipLabel = relationshipLabel;
   }
 
+  @JsonProperty
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  @PropertyRange(min = 2)
+  public String getRelationshipsLabel() {
+    return relationshipsLabel;
+  }
+
+  @JsonProperty
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  @Translatable(propertyName = "relationshipsLabel", key = "RELATIONSHIPS_LABEL")
+  public String getDisplayRelationshipsLabel() {
+    return getTranslation("RELATIONSHIPS_LABEL", getRelationshipsLabel());
+  }
+
+  public void setRelationshipsLabel(String relationshipsLabel) {
+    this.relationshipsLabel = relationshipsLabel;
+  }
+
   @JsonProperty
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
   @PropertyRange(min = 2)
@@ -987,6 +1014,24 @@ public void setNoteLabel(String noteLabel) {
     this.noteLabel = noteLabel;
   }
 
+  @JsonProperty
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  @PropertyRange(min = 2)
+  public String getNotesLabel() {
+    return notesLabel;
+  }
+
+  @JsonProperty
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  @Translatable(propertyName = "notesLabel", key = "NOTES_LABEL")
+  public String getDisplayNotesLabel() {
+    return getTranslation("NOTES_LABEL", getNotesLabel());
+  }
+
+  public void setNotesLabel(String notesLabel) {
+    this.notesLabel = notesLabel;
+  }
+
   @JsonProperty
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
   @PropertyRange(min = 2)
@@ -1007,6 +1052,26 @@ public void setTrackedEntityAttributeLabel(String trackedEntityAttributeLabel) {
     this.trackedEntityAttributeLabel = trackedEntityAttributeLabel;
   }
 
+  @JsonProperty
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  @PropertyRange(min = 2)
+  public String getTrackedEntityAttributesLabel() {
+    return trackedEntityAttributesLabel;
+  }
+
+  @JsonProperty
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  @Translatable(
+      propertyName = "trackedEntityAttributesLabel",
+      key = "TRACKED_ENTITY_ATTRIBUTES_LABEL")
+  public String getDisplayTrackedEntityAttributesLabel() {
+    return getTranslation("TRACKED_ENTITY_ATTRIBUTES_LABEL", getTrackedEntityAttributesLabel());
+  }
+
+  public void setTrackedEntityAttributesLabel(String trackedEntityAttributesLabel) {
+    this.trackedEntityAttributesLabel = trackedEntityAttributesLabel;
+  }
+
   @JsonProperty
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
   @PropertyRange(min = 2)
@@ -1513,14 +1578,17 @@ private static void setShallowCopyValues(
     copy.setEnrollmentLabel(original.getEnrollmentLabel());
     copy.setEnrollmentsLabel(original.getEnrollmentsLabel());
     copy.setNoteLabel(original.getNoteLabel());
+    copy.setNotesLabel(original.getNotesLabel());
     copy.setFollowUpLabel(original.getFollowUpLabel());
     copy.setOrgUnitLabel(original.getOrgUnitLabel());
     copy.setTrackedEntityAttributeLabel(original.getTrackedEntityAttributeLabel());
+    copy.setTrackedEntityAttributesLabel(original.getTrackedEntityAttributesLabel());
     copy.setProgramStageLabel(original.getProgramStageLabel());
     copy.setProgramStagesLabel(original.getProgramStagesLabel());
     copy.setEventLabel(original.getEventLabel());
     copy.setEventsLabel(original.getEventsLabel());
     copy.setRelationshipLabel(original.getRelationshipLabel());
+    copy.setRelationshipsLabel(original.getRelationshipsLabel());
     copy.setEnableChangeLog(original.isEnableChangeLog());
   }
 

dhis-2/dhis-api/src/main/java/org/hisp/dhis/user/UserRoleAuthoritiesChangedEvent.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2004-2022, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors 
+ * may be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.user;
+
+import org.hisp.dhis.common.UID;
+
+/**
+ * Event published after the authorities of a {@link UserRole} have changed, so that active sessions
+ * of users with that role can be invalidated asynchronously, outside the updating transaction and
+ * off the request thread.
+ *
+ * @author Morten Svanæs <msvanaes@dhis2.org>
+ */
+public record UserRoleAuthoritiesChangedEvent(UID userRoleUid) {}

dhis-2/dhis-api/src/main/java/org/hisp/dhis/user/UserService.java

@@ -584,6 +584,14 @@ boolean canCurrentUserCanModify(
    */
   void invalidateUserSessions(String username);
 
+  /**
+   * Returns the usernames of all users that are members of the user role with the given UID.
+   *
+   * @param roleUid the UID of the user role.
+   * @return a list of usernames.
+   */
+  List<String> getUsernamesByUserRole(@Nonnull UID roleUid);
+
   /**
    * Register a account recovery attempt for the given user account.
    *