[2.43] fix: Stage.ou dimension metadata with LEVEL/OU_GROUP selectors [DHIS2-21595]

.github/workflows/run-api-analytics-tests-clickhouse.yml

@@ -0,0 +1,139 @@
+name: Run analytics api tests [ClickHouse]
+env:
+  # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
+  # PR → quiet (unless ci-verbose label), push/schedule → info, workflow_dispatch → user choice
+  MAVEN_ARGS: ${{ (inputs.maven_log == 'debug' && '-X') || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-verbose') && '-q') || (inputs.maven_log == 'quiet' && '-q') || '' }}
+
+on:
+  pull_request:
+    types: [opened, labeled, synchronize]
+  schedule:
+    - cron: "26 1 * * *" # Run at 01:36 AM
+  workflow_dispatch:
+    inputs:
+      maven_log:
+        description: "Maven log verbosity"
+        required: false
+        default: "quiet"
+        type: choice
+        options:
+          - quiet
+          - info
+          - debug
+concurrency:
+  group: ${{ github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  api-analytics-test-clickhouse:
+    env:
+      CORE_IMAGE_NAME: "dhis2/core-dev:local"
+      PR_NUMBER: ${{ github.event.number }}
+      DOCKER_CHANNEL: "dhis2/core-pr"
+      SIERRA_LEONE_DB_PATH: "/tmp/db/sierra-leone"
+      SIERRA_LEONE_DB_VERSION: "2.39.6"
+      GITHUB_TOKEN: ${{ secrets.DHIS2_BOT_GITHUB_TOKEN }}
+      SSH_SIGNING_KEY: ${{ secrets.DHIS2_BOT_SSH_SIGNING_KEY }}
+      SSH_SIGNING_PASSPHRASE: ${{ secrets.DHIS2_BOT_SSH_SIGNING_PASSPHRASE }}
+
+    runs-on: ubuntu-22.04
+    if: contains(github.event.pull_request.labels.*.name, 'run-api-analytics-tests-clickhouse') || (github.event_name == 'workflow_dispatch')
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Free disk space on runner (Ubuntu)
+        shell: bash
+        run: |
+          # Remove Android and .NET libs (approx. 14GB)
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v5
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: maven
+
+      - name: Build container image
+        run: |
+          # only build image for running api tests in this workflow
+          mvn clean package --threads 2C --batch-mode --no-transfer-progress \
+            -DskipTests -Dmaven.test.skip=true --update-snapshots --file dhis-2/pom.xml \
+            --projects dhis-web-server --also-make --activate-profiles jibDockerBuild \
+            -Djib.to.image=$CORE_IMAGE_NAME
+
+      - name: Get Sierra Leone DB from cache
+        uses: actions/cache@v5
+        id: cache-sierra-leone-db
+        with:
+          path: ${{ env.SIERRA_LEONE_DB_PATH }}
+          key: sierra-leone-db-analytics-${{ env.SIERRA_LEONE_DB_VERSION }}
+
+      - name: Download Sierra Leone DB
+        if: ${{ steps.cache-sierra-leone-db.outputs.cache-hit != 'true' }}
+        run: |
+          mkdir -p ${{ env.SIERRA_LEONE_DB_PATH }}
+          wget https://databases.dhis2.org/sierra-leone/${{ env.SIERRA_LEONE_DB_VERSION }}/analytics_be/dhis2-db-sierra-leone.sql.gz --no-clobber --directory-prefix ${{ env.SIERRA_LEONE_DB_PATH }}
+
+      - name: Run tests
+        run: |
+          cd dhis-2/dhis-test-e2e
+          source setup_clickhouse_links
+          DHIS2_IMAGE="$CORE_IMAGE_NAME" \
+          DHIS2_E2E_TEST_DUMP_PATH="$SIERRA_LEONE_DB_PATH" \
+          CLICKHOUSE_VERSION=24.8 \
+          docker compose -f docker-compose.yml -f docker-compose.clickhouse.yml -f docker-compose.e2e.yml \
+            -f docker-compose.e2e-analytics.yml -f docker-compose.clickhouse-analytics.yml \
+            up --attach test --remove-orphans --exit-code-from test
+
+      # - name: Push surefire reports to repository
+      #   if: always()
+      #   run: |
+      #     cd dhis-2/dhis-test-e2e
+      #     DHIS2_IMAGE="$CORE_IMAGE_NAME" \
+      #     DHIS2_E2E_TEST_DUMP_PATH="$SIERRA_LEONE_DB_PATH" \
+      #     CLICKHOUSE_VERSION=24.8 \
+      #     TEST_TYPE=e2e-analytics \
+      #     DB_TYPE=clickhouse \
+      #     docker compose -f docker-compose.yml -f docker-compose.clickhouse.yml -f docker-compose.e2e.yml -f docker-compose.e2e-analytics.yml -f docker-compose.clickhouse-analytics.yml --profile reports run reports-processor
+
+      # - name: Add a comment to the pull request
+      #   if: always()
+      #   uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9
+      #   with:
+      #     issue-number: ${{ github.event.pull_request.number }}
+      #     body: |
+      #       Surefire reports for analytics API tests against ClickHouse have been pushed to the repository. [View reports](https://dhis2.github.io/dhis2-test-reports/reports/?component=core&testType=e2e-analytics)
+
+      - name: Upload logs
+        if: failure()
+        run: |
+          cd dhis-2/dhis-test-e2e
+          docker compose logs web > ~/logs.txt
+
+      - uses: actions/upload-artifact@v7
+        if: failure()
+        with:
+          name: "tomcat_logs"
+          path: "~/logs.txt"
+
+  send-slack-message:
+    runs-on: ubuntu-22.04
+    if: |
+      always() &&
+      contains(needs.*.result, 'failure') &&
+      github.ref == 'refs/heads/master'
+
+    needs: [api-analytics-test-clickhouse]
+    steps:
+      - uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_BACKEND_WEBHOOK }}
+          SLACK_CHANNEL: "team-backend"
+          SLACK_MESSAGE: "Latest analytics e2e test run against ClickHouse on master failed and needs investigation :detective-duck:. \n Commit message: ${{ github.event.head_commit.message }}"
+          SLACK_COLOR: "#ff0000"

.github/workflows/run-api-analytics-tests-doris.yml

@@ -33,6 +33,8 @@ jobs:
       SIERRA_LEONE_DB_PATH: "/tmp/db/sierra-leone"
       SIERRA_LEONE_DB_VERSION: "2.39.6"
       GITHUB_TOKEN: ${{ secrets.DHIS2_BOT_GITHUB_TOKEN }}
+      SSH_SIGNING_KEY: ${{ secrets.DHIS2_BOT_SSH_SIGNING_KEY }}
+      SSH_SIGNING_PASSPHRASE: ${{ secrets.DHIS2_BOT_SSH_SIGNING_PASSPHRASE }}
 
     runs-on: ubuntu-22.04
     if: contains(github.event.pull_request.labels.*.name, 'run-api-analytics-tests-doris') || (github.event_name == 'workflow_dispatch')
@@ -85,26 +87,6 @@ jobs:
             -f docker-compose.e2e-analytics.yml -f docker-compose.doris-analytics.yml \
             up --remove-orphans --exit-code-from test
 
-      - name: Push surefire reports to repository
-        if: always()
-        run: |
-          cd dhis-2/dhis-test-e2e
-          DHIS2_IMAGE="$CORE_IMAGE_NAME" \
-          DHIS2_E2E_TEST_DUMP_PATH="$SIERRA_LEONE_DB_PATH" \
-          DORIS_VERSION=3.0.7 \
-          TEST_TYPE=e2e-analytics \
-          DB_TYPE=doris \
-          docker compose -f docker-compose.yml -f docker-compose.doris.yml -f docker-compose.e2e.yml -f docker-compose.e2e-analytics.yml -f docker-compose.doris-analytics.yml --profile reports run reports-processor
-
-      - name: Add a comment to the pull request
-        if: always()
-        uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          body: |
-            Surefire reports for analytics API tests against Doris have been pushed to the repository. [View reports](https://dhis2.github.io/dhis2-test-reports/reports/?component=core&testType=e2e-analytics)
-
-
       - name: Upload logs
         if: failure()
         run: |
@@ -132,4 +114,3 @@ jobs:
           SLACK_CHANNEL: "team-backend"
           SLACK_MESSAGE: "Latest analytics e2e test run against Doris on master failed and needs investigation :detective-duck:. \n Commit message: ${{ github.event.head_commit.message }}"
           SLACK_COLOR: "#ff0000"
-

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DimensionalObjectProvider.java

@@ -459,18 +459,50 @@ public List<String> getOrgUnitDimensionUid(
    *     its keywords.
    * @param userOrgUnits the list of organisation units associated with the current user.
    * @param expandGroupsAndLevels if true, expands LEVEL-X and OU_GROUP-X to their member org units.
-   *     This is needed for SQL filtering but should be false for metadata generation.
+   *     Explicit org units are kept in the result (union semantics). This is needed for SQL
+   *     filtering but should be false for metadata generation.
    * @return a list of {@link OrganisationUnit} UIDs.
    */
   public List<String> getOrgUnitDimensionUid(
       List<String> items, List<OrganisationUnit> userOrgUnits, boolean expandGroupsAndLevels) {
+    return getOrgUnitDimensionUid(items, userOrgUnits, expandGroupsAndLevels, false);
+  }
+
+  /**
+   * This method will return a list of {@link OrganisationUnit} UIDs based on the given items and
+   * user organisation units.
+   *
+   * @param items the list of items that might be included into the resulting organisation unit and
+   *     its keywords.
+   * @param userOrgUnits the list of organisation units associated with the current user.
+   * @param expandGroupsAndLevels if true, expands LEVEL-X and OU_GROUP-X to their member org units.
+   *     This is needed for SQL filtering but should be false for metadata generation.
+   * @param orgUnitsAsBoundaries if true and LEVEL-X / OU_GROUP-X selectors are present, explicit
+   *     org units only scope the expansion and are not included in the result (stage.ou dimension
+   *     semantics). If false, explicit org units are kept alongside the expanded members (union
+   *     semantics, used by org unit typed data element filters). Only relevant when
+   *     expandGroupsAndLevels is true.
+   * @return a list of {@link OrganisationUnit} UIDs.
+   */
+  public List<String> getOrgUnitDimensionUid(
+      List<String> items,
+      List<OrganisationUnit> userOrgUnits,
+      boolean expandGroupsAndLevels,
+      boolean orgUnitsAsBoundaries) {
     List<Integer> levels = new ArrayList<>();
     List<OrganisationUnitGroup> groups = new ArrayList<>();
 
     List<DimensionalItemObject> ous =
         getOrgUnitDimensionItems(items, userOrgUnits, IdScheme.UID, levels, groups);
 
-    List<String> result = new ArrayList<>(ous.stream().map(DimensionalItemObject::getUid).toList());
+    boolean hasLevelsOrGroups = !levels.isEmpty() || !groups.isEmpty();
+
+    List<String> result = new ArrayList<>();
+
+    // With boundary semantics, OUs combined with levels / groups only scope the expansion
+    if (!(expandGroupsAndLevels && orgUnitsAsBoundaries && hasLevelsOrGroups)) {
+      result.addAll(ous.stream().map(DimensionalItemObject::getUid).toList());
+    }
 
     if (expandGroupsAndLevels) {
       List<OrganisationUnit> ousList = asTypedList(ous);

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java

@@ -1816,7 +1816,7 @@ public String toSql(QueryItem item, QueryFilter filter, EventQueryParams params)
 
     String filterString =
         item.getValueType() == ValueType.ORGANISATION_UNIT
-            ? organisationUnitResolver.resolveOrgUnits(filter, params.getUserOrgUnits())
+            ? organisationUnitResolver.resolveOrgUnits(filter, params.getUserOrgUnits(), item)
             : filter.getFilter();
 
     if (IN.equals(filter.getOperator())) {
@@ -3140,7 +3140,7 @@ private Condition buildInFilterCondition(
     // Resolve special org unit keywords like USER_ORGUNIT if applicable
     String resolvedFilter =
         filterBuilder.requiresOrgUnitResolution(item)
-            ? organisationUnitResolver.resolveOrgUnits(filter, params.getUserOrgUnits())
+            ? organisationUnitResolver.resolveOrgUnits(filter, params.getUserOrgUnits(), item)
             : filter.getFilter();
 
     InQueryCteFilter inQueryCteFilter =
@@ -3191,7 +3191,7 @@ private String getSqlFilterValue(QueryFilter filter, QueryItem item, EventQueryP
     // Resolve special org unit keywords like USER_ORGUNIT if applicable
     String filterValue =
         filterBuilder.requiresOrgUnitResolution(item)
-            ? organisationUnitResolver.resolveOrgUnits(filter, params.getUserOrgUnits())
+            ? organisationUnitResolver.resolveOrgUnits(filter, params.getUserOrgUnits(), item)
             : filter.getFilter();
 
     if ("NV".equals(filterValue)) {

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/OrganisationUnitResolver.java

@@ -89,16 +89,38 @@ public class OrganisationUnitResolver {
   /**
    * Resolve organisation units like ou:USER_ORGUNIT;USER_ORGUNIT_CHILDREN;LEVEL-XXX;OUGROUP-XXX
    * into a list of organisation unit dimension uids. Groups and levels are expanded to their member
-   * org units for SQL filtering purposes.
+   * org units for SQL filtering purposes. Explicit org units are kept in the result (union
+   * semantics, used by org unit typed data element filters).
    *
    * @param queryFilter the query filter containing the organisation unit filter
    * @param userOrgUnits the user organisation units
    * @return the organisation unit dimension uids
    */
   public String resolveOrgUnits(QueryFilter queryFilter, List<OrganisationUnit> userOrgUnits) {
+    return resolveOrgUnits(queryFilter, userOrgUnits, false);
+  }
+
+  /**
+   * Resolve organisation units from the given query filter, applying boundary semantics when the
+   * item is a stage.ou dimension (explicit org units only scope the LEVEL-/OU_GROUP- expansion) and
+   * union semantics otherwise.
+   *
+   * @param queryFilter the query filter containing the organisation unit filter
+   * @param userOrgUnits the user organisation units
+   * @param item the query item the filter belongs to
+   * @return the organisation unit dimension uids
+   */
+  public String resolveOrgUnits(
+      QueryFilter queryFilter, List<OrganisationUnit> userOrgUnits, QueryItem item) {
+    return resolveOrgUnits(queryFilter, userOrgUnits, isStageOuDimension(item));
+  }
+
+  private String resolveOrgUnits(
+      QueryFilter queryFilter, List<OrganisationUnit> userOrgUnits, boolean orgUnitsAsBoundaries) {
     List<String> filterItem = QueryFilter.getFilterItems(queryFilter.getFilter());
     List<String> orgUnitDimensionUid =
-        dimensionalObjectProducer.getOrgUnitDimensionUid(filterItem, userOrgUnits, true);
+        dimensionalObjectProducer.getOrgUnitDimensionUid(
+            filterItem, userOrgUnits, true, orgUnitsAsBoundaries);
 
     // Throw E7143 if no valid org units were resolved (mirrors standard ou: dimension behavior)
     if (orgUnitDimensionUid.isEmpty()) {
@@ -120,7 +142,8 @@ public Map<String, MetadataItem> getMetadataItemsForOrgUnitDataElements(EventQue
     for (QueryItem queryItem : params.getItems()) {
       if (queryItem.getValueType().isOrganisationUnit()) {
         for (QueryFilter queryFilter : queryItem.getFilters()) {
-          String resolveOrgUnits = resolveOrgUnits(queryFilter, params.getUserOrgUnits());
+          String resolveOrgUnits =
+              resolveOrgUnits(queryFilter, params.getUserOrgUnits(), queryItem);
           if (StringUtils.isNotBlank(resolveOrgUnits)) {
             orgUnitIds.addAll(Arrays.asList(resolveOrgUnits.split(OPTION_SEP)));
           }
@@ -162,7 +185,7 @@ private MetadataItem toMetadataItem(OrganisationUnit orgUnit, EventQueryParams p
    */
   public List<String> resolveOrgUnits(EventQueryParams params, QueryItem item) {
     return item.getFilters().stream()
-        .map(queryFilter -> resolveOrgUnits(queryFilter, params.getUserOrgUnits()))
+        .map(queryFilter -> resolveOrgUnits(queryFilter, params.getUserOrgUnits(), item))
         .map(s -> s.split(DimensionConstants.OPTION_SEP))
         .flatMap(Arrays::stream)
         .distinct()
@@ -224,9 +247,12 @@ public Map<Integer, List<OrganisationUnit>> resolveOrgUnitsGroupedByLevel(
   public DimensionalItemObject loadOrgUnitDimensionalItem(
       @Nonnull String dimensionUid, @Nonnull IdScheme idScheme) {
     if (dimensionUid.startsWith(KEY_LEVEL)) {
+      String levelToken = substringAfterLast(dimensionUid, SEPARATOR);
       OrganisationUnitLevel level =
-          idObjectManager.getObject(
-              OrganisationUnitLevel.class, idScheme, substringAfterLast(dimensionUid, SEPARATOR));
+          StringUtils.isNumeric(levelToken)
+              ? organisationUnitService.getOrganisationUnitLevelByLevel(
+                  Integer.parseInt(levelToken))
+              : idObjectManager.getObject(OrganisationUnitLevel.class, idScheme, levelToken);
 
       if (level != null) {
         BaseDimensionalItemObject dim = new BaseDimensionalItemObject();

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/QueryItemFilterBuilder.java

@@ -101,7 +101,7 @@ public String extractFiltersAsSql(QueryItem item, String columnName, EventQueryP
               boolean needsResolution = requiresOrgUnitResolution(item);
               String resolvedFilter =
                   needsResolution
-                      ? organisationUnitResolver.resolveOrgUnits(f, params.getUserOrgUnits())
+                      ? organisationUnitResolver.resolveOrgUnits(f, params.getUserOrgUnits(), item)
                       : f.getFilter();
 
               return buildFilterCondition(f, item, columnName, resolvedFilter);
@@ -228,7 +228,7 @@ public String extractNonNvFiltersAsSql(
               boolean needsResolution = requiresOrgUnitResolution(item);
               String resolvedFilter =
                   needsResolution
-                      ? organisationUnitResolver.resolveOrgUnits(f, params.getUserOrgUnits())
+                      ? organisationUnitResolver.resolveOrgUnits(f, params.getUserOrgUnits(), item)
                       : f.getFilter();
 
               // For IN operator with mixed values, only include non-NV values