Fix continuous analytics event export (#23857)

Author: jason-p-pickering

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java

@@ -175,10 +175,10 @@ private List<AnalyticsTable> getRegularAnalyticsTables(
 
     List<Program> programs =
         params.isSkipPrograms()
-            ? idObjectManager.getAllNoAcl(Program.class)
-            : idObjectManager.getAllNoAcl(Program.class).stream()
+            ? idObjectManager.getAllNoAcl(Program.class).stream()
                 .filter(p -> !params.getSkipPrograms().contains(p.getUid()))
-                .toList();
+                .toList()
+            : idObjectManager.getAllNoAcl(Program.class);
 
     Integer firstDataYear = availableDataYears.get(0);
     Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1);
@@ -314,12 +314,12 @@ where ax.psi in ( \
               and psi.lastupdated >= '${startDate}' \
               and psi.lastupdated < '${endDate}');""",
               Map.of(
-                  "tableName", quote(table.getName()),
+                  "tableName", quote(table.getMainName()),
                   "programId", String.valueOf(table.getProgram().getId()),
                   "startDate", toLongDate(partition.getStartDate()),
                   "endDate", toLongDate(partition.getEndDate())));
 
-      invokeTimeAndLog(sql, "Remove updated events for table: '{}'", table.getName());
+      invokeTimeAndLog(sql, "Remove updated events for table: '{}'", table.getMainName());
     }
   }
 

dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java

@@ -31,8 +31,10 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hisp.dhis.DhisConvenienceTest.createCategory;
 import static org.hisp.dhis.DhisConvenienceTest.createCategoryCombo;
@@ -1168,6 +1170,80 @@ void verifyTeaTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() {
     assertThat(sql.getValue(), containsString(String.format(ouQuery, "name", quote(tea.getUid()))));
   }
 
+  @Test
+  @DisplayName("removeUpdatedData deletes from the live table, not the staging table")
+  void removeUpdatedDataTargetsLiveTable() {
+    ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+    Program program = createProgram('A');
+
+    Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate();
+    Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate();
+    Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate();
+
+    AnalyticsTableUpdateParams params =
+        AnalyticsTableUpdateParams.newBuilder()
+            .withStartTime(startTime)
+            .withLatestPartition()
+            .build();
+
+    List<Map<String, Object>> queryResp = new ArrayList<>();
+    queryResp.add(Map.of("eventid", 1));
+
+    when(systemSettingManager.getDateSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_UPDATE))
+        .thenReturn(lastFullTableUpdate);
+    when(systemSettingManager.getDateSetting(
+            SettingKey.LAST_SUCCESSFUL_LATEST_ANALYTICS_PARTITION_UPDATE))
+        .thenReturn(lastLatestPartitionUpdate);
+    when(jdbcTemplate.queryForList(Mockito.anyString())).thenReturn(queryResp);
+    when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program));
+
+    List<AnalyticsTable> tables = subject.getAnalyticsTables(params);
+    assertThat(tables, hasSize(1));
+
+    subject.removeUpdatedData(tables);
+
+    verify(jdbcTemplate).execute(sql.capture());
+
+    String mainTableName = TABLE_PREFIX + program.getUid().toLowerCase();
+    assertThat(sql.getValue(), containsString(quote(mainTableName)));
+    assertThat(sql.getValue(), not(containsString(quote(mainTableName + STAGING_TABLE_SUFFIX))));
+  }
+
+  @Test
+  @DisplayName("getRegularAnalyticsTables excludes programs listed in skipPrograms")
+  void getRegularAnalyticsTablesExcludesSkippedPrograms() {
+    Program prA = createProgram('A');
+    Program prB = createProgram('B');
+    Program prC = createProgram('C');
+    Program prD = createProgram('D');
+
+    Set<String> skipPrograms = new HashSet<>();
+    skipPrograms.add(prC.getUid());
+    skipPrograms.add(prD.getUid());
+
+    AnalyticsTableUpdateParams params =
+        AnalyticsTableUpdateParams.newBuilder()
+            .withLastYears(2)
+            .withStartTime(START_TIME)
+            .withToday(today)
+            .withSkipPrograms(skipPrograms)
+            .build();
+
+    when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(prA, prB, prC, prD));
+    when(periodDataProvider.getAvailableYears(DATABASE))
+        .thenReturn(List.of(2018, 2019, now().getYear()));
+    when(jdbcTemplate.queryForList(Mockito.anyString(), Mockito.eq(Integer.class)))
+        .thenReturn(List.of(2018, 2019));
+
+    List<AnalyticsTable> tables = subject.getAnalyticsTables(params);
+
+    assertThat(tables, hasSize(2));
+
+    List<String> programUids = tables.stream().map(t -> t.getProgram().getUid()).toList();
+    assertThat(programUids, not(hasItem(prC.getUid())));
+    assertThat(programUids, not(hasItem(prD.getUid())));
+  }
+
   private String toSelectExpression(String template, String uid) {
     return String.format(template, uid, uid, uid);
   }