Merge pull request #1550 from utmstack/release/v11.1.6

Release/v11.1.6

Author: mjabascal10

CHANGELOG.md

@@ -1,8 +1,8 @@
-# UTMStack 11.1.5 – Release Notes
+# UTMStack 11.1.6 – Release Notes
 
-The **UTMStack v11.1.5** update delivers important fixes and usability improvements to enhance stability and user experience.
+The **UTMStack v11.1.6** update delivers important fixes and usability improvements to enhance stability and user experience.
 
 ## Improvements & Fixes
-- Standardized `utm_visualization` field names by replacing legacy O365 keys with new conventions.
-- Enhanced responsive behavior for TFA enrollment components based on viewport height.
+- Enhanced Threat and Windows activity dashboards with new filters and aggregations for better data analysis.
+- Improved email notifications for alerts, providing clearer information and enhanced formatting for better user experience.
 

backend/src/main/java/com/park/utmstack/domain/shared_types/alert/Event.java

@@ -3,12 +3,20 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.park.utmstack.util.MapUtil;
 import lombok.Data;
+import lombok.Getter;
+import org.springframework.util.StringUtils;
 
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 @Data
+@Getter
 @JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class Event {
@@ -39,5 +47,18 @@ public class Event {
 
     private List<String> errors;
     private Map<String, ComplianceValues> compliance;
+
+    public Map<String, String> getLogxFlatted() {
+        return MapUtil.flattenToStringMap(log, true);
+    }
+
+    public String getTimestampFormatted() {
+        try {
+            return StringUtils.hasText(timestamp) ? DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withLocale(Locale.getDefault()).withZone(
+                    ZoneId.systemDefault()).format(Instant.parse(timestamp)) : null;
+        } catch (Exception e) {
+            return null;
+        }
+    }
 }
 

backend/src/main/java/com/park/utmstack/repository/network_scan/UtmNetworkScanRepository.java

@@ -114,8 +114,8 @@ void updateGroup(@Param("assetIds") List<Long> assetIds,
 
     @Modifying
     @Query(nativeQuery = true, value = "with sources as (select ds.source from utm_data_input_status ds where ds.data_type in :types)" +
-        " delete from utm_network_scan asset where asset.asset_ip in (select src.source from sources src) " +
-        "or asset.asset_name in (select src.source from sources src)")
+        " delete from utm_network_scan asset where (asset.asset_ip in (select src.source from sources src) " +
+        "or asset.asset_name in (select src.source from sources src)) and asset.is_agent is false")
     void deleteAllAssetsByDataType(@Param("types") List<String> types);
 
     @Query("select distinct n.assetOsPlatform from UtmNetworkScan n where n.assetOsPlatform is not null and n.isAgent is true")
@@ -129,4 +129,11 @@ void updateGroup(@Param("assetIds") List<Long> assetIds,
             "JOIN UtmAssetGroup ag ON ns.groupId = ag.id " +
             "WHERE ns.groupId IS NOT NULL")
     List<Object[]> findAllAssetGroupMappings();
+
+    List<UtmNetworkScan> findByAssetIpInOrAssetNameIn(List<String> assetIps, List<String> assetNames);
+
+    List<UtmNetworkScan> findByIsAgentTrue();
+
+    List<UtmNetworkScan> findByAssetNameIn(List<String> assetNames);
+
 }

backend/src/main/java/com/park/utmstack/security/saml/Saml2LoginSuccessHandler.java

@@ -59,13 +59,11 @@ public void onAuthenticationSuccess(HttpServletRequest request,
 
         Collection<? extends GrantedAuthority> authorities = Objects.requireNonNull(user.getAuthorities())
                 .stream()
-                .map(Objects::toString)
-                .filter(r -> r.startsWith("ROLE_"))
-                .map(SimpleGrantedAuthority::new)
+                .map(a -> new SimpleGrantedAuthority(a.getName()))
                 .toList();
 
         UsernamePasswordAuthenticationToken auth =
-                new UsernamePasswordAuthenticationToken((Object) username, null, authorities);
+                new UsernamePasswordAuthenticationToken(username, null, authorities);
 
         SecurityContextHolder.getContext().setAuthentication(auth);
 

backend/src/main/java/com/park/utmstack/service/MailService.java

@@ -5,6 +5,7 @@
 import com.park.utmstack.domain.application_events.enums.ApplicationEventType;
 import com.park.utmstack.domain.incident.UtmIncident;
 import com.park.utmstack.domain.mail_sender.MailConfig;
+import com.park.utmstack.domain.shared_types.alert.Event;
 import com.park.utmstack.domain.shared_types.alert.UtmAlert;
 import com.park.utmstack.domain.shared_types.LogType;
 import com.park.utmstack.service.application_events.ApplicationEventService;
@@ -250,7 +251,7 @@ public void sendPasswordResetMail(User user) {
     }
 
     @Async
-    public void sendAlertEmail(List<String> emailsTo, UtmAlert alert, List<LogType> relatedLogs) {
+    public void sendAlertEmail(List<String> emailsTo, UtmAlert alert, List<Event> relatedLogs) {
         final String ctx = CLASS_NAME + ".sendAlertEmail";
         try {
             JavaMailSender javaMailSender = getJavaMailSender();
@@ -333,7 +334,7 @@ public void sendIncidentEmail(List<String> emailsTo, List<UtmAlert> alerts, UtmI
      * @throws Exception In case of any error
      */
     private ByteArrayResource buildAlertEmailAttachment(Context context, UtmAlert alert,
-                                                        List<LogType> relatedLogs) throws Exception {
+                                                        List<Event> relatedLogs) throws Exception {
         final String ctx = CLASS_NAME + ".buildAlertEmailAttachment";
         try {
             ByteArrayOutputStream bout = new ByteArrayOutputStream();
@@ -351,9 +352,9 @@ private ByteArrayResource buildAlertEmailAttachment(Context context, UtmAlert al
         }
     }
 
-    private void buildRelatedEventCsvAttachment(List<LogType> relatedLogs, ZipOutputStream zipOut) {
+    private void buildRelatedEventCsvAttachment(List<Event> relatedLogs, ZipOutputStream zipOut) {
         final String ctx = CLASS_NAME + ".buildRelatedEventCsvAttachment";
-        Map<String, List<LogType>> evtTypes = new HashMap<>();
+        Map<String, List<Event>> evtTypes = new HashMap<>();
 
         // Separating event types
         relatedLogs.forEach(doc -> {

backend/src/main/java/com/park/utmstack/service/UtmAlertTagRuleService.java

@@ -269,7 +269,7 @@ private void assignAssetGroupsToReviewAlerts() {
                                     ctx._source.assetGroupName = '%s';
                                   }
                                 """,
-                        assetName.replace("'", "\\'"), // Escapar comillas simples
+                        assetName.replace("'", "\\'"),
                         groupId,
                         groupName.replace("'", "\\'")
                 ));
@@ -282,7 +282,7 @@ private void assignAssetGroupsToReviewAlerts() {
             List<FilterType> filters = new ArrayList<>();
             filters.add(new FilterType(Constants.alertStatus, OperatorType.IS,
                     AlertStatus.AUTOMATIC_REVIEW.getCode()));
-            filters.add(new FilterType("dataSource", OperatorType.IS_NOT, null));
+            filters.add(new FilterType("dataSource", OperatorType.EXIST, null));
 
             Query query = SearchUtil.toQuery(filters);
             String indexPattern = Constants.SYS_INDEX_PATTERN.get(SystemIndexPattern.ALERTS);

backend/src/main/java/com/park/utmstack/service/UtmDataInputStatusService.java

@@ -38,6 +38,7 @@
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 
@@ -157,7 +158,7 @@ private void checkDataInputStatus(List<UtmDataInputStatus> inputs, String server
 
             long currentTimeInSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
             List<UtmDataInputStatus> inTime = inputs.stream().filter(row -> (currentTimeInSeconds - row.getTimestamp()) < 3600)
-                    .collect(Collectors.toList());
+                    .toList();
             if (!CollectionUtils.isEmpty(inTime))
                 return;
 
@@ -219,7 +220,7 @@ public void syncDataInputStatus() {
      * Gets the sources from utm_data_input_status that are not registered in utm_network_scan table
      * and create new assets with it. This method is a schedule with a delay of 1 hour
      */
-    @Scheduled(fixedDelay = 15000, initialDelay = 30000)
+    @Scheduled(fixedDelay = 30000, initialDelay = 60000)
     public void syncSourcesToAssets() {
         final String ctx = CLASSNAME + ".syncSourcesToAssets";
         try {
@@ -231,55 +232,73 @@ public void syncSourcesToAssets() {
         }
     }
 
+    @Transactional
     public void synchronizeSourcesToAssets() {
         final String ctx = CLASSNAME + ".syncSourcesToAssets";
+
         try {
-            final List<String> excludeOfTypes = dataTypesRepository.findAllByIncludedFalse().stream()
-                    .map(UtmDataTypes::getDataType).collect(Collectors.toList());
-            excludeOfTypes.addAll(Arrays.asList("utmstack", "UTMStack", DataSourceConstants.IBM_AS400_TYPE));
-
-            List<UtmDataInputStatus> sources = dataInputStatusRepository.extractSourcesToExport(excludeOfTypes);
-            if (!CollectionUtils.isEmpty(sources)) {
-                //return;
-
-                Map<String, Boolean> sourcesWithStatus = extractSourcesWithUpDownStatus(sources);
-                List<UtmNetworkScan> assets = networkScanService.findAll();
-
-                List<UtmNetworkScan> saveOrUpdate = new ArrayList<>();
-                sourcesWithStatus.forEach((key, value) -> {
-                    Optional<UtmNetworkScan> assetOpt = assets.stream()
-                            .filter(asset -> ((StringUtils.hasText(asset.getAssetIp()) && asset.getAssetIp().equals(key))
-                                    || (StringUtils.hasText(asset.getAssetName()) && asset.getAssetName().equals(key))))
-                            .findFirst();
-                    if (assetOpt.isPresent()) {
-                        UtmNetworkScan utmAsset = assetOpt.get();
-                        if (Objects.isNull(utmAsset.getUpdateLevel())
-                                || utmAsset.getUpdateLevel().equals(UpdateLevel.DATASOURCE)) {
-                            utmAsset.assetAlive(value)
-                                    .updateLevel(UpdateLevel.DATASOURCE)
-                                    .assetStatus(AssetStatus.CHECK)
-                                    .modifiedAt(LocalDateTime.now().toInstant(ZoneOffset.UTC));
-                            saveOrUpdate.add(utmAsset);
-                        }
-                    } else {
-                        saveOrUpdate.add(new UtmNetworkScan(key, value));
-                    }
-                });
-
-                assets.forEach(asset -> {
-                    if (!sourcesWithStatus.containsKey(asset.getAssetIp()) && !sourcesWithStatus.containsKey(asset.getAssetName())
-                            && !Objects.isNull(asset.getUpdateLevel()) && asset.getUpdateLevel().equals(UpdateLevel.DATASOURCE)) {
-                        asset.assetStatus(AssetStatus.MISSING).updateLevel(null)
-                                .modifiedAt(LocalDateTime.now().toInstant(ZoneOffset.UTC));
-                        saveOrUpdate.add(asset);
-                    }
-                });
 
-                networkScanService.saveAll(saveOrUpdate);
+            List<String> excludeDataTypes = dataTypesRepository.findAllByIncludedFalse()
+                    .stream()
+                    .map(UtmDataTypes::getDataType)
+                    .collect(Collectors.toList());
+
+            excludeDataTypes.addAll(Arrays.asList("utmstack", "UTMStack", DataSourceConstants.IBM_AS400_TYPE));
+
+            List<UtmDataInputStatus> sources = dataInputStatusRepository.extractSourcesToExport(excludeDataTypes);
+            if (CollectionUtils.isEmpty(sources)) {
+                return;
             }
-            // Finally, delete excluded assets
-            networkScanRepository.deleteAllAssetsByDataType(excludeOfTypes);
+
+            Map<String, Boolean> sourcesWithStatus = extractSourcesWithUpDownStatus(sources);
+
+            List<String> keys = new ArrayList<>(sourcesWithStatus.keySet());
+            List<UtmNetworkScan> assets = networkScanRepository.findByAssetIpInOrAssetNameIn(keys, keys);
+
+            Map<String, UtmNetworkScan> assetsByKey = new HashMap<>();
+
+            assets.forEach(a -> {
+                if (StringUtils.hasText(a.getAssetIp())) assetsByKey.put(a.getAssetIp(), a);
+                if (StringUtils.hasText(a.getAssetName())) assetsByKey.put(a.getAssetName(), a);
+            });
+
+            for (Map.Entry<String, Boolean> entry : sourcesWithStatus.entrySet()) {
+                String key = entry.getKey();
+                Boolean alive = entry.getValue();
+
+                UtmNetworkScan asset = assetsByKey.get(key);
+
+                if (asset != null) {
+                    if (asset.getUpdateLevel() == null || asset.getUpdateLevel().equals(UpdateLevel.DATASOURCE) || asset.getUpdateLevel().equals(UpdateLevel.AGENT)) {
+                        asset.assetAlive(alive)
+                             .updateLevel(UpdateLevel.DATASOURCE)
+                             .assetStatus(AssetStatus.CHECK)
+                             .modifiedAt(LocalDateTime.now().toInstant(ZoneOffset.UTC));
+
+                        networkScanService.save(asset);
+                    }
+                } else {
+                    networkScanService.save(new UtmNetworkScan(key, alive));
+                }
+            }
+
+            assets.forEach(asset -> {
+                boolean missing = !sourcesWithStatus.containsKey(asset.getAssetIp())
+                        && !sourcesWithStatus.containsKey(asset.getAssetName());
+
+                if (missing && UpdateLevel.DATASOURCE.equals(asset.getUpdateLevel())) {
+                    asset.assetStatus(AssetStatus.MISSING)
+                          .updateLevel(null)
+                          .modifiedAt(LocalDateTime.now().toInstant(ZoneOffset.UTC));
+
+                    networkScanService.save(asset);
+                }
+            });
+
+           networkScanRepository.deleteAllAssetsByDataType(excludeDataTypes);
+
         } catch (Exception e) {
+            log.error("{}: Error synchronizing sources to assets - {}", ctx, e.getMessage(), e);
             throw new RuntimeException(ctx + ": " + e.getLocalizedMessage());
         }
     }