feat[backend](updated filters and rules): added removed rules and filters routines

Author: AlexSanchez-bit

backend/src/main/java/com/park/utmstack/domain/application_modules/enums/ModuleName.java

@@ -66,5 +66,7 @@ public enum ModuleName {
     ORACLE,
     SURICATA,
     UTMSTACK,
-    CROWDSTRIKE
+    CROWDSTRIKE,
+    SYSLOG_GENERIC,
+    WINDOWS_EVENTS
 }

backend/src/main/java/com/park/utmstack/domain/logstash_filter/UtmLogstashFilter.java

@@ -65,8 +65,9 @@ public class UtmLogstashFilter implements Serializable {
     private Instant updatedAt;
 
     @JsonIgnore
-    @ManyToOne(fetch = FetchType.LAZY)
-    @JoinColumn(name = "module_name", referencedColumnName = "module_name",insertable = false, updatable = false)
+    @ManyToOne(fetch = FetchType.LAZY, optional = true)
+    @org.hibernate.annotations.NotFound(action = org.hibernate.annotations.NotFoundAction.IGNORE)
+    @JoinColumn(name = "module_name", referencedColumnName = "module_name", insertable = false, updatable = false, nullable = true)
     private UtmModule module;
 
     public UtmLogstashFilter() {

backend/src/main/java/com/park/utmstack/repository/correlation/rules/UtmCorrelationRulesRepository.java

@@ -56,5 +56,7 @@ Page<UtmCorrelationRules> searchByFilters(@Param("ruleName") String ruleName,
 
     Optional<UtmCorrelationRules> findOneByRuleName(String ruleName);
 
+    List<UtmCorrelationRules> findAllBySystemOwnerIsTrue();
+
     Optional<UtmCorrelationRules> findFirstBySystemOwnerIsTrueOrderByIdDesc();
 }

backend/src/main/java/com/park/utmstack/repository/logstash_filter/UtmLogstashFilterRepository.java

@@ -20,6 +20,8 @@ public interface UtmLogstashFilterRepository extends JpaRepository<UtmLogstashFi
 
     void deleteAllBySystemOwnerIsTrueAndIdNotIn(List<Long> ids);
 
+    List<UtmLogstashFilter> findAllBySystemOwnerIsTrue();
+
     @Query(nativeQuery = true, value = "select utm_logstash_filter.* from utm_logstash_filter where :nameShort = any(string_to_array(utm_logstash_filter.module_name, ','))")
     List<UtmLogstashFilter> findAllByModuleName(@Param("nameShort") String nameShort);
 

backend/src/main/java/com/park/utmstack/service/DefinitionSyncService.java

@@ -21,7 +21,9 @@
 import org.springframework.transaction.annotation.Transactional;
 import org.yaml.snakeyaml.Yaml;
 
+import javax.annotation.PostConstruct;
 import java.io.IOException;
+import java.lang.Exception;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -44,26 +46,37 @@ public class DefinitionSyncService implements CommandLineRunner {
     private final UtmLogstashFilterService filterService;
 
     @Override
-    @Transactional
     public void run(String... args) {
-        log.info("Starting definition sync from filesystem...");
-        syncFilters();
-        syncRules();
-        log.info("Definition sync completed.");
+        log.info("Starting definition sync from filesystem... ---");
+        try {
+            Set<String> filesystemFilters = syncFilters();
+            Set<String> filesystemRules = syncRules();
+
+            cleanupOrphanedFilters(filesystemFilters);
+            cleanupOrphanedRules(filesystemRules);
+
+            log.info("Definition sync completed successfully. ---");
+        } catch (Exception e) {
+            log.error("CRITICAL: Definition sync failed. Reason: {} ---", e.getMessage(), e);
+        }
     }
 
-    private void syncFilters() {
+    private Set<String> syncFilters() {
+        Set<String> foundModules = new HashSet<>();
         Path filtersPath = Paths.get(".",Constants.APP_FILTER_DEFINITIONS);
         if (!Files.exists(filtersPath) || !Files.isDirectory(filtersPath)) {
             log.warn("Filters directory not found: {}", Constants.APP_FILTER_DEFINITIONS);
-            return;
+            return foundModules;
         }
 
         try (Stream<Path> paths = Files.walk(filtersPath)) {
             paths.filter(path -> Files.isRegularFile(path) && isYamlFile(path)).forEach(path -> {
-                String moduleName = getFileNameWithoutExtension(path);
+                String rawName = getFileNameWithoutExtension(path);
+                String moduleName = rawName.toUpperCase().replace("-", "_");
+                foundModules.add(moduleName);
                 try {
                     String content = Files.readString(path);
+
                     Optional<UtmLogstashFilter> filterOpt = filterRepository.findOneByModuleName(moduleName);
 
                     if (filterOpt.isPresent()) {
@@ -75,7 +88,6 @@ private void syncFilters() {
                             filterService.save(filter, true);
                         }
                     } else {
-                        log.info("Inserting new filter for module: {}", moduleName);
                         UtmLogstashFilter filter = new UtmLogstashFilter();
                         filter.setModuleName(moduleName);
                         filter.setFilterName(moduleName + " Filter");
@@ -99,13 +111,15 @@ private void syncFilters() {
         } catch (IOException e) {
             log.error("Error listing filters directory: {}", e.getMessage());
         }
+        return foundModules;
     }
 
-    private void syncRules() {
+    private Set<String> syncRules() {
+        Set<String> foundRules = new HashSet<>();
         Path rulesPath = Paths.get(".",Constants.APP_RULE_DEFINITIONS);
         if (!Files.exists(rulesPath) || !Files.isDirectory(rulesPath)) {
             log.warn("Rules directory not found: {}", Constants.APP_RULE_DEFINITIONS);
-            return;
+            return foundRules;
         }
 
         Yaml yaml = new Yaml();
@@ -120,6 +134,7 @@ private void syncRules() {
                         return;
                     }
 
+                    foundRules.add(ruleYaml.getName());
                     Optional<UtmCorrelationRules> ruleOpt = rulesRepository.findOneByRuleName(ruleYaml.getName());
                     UtmCorrelationRulesDTO ruleDto = new UtmCorrelationRulesDTO();
 
@@ -173,6 +188,35 @@ private void syncRules() {
         } catch (IOException e) {
             log.error("Error walking rules directory: {}", e.getMessage());
         }
+        return foundRules;
+    }
+
+    private void cleanupOrphanedFilters(Set<String> currentFilesystemModules) {
+        if (currentFilesystemModules.isEmpty()) return;
+
+        List<UtmLogstashFilter> systemFilters = filterRepository.findAllBySystemOwnerIsTrue();
+        systemFilters.stream()
+            .filter(filter -> !currentFilesystemModules.contains(filter.getModuleName()))
+            .forEach(filter -> {
+                log.info("Deleting orphaned system filter: {}", filter.getModuleName());
+                filterService.delete(filter.getId());
+            });
+    }
+
+    private void cleanupOrphanedRules(Set<String> currentFilesystemRules) {
+        if (currentFilesystemRules.isEmpty()) return;
+
+        List<UtmCorrelationRules> systemRules = rulesRepository.findAllBySystemOwnerIsTrue();
+        systemRules.stream()
+            .filter(rule -> !currentFilesystemRules.contains(rule.getRuleName()))
+            .forEach(rule -> {
+                log.info("Deleting orphaned system rule: {}", rule.getRuleName());
+                try {
+                    rulesService.deleteRule(rule.getId(), true);
+                } catch (Exception e) {
+                    log.error("Error deleting orphaned system rule {}: {}", rule.getRuleName(), e.getMessage());
+                }
+            });
     }
 
     private boolean isYamlFile(Path path) {

backend/src/main/java/com/park/utmstack/service/correlation/rules/UtmCorrelationRulesService.java

@@ -158,12 +158,17 @@ public void setRuleActivation(Long ruleId, boolean setActive) throws Exception {
      * */
     @Transactional
     public void deleteRule (Long id) throws Exception {
+        deleteRule(id, false);
+    }
+
+    @Transactional
+    public void deleteRule (Long id, boolean forcedSystemMode) throws Exception {
         final String ctx = CLASSNAME + ".deleteRule";
         Optional<UtmCorrelationRules> find = utmCorrelationRulesRepository.findById(id);
         if (find.isEmpty()) {
             throw new BadRequestException(ctx + ": The rule you're trying to delete is not present in database.");
         }
-        if(find.get().getSystemOwner()) {
+        if(find.get().getSystemOwner() && !forcedSystemMode) {
             throw new BadRequestException(ctx + ": System's rules can't be removed.");
         }
         utmCorrelationRulesRepository.deleteById(id);