fix[backend](index-removal): added index verification before removal

Author: AlexSanchez-bit

backend/src/main/java/com/park/utmstack/service/elasticsearch/ElasticsearchService.java

@@ -10,6 +10,7 @@
 import com.park.utmstack.service.MailService;
 import com.park.utmstack.service.UtmSpaceNotificationControlService;
 import com.park.utmstack.service.application_events.ApplicationEventService;
+import com.park.utmstack.service.index_policy.IndexPolicyService;
 import com.park.utmstack.util.chart_builder.IndexPropertyType;
 import com.park.utmstack.util.exceptions.OpenSearchIndexNotFoundException;
 import com.park.utmstack.util.exceptions.UtmElasticsearchException;
@@ -57,16 +58,19 @@ public class ElasticsearchService {
     private final UserRepository userRepository;
     private final MailService mailService;
     private final UtmSpaceNotificationControlService spaceNotificationControlService;
+    private final IndexPolicyService indexPolicyService;
     private final OpensearchClientBuilder client;
 
     public ElasticsearchService(ApplicationEventService eventService, UserRepository userRepository,
                                 MailService mailService,
                                 UtmSpaceNotificationControlService spaceNotificationControlService,
+                                IndexPolicyService indexPolicyService,
                                 OpensearchClientBuilder client) {
         this.eventService = eventService;
         this.userRepository = userRepository;
         this.mailService = mailService;
         this.spaceNotificationControlService = spaceNotificationControlService;
+        this.indexPolicyService = indexPolicyService;
         this.client = client;
     }
 
@@ -284,6 +288,11 @@ private void deleteOldestIndices() {
                 if (opt.isEmpty() || opt.get().getResume().getDiskUsedPercent() < 70)
                     break;
 
+                if (!indexPolicyService.isIndexRemovable(index.index())) {
+                    log.info("{} Skipping index {} because it is not in a removable state", ctx, index.index());
+                    continue;
+                }
+
                 try {
                     // Delete oldest indices
                     deleteIndex(Collections.singletonList(index.index()));

backend/src/main/java/com/park/utmstack/service/index_policy/IndexPolicyService.java

@@ -1,6 +1,7 @@
 package com.park.utmstack.service.index_policy;
 
 import com.google.gson.Gson;
+import com.google.gson.JsonObject;
 import com.park.utmstack.config.Constants;
 import com.park.utmstack.domain.index_pattern.enums.SystemIndexPattern;
 import com.park.utmstack.domain.index_policy.*;
@@ -37,6 +38,45 @@ public IndexPolicyService(ApplicationContext applicationContext,
         this.client = client;
     }
 
+    /**
+     * Check if an index is in a removable state (STATE_DELETE or STATE_SAFE_DELETE)
+     *
+     * @param index: Index name
+     * @return True if removable, false otherwise
+     */
+    public boolean isIndexRemovable(String index) {
+        final String ctx = CLASSNAME + ".isIndexRemovable";
+        try {
+            final String url = String.format("_plugins/_ism/explain/%1$s", index);
+            try (Response rs = client.getClient().executeHttpRequest(url, null, null, HttpMethod.GET)) {
+                if (!rs.isSuccessful() || rs.body() == null)
+                    return false;
+
+                String body = rs.body().string();
+                JsonObject json = new Gson().fromJson(body, JsonObject.class);
+                if (json != null && json.has(index)) {
+                    JsonObject indexInfo = json.getAsJsonObject(index);
+                    String state = null;
+                    if (indexInfo.has("index.plugins.index_state_management.current_state")) {
+                        state = indexInfo.get("index.plugins.index_state_management.current_state").getAsString();
+                    } else if (indexInfo.has("index.opendistro.index_state_management.current_state")) {
+                        state = indexInfo.get("index.opendistro.index_state_management.current_state").getAsString();
+                    } else if (indexInfo.has("opendistro.index_state_management.current_state")) {
+                        state = indexInfo.get("opendistro.index_state_management.current_state").getAsString();
+                    }
+
+                    if (state != null) {
+                        return Constants.STATE_DELETE.equals(state) || Constants.STATE_SAFE_DELETE.equals(state);
+                    }
+                }
+            }
+            return false;
+        } catch (Exception e) {
+            log.error("{}: {}", ctx, e.getMessage());
+            return false;
+        }
+    }
+
     @EventListener(IndexPatternsReadyEvent.class)
     public void init() throws Exception {
         final String ctx = CLASSNAME + ".init";