use message broker to process server info

Author: slamdev

README.md

@@ -8,9 +8,9 @@ Inventory manager for [Hetzner Robot](https://robot.your-server.de)
 export HETZNER_IROBO_ROBOT_USERNAME="xxx"
 export HETZNER_IROBO_ROBOT_PASSWORD="xxx"
 export HETZNER_IROBO_ROBOT_SSH_KEY="xxx"
+export HETZNER_IROBO_ZABBIX_URL="xxx"
 export HETZNER_IROBO_ZABBIX_USERNAME="xxx"
 export HETZNER_IROBO_ZABBIX_PASSWORD="xxx"
-export ZABBIX_API_URL="xxx"
 docker-compose --profile=complete up
 ```
 

build.gradle

@@ -27,7 +27,7 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-validation'
     implementation 'org.springframework.cloud:spring-cloud-starter-sleuth'
     implementation 'org.springframework.cloud:spring-cloud-sleuth-zipkin'
-    implementation 'org.springframework.retry:spring-retry'
+    implementation 'io.nats:nats-spring-cloud-stream-binder:0.5.0'
     implementation 'ch.qos.logback:logback-access'
     compileOnly 'org.projectlombok:lombok'
     developmentOnly 'org.springframework.boot:spring-boot-devtools'
@@ -37,7 +37,6 @@ dependencies {
     implementation 'com.github.spotbugs:spotbugs-annotations:4.4.1'
     implementation 'com.google.guava:guava:31.0.1-jre'
     implementation 'com.google.mug:mug:5.5'
-    implementation 'io.github.mlniang:spring-zabbix-api-client:0.1.1'
     implementation 'com.hierynomus:sshj:0.31.0'
     implementation 'org.postgresql:postgresql'
     runtimeOnly 'org.flywaydb:flyway-core'
@@ -53,4 +52,6 @@ dependencies {
     testCompileOnly 'org.projectlombok:lombok'
     testImplementation 'org.testcontainers:junit-jupiter'
     testImplementation 'org.testcontainers:cockroachdb'
+    testImplementation group: 'org.springframework.cloud', name: 'spring-cloud-stream', classifier: 'test-binder'
+    testRuntimeOnly 'com.h2database:h2'
 }

docker-compose.yaml

@@ -1,5 +1,10 @@
 version: "3.8"
 services:
+  nats:
+    image: nats:2.6.1-alpine3.14
+    ports:
+      - "4222:4222"
+      - "8222:8222"
   cockroachdb:
     image: cockroachdb/cockroach:v21.1.9
     command:
@@ -28,13 +33,14 @@ services:
     profiles: [ "complete" ]
     depends_on:
       - create-db
+      - nats
     environment:
       - HETZNER_IROBO_ROBOT_USERNAME
       - HETZNER_IROBO_ROBOT_PASSWORD
       - HETZNER_IROBO_ROBOT_SSH_KEY
+      - HETZNER_IROBO_ZABBIX_URL
       - HETZNER_IROBO_ZABBIX_USERNAME
       - HETZNER_IROBO_ZABBIX_PASSWORD
-      - ZABBIX_API_URL
       - SPRING_PROFILES_ACTIVE=cloud
     deploy:
       restart_policy:

lombok.config

@@ -1,3 +1,4 @@
 lombok.extern.findbugs.addSuppressFBWarnings=true
 lombok.anyConstructor.addConstructorProperties=true
 lombok.anyConstructor.suppressConstructorProperties=false
+lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Qualifier

src/main/java/com/github/slamdev/hetzner/irobo/HetznerIroboApplication.java

@@ -1,19 +1,13 @@
 package com.github.slamdev.hetzner.irobo;
 
 import com.github.slamdev.hetzner.irobo.integration.AppConfig;
-import net.javacrumbs.shedlock.spring.annotation.EnableSchedulerLock;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.data.jdbc.repository.config.EnableJdbcAuditing;
-import org.springframework.retry.annotation.EnableRetry;
-import org.springframework.scheduling.annotation.EnableScheduling;
 
 @SpringBootApplication
 @EnableJdbcAuditing
-@EnableScheduling
-@EnableRetry
-@EnableSchedulerLock(defaultLockAtMostFor = "1m")
 @EnableConfigurationProperties(AppConfig.class)
 public class HetznerIroboApplication {
 

src/main/java/com/github/slamdev/hetzner/irobo/business/boundary/HetznerScheduledUpdater.java

@@ -0,0 +1,51 @@
+package com.github.slamdev.hetzner.irobo.business.boundary;
+
+import com.github.slamdev.hetzner.irobo.business.control.HetznerServerUpdater;
+import com.github.slamdev.hetzner.irobo.business.control.UpdateHistoryRepository;
+import com.github.slamdev.hetzner.irobo.business.entity.UpdateHistoryModel;
+import com.github.slamdev.hetzner.irobo.integration.AppConfig;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.time.Instant;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+import static com.github.slamdev.hetzner.irobo.business.entity.UpdateHistoryModel.UpdateType.HETZNER_SERVER_LIST;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class HetznerScheduledUpdater {
+
+    private final HetznerServerUpdater hetznerServerUpdater;
+    private final UpdateHistoryRepository updateHistoryRepository;
+    private final AppConfig appConfig;
+
+    @Scheduled(fixedDelay = 5, timeUnit = TimeUnit.MINUTES, initialDelay = 1)
+    @SchedulerLock(name = "hetzner-servers-list", lockAtLeastFor = "10s", lockAtMostFor = "30s")
+    public void scheduleUpdateServersList() {
+        Optional<UpdateHistoryModel> history = updateHistoryRepository.findFirstByUpdateTypeOrderByExecutedDateDesc(HETZNER_SERVER_LIST);
+        boolean updateRequired = history
+                .map(UpdateHistoryModel::getExecutedDate)
+                .map(d -> d.plus(appConfig.getUpdateInterval().getHetznerServersList()).isBefore(Instant.now()))
+                .orElse(true);
+
+        if (!updateRequired) {
+            log.debug("last update {} is newer than {}; no action will be performed", history, appConfig.getUpdateInterval().getHetznerServersList());
+            return;
+        }
+
+        hetznerServerUpdater.updateServers();
+
+        UpdateHistoryModel updatedHistory = updateHistoryRepository.save(UpdateHistoryModel.builder()
+                .updateType(HETZNER_SERVER_LIST)
+                .executedDate(Instant.now())
+                .build());
+
+        log.info("servers list updated: {}", updatedHistory);
+    }
+}

src/main/java/com/github/slamdev/hetzner/irobo/business/boundary/HetznerServerConsumer.java

@@ -0,0 +1,34 @@
+package com.github.slamdev.hetzner.irobo.business.boundary;
+
+import com.github.slamdev.hetzner.irobo.business.control.ServerFactory;
+import com.github.slamdev.hetzner.irobo.business.control.ServerRepository;
+import com.github.slamdev.hetzner.irobo.business.entity.HetznerServerMessage;
+import com.github.slamdev.hetzner.irobo.business.entity.ServerModel;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+import java.util.function.Consumer;
+
+@Slf4j
+@RequiredArgsConstructor
+@Component("hetznerServerConsumer")
+public class HetznerServerConsumer implements Consumer<HetznerServerMessage> {
+
+    private final ServerRepository serverRepository;
+    private final ServerFactory serverFactory;
+
+    @Override
+    public void accept(HetznerServerMessage msg) {
+        if (msg.isCancelled()) {
+            serverRepository.deleteById(msg.getId());
+            log.info("server {} is deleted because it is canceled", msg.getId());
+            return;
+        }
+
+        ServerModel server = serverFactory.create(msg);
+        server = serverRepository.save(server);
+
+        log.info("server updated: {}", server);
+    }
+}

src/main/java/com/github/slamdev/hetzner/irobo/business/boundary/HetznerServerProducer.java

@@ -0,0 +1,18 @@
+package com.github.slamdev.hetzner.irobo.business.boundary;
+
+import com.github.slamdev.hetzner.irobo.business.entity.HetznerServerMessage;
+import lombok.RequiredArgsConstructor;
+import org.springframework.cloud.stream.function.StreamBridge;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+public class HetznerServerProducer {
+
+    private static final String DESTINATION = "hetznerServerProducer-out-0";
+    private final StreamBridge streamBridge;
+
+    public void sendHetznerServer(HetznerServerMessage server) {
+        streamBridge.send(DESTINATION, server);
+    }
+}

src/main/java/com/github/slamdev/hetzner/irobo/business/boundary/IndexController.java

@@ -2,11 +2,9 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.slamdev.hetzner.irobo.business.control.ServerRepository;
-import com.github.slamdev.hetzner.irobo.business.control.ServerSearchRepository;
 import com.github.slamdev.hetzner.irobo.business.entity.ServerListViewModel;
 import com.github.slamdev.hetzner.irobo.business.entity.ServerModel;
 import com.github.slamdev.hetzner.irobo.integration.AppConfig;
-import com.github.slamdev.hetzner.irobo.integration.Streams;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import org.springframework.data.domain.Sort;
@@ -27,18 +25,17 @@
 public class IndexController {
 
     private final ServerRepository serverRepository;
-    private final ServerSearchRepository serverSearchRepository;
     private final ObjectMapper objectMapper;
     private final AppConfig appConfig;
 
     @GetMapping
-    public ModelAndView indexPage(@RequestParam(defaultValue = "serverNumber") String sortField,
+    public ModelAndView indexPage(@RequestParam(defaultValue = "id") String sortField,
                                   @RequestParam(defaultValue = "ASC") Sort.Direction sortDirection,
                                   @RequestParam(defaultValue = "") String filter) {
         String[] fields;
         switch (sortField) {
             case "externalIp":
-                fields = new String[]{"serverIpV4", "serverIpV6"};
+                fields = new String[]{"ipV4", "ipV6"};
                 break;
             case "internalIp":
                 fields = new String[]{"zabbixIp"};
@@ -51,19 +48,19 @@ public ModelAndView indexPage(@RequestParam(defaultValue = "serverNumber") Strin
                 break;
         }
         Sort sort = Sort.by(sortDirection, fields);
-        Iterable<ServerModel> results;
+        List<ServerModel> results;
         if (filter.trim().isEmpty()) {
             results = serverRepository.findAll(sort);
         } else {
-            List<Integer> ids = serverSearchRepository.findAllBySearchDataLike("%" + filter.toLowerCase(Locale.ROOT) + "%");
-            results = serverRepository.findAllByServerNumberIsIn(ids, sort);
+            String searchKeywords = "%" + filter.trim().toLowerCase(Locale.ROOT) + "%";
+            results = serverRepository.findAllBySearchKeywordsLike(searchKeywords, sort);
         }
-        List<ServerListViewModel> servers = Streams.stream(results)
+        List<ServerListViewModel> servers = results.stream()
                 .map(s -> ServerListViewModel.builder()
-                        .serverNumber(s.getServerNumber())
-                        .serverName(s.getServerName())
+                        .id(s.getId())
+                        .name(s.getName())
                         .hostName(s.getZabbixHost())
-                        .externalIp(s.getServerIpV4() == null ? s.getServerIpV6() : s.getServerIpV4())
+                        .externalIp(s.getIpV4() == null ? s.getIpV6() : s.getIpV4())
                         .internalIp(s.getZabbixIp())
                         .product(s.getProduct())
                         .dc(s.getDc())
@@ -83,7 +80,7 @@ private String buildZabbixUrl(ServerModel server) {
     }
 
     private String buildHetznerUrl(ServerModel server) {
-        return appConfig.getRobot().getWebUrl() + "/server?text=" + server.getServerNumber();
+        return appConfig.getRobot().getWebUrl() + "/server?text=" + server.getId();
     }
 
     @GetMapping("details")