Merge branch 'release/v10.8.1' of https://github.com/utmstack/UTMStack into release/v10.8.1

Author: Kbayero

CHANGELOG.md

@@ -5,4 +5,10 @@
 - Added support for RedHat; UTMStack can now be installed on both Ubuntu and RedHat.
 - Improved log delivery from ARM-based agents on Windows, now sending native system logs.
 - Added support for macOS ARM64; agents can now be installed on that platform.
-- Improved agent information displayed in the Sources panel, providing more accurate OS details and agent versions.
+- Improved agent information displayed in the Sources panel, providing more accurate OS details and agent versions.
+
+
+### Bug Fixes
+-- Compliance Report Scheduling: Improved the stability of the selection process when creating new report schedules.
+-- Improved field rendering in Log Explorer by consolidating list-based fields into a single entry for better readability and consistency.
+-- Improved field rendering for tags and note fields in Alerts.

aws/configuration/const.go

@@ -3,8 +3,9 @@ package configuration
 import "github.com/utmstack/UTMStack/aws/utils"
 
 const (
-	CORRELATIONURL       = "http://correlation:8080/v1/newlog"
 	URL_CHECK_CONNECTION = "https://sts.amazonaws.com"
+	LogstashEndpoint     = "http://%s:%s"
+	UTMLogSeparator      = "<utm-log-separator>"
 )
 
 func GetInternalKey() string {
@@ -14,3 +15,11 @@ func GetInternalKey() string {
 func GetPanelServiceName() string {
 	return utils.Getenv("PANEL_SERV_NAME")
 }
+
+func GetLogstashHost() string {
+	return utils.Getenv("UTM_LOGSTASH_HOST")
+}
+
+func GetLogstashPort() string {
+	return utils.Getenv("UTM_LOGSTASH_PORT")
+}

aws/processor/pull.go

@@ -18,7 +18,7 @@ func PullLogs(startTime time.Time, endTime time.Time, group types.ModuleGroup) *
 		return err
 	}
 
-	err = SendToCorrelation(logs)
+	err = SendToLogstash(logs)
 	if err != nil {
 		return err
 	}

aws/processor/sendData.go

@@ -1,32 +1,71 @@
 package processor
 
 import (
+	"bytes"
+	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"net/http"
+	"strings"
+	"time"
 
 	"github.com/threatwinds/logger"
 	"github.com/utmstack/UTMStack/aws/configuration"
 	"github.com/utmstack/UTMStack/aws/utils"
 )
 
-func SendToCorrelation(data []TransformedLog) *logger.Error {
+var transport = &http.Transport{
+	MaxIdleConns:          100,
+	IdleConnTimeout:       2 * time.Second,
+	ResponseHeaderTimeout: 2 * time.Second,
+	ForceAttemptHTTP2:     true,
+	TLSClientConfig: &tls.Config{
+		InsecureSkipVerify: true,
+	},
+}
+
+var client = &http.Client{Transport: transport, Timeout: 2 * time.Second}
+
+func SendToLogstash(data []TransformedLog) *logger.Error {
+	var logStrings []string
 	for _, log := range data {
 		body, err := json.Marshal(log)
 		if err != nil {
 			utils.Logger.ErrorF("error encoding log to JSON: %v", err)
 			continue
 		}
+		logStrings = append(logStrings, string(body))
+	}
 
-		_, status, e := utils.DoReq[map[string]interface{}](configuration.CORRELATIONURL, body, http.MethodPost, map[string]string{})
-		if e != nil {
-			utils.Logger.ErrorF("error sending log to correlation engine: %v", e)
-			continue
-		} else if status != http.StatusOK && status != http.StatusCreated {
-			utils.Logger.ErrorF("error sending log to correlation engine: status %v", status)
-			continue
+	if len(logStrings) == 0 {
+		return nil
+	}
+
+	var logs string
+	for _, str := range logStrings {
+		logs += str + configuration.UTMLogSeparator
+	}
+
+	url := fmt.Sprintf(configuration.LogstashEndpoint, configuration.GetLogstashHost(), configuration.GetLogstashPort())
+
+	req, err := http.NewRequest("POST", url, bytes.NewBufferString(logs))
+	if err != nil {
+		return utils.Logger.ErrorF("error creating request: %v", err.Error())
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		if !strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
+			utils.Logger.ErrorF("error sending logs with error: %v", err.Error())
 		}
+		return utils.Logger.ErrorF("error sending logs: %v", err.Error())
+	}
+	defer resp.Body.Close()
 
+	if resp.StatusCode != http.StatusOK {
+		return utils.Logger.ErrorF("error sending logs with http code %d", resp.StatusCode)
 	}
 
+	utils.Logger.Info("successfully sent %d logs to Logstash", len(logStrings))
 	return nil
 }

backend/src/main/resources/config/liquibase/changelog/20250507001_add_aws_pipeline.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+
+    <changeSet id="20250507001" author="Manuel">
+        <sql dbms="postgresql" splitStatements="true" stripComments="true">
+
+            INSERT INTO utm_logstash_pipeline (id, pipeline_id, pipeline_name, parent_pipeline, pipeline_status, module_name, system_owner, pipeline_description, pipeline_internal, events_in, events_filtered, events_out, reloads_successes, reloads_failures, reloads_last_failure_timestamp, reloads_last_error, reloads_last_success_timestamp)
+            VALUES (55, 'aws', 'AWS', null, 'up', 'AWS', true, null, false, 0, 0, 0, 0, 0, null, null, null);
+
+            INSERT INTO utm_group_logstash_pipeline_filters (filter_id, pipeline_id, relation)
+            VALUES (101, 55, 'PIPELINE_FILTER');
+
+            INSERT INTO utm_logstash_input (id, pipeline_id, input_pretty_name, input_plugin, input_with_ssl, system_owner)
+            VALUES (68, 55, 'HTTP', 'http', false, true);
+
+            INSERT INTO utm_logstash_input_configuration (id, input_id, conf_key, conf_value, conf_type, conf_required, conf_validation_regex, system_owner)
+            VALUES (68, 68, 'http_port', '10048', 'port', true, '^((6553[0-5])|(655[0-2][0-9])|(65[0-4][0-9]{2})|(6[0-4][0-9]{3})|([1-5][0-9]{4})|([0-5]{0,5})|([0-9]{1,4}))$', true);
+
+
+        </sql>
+    </changeSet>
+</databaseChangeLog>

backend/src/main/resources/config/liquibase/changelog/20250507002_add_sophos_central_pipeline.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+
+    <changeSet id="20250507002" author="Manuel">
+        <sql dbms="postgresql" splitStatements="true" stripComments="true">
+
+            INSERT INTO utm_logstash_pipeline (id, pipeline_id, pipeline_name, parent_pipeline, pipeline_status, module_name, system_owner, pipeline_description, pipeline_internal, events_in, events_filtered, events_out, reloads_successes, reloads_failures, reloads_last_failure_timestamp, reloads_last_error, reloads_last_success_timestamp)
+            VALUES (56, 'sophos-central', 'Sophos Central', null, 'up', 'AWS', true, null, false, 0, 0, 0, 0, 0, null, null, null);
+
+            INSERT INTO utm_group_logstash_pipeline_filters (filter_id, pipeline_id, relation)
+            VALUES (102, 56, 'PIPELINE_FILTER');
+
+            INSERT INTO utm_logstash_input (id, pipeline_id, input_pretty_name, input_plugin, input_with_ssl, system_owner)
+            VALUES (69, 56, 'HTTP', 'http', false, true);
+
+            INSERT INTO utm_logstash_input_configuration (id, input_id, conf_key, conf_value, conf_type, conf_required, conf_validation_regex, system_owner)
+            VALUES (69, 69, 'http_port', '10049', 'port', true, '^((6553[0-5])|(655[0-2][0-9])|(65[0-4][0-9]{2})|(6[0-4][0-9]{3})|([1-5][0-9]{4})|([0-5]{0,5})|([0-9]{1,4}))$', true);
+
+
+
+        </sql>
+    </changeSet>
+</databaseChangeLog>

backend/src/main/resources/config/liquibase/changelog/20250507003_add_o365_pipeline.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+
+    <changeSet id="20250507003" author="Manuel">
+        <sql dbms="postgresql" splitStatements="true" stripComments="true">
+
+            INSERT INTO utm_logstash_pipeline (id, pipeline_id, pipeline_name, parent_pipeline, pipeline_status, module_name, system_owner, pipeline_description, pipeline_internal, events_in, events_filtered, events_out, reloads_successes, reloads_failures, reloads_last_failure_timestamp, reloads_last_error, reloads_last_success_timestamp)
+            VALUES (57, 'o365', 'Office 365', null, 'up', 'AWS', true, null, false, 0, 0, 0, 0, 0, null, null, null);
+
+            INSERT INTO utm_group_logstash_pipeline_filters (filter_id, pipeline_id, relation)
+            VALUES (103, 57, 'PIPELINE_FILTER');
+
+            INSERT INTO utm_logstash_input (id, pipeline_id, input_pretty_name, input_plugin, input_with_ssl, system_owner)
+            VALUES (70, 57, 'HTTP', 'http', false, true);
+
+            INSERT INTO utm_logstash_input_configuration (id, input_id, conf_key, conf_value, conf_type, conf_required, conf_validation_regex, system_owner)
+            VALUES (70, 70, 'http_port', '10050', 'port', true, '^((6553[0-5])|(655[0-2][0-9])|(65[0-4][0-9]{2})|(6[0-4][0-9]{3})|([1-5][0-9]{4})|([0-5]{0,5})|([0-9]{1,4}))$', true);
+
+
+
+        </sql>
+    </changeSet>
+</databaseChangeLog>

backend/src/main/resources/config/liquibase/master.xml

@@ -91,5 +91,11 @@
 
     <include file="/config/liquibase/changelog/20250418001_add_options_module_group_config.xml" relativeToChangelogFile="false"/>
 
+    <include file="/config/liquibase/changelog/20250507001_add_aws_pipeline.xml" relativeToChangelogFile="false"/>
+
+    <!--<include file="/config/liquibase/changelog/20250507002_add_sophos_central_pipeline.xml" relativeToChangelogFile="false"/>
+
+    <include file="/config/liquibase/changelog/20250507003_add_o365_pipeline.xml" relativeToChangelogFile="false"/>-->
+
 
 </databaseChangeLog>

frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.html

@@ -21,7 +21,7 @@ <h4 class="card-title mb-0 text-primary">
               <div [innerHtml]="step.name"></div>
               <ng-container *ngIf="step.content">
 
-                <ng-template [ngIf]="step.content.id === 'stepContent2'">
+                <ng-template [ngIf]="step.content.id === 'stepContent3'">
                   <app-agent-install-selector [platforms]="architectures" [_selectedPlatform]="architectures[0]"></app-agent-install-selector>
                 </ng-template>
               </ng-container>

frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.ts

@@ -42,14 +42,16 @@ export class GuideMacosAgentComponent implements OnInit {
   getCommandARM(installerName: string): string {
     const ip = window.location.host.includes(':') ? window.location.host.split(':')[0] : window.location.host;
 
-    return `sudo bash -c "./${installerName} ${ip} <secret>${this.token}</secret> yes"`;
+    return `sudo bash -c "/opt/utmstack/${installerName} ${ip} <secret>${this.token}</secret> yes"`;
   }
 
 
   getUninstallCommand(installerName: string): string {
-    return `sudo bash -c "./utmstack_agent_service uninstall"`;
+    // tslint:disable-next-line:max-line-length
+    return `sudo bash -c "/opt/utmstack/${installerName} uninstall; launchctl bootout system /Library/LaunchDaemons/UTMStackAgent.plist 2>/dev/null; rm /Library/LaunchDaemons/UTMStackAgent.plist; rm -rf /opt/utmstack"`;
   }
 
+
   private loadArchitectures() {
     this.architectures = [
       {