Release/v10.8.1 (#1225)

* fix(compliance-schedule): fix standard and section selection issue in report creation

* chore: Update CHANGELOG.md

* chore: update version.yml

* fix(ui): display array fields as a single field without numeric suffixes

* chore: update CHANGELOG.md

* fix(alert-field-render): resolve persistent loading spinner when displaying "tags" column

* chore: Update CHANGELOG.md

* fix: Resolve false positive checkbox selection when editing tagging rules

* feat: implement alert correlation and context building for enhanced alert analysis

* add debug logging for GPT request

* feat: add debug logging for alert processing and related alerts retrieval

* fix: update to return schema.Alert and adjust related logic

* refactor: simplify body creation in ElasticSearch function and remove unnecessary debug logs

* fix: optimize alert correlation logic and improve classification handling

* fix: update of the logic of correlation of alerts and construction of the historical context based on counts

* fix: improve log handling in GPT request and ensure last log entry is used

* feat: update macOS install steps with `utmstack-macos-agent.pkg`

* fix: add pipeline for aws, sophos-central and o365 integrations

* fix: remove logging of debug

* Migrate from correlation service to direct Logstash connection in aws integration.

* Migrate from correlation service to direct Logstash connection in office365 integration.

* Migrate from correlation service to direct Logstash connection in sophos integration.

* fix: add pipeline for aws, sophos-central and o365 integrations

* chore: resolve merge conflicts

* chore: integrate recent UI improvements

* add datasource in macos agent logs

* include logstash ports in installer for aws, o365 and sophos

* fix: update TagRulesApplied field type to slice and join in conversion

* fix: change TagRulesApplied field type from string to slice of int

* Refactoring the event sending format to Logstash in the AWS plugin.

* Refactoring the event sending format to Logstash in the Sophos plugin.

* Refactoring the event sending format to Logstash in the office365 plugin.

* fix: add pipeline for aws, sophos-central and o365 integrations

* "Update blocklist processing to support severity levels and enhance IP threat intelligence integration."

* "Add IP validation using net.ParseIP to ensure proper processing of source and destination IPs."

* "Fix path in Dockerfile COPY command for the correlation binary."

* fix: add pipeline for aws, sophos-central and o365 integrations

* Update correlation Dockerfile

* fix: add pipeline for aws, sophos-central and o365 integrations

* fix: add pipeline for aws, sophos-central and o365 integrations

* fix: filter only valid IPs when parsing coordinate map chart data

* fix: update display name for Sophos integration

* Implement Sophos Central filter (v1.0.0).

* Refactor AWS filter (v2.0.0) to use JSON instead of Grok.

* Refactor Office 365 filter (v2.0.0) by simplifying the structure.

* fix: corrected typo in compliance status label from "Complaint" to "Compliant"

* send logs from new windows channels in arm agent

* fix: hide sorting action for assets filters

* fix: improve CSV export limit parameters

* fix: correct uninstalling command for macOs agent

* feat: add Windows ARM64 support to agent installation platforms

* set correct api url environment

* fix: update filter for winevent log agent

* fix: update wineventlog filter

* fix: update wineventlog filter

* update version and changelog

---------

Co-authored-by: Manuel Abascal <mjabascal10@gmail.com>
Co-authored-by: Yadian Llada Lopez <yadian.llada@gmail.com>
Co-authored-by: JocLRojas <joc.l.rojas02@gmail.com>
Co-authored-by: Osmany Montero <osmontero@icloud.com>

Author: 5 people

CHANGELOG.md

@@ -1,8 +1,10 @@
-# UTMStack 10.8.0 Release Notes
-- Updated Soc-AI models and released the code as open source.
-- Added the ability for users to choose which model to use with Soc-AI.
-- Enhanced the prompt sent to OpenAI by including additional contextual details.
-- Added support for RedHat; UTMStack can now be installed on both Ubuntu and RedHat.
-- Improved log delivery from ARM-based agents on Windows, now sending native system logs.
-- Added support for macOS ARM64; agents can now be installed on that platform.
-- Improved agent information displayed in the Sources panel, providing more accurate OS details and agent versions.
+# UTMStack 10.8.1 Release Notes
+
+- Improved log parsing and processing for AWS, O365, and Sophos Central integrations.
+- Updated Sophos XG integration from legacy mode to support newer versions
+- Enhanced log processing and parsing for Windows Agent on ARM architectures.
+- Added support for new log channels using the Windows API to retrieve additional logs.
+- Compliance Report Scheduling: Improved the stability of the selection process when creating new report schedules.
+- Improved field rendering in Log Explorer by consolidating list-based fields into a single entry for better readability and consistency.
+- Improved field rendering for tags and note fields in Alerts.
+- Improved export functionality to better handle large data sets and avoid performance issues during report generation.

agent/collectors/macos.go

@@ -5,6 +5,7 @@ package collectors
 
 import (
 	"bufio"
+	"os"
 	"os/exec"
 	"path/filepath"
 
@@ -29,6 +30,11 @@ func getCollectorsInstances() []Collector {
 func (d Darwin) SendLogs() {
 	path := utils.GetMyPath()
 	collectorPath := filepath.Join(path, "utmstack-collector-mac")
+	host, err := os.Hostname()
+	if err != nil {
+		utils.Logger.ErrorF("error getting hostname: %v", err)
+		host = "unknown"
+	}
 
 	cmd := exec.Command(collectorPath)
 
@@ -62,9 +68,11 @@ func (d Darwin) SendLogs() {
 				continue
 			}
 
+			messageWithHost := config.GetMessageFormated(host, validatedLog)
+
 			logservice.LogQueue <- logservice.LogPipe{
 				Src:  string(config.DataTypeMacOs),
-				Logs: []string{validatedLog},
+				Logs: []string{messageWithHost},
 			}
 		}
 

agent/collectors/windows_arm64.go

@@ -7,12 +7,13 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
-	"log"
 	"os"
 	"os/signal"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
+	"time"
 	"unsafe"
 
 	"github.com/threatwinds/validations"
@@ -74,8 +75,10 @@ type EventSubscription struct {
 	Channel      string
 	Query        string
 	Errors       chan error
-	Callback     func(event *Event)
 	winAPIHandle windows.Handle
+
+	mu      sync.Mutex
+	running bool
 }
 
 const (
@@ -90,9 +93,13 @@ var (
 	procEvtSubscribe = modwevtapi.NewProc("EvtSubscribe")
 	procEvtRender    = modwevtapi.NewProc("EvtRender")
 	procEvtClose     = modwevtapi.NewProc("EvtClose")
+	incomingEvents   = make(chan string, 1024)
 )
 
 func (evtSub *EventSubscription) Create() error {
+	evtSub.mu.Lock()
+	defer evtSub.mu.Unlock()
+
 	if evtSub.winAPIHandle != 0 {
 		return fmt.Errorf("windows_events: subscription has already been created")
 	}
@@ -109,8 +116,6 @@ func (evtSub *EventSubscription) Create() error {
 
 	callback := syscall.NewCallback(evtSub.winAPICallback)
 
-	log.Printf("Debug - Subscribing to channel: %s", evtSub.Channel)
-
 	handle, _, err := procEvtSubscribe.Call(
 		0,
 		0,
@@ -131,8 +136,11 @@ func (evtSub *EventSubscription) Create() error {
 }
 
 func (evtSub *EventSubscription) Close() error {
+	evtSub.mu.Lock()
+	defer evtSub.mu.Unlock()
+
 	if evtSub.winAPIHandle == 0 {
-		return fmt.Errorf("windows_events: no active subscription to close")
+		return nil
 	}
 	ret, _, err := procEvtClose.Call(uintptr(evtSub.winAPIHandle))
 	if ret == 0 {
@@ -145,47 +153,101 @@ func (evtSub *EventSubscription) Close() error {
 func (evtSub *EventSubscription) winAPICallback(action, userContext, event uintptr) uintptr {
 	switch action {
 	case evtSubscribeActionError:
-		evtSub.Errors <- fmt.Errorf("windows_events: error in callback, code: %x", uint16(event))
-	case evtSubscribeActionDeliver:
-		bufferSize := uint32(4096)
-		for {
-			renderSpace := make([]uint16, bufferSize/2)
-			bufferUsed := uint32(0)
-			propertyCount := uint32(0)
-			ret, _, err := procEvtRender.Call(
-				0,
-				event,
-				evtRenderEventXML,
-				uintptr(bufferSize),
-				uintptr(unsafe.Pointer(&renderSpace[0])),
-				uintptr(unsafe.Pointer(&bufferUsed)),
-				uintptr(unsafe.Pointer(&propertyCount)),
-			)
-			if ret == 0 {
-				if err == windows.ERROR_INSUFFICIENT_BUFFER {
-					bufferSize *= 2
-					continue
+		err := fmt.Errorf("windows_events: error in callback, code: %x", uint16(event))
+		evtSub.Errors <- err
+
+		go func(channel string) {
+			utils.Logger.LogF(100, "Attempting to resubscribe to channel: %s after error: %v", channel, err)
+			evtSub.mu.Lock()
+			defer evtSub.mu.Unlock()
+
+			_ = evtSub.Close()
+
+			for {
+				time.Sleep(5 * time.Second)
+				if err := evtSub.Create(); err != nil {
+					utils.Logger.ErrorF("Retry failed for channel %s: %s", channel, err)
+				} else {
+					utils.Logger.LogF(100, "Resubscribed to channel: %s", channel)
+					break
 				}
-				evtSub.Errors <- fmt.Errorf("windows_events: failed to render event: %w", err)
-				return 0
-			}
-			xmlStr := windows.UTF16ToString(renderSpace)
-			xmlStr = cleanXML(xmlStr)
-
-			dataParsed := new(Event)
-			if err := xml.Unmarshal([]byte(xmlStr), dataParsed); err != nil {
-				evtSub.Errors <- fmt.Errorf("windows_events: failed to parse XML: %s", err)
-			} else {
-				evtSub.Callback(dataParsed)
 			}
+		}(evtSub.Channel)
+
+	case evtSubscribeActionDeliver:
+		utils.Logger.LogF(100, "Received event from channel: %s", evtSub.Channel)
+		xmlStr, err := quickRenderXML(event)
+		if err != nil {
+			evtSub.Errors <- fmt.Errorf("render in callback: %v", err)
 			break
 		}
+		select {
+		case incomingEvents <- xmlStr:
+		default:
+			utils.Logger.ErrorF("incomingEvents lleno: evento descartado")
+		}
 	default:
 		evtSub.Errors <- fmt.Errorf("windows_events: unsupported action in callback: %x", uint16(action))
 	}
 	return 0
 }
 
+func eventWorker() {
+	for xmlStr := range incomingEvents {
+		ev := new(Event)
+		if err := xml.Unmarshal([]byte(xmlStr), ev); err != nil {
+			utils.Logger.ErrorF("unmarshal error: %v", err)
+			continue
+		}
+
+		eventJSON, err := convertEventToJSON(ev)
+		if err != nil {
+			utils.Logger.ErrorF("toJSON error: %v", err)
+			continue
+		}
+
+		validatedLog, _, err := validations.ValidateString(eventJSON, false)
+		if err != nil {
+			utils.Logger.LogF(100, "validation error: %s: %v", eventJSON, err)
+			continue
+		}
+
+		select {
+		case logservice.LogQueue <- logservice.LogPipe{
+			Src:  string(config.DataTypeWindowsAgent),
+			Logs: []string{validatedLog},
+		}:
+		default:
+			utils.Logger.LogF(100, "LogQueue full: event discarded")
+		}
+	}
+}
+
+func quickRenderXML(h uintptr) (string, error) {
+	bufSize := uint32(4096)
+	for {
+		space := make([]uint16, bufSize/2)
+		used := uint32(0)
+		prop := uint32(0)
+
+		ret, _, err := procEvtRender.Call(
+			0, h, evtRenderEventXML,
+			uintptr(bufSize),
+			uintptr(unsafe.Pointer(&space[0])),
+			uintptr(unsafe.Pointer(&used)),
+			uintptr(unsafe.Pointer(&prop)),
+		)
+		if ret == 0 {
+			if err == windows.ERROR_INSUFFICIENT_BUFFER {
+				bufSize *= 2
+				continue
+			}
+			return "", err
+		}
+		return cleanXML(windows.UTF16ToString(space)), nil
+	}
+}
+
 func cleanXML(xmlStr string) string {
 	xmlStr = strings.TrimSpace(xmlStr)
 	if idx := strings.Index(xmlStr, "<?xml"); idx > 0 {
@@ -210,36 +272,23 @@ func getCollectorsInstances() []Collector {
 
 func (w Windows) SendLogs() {
 	errorsChan := make(chan error, 10)
+	go eventWorker()
 
-	callback := func(event *Event) {
-		eventJSON, err := convertEventToJSON(event)
-		if err != nil {
-			utils.Logger.ErrorF("error converting event to JSON: %v", err)
-			return
-		}
-		validatedLog, _, err := validations.ValidateString(eventJSON, false)
-		if err != nil {
-			utils.Logger.LogF(100, "error validating log: %s: %v", eventJSON, err)
-			return
-		}
-		logservice.LogQueue <- logservice.LogPipe{
-			Src:  string(config.DataTypeWindowsAgent),
-			Logs: []string{validatedLog},
-		}
+	channels := []string{
+		"Security", "Application", "System", "Microsoft-Windows-Sysmon/Operational", "Windows Powershell",
+		"Microsoft-Windows-Powershell/Operational", "ForwardedEvents", "Microsoft-Windows-WinLogon/Operational",
+		"Microsoft-Windows-Windows Firewall With Advanced Security/Firewall", "Microsoft-Windows-Windows Defender/Operational",
 	}
-
-	channels := []string{"Security", "Application", "System"}
 	var subscriptions []*EventSubscription
 
 	for _, channel := range channels {
 		sub := &EventSubscription{
-			Channel:  channel,
-			Query:    "*",
-			Errors:   errorsChan,
-			Callback: callback,
+			Channel: channel,
+			Query:   "*",
+			Errors:  errorsChan,
 		}
 		if err := sub.Create(); err != nil {
-			utils.Logger.ErrorF("Error subscribing to channel %s: %s", channel, err)
+			utils.Logger.LogF(100, "Error subscribing to channel %s: %s", channel, err)
 			continue
 		}
 		subscriptions = append(subscriptions, sub)
@@ -255,6 +304,7 @@ func (w Windows) SendLogs() {
 	exitChan := make(chan os.Signal, 1)
 	signal.Notify(exitChan, os.Interrupt)
 	<-exitChan
+	close(incomingEvents)
 	utils.Logger.LogF(100, "Interrupt received, closing subscriptions...")
 	for _, sub := range subscriptions {
 		if err := sub.Close(); err != nil {

aws/configuration/const.go

@@ -3,8 +3,9 @@ package configuration
 import "github.com/utmstack/UTMStack/aws/utils"
 
 const (
-	CORRELATIONURL       = "http://correlation:8080/v1/newlog"
 	URL_CHECK_CONNECTION = "https://sts.amazonaws.com"
+	LogstashEndpoint     = "http://%s:%s"
+	UTMLogSeparator      = "<utm-log-separator>"
 )
 
 func GetInternalKey() string {
@@ -14,3 +15,11 @@ func GetInternalKey() string {
 func GetPanelServiceName() string {
 	return utils.Getenv("PANEL_SERV_NAME")
 }
+
+func GetLogstashHost() string {
+	return utils.Getenv("UTM_LOGSTASH_HOST")
+}
+
+func GetLogstashPort() string {
+	return utils.Getenv("UTM_LOGSTASH_PORT")
+}

aws/processor/pull.go

@@ -18,7 +18,7 @@ func PullLogs(startTime time.Time, endTime time.Time, group types.ModuleGroup) *
 		return err
 	}
 
-	err = SendToCorrelation(logs)
+	err = SendToLogstash(logs)
 	if err != nil {
 		return err
 	}