Skip to content

Commit 4740312

Browse files
mjabascal10mjabascal10
committed
Merge remote-tracking branch 'origin/release/v10.8.1' into release/v10.8.1
2 parents adb56a7 + fdbc29b commit 4740312

9 files changed

Lines changed: 178 additions & 37 deletions

File tree

‎aws/configuration/const.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ package configuration
33
import "github.com/utmstack/UTMStack/aws/utils"
44

55
const (
6-
CORRELATIONURL = "http://correlation:8080/v1/newlog"
76
URL_CHECK_CONNECTION = "https://sts.amazonaws.com"
7+
LogstashEndpoint = "http://%s:%s"
8+
UTMLogSeparator = "<utm-log-separator>"
89
)
910

1011
func GetInternalKey() string {
@@ -14,3 +15,11 @@ func GetInternalKey() string {
1415
func GetPanelServiceName() string {
1516
return utils.Getenv("PANEL_SERV_NAME")
1617
}
18+
19+
func GetLogstashHost() string {
20+
return utils.Getenv("UTM_LOGSTASH_HOST")
21+
}
22+
23+
func GetLogstashPort() string {
24+
return utils.Getenv("UTM_LOGSTASH_PORT")
25+
}

‎aws/processor/pull.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ func PullLogs(startTime time.Time, endTime time.Time, group types.ModuleGroup) *
1818
return err
1919
}
2020

21-
err = SendToCorrelation(logs)
21+
err = SendToLogstash(logs)
2222
if err != nil {
2323
return err
2424
}

‎aws/processor/sendData.go‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,71 @@
11
package processor
22

33
import (
4+
"bytes"
5+
"crypto/tls"
46
"encoding/json"
7+
"fmt"
58
"net/http"
9+
"strings"
10+
"time"
611

712
"github.com/threatwinds/logger"
813
"github.com/utmstack/UTMStack/aws/configuration"
914
"github.com/utmstack/UTMStack/aws/utils"
1015
)
1116

12-
func SendToCorrelation(data []TransformedLog) *logger.Error {
17+
var transport = &http.Transport{
18+
MaxIdleConns: 100,
19+
IdleConnTimeout: 2 * time.Second,
20+
ResponseHeaderTimeout: 2 * time.Second,
21+
ForceAttemptHTTP2: true,
22+
TLSClientConfig: &tls.Config{
23+
InsecureSkipVerify: true,
24+
},
25+
}
26+
27+
var client = &http.Client{Transport: transport, Timeout: 2 * time.Second}
28+
29+
func SendToLogstash(data []TransformedLog) *logger.Error {
30+
var logStrings []string
1331
for _, log := range data {
1432
body, err := json.Marshal(log)
1533
if err != nil {
1634
utils.Logger.ErrorF("error encoding log to JSON: %v", err)
1735
continue
1836
}
37+
logStrings = append(logStrings, string(body))
38+
}
1939

20-
_, status, e := utils.DoReq[map[string]interface{}](configuration.CORRELATIONURL, body, http.MethodPost, map[string]string{})
21-
if e != nil {
22-
utils.Logger.ErrorF("error sending log to correlation engine: %v", e)
23-
continue
24-
} else if status != http.StatusOK && status != http.StatusCreated {
25-
utils.Logger.ErrorF("error sending log to correlation engine: status %v", status)
26-
continue
40+
if len(logStrings) == 0 {
41+
return nil
42+
}
43+
44+
var logs string
45+
for _, str := range logStrings {
46+
logs += str + configuration.UTMLogSeparator
47+
}
48+
49+
url := fmt.Sprintf(configuration.LogstashEndpoint, configuration.GetLogstashHost(), configuration.GetLogstashPort())
50+
51+
req, err := http.NewRequest("POST", url, bytes.NewBufferString(logs))
52+
if err != nil {
53+
return utils.Logger.ErrorF("error creating request: %v", err.Error())
54+
}
55+
56+
resp, err := client.Do(req)
57+
if err != nil {
58+
if !strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
59+
utils.Logger.ErrorF("error sending logs with error: %v", err.Error())
2760
}
61+
return utils.Logger.ErrorF("error sending logs: %v", err.Error())
62+
}
63+
defer resp.Body.Close()
2864

65+
if resp.StatusCode != http.StatusOK {
66+
return utils.Logger.ErrorF("error sending logs with http code %d", resp.StatusCode)
2967
}
3068

69+
utils.Logger.Info("successfully sent %d logs to Logstash", len(logStrings))
3170
return nil
3271
}

‎office365/configuration/const.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ const (
1414
endPointStartSubscription = "/activity/feed/subscriptions/start"
1515
endPointContent = "/activity/feed/subscriptions/content"
1616
BASEURL = "https://manage.office.com/api/v1.0/"
17-
CORRELATIONURL = "http://correlation:8080/v1/newlog"
17+
LogstashEndpoint = "http://%s:%s"
18+
UTMLogSeparator = "<utm-log-separator>"
1819
)
1920

2021
func GetInternalKey() string {
@@ -36,3 +37,11 @@ func GetStartSubscriptionLink(tenant string) string {
3637
func GetContentLink(tenant string) string {
3738
return fmt.Sprintf("%s%s%s", BASEURL, tenant, endPointContent)
3839
}
40+
41+
func GetLogstashHost() string {
42+
return utils.Getenv("UTM_LOGSTASH_HOST")
43+
}
44+
45+
func GetLogstashPort() string {
46+
return utils.Getenv("UTM_LOGSTASH_PORT")
47+
}

‎office365/processor/processor.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,9 @@ func (o *OfficeProcessor) GetLogs(startTime string, endTime string, group types.
146146
if len(details) > 0 {
147147
logsCounter += len(details)
148148
cleanLogs := ETLProcess(details, group)
149-
err = SendToCorrelation(cleanLogs)
149+
err := SendToLogstash(cleanLogs)
150150
if err != nil {
151-
utils.Logger.ErrorF("error sending logs to correlation: %v", err) // Debug
151+
utils.Logger.ErrorF("error sending logs to logstash: %v", err) // Debug
152152
continue
153153
}
154154
}

‎office365/processor/sendData.go‎

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,71 @@
11
package processor
22

33
import (
4+
"bytes"
5+
"crypto/tls"
46
"encoding/json"
7+
"fmt"
58
"net/http"
9+
"strings"
10+
"time"
611

12+
"github.com/threatwinds/logger"
713
"github.com/utmstack/UTMStack/office365/configuration"
814
"github.com/utmstack/UTMStack/office365/utils"
915
)
1016

11-
func SendToCorrelation(data []TransformedLog) error {
12-
utils.Logger.Info("uploading %d logs...", len(data))
17+
var transport = &http.Transport{
18+
MaxIdleConns: 100,
19+
IdleConnTimeout: 2 * time.Second,
20+
ResponseHeaderTimeout: 2 * time.Second,
21+
ForceAttemptHTTP2: true,
22+
TLSClientConfig: &tls.Config{
23+
InsecureSkipVerify: true,
24+
},
25+
}
26+
27+
var client = &http.Client{Transport: transport, Timeout: 2 * time.Second}
1328

29+
func SendToLogstash(data []TransformedLog) *logger.Error {
30+
var logStrings []string
1431
for _, log := range data {
1532
body, err := json.Marshal(log)
1633
if err != nil {
1734
utils.Logger.ErrorF("error encoding log to JSON: %v", err)
1835
continue
1936
}
37+
logStrings = append(logStrings, string(body))
38+
}
2039

21-
_, status, e := utils.DoReq[map[string]interface{}](configuration.CORRELATIONURL, body, http.MethodPost, map[string]string{})
22-
if e != nil {
23-
utils.Logger.ErrorF("error sending log to correlation engine: %v", e)
24-
continue
25-
} else if status != http.StatusOK && status != http.StatusCreated {
26-
utils.Logger.ErrorF("error sending log to correlation engine: status code %d", status)
27-
continue
40+
if len(logStrings) == 0 {
41+
return nil
42+
}
43+
44+
var logs string
45+
for _, str := range logStrings {
46+
logs += str + configuration.UTMLogSeparator
47+
}
48+
49+
url := fmt.Sprintf(configuration.LogstashEndpoint, configuration.GetLogstashHost(), configuration.GetLogstashPort())
50+
51+
req, err := http.NewRequest("POST", url, bytes.NewBufferString(logs))
52+
if err != nil {
53+
return utils.Logger.ErrorF("error creating request: %v", err.Error())
54+
}
55+
56+
resp, err := client.Do(req)
57+
if err != nil {
58+
if !strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
59+
utils.Logger.ErrorF("error sending logs with error: %v", err.Error())
2860
}
61+
return utils.Logger.ErrorF("error sending logs: %v", err.Error())
62+
}
63+
defer resp.Body.Close()
2964

30-
utils.Logger.Info("log successfully sent to correlation engine")
65+
if resp.StatusCode != http.StatusOK {
66+
return utils.Logger.ErrorF("error sending logs with http code %d", resp.StatusCode)
3167
}
3268

33-
utils.Logger.Info("all logs were sent to correlation")
69+
utils.Logger.Info("successfully sent %d logs to Logstash", len(logStrings))
3470
return nil
3571
}

‎sophos/configuration/const.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@ package configuration
33
import "github.com/utmstack/UTMStack/sophos/utils"
44

55
const (
6-
CORRELATIONURL = "http://correlation:8080/v1/newlog"
7-
AUTHURL = "https://id.sophos.com/api/v2/oauth2/token"
8-
WHOAMIURL = "https://api.central.sophos.com/whoami/v1"
9-
CHECKCON = "https://id.sophos.com"
6+
AUTHURL = "https://id.sophos.com/api/v2/oauth2/token"
7+
WHOAMIURL = "https://api.central.sophos.com/whoami/v1"
8+
CHECKCON = "https://id.sophos.com"
9+
LogstashEndpoint = "http://%s:%s"
10+
UTMLogSeparator = "<utm-log-separator>"
1011
)
1112

1213
func GetInternalKey() string {
@@ -16,3 +17,11 @@ func GetInternalKey() string {
1617
func GetPanelServiceName() string {
1718
return utils.Getenv("PANEL_SERV_NAME")
1819
}
20+
21+
func GetLogstashHost() string {
22+
return utils.Getenv("UTM_LOGSTASH_HOST")
23+
}
24+
25+
func GetLogstashPort() string {
26+
return utils.Getenv("UTM_LOGSTASH_PORT")
27+
}

‎sophos/processor/pull.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func PullLogs(group types.ModuleGroup) *logger.Error {
3636

3737
nextKeys[group.ModuleID] = newNextKey
3838

39-
err = SendToCorrelation(logs)
39+
err = SendToLogstash(logs)
4040
if err != nil {
4141
return err
4242
}

‎sophos/processor/sendData.go‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,71 @@
11
package processor
22

33
import (
4+
"bytes"
5+
"crypto/tls"
46
"encoding/json"
7+
"fmt"
58
"net/http"
9+
"strings"
10+
"time"
611

712
"github.com/threatwinds/logger"
813
"github.com/utmstack/UTMStack/sophos/configuration"
914
"github.com/utmstack/UTMStack/sophos/utils"
1015
)
1116

12-
func SendToCorrelation(data []TransformedLog) *logger.Error {
17+
var transport = &http.Transport{
18+
MaxIdleConns: 100,
19+
IdleConnTimeout: 2 * time.Second,
20+
ResponseHeaderTimeout: 2 * time.Second,
21+
ForceAttemptHTTP2: true,
22+
TLSClientConfig: &tls.Config{
23+
InsecureSkipVerify: true,
24+
},
25+
}
26+
27+
var client = &http.Client{Transport: transport, Timeout: 2 * time.Second}
28+
29+
func SendToLogstash(data []TransformedLog) *logger.Error {
30+
var logStrings []string
1331
for _, log := range data {
1432
body, err := json.Marshal(log)
1533
if err != nil {
1634
utils.Logger.ErrorF("error encoding log to JSON: %v", err)
1735
continue
1836
}
37+
logStrings = append(logStrings, string(body))
38+
}
1939

20-
_, status, e := utils.DoReq[map[string]interface{}](configuration.CORRELATIONURL, body, http.MethodPost, map[string]string{})
21-
if e != nil {
22-
utils.Logger.ErrorF("error sending log to correlation engine: %v", e)
23-
continue
24-
} else if status != http.StatusOK && status != http.StatusCreated {
25-
utils.Logger.ErrorF("error sending log to correlation engine: status %v", status)
26-
continue
40+
if len(logStrings) == 0 {
41+
return nil
42+
}
43+
44+
var logs string
45+
for _, str := range logStrings {
46+
logs += str + configuration.UTMLogSeparator
47+
}
48+
49+
url := fmt.Sprintf(configuration.LogstashEndpoint, configuration.GetLogstashHost(), configuration.GetLogstashPort())
50+
51+
req, err := http.NewRequest("POST", url, bytes.NewBufferString(logs))
52+
if err != nil {
53+
return utils.Logger.ErrorF("error creating request: %v", err.Error())
54+
}
55+
56+
resp, err := client.Do(req)
57+
if err != nil {
58+
if !strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
59+
utils.Logger.ErrorF("error sending logs with error: %v", err.Error())
2760
}
61+
return utils.Logger.ErrorF("error sending logs: %v", err.Error())
62+
}
63+
defer resp.Body.Close()
2864

65+
if resp.StatusCode != http.StatusOK {
66+
return utils.Logger.ErrorF("error sending logs with http code %d", resp.StatusCode)
2967
}
3068

69+
utils.Logger.Info("successfully sent %d logs to Logstash", len(logStrings))
3170
return nil
3271
}

0 commit comments

Comments
 (0)