Merge pull request #41 from BrainNotFoundException/Lavnish/ElasticSearch

Lavnish/elastic search

Author: abhitrueprogrammer

.env.example

@@ -3,16 +3,31 @@ JWT_SECRET=
 
 FRONTEND_URL=http://localhost:3000
 
-POSTGRES_HOST=
-POSTGRES_PORT=
-POSTGRES_USER=abcd
-POSTGRES_PASSWORD=abcd
-POSTGRES_DB=abcd
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_USER=devsoc
+POSTGRES_PASSWORD=devsoc
+POSTGRES_DB=devsoc
 
 REDIS_HOST=
 REDIS_PORT=
 REDIS_PASSWORD=
 
+ELASTIC_HOST=elasticsearch
+ELASTIC_PORT=9200
+ELASTIC_SCHEME=https
+ELASTIC_USERNAME=elastic
+ELASTIC_PASSWORD=gyattpass
+ELASTIC_INDEX=logs
+logger.level=ERROR #otherwise elastic search will keep yapping - i couldn't read shit b4 -- set to WARN if you wanna read sm not so imp stuff
+discovery.type=single-node #idk but sm sort of bootstrap issue
+
+xpack.security.enabled=false
+
+bootstrap.memory_lock=true
+ES_JAVA_OPTS=-Xms2g -Xmx2g #this is 4 sm memory related issues that might occur in docker fk elasticsearch
+# ts stands for -initial -max (heap sizes) and 2g signifies 2GB idk how much we'll actually need for logs but jus change that part as req ig
+
 ENV=
 
 GOOSE_MIGRATION_DIR=database/schema

.gitignore

@@ -30,4 +30,4 @@ dump.txt
 # Project build
 main
 *templ.go
-.DS_Store
+.DS_Store

cmd/api/main.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"context"
 	"log"
 	"net/http"
 	"os"
 
+	"github.com/CodeChefVIT/devsoc-backend-26/pkg/elastic"
 	"github.com/CodeChefVIT/devsoc-backend-26/pkg/middlewares"
 	"github.com/CodeChefVIT/devsoc-backend-26/pkg/redis"
 	"github.com/CodeChefVIT/devsoc-backend-26/pkg/router"
@@ -15,6 +17,13 @@ import (
 func main() {
 	utils.LoadConfig()
 	utils.InitDB()
+	elastic.InitElasticClient()
+
+	ctx := context.Background()
+
+	if err := elastic.CheckLogsIndex(ctx); err != nil {
+		log.Fatal(err)
+	}
 	redis.InitRedis()
 	redis.InitRequestManager()
 

docker-compose.yml

@@ -1,4 +1,31 @@
 services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.2.3
+    container_name: elastic
+    env_file:
+      - .env
+    ports:
+      - 127.0.0.1:9200:9200
+    volumes:
+      - ./elasticsearch/certs:/usr/share/elasticsearch/config/certs:ro
+      - elastic_data:/usr/share/elasticsearch/data
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl http://localhost:9200/_cluster/health || exit 1"
+        ]
+      interval: 5s
+      timeout: 2s
+      retries: 20
+
   postgres:
     image: postgres:16
     container_name: devsoc_postgres
@@ -32,9 +59,14 @@ services:
       - GOFLAGS=-buildvcs=false
     restart: on-failure
     depends_on:
-      - postgres
-      - redis
+      elasticsearch:
+        condition: service_healthy
+      postgres:
+        condition: service_started
+      redis:
+        condition: service_started
 
 volumes:
   postgres_data:
   redis_data:
+  elastic_data:

go.mod

@@ -18,7 +18,11 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/elastic/elastic-transport-go/v8 v8.8.0 // indirect
+	github.com/elastic/go-elasticsearch/v9 v9.2.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.12 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
@@ -38,6 +42,10 @@ require (
 	github.com/stretchr/testify v1.11.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	golang.org/x/crypto v0.46.0 // indirect
 	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/sync v0.19.0 // indirect

go.sum

@@ -15,8 +15,17 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvw
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/elastic/elastic-transport-go/v8 v8.8.0 h1:7k1Ua+qluFr6p1jfJjGDl97ssJS/P7cHNInzfxgBQAo=
+github.com/elastic/elastic-transport-go/v8 v8.8.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
+github.com/elastic/go-elasticsearch/v9 v9.2.1 h1:/H8RKblXQbnVlFAkc0J5/FfSgVug60CU/DxlRcMdQf4=
+github.com/elastic/go-elasticsearch/v9 v9.2.1/go.mod h1:LvMSwNhRGZgkWWmErHS0IkT10wKzU+PRkOkQHGy3Wz0=
 github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw=
 github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@@ -78,6 +87,14 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
 github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
+go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
+go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
+go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
+go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
+go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
 golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
 golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=

pkg/controllers/logs.go

@@ -3,10 +3,11 @@ package controllers
 import (
 	"encoding/json"
 	"net/http"
-	"time"
 
 	"github.com/CodeChefVIT/devsoc-backend-26/pkg/db"
 	"github.com/CodeChefVIT/devsoc-backend-26/pkg/dto"
+	"github.com/CodeChefVIT/devsoc-backend-26/pkg/elastic"
+	"github.com/CodeChefVIT/devsoc-backend-26/pkg/models"
 	"github.com/labstack/echo/v4"
 )
 
@@ -33,19 +34,38 @@ func GetLogs(c echo.Context) error {
 			createdBy = &s
 		}
 
-		createdAt := ""
-		if l.CreatedAt.Valid {
-			createdAt = l.CreatedAt.Time.Format(time.RFC3339)
-		}
-
 		resp = append(resp, dto.Log{
 			ID:        l.ID.String(),
 			Type:      l.Type,
 			Body:      body,
 			CreatedBy: createdBy,
-			CreatedAt: createdAt,
+			CreatedAt: l.CreatedAt.Time,
 		})
 	}
 
 	return c.JSON(http.StatusOK, resp)
 }
+
+func GetLog(c echo.Context) error {
+
+	query := c.QueryParam("q")
+
+	if query == "" {
+		return c.JSON(http.StatusBadRequest, &models.Response{
+			Status:  "fail",
+			Message: "The search query is required to search the logs!",
+		})
+	}
+
+	logs, err := elastic.SearchLogs(c.Request().Context(), query)
+
+	if err != nil {
+		return c.JSON(http.StatusInternalServerError, &models.Response{
+			Status:  "fail",
+			Message: err.Error(),
+		})
+	}
+
+	return c.JSON(http.StatusOK, logs)
+
+}

pkg/dto/logs.go

@@ -1,9 +1,15 @@
 package dto
 
+import "time"
+
 type Log struct {
 	ID        string      `json:"id"`
 	Type      string      `json:"type"`
+	Method    string      `json:"method"`
+	URL       string      `json:"url"`
+	IPAddress string      `json:"ip_address"`
+	UserAgent string      `json:"user_agent"`
 	Body      interface{} `json:"body"`
 	CreatedBy *string     `json:"created_by"`
-	CreatedAt string      `json:"created_at"`
+	CreatedAt time.Time   `json:"created_at"`
 }

pkg/elastic/client.go

@@ -0,0 +1,42 @@
+package elastic
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/CodeChefVIT/devsoc-backend-26/pkg/utils"
+	es "github.com/elastic/go-elasticsearch/v9"
+)
+
+var Client *es.Client
+
+func InitElasticClient() {
+
+	cfg := utils.Config
+
+	elasticURL := fmt.Sprintf("%s://%s:%d", cfg.ElasticScheme, cfg.ElasticHost, cfg.ElasticPort)
+
+	esCfg := es.Config{
+		Addresses:     []string{elasticURL},
+		MaxRetries:    cfg.ElasticMaxRetries,
+		RetryOnStatus: []int{502, 503, 504},
+		RetryBackoff: func(i int) time.Duration {
+			return time.Duration(i) * time.Second
+		},
+	}
+
+	client, err := es.NewClient(esCfg)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to create ElasticSearch Client: %v", err))
+	}
+
+	res, err := client.Info(client.Info.WithContext(context.Background()))
+	if err != nil {
+		panic(fmt.Sprintf("ElasticSearch Health Check Faileed: %v", err))
+	}
+	defer res.Body.Close()
+
+	fmt.Println("ElasticSearch Connected Successfully")
+	Client = client
+}

pkg/elastic/index.go

@@ -0,0 +1,102 @@
+package elastic
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/CodeChefVIT/devsoc-backend-26/pkg/dto"
+)
+
+const logsIndex = "logs"
+
+func CheckLogsIndex(ctx context.Context) error {
+	res, err := Client.Indices.Exists([]string{logsIndex})
+	if err != nil {
+		return err
+	}
+
+	defer res.Body.Close()
+
+	if res.StatusCode == 200 {
+		return nil
+	}
+
+	mapping := map[string]interface{}{
+		"mappings": map[string]interface{}{
+			"properties": map[string]interface{}{
+				"type": map[string]interface{}{
+					"type": "keyword",
+				},
+				"method": map[string]interface{}{
+					"type": "keyword",
+				},
+				"url": map[string]interface{}{
+					"type": "text",
+				},
+				"ip_address": map[string]interface{}{
+					"type": "ip",
+				},
+				"user_agent": map[string]interface{}{
+					"type": "text",
+					"fields": map[string]interface{}{
+						"keyword": map[string]interface{}{
+							"type": "keyword",
+						},
+					},
+				},
+				"created_by": map[string]interface{}{
+					"type": "keyword",
+				},
+				"created_at": map[string]interface{}{
+					"type": "date",
+				},
+				"body": map[string]interface{}{
+					"type": "object",
+				},
+			},
+		},
+	}
+
+	body, _ := json.Marshal(mapping)
+
+	createRes, err := Client.Indices.Create(
+		logsIndex,
+		Client.Indices.Create.WithContext(ctx),
+		Client.Indices.Create.WithBody(bytes.NewReader((body))),
+	)
+
+	if err != nil {
+		return err
+	}
+
+	defer createRes.Body.Close()
+
+	if createRes.IsError() {
+		return fmt.Errorf("Failed to create logs indexing for elastic client")
+	}
+
+	return nil
+}
+
+func IndexLog(ctx context.Context, log dto.Log) error {
+	body, err := json.Marshal(log)
+	if err != nil {
+		return err
+	}
+
+	res, err := Client.Index(
+		logsIndex,
+		bytes.NewReader(body),
+		Client.Index.WithContext(ctx),
+	)
+
+	if err != nil {
+		return err
+	}
+
+	defer res.Body.Close()
+
+	return nil
+}