Merge pull request #14 from d-Rickyy-b/prometheus

Implement prometheus support

Author: d-Rickyy-b

CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+### Added
+### Changed
+### Fixed
+### Docs
+
+## [1.1.0] - 2022-03-28
+
+### Added
+- Support for prometheus database 
+
+## [1.0.0] - 2022-02-14
+Initial release! First stable version of GoGeizhalsBot is published as v1.0.0 
+
+[unreleased]: https://github.com/d-Rickyy-b/GoGeizhalsBot/compare/v1.1.0...HEAD
+[1.1.0]: https://github.com/d-Rickyy-b/GoGeizhalsBot/compare/v1.0.0...v1.1.0
+[1.0.0]: https://github.com/d-Rickyy-b/GoGeizhalsBot/tree/v1.0.0

go.mod

@@ -5,6 +5,7 @@ go 1.16
 require (
 	github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.4
 	github.com/PuerkitoBio/goquery v1.8.0
+	github.com/VictoriaMetrics/metrics v1.18.1
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/glebarez/sqlite v1.3.3
 	github.com/jinzhu/now v1.1.4 // indirect

go.sum

@@ -2,6 +2,8 @@ github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.4 h1:1hKv9DvEwn//VsJgcCGt9+IAq4n+P
 github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.4/go.mod h1:r815fYWTudnU9JhtsJAxUtuV7QrSgKpChJkfTSMFpfg=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0=
+github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -33,6 +35,10 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
+github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
+github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
+github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
 github.com/wcharczuk/go-chart/v2 v2.1.0 h1:tY2slqVQ6bN+yHSnDYwZebLQFkphK4WNrVwnt7CJZ2I=
 github.com/wcharczuk/go-chart/v2 v2.1.0/go.mod h1:yx7MvAVNcP/kN9lKXM/NTce4au4DFN99j6i1OwDclNA=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

internal/bot/bot.go

@@ -6,10 +6,12 @@ import (
 	"GoGeizhalsBot/internal/config"
 	"GoGeizhalsBot/internal/database"
 	"GoGeizhalsBot/internal/geizhals"
+	"GoGeizhalsBot/internal/prometheus"
 	"fmt"
 	"log"
 	"net/http"
 	"net/url"
+	"time"
 
 	"github.com/PaulSonOfLars/gotgbot/v2/ext/handlers/filters/message"
 
@@ -218,6 +220,7 @@ func showPriceagentDetail(b *gotgbot.Bot, ctx *ext.Context) error {
 		},
 	}
 
+	// Check if the initial message contained a photo, if yes, we're coming from the price history graph
 	if len(cb.Message.Photo) > 0 {
 		bot.DeleteMessage(ctx.EffectiveChat.Id, cb.Message.MessageId)
 
@@ -344,6 +347,7 @@ func deletePriceagentHandler(b *gotgbot.Bot, ctx *ext.Context) error {
 }
 
 func newUserHandler(_ *gotgbot.Bot, ctx *ext.Context) error {
+	prometheus.TotalUserInteractions.Inc()
 	// Create user in databse if they don't exist already
 	if !ctx.EffectiveSender.IsUser() {
 		return nil
@@ -480,5 +484,22 @@ func Start(botConfig config.Config) {
 
 	log.Printf("Bot has been started as @%s...\n", bot.User.Username)
 
+	if botConfig.Prometheus.Enabled {
+		// Periodically update the metrics from the database
+		go func() {
+			for {
+				prometheus.TotalUniquePriceagentsValue = database.GetPriceAgentCount()
+				prometheus.TotalUniqueUsersValue = database.GetUserCount()
+				prometheus.TotalUniqueWishlistPriceagentsValue = database.GetPriceAgentWishlistCount()
+				prometheus.TotalUniqueProductPriceagentsValue = database.GetPriceAgentProductCount()
+				time.Sleep(time.Second * 60)
+			}
+		}()
+
+		exportAddr := fmt.Sprintf("%s:%d", botConfig.Prometheus.ExportIP, botConfig.Prometheus.ExportPort)
+		log.Printf("Starting prometheus exporter on %s...\n", exportAddr)
+		prometheus.StartPrometheusExporter(exportAddr)
+	}
+
 	updater.Idle()
 }

internal/bot/notification.go

@@ -4,6 +4,7 @@ import (
 	"GoGeizhalsBot/internal/bot/models"
 	"GoGeizhalsBot/internal/database"
 	"GoGeizhalsBot/internal/geizhals"
+	"GoGeizhalsBot/internal/prometheus"
 	"fmt"
 	"log"
 	"time"
@@ -23,7 +24,7 @@ func updateEntityPrices() {
 	// For each price agent, update prices and store updated prices in the entity in the database.
 	// Also update price history with the new prices.
 	for _, entity := range allEntities {
-		log.Println("Updating prices for:", entity.Name)
+		log.Println("Updating prices for:", entity.URL)
 
 		// If there are two price agents with the same entity, we currently fetch it twice
 		updatedEntity, updateErr := geizhals.UpdateEntity(entity)
@@ -84,6 +85,7 @@ func notifyUsers(priceAgent models.PriceAgent, oldEntity, updatedEntity geizhals
 	}
 
 	log.Println("Sending notification to user:", user.ID)
+	prometheus.PriceagentNotifications.Inc()
 
 	markup := gotgbot.InlineKeyboardMarkup{
 		InlineKeyboard: [][]gotgbot.InlineKeyboardButton{

internal/bot/pricehistory.go

@@ -4,6 +4,7 @@ import (
 	"GoGeizhalsBot/internal/bot/models"
 	"GoGeizhalsBot/internal/database"
 	"GoGeizhalsBot/internal/geizhals"
+	"GoGeizhalsBot/internal/prometheus"
 	"bytes"
 	"fmt"
 	"io"
@@ -145,6 +146,7 @@ func getPriceagentFromContext(ctx *ext.Context) (models.PriceAgent, error) {
 
 // renderChart renders a price history chart to the given writer.
 func renderChart(priceagent models.PriceAgent, history geizhals.PriceHistory, since time.Time, w io.Writer) {
+	prometheus.GraphsRendered.Inc()
 	darkFontColor := drawing.ColorFromHex("c2c2c2")
 	fontColor := darkFontColor
 

internal/bot/version.go

@@ -1,3 +1,3 @@
 package bot
 
-var version = "1.0.0-dev (compiled manually)"
+var version = "1.1.0-dev (compiled manually)"

internal/database/database.go

@@ -53,11 +53,41 @@ func GetPriceAgentCountForUser(userID int64) int64 {
 	return count
 }
 
+func GetPriceAgentCount() int64 {
+	var count int64
+	db.Model(&models.PriceAgent{}).Count(&count)
+	return count
+}
+
+func GetPriceAgentProductCount() int64 {
+	var count int64
+	db.Model(&models.PriceAgent{}).Joins("JOIN entities on price_agents.entity_id = entities.id").Where("entities.type = ?", geizhals.Product).Count(&count)
+	return count
+}
+
+func GetPriceAgentWishlistCount() int64 {
+	var count int64
+	db.Model(&models.PriceAgent{}).Joins("JOIN entities on price_agents.entity_id = entities.id").Where("entities.type = ?", geizhals.Wishlist).Count(&count)
+	return count
+}
+
+func GetUserCount() int64 {
+	var count int64
+	db.Model(&models.User{}).Count(&count)
+	return count
+}
+
 func CreateUser(user models.User) error {
 	tx := db.Create(&user)
 	return tx.Error
 }
 
+func GetAllUsers() []models.User {
+	var users []models.User
+	db.Find(&users)
+	return users
+}
+
 func DeletePriceAgentForUser(priceAgent models.PriceAgent) error {
 	log.Println("Delete priceagent!")
 

internal/geizhals/geizhals.go

@@ -1,6 +1,7 @@
 package geizhals
 
 import (
+	"GoGeizhalsBot/internal/prometheus"
 	"fmt"
 	"log"
 	"net/http"
@@ -72,9 +73,11 @@ func downloadHTML(entityURL string) (*goquery.Document, int, error) {
 		log.Println("Using proxy: ", proxyURL)
 	}
 
+	prometheus.GeizhalsHTTPRequests.Inc()
 	resp, getErr := httpClient.Get(entityURL)
 	if getErr != nil {
 		log.Println(getErr)
+		prometheus.HttpErrors.Inc()
 		return nil, 0, fmt.Errorf("error while downloading content from Geizhals: %w", getErr)
 	}
 	// Cleanup when this function ends

internal/prometheus/prometheus.go

@@ -0,0 +1,45 @@
+package prometheus
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	TotalUniqueUsersValue int64
+	totalUniqueUsers      = metrics.NewGauge("gogeizhalsbot_unique_users_total", func() float64 {
+		return float64(TotalUniqueUsersValue)
+	})
+
+	TotalUniquePriceagentsValue int64
+	totalUniquePriceagents      = metrics.NewGauge("gogeizhalsbot_unique_priceagents", func() float64 {
+		return float64(TotalUniquePriceagentsValue)
+	})
+
+	TotalUniqueProductPriceagentsValue int64
+	totalUniqueProductPriceagents      = metrics.NewGauge("gogeizhalsbot_unique_priceagents{type=\"product\"}", func() float64 {
+		return float64(TotalUniqueProductPriceagentsValue)
+	})
+
+	TotalUniqueWishlistPriceagentsValue int64
+	totalUniqueWishlistPriceagents      = metrics.NewGauge("gogeizhalsbot_unique_priceagents{type=\"wishlist\"}", func() float64 {
+		return float64(TotalUniqueWishlistPriceagentsValue)
+	})
+
+	TotalUserInteractions   = metrics.NewCounter("gogeizhalsbot_user_interactions_total")
+	GeizhalsHTTPRequests    = metrics.NewCounter("gogeizhalsbot_geizhals_http_requests_total")
+	PriceagentNotifications = metrics.NewCounter("gogeizhalsbot_priceagent_notifications_total")
+	HttpErrors              = metrics.NewCounter("gogeizhalsbot_http_errors_total")
+	GraphsRendered          = metrics.NewCounter("gogeizhalsbot_graphs_rendered_total")
+)
+
+// var backgroundUpdateChecks = metrics.NewSummary("gogeizhalsbot_total_requests")
+
+func StartPrometheusExporter(addr string) {
+	// Expose the registered metrics at `/metrics` path.
+	http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
+		metrics.WritePrometheus(w, true)
+	})
+	http.ListenAndServe(addr, nil)
+}