Merge branch 'main' into workflows-merging-main

Author: grussorusso

.github/workflows/makefile.yml

@@ -15,7 +15,7 @@ jobs:
      - name: Setup Go
        uses: actions/setup-go@v3
        with:
-          go-version: "1.20.0"
+          go-version: "1.21.0"
 
      - uses: actions/checkout@v2
 

.gitignore

@@ -3,3 +3,5 @@ bin/
 .run/
 __pycache__*
 images/*/executor
+*.backup
+default.etcd

README.md

@@ -50,19 +50,9 @@ As functions are executed within Docker containers, you need Docker to
 be installed on the host. Furthermore, Serverledge needs
 permissions to create containers.
 
-If you have more than one context (docker context ls), be sure to set up the DOCKER_HOST environment variable to the correct Host context.
-To set correctly, use the following three commands from cli:
-
-1) docker context ls
-2) docker context inspect <current-context>
-3) Take the Endpoints.docker.Host value and save it into the DOCKER_HOST environment variable. The default value is unix:///var/run/docker.sock
-
-
-
-You also need an **etcd** server to run Serverledge. To quickly start a local
-server:
-
-	$ ./scripts/start-etcd.sh   # stop it with ./scripts/stop-etcd.sh
+You also need an `etcd` server to run Serverledge. To quickly start a local
+server, just download [a recent
+release](https://github.com/etcd-io/etcd/releases) of `etcd` and just run `./etcd`.
 
 Start a local Serverledge node:
 
@@ -117,7 +107,7 @@ Note that we currently support output capture only for some runtimes (e.g., Pyth
 
 ## Distributed Deployment
 
-[This repository](https://github.com/grussorusso/serverledge-deploy) provides an
+[This repository](https://github.com/serverledge-faas/serverledge-deploy) provides an
 Ansible playbook to deploy Serverledge in a distributed configuration.
 
 In this case, you can instruct `serverledge-cli` to

cmd/serverledge/main.go

@@ -10,17 +10,18 @@ import (
 	"os/signal"
 	"time"
 
+	"github.com/serverledge-faas/serverledge/internal/node"
+
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
 	"github.com/serverledge-faas/serverledge/internal/api"
 	"github.com/serverledge-faas/serverledge/internal/cache"
 	"github.com/serverledge-faas/serverledge/internal/config"
 	"github.com/serverledge-faas/serverledge/internal/metrics"
-	"github.com/serverledge-faas/serverledge/internal/node"
 	"github.com/serverledge-faas/serverledge/internal/registration"
 	"github.com/serverledge-faas/serverledge/internal/scheduling"
+	"github.com/serverledge-faas/serverledge/internal/telemetry"
 	"github.com/serverledge-faas/serverledge/utils"
-
-	"github.com/labstack/echo/v4"
-	"github.com/labstack/echo/v4/middleware"
 )
 
 func startAPIServer(e *echo.Echo) {
@@ -133,6 +134,25 @@ func main() {
 
 	go metrics.Init()
 
+	if config.GetBool(config.TRACING_ENABLED, false) {
+		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+		defer stop()
+
+		tracesOutfile := config.GetString(config.TRACING_OUTFILE, "")
+		if len(tracesOutfile) < 1 {
+			tracesOutfile = fmt.Sprintf("traces-%s.json", time.Now().Format("20060102-150405"))
+		}
+		log.Printf("Enabling tracing to %s\n", tracesOutfile)
+		otelShutdown, err := telemetry.SetupOTelSDK(ctx, tracesOutfile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		// Handle shutdown properly so nothing leaks.
+		defer func() {
+			err = errors.Join(err, otelShutdown(context.Background()))
+		}()
+	}
+
 	e := echo.New()
 
 	// Register a signal handler to cleanup things on termination
@@ -151,3 +171,17 @@ func main() {
 	api.StartAPIServer(e)
 
 }
+
+func createSchedulingPolicy() scheduling.Policy {
+	policyConf := config.GetString(config.SCHEDULING_POLICY, "default")
+	log.Printf("Configured policy: %s\n", policyConf)
+	if policyConf == "cloudonly" {
+		return &scheduling.CloudOnlyPolicy{}
+	} else if policyConf == "edgecloud" {
+		return &scheduling.CloudEdgePolicy{}
+	} else if policyConf == "edgeonly" {
+		return &scheduling.EdgePolicy{}
+	} else {
+		return &scheduling.DefaultLocalPolicy{}
+	}
+}

docs/api.md

@@ -156,8 +156,68 @@ An example response for a successful **asynchronous** request:
 > | `503`         | `text/plain`              |  |    Prewarming failed                        |
 
 ------------------------------------------------------------------------------------------
+### Status information
 
-<!--
-status API
-function API
--->
+ <code>GET</code> <code><b>/status</b></code> (get current status of the node)
+
+##### Parameters
+
+None
+
+##### Responses
+
+> | http code     | content-type                      | response                        | comments                                    |
+> |---------------|-----------------------------------|---------------------------------|-----------------------------------|
+> | `200`         | `application/json`        | *See response example.*    |                            |
+> | `500`         | `text/plain`              | `Could not retrieve results` |    
+
+An example response for a successful request:
+
+```
+{
+	"Url": "http://192.168.1.23:1323",
+	"AvailableWarmContainers": {
+		"isprime": 1,
+		"sleepSec": 2
+	},
+	"AvailableMemMB": 3072,
+	"AvailableCPUs": 8,
+	"Coordinates": {
+		"Vec": [
+			0,
+			0,
+			0
+		],
+		"Error": 1.5,
+		"Height": 0.00001
+	},
+	"LoadAvg": [
+		1.34,
+		1.43,
+		0.94
+	]
+}
+```
+
+`AvailableMemMB` and `AvailableCPUs` indicate the currently amount of memory
+and CPUs (intended as vCPUs or fractions of them) that can be allocated to new
+requests in the node. `Coordinates.Vec` reports the coordinates of this node
+in the virtual coordinate space computed by Vivaldi algorithm.
+
+### Listing functions
+
+ <code>GET</code> <code><b>/functions</b></code> (get list of registered functions)
+
+##### Parameters
+
+None
+
+##### Responses
+
+> | http code     | content-type                      | response                        | comments                                    |
+> |---------------|-----------------------------------|---------------------------------|-----------------------------------|
+> | `200`         | `application/json`        | *List of names of existing functions.*    |                            |
+> | `500`         | `text/plain`              | `Could not retrieve results` |    
+
+Note that functions are globally registered in the system. Therefore, the same
+list is returned by every node in the same cluster.

docs/tracing.md

@@ -0,0 +1,15 @@
+Serverledge relies on [OpenTelemetry](https://opentelemetry.io) for optional
+request tracing, aimed at performance investigations.
+
+## Enabling tracing
+
+Tracing is disabled by default. It can enabled with the following configuration
+line:
+
+    tracing.enabled: true
+
+By default, JSON-encoded traces are written to `./traces-<timestamp>.json`.
+The following line sets a custom output file:
+
+    tracing.outfile: /path/to/file.json
+

go.mod

@@ -1,6 +1,8 @@
 module github.com/serverledge-faas/serverledge
 
-go 1.20
+go 1.21
+
+toolchain go1.22.5
 
 require (
 	github.com/LK4D4/trylock v0.0.0-20191027065348-ff7e133a5c54
@@ -11,10 +13,14 @@ require (
 	github.com/labstack/echo/v4 v4.6.1
 	github.com/labstack/gommon v0.3.0
 	github.com/lithammer/shortuuid v3.0.0+incompatible
+	github.com/mikoim/go-loadavg v0.0.0-20150917074714-35ece5f6d547
 	github.com/prometheus/client_golang v1.14.0
-	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.4.0
 	go.etcd.io/etcd/client/v3 v3.5.1
+	go.opentelemetry.io/otel v1.28.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0
+	go.opentelemetry.io/otel/sdk v1.28.0
+	go.opentelemetry.io/otel/trace v1.28.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	golang.org/x/net v0.15.0
 	google.golang.org/grpc v1.57.0
@@ -30,12 +36,14 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.8 // indirect
@@ -54,19 +62,20 @@ require (
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cast v1.3.0 // indirect
+	github.com/spf13/cobra v1.9.1 // indirect
 	github.com/spf13/jwalterweatherman v1.0.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.1 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.1 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
 	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	golang.org/x/tools v0.13.0 // indirect