use urls from zos-config if exists

pkg/environment/config.go

@@ -30,7 +30,23 @@ type Config struct {
 	RolloutUpgrade struct {
 		TestFarms []uint32 `json:"test_farms"`
 	} `json:"rollout_upgrade"`
-	RelaysURLs []string `json:"relays_urls"`
+	SubstrateURL  []string `json:"substrate_urls"`
+	RelaysURLs    []string `json:"relays_urls"`
+	ActivationURL []string `json:"activation_urls"`
+	GraphQL       []string `json:"graphql_urls"`
+	KycURL        string   `json:"kyc_url"`
+	RegistrarURL  string   `json:"registrar_url"`
+	BinRepo       string   `json:"bin_repo"`
+	GeoipURLs     []string `json:"geoip_urls"`
+
+	// FlistURL   string `json:"flist_url"`
+	// V4FlistURL string `json:"v4_flist_url"`
+
+	HubURL   string `json:"hub_url"`
+	V4HubURL string `json:"v4_hub_url"`
+
+	// HubStorage   string `json:"hub_storage"`
+	// V4HubStorage string `json:"v4_hub_storage"`
 }
 
 // Merge, updates current config with cfg merging and override config

pkg/environment/environment.go

@@ -1,6 +1,7 @@
 package environment
 
 import (
+	"net/http"
 	"os"
 	"slices"
 	"strconv"
@@ -16,8 +17,24 @@ import (
 
 const (
 	baseExtendedURL = "https://raw.githubusercontent.com/threefoldtech/zos-config/main/"
+
+	defaultHubURL   = "https://hub.threefold.me"
+	defaultV4HubURL = "https://v4.hub.threefold.me"
+
+	defaultFlistURL   = "redis://hub.threefold.me:9900"
+	defaultV4FlistURL = "redis://v4.hub.threefold.me:9940"
+
+	defaultHubStorage   = "zdb://hub.threefold.me:9900"
+	defaultV4HubStorage = "zdb://v4.hub.threefold.me:9940"
 )
 
+var defaultGeoipURLs = []string{
+	"https://geoip.threefold.me/",
+	"https://geoip.grid.tf/",
+	"https://02.geoip.grid.tf/",
+	"https://03.geoip.grid.tf/",
+}
+
 // PubMac specify how the mac address of the public nic
 // (in case of public-config) is calculated
 type PubMac string
@@ -37,8 +54,12 @@ const (
 type Environment struct {
 	RunningMode RunMode
 
-	FlistURL string
-	BinRepo  string
+	FlistURL   string
+	HubStorage string
+	BinRepo    string
+
+	HubURL   string
+	V4HubURL string
 
 	FarmID pkg.FarmID
 	Orphan bool
@@ -51,6 +72,7 @@ type Environment struct {
 	relaysURLs    []string
 	ActivationURL []string
 	GraphQL       []string
+	GeoipURLs     []string
 	KycURL        string
 	RegistrarURL  string
 
@@ -121,14 +143,18 @@ var (
 			"https://activation.dev.grid.tf/activation/activate",
 			"https://activation.02.dev.grid.tf/activation/activate",
 		},
-		FlistURL: "redis://hub.grid.tf:9900",
-		BinRepo:  "tf-zos-v3-bins.dev",
+		HubURL:     defaultHubURL,
+		V4HubURL:   defaultV4HubURL,
+		FlistURL:   defaultFlistURL,
+		HubStorage: defaultHubStorage,
+		BinRepo:    "tf-zos-v3-bins.dev",
 		GraphQL: []string{
 			"https://graphql.dev.grid.tf/graphql",
 			"https://graphql.02.dev.grid.tf/graphql",
 		},
 		KycURL:       "https://kyc.dev.grid.tf",
 		RegistrarURL: "http://registrar.dev4.grid.tf",
+		GeoipURLs:    defaultGeoipURLs,
 	}
 
 	envTest = Environment{
@@ -145,14 +171,18 @@ var (
 			"https://activation.test.grid.tf/activation/activate",
 			"https://activation.02.test.grid.tf/activation/activate",
 		},
-		FlistURL: "redis://hub.grid.tf:9900",
-		BinRepo:  "tf-zos-v3-bins.test",
+		HubURL:     defaultHubURL,
+		V4HubURL:   defaultV4HubURL,
+		FlistURL:   defaultFlistURL,
+		HubStorage: defaultHubStorage,
+		BinRepo:    "tf-zos-v3-bins.test",
 		GraphQL: []string{
 			"https://graphql.test.grid.tf/graphql",
 			"https://graphql.02.test.grid.tf/graphql",
 		},
 		KycURL:       "https://kyc.test.grid.tf",
 		RegistrarURL: "http://registrar.test4.grid.tf",
+		GeoipURLs:    defaultGeoipURLs,
 	}
 
 	envQA = Environment{
@@ -169,14 +199,18 @@ var (
 			"https://activation.qa.grid.tf/activation/activate",
 			"https://activation.02.qa.grid.tf/activation/activate",
 		},
-		FlistURL: "redis://hub.grid.tf:9900",
-		BinRepo:  "tf-zos-v3-bins.qanet",
+		HubURL:     defaultHubURL,
+		V4HubURL:   defaultV4HubURL,
+		FlistURL:   defaultFlistURL,
+		HubStorage: defaultHubStorage,
+		BinRepo:    "tf-zos-v3-bins.qanet",
 		GraphQL: []string{
 			"https://graphql.qa.grid.tf/graphql",
 			"https://graphql.02.qa.grid.tf/graphql",
 		},
 		KycURL:       "https://kyc.qa.grid.tf",
 		RegistrarURL: "https://registrar.qa4.grid.tf",
+		GeoipURLs:    defaultGeoipURLs,
 	}
 
 	envProd = Environment{
@@ -195,15 +229,21 @@ var (
 		ActivationURL: []string{
 			"https://activation.grid.tf/activation/activate",
 			"https://activation.02.grid.tf/activation/activate",
+			"https://activation.grid.threefold.me/activation/activate",
 		},
-		FlistURL: "redis://hub.grid.tf:9900",
-		BinRepo:  "tf-zos-v3-bins",
+		HubURL:     defaultHubURL,
+		V4HubURL:   defaultV4HubURL,
+		FlistURL:   defaultFlistURL,
+		HubStorage: defaultHubStorage,
+		BinRepo:    "tf-zos-v3-bins",
 		GraphQL: []string{
 			"https://graphql.grid.tf/graphql",
 			"https://graphql.02.grid.tf/graphql",
+			"https://graphql.grid.threefold.me/graphql",
 		},
-		KycURL:       "https://kyc.grid.tf",
-		RegistrarURL: "https://registrar.prod4.grid.tf",
+		KycURL:       "https://kyc.threefold.me",
+		RegistrarURL: "https://registrar.prod4.threefold.me",
+		GeoipURLs:    defaultGeoipURLs,
 	}
 )
 
@@ -283,22 +323,82 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
 		env = envProd
 	}
 
-	if substrate, ok := params.Get("substrate"); ok {
-		if len(substrate) > 0 {
-			env.SubstrateURL = substrate
-		}
+	config, err := getConfig(env.RunningMode, baseExtendedURL, http.DefaultClient)
+	if err != nil {
+		// maybe the node can't reach the internet right now
+		// this will enforce node to skip config
+		// or we can keep retrying untill it can fetch config
+		config = Config{}
 	}
 
-	if relay, ok := params.Get("relay"); ok {
-		if len(relay) > 0 {
-			env.relaysURLs = relay
-		}
+	if substrate, ok := params.Get("substrate"); ok && len(substrate) > 0 {
+		env.SubstrateURL = substrate
+	} else if substrate := config.SubstrateURL; len(substrate) > 0 {
+		env.SubstrateURL = substrate
 	}
 
-	if activation, ok := params.Get("activation"); ok {
-		if len(activation) > 0 {
-			env.ActivationURL = activation
-		}
+	if relay, ok := params.Get("relay"); ok && len(relay) > 0 {
+		env.relaysURLs = relay
+	} else if relay := config.RelaysURLs; len(relay) > 0 {
+		env.relaysURLs = relay
+	}
+
+	if activation, ok := params.Get("activation"); ok && len(activation) > 0 {
+		env.ActivationURL = activation
+	} else if activation := config.ActivationURL; len(activation) > 0 {
+		env.ActivationURL = activation
+	}
+
+	if graphql := config.GraphQL; len(graphql) > 0 {
+		env.GraphQL = graphql
+	}
+
+	if bin := config.BinRepo; len(bin) > 0 {
+		env.BinRepo = bin
+	}
+
+	if kyc := config.KycURL; len(kyc) > 0 {
+		env.KycURL = kyc
+	}
+
+	if registrar := config.RegistrarURL; len(registrar) > 0 {
+		env.RegistrarURL = registrar
+	}
+
+	if geoip := config.GeoipURLs; len(geoip) > 0 {
+		env.GeoipURLs = geoip
+	}
+	// flist url and hub urls shouldn't listen to changes in config as long as we can't change it at run time.
+	// it would cause breakage in vmd that needs a reboot to be recovered.
+	// if flist := config.FlistURL; len(flist) > 0 {
+	// 	env.FlistURL = flist
+	// }
+	//
+	// if storage := config.HubStorage; len(storage) > 0 {
+	// 	env.HubStorage = storage
+	// }
+
+	if hub := config.HubURL; len(hub) > 0 {
+		env.HubURL = hub
+	}
+
+	// some modules needs v3 hub url even if the node is of v4
+	if hub := config.V4HubURL; len(hub) > 0 {
+		env.V4HubURL = hub
+	}
+
+	// if the node running v4 chage urls to use v4 hub
+	if params.IsV4() {
+		env.FlistURL = defaultV4FlistURL
+		// if flist := config.V4FlistURL; len(flist) > 0 {
+		// 	env.FlistURL = flist
+		// }
+
+		env.HubStorage = defaultV4HubStorage
+		// if storage := config.V4HubStorage; len(storage) > 0 {
+		// 	env.HubStorage = storage
+		// }
+
 	}
 
 	if farmSecret, ok := params.Get("secret"); ok {
@@ -378,10 +478,5 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
 		env.BinRepo = e
 	}
 
-	// if the node running v4 chage flisturl to use v4.hub.grid.tf
-	if params.IsV4() {
-		env.FlistURL = "redis://v4.hub.grid.tf:9940"
-	}
-
 	return env, nil
 }

pkg/flist/flist_test.go

@@ -3,6 +3,7 @@ package flist
 import (
 	"bytes"
 	"context"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -15,6 +16,7 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	"github.com/threefoldtech/zosbase/pkg"
+	"github.com/threefoldtech/zosbase/pkg/environment"
 	"github.com/threefoldtech/zosbase/pkg/gridtypes"
 )
 
@@ -159,7 +161,10 @@ func TestMountUnmount(t *testing.T) {
 
 	sys.On("Mount", "overlay", filepath.Join(root, "mountpoint", name), "overlay", uintptr(syscall.MS_NOATIME), mock.Anything).Return(nil)
 
-	mnt, err := flister.mountInNamespace(name, "https://hub.grid.tf/thabet/redis.flist", pkg.DefaultMountOptions, "")
+	redisFlist, err := url.JoinPath(environment.MustGet().HubURL, "thabet", "redis.flist")
+	require.NoError(t, err)
+
+	mnt, err := flister.mountInNamespace(name, redisFlist, pkg.DefaultMountOptions, "")
 	require.NoError(t, err)
 
 	// Trick flister into thinking that 0-fs has exited
@@ -186,7 +191,10 @@ func TestMountUnmountRO(t *testing.T) {
 	flist := mock.Anything
 	sys.On("Mount", flist, filepath.Join(root, "mountpoint", name), "bind", uintptr(syscall.MS_BIND), "").Return(nil)
 
-	mnt, err := flister.mountInNamespace(name, "https://hub.grid.tf/thabet/redis.flist", pkg.ReadOnlyMountOptions, "")
+	redisFlist, err := url.JoinPath(environment.MustGet().HubURL, "thabet", "redis.flist")
+	require.NoError(t, err)
+
+	mnt, err := flister.mountInNamespace(name, redisFlist, pkg.ReadOnlyMountOptions, "")
 	require.NoError(t, err)
 
 	// Trick flister into thinking that 0-fs has exited
@@ -223,11 +231,14 @@ func TestIsolation(t *testing.T) {
 	name2 := "test2"
 	sys.On("Mount", "overlay", filepath.Join(root, "mountpoint", name2), "overlay", uintptr(syscall.MS_NOATIME), mock.Anything).Return(nil)
 
-	path1, err := flister.mountInNamespace(name1, "https://hub.grid.tf/thabet/redis.flist", pkg.DefaultMountOptions, "")
+	redisFlist, err := url.JoinPath(environment.MustGet().HubURL, "thabet", "redis.flist")
+	require.NoError(err)
+
+	path1, err := flister.mountInNamespace(name1, redisFlist, pkg.DefaultMountOptions, "")
 	require.NoError(err)
 	args1 := cmder.m
 
-	path2, err := flister.mountInNamespace(name2, "https://hub.grid.tf/thabet/redis.flist", pkg.DefaultMountOptions, "")
+	path2, err := flister.mountInNamespace(name2, redisFlist, pkg.DefaultMountOptions, "")
 	require.NoError(err)
 	args2 := cmder.m
 
@@ -246,14 +257,17 @@ func TestDownloadFlist(t *testing.T) {
 
 	f := newFlister(root, strg, cmder, sys)
 
-	hash1, path1, err := f.downloadFlist("https://hub.grid.tf/thabet/redis.flist", "")
+	redisFlist, err := url.JoinPath(environment.MustGet().HubURL, "thabet", "redis.flist")
+	require.NoError(err)
+
+	hash1, path1, err := f.downloadFlist(redisFlist, "")
 	require.NoError(err)
 
 	// now corrupt the flist
 	err = os.Truncate(string(path1), 512)
 	require.NoError(err)
 
-	hash2, path2, err := f.downloadFlist("https://hub.grid.tf/thabet/redis.flist", "")
+	hash2, path2, err := f.downloadFlist(redisFlist, "")
 	require.NoError(err)
 
 	require.EqualValues(path1, path2)

pkg/gateway/flist.go

@@ -3,24 +3,29 @@ package gateway
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zbus"
 	"github.com/threefoldtech/zosbase/pkg"
+	"github.com/threefoldtech/zosbase/pkg/environment"
 	"github.com/threefoldtech/zosbase/pkg/stubs"
 )
 
-const (
-	flist = "https://hub.grid.tf/tf-autobuilder/traefik-2.9.9.flist"
-)
-
 // ensureTraefikBin makes sure traefik flist is mounted.
 // TODO: we need to "update" traefik and restart the service
 // if new version is available!
 func ensureTraefikBin(ctx context.Context, cl zbus.Client) (string, error) {
 	const bin = "traefik"
 	flistd := stubs.NewFlisterStub(cl)
+
+	env := environment.MustGet()
+	flist, err := url.JoinPath(env.HubURL, "tf-autobuilder", "traefik-2.9.9.flist")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to constract traefik flist url")
+	}
+
 	hash, err := flistd.FlistHash(ctx, flist)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to get traefik flist hash")