diff --git a/Makefile b/Makefile index 4a37c658..e6b72203 100644 --- a/Makefile +++ b/Makefile @@ -148,6 +148,7 @@ lint: tidy fmt vet golangci-lint ## Run the lint check pr: lint test yaml ## Run targets required for PR ##@ Test +ARCH = $(shell go env GOARCH) TEST_BIN_DIR = $(BASE_DIR)/tools/testbin TEST_RESULT_DIR = $(BASE_DIR)/testing ENVTEST ?= $(TEST_BIN_DIR)/setup-envtest @@ -307,9 +308,9 @@ catalog-push: ## Push a catalog image. $(MAKE) docker-push IMG=$(CATALOG_IMG) test-daemon: - $(DOCKER) build -t daemon-test:latest -f ./daemon/dockerfiles/Dockerfile.multi-nicd-test . - $(DOCKER) run -i --privileged daemon-test /bin/bash -c "cd /usr/local/build/cni&&make test" - $(DOCKER) run -i --privileged daemon-test /bin/bash -c "cd /usr/local/build/daemon/src&&make test-verbose" + $(DOCKER) build --platform linux/$(ARCH) -t daemon-test:latest -f ./daemon/dockerfiles/Dockerfile.multi-nicd-test . + $(DOCKER) run --platform linux/$(ARCH) -i --rm --privileged daemon-test /bin/bash -c "cd /usr/local/build/cni&&make test" + $(DOCKER) run --platform linux/$(ARCH) -i --rm -v ./testing:/usr/local/build/daemon/src/testing --privileged daemon-test /bin/bash -c "cd /usr/local/build/daemon/src&&make test" build-push-kbuilder-base: $(DOCKER) build -t $(IMAGE_TAG_BASE)-kbuilder:v$(VERSION) -f ./daemon/dockerfiles/Dockerfile.kbuilder . diff --git a/daemon/Makefile b/daemon/Makefile index 01ee1f42..82ff046c 100644 --- a/daemon/Makefile +++ b/daemon/Makefile @@ -12,8 +12,8 @@ DAEMON_IMG ?= $(IMAGE_TAG_BASE)-daemon:v1.2.9 test-verbose: $(MAKE) -C src test-verbose -test-env: - $(MAKE) -C src test-env +test: + $(MAKE) -C src test update-cni-local: ifeq ($(shell uname), Linux) diff --git a/daemon/dockerfiles/Dockerfile.kbuilder b/daemon/dockerfiles/Dockerfile.kbuilder index a4fde797..190f8968 100644 --- a/daemon/dockerfiles/Dockerfile.kbuilder +++ b/daemon/dockerfiles/Dockerfile.kbuilder @@ -1,7 +1,7 @@ FROM ubuntu:20.04 RUN apt-get update \ - && apt-get install -y wget tar build-essential net-tools iproute2 git curl \ + && apt-get install -y wget tar build-essential net-tools iproute2 git curl jq \ && rm -rf /var/lib/apt/lists/* # install go diff --git a/daemon/dockerfiles/Dockerfile.multi-nicd-test b/daemon/dockerfiles/Dockerfile.multi-nicd-test index b5916293..8e3c4da2 100644 --- a/daemon/dockerfiles/Dockerfile.multi-nicd-test +++ b/daemon/dockerfiles/Dockerfile.multi-nicd-test @@ -1,5 +1,9 @@ FROM ghcr.io/foundation-model-stack/multi-nic-cni-kbuilder:v1.2.9 +RUN apt-get update \ + && apt-get install -y jq \ + && rm -rf /var/lib/apt/lists/* + RUN mkdir -p /usr/local/app RUN mkdir -p /host/opt/cni/bin @@ -18,12 +22,9 @@ RUN cd cni && go mod tidy && go build -o /usr/local/app/aws-ipvlan ./plugins/mai RUN cd cni && make ginkgo-set # build Daemon COPY daemon daemon +COPY hack daemon/hack RUN mkdir -p /usr/local/kubebuilder RUN mkdir -p /usr/local/build/daemon/src/test-bin -RUN curl -sSLo setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh -RUN cp setup-envtest.sh /usr/local/build/daemon/src/test-bin/setup-envtest.sh -RUN cd daemon/src && make test-env && cp -r /usr/local/build/daemon/src/test-bin/bin /usr/local/kubebuilder/bin -RUN mkdir -p daemon/src/test-bin RUN cd daemon/src && go mod tidy RUN cd daemon/src && go build -o /usr/local/app/daemon RUN cp /usr/local/app/multi-nic /usr/local/bin \ diff --git a/daemon/src/Makefile b/daemon/src/Makefile index 9b642c96..01aa4abc 100644 --- a/daemon/src/Makefile +++ b/daemon/src/Makefile @@ -4,19 +4,52 @@ # all: build -ENVTEST_ASSETS_DIR=$(PWD)/test-bin -export PATH := $(PATH):$(ENVTEST_ASSETS_DIR) - -test-env: SHELL := /bin/bash -test-env: - mkdir -p ${ENVTEST_ASSETS_DIR} - test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh - source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); - cp -r ${ENVTEST_ASSETS_DIR}/bin /usr/local/kubebuilder/bin - -test: SHELL := /bin/bash -test: test-env ## Run tests. - go test ./... -coverprofile cover.out +BASE_DIR=$(shell pwd) + +##@ Test +TEST_BIN_DIR = $(BASE_DIR)/tools/testbin +TEST_RESULT_DIR = $(BASE_DIR)/testing +ENVTEST ?= $(TEST_BIN_DIR)/setup-envtest +GINKGO ?= $(TEST_BIN_DIR)/ginkgo +ENVTEST_K8S_VERSION ?= 1.28.3 + +$(TEST_BIN_DIR): + mkdir -p $(TEST_BIN_DIR) + +$(TEST_RESULT_DIR): + mkdir -p $(TEST_RESULT_DIR) + +# go-get-tool will 'go get' any package $3 and install it to $2 in folder $1. +define go-get-tool +@[ -f $(2) ] || { \ +set -e ;\ +TMP_DIR=$$(mktemp -d) ;\ +cd $$TMP_DIR ;\ +go mod init tmp ;\ +echo "Downloading $(3)" ;\ +GOBIN=$(1) go install $(3) ;\ +ls $$TMP_DIR;\ +echo $(2);\ +rm -rf $$TMP_DIR ;\ +} +endef + +.PHONY: ginkgo +ginkgo: $(GINKGO) ## Download and install ginkgo locally if necessary. +$(GINKGO): $(TEST_BIN_DIR) + $(call go-get-tool,$(TEST_BIN_DIR),$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo@v2.21.0) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download and install setup-envtest locally if necessary. +$(ENVTEST): $(TEST_BIN_DIR) + $(call go-get-tool,$(TEST_BIN_DIR),$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20240624150636-162a113134de) + +.PHONY: test +test: $(TEST_RESULT_DIR) ginkgo envtest + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(TEST_BIN_DIR)/ginkgo run --cover --coverprofile=multi-nicd-coverage --json-report multi-nicd-unittest-report.json ./... + @../hack/json-report-to-markdown.sh multi-nicd-unittest-report "Unit Test" + @rm multi-nicd-unittest-report.json + @../hack/coverage-to-markdown.sh multi-nicd-coverage "Multi-NIC Daemon Unit Test Coverage" build: test @go build -o ../bin/daemon . diff --git a/daemon/src/allocator/allocator_test.go b/daemon/src/allocator/allocator_test.go index a72b0ee9..eefd288d 100644 --- a/daemon/src/allocator/allocator_test.go +++ b/daemon/src/allocator/allocator_test.go @@ -6,11 +6,13 @@ package allocator import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "testing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "time" + "github.com/foundation-model-stack/multi-nic-cni/daemon/backend" ) diff --git a/daemon/src/go.mod b/daemon/src/go.mod index d17de458..201b783f 100644 --- a/daemon/src/go.mod +++ b/daemon/src/go.mod @@ -5,8 +5,8 @@ go 1.23 require ( github.com/gorilla/mux v1.8.0 github.com/jaypipes/ghw v0.14.0 - github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.18.1 + github.com/onsi/ginkgo/v2 v2.21.0 + github.com/onsi/gomega v1.36.1 github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 k8s.io/api v0.23.3 k8s.io/apiextensions-apiserver v0.23.0 diff --git a/daemon/src/main_test.go b/daemon/src/main_test.go index b5cbb4b4..d21870de 100644 --- a/daemon/src/main_test.go +++ b/daemon/src/main_test.go @@ -8,48 +8,26 @@ package main import ( "bytes" "encoding/json" - "fmt" "io/ioutil" "net/http" "net/http/httptest" - "testing" da "github.com/foundation-model-stack/multi-nic-cni/daemon/allocator" - backend "github.com/foundation-model-stack/multi-nic-cni/daemon/backend" di "github.com/foundation-model-stack/multi-nic-cni/daemon/iface" dr "github.com/foundation-model-stack/multi-nic-cni/daemon/router" ds "github.com/foundation-model-stack/multi-nic-cni/daemon/selector" - "k8s.io/client-go/kubernetes" v1 "k8s.io/api/core/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/client-go/discovery" - "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" - "k8s.io/client-go/restmapper" - "context" - "path/filepath" - - "github.com/vishvananda/netlink" "k8s.io/apimachinery/pkg/runtime" "log" - "strings" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - "os" //+kubebuilder:scaffold:imports ) @@ -115,99 +93,6 @@ var GPU_BUS_MAP map[string]string = map[string]string{ "GPU-581b17ed-1c48-9b8c-6a9b-e2e6f99500dc": "0000:0c:05.0", } -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "MultiNic CNI Suite") -} - -func setTestLatestInterfaces() { - for index, master := range MASTER_INTERFACES { - netAddress := MASTER_NETADDRESSES[index] - vendor := MASTER_VENDORS[index] - product := MASTER_PRODUCTS[index] - pciAddress := MASTER_PCIADDRESS[index] - iface := di.InterfaceInfoType{ - InterfaceName: master, - NetAddress: netAddress, - Vendor: vendor, - Product: product, - PciAddress: pciAddress, - } - di.SetInterfaceInfoCache(master, iface) - } - interfaceMap := di.GetInterfaceInfoCache() - Expect(len(interfaceMap)).To(Equal(2)) -} - -func replacePodUID(clientset *kubernetes.Clientset) { - var err error - // Get PodUID - targetPod, err = clientset.CoreV1().Pods(POD_NAMESPACE).Get(context.TODO(), POD_NAME, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - // Replace checkpoint file - - // Read the file content - content, err := ioutil.ReadFile(EXAMPLE_CHECKPOINT) - Expect(err).NotTo(HaveOccurred()) - - // Perform text replacements - modifiedContent := strings.ReplaceAll(string(content), TO_REPLACE_POD_UID, string(targetPod.UID)) - - // Write the modified content back to the file - err = ioutil.WriteFile(EXAMPLE_CHECKPOINT, []byte(modifiedContent), 0644) - Expect(err).NotTo(HaveOccurred()) -} - -var _ = BeforeSuite(func() { - os.Setenv("TEST_MODE", "true") - - // this env should be set by config.multinic when creating the daemonset - os.Setenv(NODENAME_ENV, FULL_HOST_NAME) - initHostName() - Expect(hostName).To(Equal(FULL_HOST_NAME)) - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - install.Install(scheme) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "example", "crd")}, - ErrorIfCRDPathMissing: true, - Scheme: scheme, - } - - err := apiextensionsv1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme - - cfg, err := testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - dyn, err = dynamic.NewForConfig(cfg) - Expect(err).NotTo(HaveOccurred()) - dc, err = discovery.NewDiscoveryClientForConfig(cfg) - Expect(err).NotTo(HaveOccurred()) - - da.IppoolHandler = backend.NewIPPoolHandler(cfg) - ds.MultinicnetHandler = backend.NewMultiNicNetworkHandler(cfg) - ds.NetAttachDefHandler = backend.NewNetAttachDefHandler(cfg) - ds.DeviceClassHandler = backend.NewDeviceClassHandler(cfg) - da.K8sClientset, _ = kubernetes.NewForConfig(cfg) - ds.K8sClientset, _ = kubernetes.NewForConfig(cfg) - - deployExamples(EXAMPLE_RESOURCE_FOLDER, false) - addMasterInterfaces() - replacePodUID(ds.K8sClientset) -}, 60) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - deleteExamples(EXAMPLE_RESOURCE_FOLDER, true) - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) - deleteMasterInterfaces() -}) - var _ = Describe("Test L3Config Add/Delete", func() { It("apply/delete l3config", func() { l3config, err := json.Marshal(requestL3Config) @@ -505,109 +390,23 @@ var _ = Describe("Test NIC Select", func() { }) }) -func deployExamples(folder string, ignoreErr bool) { - files, err := ioutil.ReadDir(folder) - Expect(err).NotTo(HaveOccurred()) - - for _, file := range files { - fileLocation := folder + "/" + file.Name() - obj, dr := getDR(fileLocation, ignoreErr) - if dr == nil { - fmt.Println("No DR, deploy") - continue - } - _, err = dr.Create(context.TODO(), obj, metav1.CreateOptions{}) - fmt.Printf("Deploy %s (%v): %v\n", fileLocation, ignoreErr, err) - if !ignoreErr { - Expect(err).NotTo(HaveOccurred()) - } - } - -} - -func getDR(fileName string, ignoreErr bool) (*unstructured.Unstructured, dynamic.ResourceInterface) { - bodyBytes, err := ioutil.ReadFile(fileName) - if ignoreErr && err != nil { - return nil, nil - } - Expect(err).NotTo(HaveOccurred()) - obj := &unstructured.Unstructured{} - dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) - _, gvk, err := dec.Decode(bodyBytes, nil, obj) - if ignoreErr && err != nil { - return nil, nil - } - Expect(err).NotTo(HaveOccurred()) - return obj, getResourceInterface(gvk, obj.GetNamespace(), ignoreErr) -} - -func getResourceInterface(gvk *schema.GroupVersionKind, ns string, ignoreErr bool) dynamic.ResourceInterface { - mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) - mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if !ignoreErr { - Expect(err).NotTo(HaveOccurred()) - } - if err != nil { - return nil - } - - var dr dynamic.ResourceInterface - if mapping.Scope.Name() == meta.RESTScopeNameNamespace { - dr = dyn.Resource(mapping.Resource).Namespace(ns) - } else { - dr = dyn.Resource(mapping.Resource) - } - return dr -} - -func deleteExamples(folder string, ignoreErr bool) { - files, err := ioutil.ReadDir(folder) - Expect(err).NotTo(HaveOccurred()) - - for _, file := range files { - fileLocation := folder + "/" + file.Name() - fmt.Printf("Delete %s (%v)\n", fileLocation, ignoreErr) - obj, dr := getDR(fileLocation, ignoreErr) - if dr == nil { - fmt.Println("No DR, delete") - continue - } - err = dr.Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{}) - if !ignoreErr { - Expect(err).NotTo(HaveOccurred()) - } - } -} - -func addMasterInterfaces() { - // Add master +func setTestLatestInterfaces() { for index, master := range MASTER_INTERFACES { - linkAttrs := netlink.LinkAttrs{ - Name: master, + netAddress := MASTER_NETADDRESSES[index] + vendor := MASTER_VENDORS[index] + product := MASTER_PRODUCTS[index] + pciAddress := MASTER_PCIADDRESS[index] + iface := di.InterfaceInfoType{ + InterfaceName: master, + NetAddress: netAddress, + Vendor: vendor, + Product: product, + PciAddress: pciAddress, } - err := netlink.LinkAdd(&netlink.Dummy{ - linkAttrs, - }) - Expect(err).NotTo(HaveOccurred()) - masterLink, err := netlink.LinkByName(master) - Expect(err).NotTo(HaveOccurred()) - - addr, _ := netlink.ParseAddr(MASTER_IPS[index]) - netlink.AddrAdd(masterLink, addr) - Expect(err).NotTo(HaveOccurred()) - err = netlink.LinkSetUp(masterLink) - Expect(err).NotTo(HaveOccurred()) - } -} - -func deleteMasterInterfaces() { - // Add master - for _, master := range MASTER_INTERFACES { - masterLink, err := netlink.LinkByName(master) - Expect(err).NotTo(HaveOccurred()) - netlink.LinkSetDown(masterLink) - netlink.LinkDel(masterLink) + di.SetInterfaceInfoCache(master, iface) } + interfaceMap := di.GetInterfaceInfoCache() + Expect(len(interfaceMap)).To(Equal(2)) } func MakeIPRequest(ipJson []byte, path string, handler http.HandlerFunc, shouldResponse bool) []da.IPResponse { diff --git a/daemon/src/router/router_test.go b/daemon/src/router/router_test.go index 2c496426..25ef2c68 100644 --- a/daemon/src/router/router_test.go +++ b/daemon/src/router/router_test.go @@ -3,26 +3,22 @@ * SPDX-License-Identifier: Apache2.0 */ - package router +package router - import ( - "testing" +import ( + "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" - "github.com/vishvananda/netlink" - "fmt" - "os" - ) + "fmt" + "os" - func TestRouter(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Router Test Suite") -} + "github.com/vishvananda/netlink" +) const ( - LOCAL_TABLE_ID = 255 + LOCAL_TABLE_ID = 255 LOCAL_TABLE_NAME = "local" NEW_TABLE_NAME = "newtable" @@ -30,6 +26,21 @@ const ( POD_RT_PATH = "/opt/rt_tables" ) +func TestRouter(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Router Test Suite") +} + +var _ = BeforeSuite(func() { + DeleteTable(NEW_TABLE_NAME, 100) + os.Unsetenv("RT_TABLE_PATH") +}) + +var _ = AfterSuite(func() { + DeleteTable(NEW_TABLE_NAME, 100) + os.Unsetenv("RT_TABLE_PATH") +}) + var _ = Describe("Test Path", func() { It("Set RT path", func() { SetRTTablePath() @@ -39,7 +50,7 @@ var _ = Describe("Test Path", func() { Expect(RT_TABLE_PATH).To(Equal(POD_RT_PATH)) os.Unsetenv("RT_TABLE_PATH") SetRTTablePath() - Expect(RT_TABLE_PATH).To(Equal(DEFAULT_RT_TABLE_PATH)) + Expect(RT_TABLE_PATH).To(Equal(DEFAULT_RT_TABLE_PATH)) }) }) @@ -73,13 +84,3 @@ var _ = Describe("Test RT Table", func() { fmt.Printf("Rules: %v\n", rules) }) }) - -var _ = BeforeSuite(func() { - DeleteTable(NEW_TABLE_NAME, 100) - os.Unsetenv("RT_TABLE_PATH") -}) - -var _ = AfterSuite(func() { - DeleteTable(NEW_TABLE_NAME, 100) - os.Unsetenv("RT_TABLE_PATH") -}) \ No newline at end of file diff --git a/daemon/src/suite_test.go b/daemon/src/suite_test.go new file mode 100644 index 00000000..d870c194 --- /dev/null +++ b/daemon/src/suite_test.go @@ -0,0 +1,217 @@ +/* + * Copyright 2022- IBM Inc. All rights reserved + * SPDX-License-Identifier: Apache2.0 + */ + +package main + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + da "github.com/foundation-model-stack/multi-nic-cni/daemon/allocator" + "github.com/foundation-model-stack/multi-nic-cni/daemon/backend" + ds "github.com/foundation-model-stack/multi-nic-cni/daemon/selector" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vishvananda/netlink" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + //+kubebuilder:scaffold:imports +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Multi-NIC Daemon Suite") +} + +var _ = BeforeSuite(func() { + os.Setenv("TEST_MODE", "true") + + // this env should be set by config.multinic when creating the daemonset + os.Setenv(NODENAME_ENV, FULL_HOST_NAME) + initHostName() + Expect(hostName).To(Equal(FULL_HOST_NAME)) + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + install.Install(scheme) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "example", "crd")}, + ErrorIfCRDPathMissing: true, + Scheme: scheme, + } + + err := apiextensionsv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme + + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + dyn, err = dynamic.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + dc, err = discovery.NewDiscoveryClientForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + + da.IppoolHandler = backend.NewIPPoolHandler(cfg) + ds.MultinicnetHandler = backend.NewMultiNicNetworkHandler(cfg) + ds.NetAttachDefHandler = backend.NewNetAttachDefHandler(cfg) + ds.DeviceClassHandler = backend.NewDeviceClassHandler(cfg) + da.K8sClientset, _ = kubernetes.NewForConfig(cfg) + ds.K8sClientset, _ = kubernetes.NewForConfig(cfg) + + deployExamples(EXAMPLE_RESOURCE_FOLDER, false) + addMasterInterfaces() + replacePodUID(ds.K8sClientset) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + deleteExamples(EXAMPLE_RESOURCE_FOLDER, true) + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) + deleteMasterInterfaces() +}) + +func deployExamples(folder string, ignoreErr bool) { + files, err := os.ReadDir(folder) + Expect(err).NotTo(HaveOccurred()) + + for _, file := range files { + fileLocation := folder + "/" + file.Name() + obj, dr := getDR(fileLocation, ignoreErr) + if dr == nil { + fmt.Println("No DR, deploy") + continue + } + _, err = dr.Create(context.TODO(), obj, metav1.CreateOptions{}) + fmt.Printf("Deploy %s (%v): %v\n", fileLocation, ignoreErr, err) + if !ignoreErr { + Expect(err).NotTo(HaveOccurred()) + } + } +} + +func addMasterInterfaces() { + // Add master + for index, master := range MASTER_INTERFACES { + linkAttrs := netlink.LinkAttrs{ + Name: master, + } + err := netlink.LinkAdd(&netlink.Dummy{ + linkAttrs, + }) + Expect(err).NotTo(HaveOccurred()) + masterLink, err := netlink.LinkByName(master) + Expect(err).NotTo(HaveOccurred()) + + addr, _ := netlink.ParseAddr(MASTER_IPS[index]) + netlink.AddrAdd(masterLink, addr) + Expect(err).NotTo(HaveOccurred()) + err = netlink.LinkSetUp(masterLink) + Expect(err).NotTo(HaveOccurred()) + } +} + +func replacePodUID(clientset *kubernetes.Clientset) { + var err error + // Get PodUID + targetPod, err = clientset.CoreV1().Pods(POD_NAMESPACE).Get(context.TODO(), POD_NAME, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + // Replace checkpoint file + + // Read the file content + content, err := os.ReadFile(EXAMPLE_CHECKPOINT) + Expect(err).NotTo(HaveOccurred()) + + // Perform text replacements + modifiedContent := strings.ReplaceAll(string(content), TO_REPLACE_POD_UID, string(targetPod.UID)) + + // Write the modified content back to the file + err = os.WriteFile(EXAMPLE_CHECKPOINT, []byte(modifiedContent), 0644) + Expect(err).NotTo(HaveOccurred()) +} + +func deleteExamples(folder string, ignoreErr bool) { + files, err := os.ReadDir(folder) + Expect(err).NotTo(HaveOccurred()) + + for _, file := range files { + fileLocation := folder + "/" + file.Name() + fmt.Printf("Delete %s (%v)\n", fileLocation, ignoreErr) + obj, dr := getDR(fileLocation, ignoreErr) + if dr == nil { + fmt.Println("No DR, delete") + continue + } + err = dr.Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{}) + if !ignoreErr { + Expect(err).NotTo(HaveOccurred()) + } + } +} + +func deleteMasterInterfaces() { + // Add master + for _, master := range MASTER_INTERFACES { + masterLink, err := netlink.LinkByName(master) + Expect(err).NotTo(HaveOccurred()) + netlink.LinkSetDown(masterLink) + netlink.LinkDel(masterLink) + } +} + +func getDR(fileName string, ignoreErr bool) (*unstructured.Unstructured, dynamic.ResourceInterface) { + bodyBytes, err := ioutil.ReadFile(fileName) + if ignoreErr && err != nil { + return nil, nil + } + Expect(err).NotTo(HaveOccurred()) + obj := &unstructured.Unstructured{} + dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + _, gvk, err := dec.Decode(bodyBytes, nil, obj) + if ignoreErr && err != nil { + return nil, nil + } + Expect(err).NotTo(HaveOccurred()) + return obj, getResourceInterface(gvk, obj.GetNamespace(), ignoreErr) +} + +func getResourceInterface(gvk *schema.GroupVersionKind, ns string, ignoreErr bool) dynamic.ResourceInterface { + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if !ignoreErr { + Expect(err).NotTo(HaveOccurred()) + } + if err != nil { + return nil + } + + var dr dynamic.ResourceInterface + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + dr = dyn.Resource(mapping.Resource).Namespace(ns) + } else { + dr = dyn.Resource(mapping.Resource) + } + return dr +} diff --git a/testing/multi-nicd-coverage.md b/testing/multi-nicd-coverage.md new file mode 100644 index 00000000..0f834e97 --- /dev/null +++ b/testing/multi-nicd-coverage.md @@ -0,0 +1,54 @@ +# Multi-NIC Daemon Unit Test Coverage + +File | Function | Coverage +---|---|--- +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:49: | Expired | 100.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:56: | FindAvailableIndex | 86.7% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:82: | valueToAddr | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:91: | valueToAddrStr | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:96: | addrToValue | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:107: | getIPValue | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:112: | getAddressByIndex | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:123: | Contains | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:127: | getExcludeRanges | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:155: | GenerateAllocateIndexes | 100.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:170: | AllocateIP | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:292: | getPod | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:296: | CleanHangingAllocation | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:324: | DeallocateIP | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/allocator/allocator.go:380: | FlushExpiredHistory | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:59: | handleRequests | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:74: | Join | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:101: | Greet | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:133: | GreetAck | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:148: | GetInterface | 100.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:153: | AddRoute | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:158: | ApplyL3Config | 100.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:163: | DeleteL3Config | 100.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:168: | DeleteRoute | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:173: | SelectNic | 93.3% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:196: | Allocate | 93.3% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:219: | Deallocate | 100.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:235: | InitClient | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:260: | initHostName | 62.5% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/main.go:273: | main | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:37: | ApplyL3Config | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:74: | DeleteL3Config | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:81: | deleteL3Config | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:99: | AddRoute | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:133: | DeleteRoute | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:157: | GetRoutes | 100.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:166: | isRouteExist | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:181: | getRoutesFromRequest | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/router.go:232: | getRouteFromRequest | 0.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:30: | SetRTTablePath | 100.0% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:39: | GetTableID | 66.7% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:59: | DeleteTable | 64.3% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:79: | addRule | 88.9% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:92: | isRuleExist | 87.5% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:106: | getTableIDAndReservedIDs | 82.6% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:142: | addTable | 92.3% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:165: | deleteRule | 85.7% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:176: | deleteRoutes | 54.5% | +| github.com/foundation-model-stack/multi-nic-cni/daemon/router/rt_table.go:194: | getTableLine | 100.0% | +| total: | (statements) | 27.4% | diff --git a/testing/multi-nicd-unittest-report.md b/testing/multi-nicd-unittest-report.md new file mode 100644 index 00000000..45f6a1b1 --- /dev/null +++ b/testing/multi-nicd-unittest-report.md @@ -0,0 +1,24 @@ +# Unit Test + +Test | Description | File +---|---|--- +| Test Allocation | anomaly allocate after some allocations | /usr/local/build/daemon/src/main_test.go | +| Test Allocation | anomaly allocate from begining | /usr/local/build/daemon/src/main_test.go | +| Test Allocation | normaly allocate | /usr/local/build/daemon/src/main_test.go | +| Test Allocator | find next available index with exclude range over consecutive order | /usr/local/build/daemon/src/allocator/allocator_test.go | +| Test Allocator | find next available index with exclude range over non-consecutive and then consecutive order | /usr/local/build/daemon/src/allocator/allocator_test.go | +| Test Allocator | find next available index with exclude range over non-consecutive order | /usr/local/build/daemon/src/allocator/allocator_test.go | +| Test Allocator | find simple next available index | /usr/local/build/daemon/src/allocator/allocator_test.go | +| Test Allocator | force expired | /usr/local/build/daemon/src/allocator/allocator_test.go | +| Test Get Interfaces | get interfaces | /usr/local/build/daemon/src/main_test.go | +| Test L3Config Add/Delete | apply/delete l3config | /usr/local/build/daemon/src/main_test.go | +| Test NIC Select | Get resource map | /usr/local/build/daemon/src/main_test.go | +| Test NIC Select | Init NumaAwareSelector from sysfs | /usr/local/build/daemon/src/main_test.go | +| Test NIC Select | Init NumaAwareSelector from topology file | /usr/local/build/daemon/src/main_test.go | +| Test NIC Select | select all nic | /usr/local/build/daemon/src/main_test.go | +| Test NIC Select | select nic by NumaAwareSelector (topology) | /usr/local/build/daemon/src/main_test.go | +| Test NIC Select | select nic by dev class | /usr/local/build/daemon/src/main_test.go | +| Test NIC Select | select one nic | /usr/local/build/daemon/src/main_test.go | +| Test Path | Set RT path | /usr/local/build/daemon/src/router/router_test.go | +| Test RT Table | Add then delete new table. | /usr/local/build/daemon/src/router/router_test.go | +| Test RT Table | Get local RT | /usr/local/build/daemon/src/router/router_test.go |