From 21f679fedcb6e41b9ac795ad5d5a1d96cb6bf261 Mon Sep 17 00:00:00 2001 From: Dean Roehrich Date: Wed, 23 Jul 2025 15:57:59 -0500 Subject: [PATCH 1/6] Use Go 1.23 Signed-off-by: Dean Roehrich --- Dockerfile | 2 +- go.mod | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index be21ebe..3236d94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.22 AS builder +FROM golang:1.23 AS builder WORKDIR /workspace diff --git a/go.mod b/go.mod index dc4cdf1..fad46ed 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/NearNodeFlash/nnf-ec -go 1.22.0 - -toolchain go1.22.5 +go 1.23.9 require ( github.com/HewlettPackard/structex v1.0.4 From ddc430c630028fb1e2bb849906fd07484dc0256d Mon Sep 17 00:00:00 2001 From: Anthony Floeder Date: Wed, 23 Jul 2025 08:12:33 -0500 Subject: [PATCH 2/6] Add periodic garbage collector for key-value store Signed-off-by: Anthony Floeder --- pkg/manager-nnf/manager.go | 1 + pkg/persistent/storage_api.go | 28 +++++++++++-- pkg/persistent/storage_json.go | 13 +++--- pkg/persistent/storage_local.go | 74 +++++++++++++++++++++++++++++---- 4 files changed, 100 insertions(+), 16 deletions(-) diff --git a/pkg/manager-nnf/manager.go b/pkg/manager-nnf/manager.go index 12f0092..bd49fef 100644 --- a/pkg/manager-nnf/manager.go +++ b/pkg/manager-nnf/manager.go @@ -500,6 +500,7 @@ func (s *StorageService) Initialize(log ec.Logger, ctrl NnfControllerInterface) // Create the key-value storage database { path := "nnf.db" + persistent.SetLogger(log) s.store, err = persistent.Open(path, false) if err != nil { log.Error(err, "Unable to open database", "path", path) diff --git a/pkg/persistent/storage_api.go b/pkg/persistent/storage_api.go index b6c69ed..24812dc 100644 --- a/pkg/persistent/storage_api.go +++ b/pkg/persistent/storage_api.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 Hewlett Packard Enterprise Development LP + * Copyright 2022-2025 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -17,15 +17,37 @@ * limitations under the License. */ +// Package persistent provides persistent storage functionality for key-value operations. +// This package implements durable storage mechanisms with support for various backends. package persistent +import ( + "github.com/NearNodeFlash/nnf-ec/pkg/ec" + "github.com/go-logr/logr" +) + +// Package-level logger that can be configured +var packageLogger ec.Logger = logr.Discard() + +// SetLogger configures the package-level logger for all persistent storage operations +func SetLogger(log ec.Logger) { + packageLogger = log.WithName("persistent") +} + +// GetLogger returns the current package logger +func GetLogger() ec.Logger { + return packageLogger +} + +// StorageProvider is the default storage provider instance for the package var StorageProvider = NewLocalPersistentStorageProvider() +// PersistentStorageProvider provides methods for creating persistent storage interfaces type PersistentStorageProvider interface { NewPersistentStorageInterface(path string, readOnly bool) (PersistentStorageApi, error) } -// Persistent Storage API provides an interface for interacting with persistent storage +// PersistentStorageApi provides an interface for interacting with persistent storage. type PersistentStorageApi interface { View(func(txn PersistentStorageTransactionApi) error) error Update(func(txn PersistentStorageTransactionApi) error) error @@ -34,7 +56,7 @@ type PersistentStorageApi interface { Close() error } -// Persistent Storage Transaction API provides an interface for interacting with persistent storage transactions +// PersistentStorageTransactionApi provides an interface for interacting with persistent storage transactions type PersistentStorageTransactionApi interface { NewIterator(prefix string) PersistentStorageIteratorApi Set(key string, value []byte) error diff --git a/pkg/persistent/storage_json.go b/pkg/persistent/storage_json.go index 1f9e5f8..dc53573 100644 --- a/pkg/persistent/storage_json.go +++ b/pkg/persistent/storage_json.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 Hewlett Packard Enterprise Development LP + * Copyright 2022-2025 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -22,7 +22,6 @@ package persistent import ( "encoding/json" "io/ioutil" - ) func NewJsonFilePersistentStorageProvider(filename string) PersistentStorageProvider { @@ -34,16 +33,22 @@ type jsonFilePersisentStorageProvider struct { } func (p *jsonFilePersisentStorageProvider) NewPersistentStorageInterface(name string, readOnly bool) (PersistentStorageApi, error) { + log := GetLogger() + log.Info("Opening JSON file storage", "file", p.filename, "name", name, "readOnly", readOnly) + content, err := ioutil.ReadFile(p.filename) if err != nil { + log.Error(err, "Failed to read JSON file", "file", p.filename) return nil, err } var payload map[string]map[string]string if err := json.Unmarshal(content, &payload); err != nil { + log.Error(err, "Failed to unmarshal JSON content", "file", p.filename) return nil, err } + log.Info("Successfully opened JSON file storage", "name", name) return &jsonPersistentStorageInterface{data: payload[name]}, nil } @@ -66,7 +71,3 @@ func (*jsonPersistentStorageInterface) Delete(key string) error { func (*jsonPersistentStorageInterface) Close() error { panic("unimplemented") } - - - - diff --git a/pkg/persistent/storage_local.go b/pkg/persistent/storage_local.go index 8a788d3..60676bd 100644 --- a/pkg/persistent/storage_local.go +++ b/pkg/persistent/storage_local.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 Hewlett Packard Enterprise Development LP + * Copyright 2022-2025 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -20,9 +20,13 @@ package persistent import ( + "time" + "github.com/dgraph-io/badger/v3" ) +const garbageCollectPeriod = 24 * time.Hour + func NewLocalPersistentStorageProvider() PersistentStorageProvider { return &localPersistentStorageProvider{} } @@ -40,27 +44,51 @@ type localPersistentStorage struct { } func (s *localPersistentStorage) open(path string, readOnly bool) (err error) { + log := GetLogger().WithValues("path", path, "readOnly", readOnly) + log.Info("BadgerDB: Opening database") + opts := badger.DefaultOptions(path) opts.SyncWrites = true - //opts.ReadOnly = readOnly // Causes ErrLogTruncate opts.BypassLockGuard = readOnly + opts.VerifyValueChecksum = true - // Shrink the in-memory and on-disk size to a more manageable 8 MiB and 16 MiB, respectively; + // Shrink the in-memory and on-disk size to a more manageable 8 MiB and 32 MiB, respectively; // We use very little data and the 64 MiB and 256 MiB defaults will cause OOM issues in kubernetes. - // 8MiB seems to be the lower limit within badger, anything smaller and badger will complain with + // 8MiB seems to be the lower limit within badger, anything smaller and badger complains with // """ // Valuethreshold 1048576 greater than max batch size of 629145. Either reduce opt.ValueThreshold // or increase opt.MaxTableSize. // """ opts.MemTableSize = 8 << 20 - opts.BlockCacheSize = 16 << 20 + opts.BlockCacheSize = 32 << 20 // Increased to 32 MiB for better cache hit ratio s.DB, err = badger.Open(opts) - return err + if err != nil { + log.Error(err, "BadgerDB: Failed to open database") + return err + } + + log.WithValues("mem_table_size", opts.MemTableSize, "block_cache_size", opts.BlockCacheSize).Info("BadgerDB: Database opened successfully") + + // Run garbage collection on existing database during initialization + // Skip GC for read-only databases to avoid potential issues + if !readOnly { + s.RunPeriodicGC(garbageCollectPeriod) + } + + return nil } func (s *localPersistentStorage) Close() error { - return s.DB.Close() + log := GetLogger() + log.Info("BadgerDB: Closing database") + err := s.DB.Close() + if err != nil { + log.Error(err, "BadgerDB: Failed to close database") + } else { + log.Info("BadgerDB: Database closed successfully") + } + return err } func (s *localPersistentStorage) View(fn func(PersistentStorageTransactionApi) error) error { @@ -84,6 +112,38 @@ func (s *localPersistentStorage) Delete(key string) error { return txn.Commit() } +func (s *localPersistentStorage) RunGC() error { + log := GetLogger().WithName("gc") + log.Info("BadgerDB: Starting garbage collection") + + err := s.DB.RunValueLogGC(0.5) + if err != nil { + if err == badger.ErrNoRewrite { + log.Info("BadgerDB: GC completed - no rewrite needed") + return nil + } + log.Error(err, "BadgerDB: GC failed") + return err + } + log.Info("BadgerDB: GC completed successfully") + return nil +} + +func (s *localPersistentStorage) RunPeriodicGC(interval time.Duration) { + log := GetLogger().WithName("periodic-gc").WithValues("interval", interval) + log.Info("BadgerDB: Starting periodic GC") + + ticker := time.NewTicker(interval) + go func() { + defer ticker.Stop() + for range ticker.C { + if err := s.RunGC(); err != nil { + log.Error(err, "BadgerDB: Periodic GC encountered error") + } + } + }() +} + type localPersistentStorageTransaction struct { *badger.Txn } From adc23260f047ab58bccc5d67f387e0394186f0c5 Mon Sep 17 00:00:00 2001 From: Anthony Floeder Date: Thu, 24 Jul 2025 15:50:40 -0500 Subject: [PATCH 3/6] When a replacement Volume is added to a pool, ensure the volume is not used in other pools. Signed-off-by: Anthony Floeder --- pkg/manager-nnf/storage_pool.go | 106 +++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 16 deletions(-) diff --git a/pkg/manager-nnf/storage_pool.go b/pkg/manager-nnf/storage_pool.go index b7d3af0..09e3a62 100644 --- a/pkg/manager-nnf/storage_pool.go +++ b/pkg/manager-nnf/storage_pool.go @@ -49,7 +49,7 @@ type StoragePool struct { missingVolumes []storagePoolPersistentVolumeInfo // Original persistent volume information from KV store - persistentVolumes []storagePoolPersistentVolumeInfo + persistedVolumes []storagePoolPersistentVolumeInfo storageGroupIds []string fileSystemId string @@ -123,17 +123,34 @@ func (p *StoragePool) findStorageGroupByEndpoint(endpoint *Endpoint) *StorageGro return nil } +// recoverVolumes attempts to restore the state of volumes in the storage pool based on +// the provided slice of persisted volume information. It updates the pool's internal +// lists of persisted, missing, and providing volumes by: +// - Skipping and marking as missing any volumes with an invalid namespace ID. +// - Attempting to locate the corresponding storage device and namespace for each volume. +// - Marking volumes as missing if the storage device or namespace cannot be found. +// - Adding successfully located volumes to the providingVolumes list. +// +// Finally, it updates the allocatedVolume field to reflect the current pool capacity. func (p *StoragePool) recoverVolumes(volumes []storagePoolPersistentVolumeInfo) error { log := p.storageService.log.WithValues(storagePoolIdKey, p.id) log.Info("recover volumes") - // Store the persistent volumes information for later use - p.persistentVolumes = make([]storagePoolPersistentVolumeInfo, len(volumes)) - copy(p.persistentVolumes, volumes) + // Store the persisted volumes information for later use + p.persistedVolumes = make([]storagePoolPersistentVolumeInfo, len(volumes)) + copy(p.persistedVolumes, volumes) for _, volumeInfo := range volumes { log := log.WithValues("serialNumber", volumeInfo.SerialNumber, "namespaceId", volumeInfo.NamespaceID) + // Consider volumes that have been invalidated (marked with invalid namespace ID) as missing. + // They need a replacement. + if volumeInfo.NamespaceID == invalidNamespaceID { + log.V(2).Info("skipping invalidated volume during recovery") + p.missingVolumes = append(p.missingVolumes, volumeInfo) + continue + } + // Locate the NVMe Storage device by Serial Number storage := p.storageService.findStorage(volumeInfo.SerialNumber) if storage == nil { @@ -165,16 +182,31 @@ func (p *StoragePool) recoverVolumes(volumes []storagePoolPersistentVolumeInfo) return nil } -// Rescan namespaces associated to set the missing volumes list +// checkVolumes validates the current state of volumes in the storage pool by rescanning +// associated storage devices and updating the pool's volume tracking lists. +// +// This method performs the following operations: +// 1. Rescans all storage devices that should contain volumes for this pool to refresh +// their namespace information and detect any changes in device state +// 2. Clears and rebuilds both the missingVolumes and providingVolumes lists to ensure +// they accurately reflect the current state +// 3. Attempts to recover volumes from persistent storage information, identifying +// which volumes are still available and which are missing or invalid +// 4. Updates the pool's internal state to reflect any volumes that may have become +// unavailable due to device failures, disconnections, or other issues +// +// The method is typically called during pool initialization, recovery operations, +// or when storage device states may have changed. It ensures the pool maintains +// an accurate view of its available storage resources. func (p *StoragePool) checkVolumes() error { log := p.storageService.log.WithValues(storagePoolIdKey, p.id) log.Info("check volumes") // Rescan the storages to ensure our namespace information is up to date - volumes := make([]storagePoolPersistentVolumeInfo, len(p.persistentVolumes)) - copy(volumes, p.persistentVolumes) + volumesInPool := make([]storagePoolPersistentVolumeInfo, len(p.persistedVolumes)) + copy(volumesInPool, p.persistedVolumes) - for _, pv := range volumes { + for _, pv := range volumesInPool { log := log.WithValues("serialNumber", pv.SerialNumber, "namespaceId", pv.NamespaceID) storage := p.storageService.findStorage(pv.SerialNumber) if storage == nil { @@ -187,7 +219,7 @@ func (p *StoragePool) checkVolumes() error { p.missingVolumes = p.missingVolumes[:0] // Clear the missing volumes list p.providingVolumes = p.providingVolumes[:0] // Clear the providing volumes list - if err := p.recoverVolumes(volumes); err != nil { + if err := p.recoverVolumes(volumesInPool); err != nil { log.Error(err, "Failed to recover volumes") return err } @@ -268,11 +300,11 @@ func (p *StoragePool) replaceMissingVolumes() error { pv.Storage.SerialNumber(), pv.VolumeId), p) - // TODO: Find the serialnumber/volumeid in any other storage pools and - // invalidate that volumeid to prevent reuse. Should probably store - // that new value some how too. - // OR - // Patch all the storage pools and don't allow a particular storeage pool to be patched in isolation + // Invalidate this volume in any other storage pools to prevent reuse + if err := p.invalidateVolumeInOtherPools(pv.Storage.SerialNumber(), pv.Storage.FindVolume(pv.VolumeId).GetNamespaceId()); err != nil { + log.Error(err, "Failed to invalidate volume in other storage pools", "serialNumber", pv.Storage.SerialNumber(), "namespaceId", pv.Storage.FindVolume(pv.VolumeId).GetNamespaceId()) + // This is not a fatal error, continue with the replacement + } } // We've replaced all the missing volumes, so clear the list @@ -281,6 +313,45 @@ func (p *StoragePool) replaceMissingVolumes() error { return nil } +// invalidateVolumeInOtherPools finds and invalidates the specified volume in any other storage pools +// to prevent reuse of the same volume in multiple pools. It replaces the namespace ID with a +// nonsense value (0xFFFFFFFF) in the persistent volume information of other pools. +func (p *StoragePool) invalidateVolumeInOtherPools(serialNumber string, namespaceID nvme2.NamespaceIdentifier) error { + log := p.storageService.log.WithValues(storagePoolIdKey, p.id, "targetSerialNumber", serialNumber, "targetNamespaceId", namespaceID) + log.V(2).Info("invalidating volume in other storage pools") + + invalidatedPools := 0 + + // Check all other storage pools in the service + for i := range p.storageService.pools { + otherPool := &p.storageService.pools[i] + if otherPool.id == p.id { + continue // Skip self + } + + poolModified := false + + // Check persistent volumes + for j := range otherPool.persistedVolumes { + persistentVol := &otherPool.persistedVolumes[j] + if persistentVol.SerialNumber == serialNumber && persistentVol.NamespaceID == namespaceID { + log.Info("invalidating persistent volume in storage pool", "otherPoolId", otherPool.id, "originalNamespaceId", persistentVol.NamespaceID) + + // Replace with a nonsense namespace ID to prevent reuse + persistentVol.NamespaceID = invalidNamespaceID + poolModified = true + } + } + + if poolModified { + invalidatedPools++ + } + } + + log.V(2).Info("volume invalidation complete", "invalidatedPools", invalidatedPools) + return nil +} + // Locate Storages not providing a volume for this pool func (p *StoragePool) locateUnusedStorage() []*nvme.Storage { log := p.storageService.log.WithValues(storagePoolIdKey, p.id) @@ -356,6 +427,9 @@ func (p *StoragePool) deallocateVolumes() error { const storagePoolRegistryPrefix = "SP" +// Invalid namespace ID used to mark volumes as unusable +const invalidNamespaceID = nvme2.NamespaceIdentifier(0xFFFFFFFF) + const ( // Allocation Log Entry is recorded after the storage pool successfully allocates the backing storage resources (i.e. NVMe Namespaces) storagePoolStorageCreateStartLogEntryType uint32 = iota @@ -430,8 +504,8 @@ func (p *StoragePool) GenerateStateData(state uint32) ([]byte, error) { } // Store the persistent volumes information for later use - p.persistentVolumes = make([]storagePoolPersistentVolumeInfo, len(entry.Volumes)) - copy(p.persistentVolumes, entry.Volumes) + p.persistedVolumes = make([]storagePoolPersistentVolumeInfo, len(entry.Volumes)) + copy(p.persistedVolumes, entry.Volumes) return json.Marshal(entry) } From 3ce827d12a6b15d8082cf23418c144b61e0159f5 Mon Sep 17 00:00:00 2001 From: Anthony Floeder Date: Wed, 30 Jul 2025 11:03:10 -0500 Subject: [PATCH 4/6] Tune up dockerfile removing layers introduced by COPY Remove unused cruft from the repo Signed-off-by: Anthony Floeder --- .dockerignore | 6 ++ Dockerfile | 11 +--- Makefile | 32 ++++++++++- kubernetes/nnf-ec/Chart.yaml | 5 -- .../nnf-ec/charts/cray-service-2.0.0.tgz | Bin 11613 -> 0 bytes kubernetes/nnf-ec/templates/NOTES.txt | 0 kubernetes/nnf-ec/templates/_helpers.tpl | 45 --------------- kubernetes/nnf-ec/templates/role.yaml | 32 ----------- kubernetes/nnf-ec/templates/role_binding.yaml | 17 ------ .../nnf-ec/templates/service_account.yaml | 10 ---- kubernetes/nnf-ec/values.yaml | 53 ------------------ 11 files changed, 40 insertions(+), 171 deletions(-) create mode 100644 .dockerignore delete mode 100644 kubernetes/nnf-ec/Chart.yaml delete mode 100644 kubernetes/nnf-ec/charts/cray-service-2.0.0.tgz delete mode 100644 kubernetes/nnf-ec/templates/NOTES.txt delete mode 100644 kubernetes/nnf-ec/templates/_helpers.tpl delete mode 100644 kubernetes/nnf-ec/templates/role.yaml delete mode 100644 kubernetes/nnf-ec/templates/role_binding.yaml delete mode 100644 kubernetes/nnf-ec/templates/service_account.yaml delete mode 100644 kubernetes/nnf-ec/values.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a12dbe3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore all files which are not go type or shell scripts +!**/*.go +!**/*.mod +!**/*.sum +!**/*.sh \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3236d94..bfdf9c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,14 +19,9 @@ FROM golang:1.23 AS builder WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum - -# Copy the Go source tree -COPY cmd/ cmd/ -COPY internal/ internal/ -COPY pkg/ pkg/ +# Copy all source files, avoid producing multiple layers +# See .dockerignore for exclusions +COPY . . # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o nnf-ec ./cmd/nnf_ec.go diff --git a/Makefile b/Makefile index 30d6fcc..6b56532 100644 --- a/Makefile +++ b/Makefile @@ -62,9 +62,15 @@ codestyle: clean-lint: docker rmi $(DTR_IMGPATH)-lint:$(PROD_VERSION) || true + docker container rm lint || true clean-codestyle: docker rmi $(DTR_IMGPATH)-codestyle:$(PROD_VERSION) || true + docker container rm codestyle || true + +clean-container-unit-test: + docker rmi $(DTR_IMGPATH)-container-unit-test:$(PROD_VERSION) || true + docker container rm container-unit-test || true # push: # docker push $(DTR_IMGPATH):$(PROD_VERSION) @@ -75,8 +81,32 @@ kind-push: image clean: clean-db docker container prune --force docker image prune --force - docker rmi $(DTR_IMGPATH):$(PROD_VERSION) + docker rmi $(DTR_IMGPATH):$(PROD_VERSION) || true go clean all +# Clean all Docker images related to this project +clean-images: + docker rmi $(DTR_IMGPATH):$(PROD_VERSION) || true + docker rmi $(DTR_IMGPATH)-container-unit-test:$(PROD_VERSION) || true + docker rmi $(DTR_IMGPATH)-lint:$(PROD_VERSION) || true + docker rmi $(DTR_IMGPATH)-codestyle:$(PROD_VERSION) || true + +# Aggressive Docker cleanup - removes all unused images, containers, networks, and build cache +clean-docker-all: + docker system prune -a --volumes --force + +# Clean only unused Docker images (keeps tagged images) +clean-docker-images: + docker image prune --force + +# Clean Docker build cache +clean-docker-cache: + docker builder prune --force + +# Clean everything Docker related for this project +clean-project-docker: clean-images + docker container prune --force + docker image prune --force + clean-db: find . -name "*.db" -type d -exec rm -rf {} + diff --git a/kubernetes/nnf-ec/Chart.yaml b/kubernetes/nnf-ec/Chart.yaml deleted file mode 100644 index f9ad5ae..0000000 --- a/kubernetes/nnf-ec/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: A Helm chart for Kubernetes -name: nnf-ec -version: 1.0.0 diff --git a/kubernetes/nnf-ec/charts/cray-service-2.0.0.tgz b/kubernetes/nnf-ec/charts/cray-service-2.0.0.tgz deleted file mode 100644 index 0a69ccf52a55448e94861de3b88c892c7c4f895f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11613 zcmV-jEuzvNiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMbK5wwFub4nEBeqmm1JIvy4a30n#%he$M$BuGqJt4GgDhr zQw~Hz5@L#A08qBZ@$=iC!i@weQj#xmCVR{eiA4eppaJv^jqZ@4S&L&fB_VF_ju8{V z3{B#{J>=8rbUNExTk8K#r&ImE)7#$oTX$o-_o}zi+3aoqt<&9j)#>~VIuF{5h9~0! zvA=ch-BxvQ-^qhg@+)SX(4-I3t`}h*GLj1Q`)o`&gmO&)AJZ(3Ai-12V1R)nT%b6{ z5k!RHP{gx!7$A;eMl%LH#UUAz5Ox`wc>rd8X;>WSa0o)erU~}Obb|X3#x#rCYBB0U zUV_wnXL)*4+tFUo2|C`lM%{nJQ`-Mi6la(}I00C^|GV4UTb;`O-|TF@dfNYw@w8ek zZw>bF5M{9dy#QcHS!w45-kK%`gad+q@4Ww|HKc5UguoFDNvsO&;V?!F!xS-s1~KL! zs3a9B;v7dn5 z#g}NtC0vdL&;-Ufo>+3ENKho-+7}^B1iljM8jk8d!-&&F_7EaHY~73FERIic$gtpj z`27!$3(C+4@5YGpKKQaTBoR1?aAZctv1rpNW{gBQ0VMKP1)1P=pbQc!R6mgb^`1h; zF^F-|w8ZSu-ccWhBq4kZ!z_-0XK6~AQ1fsqkif$%K9yjS2=)-3(Bu>gPs~!>hdrFe zbT+|>fHmjG7pLBrqK>4hN2beWP_j4jHPE4<-iF-DuGEy*Baqls<>3+~LXIqDbV?!& z2qu)_R!ApljD>-%RupCekpyBiz%g$pXhM<^yqID(1FBdKieGvRr!fi9F3l3rhpv|* zCW!oa5XlaBYY=8!&lP@P&pa=;h*AT;;W-OHJR4#wau!4qSPJxOsW*pEIkytGyBG+6*rXsd;kyr+SkkTs9L}iV#@z2X2K+BE}>E1W~@a*CiIz^5$5q zL!{QX`V)&V@KPG>%=IJ3+t1kmvjhvwA)ygI#W4;AW$PeljG3;puF*P_yHd=64nE;f zXdK8Mdu~Mq->IvphpnnUnjN(hhH_OOnsq7($`DkqC7r;NRLs^jm6J3>u3;-UAzjlF z5ZMh)L@Ywu3-RH(lmR}ZBZeXjr-C7YM{>Cw$vI&-##6bO$q?pf3*oX;4j)k6%TfCJ z)G2HM8?M4bYHjEiqUe@ zKx^75eW9@J&h^D>4ssrLX(DM4X1w|}v|l6vOED8_k7)voqy7VQz0+RH8<9N@ zDZ^g5Q)Y=ya*o@Q-&=XLKCA&^10on=Gq}JrMOD2u0C1=tfMR5Q^(O$sDdj{^HZ!XY z2ctmo^>)#aDJ#&Z4~i)Z^Cr#W_?X5dob};gcu2)D!yG62a}WgTe?%8nc@UB;tM8 zvIE16&?tYsp$%aC#@PgO$-LDFCG$%$vt-nKHtSjcTDpWM zvsOfyUQCj~>=8CcB%LmB9A=D&*{+tGExiv>A_d-zOBPcV{N(r;jTJ}uaIn{Ycd!Rp z5@80HV-k+Fcu`=QWr-xhl)(fgXoTeuCxmk~T*=uclGWkJqQhH*Q;Z*m4yQD-BQ%;p zJ|}C{vFp{yvSf#U;F1J)JDpDP?j569+J~NfIV`=gEWhu`O)MqEs1Jf=*aO2`cjJr; z%npt{fCP(6$}aj!9%*T`7>T~}=F1qd=n}E=J2D!#c#3iKMa#yRwNMyh&OHD=Mhr&> zX&;6t=Cb`Y_>hEjqS^WoGA;$WUnq*+q8KG1W~zru0J1`(76xZP#B&a+%WJVy;ow+t zAPKsDQE4eg=`?C_tF`UrG)+^qe3R=v?+~~;A_j(@vJn=V-kG9~FbYr=M?EaU$e_M7 zHx@ZOK`9?oVf7EG%S**dipN7nCswkS+LHR%fmRDJ;zVxL1SWVw*-SwmQWoN!M39|a z7L+Rh*5L5yY+q9QLco;bl@_>hR4A)tlu(8vcp<02=!63lUR+89O9cxFG9WP#vzH#g z1YP|rqXL=1K%v%umaiqi-n+N5yBrHCx67GTJ>pn!Q%LGS=n7tRJMZ2~pq%`PB@qS4 zt#AHDuo3inzMSr2D3T(CW0Z`vcDN2W;ThA)uBocZNEJ$zGaZn_Otiul)4{U&1DKP% z0ZOtqz?xc!yENh1L@H_2SR1uEB}ti8+NE?HL8zC5QdF$s1~His!2wB<6vy;Ztx^p+ zSMJm+#Y<&b`wJjUvp)DeKYyVYMIU6 zOqnPmDmM&DW>Og*DWzC1>zP#Z23c;SA7E`N2$fwz_Q&)y6Jv;QNVp-sYj!Zf2nl?t zEcJ%pm(2YPs}2mzzQqw_Ni1<7xAR3pFX3{Gg#0Btt@;^w00w9ud_(_v%e?YSji(F` z$yL#|(JdLx1O{s}v@Ar78O=sxRl-2iWLdtTncyVC0UQXsWYx^KQA&9G+QLuCk zF3|}em8J8eq8zY)g(hi?`{pf~@Q-k*aAE1bpzu)xR3FI@^Z?+^8_fgt&zvFJVKO|z zdXxu7>EAirQ{6W8wULwX|cIkxX0>Q6>pssHnMv7WI?p ziB`xAU`J81+*Vpz*i{_Rbz+)(zY?FA;61EWtF?2$w*h0)BUn`{#9c;5TZCP7L47@ zg2HuqIo`B~l(q+mnb|L|#-ewK7d10R@@nP9w8JGUs;qdqVDg2u?jpv1S%-J~XKHl) zjn3xgMqau-?aGFr_5TBz@bg0D)}pTbCdmpT2JMSQBK|pEBoQg2b`oME5p5{3jSJb^ z{N&K@c4a%7&ikeEL5+aWHT9|z!d*t2B8DbdDBE_;;EK<-V_Lm9A7ql{aVZ@3 z-oLeDo1iPXI1{;h)pGN@9s4b&qjP1nlzK=Xd=$rudKN9H#pwygf1~W8VR2rnPiIP_ z@}&ac=P!%JIqmwMcY%f%Xwf=fTU&#_QwCV?T7@sg*d(S#)JmxuCz2R(Gc1jn7)!Fm zF)jd*q_7~2h!|&sAf%Hv7mQ5)B=-N7>RF4XQgfxOEf~h_36lM6TTpFP#sj+=PqhGV zc7t;+-V3^(u}Mh@fb0DVO6I~zz83+~7|HIbg`yOxoIWT!(vXa7Nje-#{U8uDC1GFy z%Yg?&q=oPe3xlDABZZVeO>Gd<(H&|`ND>gv`p_$htG14s>`wz4nHSbBVt%%OM18PY zq%EIH97!CARv05P$v<<4TInakF&21NmMT`qpOju;oK{g^#dB?68``e6T+^;sRQ*+r zQhm)&RtmT)T7%|lsBeY7;Mf;SdV{gux%9>JMll_rSPm7XX|daUhpU^j&p}4wNS%rX zYO{zj=K3(r95@RKD2fy>=3~l)mTb6IO4Ru-SR;?zd4YCR`oUHw3Gi%SJV)9oQnI)H zrRh}7E|hTfKZ;B6_U1-+bLGMtKlbDbQ*G$Ub}Ytb)LMg4!B9>+N^)=u`w%>2JudhwhH#gVU z5EKpR5?=LBsehB9mF+qBhcv>+lnFoYxpjw8_E7~7Un!&9;nCjy`SH=o*_&rCF0ug* zMGPY>pry{4c#1*{vJ)p*i3!1B{`TeuS`onkqA6d-IGi^f5p&xPsq=1RkdC`T`l!y3IDfj{x%5KTPeQ<27Ot}L`kh6aP z_&3dz)OV>AZFABsGR2RaUGdySCZC4Tx?FW`o)>d-x_|QP!EVj;D2S^}&Wdf8Ch_^T zojcc%Z}*d&Mv~qEVe*a@4xQ2v~3CfZ{YvuT4!!K#)3ekck~Kf=KsC1 zRq_AssaH?@{}G<+>-G=c4(RFDK9xhLc*^C@T8>BD4{fv0)p=fZC(W-OA6hpzp7QJ@ zv~5pEn@Lx(7rpCiZSA&1!|h#5e;4%MXc9wUe(S|w{rg*^`Y-t`+5fp|@$S2S>Hgn- z)$3IEf9Fa5f1Ibh{|m#mltPv4BsmdDU_iy#IYLvXUKPi>!i6=$JSFxA(V|&Q=>;GT zVobBqcs2Twx93YbV11q>1J0XOghGoB+9?bCMw+4uw6yizhv0-#VQ7lc5elfPXi#`| zNtL3mgg_(slAjH|sBaggY0IedrdfC?r3TyQW5)%ou93n;%xTMaoyGh@yCFlM3|_dS z3mm5htE0WF6e$h?no76^RJTmgTbxW^8YwQBYO9D<6qoeIO7nJ#_s@3s&i`?AdS*?0 zvgV?xHS9v+_U5J~n`nhGk*JvmtQ3zj;NHgekAB``BRVw=U>;Z1{vRKmp1nKSKfTWw ztWDmphg>s=t<8@`+6`{i`L_{PG)m)5Bg2<%rYJ(Np)frKOxP6TY0N?UQF?$=y9e`j=(ZwdLMzqnOZ ze!k^pEg8#q;>;bDAuX@Ecw_d5eV)K$AQ0BrZImf7PO9Oda8zy+SwVJE2VYR-#VX0*4)js?&*rr1VG z{Qz8RRpw;QQd=Qvg_wz3k4g2VZJ;b-JqxzpLM~J)d4G0PTuL)U$8l6>rZ=|EU8w93 zhqExwDXjUqn7Z0Jw{NfFzpY!HbLwtxyITJaJrEH|OuO>WsyOd_UkTJv5@H9O{82_8;vzb|2%Dnv{A+73tLWBgKECPC9U0Y^ z*Di!nYcMONC})(N+y|x}VD8VM=tH%NLmNux(q+rHnV028Z})KCxjpC$BPU(df6^-*e-tvxwO4*| zo24LV%CV96OA#5IpaV`Yf)&|?Y}C)G%rhocG^yF=uN;DAV|^&*P{jRkh?rBtulo5o zfab$8$U?Qj8s?oH%C-EG&x10+FEp$#GR!yi;S(M3mU2mv>SkqF&&r1}zu;sf#u9}w zP8`oGw~UKh+2fIDfYN!XO+?M+2lsQGd4GxF=;o#x51~3$O?to8`Ta{oltY9zIs3Oi z4pM8bI1Y-*fFU+(4>CR*(5rqo=)Dd)tx-${q3;-3L2e*(Yl}?MSl`>#F>rv>bWFG) zArd$O*~N7^s04>nb7C`4dqdkU+5?ib`Plqwh31be5hR9$$CO@V=_;;1dh^U1-+zLL zYUdN#5t5A5@t!JV-O7Ig9%GzBPnJ#aPsUm0`Om7*2}aTE`A>^BRD)s4EYLJln>r$) z0RNwKaH#tGBB9!A=S7GbkI*W^dkKCkpYi;lF~U#pfljcqpYtxV%h9B7;SzP<1xpoSFyT$@Mtq^%*!N+@YQ;SD3(20OZ2qBhHki2 zN8`oqyuVxOirdC&<8p^mZLvXim-2sw%2oy&)(td#m_deLXMi_1o~OvLwcy#XZ;;C# zC9XebCaf_MI{H;Zy~1kLtL==9=8^Z|nR!#sn{DwLrd+PKoEtA>n$0IHiVr)NX)1VY zI?$;(k7+)Nx#K~MHo4ndHTu`#?E$S~FJFO`nG=>(4p;7Yw6bu#^?-ughz7RNAY@oB zVS-}rRgN#_eykvlt$qm8bO5t(Xg2GfjUAnZ}n;apnjAHk1A3!BDn!HB)n2U4X4KYmi-Mx0Afs<>Y+@57rn@O;-q_I+Mc%U*A>Wg@@y zalkOTaNZ1wSFQP*W1J9UeZ$AWk6eZ3CVr0G;yRED^P^i{hcILMBCy375Dt$sPo+r# zV^!J6-(VXu$sPFfYR8_#5D_U@Dei@@R5Z;9*UX`b8q2U?vqkOZ=#FTD=UqVDpc7z2 zPHc4SeRUge!tcE&zKJSeLtEstCXw>q-I_su8&Q=caOKgMy2*?mv`^|OTkZ*{ge ztLH!6?d>Q2{}_)pRgg(vo&66b zBDIYZtpbJw$5G&YQ1P{#$d0~|79>NtkTjIj2?kD?`f5Wk@^TEuWjH8Z1F51SZ_(mG z_dh_ckHDRgVOtqyaXf?18H!b;Nr?(Imx1>i*3eaXA={Ai%3***lyR&P;6_eYQT-+Q zxQ?+eS%D5ep@Iy(7^vv>uy=S0CeptYDqjZP!B9mSN8Bu<{DQEKOQhZrp*fn4IUDJ? zUn(_%?(Jek7Y2`-)5B_3gS<>*ftBzc^8(T{HHQA1)z`Vx&m0{ZO4(HG`>WQr3al>-wgQAHQB5fmp_pg11nA&F8- zV;~8aJC!mUW)cwO&QeRTX#lniz?KA9OPs2{T6OEH5<0=ejxG@E!uSIA_{}2e^~;4M zp$YnuZN40*14>0JSF?1~I5;@I7g|d&TN-AIq<6)vUxMd1VRR)tR$(p2-#eAxM;kv( z$jv3gFPlN!F&9~tYR203YYWtlVsXd6we0%z-2Z9P|56$~Oz@8t;(xsAR`Wk@zuJ7d z|NkhDqCLlz6iv0>f6C==2}!D~8cjZOnZi0AXM>o8wN7=;S)yNcV=m=%vVP8`gPr;! zi(`WnRBxGIk1S^bxF}=j>CHVPL(7V}97>m-B zx2Ik2f+SJj$xLXSG5Q`Xa|bAgC@W%B>qG{%rufyO2HDGB3muH?EtQ~j2kfRACVMqt zCBiock*8QCY~apOJLpjEzc@p_|Dfw2EWZ5$c}h;xDQEhWnZMM(hv?y5`lxDECuQ-( z%&{YPR$k}MMP_DJnq=*$?!tLh@&5YOJNkA{6aCK}`yYS-EIa?-n5X~SPxSwBo)zeS zz7&=+fV|`r2Ux%XikRV!0PLo{t@Et6DK*V)aFg!RbZgF4)s~(cQ|`&zoq2{s zWo69WFxTJ^W2q|Sk)?srI3~&HLmEl;0ZC@k9Ge`YrIcAGZdn5`?~i|Eer0i0qs z#mt^*f5M?KW;r&(k#pxrA6n0Mpc;yy?$7|DOgk0#VY}BEEB>6|%eQz)8SX=GQ@)~Xgc9ixyU`T3+H_5MLGs<5BK^n(p6Rqoz0T%q{d*@KQAWgg(g((oez%{5G$P5U z59vk7yD9+_8sR*}CMTmrXXui@F7OO+5|VU`SwtiXvL? zx7!xwP4J*g=&!1vER_Gv*zRiqTrB^0ds|yo``>nV7x(-%YUBZKNki(YqwR#sf)W3z_d4PIFsD6OhexVnWK zRa>-8o`hRm+skk4=@r_x`!=N}bh!z=b8yS+fGdaasWoTLd3KQG6AmS2l|bSzU+vZV z)k=1#+_45|yk7jO8V$J`3A%2BIXADD@5V}D_p0_?`=#7ymiH4UL&`$DlL)f2q!-GU zrzlqJMTS3TgyCpUr7AxS$2iJjl8g>U3C&;cU*Rw-L{=@`DI!~)1+(Fi2*>+ZDZ`wX zjoZ1r*jIG*+z`*w_hK9N;UKAe(kV*&@cZh_>kkKnpec>%X!f)0L79cwC};C5p)r(p z&77puuo_6aHtFnX)k^OkPN}4~yD{RtI4f}9mdHqb@k);?S!wM=G*BZsh@w0=;oD0W z9HHit;}Y~t(AB>(Dv*12Ql$BXZucEgiRA}G=xntpVw6q*4d@g@xAX3;itLGCg03np zx|^l~cEK=BnR3ND&weE)_!81FS9%V=cOo7|Pi15KHg`-ZXUooqv!nCfqxZ)0z_}7P{ zvz_y|JG(#cAMTyM{rA~^HGN69v%Rss*?ryH?D(*z7tV_bx&of1DP;mjy6DR)+U6v4 z%bE9!C;A#)s4-d{TW9 z#p&*ZMz{}q2d6u4f7$;lOzzXHE><(|Go1q|b`<_I|D=Qa*b7VH0ZWFq6FeokT7A!{ zT-0ce$Ho2(wO#}(3Er-&0K;4$CT!xgLp;sJGB?DXg`dY8QP|1Ds1`i*wqZx=^q2m0 zBiIOfJr`~%9h2>g8*{|IGm%G@iX?MdYym3JS{;Ax)h|s0auI*oiG>|L*(1bNuzBlq}d`y|J^$H-h7yB!| z#fwB;BQX6+n8;9Ua&k1tPDQH^ApJa`dvT4fJ$DRHt4wb$Ql`t14$W-d3+^0L*PP!h zjW;(?W7?N;x8rOer)h))zC^S9t)r*gVVNQts^avftTN}hqF=^HndvqxdoA3RITo1P z*kxP6Tn&E?zRdvK; z4J<%^#!4NRvsQ}{UC#Gzj|&a5T~l?w5K2jUnul z7TRZ(W;qi2J{~zPI zULmk^_12GG99%Fm`I9(o+1fEwx`=yNG9KDSVWVAt4VGO(*xiruEhALm`YSYLG}2Qn zq^4S%NOTcvH$TgE8t3d4YRop!WV(S>%{KrI2{{c(&5|AdflE@>?Q}X_=hKkCV>CmDf{59XJ*d>fp(zRTS8%NdnQ_eKUVg75{{|NH3iYX^ z;dvN?`r8t!e|>#>TT_Gl zXIbXw_^aRl?smJ|TXXlnH=pvqKFU*d<(N-D7YDc3OP1pwSfsg(M38L@`Q&NbWPs za|`NzA7-29-TRKzH3zTvSJnhZlY{7XdaXX3GET=;hi0dDy4-2*)vRqiN5cbiOP@DY zwAc9aND_-uTU8vo7NM(BwBq@-n#Ff650X|kc!LTzms#sh{_)lmeuY-@qIYRYxh~s% zrQ51=qIR0I#ebcTe%tN(ZccEhak_NiAciJbC_*N7QG{SOG_JMeE z$=;b~He82i>ikK&3A`YQSh{guT6(GprafVDsq3`+VB*R3wd|oJ5cN^Q+wU!T;n`Jl_)sF)@-U@CEw9U-Vwek<%q6tqySes z@-efOBj326sg0l4sLSej4+phMLhBHPW1Q>t{cb1NsPj^T-+eyz*THXv@?T9dQ~o*5 znOnczes+9xdiL&Q|MdLB>HbN-_<6i@divYZ$=?4Iue8%uQR3+2%&EWk7Fr=Z*W}Lf z%NH+ydS0Kc+V%mW0k}Y#5?AL%jB{4$YDL8p%g$11Y$eO3@`3t0qm$|ZoaSCX<5@1z zyHCDh^lJKA$vexBy0IeL%WsG@jM7fo_dU`Saehfzv}mMrzIC_otefkSzYwQY|2f-| z2b?@9T@xt!#y1ZQgj9G&#bzW$zE z;{I1XP4?eu_(0BotHgh<=Kt(&ZFQgWe?7|6VA-uCg-@rUn+&=3f*`Lo_T~cYab2DJ z3jC5TrqfWTo>^UZU3S4^5&T=2q%@d^zPwTk)r!wX`7R5pbN4OXH-*lilh#NAMt7cZVSyf6e>C%$4nZ@PM zcchy3`!C1&8tA`H-EUIT-W~{y)l7wbrS5w~5g!++d1gmkX53 zx>@Z}97v+?yu+WJ*Z!$f)3?SCH~Bq^VZ|JC*WWsU#&`ZS?tjgI9_Ri?w=G=)X5)pSO_gA^8ZA&%)~k-KkEFB;SRl{->*F4cHL z#=?G}5LI#l%BOwq%((LDinFJx;46Vyg5tB1FBt6Rp=jy^;JH=oz&$uNo3He)w&Qix zO3me`AS>R1Xqkhg`pXKkPx0R$<+)y(|HWAf%M!qx znxaoRz?S6zQ``rsuus@}FQl7*Dv)=Bfk!mM;IDot$b2_qqURcZ^#z`^)F($ki7BGzBcm@f^5eK3D6O?m;7-M~-O1;*Z z`QsUk(Gf<>n%r$kG7LWHW@r0EZ~!7)Syk;#h%IH!7{iHQ^A_ zWJpFC!;xxA>QGM81DrR7R2JVEAci61IP%u?a4u;UN1z)*3S3M=A={MA*>IL{IVR!Q z6dqt0peT3@CnbFgMi1!G9hKb1|0)5k*7WKyT?Z6~irveVU?$qk+}X)#F?&uDM|(7( z0qp4d=IEz{?Mh&*g^{#sQT8AqwPHFHe1ej8M8gZrT5@7T5ibd=RUcSuF0__>aXBV2 zhTp;eXh1_N$uEb&2n&4^R}U$uUU-&7uyvb^2aBoJq9O&2YO!z%EefAFO;RMrZ=PQV zQZ~LF<+XxJRXCMqhEzytrMC%>XQJtg@+7~^u)O@FGiLNMg*r9|C@vdOrjMbm9FFh)L@G@PEBE_!TiiGY$uP``imsMx`j{ z1th|#t;;d`$qS}Dq!Dg=0&~IJ-vb>g?f>0>>>R$^|K;diFo_nFX1aCuN-A!K(iz|RtHVc+t~hb zy|?kY^U+befN&20MUf;Yl6Yy&UBzxiN#rVMQWZ{~3!qcX7>OdB1aL5{mheCoxMV~K zoG2Iy7~x8d)&q9hA0w-I>rM=2N*&8MW>WV$7|zv@zhWi4$dXLy`AUc5Sf8mW3xf+Z z$Whq|I<=@Ya^oB(CA~)TJZycMzWpD{1{D8#Vk-9pft6z;*j7t@)W4|WWD2p(YRVL~!&X=Mw z?X0W&w`#WXq#&|PL>20XG>+*dNk)CI1v)!8%xK2s%9hwQRpc7h4sNopiW)5cu<32o zh=C3jNacO6s(URs!4o>w)|roLxPd^Uk2>MGYG;B-;^@yN5r}HaYr%fPGll^rGda0R z%15zVoFp2IQw_uz3E*R8j!CG{%vBewB2=55s=ZUdAf;&*BY`8YLjR9`JKR4xeQ+AH zO#bV3tMtG3s`KRk_b88CrH+NC1#E!-j7W$t{>NH))oT>fDUM^o%CA$E&?kTQtw;0p bJUvg()ARJK`ux8D00960BjtjY00sd7o;B+S diff --git a/kubernetes/nnf-ec/templates/NOTES.txt b/kubernetes/nnf-ec/templates/NOTES.txt deleted file mode 100644 index e69de29..0000000 diff --git a/kubernetes/nnf-ec/templates/_helpers.tpl b/kubernetes/nnf-ec/templates/_helpers.tpl deleted file mode 100644 index 3b2fe98..0000000 --- a/kubernetes/nnf-ec/templates/_helpers.tpl +++ /dev/null @@ -1,45 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "csds-dpapi.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "csds-dpapi.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "csds-dpapi.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "csds-api.labels" -}} -app.kubernetes.io/name: {{ include "csds-dpapi.name" . }} -helm.sh/chart: {{ include "csds-dpapi.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} diff --git a/kubernetes/nnf-ec/templates/role.yaml b/kubernetes/nnf-ec/templates/role.yaml deleted file mode 100644 index be81d4e..0000000 --- a/kubernetes/nnf-ec/templates/role.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "csds-dpapi.fullname" . }} - labels: - app.kubernetes.io/name: {{ include "cray-service.name" . }} - {{- include "cray-service.common-labels" . | nindent 4 }} - annotations: - {{- include "cray-service.common-annotations" . | nindent 4 }} -rules: -- apiGroups: [""] - resources: ["persistentvolumeclaims", "events", "secrets", "pods"] - verbs: ["*"] -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["get"] -- apiGroups: [""] - resources: ["configmaps", "services"] - verbs: ["get", "list", "create", "delete", "update"] -- apiGroups: ["apps"] - resources: ["daemonsets"] - verbs: ["get", "list"] -- apiGroups: ["monitoring.coreos.com"] - resources: ["servicemonitors"] - verbs: ["get", "create"] -- apiGroups: ["apps"] - resourceNames: ["csds-dpapi"] - resources: ["deployments/finalizers"] - verbs: ["update"] -- apiGroups: ["dpm.cray.com"] - resources: ["*"] - verbs: ["*"] diff --git a/kubernetes/nnf-ec/templates/role_binding.yaml b/kubernetes/nnf-ec/templates/role_binding.yaml deleted file mode 100644 index d6f658b..0000000 --- a/kubernetes/nnf-ec/templates/role_binding.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "csds-dpapi.fullname" . }} - labels: - app.kubernetes.io/name: {{ include "cray-service.name" . }} - {{- include "cray-service.common-labels" . | nindent 4 }} - annotations: - {{- include "cray-service.common-annotations" . | nindent 4 }} -subjects: -- kind: ServiceAccount - name: {{ index .Values "cray-service" "serviceAccountName" }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "csds-dpapi.fullname" . }} - apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/nnf-ec/templates/service_account.yaml b/kubernetes/nnf-ec/templates/service_account.yaml deleted file mode 100644 index ea7dab0..0000000 --- a/kubernetes/nnf-ec/templates/service_account.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ index .Values "cray-service" "serviceAccountName" }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "cray-service.name" . }} - {{- include "cray-service.common-labels" . | nindent 4 }} - annotations: - {{- include "cray-service.common-annotations" . | nindent 4 }} diff --git a/kubernetes/nnf-ec/values.yaml b/kubernetes/nnf-ec/values.yaml deleted file mode 100644 index 797ad18..0000000 --- a/kubernetes/nnf-ec/values.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Note that cray-service.containers[*].image and cray-service.initContainers[*].image map values are one of the only structures that -# differ from the standard kubernetes container spec: -# image: -# repository: "" -# tag: "" (default = "latest") -# pullPolicy: "" (default = "IfNotPresent") -cray-service: - # imagesHost: "s-lmo-anastl:5000" - nameOverride: nnf-ec - type: DaemonSet - service: - enabled: False - replicaCount: 1 - serviceAccountName: nnf-ec - tolerations: - - key: "node-type" - operator: "Equal" - value: "storage" - effect: "NoSchedule" - containers: - cray-nnf-ec: - name: cray-nnf-ec - image: - repository: cray/cray-nnf-ec - tag: latest - pullPolicy: Never - command: ["nnf-ec"] - args: ["--http", "--port=8080", "--mock", "--log", "--verbose"] - ports: - - name: nnf-api-service - containerPort: 8080 - securityContext: - privileged: true - volumeMounts: - - mountPath: /dev - name: dev-dir - - mountPath: /mnt - name: mnt-dir - mountPropagation: Bidirectional - volumes: - - name: dev-dir - hostPath: - path: /dev - - name: mnt-dir - hostPath: - path: /mnt - nodeSelector: - cray.dpm.dg.dpm-nnf-node: "true" - -global: - chart: - name: "" # set at deploy time automatically, no need to ever set explicitly - version: "" # set at deploy time automatically, no need to ever set explicitly From 62758e5c7eb459e22019e25de45b066097cbdd05 Mon Sep 17 00:00:00 2001 From: Anthony Floeder Date: Wed, 20 Aug 2025 11:29:43 -0500 Subject: [PATCH 5/6] During NVMe operations, when system-level errors occur such as: - device communication errors - timeout errors - device not found errors Take the following actions: - record the state of the storage (NVMe device) as offline - publish event to trigger nnf-sos to query the storage state Signed-off-by: Anthony Floeder --- Makefile | 69 +++++++++++-------- pkg/manager-nvme/manager.go | 133 ++++++++++++++++++++++++++++++++++-- 2 files changed, 169 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 6b56532..726671b 100644 --- a/Makefile +++ b/Makefile @@ -21,92 +21,105 @@ PROD_VERSION=$(shell sed 1q .version) DEV_IMGNAME=nnf-ec DTR_IMGPATH=ghcr.io/nearnodeflash/$(DEV_IMGNAME) -all: image - -vendor: +.DEFAULT_GOAL := help + +## Display this help message +help: + @echo "NNF Element Controller (nnf-ec) Makefile" + @echo "" + @echo "Available targets:" + @echo "" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @echo "" + @echo "Docker cleanup targets:" + @echo " clean-images Remove all project-related Docker images" + @echo " clean-docker-all Remove ALL unused Docker resources (aggressive)" + @echo " clean-docker-images Remove unused Docker images only" + @echo " clean-docker-cache Remove Docker build cache" + @echo " clean-project-docker Remove project Docker images and unused resources" + @echo "" + +all: image ## Build the default Docker image + +vendor: ## Download Go module dependencies go mod vendor -vet: +vet: ## Run Go vet on all packages go vet `go list -f {{.Dir}} ./...` -fmt: +fmt: ## Format Go source code go fmt `go list -f {{.Dir}} ./...` -generate: +generate: ## Generate code and redfish/swordfish message registries ( cd ./pkg/manager-message-registry/generator && go build msgenerator.go ) go generate ./... go fmt ./pkg/manager-message-registry/registries -test: +test: ## Run Go unit tests locally go test -v ./... -linux: +linux: ## Build Linux binary env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${DEV_IMGNAME} ./cmd/nnf_ec.go -image: +image: ## Build Docker image docker build --file Dockerfile --label $(DTR_IMGPATH):$(PROD_VERSION) --tag $(DTR_IMGPATH):$(PROD_VERSION) . -container-unit-test: +container-unit-test: ## Run containerized unit tests docker build -f Dockerfile --label $(DTR_IMGPATH)-$@:$(PROD_VERSION)-$@ -t $(DTR_IMGPATH)-$@:$(PROD_VERSION) --target $@ . docker run --rm -t --name $@ -v $(PWD):$(PWD):rw,z $(DTR_IMGPATH)-$@:$(PROD_VERSION) # This and the corresponding clean-lint should go away and move to git pre-commit hook -lint: +lint: ## Run linting checks in Docker container docker build -f Dockerfile --label $(DTR_IMGPATH)-$@:$(PROD_VERSION)-$@ -t $(DTR_IMGPATH)-$@:$(PROD_VERSION) --target $@ . docker run --rm -t --name $@ -v $(PWD):$(PWD):rw,z $(DTR_IMGPATH)-$@:$(PROD_VERSION) # This and the cooresponding clean-codestyle should go away and move to git pre-commit hook -codestyle: +codestyle: ## Run code style checks in Docker container docker build -f Dockerfile --label $(DTR_IMGPATH)-$@:$(PROD_VERSION) -t $(DTR_IMGPATH)-$@:$(PROD_VERSION) --target $@ . docker run --rm -t --name $@ -v $(PWD):$(PWD):rw,z $(DTR_IMGPATH)-$@:$(PROD_VERSION) -clean-lint: +clean-lint: ## Clean lint Docker images and containers docker rmi $(DTR_IMGPATH)-lint:$(PROD_VERSION) || true docker container rm lint || true -clean-codestyle: +clean-codestyle: ## Clean codestyle Docker images and containers docker rmi $(DTR_IMGPATH)-codestyle:$(PROD_VERSION) || true docker container rm codestyle || true -clean-container-unit-test: +clean-container-unit-test: ## Clean container-unit-test Docker images and containers docker rmi $(DTR_IMGPATH)-container-unit-test:$(PROD_VERSION) || true docker container rm container-unit-test || true # push: # docker push $(DTR_IMGPATH):$(PROD_VERSION) -kind-push: image +kind-push: image ## Load Docker image into kind cluster kind load docker-image $(DTR_IMGPATH):$(PROD_VERSION) -clean: clean-db +clean: clean-db ## Clean build artifacts and Docker resources docker container prune --force docker image prune --force docker rmi $(DTR_IMGPATH):$(PROD_VERSION) || true go clean all -# Clean all Docker images related to this project -clean-images: +clean-images: ## Clean all Docker images related to this project docker rmi $(DTR_IMGPATH):$(PROD_VERSION) || true docker rmi $(DTR_IMGPATH)-container-unit-test:$(PROD_VERSION) || true docker rmi $(DTR_IMGPATH)-lint:$(PROD_VERSION) || true docker rmi $(DTR_IMGPATH)-codestyle:$(PROD_VERSION) || true -# Aggressive Docker cleanup - removes all unused images, containers, networks, and build cache -clean-docker-all: +clean-docker-all: ## Remove ALL unused Docker resources (aggressive) docker system prune -a --volumes --force -# Clean only unused Docker images (keeps tagged images) -clean-docker-images: +clean-docker-images: ## Remove unused Docker images only docker image prune --force -# Clean Docker build cache -clean-docker-cache: +clean-docker-cache: ## Remove Docker build cache docker builder prune --force -# Clean everything Docker related for this project -clean-project-docker: clean-images +clean-project-docker: clean-images ## Remove project Docker images and unused resources docker container prune --force docker image prune --force -clean-db: +clean-db: ## Remove all database files find . -name "*.db" -type d -exec rm -rf {} + diff --git a/pkg/manager-nvme/manager.go b/pkg/manager-nvme/manager.go index efde722..fe37caf 100644 --- a/pkg/manager-nvme/manager.go +++ b/pkg/manager-nvme/manager.go @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 Hewlett Packard Enterprise Development LP + * Copyright 2020-2025 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -364,6 +364,7 @@ func (s *Storage) initialize() error { ctrl, err := s.device.IdentifyController(0) if err != nil { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) return fmt.Errorf("Initialize Storage %s: Failed to identify common controller: Error: %w", s.id, err) } @@ -420,6 +421,8 @@ func (s *Storage) initialize() error { ns, err = s.device.IdentifyNamespace(CommonNamespaceIdentifier) if err != nil { if retryCount >= 2 { + // After retries, this is likely a system-level failure + s.notify(sf.UNAVAILABLE_OFFLINE_RST) return fmt.Errorf("Initialize Storage %s: Failed to identify common namespace, retried %d times: Error: %w", s.id, retryCount, err) } @@ -464,11 +467,19 @@ func (s *Storage) purge() error { namespaces, err := s.device.ListNamespaces(0) if err != nil { + // System-level failure when listing namespaces + if isSystemLevelError(err) { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } for _, nsid := range namespaces { if err := s.device.DeleteNamespace(nsid); err != nil { + // System-level failure when deleting namespace during purge + if isSystemLevelError(err) { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } } @@ -540,19 +551,23 @@ func (s *Storage) createVolume(desiredCapacityInBytes uint64) (*Volume, error) { actualCapacityBytes := roundUpToMultiple(desiredCapacityInBytes, s.blockSizeBytes) s.log.V(2).Info("Creating namespace", "capacityInBytes", actualCapacityBytes, "formatIndex", s.lbaFormatIndex) - namespaceId, guid, err := s.device.CreateNamespace(actualCapacityBytes/s.blockSizeBytes, s.lbaFormatIndex) + namespaceID, guid, err := s.device.CreateNamespace(actualCapacityBytes/s.blockSizeBytes, s.lbaFormatIndex) if err != nil { + // Only publish error events for system-level failures + if isSystemLevelError(err) { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return nil, err } - id := strconv.Itoa(int(namespaceId)) + id := strconv.Itoa(int(namespaceID)) - log := s.log.WithName(id).WithValues(namespaceIdKey, namespaceId) + log := s.log.WithName(id).WithValues(namespaceIdKey, namespaceID) log.V(1).Info("Created namespace") volume := Volume{ id: id, - namespaceId: namespaceId, + namespaceId: namespaceID, guid: guid, capacityBytes: actualCapacityBytes, storage: s, @@ -575,6 +590,10 @@ func (s *Storage) deleteVolume(volumeId string) error { log.V(2).Info("Deleting namespace") err := s.device.DeleteNamespace(volume.namespaceId) if err != nil { + // Only publish error events for system-level failures + if isSystemLevelError(err) { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) + } log.Error(err, "Delete namespace failed, removing from in-memory state anyway", "capacity", volume.capacityBytes) } else { log.V(1).Info("Deleted namespace", "capacity", volume.capacityBytes) @@ -599,6 +618,10 @@ func (s *Storage) formatVolume(volumeID string) error { log.V(2).Info("Format namespace") if err := s.device.FormatNamespace(volume.namespaceId); err != nil { + // Only publish error events for system-level failures + if isSystemLevelError(err) { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) + } log.Error(err, "Format namespace failure") return err } @@ -614,6 +637,10 @@ func (s *Storage) formatVolume(volumeID string) error { func (s *Storage) recoverStorageVolumes() error { namespaces, err := s.device.ListNamespaces(0) if err != nil { + // Only publish error events for system-level failures + if isSystemLevelError(err) { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) + } s.log.Error(err, "Failed to list device namespaces") } @@ -623,6 +650,10 @@ func (s *Storage) recoverStorageVolumes() error { ns, err := s.device.IdentifyNamespace(nsid) if err != nil { + // Only publish error events for system-level failures + if isSystemLevelError(err) { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) + } log.Error(err, "Failed to identify namespace") continue } @@ -649,6 +680,66 @@ func (s *Storage) recoverStorageVolumes() error { return nil } +// isSystemLevelError determines if an error represents a system-level failure +// (hardware, communication, etc.) rather than an expected operational failure +// (insufficient capacity, namespace already exists, etc.). +// System-level errors should trigger ResourceState event publishing. +func isSystemLevelError(err error) bool { + if err == nil { + return false + } + + // Check if it's an NVMe command error with a specific status code + var cmdErr *nvme.CommandError + if errors.As(err, &cmdErr) { + switch cmdErr.StatusCode { + // Expected operational failures - don't publish events + case nvme.NamespaceAlreadyAttached: + return false + case nvme.NamespaceNotAttached: + return false + case 0x0A: // Namespace Insufficient Capacity + return false + case 0x0B: // Namespace Identifier Unavailable + return false + case 0x0C: // Namespace Already Exists + return false + case 0x15: // Invalid Namespace Identifier (operational - bad request) + return false + case 0x02: // Invalid Field in Command (operational - bad parameters) + return false + default: + // Unknown status codes are considered system-level errors + // This includes hardware failures, communication timeouts, etc. + return true + } + } + + // Check for mock device operational errors + errMsg := err.Error() + if strings.Contains(errMsg, "Insufficient capacity") || + strings.Contains(errMsg, "Could not find free namespace") || + strings.Contains(errMsg, "Namespace Already Exists") || + strings.Contains(errMsg, "already has controller") || + strings.Contains(errMsg, "already has controller") || + strings.Contains(errMsg, "not found") { + return false + } + + // Check for CLI parsing errors (not system-level) + if strings.Contains(errMsg, "NSID not found in response output") { + return false + } + + // All other errors are considered system-level failures: + // - Device communication errors + // - Hardware failures + // - Timeout errors + // - Permission/access errors + // - Unknown NVMe status codes + return true +} + func (s *Storage) findVolume(volumeId string) *Volume { for idx, v := range s.volumes { if v.id == volumeId { @@ -727,6 +818,10 @@ func (v *Volume) WaitFormatComplete() error { log.V(2).Info("Wait for format completion") ns, err := v.storage.device.IdentifyNamespace(v.namespaceId) if err != nil { + // System-level failure when identifying namespace during format wait + if isSystemLevelError(err) { + v.storage.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } if ns == nil { @@ -746,6 +841,10 @@ func (v *Volume) WaitFormatComplete() error { ns, err = v.storage.device.IdentifyNamespace(v.namespaceId) if err != nil { + // System-level failure when re-identifying namespace during format wait + if isSystemLevelError(err) { + v.storage.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } if ns == nil { @@ -808,9 +907,17 @@ func (v *Volume) attach(controllerIndex uint16) error { var cmdErr *nvme.CommandError if errors.As(err, &cmdErr) { if cmdErr.StatusCode != nvme.NamespaceAlreadyAttached { + // Only publish error events for system-level failures + if isSystemLevelError(err) { + v.storage.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } } else { + // Non-CommandError types - check if system-level + if isSystemLevelError(err) { + v.storage.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } } @@ -834,9 +941,17 @@ func (v *Volume) detach(controllerIndex uint16) error { var cmdErr *nvme.CommandError if errors.As(err, &cmdErr) { if cmdErr.StatusCode != nvme.NamespaceNotAttached { + // Only publish error events for system-level failures + if isSystemLevelError(err) { + v.storage.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } } else { + // Non-CommandError types - check if system-level + if isSystemLevelError(err) { + v.storage.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } } @@ -866,6 +981,10 @@ func (v *Volume) AttachControllerIfUnattached(controllerIndex uint16) error { attachedControllers, err := v.listAttachedControllers() if err != nil { log.Error(err, "failed to list attached controllers") + // System-level failure when listing attached controllers + if isSystemLevelError(err) { + v.storage.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return err } @@ -1332,6 +1451,10 @@ func (mgr *Manager) StorageIdVolumeIdGet(storageId, volumeId string, model *sf.V ns, err := s.device.IdentifyNamespace(nvme.NamespaceIdentifier(v.namespaceId)) if err != nil { v.log.Error(err, "Identify Namespace Failed") + // System-level failure when identifying namespace for volume retrieval + if isSystemLevelError(err) { + s.notify(sf.UNAVAILABLE_OFFLINE_RST) + } return ec.NewErrInternalServerError() } if ns == nil { From 89f75451d5ca2a3289f0c24f73406f54cf1a0df5 Mon Sep 17 00:00:00 2001 From: Anthony Floeder Date: Thu, 28 Aug 2025 16:51:15 -0500 Subject: [PATCH 6/6] Use the available drive count to calculate the size of each allocation. Signed-off-by: Anthony Floeder --- pkg/manager-nnf/allocation_policy.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/manager-nnf/allocation_policy.go b/pkg/manager-nnf/allocation_policy.go index da0d3d8..636ece2 100644 --- a/pkg/manager-nnf/allocation_policy.go +++ b/pkg/manager-nnf/allocation_policy.go @@ -161,8 +161,9 @@ func (p *SpareAllocationPolicy) CheckAndAdjustCapacity() error { if p.compliance != RelaxedAllocationComplianceType { - if len(p.storage) < SpareAllocationPolicyMinimumDriveCount { - return fmt.Errorf("Insufficient drive count. Required: %d Available: %d", SpareAllocationPolicyMinimumDriveCount, len(p.storage)) + driveCount := uint64(len(p.storage)) + if driveCount < SpareAllocationPolicyMinimumDriveCount { + return fmt.Errorf("Insufficient drive count. Required: %d Available: %d", SpareAllocationPolicyMinimumDriveCount, driveCount) } roundUpToMultiple := func(n, m uint64) uint64 { // Round 'n' up to a multiple of 'm' @@ -170,8 +171,8 @@ func (p *SpareAllocationPolicy) CheckAndAdjustCapacity() error { } // Validate each drive can contribute sufficient capacity towards the entire pool. - poolCapacityBytes := roundUpToMultiple(p.capacityBytes, SpareAllocationPolicyMinimumDriveCount) - driveCapacityBytes := roundUpToMultiple(poolCapacityBytes/SpareAllocationPolicyMinimumDriveCount, 4096) + poolCapacityBytes := roundUpToMultiple(p.capacityBytes, driveCount) + driveCapacityBytes := roundUpToMultiple(poolCapacityBytes/driveCount, 4096) for _, s := range p.storage { if driveCapacityBytes > s.UnallocatedBytes() { @@ -180,7 +181,7 @@ func (p *SpareAllocationPolicy) CheckAndAdjustCapacity() error { } // Adjust the pool's capacity such that it is a multiple of the number drives. - p.capacityBytes = driveCapacityBytes * uint64(len(p.storage)) + p.capacityBytes = driveCapacityBytes * driveCount } if availableBytes < p.capacityBytes { @@ -193,7 +194,8 @@ func (p *SpareAllocationPolicy) CheckAndAdjustCapacity() error { // Allocate - allocate the storage func (p *SpareAllocationPolicy) Allocate() ([]nvme.ProvidingVolume, error) { - perStorageCapacityBytes := p.capacityBytes / uint64(len(p.storage)) + driveCount := uint64(len(p.storage)) + perStorageCapacityBytes := p.capacityBytes / driveCount remainingCapacityBytes := p.capacityBytes volumes := []nvme.ProvidingVolume{} @@ -204,7 +206,7 @@ func (p *SpareAllocationPolicy) Allocate() ([]nvme.ProvidingVolume, error) { // Leftover bytes are placed on trailing volume; note that this // is never the case for strict allocation in which the requested // allocation must be a multiple of the storage size. - if idx == len(p.storage)-1 { + if idx == int(driveCount-1) { capacityBytes = remainingCapacityBytes }