Merge branch 'master' into release-v0.1.10

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>

Author: ajfloeder

internal/switchtec/cmd/fabric/perf.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -51,7 +51,7 @@ func (cmd *BandwidthCmd) Run() error {
 		return err
 	}
 
-	// Resetting the bandwidht counter takes about 1s
+	// Resetting the bandwidth counter takes about 1s
 	time.Sleep(time.Second)
 
 	// Record the bandwidth counters

internal/switchtec/pkg/nvme/nvme.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -22,6 +22,7 @@ package nvme
 import (
 	"encoding/binary"
 	"fmt"
+	"math/rand"
 	"path"
 	"regexp"
 	"strconv"
@@ -33,6 +34,8 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec"
+	"github.com/NearNodeFlash/nnf-ec/pkg/ec"
+	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
 )
 
 // Device describes the NVMe Device and its attributes
@@ -532,7 +535,7 @@ type id_ctrl struct {
 	NVMSetIdentifierMaximum                   uint16               // NSETIDMAX
 	EnduraceGroupIdentifierMaximum            uint16               // ENDGIDMAX
 	ANATranstitionTime                        uint8                // ANATT
-	AsymentricNamespaceAccessCapabilities     uint8                // ANACAP
+	AsymmetricNamespaceAccessCapabilities     uint8                // ANACAP
 	ANAGroupIdenfierMaximum                   uint32               // ANAGRPMAX
 	NumberOfANAGroupIdentifiers               uint32               // NANAGRPID
 	PersistentEventLogSize                    uint32               // PELS
@@ -1101,7 +1104,7 @@ type SmartLog struct {
 	DataUnitsWrittenLo                           uint64
 	DataUnitsWrittenHi                           uint64
 	HostReadsLo                                  uint64
-	HistReadsHi                                  uint64
+	HostReadsHi                                  uint64
 	HostWritesLo                                 uint64
 	HostWritesHi                                 uint64
 	ControllerBusyTimeLo                         uint64
@@ -1126,6 +1129,7 @@ type SmartLog struct {
 	Reserved232                                  [280]uint8
 }
 
+// GetSmartLog - retrieve the smartlog information
 func (dev *Device) GetSmartLog() (*SmartLog, error) {
 
 	log := new(SmartLog)
@@ -1146,6 +1150,53 @@ func (dev *Device) GetSmartLog() (*SmartLog, error) {
 	return log, nil
 }
 
+// InterpretSmartLog - calculate the drive's swordfish status
+func InterpretSmartLog(log *SmartLog) sf.ResourceState {
+	// Check critical warnings first - check individual bitfield members
+	if log.CriticalWarning.SpareCapacity != 0 { // Spare capacity
+		return sf.DISABLED_RST
+	}
+	if log.CriticalWarning.ReadOnly != 0 { // Read-only mode
+		return sf.DISABLED_RST
+	}
+
+	// Check other critical warnings for degraded state
+	if log.CriticalWarning.Temperature != 0 ||
+		log.CriticalWarning.Degraded != 0 ||
+		log.CriticalWarning.BackupFailed != 0 ||
+		log.CriticalWarning.PersistentMemoryRegionReadOnly != 0 {
+		return sf.STANDBY_OFFLINE_RST
+	}
+
+	return sf.ENABLED_RST // Healthy
+}
+
+func MangleSmartLog(log *SmartLog) {
+
+	// Occaionally managle the smart log data
+	if rand.Intn(100) < 10 { // 10% chance of simulating critical condition
+		warningType := rand.Intn(6)
+		switch warningType {
+		case 0:
+			log.CriticalWarning.SpareCapacity = 1
+			log.AvailableSpare = 5 // Below threshold
+		case 1:
+			log.CriticalWarning.Temperature = 1
+			log.CompositeTemperature = 358 // ~85°C
+		case 2:
+			log.CriticalWarning.Degraded = 1
+		case 3:
+			log.CriticalWarning.ReadOnly = 1
+		case 4:
+			log.CriticalWarning.BackupFailed = 1
+		case 5:
+			log.CriticalWarning.PersistentMemoryRegionReadOnly = 1
+		}
+	}
+
+}
+
+// Log contstants
 const (
 	LogCdw10LogPageIdentiferMask         = 0xFF
 	LogCdw10LogPageIdentiferShift        = 0
@@ -1166,7 +1217,7 @@ const (
 )
 
 func (dev *Device) getNsidLog(logPageIdentifier uint8, retainAsynchronousEvent uint8, nsid uint32, buf []byte) error {
-	return dev.getLog(logPageIdentifier, 0, 0, nsid, 0, buf)
+	return dev.getLog(logPageIdentifier, 0, retainAsynchronousEvent, nsid, 0, buf)
 }
 
 func (dev *Device) getLog(logPageIdentifier uint8, logSpecificField uint8, retainAsynchronousEvent uint8, nsid uint32, logPageOffset uint64, buf []byte) error {
@@ -1200,3 +1251,26 @@ func (dev *Device) getLog(logPageIdentifier uint8, logSpecificField uint8, retai
 	return dev.ops.submitAdminPassthru(dev, &cmd, buf)
 
 }
+
+// LogSmartLog outputs the SMART log data in a structured way for logging.
+func LogSmartLog(log ec.Logger, smartLog *SmartLog, context ...interface{}) {
+	if smartLog == nil {
+		log.Error(nil, "SMART log is nil", context...)
+		return
+	}
+	log.Info("SMART log data",
+		append(context,
+			"criticalWarning.spareCapacity", smartLog.CriticalWarning.SpareCapacity,
+			"criticalWarning.temperature", smartLog.CriticalWarning.Temperature,
+			"criticalWarning.degraded", smartLog.CriticalWarning.Degraded,
+			"criticalWarning.readOnly", smartLog.CriticalWarning.ReadOnly,
+			"criticalWarning.backupFailed", smartLog.CriticalWarning.BackupFailed,
+			"criticalWarning.persistentMemoryRegionReadOnly", smartLog.CriticalWarning.PersistentMemoryRegionReadOnly,
+			"availableSpare", smartLog.AvailableSpare,
+			"availableSpareThreshold", smartLog.AvailableSpareThreshold,
+			"percentageUsed", smartLog.PercentageUsed,
+			"compositeTemperature", smartLog.CompositeTemperature,
+			"mediaErrorsLo", smartLog.MediaErrorsLo,
+			"numberErrorLogEntriesLo", smartLog.NumberErrorLogEntriesLo,
+		)...)
+}

pkg/manager-fabric/manager.go

@@ -782,7 +782,11 @@ func (p *Port) bind() error {
 							if endpoint.boundPaxId != uint8(s.paxId) ||
 								endpoint.boundHvdPhyId != uint8(initiatorPort.config.Port) ||
 								endpoint.boundHvdLogId != uint8(logicalPortId) {
-								panic(fmt.Sprintf("Already Bound: Misconfigured Port: PAX: %d, Physical Port: %d, Logical Port: %d, PDFID: %#04x", endpoint.boundPaxId, endpoint.boundHvdPhyId, endpoint.boundHvdLogId, endpoint.pdfid))
+								panic(fmt.Sprintf("Already Bound: Misconfigured Port: PAX: %d(%d), Physical Port: %d(%d), Logical Port: %d(%d), PDFID: %#04x",
+									endpoint.boundPaxId, uint8(s.paxId),
+									endpoint.boundHvdPhyId, uint8(initiatorPort.config.Port),
+									endpoint.boundHvdLogId, uint8(logicalPortId),
+									endpoint.pdfid))
 							}
 
 							break
@@ -1121,7 +1125,7 @@ func Start() error {
 	event.EventManager.Publish(msgreg.FabricReadyNnf(m.id))
 
 	// Run the Fabric Monitor in a background thread.
-	go NewFabricMonitor(m).Run()
+	StartFabricMonitor(m)
 
 	return nil
 }

pkg/manager-fabric/monitor.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -27,23 +27,53 @@ import (
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec"
 )
 
-// The Fabric Monitor is responsible for ensuring that the fabriic and related sub-resource
+// NewFabricMonitor - The Fabric Monitor is responsible for ensuring that the fabriic and related sub-resource
 // are updated with the latest information from the switch. This runs as a background
 // thread, and periodically queries the fabric.
-func NewFabricMonitor(f *Fabric) *monitor {
-	return &monitor{fabric: f}
+func NewMonitor(f *Fabric, i time.Duration) *monitor {
+	return &monitor{fabric: f, interval: i}
 }
 
 type monitor struct {
-	fabric *Fabric
+	fabric   *Fabric
+	interval time.Duration
+}
+
+// StartFabricMonitor starts the fabric monitor in a background goroutine if the period is non-zero.
+func StartFabricMonitor(fabric *Fabric) {
+	defaultFabricMonitorPeriod := 60 * time.Second
+	fabricMonitorPeriod := defaultFabricMonitorPeriod
+	if periodStr := os.Getenv("NNF_FABRIC_MONITOR_PERIOD"); periodStr != "" {
+		if d, err := time.ParseDuration(periodStr); err == nil {
+			fabricMonitorPeriod = d
+		} else {
+			if fabric != nil && !fabric.log.IsZero() {
+				fabric.log.Info("Invalid NNF_FABRIC_MONITOR_PERIOD, using default", "value", fabricMonitorPeriod, "error", err)
+			}
+		}
+	}
+
+	// A period of 0 means don't start the monitor.
+	if fabricMonitorPeriod == 0 {
+		if fabric != nil && !fabric.log.IsZero() {
+			fabric.log.Info("Not starting fabric monitor", "monitorPeriod", fabricMonitorPeriod)
+		}
+		return
+	}
+
+	mon := NewMonitor(fabric, fabricMonitorPeriod)
+	go mon.Run()
+	if fabric != nil && !fabric.log.IsZero() {
+		fabric.log.Info("Started fabric monitor", "monitorPeriod", fabricMonitorPeriod)
+	}
+
 }
 
 // Run Fabric Monitor forever
 func (m *monitor) Run() {
 
 	for {
-
-		time.Sleep(time.Second * 60)
+		time.Sleep(m.interval)
 
 		for idx := range m.fabric.switches {
 			s := &m.fabric.switches[idx]
@@ -62,13 +92,13 @@ func (m *monitor) Run() {
 					}
 
 					for _, event := range events {
-						physPortId, isDown := m.getEventInfo(event)
+						physPortID, isDown := m.getEventInfo(event)
 
-						if physPortId == invalidPhysicalPortId {
+						if physPortID == invalidPhysicalPortId {
 							continue
 						}
 
-						if p := s.findPortByPhysicalPortId(physPortId); p != nil {
+						if p := s.findPortByPhysicalPortId(physPortID); p != nil {
 							p.notify(isDown)
 						}
 					}

pkg/manager-message-registry/registries/Nnf.1.0.0.go

@@ -54,7 +54,7 @@
             "Message": "The storage pool '%1' patched, SerialNumber '%2', Namespace '%3' replaced with SerialNumber '%4', Namespace '%5'",
             "Severity": "OK",
             "MessageSeverity": "OK",
-            "NumberOfArgs": 1,
+            "NumberOfArgs": 5,
             "ParamTypes": [
                 "string",
                 "string",
@@ -77,6 +77,30 @@
                 "This argument shall contain the namespace id on the new storage device."
             ],
             "Resolution": "None"
+        },
+        "NvmeStateChange": {
+            "Description": "Indicates that an NVMe device has changed state",
+            "LongDescription": "This message shall be used to indicate that an NVMe device has changed state",
+            "Message": "The Nvme State changed slot '%1', model '%2', serial '%3'",
+            "Severity": "Warning",
+            "MessageSeverity": "Warning",
+            "NumberOfArgs": 3,
+            "ParamTypes": [
+                "number",
+                "string",
+                "string"
+            ],
+            "ArgDescriptions": [
+                "The slot identifier.",
+                "The model number.",
+                "The serial number."
+            ],
+            "ArgLongDescriptions": [
+                "This argument shall contain the NVMe slot number.",
+                "This argument shall contain the NVMe model number.",
+                "This argument shall contain the NVMe serial number."
+            ],
+            "Resolution": "None"
         }
     }
 }

pkg/manager-message-registry/registries/Nnf.1.0.0.json

@@ -112,8 +112,8 @@ func NewAllocationPolicy(config AllocationConfig, oem map[string]interface{}) Al
 
 /* ------------------------------ Spare Allocation Policy --------------------- */
 
-// Required number of drives for the Spare Allocation Policy
-const SpareAllocationPolicyRequiredDriveCount = 16
+const SpareAllocationPolicyExpectedDriveCount = 16
+const SpareAllocationPolicyMinimumDriveCount = 14
 
 type SpareAllocationPolicy struct {
 	compliance     AllocationComplianceType
@@ -137,7 +137,7 @@ func (p *SpareAllocationPolicy) Initialize(capacityBytes uint64) error {
 		return !!!(storage[i].UnallocatedBytes() < storage[j].UnallocatedBytes())
 	})
 
-	count := SpareAllocationPolicyRequiredDriveCount
+	count := SpareAllocationPolicyExpectedDriveCount
 	if len(storage) < count {
 		count = len(storage)
 	}
@@ -161,17 +161,17 @@ func (p *SpareAllocationPolicy) CheckAndAdjustCapacity() error {
 
 	if p.compliance != RelaxedAllocationComplianceType {
 
-		if len(p.storage) != SpareAllocationPolicyRequiredDriveCount {
-			return fmt.Errorf("Insufficient drive count. Required: %d Available: %d", SpareAllocationPolicyRequiredDriveCount, len(p.storage))
+		if len(p.storage) < SpareAllocationPolicyMinimumDriveCount {
+			return fmt.Errorf("Insufficient drive count. Required: %d Available: %d", SpareAllocationPolicyMinimumDriveCount, len(p.storage))
 		}
 
 		roundUpToMultiple := func(n, m uint64) uint64 { // Round 'n' up to a multiple of 'm'
 			return ((n + m - 1) / m) * m
 		}
 
 		// Validate each drive can contribute sufficient capacity towards the entire pool.
-		poolCapacityBytes := roundUpToMultiple(p.capacityBytes, SpareAllocationPolicyRequiredDriveCount)
-		driveCapacityBytes := roundUpToMultiple(poolCapacityBytes/SpareAllocationPolicyRequiredDriveCount, 4096)
+		poolCapacityBytes := roundUpToMultiple(p.capacityBytes, SpareAllocationPolicyMinimumDriveCount)
+		driveCapacityBytes := roundUpToMultiple(poolCapacityBytes/SpareAllocationPolicyMinimumDriveCount, 4096)
 
 		for _, s := range p.storage {
 			if driveCapacityBytes > s.UnallocatedBytes() {

pkg/manager-nnf/allocation_policy.go

@@ -625,6 +625,8 @@ func (s *StorageService) EventHandler(e event.Event) error {
 		}
 
 		log.Info("Storage Service Enabled", "health", s.health)
+
+		nvme.StartNVMeMonitor(s.log)
 	}
 
 	// Check for storage pool events