Merge pull request #138 from NearNodeFlash/monitor-drives

Add NVMe monitoring with configurable periods and SMART log health ch…

Author: ajfloeder

internal/switchtec/cmd/fabric/perf.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -51,7 +51,7 @@ func (cmd *BandwidthCmd) Run() error {
 		return err
 	}
 
-	// Resetting the bandwidht counter takes about 1s
+	// Resetting the bandwidth counter takes about 1s
 	time.Sleep(time.Second)
 
 	// Record the bandwidth counters

internal/switchtec/pkg/nvme/nvme.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -22,6 +22,7 @@ package nvme
 import (
 	"encoding/binary"
 	"fmt"
+	"math/rand"
 	"path"
 	"regexp"
 	"strconv"
@@ -33,6 +34,8 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec"
+	"github.com/NearNodeFlash/nnf-ec/pkg/ec"
+	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
 )
 
 // Device describes the NVMe Device and its attributes
@@ -532,7 +535,7 @@ type id_ctrl struct {
 	NVMSetIdentifierMaximum                   uint16               // NSETIDMAX
 	EnduraceGroupIdentifierMaximum            uint16               // ENDGIDMAX
 	ANATranstitionTime                        uint8                // ANATT
-	AsymentricNamespaceAccessCapabilities     uint8                // ANACAP
+	AsymmetricNamespaceAccessCapabilities     uint8                // ANACAP
 	ANAGroupIdenfierMaximum                   uint32               // ANAGRPMAX
 	NumberOfANAGroupIdentifiers               uint32               // NANAGRPID
 	PersistentEventLogSize                    uint32               // PELS
@@ -1101,7 +1104,7 @@ type SmartLog struct {
 	DataUnitsWrittenLo                           uint64
 	DataUnitsWrittenHi                           uint64
 	HostReadsLo                                  uint64
-	HistReadsHi                                  uint64
+	HostReadsHi                                  uint64
 	HostWritesLo                                 uint64
 	HostWritesHi                                 uint64
 	ControllerBusyTimeLo                         uint64
@@ -1126,6 +1129,7 @@ type SmartLog struct {
 	Reserved232                                  [280]uint8
 }
 
+// GetSmartLog - retrieve the smartlog information
 func (dev *Device) GetSmartLog() (*SmartLog, error) {
 
 	log := new(SmartLog)
@@ -1146,6 +1150,53 @@ func (dev *Device) GetSmartLog() (*SmartLog, error) {
 	return log, nil
 }
 
+// InterpretSmartLog - calculate the drive's swordfish status
+func InterpretSmartLog(log *SmartLog) sf.ResourceState {
+	// Check critical warnings first - check individual bitfield members
+	if log.CriticalWarning.SpareCapacity != 0 { // Spare capacity
+		return sf.DISABLED_RST
+	}
+	if log.CriticalWarning.ReadOnly != 0 { // Read-only mode
+		return sf.DISABLED_RST
+	}
+
+	// Check other critical warnings for degraded state
+	if log.CriticalWarning.Temperature != 0 ||
+		log.CriticalWarning.Degraded != 0 ||
+		log.CriticalWarning.BackupFailed != 0 ||
+		log.CriticalWarning.PersistentMemoryRegionReadOnly != 0 {
+		return sf.STANDBY_OFFLINE_RST
+	}
+
+	return sf.ENABLED_RST // Healthy
+}
+
+func MangleSmartLog(log *SmartLog) {
+
+	// Occaionally managle the smart log data
+	if rand.Intn(100) < 10 { // 10% chance of simulating critical condition
+		warningType := rand.Intn(6)
+		switch warningType {
+		case 0:
+			log.CriticalWarning.SpareCapacity = 1
+			log.AvailableSpare = 5 // Below threshold
+		case 1:
+			log.CriticalWarning.Temperature = 1
+			log.CompositeTemperature = 358 // ~85°C
+		case 2:
+			log.CriticalWarning.Degraded = 1
+		case 3:
+			log.CriticalWarning.ReadOnly = 1
+		case 4:
+			log.CriticalWarning.BackupFailed = 1
+		case 5:
+			log.CriticalWarning.PersistentMemoryRegionReadOnly = 1
+		}
+	}
+
+}
+
+// Log contstants
 const (
 	LogCdw10LogPageIdentiferMask         = 0xFF
 	LogCdw10LogPageIdentiferShift        = 0
@@ -1166,7 +1217,7 @@ const (
 )
 
 func (dev *Device) getNsidLog(logPageIdentifier uint8, retainAsynchronousEvent uint8, nsid uint32, buf []byte) error {
-	return dev.getLog(logPageIdentifier, 0, 0, nsid, 0, buf)
+	return dev.getLog(logPageIdentifier, 0, retainAsynchronousEvent, nsid, 0, buf)
 }
 
 func (dev *Device) getLog(logPageIdentifier uint8, logSpecificField uint8, retainAsynchronousEvent uint8, nsid uint32, logPageOffset uint64, buf []byte) error {
@@ -1200,3 +1251,26 @@ func (dev *Device) getLog(logPageIdentifier uint8, logSpecificField uint8, retai
 	return dev.ops.submitAdminPassthru(dev, &cmd, buf)
 
 }
+
+// LogSmartLog outputs the SMART log data in a structured way for logging.
+func LogSmartLog(log ec.Logger, smartLog *SmartLog, context ...interface{}) {
+	if smartLog == nil {
+		log.Error(nil, "SMART log is nil", context...)
+		return
+	}
+	log.Info("SMART log data",
+		append(context,
+			"criticalWarning.spareCapacity", smartLog.CriticalWarning.SpareCapacity,
+			"criticalWarning.temperature", smartLog.CriticalWarning.Temperature,
+			"criticalWarning.degraded", smartLog.CriticalWarning.Degraded,
+			"criticalWarning.readOnly", smartLog.CriticalWarning.ReadOnly,
+			"criticalWarning.backupFailed", smartLog.CriticalWarning.BackupFailed,
+			"criticalWarning.persistentMemoryRegionReadOnly", smartLog.CriticalWarning.PersistentMemoryRegionReadOnly,
+			"availableSpare", smartLog.AvailableSpare,
+			"availableSpareThreshold", smartLog.AvailableSpareThreshold,
+			"percentageUsed", smartLog.PercentageUsed,
+			"compositeTemperature", smartLog.CompositeTemperature,
+			"mediaErrorsLo", smartLog.MediaErrorsLo,
+			"numberErrorLogEntriesLo", smartLog.NumberErrorLogEntriesLo,
+		)...)
+}

pkg/manager-fabric/manager.go

@@ -1125,7 +1125,7 @@ func Start() error {
 	event.EventManager.Publish(msgreg.FabricReadyNnf(m.id))
 
 	// Run the Fabric Monitor in a background thread.
-	go NewFabricMonitor(m).Run()
+	StartFabricMonitor(m)
 
 	return nil
 }

pkg/manager-fabric/monitor.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -27,23 +27,53 @@ import (
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec"
 )
 
-// The Fabric Monitor is responsible for ensuring that the fabriic and related sub-resource
+// NewFabricMonitor - The Fabric Monitor is responsible for ensuring that the fabriic and related sub-resource
 // are updated with the latest information from the switch. This runs as a background
 // thread, and periodically queries the fabric.
-func NewFabricMonitor(f *Fabric) *monitor {
-	return &monitor{fabric: f}
+func NewMonitor(f *Fabric, i time.Duration) *monitor {
+	return &monitor{fabric: f, interval: i}
 }
 
 type monitor struct {
-	fabric *Fabric
+	fabric   *Fabric
+	interval time.Duration
+}
+
+// StartFabricMonitor starts the fabric monitor in a background goroutine if the period is non-zero.
+func StartFabricMonitor(fabric *Fabric) {
+	defaultFabricMonitorPeriod := 60 * time.Second
+	fabricMonitorPeriod := defaultFabricMonitorPeriod
+	if periodStr := os.Getenv("NNF_FABRIC_MONITOR_PERIOD"); periodStr != "" {
+		if d, err := time.ParseDuration(periodStr); err == nil {
+			fabricMonitorPeriod = d
+		} else {
+			if fabric != nil && !fabric.log.IsZero() {
+				fabric.log.Info("Invalid NNF_FABRIC_MONITOR_PERIOD, using default", "value", fabricMonitorPeriod, "error", err)
+			}
+		}
+	}
+
+	// A period of 0 means don't start the monitor.
+	if fabricMonitorPeriod == 0 {
+		if fabric != nil && !fabric.log.IsZero() {
+			fabric.log.Info("Not starting fabric monitor", "monitorPeriod", fabricMonitorPeriod)
+		}
+		return
+	}
+
+	mon := NewMonitor(fabric, fabricMonitorPeriod)
+	go mon.Run()
+	if fabric != nil && !fabric.log.IsZero() {
+		fabric.log.Info("Started fabric monitor", "monitorPeriod", fabricMonitorPeriod)
+	}
+
 }
 
 // Run Fabric Monitor forever
 func (m *monitor) Run() {
 
 	for {
-
-		time.Sleep(time.Second * 60)
+		time.Sleep(m.interval)
 
 		for idx := range m.fabric.switches {
 			s := &m.fabric.switches[idx]
@@ -62,13 +92,13 @@ func (m *monitor) Run() {
 					}
 
 					for _, event := range events {
-						physPortId, isDown := m.getEventInfo(event)
+						physPortID, isDown := m.getEventInfo(event)
 
-						if physPortId == invalidPhysicalPortId {
+						if physPortID == invalidPhysicalPortId {
 							continue
 						}
 
-						if p := s.findPortByPhysicalPortId(physPortId); p != nil {
+						if p := s.findPortByPhysicalPortId(physPortID); p != nil {
 							p.notify(isDown)
 						}
 					}

pkg/manager-nnf/manager.go

@@ -625,6 +625,8 @@ func (s *StorageService) EventHandler(e event.Event) error {
 		}
 
 		log.Info("Storage Service Enabled", "health", s.health)
+
+		nvme.StartNVMeMonitor(s.log)
 	}
 
 	// Check for storage pool events

pkg/manager-nvme/manager.go

@@ -1239,7 +1239,7 @@ func (mgr *Manager) StorageIdControllersControllerIdGet(storageId, controllerId
 		return ec.NewErrNotFound().WithError(err).WithCause(fmt.Sprintf("Storage Controller fabric endpoint not found: Storage: %s Controller: %s", storageId, controllerId))
 	}
 
-	percentageUsage, err := s.device.GetWearLevelAsPercentageUsed()
+	percentageUsage, err := GetWearLevelAsPercentageUsed(s.device)
 	if err != nil {
 		return ec.NewErrInternalServerError().WithError(err).WithCause(fmt.Sprintf("Storage Controller failed to retrieve SMART data: Storage: %s", storageId))
 	}

pkg/manager-nvme/monitor.go

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2025 Hewlett Packard Enterprise Development LP
+ * Other additional copyright holders may be indicated within.
+ *
+ * The entirety of this work is licensed under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ *
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nvme
+
+import (
+	"os"
+	"time"
+
+	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme"
+	"github.com/NearNodeFlash/nnf-ec/pkg/ec"
+	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
+)
+
+// StartNVMeMonitor starts a background goroutine that periodically queries all NVMe devices for their SMART log.
+func StartNVMeMonitor(log ec.Logger) {
+	// Read drive monitor period from environment variable
+	defaultDriveMonitorPeriod := 1 * time.Hour
+	driveMonitorPeriod := defaultDriveMonitorPeriod
+	if periodStr := os.Getenv("NNF_DRIVE_MONITOR_PERIOD"); periodStr != "" {
+		if d, err := time.ParseDuration(periodStr); err == nil {
+			driveMonitorPeriod = d
+		} else {
+			log.Info("Invalid NNF_DRIVE_MONITOR_PERIOD, using default", "value", driveMonitorPeriod, "error", err)
+		}
+	}
+
+	// A period of 0 means don't start the monitor.
+	if driveMonitorPeriod == 0 {
+		log.Info("Not starting NVMe monitor", "monitorPeriod", driveMonitorPeriod)
+		return
+	}
+
+	monitor := NewMonitor(log, driveMonitorPeriod)
+	go monitor.Run()
+	log.Info("Started NVMe monitor", "monitorPeriod", driveMonitorPeriod)
+}
+
+// Monitor periodically queries all NVMe devices for their SMART log
+// and can be started as a background goroutine.
+type Monitor struct {
+	Log        ec.Logger
+	Interval   time.Duration
+	GetDevices func() []*nvme.Device // Function to return all NVMe devices
+}
+
+// NewMonitor creates a new NVMe monitor with the given polling interval and device getter.
+func NewMonitor(log ec.Logger, interval time.Duration) *Monitor {
+	return &Monitor{
+		Log:      log,
+		Interval: interval,
+	}
+}
+
+// Run starts the monitor loop. It periodically calls GetSmartLog on all devices.
+func (m *Monitor) Run() {
+	for {
+		time.Sleep(m.Interval)
+
+		storages := GetStorage()
+		for _, storage := range storages {
+			if !storage.IsEnabled() {
+				continue
+			}
+
+			m.checkSmartLog(storage)
+		}
+	}
+}
+
+func (m *Monitor) checkSmartLog(storage *Storage) {
+	log := m.Log
+
+	smartLog, err := storage.device.GetSmartLog()
+	if err != nil {
+		log.Error(err, "smartlog request failed", "serial", storage.SerialNumber(), "slot", storage.Slot())
+		storage.state = sf.DISABLED_RST
+	}
+
+	// nvme.MangleSmartLog(smartLog)
+
+	state := nvme.InterpretSmartLog(smartLog)
+	if state != storage.state {
+		log.Info("smartlog state change", "old state", storage.state, "new state", state, "serial", storage.SerialNumber(), "slot", storage.Slot())
+		storage.state = state
+
+		nvme.LogSmartLog(log, smartLog)
+	}
+}