Refactor NVMe API to provide the SMART log page data

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>

Author: ajfloeder

internal/switchtec/pkg/nvme/nvme.go

@@ -22,6 +22,7 @@ package nvme
 import (
 	"encoding/binary"
 	"fmt"
+	"math/rand"
 	"path"
 	"regexp"
 	"strconv"
@@ -33,6 +34,7 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec"
+	"github.com/NearNodeFlash/nnf-ec/pkg/ec"
 	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
 )
 
@@ -1169,6 +1171,31 @@ func InterpretSmartLog(log *SmartLog) sf.ResourceState {
 	return sf.ENABLED_RST // Healthy
 }
 
+func MangleSmartLog(log *SmartLog) {
+
+	// Occaionally managle the smart log data
+	if rand.Intn(100) < 10 { // 10% chance of simulating critical condition
+		warningType := rand.Intn(6)
+		switch warningType {
+		case 0:
+			log.CriticalWarning.SpareCapacity = 1
+			log.AvailableSpare = 5 // Below threshold
+		case 1:
+			log.CriticalWarning.Temperature = 1
+			log.CompositeTemperature = 358 // ~85°C
+		case 2:
+			log.CriticalWarning.Degraded = 1
+		case 3:
+			log.CriticalWarning.ReadOnly = 1
+		case 4:
+			log.CriticalWarning.BackupFailed = 1
+		case 5:
+			log.CriticalWarning.PersistentMemoryRegionReadOnly = 1
+		}
+	}
+
+}
+
 // Log contstants
 const (
 	LogCdw10LogPageIdentiferMask         = 0xFF
@@ -1224,3 +1251,26 @@ func (dev *Device) getLog(logPageIdentifier uint8, logSpecificField uint8, retai
 	return dev.ops.submitAdminPassthru(dev, &cmd, buf)
 
 }
+
+// LogSmartLog outputs the SMART log data in a structured way for logging.
+func LogSmartLog(log ec.Logger, smartLog *SmartLog, context ...interface{}) {
+	if smartLog == nil {
+		log.Error(nil, "SMART log is nil", context...)
+		return
+	}
+	log.Info("SMART log data",
+		append(context,
+			"criticalWarning.spareCapacity", smartLog.CriticalWarning.SpareCapacity,
+			"criticalWarning.temperature", smartLog.CriticalWarning.Temperature,
+			"criticalWarning.degraded", smartLog.CriticalWarning.Degraded,
+			"criticalWarning.readOnly", smartLog.CriticalWarning.ReadOnly,
+			"criticalWarning.backupFailed", smartLog.CriticalWarning.BackupFailed,
+			"criticalWarning.persistentMemoryRegionReadOnly", smartLog.CriticalWarning.PersistentMemoryRegionReadOnly,
+			"availableSpare", smartLog.AvailableSpare,
+			"availableSpareThreshold", smartLog.AvailableSpareThreshold,
+			"percentageUsed", smartLog.PercentageUsed,
+			"compositeTemperature", smartLog.CompositeTemperature,
+			"mediaErrorsLo", smartLog.MediaErrorsLo,
+			"numberErrorLogEntriesLo", smartLog.NumberErrorLogEntriesLo,
+		)...)
+}

pkg/manager-nvme/manager.go

@@ -1239,7 +1239,7 @@ func (mgr *Manager) StorageIdControllersControllerIdGet(storageId, controllerId
 		return ec.NewErrNotFound().WithError(err).WithCause(fmt.Sprintf("Storage Controller fabric endpoint not found: Storage: %s Controller: %s", storageId, controllerId))
 	}
 
-	percentageUsage, err := s.device.GetWearLevelAsPercentageUsed()
+	percentageUsage, err := GetWearLevelAsPercentageUsed(s.device)
 	if err != nil {
 		return ec.NewErrInternalServerError().WithError(err).WithCause(fmt.Sprintf("Storage Controller failed to retrieve SMART data: Storage: %s", storageId))
 	}

pkg/manager-nvme/monitor.go

@@ -70,27 +70,36 @@ func NewMonitor(log ec.Logger, interval time.Duration) *Monitor {
 
 // Run starts the monitor loop. It periodically calls GetSmartLog on all devices.
 func (m *Monitor) Run() {
-	log := m.Log
 	for {
 		time.Sleep(m.Interval)
 
 		storages := GetStorage()
-		// log.V(2).Info("Drive monitor tick", "period", m.Interval, "device count", len(storages))
 		for _, storage := range storages {
 			if !storage.IsEnabled() {
-				// log.V(2).Info("storage disabled", "slot", storage.Slot())
 				continue
 			}
 
-			state, err := storage.device.CheckSmartLogForStatus()
-			if err != nil {
-				log.Error(err, "smartlog request failed", "serial", storage.SerialNumber(), "slot", storage.Slot())
-				storage.state = sf.DISABLED_RST
-			}
-			if state != storage.state {
-				log.Info("smartlog state change, (see 'nnf-nvme.sh cmd smartlog' for details)", "old state", storage.state, "new state", state, "serial", storage.SerialNumber(), "slot", storage.Slot())
-				storage.state = state
-			}
+			m.checkSmartLog(storage)
 		}
 	}
 }
+
+func (m *Monitor) checkSmartLog(storage *Storage) {
+	log := m.Log
+
+	smartLog, err := storage.device.GetSmartLog()
+	if err != nil {
+		log.Error(err, "smartlog request failed", "serial", storage.SerialNumber(), "slot", storage.Slot())
+		storage.state = sf.DISABLED_RST
+	}
+
+	// nvme.MangleSmartLog(smartLog)
+
+	state := nvme.InterpretSmartLog(smartLog)
+	if state != storage.state {
+		log.Info("smartlog state change", "old state", storage.state, "new state", state, "serial", storage.SerialNumber(), "slot", storage.Slot())
+		storage.state = state
+
+		nvme.LogSmartLog(log, smartLog)
+	}
+}

pkg/manager-nvme/nvme_api.go

@@ -21,7 +21,6 @@ package nvme
 
 import (
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme"
-	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
 )
 
 type NvmeController interface {
@@ -65,9 +64,8 @@ type NvmeDeviceApi interface {
 	SetNamespaceFeature(namespaceId nvme.NamespaceIdentifier, data []byte) error
 	GetNamespaceFeature(namespaceId nvme.NamespaceIdentifier) ([]byte, error)
 
-	GetWearLevelAsPercentageUsed() (uint8, error)
-
-	CheckSmartLogForStatus() (sf.ResourceState, error)
+	// GetSmartLog returns the raw SMART log page data
+	GetSmartLog() (*nvme.SmartLog, error)
 }
 
 // SecondaryControllersInitFunc -
@@ -83,3 +81,12 @@ const (
 	VQResourceType SecondaryControllerResourceType = iota
 	VIResourceType
 )
+
+// GetWearLevelAsPercentageUsed returns the PercentageUsed field from the SMART log page.
+func GetWearLevelAsPercentageUsed(dev NvmeDeviceApi) (uint8, error) {
+	smartLog, err := dev.GetSmartLog()
+	if err != nil {
+		return 0, err
+	}
+	return smartLog.PercentageUsed, nil
+}

pkg/manager-nvme/nvme_cli.go

@@ -33,7 +33,6 @@ import (
 
 	"github.com/NearNodeFlash/nnf-ec/pkg/logging"
 	fabric "github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric"
-	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
 )
 
 func NewCliNvmeController() NvmeController {
@@ -376,33 +375,20 @@ func (*cliDevice) GetNamespaceFeature(namespaceId nvme.NamespaceIdentifier) ([]b
 	return nil, nil
 }
 
-func (d *cliDevice) GetWearLevelAsPercentageUsed() (uint8, error) {
+// GetSmartLog returns the raw SMART log page data
+func (d *cliDevice) GetSmartLog() (*nvme.SmartLog, error) {
 	rsp, err := d.run(fmt.Sprintf("smart-log %s --output-format=binary", d.dev()))
 	if err != nil {
-		return 0, err
-	}
-
-	log := new(nvme.SmartLog)
-
-	err = structex.DecodeByteBuffer(bytes.NewBuffer([]byte(rsp)), log)
-
-	return log.PercentageUsed, err
-}
-
-func (d *cliDevice) CheckSmartLogForStatus() (sf.ResourceState, error) {
-	rsp, err := d.run(fmt.Sprintf("smart-log %s --output-format=binary", d.dev()))
-	if err != nil {
-		return sf.DISABLED_RST, err
+		return nil, err
 	}
 
 	log := new(nvme.SmartLog)
-
 	err = structex.DecodeByteBuffer(bytes.NewBuffer([]byte(rsp)), log)
 	if err != nil {
-		return sf.DISABLED_RST, err
+		return nil, err
 	}
 
-	return nvme.InterpretSmartLog(log), err
+	return log, nil
 }
 
 func (d *cliDevice) run(cmd string) (string, error) {

pkg/manager-nvme/nvme_device.go

@@ -27,7 +27,6 @@ import (
 	fabric "github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric"
 
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme"
-	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
 )
 
 type SwitchtecNvmeController struct{}
@@ -248,22 +247,7 @@ func (d *nvmeDevice) GetNamespaceFeature(namespaceId nvme.NamespaceIdentifier) (
 	return buf, nil
 }
 
-func (d *nvmeDevice) GetWearLevelAsPercentageUsed() (uint8, error) {
-
-	log, err := d.dev.GetSmartLog()
-	if err != nil {
-		return 0, err
-	}
-
-	return log.PercentageUsed, nil
-}
-
-func (d *nvmeDevice) CheckSmartLogForStatus() (sf.ResourceState, error) {
-
-	sl, err := d.dev.GetSmartLog()
-	if err != nil {
-		return sf.DISABLED_RST, err
-	}
-
-	return nvme.InterpretSmartLog(sl), nil
+// GetSmartLog returns the raw SMART log page data
+func (d *nvmeDevice) GetSmartLog() (*nvme.SmartLog, error) {
+	return d.dev.GetSmartLog()
 }

pkg/manager-nvme/nvme_direct_device.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme"
 	fabric "github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric"
-	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
 )
 
 type DirectNvmeController struct {
@@ -143,10 +142,7 @@ func (*nvmeDirectDevice) SetNamespaceFeature(namespaceId nvme.NamespaceIdentifie
 	panic("unimplemented")
 }
 
-func (d *nvmeDirectDevice) GetWearLevelAsPercentageUsed() (uint8, error) {
-	return d.cliDevice.GetWearLevelAsPercentageUsed()
-}
-
-func (d *nvmeDirectDevice) CheckSmartLogForStatus() (sf.ResourceState, error) {
-	return d.cliDevice.CheckSmartLogForStatus()
+// GetSmartLog returns the raw SMART log page data
+func (d *nvmeDirectDevice) GetSmartLog() (*nvme.SmartLog, error) {
+	return d.cliDevice.GetSmartLog()
 }

pkg/manager-nvme/nvme_mock.go

@@ -28,7 +28,6 @@ import (
 	"strings"
 
 	"github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme"
-	sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models"
 )
 
 type MockNvmeController struct {
@@ -445,10 +444,6 @@ func (d *mockDevice) GetNamespaceFeature(namespaceId nvme.NamespaceIdentifier) (
 	return ns.metadata, nil
 }
 
-func (*mockDevice) GetWearLevelAsPercentageUsed() (uint8, error) {
-	return 0, nil
-}
-
 func (d *mockDevice) findNamespace(namespaceId nvme.NamespaceIdentifier) *mockNamespace {
 
 	for idx, ns := range d.namespaces {
@@ -460,6 +455,32 @@ func (d *mockDevice) findNamespace(namespaceId nvme.NamespaceIdentifier) *mockNa
 	return nil
 }
 
-func (*mockDevice) CheckSmartLogForStatus() (sf.ResourceState, error) {
-	return sf.ENABLED_RST, nil
+// GetSmartLog returns mock SMART log page data
+func (d *mockDevice) GetSmartLog() (*nvme.SmartLog, error) {
+	// Create a mock SMART log with typical healthy values
+	log := &nvme.SmartLog{
+		CriticalWarning: struct {
+			SpareCapacity                  uint8 `bitfield:"1"`
+			Temperature                    uint8 `bitfield:"1"`
+			Degraded                       uint8 `bitfield:"1"`
+			ReadOnly                       uint8 `bitfield:"1"`
+			BackupFailed                   uint8 `bitfield:"1"`
+			PersistentMemoryRegionReadOnly uint8 `bitfield:"1"`
+			Reserved                       uint8 `bitfield:"2"`
+		}{
+			SpareCapacity:                  0,
+			Temperature:                    0,
+			Degraded:                       0,
+			ReadOnly:                       0,
+			BackupFailed:                   0,
+			PersistentMemoryRegionReadOnly: 0,
+			Reserved:                       0,
+		},
+		CompositeTemperature:    308, // ~35°C (273 + 35)
+		AvailableSpare:          95,  // 95% spare capacity available
+		AvailableSpareThreshold: 10,  // Warning at 10%
+		PercentageUsed:          25,  // 25% of device life used
+	}
+
+	return log, nil
 }