Skip to content

Commit b39a46d

Browse files
authored
Timeouts, Critical Warn & Testmode (#37)
On-behalf-of: SAP <filipp.akinfiev@clyso.com> Signed-off-by: Filipp Akinfiev <filipp.akinfiev@clyso.com>
1 parent 86c7151 commit b39a46d

27 files changed

Lines changed: 2740 additions & 117 deletions

pkg/commands/producer_disk_health_metrics.go

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ var (
3131
dhmReallocatedSectorsThreshold int64
3232
dhmLifetimeUsedThreshold int64
3333
dhmCephOSDBasePath string
34+
dhmTestMode bool
35+
dhmTestDataPath string
36+
dhmTestScenario string
37+
dhmTestDevices string
3438
)
3539

3640
var diskHealthMetricsCmd = &cobra.Command{
@@ -54,6 +58,14 @@ var diskHealthMetricsCmd = &cobra.Command{
5458
ReallocatedSectorsThreshold: dhmReallocatedSectorsThreshold,
5559
LifetimeUsedThreshold: dhmLifetimeUsedThreshold,
5660
CephOSDBasePath: dhmCephOSDBasePath,
61+
TestMode: dhmTestMode,
62+
TestDataPath: dhmTestDataPath,
63+
TestScenario: dhmTestScenario,
64+
}
65+
66+
// Parse test devices if provided
67+
if dhmTestDevices != "" {
68+
config.TestDevices = strings.Split(dhmTestDevices, ",")
5769
}
5870

5971
config = mergeDiskHealthMetricsConfigWithEnv(config)
@@ -104,6 +116,16 @@ func mergeDiskHealthMetricsConfigWithEnv(cfg diskhealthmetrics.DiskHealthMetrics
104116
cfg.ReallocatedSectorsThreshold = getEnvInt64("REALLOCATED_SECTORS_THRESHOLD", cfg.ReallocatedSectorsThreshold)
105117
cfg.LifetimeUsedThreshold = getEnvInt64("LIFETIME_USED_THRESHOLD", cfg.LifetimeUsedThreshold)
106118
cfg.CephOSDBasePath = getEnv("CEPH_OSD_BASE_PATH", cfg.CephOSDBasePath)
119+
120+
// Test mode environment variables
121+
cfg.TestMode = getEnvBool("TEST_MODE", cfg.TestMode)
122+
cfg.TestDataPath = getEnv("TEST_DATA_PATH", cfg.TestDataPath)
123+
cfg.TestScenario = getEnv("TEST_SCENARIO", cfg.TestScenario)
124+
125+
testDevicesEnv := getEnv("TEST_DEVICES", "")
126+
if testDevicesEnv != "" {
127+
cfg.TestDevices = strings.Split(testDevicesEnv, ",")
128+
}
107129

108130
return cfg
109131
}
@@ -122,13 +144,20 @@ func init() {
122144
diskHealthMetricsCmd.Flags().Int64Var(&dhmReallocatedSectorsThreshold, "reallocated-sectors-threshold", 10, "Threshold for reallocated sectors to trigger a warning")
123145
diskHealthMetricsCmd.Flags().Int64Var(&dhmLifetimeUsedThreshold, "lifetime-used-threshold", 80, "Threshold for SSD lifetime used percentage to trigger a critical alert")
124146
diskHealthMetricsCmd.Flags().StringVar(&dhmCephOSDBasePath, "ceph-osd-base-path", "/var/lib/rook/rook-ceph/", "Base path for mapping devices to Ceph OSD numbers")
147+
148+
// Test mode flags
149+
diskHealthMetricsCmd.Flags().BoolVar(&dhmTestMode, "test-mode", false, "Enable test mode with simulated data (no smartctl required)")
150+
diskHealthMetricsCmd.Flags().StringVar(&dhmTestDataPath, "test-data-path", "", "Path to test data directory (default: pkg/producers/diskhealthmetrics/testdata)")
151+
diskHealthMetricsCmd.Flags().StringVar(&dhmTestScenario, "test-scenario", "mixed", "Test scenario: healthy, failing, mixed")
152+
diskHealthMetricsCmd.Flags().StringVar(&dhmTestDevices, "test-devices", "", "Comma-separated list of test device names (default: nvme0,nvme1,sda,sdb)")
125153
}
126154

127155
func validateDiskHealthMetricsConfig(config diskhealthmetrics.DiskHealthMetricsConfig) {
128156
missingParams := false
129157

130-
if len(config.Disks) == 0 {
131-
fmt.Println("Warning: --disks or DISKS must be set")
158+
// In test mode, disks are optional (will use default test devices)
159+
if !config.TestMode && len(config.Disks) == 0 {
160+
fmt.Println("Warning: --disks or DISKS must be set (or use --test-mode)")
132161
missingParams = true
133162
}
134163

pkg/commands/producer_ops_log.go

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ var (
7777
opsTrackErrorsPerTenant bool
7878
opsTrackErrorsPerStatus bool
7979
opsTrackErrorsByIP bool
80+
opsTrackTimeoutErrors bool
81+
opsTrackErrorsByCategory bool
8082

8183
// IP-based metrics flags
8284
opsTrackRequestsByIPDetailed bool
@@ -178,11 +180,13 @@ Following this configuration change, the RadosGW will log operations to the file
178180
TrackBytesReceivedPerTenant: opsTrackBytesReceivedPerTenant,
179181

180182
// Error metrics
181-
TrackErrorsDetailed: opsTrackErrorsDetailed,
182-
TrackErrorsPerUser: opsTrackErrorsPerUser,
183-
TrackErrorsPerBucket: opsTrackErrorsPerBucket,
184-
TrackErrorsPerTenant: opsTrackErrorsPerTenant,
185-
TrackErrorsPerStatus: opsTrackErrorsPerStatus,
183+
TrackErrorsDetailed: opsTrackErrorsDetailed,
184+
TrackErrorsPerUser: opsTrackErrorsPerUser,
185+
TrackErrorsPerBucket: opsTrackErrorsPerBucket,
186+
TrackErrorsPerTenant: opsTrackErrorsPerTenant,
187+
TrackErrorsPerStatus: opsTrackErrorsPerStatus,
188+
TrackTimeoutErrors: opsTrackTimeoutErrors,
189+
TrackErrorsByCategory: opsTrackErrorsByCategory,
186190

187191
// IP-based metrics
188192
TrackRequestsByIPDetailed: opsTrackRequestsByIPDetailed,
@@ -591,6 +595,8 @@ func mergeOpsLogConfigWithEnv(cfg opslog.OpsLogConfig) opslog.OpsLogConfig {
591595
cfg.MetricsConfig.TrackErrorsPerTenant = getEnvBool("TRACK_ERRORS_PER_TENANT", cfg.MetricsConfig.TrackErrorsPerTenant)
592596
cfg.MetricsConfig.TrackErrorsPerStatus = getEnvBool("TRACK_ERRORS_PER_STATUS", cfg.MetricsConfig.TrackErrorsPerStatus)
593597
cfg.MetricsConfig.TrackErrorsByIP = getEnvBool("TRACK_ERRORS_BY_IP", cfg.MetricsConfig.TrackErrorsByIP)
598+
cfg.MetricsConfig.TrackTimeoutErrors = getEnvBool("TRACK_TIMEOUT_ERRORS", cfg.MetricsConfig.TrackTimeoutErrors)
599+
cfg.MetricsConfig.TrackErrorsByCategory = getEnvBool("TRACK_ERRORS_BY_CATEGORY", cfg.MetricsConfig.TrackErrorsByCategory)
594600

595601
// IP-based metrics
596602
cfg.MetricsConfig.TrackRequestsByIPDetailed = getEnvBool("TRACK_REQUESTS_BY_IP_DETAILED", cfg.MetricsConfig.TrackRequestsByIPDetailed)
@@ -679,6 +685,8 @@ func init() {
679685
opsLogCmd.Flags().BoolVar(&opsTrackErrorsPerBucket, "track-errors-per-bucket", false, "Track errors per bucket")
680686
opsLogCmd.Flags().BoolVar(&opsTrackErrorsPerTenant, "track-errors-per-tenant", false, "Track errors per tenant")
681687
opsLogCmd.Flags().BoolVar(&opsTrackErrorsPerStatus, "track-errors-per-status", false, "Track errors per HTTP status")
688+
opsLogCmd.Flags().BoolVar(&opsTrackTimeoutErrors, "track-timeout-errors", false, "Track timeout errors (408, 504, 598, 499) separately for OSD issues")
689+
opsLogCmd.Flags().BoolVar(&opsTrackErrorsByCategory, "track-errors-by-category", false, "Track errors by category (timeout, connection, client, server)")
682690

683691
// IP-based metrics
684692
opsLogCmd.Flags().BoolVar(&opsTrackRequestsByIPDetailed, "track-requests-by-ip-detailed", false, "Track requests by IP")

pkg/producers/diskhealthmetrics/README.md

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ Prometheus metrics for monitoring dashboards and NATS subjects for alerting.
1515
and consistent data representation across various drives in your environment.
1616
- **Device Info Normalization**: Standardizes device information such as product name, capacity,
1717
vendor, and media type to maintain consistency across different systems and databases.
18+
- **Enhanced NVMe Support**:
19+
- Exposes critical NVMe-specific attributes including `critical_warning` bitfield for early failure detection
20+
- Vendor IDs displayed in standard hexadecimal format (e.g., "0x144D" for Samsung)
21+
- Available spare percentage and threshold monitoring
22+
- Media error tracking and error log entry counts
1823
- **Ceph OSD Integration**: Automatically maps physical disk devices to their corresponding Ceph OSD
1924
numbers, providing enhanced observability for Ceph storage clusters. Works seamlessly with both
2025
direct block devices and LVM logical volumes.
@@ -54,14 +59,48 @@ monitoring and alerting based on specific OSD performance and health metrics.
5459
All metrics include standard labels (`disk`, `node`, `instance`) and an optional `osd_id` label when
5560
Ceph integration is enabled:
5661

57-
- **smart_attributes**: Gauges various SMART attributes of the disk.
58-
- **disk_temperature_celsius**: Monitors disk temperature in Celsius.
59-
- **disk_reallocated_sectors**: Tracks the number of reallocated sectors.
60-
- **disk_pending_sectors**: Monitors the number of pending sectors.
61-
- **disk_power_on_hours_total**: Reports the cumulative number of hours the disk has been powered on.
62-
- **ssd_life_used_percentage**: Indicates the percentage of SSD life used.
63-
- **disk_error_counts_total**: Tracks various error counts for the disk.
64-
- **disk_capacity_gb**: Reports the capacity of the disk in GB.
62+
### Core Metrics
63+
- **smart_attributes**: Gauges various SMART attributes of the disk with `attribute` label
64+
- Includes NVMe-specific attributes like `critical_warning`, `available_spare`, `available_spare_threshold`
65+
- Vendor IDs stored as decimal values (e.g., `nvme_vendor_id`, `nvme_subsystem_vendor_id`)
66+
- **disk_temperature_celsius**: Monitors disk temperature in Celsius
67+
- **disk_reallocated_sectors**: Tracks the number of reallocated sectors
68+
- **disk_pending_sectors**: Monitors the number of pending sectors
69+
- **disk_power_on_hours_total**: Reports the cumulative number of hours the disk has been powered on
70+
- **ssd_life_used_percentage**: Indicates the percentage of SSD life used (normalized from various wear indicators)
71+
- **disk_error_counts_total**: Tracks various error counts for the disk with `error_type` label
72+
- **disk_capacity_gb**: Reports the capacity of the disk in GB
73+
74+
### Device Information Metric
75+
- **disk_info**: Static information about the disk device (value always 1) with labels:
76+
- `vendor`: Vendor/manufacturer name
77+
- `vendor_id`: NVMe Vendor ID in hex format (e.g., "0x144D" for Samsung)
78+
- `subsystem_vendor_id`: NVMe Subsystem Vendor ID in hex format
79+
- `model`: Device model
80+
- `serial_number`: Device serial number
81+
- `firmware_version`: Firmware version
82+
- `product`: Product name
83+
- `model_family`: Model family (for ATA devices)
84+
- `capacity_gb`: Capacity in GB
85+
- `media_type`: Media type (ssd, hdd, nvme)
86+
- `form_factor`: Physical form factor
87+
- `rpm`: Rotational speed (for HDDs)
88+
- `dwpd`: Drive Writes Per Day (for SSDs)
89+
90+
## NVMe Critical Warning Interpretation
91+
92+
The `critical_warning` attribute in `smart_attributes` is a bitfield that provides early warning for NVMe drive issues:
93+
94+
| Value | Meaning |
95+
|-------|---------|
96+
| 0x00 | No critical warnings (drive is healthy) |
97+
| 0x01 | Available spare space has fallen below threshold |
98+
| 0x02 | Temperature exceeded threshold |
99+
| 0x04 | NVM subsystem reliability has been degraded |
100+
| 0x08 | Drive is in read-only mode |
101+
| 0x10 | Volatile memory backup device has failed |
102+
103+
Multiple bits can be set simultaneously. For example, 0x06 means both temperature exceeded (0x02) and reliability degraded (0x04).
65104

66105
## Alerts and Thresholds
67106

@@ -127,6 +166,14 @@ volumes:
127166
type: Directory
128167
```
129168
169+
## Recent Improvements
170+
171+
### Version Updates
172+
- **NVMe Critical Warning Support**: Added exposure of the `critical_warning` bitfield for early failure detection
173+
- **Hexadecimal Vendor IDs**: Vendor IDs and Subsystem Vendor IDs now displayed in standard hex format (e.g., "0x144D")
174+
- **Enhanced Device Info Metric**: Added `disk_info` metric with comprehensive device metadata labels
175+
- **Improved NVMe Integration**: Better support for NVMe-specific attributes from both smartctl and nvme-cli
176+
130177
## Acknowledgment
131178

132179
The development of the local-producer disk-health-metrics was greatly supported by insights and

pkg/producers/diskhealthmetrics/config.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,10 @@ type DiskHealthMetricsConfig struct {
2424
LifetimeUsedThreshold int64 // percentage
2525

2626
CephOSDBasePath string
27+
28+
// Test mode configuration
29+
TestMode bool // Enable test mode with simulated data
30+
TestDataPath string // Path to test data directory
31+
TestScenario string // Test scenario name (e.g., "healthy", "failing", "mixed")
32+
TestDevices []string // Simulated device names in test mode
2733
}

pkg/producers/diskhealthmetrics/diskhealthmetrics.go

Lines changed: 100 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ package diskhealthmetrics
77
import (
88
"encoding/json"
99
"fmt"
10+
"os"
11+
"path/filepath"
1012
"time"
1113

1214
"github.com/nats-io/nats.go"
@@ -16,6 +18,11 @@ import (
1618
func collectDiskHealthMetrics(cfg DiskHealthMetricsConfig) []NormalizedSmartData {
1719
var allMetrics []NormalizedSmartData
1820

21+
// Check for test mode
22+
if cfg.TestMode {
23+
return collectTestDiskHealthMetrics(cfg)
24+
}
25+
1926
// Check if nvme-cli is available for enhanced NVMe support
2027
nvmeCliAvailable := checkNVMeCliInstalled()
2128
if nvmeCliAvailable {
@@ -73,6 +80,59 @@ func collectDiskHealthMetrics(cfg DiskHealthMetricsConfig) []NormalizedSmartData
7380
return allMetrics
7481
}
7582

83+
// collectTestDiskHealthMetrics collects metrics from test data files
84+
func collectTestDiskHealthMetrics(cfg DiskHealthMetricsConfig) []NormalizedSmartData {
85+
var allMetrics []NormalizedSmartData
86+
87+
// Determine test data path
88+
scenarioPath := filepath.Join(cfg.TestDataPath, "scenarios", cfg.TestScenario)
89+
90+
for _, device := range cfg.Disks {
91+
jsonFile := filepath.Join(scenarioPath, device + ".json")
92+
93+
// Check if file exists
94+
if _, err := os.Stat(jsonFile); os.IsNotExist(err) {
95+
log.Warn().Str("device", device).Str("file", jsonFile).Msg("Test data file not found, skipping")
96+
continue
97+
}
98+
99+
// Load test data
100+
rawData, err := collectSmartDataFromFile(jsonFile)
101+
if err != nil {
102+
log.Error().Err(err).Str("file", jsonFile).Msg("Error loading test data")
103+
continue
104+
}
105+
106+
// Override device name to match test device
107+
rawData.Device.Name = "/dev/" + device
108+
rawData.Device.InfoName = "/dev/" + device
109+
110+
// Process as normal
111+
deviceInfo := &DeviceInfo{}
112+
FillDeviceInfoFromSmartData(deviceInfo, rawData)
113+
NormalizeVendor(deviceInfo)
114+
NormalizeDeviceInfo(deviceInfo)
115+
116+
smartAttrs := GetSmartAttributes()
117+
ProcessAndUpdateSmartAttributes(smartAttrs, rawData)
118+
CleanupSmartAttributes(smartAttrs)
119+
120+
normalizedData := normalizeSmartData(rawData, deviceInfo, smartAttrs,
121+
cfg.NodeName, cfg.InstanceID, cfg.CephOSDBasePath)
122+
123+
// Add a note in the log that this is test data
124+
log.Debug().
125+
Str("device", device).
126+
Str("scenario", cfg.TestScenario).
127+
Interface("attributes", smartAttrs).
128+
Msg("Processed test device data")
129+
130+
allMetrics = append(allMetrics, normalizedData)
131+
}
132+
133+
return allMetrics
134+
}
135+
76136
// normalizeSmartData normalizes the raw SMART data
77137
func normalizeSmartData(smartData *SmartCtlOutput, deviceInfo *DeviceInfo, attributes map[string]SmartAttribute, nodeName, instanceID, basePath string) NormalizedSmartData {
78138
var temperatureCelsius *int64
@@ -99,9 +159,15 @@ func normalizeSmartData(smartData *SmartCtlOutput, deviceInfo *DeviceInfo, attri
99159

100160
// Initialize device-specific attributes
101161
if smartData.Device.Protocol == "ATA" && smartData.ATASMARTAttributes != nil {
102-
reallocatedSectors = &findSmartAttributeByID(smartData.ATASMARTAttributes.Table, 5).Raw.Value
103-
pendingSectors = &findSmartAttributeByID(smartData.ATASMARTAttributes.Table, 197).Raw.Value
104-
udmaCrcErrorCount = findSmartAttributeByID(smartData.ATASMARTAttributes.Table, 199).Raw.Value
162+
if attr := findSmartAttributeByID(smartData.ATASMARTAttributes.Table, 5); attr != nil {
163+
reallocatedSectors = &attr.Raw.Value
164+
}
165+
if attr := findSmartAttributeByID(smartData.ATASMARTAttributes.Table, 197); attr != nil {
166+
pendingSectors = &attr.Raw.Value
167+
}
168+
if attr := findSmartAttributeByID(smartData.ATASMARTAttributes.Table, 199); attr != nil {
169+
udmaCrcErrorCount = attr.Raw.Value
170+
}
105171
powerOnHours = &smartData.PowerOnTime.Hours
106172
} else if smartData.Device.Protocol == "SCSI" && smartData.SCSIStartStopCycleCounter != nil {
107173
powerOnHours = &smartData.PowerOnTime.Hours
@@ -141,20 +207,42 @@ func findSmartAttributeByID(attributes []SmartCtlATASMARTEntry, id int64) *Smart
141207
}
142208

143209
func StartMonitoring(cfg DiskHealthMetricsConfig) {
144-
if !checkSmartctlInstalled() {
210+
// Skip smartctl check in test mode
211+
if !cfg.TestMode && !checkSmartctlInstalled() {
145212
log.Fatal().Msg("smartctl is not installed. please install smartmontools package.")
146213
}
147214

148-
// Discover devices if wildcard (*) is used in the configuration.
149-
if len(cfg.Disks) == 1 && cfg.Disks[0] == "*" {
150-
devices, err := discoverDevices()
151-
if err != nil {
152-
log.Fatal().Err(err).Msg("Error discovering devices")
215+
// Handle test mode setup
216+
if cfg.TestMode {
217+
// Use default test devices if none specified
218+
if len(cfg.TestDevices) == 0 {
219+
cfg.TestDevices = []string{"nvme0", "nvme1", "sda", "sdb"}
153220
}
221+
cfg.Disks = cfg.TestDevices
222+
223+
// Set default test data path if not specified
224+
if cfg.TestDataPath == "" {
225+
cfg.TestDataPath = "pkg/producers/diskhealthmetrics/testdata"
226+
}
227+
228+
log.Info().
229+
Bool("test_mode", true).
230+
Str("test_scenario", cfg.TestScenario).
231+
Str("test_data_path", cfg.TestDataPath).
232+
Strs("test_devices", cfg.TestDevices).
233+
Msg("Running in test mode with simulated data")
234+
} else {
235+
// Discover devices if wildcard (*) is used in the configuration.
236+
if len(cfg.Disks) == 1 && cfg.Disks[0] == "*" {
237+
devices, err := discoverDevices()
238+
if err != nil {
239+
log.Fatal().Err(err).Msg("Error discovering devices")
240+
}
154241

155-
cfg.Disks = make([]string, len(devices.Devices))
156-
for i, device := range devices.Devices {
157-
cfg.Disks[i] = device.Name
242+
cfg.Disks = make([]string, len(devices.Devices))
243+
for i, device := range devices.Devices {
244+
cfg.Disks[i] = device.Name
245+
}
158246
}
159247
}
160248

pkg/producers/diskhealthmetrics/normalize.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,9 @@ func FillDeviceInfoFromSmartData(deviceInfo *DeviceInfo, smartData *SmartCtlOutp
6666
// NVMe-specific processing
6767
// Use the ID and SubsystemID to represent the vendor, or leave blank if unavailable
6868
if smartData.NVMePCIVendor != nil {
69-
deviceInfo.Vendor = fmt.Sprintf("Vendor ID: %d, Subsystem ID: %d", smartData.NVMePCIVendor.ID, smartData.NVMePCIVendor.SubsystemID)
69+
deviceInfo.Vendor = fmt.Sprintf("Vendor ID: 0x%04X, Subsystem ID: 0x%04X", smartData.NVMePCIVendor.ID, smartData.NVMePCIVendor.SubsystemID)
70+
deviceInfo.VendorID = fmt.Sprintf("0x%04X", smartData.NVMePCIVendor.ID)
71+
deviceInfo.SubsystemVendorID = fmt.Sprintf("0x%04X", smartData.NVMePCIVendor.SubsystemID)
7072
}
7173
deviceInfo.Product = smartData.DeviceModel
7274
deviceInfo.Media = "nvme"

0 commit comments

Comments
 (0)