Merge pull request #26 from NETWAYS/filesystem_timeout

Add timeout for reading filesystem usage to exit properly with hanging filesystems.
NETWAYS · Mar 6, 2024 · af89c79 · af89c79
2 parents f2701ce + 1bd01ca
commit af89c79
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 32 deletions.
diff --git a/cmd/filesystem.go b/cmd/filesystem.go
@@ -1,9 +1,11 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"strconv"
+	"time"
 
 	"github.com/NETWAYS/check_system_basics/internal/common/thresholds"
 	"github.com/NETWAYS/check_system_basics/internal/filesystem"
@@ -137,7 +139,10 @@ var diskCmd = &cobra.Command{
 		}
 
 		// Retrieve stats
-		err = filesystem.GetDiskUsage(filesystemList, &FsConfig)
+		internalTimeout := time.Duration(Timeout/2) * time.Second
+		ctx := context.Background()
+
+		err = filesystem.GetDiskUsage(ctx, internalTimeout, filesystemList, &FsConfig)
 		if err != nil {
 			check.ExitError(err)
 		}
@@ -146,8 +151,9 @@ var diskCmd = &cobra.Command{
 		for index := range filesystemList {
 			sc := computeFsCheckResult(&filesystemList[index], &FsConfig)
 
-			sc.Output = fmt.Sprintf("%s (%.2f%% used space, %.2f%% free inodes)", sc.Output, filesystemList[index].UsageStats.UsedPercent, 100-filesystemList[index].UsageStats.InodesUsedPercent)
-
+			if filesystemList[index].Error == nil {
+				sc.Output = fmt.Sprintf("%s (%.2f%% used space, %.2f%% free inodes)", sc.Output, filesystemList[index].UsageStats.UsedPercent, 100-filesystemList[index].UsageStats.InodesUsedPercent)
+			}
 			overall.AddSubcheck(sc)
 		}
 

diff --git a/internal/common/filter/filter.go b/internal/common/filter/filter.go
@@ -5,7 +5,7 @@ import (
 )
 
 type Filterable interface {
-	GetFilterableValue(uint) string
+	GetFilterableValue(val uint) string
 }
 
 type Options struct {

diff --git a/internal/filesystem/filesystem.go b/internal/filesystem/filesystem.go
@@ -1,9 +1,10 @@
 package filesystem
 
 import (
+	"context"
 	"errors"
 	"regexp"
-	"syscall"
+	"time"
 
 	"github.com/shirou/gopsutil/v3/disk"
 )
@@ -15,16 +16,43 @@ type FilesystemType struct {
 	Error      error
 }
 
-func GetDiskUsage(fsList []FilesystemType, _ *CheckConfig) error {
-	for index := range fsList {
-		diskUsage, err := disk.Usage(fsList[index].PartStats.Mountpoint)
+type tmpFileSystemWrapper struct {
+	usage disk.UsageStat
+	err   error
+}
+
+func GetDiskUsageSingle(ctx context.Context, timeout time.Duration, fs *FilesystemType) {
+	myCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	resChan := make(chan tmpFileSystemWrapper, 1)
+
+	go func() {
+		tmp := tmpFileSystemWrapper{}
+		usageStats, err := disk.Usage(fs.PartStats.Mountpoint)
+		tmp.usage = *usageStats
+		tmp.err = err
 
-		if errors.Is(err, syscall.Errno(13)) {
-			fsList[index].Error = err
-			continue
+		resChan <- tmp
+	}()
+
+	select {
+	case tmp := <-resChan:
+		if tmp.err != nil {
+			fs.Error = tmp.err
+			return
 		}
 
-		fsList[index].UsageStats = *diskUsage
+		fs.UsageStats = tmp.usage
+	case <-myCtx.Done():
+		err := errors.New("Timeout exceeded for fs " + fs.PartStats.Mountpoint + ". Maybe hanging network filesystem?")
+		fs.Error = err
+	}
+}
+
+func GetDiskUsage(ctx context.Context, timeout time.Duration, fsList []FilesystemType, _ *CheckConfig) error {
+	for index := range fsList {
+		GetDiskUsageSingle(ctx, timeout/time.Duration(len(fsList)), &fsList[index])
 	}
 
 	return nil

diff --git a/internal/sensors/sensors.go b/internal/sensors/sensors.go
@@ -29,6 +29,14 @@ type Device struct {
 	Sensors []Sensor
 }
 
+const (
+	inputFileSuffix         string = "_input"
+	critThresholdFileSuffix string = "_crit"
+	lowestValueFileSuffix   string = "_lowest"
+	highestValueFileSuffix  string = "_highest"
+	maxValueFileSuffix      string = "_max"
+)
+
 func (d *Device) String() string {
 	result := d.Name
 	result += ": "
@@ -263,7 +271,7 @@ func readHumiditySensor(devicePath string, index int) (Sensor, error) {
 	sensor.Perfdata.Label = label
 
 	// Look for input (the actual value)
-	value, err := readIntFromFile(basePath + "_input")
+	value, err := readIntFromFile(basePath + inputFileSuffix)
 
 	if err != nil {
 		return sensor, err
@@ -285,7 +293,7 @@ func readEnergySensor(devicePath string, index int) (Sensor, error) {
 	sensor.Perfdata.Label = label
 
 	// Look for input (the actual value)
-	value, err := readIntFromFile(basePath + "_input")
+	value, err := readIntFromFile(basePath + inputFileSuffix)
 	if err != nil {
 		return sensor, err
 	}
@@ -307,7 +315,7 @@ func readCurrSensor(devicePath string, index int) (Sensor, error) {
 	sensor.Perfdata.Label = label
 
 	// Look for input (the actual value)
-	value, err := readIntFromFile(basePath + "_input")
+	value, err := readIntFromFile(basePath + inputFileSuffix)
 	if err != nil {
 		return sensor, err
 	}
@@ -317,14 +325,14 @@ func readCurrSensor(devicePath string, index int) (Sensor, error) {
 
 	// == Min
 	// Is there a currN_lowest file? Use it for max value
-	value, err = readIntFromFile(basePath + "_lowest")
+	value, err = readIntFromFile(basePath + lowestValueFileSuffix)
 	if err == nil {
 		sensor.Perfdata.Min = float64(value) / 1000
 	}
 
 	// == Max
 	// Is there a currN_highest file? Use it for max value
-	value, err = readIntFromFile(basePath + "_highest")
+	value, err = readIntFromFile(basePath + highestValueFileSuffix)
 	if err == nil {
 		sensor.Perfdata.Max = float64(value) / 1000
 	}
@@ -343,7 +351,7 @@ func readCurrSensor(devicePath string, index int) (Sensor, error) {
 		critPresent = true
 	}
 	// Is there a currN_crit file? If yes, use that as upper critical
-	value, err = readIntFromFile(basePath + "_crit")
+	value, err = readIntFromFile(basePath + critThresholdFileSuffix)
 	if err == nil {
 		tmp.Upper = float64(value) / 1000
 		critPresent = true
@@ -391,7 +399,7 @@ func readFanSensor(devicePath string, index int) (Sensor, error) {
 	sensor.Perfdata.Label = label
 
 	// Look for input (the actual value)
-	value, err := readIntFromFile(basePath + "_input")
+	value, err := readIntFromFile(basePath + inputFileSuffix)
 	if err != nil {
 		return sensor, err
 	}
@@ -403,7 +411,7 @@ func readFanSensor(devicePath string, index int) (Sensor, error) {
 
 	// == Max
 	// Is there a tempN_highest file? Use it for max value
-	value, err = readIntFromFile(basePath + "_max")
+	value, err = readIntFromFile(basePath + maxValueFileSuffix)
 	if err == nil {
 		sensor.Perfdata.Max = float64(value)
 	}
@@ -424,7 +432,7 @@ func readVoltageSensor(_, devicePath string, index int) (Sensor, error) {
 	sensor.Perfdata.Label = label
 
 	// Look for input (the actual value)
-	value, err := readIntFromFile(basePath + "_input")
+	value, err := readIntFromFile(basePath + inputFileSuffix)
 	if err != nil {
 		return sensor, err
 	}
@@ -440,7 +448,7 @@ func readVoltageSensor(_, devicePath string, index int) (Sensor, error) {
 	}
 	warnPresent := false
 	// Is there a inN_max file? If yes, use that as warning
-	value, err = readIntFromFile(basePath + "_max")
+	value, err = readIntFromFile(basePath + maxValueFileSuffix)
 	if err == nil {
 		tmpWarn.Upper = float64(value) / 1000
 		warnPresent = true
@@ -458,7 +466,7 @@ func readVoltageSensor(_, devicePath string, index int) (Sensor, error) {
 	}
 	critPresent := false
 	// Is there a inN_crit file? If yes, use that as critical
-	value, err = readIntFromFile(basePath + "_crit")
+	value, err = readIntFromFile(basePath + critThresholdFileSuffix)
 	if err == nil {
 		tmpCrit.Upper = float64(value) / 1000
 		critPresent = true
@@ -476,14 +484,14 @@ func readVoltageSensor(_, devicePath string, index int) (Sensor, error) {
 	}
 
 	// == Min
-	value, err = readIntFromFile(basePath + "_lowest")
+	value, err = readIntFromFile(basePath + lowestValueFileSuffix)
 	if err == nil {
 		sensor.Perfdata.Min = float64(value) / 1000
 	}
 
 	// == Max
 	// Is there a tempN_highest file? Use it for max value
-	value, err = readIntFromFile(basePath + "_highest")
+	value, err = readIntFromFile(basePath + highestValueFileSuffix)
 	if err == nil {
 		sensor.Perfdata.Max = float64(value) / 1000
 	}
@@ -503,7 +511,7 @@ func readPowerSensor(_, devicePath string, index int) (Sensor, error) {
 	sensor.Perfdata.Label = label
 
 	// Look for input (the actual value)
-	value, err := readIntFromFile(basePath + "_input")
+	value, err := readIntFromFile(basePath + inputFileSuffix)
 
 	if err != nil {
 		return sensor, err
@@ -551,7 +559,7 @@ func readPowerSensor(_, devicePath string, index int) (Sensor, error) {
 	}
 	critPresent := false
 	// Is there a powerN_crit file? If yes, use that as critical
-	value, err = readIntFromFile(basePath + "_crit")
+	value, err = readIntFromFile(basePath + critThresholdFileSuffix)
 	if err == nil {
 		tmpCrit.Upper = float64(value)
 		critPresent = true
@@ -574,7 +582,7 @@ func readTempSensor(_, devicePath string, index int) (Sensor, error) {
 	sensor.Perfdata.Label = label
 
 	// Look for input (the actual value)
-	value, err := readIntFromFile(basePath + "_input")
+	value, err := readIntFromFile(basePath + inputFileSuffix)
 
 	if err != nil {
 		return sensor, err
@@ -591,7 +599,7 @@ func readTempSensor(_, devicePath string, index int) (Sensor, error) {
 	}
 	warnPresent := false
 	// Is there a tempN_max file? If yes, use that as warning
-	value, err = readIntFromFile(basePath + "_max")
+	value, err = readIntFromFile(basePath + maxValueFileSuffix)
 
 	if err == nil {
 		tmpWarn.Upper = float64(value / 1000)
@@ -611,7 +619,7 @@ func readTempSensor(_, devicePath string, index int) (Sensor, error) {
 
 	critPresent := false
 	// Is there a tempN_crit file? If yes, use that as critical
-	value, err = readIntFromFile(basePath + "_crit")
+	value, err = readIntFromFile(basePath + critThresholdFileSuffix)
 
 	if err == nil {
 		tmpCrit.Upper = float64(value / 1000)
@@ -640,15 +648,15 @@ func readTempSensor(_, devicePath string, index int) (Sensor, error) {
 
 	// == Min
 	// Is there a tempN_lowest file? Use it for min value
-	value, err = readIntFromFile(basePath + "_lowest")
+	value, err = readIntFromFile(basePath + lowestValueFileSuffix)
 
 	if err == nil {
 		sensor.Perfdata.Min = value
 	}
 
 	// == Max
 	// Is there a tempN_highest file? Use it for max value
-	value, err = readIntFromFile(basePath + "_highest")
+	value, err = readIntFromFile(basePath + highestValueFileSuffix)
 
 	if err == nil {
 		sensor.Perfdata.Max = value