Merge 8ba618fe2c into b99dcb64f9

collector/filesystem_linux.go

@@ -47,6 +47,9 @@ var statWorkerCount = kingpin.Flag("collector.filesystem.stat-workers",
 var stuckMounts = make(map[string]struct{})
 var stuckMountsMtx = &sync.Mutex{}
 
+// /proc/*/mountinfo can be observed mid-update under heavy mount churn.
+const mountInfoParseRetries = 3
+
 // GetStats returns filesystem stats.
 func (c *filesystemCollector) GetStats() ([]filesystemStats, error) {
 	mps, err := mountPointDetails(c.logger)
@@ -183,11 +186,15 @@ func mountPointDetails(logger *slog.Logger) ([]filesystemLabels, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to open procfs: %w", err)
 	}
-	mountInfo, err := fs.GetProcMounts(1)
+	mountInfo, err := getMountInfoWithRetry(logger, func() ([]*procfs.MountInfo, error) {
+		return fs.GetProcMounts(1)
+	})
 	if errors.Is(err, os.ErrNotExist) {
 		// Fallback to `/proc/self/mountinfo` if `/proc/1/mountinfo` is missing due hidepid.
 		logger.Debug("Reading root mounts failed, falling back to self mounts", "err", err)
-		mountInfo, err = fs.GetMounts()
+		mountInfo, err = getMountInfoWithRetry(logger, func() ([]*procfs.MountInfo, error) {
+			return fs.GetMounts()
+		})
 	}
 	if err != nil {
 		return nil, err
@@ -196,6 +203,26 @@ func mountPointDetails(logger *slog.Logger) ([]filesystemLabels, error) {
 	return parseFilesystemLabels(mountInfo)
 }
 
+func getMountInfoWithRetry(logger *slog.Logger, readMountInfo func() ([]*procfs.MountInfo, error)) ([]*procfs.MountInfo, error) {
+	var (
+		mountInfo []*procfs.MountInfo
+		err       error
+	)
+
+	for attempt := 1; attempt <= mountInfoParseRetries; attempt++ {
+		mountInfo, err = readMountInfo()
+		if !errors.Is(err, procfs.ErrFileParse) {
+			return mountInfo, err
+		}
+		if attempt == mountInfoParseRetries {
+			break
+		}
+		logger.Debug("Parsing mountinfo failed, retrying", "attempt", attempt, "err", err)
+	}
+
+	return mountInfo, err
+}
+
 func parseFilesystemLabels(mountInfo []*procfs.MountInfo) ([]filesystemLabels, error) {
 	var filesystems []filesystemLabels
 

collector/filesystem_linux_test.go

@@ -16,8 +16,11 @@
 package collector
 
 import (
+	"errors"
+	"fmt"
 	"io"
 	"log/slog"
+	"os"
 	"testing"
 
 	"github.com/alecthomas/kingpin/v2"
@@ -168,6 +171,60 @@ func TestMountsFallback(t *testing.T) {
 	}
 }
 
+func TestGetMountInfoWithRetry(t *testing.T) {
+	t.Run("retries parse errors", func(t *testing.T) {
+		attempts := 0
+		expected := []*procfs.MountInfo{{MountPoint: "/"}}
+
+		got, err := getMountInfoWithRetry(slog.New(slog.NewTextHandler(io.Discard, nil)), func() ([]*procfs.MountInfo, error) {
+			attempts++
+			if attempts < mountInfoParseRetries {
+				return nil, fmt.Errorf("%w: Too few fields in mount string: truncated", procfs.ErrFileParse)
+			}
+			return expected, nil
+		})
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if attempts != mountInfoParseRetries {
+			t.Fatalf("expected %d attempts, got %d", mountInfoParseRetries, attempts)
+		}
+		if len(got) != len(expected) || got[0].MountPoint != expected[0].MountPoint {
+			t.Fatalf("unexpected mountinfo: %+v", got)
+		}
+	})
+
+	t.Run("does not retry non parse errors", func(t *testing.T) {
+		attempts := 0
+
+		_, err := getMountInfoWithRetry(slog.New(slog.NewTextHandler(io.Discard, nil)), func() ([]*procfs.MountInfo, error) {
+			attempts++
+			return nil, os.ErrNotExist
+		})
+		if !errors.Is(err, os.ErrNotExist) {
+			t.Fatalf("expected os.ErrNotExist, got %v", err)
+		}
+		if attempts != 1 {
+			t.Fatalf("expected 1 attempt, got %d", attempts)
+		}
+	})
+
+	t.Run("returns parse error after max retries", func(t *testing.T) {
+		attempts := 0
+
+		_, err := getMountInfoWithRetry(slog.New(slog.NewTextHandler(io.Discard, nil)), func() ([]*procfs.MountInfo, error) {
+			attempts++
+			return nil, fmt.Errorf("%w: Too few fields in mount string: truncated", procfs.ErrFileParse)
+		})
+		if !errors.Is(err, procfs.ErrFileParse) {
+			t.Fatalf("expected procfs.ErrFileParse, got %v", err)
+		}
+		if attempts != mountInfoParseRetries {
+			t.Fatalf("expected %d attempts, got %d", mountInfoParseRetries, attempts)
+		}
+	})
+}
+
 func TestPathRootfs(t *testing.T) {
 	if _, err := kingpin.CommandLine.Parse([]string{"--path.procfs", "./fixtures_bindmount/proc", "--path.rootfs", "/host"}); err != nil {
 		t.Fatal(err)