Add check state for mdadm arrays via node_md_state metric (#1810)

* Expose metric for state=check for node_md_state
* Added new e2e output fixture including md201 which is in checking state and a the new state=check labeled metric for all other md

Signed-off-by: Christian Rohmann <github@frittentheke.de>

collector/fixtures/e2e-output.txt

@@ -1210,6 +1210,7 @@ node_md_blocks{device="md12"} 3.886394368e+09
 node_md_blocks{device="md120"} 2.095104e+06
 node_md_blocks{device="md126"} 1.855870976e+09
 node_md_blocks{device="md127"} 3.12319552e+08
+node_md_blocks{device="md201"} 1.993728e+06
 node_md_blocks{device="md219"} 7932
 node_md_blocks{device="md3"} 5.853468288e+09
 node_md_blocks{device="md4"} 4.883648e+06
@@ -1228,6 +1229,7 @@ node_md_blocks_synced{device="md12"} 3.886394368e+09
 node_md_blocks_synced{device="md120"} 2.095104e+06
 node_md_blocks_synced{device="md126"} 1.855870976e+09
 node_md_blocks_synced{device="md127"} 3.12319552e+08
+node_md_blocks_synced{device="md201"} 114176
 node_md_blocks_synced{device="md219"} 7932
 node_md_blocks_synced{device="md3"} 5.853468288e+09
 node_md_blocks_synced{device="md4"} 4.883648e+06
@@ -1264,6 +1266,9 @@ node_md_disks{device="md126",state="spare"} 0
 node_md_disks{device="md127",state="active"} 2
 node_md_disks{device="md127",state="failed"} 0
 node_md_disks{device="md127",state="spare"} 0
+node_md_disks{device="md201",state="active"} 2
+node_md_disks{device="md201",state="failed"} 0
+node_md_disks{device="md201",state="spare"} 0
 node_md_disks{device="md219",state="active"} 0
 node_md_disks{device="md219",state="failed"} 0
 node_md_disks{device="md219",state="spare"} 3
@@ -1296,6 +1301,7 @@ node_md_disks_required{device="md12"} 2
 node_md_disks_required{device="md120"} 2
 node_md_disks_required{device="md126"} 2
 node_md_disks_required{device="md127"} 2
+node_md_disks_required{device="md201"} 2
 node_md_disks_required{device="md219"} 0
 node_md_disks_required{device="md3"} 8
 node_md_disks_required{device="md4"} 0
@@ -1306,66 +1312,87 @@ node_md_disks_required{device="md9"} 4
 # HELP node_md_state Indicates the state of md-device.
 # TYPE node_md_state gauge
 node_md_state{device="md0",state="active"} 1
+node_md_state{device="md0",state="check"} 0
 node_md_state{device="md0",state="inactive"} 0
 node_md_state{device="md0",state="recovering"} 0
 node_md_state{device="md0",state="resync"} 0
 node_md_state{device="md00",state="active"} 1
+node_md_state{device="md00",state="check"} 0
 node_md_state{device="md00",state="inactive"} 0
 node_md_state{device="md00",state="recovering"} 0
 node_md_state{device="md00",state="resync"} 0
 node_md_state{device="md10",state="active"} 1
+node_md_state{device="md10",state="check"} 0
 node_md_state{device="md10",state="inactive"} 0
 node_md_state{device="md10",state="recovering"} 0
 node_md_state{device="md10",state="resync"} 0
 node_md_state{device="md101",state="active"} 1
+node_md_state{device="md101",state="check"} 0
 node_md_state{device="md101",state="inactive"} 0
 node_md_state{device="md101",state="recovering"} 0
 node_md_state{device="md101",state="resync"} 0
 node_md_state{device="md11",state="active"} 0
+node_md_state{device="md11",state="check"} 0
 node_md_state{device="md11",state="inactive"} 0
 node_md_state{device="md11",state="recovering"} 0
 node_md_state{device="md11",state="resync"} 1
 node_md_state{device="md12",state="active"} 1
+node_md_state{device="md12",state="check"} 0
 node_md_state{device="md12",state="inactive"} 0
 node_md_state{device="md12",state="recovering"} 0
 node_md_state{device="md12",state="resync"} 0
 node_md_state{device="md120",state="active"} 1
+node_md_state{device="md120",state="check"} 0
 node_md_state{device="md120",state="inactive"} 0
 node_md_state{device="md120",state="recovering"} 0
 node_md_state{device="md120",state="resync"} 0
 node_md_state{device="md126",state="active"} 1
+node_md_state{device="md126",state="check"} 0
 node_md_state{device="md126",state="inactive"} 0
 node_md_state{device="md126",state="recovering"} 0
 node_md_state{device="md126",state="resync"} 0
 node_md_state{device="md127",state="active"} 1
+node_md_state{device="md127",state="check"} 0
 node_md_state{device="md127",state="inactive"} 0
 node_md_state{device="md127",state="recovering"} 0
 node_md_state{device="md127",state="resync"} 0
+node_md_state{device="md201",state="active"} 0
+node_md_state{device="md201",state="check"} 1
+node_md_state{device="md201",state="inactive"} 0
+node_md_state{device="md201",state="recovering"} 0
+node_md_state{device="md201",state="resync"} 0
 node_md_state{device="md219",state="active"} 0
+node_md_state{device="md219",state="check"} 0
 node_md_state{device="md219",state="inactive"} 1
 node_md_state{device="md219",state="recovering"} 0
 node_md_state{device="md219",state="resync"} 0
 node_md_state{device="md3",state="active"} 1
+node_md_state{device="md3",state="check"} 0
 node_md_state{device="md3",state="inactive"} 0
 node_md_state{device="md3",state="recovering"} 0
 node_md_state{device="md3",state="resync"} 0
 node_md_state{device="md4",state="active"} 0
+node_md_state{device="md4",state="check"} 0
 node_md_state{device="md4",state="inactive"} 1
 node_md_state{device="md4",state="recovering"} 0
 node_md_state{device="md4",state="resync"} 0
 node_md_state{device="md6",state="active"} 0
+node_md_state{device="md6",state="check"} 0
 node_md_state{device="md6",state="inactive"} 0
 node_md_state{device="md6",state="recovering"} 1
 node_md_state{device="md6",state="resync"} 0
 node_md_state{device="md7",state="active"} 1
+node_md_state{device="md7",state="check"} 0
 node_md_state{device="md7",state="inactive"} 0
 node_md_state{device="md7",state="recovering"} 0
 node_md_state{device="md7",state="resync"} 0
 node_md_state{device="md8",state="active"} 0
+node_md_state{device="md8",state="check"} 0
 node_md_state{device="md8",state="inactive"} 0
 node_md_state{device="md8",state="recovering"} 0
 node_md_state{device="md8",state="resync"} 1
 node_md_state{device="md9",state="active"} 0
+node_md_state{device="md9",state="check"} 0
 node_md_state{device="md9",state="inactive"} 0
 node_md_state{device="md9",state="recovering"} 0
 node_md_state{device="md9",state="resync"} 1

collector/fixtures/proc/mdstat

@@ -20,6 +20,10 @@ md8 : active raid1 sdb1[1] sda1[0] sdc[2](S) sde[3](S)
       195310144 blocks [2/2] [UU]
       [=>...................]  resync =  8.5% (16775552/195310144) finish=17.0min speed=259783K/sec
 
+md201 : active raid1 sda3[0] sdb3[1]
+      1993728 blocks super 1.2 [2/2] [UU]
+      [=>...................]  check =  5.7% (114176/1993728) finish=0.2min speed=114176K/sec
+
 md7 : active raid6 sdb1[0] sde1[3] sdd1[2] sdc1[1](F)
       7813735424 blocks super 1.2 level 6, 512k chunk, algorithm 2 [4/3] [U_UU]
       bitmap: 0/30 pages [0KB], 65536KB chunk

collector/mdadm_linux.go

@@ -64,6 +64,12 @@ var (
 		[]string{"device"},
 		prometheus.Labels{"state": "resync"},
 	)
+	checkDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "md", "state"),
+		"Indicates the state of md-device.",
+		[]string{"device"},
+		prometheus.Labels{"state": "check"},
+	)
 
 	disksDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, "md", "disks"),
@@ -174,6 +180,13 @@ func (c *mdadmCollector) Update(ch chan<- prometheus.Metric) error {
 			mdStat.Name,
 		)
 
+		ch <- prometheus.MustNewConstMetric(
+			checkDesc,
+			prometheus.GaugeValue,
+			stateVals["checking"],
+			mdStat.Name,
+		)
+
 		ch <- prometheus.MustNewConstMetric(
 			blocksTotalDesc,
 			prometheus.GaugeValue,