Lnstat: expose metrics from /proc/net/stat (#1771)

* Lnstat initial commit

Signed-off-by: Aleksei Zakharov <zaharov@selectel.ru>
Co-authored-by: Johannes 'fish' Ziemke <github@freigeist.org>

README.md

@@ -105,6 +105,7 @@ filesystem | Exposes filesystem statistics, such as disk space used. | Darwin, D
 hwmon | Expose hardware monitoring and sensor data from `/sys/class/hwmon/`. | Linux
 infiniband | Exposes network statistics specific to InfiniBand and Intel OmniPath configurations. | Linux
 ipvs | Exposes IPVS status from `/proc/net/ip_vs` and stats from `/proc/net/ip_vs_stats`. | Linux
+lnstat | Exposes stats from `/proc/net/stat/`. | Linux
 loadavg | Exposes load average. | Darwin, Dragonfly, FreeBSD, Linux, NetBSD, OpenBSD, Solaris
 mdadm | Exposes statistics about devices in `/proc/mdstat` (does nothing if no `/proc/mdstat` present). | Linux
 meminfo | Exposes memory statistics. | Darwin, Dragonfly, FreeBSD, Linux, OpenBSD

collector/fixtures/e2e-output.txt

@@ -1252,6 +1252,184 @@ node_ksmd_run 1
 # HELP node_ksmd_sleep_seconds ksmd 'sleep_millisecs' file.
 # TYPE node_ksmd_sleep_seconds gauge
 node_ksmd_sleep_seconds 0.02
+# HELP node_lnstat_allocs_total linux network cache stats
+# TYPE node_lnstat_allocs_total counter
+node_lnstat_allocs_total{cpu="0",subsystem="arp_cache"} 1
+node_lnstat_allocs_total{cpu="0",subsystem="ndisc_cache"} 240
+node_lnstat_allocs_total{cpu="1",subsystem="arp_cache"} 13
+node_lnstat_allocs_total{cpu="1",subsystem="ndisc_cache"} 252
+# HELP node_lnstat_delete_list_total linux network cache stats
+# TYPE node_lnstat_delete_list_total counter
+node_lnstat_delete_list_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_delete_list_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_delete_list_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_delete_list_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_delete_total linux network cache stats
+# TYPE node_lnstat_delete_total counter
+node_lnstat_delete_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_delete_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_delete_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_delete_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_destroys_total linux network cache stats
+# TYPE node_lnstat_destroys_total counter
+node_lnstat_destroys_total{cpu="0",subsystem="arp_cache"} 2
+node_lnstat_destroys_total{cpu="0",subsystem="ndisc_cache"} 241
+node_lnstat_destroys_total{cpu="1",subsystem="arp_cache"} 14
+node_lnstat_destroys_total{cpu="1",subsystem="ndisc_cache"} 253
+# HELP node_lnstat_drop_total linux network cache stats
+# TYPE node_lnstat_drop_total counter
+node_lnstat_drop_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_drop_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_drop_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_drop_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_early_drop_total linux network cache stats
+# TYPE node_lnstat_early_drop_total counter
+node_lnstat_early_drop_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_early_drop_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_early_drop_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_early_drop_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_entries_total linux network cache stats
+# TYPE node_lnstat_entries_total counter
+node_lnstat_entries_total{cpu="0",subsystem="arp_cache"} 20
+node_lnstat_entries_total{cpu="0",subsystem="ndisc_cache"} 36
+node_lnstat_entries_total{cpu="0",subsystem="nf_conntrack"} 33
+node_lnstat_entries_total{cpu="1",subsystem="arp_cache"} 20
+node_lnstat_entries_total{cpu="1",subsystem="ndisc_cache"} 36
+node_lnstat_entries_total{cpu="1",subsystem="nf_conntrack"} 33
+node_lnstat_entries_total{cpu="2",subsystem="nf_conntrack"} 33
+node_lnstat_entries_total{cpu="3",subsystem="nf_conntrack"} 33
+# HELP node_lnstat_expect_create_total linux network cache stats
+# TYPE node_lnstat_expect_create_total counter
+node_lnstat_expect_create_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_expect_create_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_expect_create_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_expect_create_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_expect_delete_total linux network cache stats
+# TYPE node_lnstat_expect_delete_total counter
+node_lnstat_expect_delete_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_expect_delete_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_expect_delete_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_expect_delete_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_expect_new_total linux network cache stats
+# TYPE node_lnstat_expect_new_total counter
+node_lnstat_expect_new_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_expect_new_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_expect_new_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_expect_new_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_forced_gc_runs_total linux network cache stats
+# TYPE node_lnstat_forced_gc_runs_total counter
+node_lnstat_forced_gc_runs_total{cpu="0",subsystem="arp_cache"} 10
+node_lnstat_forced_gc_runs_total{cpu="0",subsystem="ndisc_cache"} 249
+node_lnstat_forced_gc_runs_total{cpu="1",subsystem="arp_cache"} 22
+node_lnstat_forced_gc_runs_total{cpu="1",subsystem="ndisc_cache"} 261
+# HELP node_lnstat_found_total linux network cache stats
+# TYPE node_lnstat_found_total counter
+node_lnstat_found_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_found_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_found_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_found_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_hash_grows_total linux network cache stats
+# TYPE node_lnstat_hash_grows_total counter
+node_lnstat_hash_grows_total{cpu="0",subsystem="arp_cache"} 3
+node_lnstat_hash_grows_total{cpu="0",subsystem="ndisc_cache"} 242
+node_lnstat_hash_grows_total{cpu="1",subsystem="arp_cache"} 15
+node_lnstat_hash_grows_total{cpu="1",subsystem="ndisc_cache"} 254
+# HELP node_lnstat_hits_total linux network cache stats
+# TYPE node_lnstat_hits_total counter
+node_lnstat_hits_total{cpu="0",subsystem="arp_cache"} 5
+node_lnstat_hits_total{cpu="0",subsystem="ndisc_cache"} 244
+node_lnstat_hits_total{cpu="1",subsystem="arp_cache"} 17
+node_lnstat_hits_total{cpu="1",subsystem="ndisc_cache"} 256
+# HELP node_lnstat_icmp_error_total linux network cache stats
+# TYPE node_lnstat_icmp_error_total counter
+node_lnstat_icmp_error_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_icmp_error_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_icmp_error_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_icmp_error_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_ignore_total linux network cache stats
+# TYPE node_lnstat_ignore_total counter
+node_lnstat_ignore_total{cpu="0",subsystem="nf_conntrack"} 22666
+node_lnstat_ignore_total{cpu="1",subsystem="nf_conntrack"} 22180
+node_lnstat_ignore_total{cpu="2",subsystem="nf_conntrack"} 22740
+node_lnstat_ignore_total{cpu="3",subsystem="nf_conntrack"} 22152
+# HELP node_lnstat_insert_failed_total linux network cache stats
+# TYPE node_lnstat_insert_failed_total counter
+node_lnstat_insert_failed_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_insert_failed_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_insert_failed_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_insert_failed_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_insert_total linux network cache stats
+# TYPE node_lnstat_insert_total counter
+node_lnstat_insert_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_insert_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_insert_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_insert_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_invalid_total linux network cache stats
+# TYPE node_lnstat_invalid_total counter
+node_lnstat_invalid_total{cpu="0",subsystem="nf_conntrack"} 3
+node_lnstat_invalid_total{cpu="1",subsystem="nf_conntrack"} 2
+node_lnstat_invalid_total{cpu="2",subsystem="nf_conntrack"} 1
+node_lnstat_invalid_total{cpu="3",subsystem="nf_conntrack"} 47
+# HELP node_lnstat_lookups_total linux network cache stats
+# TYPE node_lnstat_lookups_total counter
+node_lnstat_lookups_total{cpu="0",subsystem="arp_cache"} 4
+node_lnstat_lookups_total{cpu="0",subsystem="ndisc_cache"} 243
+node_lnstat_lookups_total{cpu="1",subsystem="arp_cache"} 16
+node_lnstat_lookups_total{cpu="1",subsystem="ndisc_cache"} 255
+# HELP node_lnstat_new_total linux network cache stats
+# TYPE node_lnstat_new_total counter
+node_lnstat_new_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_new_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_new_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_new_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_periodic_gc_runs_total linux network cache stats
+# TYPE node_lnstat_periodic_gc_runs_total counter
+node_lnstat_periodic_gc_runs_total{cpu="0",subsystem="arp_cache"} 9
+node_lnstat_periodic_gc_runs_total{cpu="0",subsystem="ndisc_cache"} 248
+node_lnstat_periodic_gc_runs_total{cpu="1",subsystem="arp_cache"} 21
+node_lnstat_periodic_gc_runs_total{cpu="1",subsystem="ndisc_cache"} 260
+# HELP node_lnstat_rcv_probes_mcast_total linux network cache stats
+# TYPE node_lnstat_rcv_probes_mcast_total counter
+node_lnstat_rcv_probes_mcast_total{cpu="0",subsystem="arp_cache"} 7
+node_lnstat_rcv_probes_mcast_total{cpu="0",subsystem="ndisc_cache"} 246
+node_lnstat_rcv_probes_mcast_total{cpu="1",subsystem="arp_cache"} 19
+node_lnstat_rcv_probes_mcast_total{cpu="1",subsystem="ndisc_cache"} 258
+# HELP node_lnstat_rcv_probes_ucast_total linux network cache stats
+# TYPE node_lnstat_rcv_probes_ucast_total counter
+node_lnstat_rcv_probes_ucast_total{cpu="0",subsystem="arp_cache"} 8
+node_lnstat_rcv_probes_ucast_total{cpu="0",subsystem="ndisc_cache"} 247
+node_lnstat_rcv_probes_ucast_total{cpu="1",subsystem="arp_cache"} 20
+node_lnstat_rcv_probes_ucast_total{cpu="1",subsystem="ndisc_cache"} 259
+# HELP node_lnstat_res_failed_total linux network cache stats
+# TYPE node_lnstat_res_failed_total counter
+node_lnstat_res_failed_total{cpu="0",subsystem="arp_cache"} 6
+node_lnstat_res_failed_total{cpu="0",subsystem="ndisc_cache"} 245
+node_lnstat_res_failed_total{cpu="1",subsystem="arp_cache"} 18
+node_lnstat_res_failed_total{cpu="1",subsystem="ndisc_cache"} 257
+# HELP node_lnstat_search_restart_total linux network cache stats
+# TYPE node_lnstat_search_restart_total counter
+node_lnstat_search_restart_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_search_restart_total{cpu="1",subsystem="nf_conntrack"} 2
+node_lnstat_search_restart_total{cpu="2",subsystem="nf_conntrack"} 1
+node_lnstat_search_restart_total{cpu="3",subsystem="nf_conntrack"} 4
+# HELP node_lnstat_searched_total linux network cache stats
+# TYPE node_lnstat_searched_total counter
+node_lnstat_searched_total{cpu="0",subsystem="nf_conntrack"} 0
+node_lnstat_searched_total{cpu="1",subsystem="nf_conntrack"} 0
+node_lnstat_searched_total{cpu="2",subsystem="nf_conntrack"} 0
+node_lnstat_searched_total{cpu="3",subsystem="nf_conntrack"} 0
+# HELP node_lnstat_table_fulls_total linux network cache stats
+# TYPE node_lnstat_table_fulls_total counter
+node_lnstat_table_fulls_total{cpu="0",subsystem="arp_cache"} 12
+node_lnstat_table_fulls_total{cpu="0",subsystem="ndisc_cache"} 251
+node_lnstat_table_fulls_total{cpu="1",subsystem="arp_cache"} 24
+node_lnstat_table_fulls_total{cpu="1",subsystem="ndisc_cache"} 263
+# HELP node_lnstat_unresolved_discards_total linux network cache stats
+# TYPE node_lnstat_unresolved_discards_total counter
+node_lnstat_unresolved_discards_total{cpu="0",subsystem="arp_cache"} 11
+node_lnstat_unresolved_discards_total{cpu="0",subsystem="ndisc_cache"} 250
+node_lnstat_unresolved_discards_total{cpu="1",subsystem="arp_cache"} 23
+node_lnstat_unresolved_discards_total{cpu="1",subsystem="ndisc_cache"} 262
 # HELP node_load1 1m load average.
 # TYPE node_load1 gauge
 node_load1 0.21
@@ -2803,6 +2981,7 @@ node_scrape_collector_success{collector="infiniband"} 1
 node_scrape_collector_success{collector="interrupts"} 1
 node_scrape_collector_success{collector="ipvs"} 1
 node_scrape_collector_success{collector="ksmd"} 1
+node_scrape_collector_success{collector="lnstat"} 1
 node_scrape_collector_success{collector="loadavg"} 1
 node_scrape_collector_success{collector="mdadm"} 1
 node_scrape_collector_success{collector="meminfo"} 1

collector/fixtures/proc/net/stat/arp_cache

@@ -0,0 +1,3 @@
+entries  allocs destroys hash_grows  lookups hits  res_failed  rcv_probes_mcast rcv_probes_ucast  periodic_gc_runs forced_gc_runs unresolved_discards table_fulls
+00000014  00000001 00000002 00000003  00000004 00000005  00000006  00000007 00000008  00000009 0000000a 0000000b 0000000c
+00000014  0000000d 0000000e 0000000f  00000010 00000011  00000012  00000013 00000014  00000015 00000016 00000017 00000018

collector/fixtures/proc/net/stat/ndisc_cache

@@ -0,0 +1,3 @@
+entries  allocs destroys hash_grows  lookups hits  res_failed  rcv_probes_mcast rcv_probes_ucast  periodic_gc_runs forced_gc_runs unresolved_discards table_fulls
+00000024  000000f0 000000f1 000000f2  000000f3 000000f4  000000f5  000000f6 000000f7  000000f8 000000f9 000000fa 000000fb
+00000024  000000fc 000000fd 000000fe  000000ff 00000100  00000101  00000102 00000103  00000104 00000105 00000106 00000107

collector/lnstat_linux.go

@@ -0,0 +1,71 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !nolnstat
+
+package collector
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
+)
+
+type lnstatCollector struct {
+	logger log.Logger
+}
+
+func init() {
+	registerCollector("lnstat", defaultEnabled, NewLnstatCollector)
+}
+
+func NewLnstatCollector(logger log.Logger) (Collector, error) {
+	return &lnstatCollector{logger}, nil
+}
+
+func (c *lnstatCollector) Update(ch chan<- prometheus.Metric) error {
+	const (
+		subsystem = "lnstat"
+	)
+
+	fs, err := procfs.NewFS(*procPath)
+	if err != nil {
+		return fmt.Errorf("failed to open procfs: %w", err)
+	}
+
+	netStats, err := fs.NetStat()
+	if err != nil {
+		return fmt.Errorf("Lnstat error: %s", err)
+	}
+
+	for _, netStatFile := range netStats {
+		labelNames := []string{"subsystem", "cpu"}
+		for header, stats := range netStatFile.Stats {
+			for cpu, value := range stats {
+				labelValues := []string{netStatFile.Filename, strconv.Itoa(cpu)}
+				ch <- prometheus.MustNewConstMetric(
+					prometheus.NewDesc(
+						prometheus.BuildFQName(namespace, subsystem, header+"_total"),
+						"linux network cache stats",
+						labelNames, nil,
+					),
+					prometheus.CounterValue, float64(value), labelValues...,
+				)
+			}
+		}
+	}
+	return nil
+}

end-to-end-test.sh

@@ -21,6 +21,7 @@ enabled_collectors=$(cat << COLLECTORS
   interrupts
   ipvs
   ksmd
+  lnstat
   loadavg
   mdadm
   meminfo

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.29.0
 	github.com/prometheus/exporter-toolkit v0.6.0
-	github.com/prometheus/procfs v0.7.2
+	github.com/prometheus/procfs v0.7.3
 	github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1
 	github.com/siebenmann/go-kstat v0.0.0-20210513183136-173c9b0a9973 // indirect
 	github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a

go.sum

@@ -240,8 +240,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.2 h1:zE6zJjRS9S916ptrZ326OU0++1XRwHgxkvCFflxx6Fo=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
-github.com/prometheus/procfs v0.7.2/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 h1:ZFfeKAhIQiiOrQaI3/znw0gOmYpO28Tcu1YaqMa/jtQ=
 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=