promql: Add irate() function

irate is a rate function that only looks at the most
recent two data points, and calucaltes a per-second value
from that. This produces much more granular graphs for
fast moving data, and works sanely across many scrape intervals.

It doesn't do so well for slowly moving data.

console_libraries/prom.lib

@@ -62,7 +62,7 @@ renderTemplate is the name of the template to use to render the value.
 </tr>
 <tr>
   <td>CPU</td>
-  <td>{{  template "prom_query_drilldown" (args (printf "avg by(job)(rate(process_cpu_seconds_total{job='%s'}[5m]))" .) "s/s" "humanizeNoSmallPrefix") }}</td>
+  <td>{{  template "prom_query_drilldown" (args (printf "avg by(job)(irate(process_cpu_seconds_total{job='%s'}[5m]))" .) "s/s" "humanizeNoSmallPrefix") }}</td>
 </tr>
 <tr>
   <td>Memory</td>

consoles/cassandra.html

@@ -4,15 +4,15 @@
 {{ template "prom_right_table_job_head" "cassandra" }}
 <tr>
   <td>Queries</td>
-  <td>{{ template "prom_query_drilldown" (args "sum by (job)(rate(cassandra_clientrequest_latency{job='cassandra'}[5m]))" "/s" "humanizeNoSmallPrefix") }}</td>
+  <td>{{ template "prom_query_drilldown" (args "sum by (job)(irate(cassandra_clientrequest_latency{job='cassandra'}[5m]))" "/s" "humanizeNoSmallPrefix") }}</td>
 </tr>
 <tr>
   <td>Timeout Ratio</td>
-  <td>{{ template "prom_query_drilldown" (args "sum by (job)(rate(cassandra_clientrequest_timeouts{job='cassandra'}[5m])) / sum by (job)(rate(cassandra_clientrequest_latency{job='cassandra'}[5m]))" "" "humanizeNoSmallPrefix") }}</td>
+  <td>{{ template "prom_query_drilldown" (args "sum by (job)(irate(cassandra_clientrequest_timeouts{job='cassandra'}[5m])) / sum by (job)(rate(cassandra_clientrequest_latency{job='cassandra'}[5m]))" "" "humanizeNoSmallPrefix") }}</td>
 </tr>
 <tr>
   <td>Unavailable Ratio</td>
-  <td>{{ template "prom_query_drilldown" (args "sum by (job)(rate(cassandra_clientrequest_unavailables{job='cassandra'}[5m])) / sum by (job)(rate(cassandra_clientrequest_latency{job='cassandra'}[5m]))" "" "humanizeNoSmallPrefix") }}</td>
+  <td>{{ template "prom_query_drilldown" (args "sum by (job)(irate(cassandra_clientrequest_unavailables{job='cassandra'}[5m])) / sum by (job)(rate(cassandra_clientrequest_latency{job='cassandra'}[5m]))" "" "humanizeNoSmallPrefix") }}</td>
 </tr>
 <tr><th colspan="2">Internals</th></tr>
 <tr>
@@ -26,7 +26,7 @@
 <tr><th colspan="2">Average Node Disk</th></tr>
 <tr>
   <td>Compacted</td>
-  <td>{{ template "prom_query_drilldown" (args "avg by (job)(rate(cassandra_compaction_bytescompacted{job='cassandra'}[5m]))" "B/s" "humanize1024") }}</td>
+  <td>{{ template "prom_query_drilldown" (args "avg by (job)(irate(cassandra_compaction_bytescompacted{job='cassandra'}[5m]))" "B/s" "humanize1024") }}</td>
 </tr>
 <tr>
   <td>Live CF</td>
@@ -50,7 +50,7 @@
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#queryGraph"),
-  expr: "sum by (job, clientrequest)(rate(cassandra_clientrequest_latency{job='cassandra'}[5m]))",
+  expr: "sum by (job, clientrequest)(irate(cassandra_clientrequest_latency{job='cassandra'}[5m]))",
   name: "[[clientrequest]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
   yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -64,7 +64,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#latencyGraph"),
-  expr: "sum by (job, clientrequest)(rate(cassandra_clientrequest_totallatency{job='cassandra'}[5m])) / sum by (job, clientrequest)(rate(cassandra_clientrequest_latency{job='cassandra'}[5m])) / 1000000",
+  expr: "sum by (job, clientrequest)(irate(cassandra_clientrequest_totallatency{job='cassandra'}[5m])) / sum by (job, clientrequest)(rate(cassandra_clientrequest_latency{job='cassandra'}[5m])) / 1000000",
   name: "[[clientrequest]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,

consoles/cloudwatch.html

@@ -8,7 +8,7 @@
 
 <tr>
   <td>API Requests</td>
-  <td>{{ template "prom_query_drilldown" (args "sum by (job)(rate(cloudwatch_requests_total{job='cloudwatch'}[5m]))" "/s" "humanizeNoSmallPrefix") }}</td>
+  <td>{{ template "prom_query_drilldown" (args "sum by (job)(irate(cloudwatch_requests_total{job='cloudwatch'}[5m]))" "/s" "humanizeNoSmallPrefix") }}</td>
 </tr>
 {{ template "prom_right_table_tail" }}
 

consoles/haproxy-backend.html

@@ -4,15 +4,15 @@
   <tr><th>{{ .Params.backend }}</th><th>{{ template "prom_query_drilldown" (args (printf "sum(min by (server)(haproxy_server_up{job='haproxy',backend='%s'}))" .Params.backend)) }} / {{ template "prom_query_drilldown" (args (printf "count(sum by (server)(haproxy_server_up{job='haproxy',backend='%s'}))" .Params.backend))}}</th></tr>
   <tr>
     <td>Responses</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_backend_http_responses_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_backend_http_responses_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Data In</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_backend_bytes_in_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_backend_bytes_in_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "B/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Data Out</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_backend_bytes_out_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_backend_bytes_out_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "B/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Current Sessions</td>
@@ -25,15 +25,15 @@
   <tr><th colspan="2">Server Errors</th></tr>
   <tr>
     <td>Connection Errors</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_backend_connection_errors_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_backend_connection_errors_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Response Errors</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_backend_connection_errors_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_backend_connection_errors_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Retry Warnings</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_backend_retry_warnings_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_backend_retry_warnings_total{job='haproxy',backend='%s'}[5m]))" .Params.backend) "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
 {{ template "prom_right_table_tail" }}
 
@@ -45,7 +45,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#responsesGraph"),
-    expr: "sum by (code)(rate(haproxy_backend_http_responses_total{job='haproxy',backend='{{ .Params.backend }}'}[5m]))",
+    expr: "sum by (code)(irate(haproxy_backend_http_responses_total{job='haproxy',backend='{{ .Params.backend }}'}[5m]))",
     renderer: 'area',
     yTitle: 'Queries',
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,

consoles/haproxy-backends.html

@@ -15,7 +15,7 @@
 <tr>
   <td><a href="haproxy-backend.html?backend={{ .Labels.backend }}">{{ .Labels.backend }}</a></td>
   <td>{{ template "prom_query_drilldown" (args (printf "sum(min by (server)(haproxy_server_up{job='haproxy',backend='%s'}))" .Labels.backend)) }} / {{ template "prom_query_drilldown" (args (printf "count(sum by (server)(haproxy_server_up{job='haproxy',backend='%s'}))" .Labels.backend))}}</tdd>
-  <td>{{ template "prom_query_drilldown" (args (printf "sum by(backend)(rate(haproxy_backend_http_responses_total{job='haproxy',backend='%s'}[5m]))" .Labels.backend) "/s" "humanizeNoSmallPrefix") }}</td>
+  <td>{{ template "prom_query_drilldown" (args (printf "sum by(backend)(irate(haproxy_backend_http_responses_total{job='haproxy',backend='%s'}[5m]))" .Labels.backend) "/s" "humanizeNoSmallPrefix") }}</td>
   <td>{{ template "prom_query_drilldown" (args (printf "sum by(backend)(haproxy_backend_current_sessions{job='haproxy',backend='%s'})" .Labels.backend) "" "humanize") }}</td>
   <td>{{ template "prom_query_drilldown" (args (printf "sum by(backend)(haproxy_backend_current_queue{job='haproxy',backend='%s'})" .Labels.backend) "" "humanize") }}</td>
 </tr>

consoles/haproxy-frontend.html

@@ -4,19 +4,19 @@
   <tr><th colspan="2">{{ .Params.frontend }}</th></tr>
   <tr>
     <td>Requests</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_frontend_http_requests_total{job='haproxy',frontend='%s'}[5m]))" .Params.frontend) "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_frontend_http_requests_total{job='haproxy',frontend='%s'}[5m]))" .Params.frontend) "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Requests Denied</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_frontend_requests_denied_total{job='haproxy',frontend='%s'}[5m]))" .Params.frontend) "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_frontend_requests_denied_total{job='haproxy',frontend='%s'}[5m]))" .Params.frontend) "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Data In</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_frontend_bytes_in_total{job='haproxy',frontend='%s'}[5m]))" .Params.frontend) "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_frontend_bytes_in_total{job='haproxy',frontend='%s'}[5m]))" .Params.frontend) "B/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Data Out</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(haproxy_frontend_bytes_out_total{job='haproxy',frontend='%s'}[5m]))" .Params.frontend) "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(haproxy_frontend_bytes_out_total{job='haproxy',frontend='%s'}[5m]))" .Params.frontend) "B/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Current Sessions</td>
@@ -32,7 +32,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#responsesGraph"),
-    expr: "sum by (code)(rate(haproxy_frontend_http_responses_total{job='haproxy',frontend='{{ .Params.frontend }}'}[5m]))",
+    expr: "sum by (code)(irate(haproxy_frontend_http_responses_total{job='haproxy',frontend='{{ .Params.frontend }}'}[5m]))",
     renderer: 'area',
     yTitle: 'Queries',
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,

consoles/haproxy-frontends.html

@@ -12,7 +12,7 @@
 {{ range query "count by (frontend)(haproxy_frontend_http_requests_total{job='haproxy'})" | sortByLabel "frontend" }}
 <tr>
   <td><a href="haproxy-frontend.html?frontend={{ .Labels.frontend }}">{{ .Labels.frontend }}</a></td>
-  <td>{{ template "prom_query_drilldown" (args (printf "sum by(frontend)(rate(haproxy_frontend_http_requests_total{job='haproxy',frontend='%s'}[5m]))" .Labels.frontend) "/s" "humanizeNoSmallPrefix") }}</td>
+  <td>{{ template "prom_query_drilldown" (args (printf "sum by(frontend)(irate(haproxy_frontend_http_requests_total{job='haproxy',frontend='%s'}[5m]))" .Labels.frontend) "/s" "humanizeNoSmallPrefix") }}</td>
   <td>{{ template "prom_query_drilldown" (args (printf "sum by(frontend)(haproxy_frontend_current_sessions{job='haproxy',frontend='%s'})" .Labels.frontend) "" "humanize") }}</td>
 </tr>
 {{ else }}

consoles/haproxy.html

@@ -7,7 +7,7 @@
   </tr>
   <tr>
     <td>CPU</td>
-    <td>{{ template "prom_query_drilldown" (args "avg by(job)(rate(haproxy_process_cpu_seconds_total{job='haproxy'}[5m]))" "s/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args "avg by(job)(irate(haproxy_process_cpu_seconds_total{job='haproxy'}[5m]))" "s/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Memory</td>
@@ -18,19 +18,19 @@
   </tr>
   <tr>
     <td>Requests</td>
-    <td>{{ template "prom_query_drilldown" (args "sum(rate(haproxy_frontend_http_requests_total{job='haproxy'}[5m]))" "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args "sum(irate(haproxy_frontend_http_requests_total{job='haproxy'}[5m]))" "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Requests Denied</td>
-    <td>{{ template "prom_query_drilldown" (args "sum(rate(haproxy_frontend_requests_denied_total{job='haproxy'}[5m]))" "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args "sum(irate(haproxy_frontend_requests_denied_total{job='haproxy'}[5m]))" "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Data In</td>
-    <td>{{ template "prom_query_drilldown" (args "sum(rate(haproxy_frontend_bytes_in_total{job='haproxy'}[5m]))" "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args "sum(irate(haproxy_frontend_bytes_in_total{job='haproxy'}[5m]))" "B/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Data Out</td>
-    <td>{{ template "prom_query_drilldown" (args "sum(rate(haproxy_frontend_bytes_out_total{job='haproxy'}[5m]))" "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args "sum(irate(haproxy_frontend_bytes_out_total{job='haproxy'}[5m]))" "B/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Current Sessions</td>
@@ -46,7 +46,7 @@
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#feRequestsGraph"),
-  expr: "sum by (frontend)(rate(haproxy_frontend_http_requests_total{job='haproxy'}[5m]))",
+  expr: "sum by (frontend)(irate(haproxy_frontend_http_requests_total{job='haproxy'}[5m]))",
   renderer: 'area',
   yTitle: 'Requests',
   name: '[[ frontend ]]',
@@ -60,7 +60,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#responsesGraph"),
-  expr: "sum by (backend)(rate(haproxy_backend_http_responses_total{job='haproxy'}[5m]))",
+  expr: "sum by (backend)(irate(haproxy_backend_http_responses_total{job='haproxy'}[5m]))",
   renderer: 'area',
   yTitle: 'Responses',
   name: '[[ backend ]]',

consoles/node-cpu.html

@@ -4,7 +4,7 @@
   <tr>
     <th colspan="2">CPU(s): {{ template "prom_query_drilldown" (args (printf "scalar(count(count by (cpu)(node_cpu{job='node',instance='%s'})))" .Params.instance)) }}</th>
   </tr>
-{{ range printf "sum by (mode)(rate(node_cpu{job='node',instance='%s'}[5m])) * 100 / scalar(count(count by (cpu)(node_cpu{job='node',instance='%s'})))" .Params.instance .Params.instance | query | sortByLabel "mode" }}
+{{ range printf "sum by (mode)(irate(node_cpu{job='node',instance='%s'}[5m])) * 100 / scalar(count(count by (cpu)(node_cpu{job='node',instance='%s'})))" .Params.instance .Params.instance | query | sortByLabel "mode" }}
   <tr>
     <td>{{ .Labels.mode | title }} CPU</td>
     <td>{{ .Value | printf "%.1f" }}%</td>
@@ -21,15 +21,15 @@
   </tr>
   <tr>
     <td>Forks</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_forks{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_forks{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Context Switches</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_context_switches{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_context_switches{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Interrupts</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_intr{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_intr{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>1m Loadavg</td>
@@ -47,7 +47,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(rate(node_cpu{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
+    expr: "sum by (mode)(irate(node_cpu{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
     renderer: 'area',
     max: {{ with printf "count(count by (cpu)(node_cpu{job='node',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,

consoles/node-disk.html

@@ -7,19 +7,19 @@
   <th colspan="2">{{ .Labels.device }}</th>
   <tr>
     <td>Utilization</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_disk_io_time_ms{job='node',instance='%s',device='%s'}[5m]) / 1000 * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_ms{job='node',instance='%s',device='%s'}[5m]) / 1000 * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
   </tr>
   <tr>
     <td>Throughput</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_disk_sectors_read{job='node',instance='%s',device='%s'}[5m]) * 512 + rate(node_disk_sectors_written{job='node',instance='%s',device='%s'}[5m]) * 512" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_sectors_read{job='node',instance='%s',device='%s'}[5m]) * 512 + rate(node_disk_sectors_written{job='node',instance='%s',device='%s'}[5m]) * 512" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Avg Read Time</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_disk_read_time_ms{job='node',instance='%s',device='%s'}[5m]) / 1000 / rate(node_disk_reads_completed{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_ms{job='node',instance='%s',device='%s'}[5m]) / 1000 / rate(node_disk_reads_completed{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
   </tr>
   <tr>
     <td>Avg Write Time</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_disk_write_time_ms{job='node',instance='%s',device='%s'}[5m]) / 1000 / rate(node_disk_writes_completed{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_ms{job='node',instance='%s',device='%s'}[5m]) / 1000 / rate(node_disk_writes_completed{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
   </tr>
 {{ end }}
   <th colspan="2">Filesystem Fullness</th>
@@ -46,7 +46,7 @@
   new PromConsole.Graph({
     node: document.querySelector("#diskioGraph"),
     expr: [
-      "rate(node_disk_io_time_ms{job='node',instance='{{ .Params.instance }}',device!~'^(md\\d+$|dm-)'}[5m]) / 1000 * 100",
+      "irate(node_disk_io_time_ms{job='node',instance='{{ .Params.instance }}',device!~'^(md\\d+$|dm-)'}[5m]) / 1000 * 100",
     ],
     min: 0,
     name: '[[ device ]]',

consoles/node-overview.html

@@ -4,11 +4,11 @@
   <tr><th colspan="2">Overview</th></tr>
   <tr>
     <td>User CPU</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(node_cpu{job='node',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu{job='node',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
   </tr>
   <tr>
     <td>System CPU</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(rate(node_cpu{job='node',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu{job='node',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
   </tr>
   <tr>
     <td>Memory Total</td>
@@ -24,11 +24,11 @@
 {{ range printf "node_network_receive_bytes{job='node',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }}
   <tr>
     <td>{{ .Labels.device }} Received</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_network_receive_bytes{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
   </tr>
   <tr>
     <td>{{ .Labels.device }} Transmitted</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_network_transmit_bytes{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
   </tr>
 {{ end }}
 </tr>
@@ -38,13 +38,13 @@
 {{ range printf "node_disk_io_time_ms{job='node',instance='%s',device!~'^(md\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }}
   <tr>
     <td>{{ .Labels.device }} Utilization</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_disk_io_time_ms{job='node',instance='%s',device='%s'}[5m]) / 1000 * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_ms{job='node',instance='%s',device='%s'}[5m]) / 1000 * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
   </tr>
 {{ end }}
 {{ range printf "node_disk_io_time_ms{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
   <tr>
     <td>{{ .Labels.device }} Throughput</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(node_disk_sectors_read{job='node',instance='%s',device='%s'}[5m]) * 512 + rate(node_disk_sectors_written{job='node',instance='%s',device='%s'}[5m]) * 512" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_sectors_read{job='node',instance='%s',device='%s'}[5m]) * 512 + rate(node_disk_sectors_written{job='node',instance='%s',device='%s'}[5m]) * 512" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
   </tr>
 {{ end }}
 <tr>
@@ -70,7 +70,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(rate(node_cpu{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
+    expr: "sum by (mode)(irate(node_cpu{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
     renderer: 'area',
     max: {{ with printf "count(count by (cpu)(node_cpu{job='node',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -85,7 +85,7 @@
   new PromConsole.Graph({
     node: document.querySelector("#diskioGraph"),
     expr: [
-      "rate(node_disk_io_time_ms{job='node',instance='{{ .Params.instance }}',device!~'^(md\\d+$|dm-)'}[5m]) / 1000 * 100",
+      "irate(node_disk_io_time_ms{job='node',instance='{{ .Params.instance }}',device!~'^(md\\d+$|dm-)'}[5m]) / 1000 * 100",
     ],
     min: 0,
     name: '[[ device ]]',

consoles/node.html

@@ -21,7 +21,7 @@
 <tr>
   <td><a href="node-overview.html?instance={{ .Labels.instance }}">{{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Labels.instance }}</a></td>
   <td{{ if eq (. | value) 1.0 }}>Yes{{ else }} class="alert-danger">No{{ end }}</td>
-  <td>{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance)(rate(node_cpu{job='node',mode='idle',instance='%s'}[5m])))" .Labels.instance) "%" "printf.1f") }}</td>
+  <td>{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance)(irate(node_cpu{job='node',mode='idle',instance='%s'}[5m])))" .Labels.instance) "%" "printf.1f") }}</td>
   <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree{job='node',instance='%s'} + node_memory_Cached{job='node',instance='%s'} + node_memory_Buffers{job='node',instance='%s'}" .Labels.instance .Labels.instance .Labels.instance) "B" "humanize1024") }}</td>
 </tr>
 {{ else }}

consoles/prometheus-overview.html

@@ -6,7 +6,7 @@
   </tr>
   <tr>
     <td>CPU</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Memory</td>
@@ -22,7 +22,7 @@
   </tr>
   <tr>
     <td>Ingested Samples</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(prometheus_local_storage_ingested_samples_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_local_storage_ingested_samples_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
   <tr>
     <td>Time Series</td>
@@ -54,11 +54,11 @@
   </tr>
   <tr>
     <td>Evaluation Duration</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(prometheus_evaluator_duration_milliseconds_sum{job='prometheus',instance='%s'}[5m]) / rate(prometheus_evaluator_duration_milliseconds_count{job='prometheus',instance='%s'}[5m]) / 1000" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_milliseconds_sum{job='prometheus',instance='%s'}[5m]) / rate(prometheus_evaluator_duration_milliseconds_count{job='prometheus',instance='%s'}[5m]) / 1000" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
   </tr>
   <tr>
     <td>Notification Latency</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(prometheus_notifications_latency_milliseconds_sum{job='prometheus',instance='%s'}[5m]) / rate(prometheus_notifications_latency_milliseconds_count{job='prometheus',instance='%s'}[5m]) / 1000" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_milliseconds_sum{job='prometheus',instance='%s'}[5m]) / rate(prometheus_notifications_latency_milliseconds_count{job='prometheus',instance='%s'}[5m]) / 1000" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
   </tr>
   <tr>
     <td>Notification Queue</td>
@@ -70,7 +70,7 @@
 {{ range printf "http_request_duration_microseconds_count{job='prometheus',instance='%s',handler=~'^(query.*|federate|consoles)$'}" .Params.instance | query | sortByLabel "handler" }}
   <tr>
     <td>{{ .Labels.handler }}</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "rate(http_request_duration_microseconds_count{job='prometheus',instance='%s',handler='%s'}[5m])" .Labels.instance .Labels.handler) "/s" "humanizeNoSmallPrefix") }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(http_request_duration_microseconds_count{job='prometheus',instance='%s',handler='%s'}[5m])" .Labels.instance .Labels.handler) "/s" "humanizeNoSmallPrefix") }}</td>
   </tr>
 {{ end }}
 
@@ -84,7 +84,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#samplesGraph"),
-    expr: "rate(prometheus_local_storage_ingested_samples_total{job='prometheus',instance='{{ .Params.instance }}'}[5m])",
+    expr: "irate(prometheus_local_storage_ingested_samples_total{job='prometheus',instance='{{ .Params.instance }}'}[5m])",
     name: 'Ingested Samples',
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
     yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
@@ -111,7 +111,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#serverGraph"),
-    expr: "rate(http_request_duration_microseconds_count{job='prometheus',instance='{{ .Params.instance }}',handler=~'^(query.*|federate|consoles)$'}[5m])",
+    expr: "irate(http_request_duration_microseconds_count{job='prometheus',instance='{{ .Params.instance }}',handler=~'^(query.*|federate|consoles)$'}[5m])",
     name: '[[handler]]',
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
     yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,

consoles/prometheus.html

@@ -22,7 +22,7 @@
 <tr>
   <td><a href="prometheus-overview.html?instance={{ .Labels.instance }}">{{ .Labels.instance }}</a></td>
   <td {{ if eq (. | value) 1.0 }}>Yes{{ else }} class="alert-danger">No{{ end }}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "rate(prometheus_local_storage_ingested_samples_total{job='prometheus',instance='%s'}[5m])" .Labels.instance) "/s" "humanizeNoSmallPrefix") }}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "irate(prometheus_local_storage_ingested_samples_total{job='prometheus',instance='%s'}[5m])" .Labels.instance) "/s" "humanizeNoSmallPrefix") }}</td>
   <td class="text-right">{{ template "prom_query_drilldown" (args (printf "prometheus_local_storage_memory_series{job='prometheus',instance='%s'}" .Labels.instance) "" "humanize") }}</td>
   <td class="text-right">{{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Labels.instance) "B" "humanize1024")}}</td>
 </tr>

consoles/snmp-overview.html

@@ -25,10 +25,10 @@
      {{ else if eq (. | value) 7.0}}">lowerLayerDown
      {{else}}">{{ end }}</td>
   <td class="text-right">{{ template "prom_query_drilldown" (args (printf "ifHighSpeed{job='snmp',instance='%s',ifDescr='%s'} * 1e6 or ifSpeed{job='snmp',instance='%s',ifDescr='%s'}" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "b/s" "humanize")}}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "rate(ifHCInOctets{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8 or rate(ifInOctets{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "b/s" "humanize")}}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "rate(ifHCOutOctets{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8 or rate(ifOutOctets{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "b/s" "humanize")}}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "rate(ifInDiscards{job='snmp',instance='%s',ifDescr='%s'}[5m]) + rate(ifOutDiscards{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "/s" "humanizeNoSmallPrefix")}}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "rate(ifInErrors{job='snmp',instance='%s',ifDescr='%s'}[5m]) + rate(ifOutErrors{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "/s" "humanizeNoSmallPrefix")}}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "irate(ifHCInOctets{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8 or rate(ifInOctets{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "b/s" "humanize")}}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "irate(ifHCOutOctets{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8 or rate(ifOutOctets{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "b/s" "humanize")}}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "irate(ifInDiscards{job='snmp',instance='%s',ifDescr='%s'}[5m]) + rate(ifOutDiscards{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "/s" "humanizeNoSmallPrefix")}}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "irate(ifInErrors{job='snmp',instance='%s',ifDescr='%s'}[5m]) + rate(ifOutErrors{job='snmp',instance='%s',ifDescr='%s'}[5m]) * 8" .Labels.instance .Labels.ifDescr .Labels.instance .Labels.ifDescr) "/s" "humanizeNoSmallPrefix")}}</td>
 </tr>
 {{ end }}
 {{ template "prom_content_tail" . }}

consoles/snmp.html

@@ -27,10 +27,10 @@
   <td {{ if eq (. | value) 1.0 }}>Yes{{ else }} class="alert-danger">No{{ end }}</td>
   <td class="text-right">{{ query (printf "ifOperStatus{job='snmp',instance='%s'} == 1" .Labels.instance) | len }}</td>
   <td class="text-right">{{ template "prom_query_drilldown" (args (printf "count(ifOperStatus{job='snmp',instance='%s'})" .Labels.instance) ) }}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "8 * sum by (instance)(rate(ifHCInOctets{job='snmp',instance='%s'}[5m]) or rate(ifInOctets{job='snmp',instance='%s'}[5m]))" .Labels.instance .Labels.instance) "b/s" "humanize")}}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "8 * sum by (instance)(rate(ifHCOutOctets{job='snmp',instance='%s'}[5m]) or rate(ifOutOctets{job='snmp',instance='%s'}[5m]))" .Labels.instance .Labels.instance) "b/s" "humanize")}}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "8 * sum by (instance)(rate(ifInDiscards{job='snmp',instance='%s'}[5m]) or rate(ifOutDiscards{job='snmp',instance='%s'}[5m]))" .Labels.instance .Labels.instance) "/s" "humanizeNoSmallPrefix")}}</td>
-  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "8 * sum by (instance)(rate(ifInErrors{job='snmp',instance='%s'}[5m]) or rate(ifOutErrors{job='snmp',instance='%s'}[5m]))" .Labels.instance .Labels.instance) "/s" "humanizeNoSmallPrefix")}}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "8 * sum by (instance)(irate(ifHCInOctets{job='snmp',instance='%s'}[5m]) or rate(ifInOctets{job='snmp',instance='%s'}[5m]))" .Labels.instance .Labels.instance) "b/s" "humanize")}}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "8 * sum by (instance)(irate(ifHCOutOctets{job='snmp',instance='%s'}[5m]) or rate(ifOutOctets{job='snmp',instance='%s'}[5m]))" .Labels.instance .Labels.instance) "b/s" "humanize")}}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "8 * sum by (instance)(irate(ifInDiscards{job='snmp',instance='%s'}[5m]) or rate(ifOutDiscards{job='snmp',instance='%s'}[5m]))" .Labels.instance .Labels.instance) "/s" "humanizeNoSmallPrefix")}}</td>
+  <td class="text-right">{{ template "prom_query_drilldown" (args (printf "8 * sum by (instance)(irate(ifInErrors{job='snmp',instance='%s'}[5m]) or rate(ifOutErrors{job='snmp',instance='%s'}[5m]))" .Labels.instance .Labels.instance) "/s" "humanizeNoSmallPrefix")}}</td>
 </tr>
 {{ else }}
 <tr><td colspan=4>No devices found.</td></tr>

promql/functions.go

@@ -127,6 +127,46 @@ func funcIncrease(ev *evaluator, args Expressions) model.Value {
 	return funcDelta(ev, args).(vector)
 }
 
+// === irate(node model.ValMatrix) Vector ===
+func funcIrate(ev *evaluator, args Expressions) model.Value {
+	resultVector := vector{}
+	for _, samples := range ev.evalMatrix(args[0]) {
+		// No sense in trying to compute a rate without at least two points. Drop
+		// this vector element.
+		if len(samples.Values) < 2 {
+			continue
+		}
+
+		lastSample := samples.Values[len(samples.Values)-1]
+		previousSample := samples.Values[len(samples.Values)-2]
+
+		var resultValue model.SampleValue
+		if lastSample.Value < previousSample.Value {
+			// Counter reset.
+			resultValue = lastSample.Value
+		} else {
+			resultValue = lastSample.Value - previousSample.Value
+		}
+
+		sampledInterval := lastSample.Timestamp.Sub(previousSample.Timestamp)
+		if sampledInterval == 0 {
+			// Avoid dividing by 0.
+			continue
+		}
+		// Convert to per-second.
+		resultValue /= model.SampleValue(sampledInterval.Seconds())
+
+		resultSample := &sample{
+			Metric:    samples.Metric,
+			Value:     resultValue,
+			Timestamp: ev.Timestamp,
+		}
+		resultSample.Metric.Del(model.MetricNameLabel)
+		resultVector = append(resultVector, resultSample)
+	}
+	return resultVector
+}
+
 // === sort(node model.ValVector) Vector ===
 func funcSort(ev *evaluator, args Expressions) model.Value {
 	byValueSorter := vectorByValueHeap(ev.evalVector(args[0]))
@@ -759,6 +799,12 @@ var functions = map[string]*Function{
 		ReturnType: model.ValVector,
 		Call:       funcHistogramQuantile,
 	},
+	"irate": {
+		Name:       "irate",
+		ArgTypes:   []model.ValueType{model.ValMatrix},
+		ReturnType: model.ValVector,
+		Call:       funcIrate,
+	},
 	"label_replace": {
 		Name:       "label_replace",
 		ArgTypes:   []model.ValueType{model.ValVector, model.ValString, model.ValString, model.ValString, model.ValString},

promql/testdata/functions.test

@@ -63,6 +63,21 @@ eval instant at 50m increase(http_requests[50m])
 	{path="/foo"} 100
 	{path="/bar"}  90
 
+clear
+
+# Tests for irate().
+load 5m
+	http_requests{path="/foo"}	0+10x10
+	http_requests{path="/bar"}	0+10x5 0+10x5
+
+eval instant at 50m irate(http_requests[50m])
+	{path="/foo"} .03333333333333333333
+	{path="/bar"} .03333333333333333333
+
+# Counter reset.
+eval instant at 30m irate(http_requests[50m])
+	{path="/foo"} .03333333333333333333
+	{path="/bar"} 0
 
 clear