This release changes the default for GOGC, the Go runtime control for the trade-off between excess memory use and CPU usage. We have found that Prometheus operates with minimal additional CPU usage, but greatly reduced memory by adjusting the upstream Go default from 100 to 50.
* [CHANGE] Rules: Execute 1 query instead of N (where N is the number of alerts within alert rule) when restoring alerts. #13980
* [CHANGE] Runtime: Change GOGC threshold from 100 to 50 #14176
* [FEATURE] Rules: Add new option `query_offset` for each rule group via rule group configuration file and `rule_query_offset` as part of the global configuration to have more resilience for remote write delays. #14061
* [ENHANCEMENT] Rules: Add `rule_group_last_restore_duration_seconds` to measure the time it takes to restore a rule group. #13974
* [ENHANCEMENT] OTLP: Improve remote write format translation performance by using label set hashes for metric identifiers instead of string based ones. #14006#13991
* [ENHANCEMENT] TSDB: Optimize querying with regexp matchers. #13620
* [BUGFIX] OTLP: Don't generate target_info unless at least one identifying label is defined. #13991
* [BUGFIX] OTLP: Don't generate target_info unless there are metrics. #13991
## 2.52.0-rc.1 / 2024-05-03
## 2.52.1 / 2024-05-29
* [BUGFIX] API: Fix missing comma during JSON encoding of API results. #14047
* [BUGFIX] Linode SD: Fix partial fetch when discovery would return more than 500 elements. #14141
## 2.52.0-rc.0 / 2024-04-22
## 2.52.0 / 2024-05-07
* [CHANGE] TSDB: Fix the predicate checking for blocks which are beyond the retention period to include the ones right at the retention boundary. #9633
* [FEATURE] Kubernetes SD: Add a new metric `prometheus_sd_kubernetes_failures_total` to track failed requests to Kubernetes API. #13554
make test # Make sure all the tests pass before you commit and push :)
```
We use [`golangci-lint`](https://github.com/golangci/golangci-lint) for linting the code. If it reports an issue and you think that the warning needs to be disregarded or is a false-positive, you can add a special comment `//nolint:linter1[,linter2,...]` before the offending line. Use this sparingly though, fixing the code to comply with the linter's recommendation is in general the preferred course of action.
To run a collection of Go linters through [`golangci-lint`](https://github.com/golangci/golangci-lint), do:
```bash
make lint
```
If it reports an issue and you think that the warning needs to be disregarded or is a false-positive, you can add a special comment `//nolint:linter1[,linter2,...]` before the offending line. Use this sparingly though, fixing the code to comply with the linter's recommendation is in general the preferred course of action. See [this section of the golangci-lint documentation](https://golangci-lint.run/usage/false-positives/#nolint-directive) for more information.
All our issues are regularly tagged so that you can also filter down the issues involving the components you want to work on. For our labeling policy refer [the wiki page](https://github.com/prometheus/prometheus/wiki/Label-Names-and-Descriptions).
serverOnlyFlag(a,"rules.alert.resend-delay","Minimum amount of time to wait before resending an alert to Alertmanager.").
Default("1m").SetValue(&cfg.resendDelay)
serverOnlyFlag(a,"rules.max-concurrent-evals","Global concurrency limit for independent rules that can run concurrently.").
serverOnlyFlag(a,"rules.max-concurrent-evals","Global concurrency limit for independent rules that can run concurrently. When set, \"query.max-concurrency\" may need to be adjusted accordingly.").
Default("4").Int64Var(&cfg.maxConcurrentEvals)
a.Flag("scrape.adjust-timestamps","Adjust scrape timestamps by up to `scrape.timestamp-tolerance` to align them to the intended schedule. See https://github.com/prometheus/prometheus/issues/7846 for more context. Experimental. This flag will be removed in a future release.").
tsdbDumpCmd:=tsdbCmd.Command("dump","Dump samples from a TSDB.")
dumpPath:=tsdbDumpCmd.Arg("db path","Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
dumpSandboxDirRoot:=tsdbDumpCmd.Flag("sandbox-dir-root","Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end.").Default(defaultDBPath).String()
dumpMinTime:=tsdbDumpCmd.Flag("min-time","Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64,10)).Int64()
dumpMaxTime:=tsdbDumpCmd.Flag("max-time","Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64,10)).Int64()
dumpMatch:=tsdbDumpCmd.Flag("match","Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
tsdbDumpOpenMetricsCmd:=tsdbCmd.Command("dump-openmetrics","[Experimental] Dump samples from a TSDB into OpenMetrics format. Native histograms are not dumped.")
tsdbDumpOpenMetricsCmd:=tsdbCmd.Command("dump-openmetrics","[Experimental] Dump samples from a TSDB into OpenMetrics text format, excluding native histograms and staleness markers, which are not representable in OpenMetrics.")
dumpOpenMetricsPath:=tsdbDumpOpenMetricsCmd.Arg("db path","Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
dumpOpenMetricsSandboxDirRoot:=tsdbDumpOpenMetricsCmd.Flag("sandbox-dir-root","Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end.").Default(defaultDBPath).String()
dumpOpenMetricsMinTime:=tsdbDumpOpenMetricsCmd.Flag("min-time","Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64,10)).Int64()
dumpOpenMetricsMaxTime:=tsdbDumpOpenMetricsCmd.Flag("max-time","Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64,10)).Int64()
dumpOpenMetricsMatch:=tsdbDumpOpenMetricsCmd.Flag("match","Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
// Offset the rule evaluation timestamp of this particular group by the specified duration into the past to ensure the underlying metrics have been received.
| <codeclass="text-nowrap">--rules.alert.for-outage-tolerance</code> | Max time to tolerate prometheus outage for restoring "for" state of alert. Use with server mode only. | `1h` |
| <codeclass="text-nowrap">--rules.alert.for-grace-period</code> | Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. Use with server mode only. | `10m` |
| <codeclass="text-nowrap">--rules.alert.resend-delay</code> | Minimum amount of time to wait before resending an alert to Alertmanager. Use with server mode only. | `1m` |
| <codeclass="text-nowrap">--rules.max-concurrent-evals</code> | Global concurrency limit for independent rules that can run concurrently. Use with server mode only. | `4` |
| <codeclass="text-nowrap">--rules.max-concurrent-evals</code> | Global concurrency limit for independent rules that can run concurrently. When set, "query.max-concurrency" may need to be adjusted accordingly. Use with server mode only. | `4` |
| <codeclass="text-nowrap">--alertmanager.notification-queue-capacity</code> | The capacity of the queue for pending Alertmanager notifications. Use with server mode only. | `10000` |
| <codeclass="text-nowrap">--query.lookback-delta</code> | The maximum lookback duration for retrieving metrics during expression evaluations and federation. Use with server mode only. | `5m` |
| <codeclass="text-nowrap">--query.timeout</code> | Maximum time a query may take before being aborted. Use with server mode only. | `2m` |
| <codeclass="text-nowrap">--sandbox-dir-root</code> | Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end. | `data/` |
| <codeclass="text-nowrap">--min-time</code> | Minimum timestamp to dump. | `-9223372036854775808` |
| <codeclass="text-nowrap">--max-time</code> | Maximum timestamp to dump. | `9223372036854775807` |
| <codeclass="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` |
@ -584,7 +585,7 @@ Dump samples from a TSDB.
##### `promtool tsdb dump-openmetrics`
[Experimental] Dump samples from a TSDB into OpenMetrics format. Native histograms are not dumped.
[Experimental] Dump samples from a TSDB into OpenMetrics text format, excluding native histograms and staleness markers, which are not representable in OpenMetrics.
@ -592,6 +593,7 @@ Dump samples from a TSDB.
| Flag | Description | Default |
| --- | --- | --- |
| <codeclass="text-nowrap">--sandbox-dir-root</code> | Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end. | `data/` |
| <codeclass="text-nowrap">--min-time</code> | Minimum timestamp to dump. | `-9223372036854775808` |
| <codeclass="text-nowrap">--max-time</code> | Maximum timestamp to dump. | `9223372036854775807` |
| <codeclass="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` |
# Offset the rule evaluation timestamp of this particular group by the specified duration into the past to ensure the underlying metrics have been received.
# Metric availability delays are more likely to occur when Prometheus is running as a remote write target, but can also occur when there's anomalies with scraping.
[ rule_query_offset: <duration> | default = 0s ]
# The labels to add to any time series or alerts when communicating with
# external systems (federation, remote storage, Alertmanager).
@ -117,6 +121,12 @@ global:
# that will be kept in memory. 0 means no limit.
[ keep_dropped_targets: <int> | default = 0 ]
runtime:
# Configure the Go garbage collector GOGC parameter
# See: https://tip.golang.org/doc/gc-guide#GOGC
# Lowering this number increases CPU usage.
[ gogc: <int> | default = 50 ]
# Rule files specifies a list of globs. Rules and alerts are read from
# all matching files.
rule_files:
@ -1349,7 +1359,7 @@ interface.
The following meta labels are available on targets during [relabeling](#relabel_config):
* `__meta_openstack_address_pool`: the pool of the private IP.
* `__meta_openstack_instance_flavor`: the flavor of the OpenStack instance.
* `__meta_openstack_instance_flavor`: the flavor ID of the OpenStack instance.
* `__meta_openstack_instance_id`: the OpenStack instance ID.
* `__meta_openstack_instance_image`: the ID of the image the OpenStack instance is using.
* `__meta_openstack_instance_name`: the OpenStack instance name.
@ -1357,7 +1367,7 @@ The following meta labels are available on targets during [relabeling](#relabel_
* `__meta_openstack_private_ip`: the private IP of the OpenStack instance.
* `__meta_openstack_project_id`: the project (tenant) owning this instance.
* `__meta_openstack_public_ip`: the public IP of the OpenStack instance.
* `__meta_openstack_tag_<tagkey>`: each tag value of the instance.
* `__meta_openstack_tag_<key>`: each metadata item of the instance, with any unsupported characters converted to an underscore.
* `__meta_openstack_user_id`: the user account owning the tenant.
See below for the configuration options for OpenStack discovery:
@ -1467,6 +1477,7 @@ For OVHcloud's [public cloud instances](https://www.ovhcloud.com/en/public-cloud
* `__meta_ovhcloud_dedicated_server_ipv6`: the IPv6 of the server
* `__meta_ovhcloud_dedicated_server_link_speed`: the link speed of the server
* `__meta_ovhcloud_dedicated_server_name`: the name of the server
* `__meta_ovhcloud_dedicated_server_no_intervention`: whether datacenter intervention is disabled for the server
* `__meta_ovhcloud_dedicated_server_os`: the operating system of the server
* `__meta_ovhcloud_dedicated_server_rack`: the rack of the server
* `__meta_ovhcloud_dedicated_server_reverse`: the reverse DNS name of the server
@ -3673,7 +3684,8 @@ queue_config:
[ min_shards: <int> | default = 1 ]
# Maximum number of samples per send.
[ max_samples_per_send: <int> | default = 2000]
# Maximum time a sample will wait in buffer.
# Maximum time a sample will wait for a send. The sample might wait less
# if the buffer is full. Further time might pass due to potential retries.
@ -148,6 +151,9 @@ the rule, active, pending, or inactive, are cleared as well. The event will be
recorded as an error in the evaluation, and as such no stale markers are
written.
# Rule query offset
This is useful to ensure the underlying metrics have been received and stored in Prometheus. Metric availability delays are more likely to occur when Prometheus is running as a remote write target due to the nature of distributed systems, but can also occur when there's anomalies with scraping and/or short evaluation intervals.
# Failed rule evaluations due to slow evaluation
If a rule group hasn't finished evaluating before its next evaluation is supposed to start (as defined by the `evaluation_interval`), the next evaluation will be skipped. Subsequent evaluations of the rule group will continue to be skipped until the initial evaluation either completes or times out. When this happens, there will be a gap in the metric produced by the recording rule. The `rule_group_iterations_missed_total` metric will be incremented for each missed iteration of the rule group.
'count by (cluster, job, instance, version) (prometheus_build_info{cluster=~"$cluster", job=~"$job", instance=~"$instance"})',
'max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~"$cluster", job=~"$job", instance=~"$instance"})',
] else [
'count by (job, instance, version) (prometheus_build_info{job=~"$job", instance=~"$instance"})',
'max by (job, instance) (time() - process_start_time_seconds{job=~"$job", instance=~"$instance"})',
], {
cluster: { alias: 'Cluster' },
cluster: { alias: if showMultiCluster then 'Cluster' else '' },
job: { alias: 'Job' },
instance: { alias: 'Instance' },
version: { alias: 'Version' },
@ -37,12 +48,18 @@ local template = grafana.template;
g.row('Discovery')
.addPanel(
g.panel('Target Sync') +
g.queryPanel('sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[5m])) by (cluster, job, scrape_job, instance) * 1e3', '{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}') +
g.queryPanel(if showMultiCluster then 'sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[5m])) by (cluster, job, scrape_job, instance) * 1e3'
else 'sum(rate(prometheus_target_sync_length_seconds_sum{job=~"$job",instance=~"$instance"}[5m])) by (scrape_job) * 1e3',
if showMultiCluster then '{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}'
else '{{scrape_job}}') +
{ yaxes: g.yaxes('ms') }
)
.addPanel(
g.panel('Targets') +
g.queryPanel('sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~"$cluster", job=~"$job",instance=~"$instance"})', '{{cluster}}:{{job}}:{{instance}}') +
g.queryPanel(if showMultiCluster then 'sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~"$cluster", job=~"$job",instance=~"$instance"})'
g.queryPanel(if showMultiCluster then 'rate(prometheus_engine_query_duration_seconds_count{cluster=~"$cluster",job=~"$job",instance=~"$instance",slice="inner_eval"}[5m])'
if showMultiCluster then '{{cluster}} {{job}} {{instance}}'
else '{{job}} {{instance}}') +
g.stack,
)
.addPanel(
g.panel('Stage Duration') +
g.queryPanel('max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",cluster=~"$cluster", job=~"$job",instance=~"$instance"}) * 1e3', '{{slice}}') +
g.queryPanel(if showMultiCluster then 'max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",cluster=~"$cluster", job=~"$job",instance=~"$instance"}) * 1e3'
else 'max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",job=~"$job",instance=~"$instance"}) * 1e3',
* an implementation of a test scripting language for PromQL engines
* a predefined set of tests written in that scripting language
The predefined set of tests can be run against any PromQL engine implementation by calling `promqltest.RunBuiltinTests()`.
Any other test script can be run with `promqltest.RunTest()`.
The rest of this document explains the test scripting language.
Each test script is written in plain text.
Comments can be given by prefixing the comment with a `#`, for example:
```
# This is a comment.
```
Each test file contains a series of commands. There are three kinds of commands:
* `load`
* `clear`
* `eval`
Each command is executed in the order given in the file.
## `load` command
`load` adds some data to the test environment.
The syntax is as follows:
```
load <interval>
<series><points>
...
<series><points>
```
* `<interval>` is the step between points (eg. `1m` or `30s`)
* `<series>` is a Prometheus series name in the usual `metric{label="value"}` syntax
* `<points>` is a specification of the points to add for that series, following the same expanding syntax as for `promtool unittest` documented [here](../../docs/configuration/unit_testing_rules.md#series)
...will create a single series with labels `my_metric{env="prod"}`, with the following points:
* t=0: value is 5
* t=1m: value is 2
* t=2m: value is 5
* t=3m: value is 7
* t=4m: no point
* t=5m: stale marker
* t=6m: native histogram with schema 1, sum -3, count 22 and bucket counts 5, 10 and 7
Each `load` command is additive - it does not replace any data loaded in a previous `load` command.
Use `clear` to remove all loaded data.
## `clear` command
`clear` removes all data previously loaded with `load` commands.
## `eval` command
`eval` runs a query against the test environment and asserts that the result is as expected.
Both instant and range queries are supported.
The syntax is as follows:
```
# Instant query
eval instant at <time><query>
<series><points>
...
<series><points>
# Range query
eval range from <start> to <end> step <step><query>
<series><points>
...
<series><points>
```
* `<time>` is the timestamp to evaluate the instant query at (eg. `1m`)
* `<start>` and `<end>` specify the time range of the range query, and use the same syntax as `<time>`
* `<step>` is the step of the range query, and uses the same syntax as `<time>` (eg. `30s`)
* `<series>` and `<points>` specify the expected values, and follow the same syntax as for `load` above
For example:
```
eval instant at 1m sum by (env) (my_metric)
{env="prod"} 5
{env="test"} 20
eval range from 0 to 3m step 1m sum by (env) (my_metric)
{env="prod"} 2 5 10 20
{env="test"} 10 20 30 45
```
Instant queries also support asserting that the series are returned in exactly the order specified: use `eval_ordered instant ...` instead of `eval instant ...`.
This is not supported for range queries.
It is also possible to test that queries fail: use `eval_fail instant ...` or `eval_fail range ...`.
`eval_fail` optionally takes an expected error message string or regexp to assert that the error message is as expected.
For example:
```
# Assert that the query fails for any reason without asserting on the error message.
eval_fail instant at 1m ceil({__name__=~'testmetric1|testmetric2'})
# Assert that the query fails with exactly the provided error message string.
eval_fail instant at 1m ceil({__name__=~'testmetric1|testmetric2'})
expected_fail_message vector cannot contain metrics with the same labelset
# Assert that the query fails with an error message matching the regexp provided.
eval_fail instant at 1m ceil({__name__=~'testmetric1|testmetric2'})
expected_fail_regexp (vector cannot contain metrics .*|something else went wrong)
expectedError:`expected error "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
},
"instant query expected to fail with error matching pattern, and query fails with that error":{
expectedError:`expected error matching pattern "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
},
"instant query expected to fail with error matching pattern, and pattern is not a valid regexp":{
expectedError:`expected error "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
},
"range query expected to fail with error matching pattern, and query fails with that error":{
expectedError:`expected error matching pattern "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
},
"range query expected to fail with error matching pattern, and pattern is not a valid regexp":{
require.Equal(t,len(test.result),len(filteredRes),"%d. Number of samples in expected and actual output don't match (%d vs. %d)",i,len(test.result),len(res))
varfilteredRespromql.Vector// After removing 'ALERTS' samples.
for_,smpl:=rangeres{
smplName:=smpl.Metric.Get("__name__")
ifsmplName=="ALERTS_FOR_STATE"{
filteredRes=append(filteredRes,smpl)
}else{
// If not 'ALERTS_FOR_STATE', it has to be 'ALERTS'.
require.Equal(t,"ALERTS",smplName)
}
}
fori:=rangetest.result{
test.result[i].T=timestamp.FromTime(evalTime)
// Updating the expected 'for' state.
iftest.result[i].F>=0{
test.result[i].F=forState
for_,aa:=rangerule.ActiveAlerts(){
require.Zero(t,aa.Labels.Get(model.MetricNameLabel),"%s label set on active alert: %s",model.MetricNameLabel,aa.Labels)
}
}
}
require.Equal(t,len(test.result),len(filteredRes),"%d. Number of samples in expected and actual output don't match (%d vs. %d)",i,len(test.result),len(res))
require.True(t,ok,"Series %s not returned.",metric)
require.True(t,value.IsStaleNaN(metricSample[2].F),"Appended second sample not as expected. Wanted: stale NaN Got: %x",math.Float64bits(metricSample[2].F))
require.True(t,ok,"Series %s not returned.",metric)
require.True(t,value.IsStaleNaN(metricSample[2].F),"Appended second sample not as expected. Wanted: stale NaN Got: %x",math.Float64bits(metricSample[2].F))
Jeanette Tan
2 years ago