apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Alerting: Extract ticker into shared package (#55703)

Browse Source 
* Move ticker files to dedicated package with no changes

* Fix package naming and resolve naming conflicts

* Fix up all existing references to moved objects

* Remove all alerting-specific references from shared util

* Rename TickerMetrics to simply Metrics

* Rename base ticker type to T and rename NewTicker to simply New
pull/55771/head^2
Alexander Weaver 3 years ago committed by GitHub
parent 53c61b49bf
commit bd6a5c900f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 44 additions and 47 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      pkg/services/alerting/engine.go
  2. 6
      pkg/services/ngalert/metrics/ngalert.go
  3. 8
      pkg/services/ngalert/schedule/schedule.go
  4. 14
      pkg/util/ticker/metrics.go
  5. 17
      pkg/util/ticker/ticker.go
  6. 40
      pkg/util/ticker/ticker_test.go

6
pkg/services/alerting/engine.go
Unescape View File

@ -16,7 +16,6 @@ import (
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/metrics"
"github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/datasources"
@ -25,6 +24,7 @@ import (
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/legacydata"
"github.com/grafana/grafana/pkg/util/ticker"
)
// AlertEngine is the background process that
@ -37,7 +37,7 @@ type AlertEngine struct {
Cfg *setting.Cfg
execQueue chan *Job
ticker *Ticker
ticker *ticker.T
scheduler scheduler
evalHandler evalHandler
ruleReader ruleReader
@ -90,7 +90,7 @@ func ProvideAlertEngine(renderer rendering.Service, requestValidator models.Plug
// Run starts the alerting service background process.
func (e *AlertEngine) Run(ctx context.Context) error {
reg := prometheus.WrapRegistererWithPrefix("legacy_", prometheus.DefaultRegisterer)
e.ticker = NewTicker(clock.New(), 1*time.Second, metrics.NewTickerMetrics(reg))
e.ticker = ticker.New(clock.New(), 1*time.Second, ticker.NewMetrics(reg, "alerting"))
defer e.ticker.Stop()
alertGroup, ctx := errgroup.WithContext(ctx)
alertGroup.Go(func() error { return e.alertingTicker(ctx) })

6
pkg/services/ngalert/metrics/ngalert.go
Unescape View File

@ -13,8 +13,8 @@ import (
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/models"
legacyMetrics "github.com/grafana/grafana/pkg/services/alerting/metrics"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/util/ticker"
"github.com/grafana/grafana/pkg/web"
)
@ -55,7 +55,7 @@ type Scheduler struct {
SchedulableAlertRules prometheus.Gauge
SchedulableAlertRulesHash prometheus.Gauge
UpdateSchedulableAlertRulesDuration prometheus.Histogram
Ticker *legacyMetrics.Ticker
Ticker *ticker.Metrics
EvaluationMissed *prometheus.CounterVec
}
@ -199,7 +199,7 @@ func newSchedulerMetrics(r prometheus.Registerer) *Scheduler {
Buckets: []float64{0.1, 0.25, 0.5, 1, 2, 5, 10},
},
),
Ticker: legacyMetrics.NewTickerMetrics(r),
Ticker: ticker.NewMetrics(r, "alerting"),
EvaluationMissed: promauto.With(r).NewCounterVec(
prometheus.CounterOpts{
Namespace: Namespace,

8
pkg/services/ngalert/schedule/schedule.go
Unescape View File

@ -10,7 +10,6 @@ import (
prometheusModel "github.com/prometheus/common/model"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/alerting"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
@ -21,6 +20,7 @@ import (
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util/ticker"
"github.com/benbjohnson/clock"
"golang.org/x/sync/errgroup"
@ -68,7 +68,7 @@ type schedule struct {
clock clock.Clock
ticker *alerting.Ticker
ticker *ticker.T
// evalApplied is only used for tests: test code can set it to non-nil
// function, and then it'll be called from the event loop whenever the
@ -119,7 +119,7 @@ type SchedulerCfg struct {
// NewScheduler returns a new schedule.
func NewScheduler(cfg SchedulerCfg, appURL *url.URL, stateManager *state.Manager) *schedule {
ticker := alerting.NewTicker(cfg.C, cfg.Cfg.BaseInterval, cfg.Metrics.Ticker)
ticker := ticker.New(cfg.C, cfg.Cfg.BaseInterval, cfg.Metrics.Ticker)
sch := schedule{
registry: alertRuleInfoRegistry{alertRuleInfo: make(map[ngmodels.AlertRuleKey]*alertRuleInfo)},
@ -449,7 +449,7 @@ func (sch *schedule) overrideCfg(cfg SchedulerCfg) {
sch.clock = cfg.C
sch.baseInterval = cfg.Cfg.BaseInterval
sch.ticker.Stop()
sch.ticker = alerting.NewTicker(cfg.C, cfg.Cfg.BaseInterval, cfg.Metrics.Ticker)
sch.ticker = ticker.New(cfg.C, cfg.Cfg.BaseInterval, cfg.Metrics.Ticker)
sch.evalAppliedFunc = cfg.EvalAppliedFunc
sch.stopAppliedFunc = cfg.StopAppliedFunc
}

14
pkg/services/alerting/metrics/metrics.go → pkg/util/ticker/metrics.go
Unescape View File

@ -1,33 +1,33 @@
package metrics
package ticker
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
type Ticker struct {
type Metrics struct {
LastTickTime prometheus.Gauge
NextTickTime prometheus.Gauge
IntervalSeconds prometheus.Gauge
}
func NewTickerMetrics(reg prometheus.Registerer) *Ticker {
return &Ticker{
func NewMetrics(reg prometheus.Registerer, subsystem string) *Metrics {
return &Metrics{
LastTickTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
Namespace: "grafana",
Subsystem: "alerting",
Subsystem: subsystem,
Name: "ticker_last_consumed_tick_timestamp_seconds",
Help: "Timestamp of the last consumed tick in seconds.",
}),
NextTickTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
Namespace: "grafana",
Subsystem: "alerting",
Subsystem: subsystem,
Name: "ticker_next_tick_timestamp_seconds",
Help: "Timestamp of the next tick in seconds before it is consumed.",
}),
IntervalSeconds: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
Namespace: "grafana",
Subsystem: "alerting",
Subsystem: subsystem,
Name: "ticker_interval_seconds",
Help: "Interval at which the ticker is meant to tick.",
}),

17
pkg/services/alerting/ticker.go → pkg/util/ticker/ticker.go
Unescape View File

@ -1,4 +1,4 @@
package alerting
package ticker
import (
"fmt"
@ -7,29 +7,28 @@ import (
"github.com/benbjohnson/clock"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/alerting/metrics"
)
// Ticker is a ticker to power the alerting scheduler. it's like a time.Ticker, except:
// Ticker emits ticks at regular time intervals. it's like a time.Ticker, except:
// - it doesn't drop ticks for slow receivers, rather, it queues up. so that callers are in control to instrument what's going on.
// - it ticks on interval marks or very shortly after. this provides a predictable load pattern
// (this shouldn't cause too much load contention issues because the next steps in the pipeline just process at their own pace)
// - the timestamps are used to mark "last datapoint to query for" and as such, are a configurable amount of seconds in the past
type Ticker struct {
type T struct {
C chan time.Time
clock clock.Clock
last time.Time
interval time.Duration
metrics *metrics.Ticker
metrics *Metrics
stopCh chan struct{}
}
// NewTicker returns a Ticker that ticks on interval marks (or very shortly after) starting at c.Now(), and never drops ticks. interval should not be negative or zero.
func NewTicker(c clock.Clock, interval time.Duration, metric *metrics.Ticker) *Ticker {
func New(c clock.Clock, interval time.Duration, metric *Metrics) *T {
if interval <= 0 {
panic(fmt.Errorf("non-positive interval [%v] is not allowed", interval))
}
t := &Ticker{
t := &T{
C: make(chan time.Time),
clock: c,
last: getStartTick(c, interval),
@ -47,7 +46,7 @@ func getStartTick(clk clock.Clock, interval time.Duration) time.Time {
return time.Unix(0, nano-(nano%interval.Nanoseconds()))
}
func (t *Ticker) run() {
func (t *T) run() {
logger := log.New("ticker")
logger.Info("starting", "first_tick", t.last.Add(t.interval))
LOOP:
@ -77,7 +76,7 @@ LOOP:
}
// Stop stops the ticker. It does not close the C channel
func (t *Ticker) Stop() {
func (t *T) Stop() {
select {
case t.stopCh <- struct{}{}:
default:

40
pkg/services/alerting/ticker_test.go → pkg/util/ticker/ticker_test.go
Unescape View File

@ -1,4 +1,4 @@
package alerting
package ticker
import (
"bytes"
@ -13,8 +13,6 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/services/alerting/metrics"
)
func TestTicker(t *testing.T) {
@ -53,7 +51,7 @@ func TestTicker(t *testing.T) {
interval := time.Duration(rand.Int63n(100)+10) * time.Second
clk := clock.NewMock()
clk.Add(interval) // align clock with the start tick
ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(prometheus.NewRegistry()))
ticker := New(clk, interval, NewMetrics(prometheus.NewRegistry(), "test"))
ticks := rand.Intn(9) + 1
jitter := rand.Int63n(int64(interval) - 1)
@ -87,7 +85,7 @@ func TestTicker(t *testing.T) {
t.Run("should not put anything to channel until it's time", func(t *testing.T) {
clk := clock.NewMock()
interval := time.Duration(rand.Int63n(9)+1) * time.Second
ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(prometheus.NewRegistry()))
ticker := New(clk, interval, NewMetrics(prometheus.NewRegistry(), "test"))
expectedTick := clk.Now().Add(interval)
for {
require.Empty(t, ticker.C)
@ -104,7 +102,7 @@ func TestTicker(t *testing.T) {
t.Run("should put the tick in the channel immediately if it is behind", func(t *testing.T) {
clk := clock.NewMock()
interval := time.Duration(rand.Int63n(9)+1) * time.Second
ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(prometheus.NewRegistry()))
ticker := New(clk, interval, NewMetrics(prometheus.NewRegistry(), "test"))
// We can expect the first tick to be at a consistent interval. Take a snapshot of the clock now, before we advance it.
expectedTick := clk.Now().Add(interval)
@ -133,25 +131,25 @@ func TestTicker(t *testing.T) {
clk.Set(time.Now())
interval := time.Duration(rand.Int63n(9)+1) * time.Second
registry := prometheus.NewPedanticRegistry()
ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(registry))
ticker := New(clk, interval, NewMetrics(registry, "test"))
expectedTick := getStartTick(clk, interval).Add(interval)
expectedMetricFmt := `# HELP grafana_alerting_ticker_interval_seconds Interval at which the ticker is meant to tick.
# TYPE grafana_alerting_ticker_interval_seconds gauge
grafana_alerting_ticker_interval_seconds %v
# HELP grafana_alerting_ticker_last_consumed_tick_timestamp_seconds Timestamp of the last consumed tick in seconds.
# TYPE grafana_alerting_ticker_last_consumed_tick_timestamp_seconds gauge
grafana_alerting_ticker_last_consumed_tick_timestamp_seconds %v
# HELP grafana_alerting_ticker_next_tick_timestamp_seconds Timestamp of the next tick in seconds before it is consumed.
# TYPE grafana_alerting_ticker_next_tick_timestamp_seconds gauge
grafana_alerting_ticker_next_tick_timestamp_seconds %v
expectedMetricFmt := `# HELP grafana_test_ticker_interval_seconds Interval at which the ticker is meant to tick.
# TYPE grafana_test_ticker_interval_seconds gauge
grafana_test_ticker_interval_seconds %v
# HELP grafana_test_ticker_last_consumed_tick_timestamp_seconds Timestamp of the last consumed tick in seconds.
# TYPE grafana_test_ticker_last_consumed_tick_timestamp_seconds gauge
grafana_test_ticker_last_consumed_tick_timestamp_seconds %v
# HELP grafana_test_ticker_next_tick_timestamp_seconds Timestamp of the next tick in seconds before it is consumed.
# TYPE grafana_test_ticker_next_tick_timestamp_seconds gauge
grafana_test_ticker_next_tick_timestamp_seconds %v
`
expectedMetric := fmt.Sprintf(expectedMetricFmt, interval.Seconds(), 0, float64(expectedTick.UnixNano())/1e9)
errs := make(map[string]error, 1)
require.Eventuallyf(t, func() bool {
err := testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetric), "grafana_alerting_ticker_last_consumed_tick_timestamp_seconds", "grafana_alerting_ticker_next_tick_timestamp_seconds", "grafana_alerting_ticker_interval_seconds")
err := testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetric), "grafana_test_ticker_last_consumed_tick_timestamp_seconds", "grafana_test_ticker_next_tick_timestamp_seconds", "grafana_test_ticker_interval_seconds")
if err != nil {
errs["error"] = err
}
@ -164,7 +162,7 @@ func TestTicker(t *testing.T) {
expectedMetric = fmt.Sprintf(expectedMetricFmt, interval.Seconds(), float64(actual.UnixNano())/1e9, float64(expectedTick.Add(interval).UnixNano())/1e9)
require.Eventuallyf(t, func() bool {
err := testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetric), "grafana_alerting_ticker_last_consumed_tick_timestamp_seconds", "grafana_alerting_ticker_next_tick_timestamp_seconds", "grafana_alerting_ticker_interval_seconds")
err := testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetric), "grafana_test_ticker_last_consumed_tick_timestamp_seconds", "grafana_test_ticker_next_tick_timestamp_seconds", "grafana_test_ticker_interval_seconds")
if err != nil {
errs["error"] = err
}
@ -176,7 +174,7 @@ func TestTicker(t *testing.T) {
t.Run("when it waits for the next tick", func(t *testing.T) {
clk := clock.NewMock()
interval := time.Duration(rand.Int63n(9)+1) * time.Second
ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(prometheus.NewRegistry()))
ticker := New(clk, interval, NewMetrics(prometheus.NewRegistry(), "test"))
clk.Add(interval)
readChanOrFail(t, ticker.C)
ticker.Stop()
@ -187,7 +185,7 @@ func TestTicker(t *testing.T) {
t.Run("when it waits for the tick to be consumed", func(t *testing.T) {
clk := clock.NewMock()
interval := time.Duration(rand.Int63n(9)+1) * time.Second
ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(prometheus.NewRegistry()))
ticker := New(clk, interval, NewMetrics(prometheus.NewRegistry(), "test"))
clk.Add(interval)
ticker.Stop()
require.Empty(t, ticker.C)
@ -196,7 +194,7 @@ func TestTicker(t *testing.T) {
t.Run("multiple times", func(t *testing.T) {
clk := clock.NewMock()
interval := time.Duration(rand.Int63n(9)+1) * time.Second
ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(prometheus.NewRegistry()))
ticker := New(clk, interval, NewMetrics(prometheus.NewRegistry(), "test"))
ticker.Stop()
ticker.Stop()
ticker.Stop()
Write Preview
Loading…
Cancel
Save