diff --git a/main.go b/main.go index 343b4a21d5..fd7b04052e 100644 --- a/main.go +++ b/main.go @@ -85,6 +85,7 @@ type prometheus struct { unwrittenSamples chan *extraction.Result ruleManager rules.RuleManager + targetManager retrieval.TargetManager notifications chan notification.NotificationReqs storage *metric.TieredStorage @@ -174,11 +175,15 @@ func (p *prometheus) close() { p.deletionTimer.Stop() } + // Stop any currently active curation (deletion or compaction). if len(p.stopBackgroundOperations) == 0 { p.stopBackgroundOperations <- true } p.ruleManager.Stop() + p.targetManager.Stop() + close(p.unwrittenSamples) + p.storage.Close() close(p.notifications) @@ -294,6 +299,7 @@ func main() { stopBackgroundOperations: make(chan bool, 1), ruleManager: ruleManager, + targetManager: targetManager, notifications: notifications, storage: ts, } diff --git a/retrieval/targetmanager.go b/retrieval/targetmanager.go index 5c0a62e250..f5ca82f638 100644 --- a/retrieval/targetmanager.go +++ b/retrieval/targetmanager.go @@ -29,6 +29,7 @@ type TargetManager interface { ReplaceTargets(job config.JobConfig, newTargets []Target) Remove(t Target) AddTargetsFromConfig(config config.Config) + Stop() Pools() map[string]*TargetPool } @@ -115,6 +116,13 @@ func (m *targetManager) AddTargetsFromConfig(config config.Config) { } } +func (m *targetManager) Stop() { + glog.Info("Target manager exiting...") + for _, p := range m.poolsByJob { + p.Stop() + } +} + // XXX: Not really thread-safe. Only used in /status page for now. func (m *targetManager) Pools() map[string]*TargetPool { return m.poolsByJob diff --git a/retrieval/targetpool.go b/retrieval/targetpool.go index c1ba2e5f81..b474310f70 100644 --- a/retrieval/targetpool.go +++ b/retrieval/targetpool.go @@ -47,6 +47,7 @@ func NewTargetPool(m TargetManager, p TargetProvider) *TargetPool { addTargetQueue: make(chan Target, targetAddQueueSize), replaceTargetsQueue: make(chan targets, targetReplaceQueueSize), targetProvider: p, + done: make(chan bool), } } diff --git a/rules/manager.go b/rules/manager.go index 19179e7b21..e585bf7c3f 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -91,8 +91,8 @@ func (m *ruleManager) Run() { m.runIteration(m.results) evalDurations.Add(map[string]string{intervalKey: m.interval.String()}, float64(time.Since(start)/time.Millisecond)) case <-m.done: - glog.Info("RuleManager exiting...") - break + glog.Info("Rule manager exiting...") + return } } }