[new feature] promtail: Add config reload endoint / signal to promtail (#7247)

clients/cmd/promtail/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"sync"
 
 	// embed time zone data
 	_ "time/tzdata"
@@ -20,11 +21,12 @@ import (
 	"github.com/grafana/loki/clients/pkg/logentry/stages"
 	"github.com/grafana/loki/clients/pkg/promtail"
 	"github.com/grafana/loki/clients/pkg/promtail/client"
-	"github.com/grafana/loki/clients/pkg/promtail/config"
+	promtail_config "github.com/grafana/loki/clients/pkg/promtail/config"
 
 	"github.com/grafana/loki/pkg/util"
-	_ "github.com/grafana/loki/pkg/util/build"
 	"github.com/grafana/loki/pkg/util/cfg"
+
+	_ "github.com/grafana/loki/pkg/util/build"
 	util_log "github.com/grafana/loki/pkg/util/log"
 )
 
@@ -32,16 +34,18 @@ func init() {
 	prometheus.MustRegister(version.NewCollector("promtail"))
 }
 
+var mtx sync.Mutex
+
 type Config struct {
-	config.Config   `yaml:",inline"`
+	promtail_config.Config `yaml:",inline"`
-	printVersion    bool
+	printVersion           bool
-	printConfig     bool
+	printConfig            bool
-	logConfig       bool
+	logConfig              bool
-	dryRun          bool
+	dryRun                 bool
-	checkSyntax     bool
+	checkSyntax            bool
-	configFile      string
+	configFile             string
-	configExpandEnv bool
+	configExpandEnv        bool
-	inspect         bool
+	inspect                bool
 }
 
 func (c *Config) RegisterFlags(f *flag.FlagSet) {
@@ -68,11 +72,11 @@ func (c *Config) Clone() flagext.Registerer {
 func main() {
 	// Load config, merging config file and CLI flags
 	var config Config
-	if err := cfg.DefaultUnmarshal(&config, os.Args[1:], flag.CommandLine); err != nil {
+	args := os.Args[1:]
+	if err := cfg.DefaultUnmarshal(&config, args, flag.CommandLine); err != nil {
 		fmt.Println("Unable to parse config:", err)
 		os.Exit(1)
 	}
-
 	if config.checkSyntax {
 		if config.configFile == "" {
 			fmt.Println("Invalid config file")
@@ -123,7 +127,17 @@ func main() {
 	}
 
 	clientMetrics := client.NewMetrics(prometheus.DefaultRegisterer, config.Config.Options.StreamLagLabels)
-	p, err := promtail.New(config.Config, clientMetrics, config.dryRun)
+	newConfigFunc := func() (*promtail_config.Config, error) {
+		mtx.Lock()
+		defer mtx.Unlock()
+		var config Config
+		if err := cfg.DefaultUnmarshal(&config, args, flag.NewFlagSet(os.Args[0], flag.ExitOnError)); err != nil {
+			fmt.Println("Unable to parse config:", err)
+			return nil, fmt.Errorf("unable to parse config: %w", err)
+		}
+		return &config.Config, nil
+	}
+	p, err := promtail.New(config.Config, newConfigFunc, clientMetrics, config.dryRun)
 	if err != nil {
 		level.Error(util_log.Logger).Log("msg", "error creating promtail", "error", err)
 		os.Exit(1)

clients/pkg/promtail/promtail.go

@@ -1,9 +1,15 @@
 package promtail
 
 import (
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
 	"sync"
+	"syscall"
 
 	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/grafana/loki/clients/pkg/logentry/stages"
@@ -16,6 +22,20 @@ import (
 	util_log "github.com/grafana/loki/pkg/util/log"
 )
 
+var reloadSuccessTotal = prometheus.NewCounter(prometheus.CounterOpts{
+	Namespace: "promtail",
+	Name:      "config_reload_success_total",
+	Help:      "Number of reload success times.",
+})
+
+var reloadFailTotal = prometheus.NewCounter(prometheus.CounterOpts{
+	Namespace: "promtail",
+	Name:      "config_reload_fail_total",
+	Help:      "Number of reload fail times.",
+})
+
+var errConfigNotChange = errors.New("config has not changed")
+
 // Option is a function that can be passed to the New method of Promtail and
 // customize the Promtail that is created.
 type Option func(p *Promtail)
@@ -42,17 +62,32 @@ type Promtail struct {
 	logger         log.Logger
 	reg            prometheus.Registerer
 
-	stopped bool
+	stopped      bool
-	mtx     sync.Mutex
+	mtx          sync.Mutex
+	configLoaded string
+	newConfig    func() (*config.Config, error)
+	metrics      *client.Metrics
+	dryRun       bool
 }
 
 // New makes a new Promtail.
-func New(cfg config.Config, metrics *client.Metrics, dryRun bool, opts ...Option) (*Promtail, error) {
+func New(cfg config.Config, newConfig func() (*config.Config, error), metrics *client.Metrics, dryRun bool, opts ...Option) (*Promtail, error) {
 	// Initialize promtail with some defaults and allow the options to override
 	// them.
+
 	promtail := &Promtail{
-		logger: util_log.Logger,
+		logger:  util_log.Logger,
-		reg:    prometheus.DefaultRegisterer,
+		reg:     prometheus.DefaultRegisterer,
+		metrics: metrics,
+		dryRun:  dryRun,
+	}
+	err := promtail.reg.Register(reloadSuccessTotal)
+	if err != nil {
+		return nil, fmt.Errorf("error register prometheus collector reloadSuccessTotal :%w", err)
+	}
+	err = promtail.reg.Register(reloadFailTotal)
+	if err != nil {
+		return nil, fmt.Errorf("error register prometheus collector reloadFailTotal :%w", err)
 	}
 	for _, o := range opts {
 		// todo (callum) I don't understand why I needed to add this check
@@ -61,37 +96,71 @@ func New(cfg config.Config, metrics *client.Metrics, dryRun bool, opts ...Option
 		}
 		o(promtail)
 	}
+	err = promtail.reloadConfig(&cfg)
+	if err != nil {
+		return nil, err
+	}
+	server, err := server.New(cfg.ServerConfig, promtail.logger, promtail.targetManagers, cfg.String())
+	if err != nil {
+		return nil, fmt.Errorf("error creating loki server: %w", err)
+	}
+	promtail.server = server
+	promtail.newConfig = newConfig
 
-	cfg.Setup(promtail.logger)
+	return promtail, nil
+}
 
+func (p *Promtail) reloadConfig(cfg *config.Config) error {
+	level.Debug(p.logger).Log("msg", "Reloading configuration file")
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	newConfigFile := cfg.String()
+	if newConfigFile == p.configLoaded {
+		return errConfigNotChange
+	}
+	newConf := cfg.String()
+	level.Info(p.logger).Log("msg", "Reloading configuration file", "newConf", newConf)
+	if p.targetManagers != nil {
+		p.targetManagers.Stop()
+	}
+	if p.client != nil {
+		p.client.Stop()
+	}
+
+	cfg.Setup(p.logger)
 	if cfg.LimitsConfig.ReadlineRateEnabled {
 		stages.SetReadLineRateLimiter(cfg.LimitsConfig.ReadlineRate, cfg.LimitsConfig.ReadlineBurst, cfg.LimitsConfig.ReadlineRateDrop)
 	}
 	var err error
-	if dryRun {
+	if p.dryRun {
-		promtail.client, err = client.NewLogger(metrics, cfg.Options.StreamLagLabels, promtail.logger, cfg.ClientConfigs...)
+		p.client, err = client.NewLogger(p.metrics, cfg.Options.StreamLagLabels, p.logger, cfg.ClientConfigs...)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		cfg.PositionsConfig.ReadOnly = true
 	} else {
-		promtail.client, err = client.NewMulti(metrics, cfg.Options.StreamLagLabels, promtail.logger, cfg.LimitsConfig.MaxStreams, cfg.ClientConfigs...)
+		p.client, err = client.NewMulti(p.metrics, cfg.Options.StreamLagLabels, p.logger, cfg.LimitsConfig.MaxStreams, cfg.ClientConfigs...)
 		if err != nil {
-			return nil, err
+			return err
 		}
 	}
 
-	tms, err := targets.NewTargetManagers(promtail, promtail.reg, promtail.logger, cfg.PositionsConfig, promtail.client, cfg.ScrapeConfig, &cfg.TargetConfig)
+	tms, err := targets.NewTargetManagers(p, p.reg, p.logger, cfg.PositionsConfig, p.client, cfg.ScrapeConfig, &cfg.TargetConfig)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	promtail.targetManagers = tms
+	p.targetManagers = tms
-	server, err := server.New(cfg.ServerConfig, promtail.logger, tms, cfg.String())
+
-	if err != nil {
+	promServer := p.server
-		return nil, err
+	if promServer != nil {
+		promtailServer, ok := promServer.(*server.PromtailServer)
+		if !ok {
+			return errors.New("promtailServer cast fail")
+		}
+		promtailServer.ReloadServer(p.targetManagers, cfg.String())
 	}
-	promtail.server = server
+	p.configLoaded = newConf
-	return promtail, nil
+	return nil
 }
 
 // Run the promtail; will block until a signal is received.
@@ -103,6 +172,7 @@ func (p *Promtail) Run() error {
 		return nil
 	}
 	p.mtx.Unlock() // unlock before blocking
+	go p.watchConfig()
 	return p.server.Run()
 }
 
@@ -133,3 +203,48 @@ func (p *Promtail) Shutdown() {
 func (p *Promtail) ActiveTargets() map[string][]target.Target {
 	return p.targetManagers.ActiveTargets()
 }
+
+func (p *Promtail) watchConfig() {
+	// Reload handler.
+	// Make sure that sighup handler is registered with a redirect to the channel before the potentially
+	if p.newConfig == nil {
+		level.Warn(p.logger).Log("msg", "disable watchConfig", "reason", "Promtail newConfig func is Empty")
+		return
+	}
+	promtailServer, ok := p.server.(*server.PromtailServer)
+	if !ok {
+		level.Warn(p.logger).Log("msg", "disable watchConfig", "reason", "promtailServer cast fail")
+		return
+	}
+	level.Warn(p.logger).Log("msg", "enable watchConfig")
+	hup := make(chan os.Signal, 1)
+	signal.Notify(hup, syscall.SIGHUP)
+	for {
+		select {
+		case <-hup:
+			_ = p.reload()
+		case rc := <-promtailServer.Reload():
+			if err := p.reload(); err != nil {
+				rc <- err
+			} else {
+				rc <- nil
+			}
+		}
+	}
+}
+
+func (p *Promtail) reload() error {
+	cfg, err := p.newConfig()
+	if err != nil {
+		reloadFailTotal.Inc()
+		return fmt.Errorf("Error new Config: %w", err)
+	}
+	err = p.reloadConfig(cfg)
+	if err != nil {
+		reloadFailTotal.Inc()
+		level.Error(p.logger).Log("msg", "Error reloading config", "err", err)
+		return err
+	}
+	reloadSuccessTotal.Inc()
+	return nil
+}

clients/pkg/promtail/promtail_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/grafana/dskit/flagext"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
@@ -106,7 +107,7 @@ func TestPromtail(t *testing.T) {
 		_ = server.Shutdown(context.Background())
 	}()
 
-	p, err := New(buildTestConfig(t, positionsFileName, testDir), clientMetrics, false, nil)
+	p, err := New(buildTestConfig(t, positionsFileName, testDir), nil, clientMetrics, false, nil)
 	if err != nil {
 		t.Error("error creating promtail", err)
 		return
@@ -659,7 +660,7 @@ func Test_DryRun(t *testing.T) {
 	require.NoError(t, err)
 	defer os.Remove(f.Name())
 
-	_, err = New(config.Config{}, clientMetrics, true, nil)
+	_, err = New(config.Config{}, nil, clientMetrics, true, nil)
 	require.Error(t, err)
 
 	// Set the minimum config needed to start a server. We need to do this since we
@@ -673,7 +674,8 @@ func Test_DryRun(t *testing.T) {
 		},
 	}
 
-	prometheus.DefaultRegisterer = prometheus.NewRegistry() // reset registry, otherwise you can't create 2 weavework server.
+	prometheus.DefaultRegisterer = prometheus.NewRegistry()
+
 	_, err = New(config.Config{
 		ServerConfig: serverCfg,
 		ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "string"}}},
@@ -681,7 +683,7 @@ func Test_DryRun(t *testing.T) {
 			PositionsFile: f.Name(),
 			SyncPeriod:    time.Second,
 		},
-	}, clientMetrics, true, nil)
+	}, nil, clientMetrics, true, nil)
 	require.NoError(t, err)
 
 	prometheus.DefaultRegisterer = prometheus.NewRegistry()
@@ -693,7 +695,168 @@ func Test_DryRun(t *testing.T) {
 			PositionsFile: f.Name(),
 			SyncPeriod:    time.Second,
 		},
-	}, clientMetrics, false, nil)
+	}, nil, clientMetrics, false, nil)
 	require.NoError(t, err)
 	require.IsType(t, &client.MultiClient{}, p.client)
 }
+
+func Test_Reload(t *testing.T) {
+	f, err := os.CreateTemp("/tmp", "Test_Reload")
+	require.NoError(t, err)
+	defer os.Remove(f.Name())
+
+	cfg := config.Config{
+		ServerConfig: server.Config{
+			Reload: true,
+		},
+		ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "string"}}},
+		PositionsConfig: positions.Config{
+			PositionsFile: f.Name(),
+			SyncPeriod:    time.Second,
+		},
+	}
+
+	expectCfgStr := cfg.String()
+
+	expectedConfig := &config.Config{
+		ServerConfig: server.Config{
+			Reload: true,
+		},
+		ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "reloadtesturl"}}},
+		PositionsConfig: positions.Config{
+			PositionsFile: f.Name(),
+			SyncPeriod:    time.Second,
+		},
+	}
+
+	expectedConfigReloaded := expectedConfig.String()
+
+	prometheus.DefaultRegisterer = prometheus.NewRegistry() // reset registry, otherwise you can't create 2 weavework server.
+	promtailServer, err := New(cfg, func() (*config.Config, error) {
+		return expectedConfig, nil
+	}, clientMetrics, true, nil)
+	require.NoError(t, err)
+	require.Equal(t, len(expectCfgStr), len(promtailServer.configLoaded))
+	require.Equal(t, expectCfgStr, promtailServer.configLoaded)
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err = promtailServer.Run()
+		if err != nil {
+			err = errors.Wrap(err, "Failed to start promtail")
+		}
+	}()
+	defer promtailServer.Shutdown() // In case the test fails before the call to Shutdown below.
+
+	svr := promtailServer.server.(*pserver.PromtailServer)
+
+	require.NotEqual(t, len(expectedConfig.String()), len(svr.PromtailConfig()))
+	require.NotEqual(t, expectedConfig.String(), svr.PromtailConfig())
+	result, err := reload(t, svr.Server.HTTPListenAddr())
+	require.NoError(t, err)
+	expectedReloadResult := ""
+	require.Equal(t, expectedReloadResult, result)
+	require.Equal(t, len(expectedConfig.String()), len(svr.PromtailConfig()))
+	require.Equal(t, expectedConfig.String(), svr.PromtailConfig())
+	require.Equal(t, len(expectedConfigReloaded), len(promtailServer.configLoaded))
+	require.Equal(t, expectedConfigReloaded, promtailServer.configLoaded)
+
+	pb := &dto.Metric{}
+	err = reloadSuccessTotal.Write(pb)
+	require.NoError(t, err)
+	require.Equal(t, 1.0, pb.Counter.GetValue())
+}
+
+func Test_ReloadFail_NotPanic(t *testing.T) {
+	f, err := os.CreateTemp("/tmp", "Test_Reload")
+	require.NoError(t, err)
+	defer os.Remove(f.Name())
+
+	cfg := config.Config{
+		ServerConfig: server.Config{
+			Reload: true,
+		},
+		ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "string"}}},
+		PositionsConfig: positions.Config{
+			PositionsFile: f.Name(),
+			SyncPeriod:    time.Second,
+		},
+	}
+
+	expectedConfig := &config.Config{
+		ServerConfig: server.Config{
+			Reload: true,
+		},
+		ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "reloadtesturl"}}},
+		PositionsConfig: positions.Config{
+			PositionsFile: f.Name(),
+			SyncPeriod:    time.Second,
+		},
+	}
+
+	newConfigErr := errors.New("load config fail")
+
+	prometheus.DefaultRegisterer = prometheus.NewRegistry() // reset registry, otherwise you can't create 2 weavework server.
+	promtailServer, err := New(cfg, func() (*config.Config, error) {
+		return nil, newConfigErr
+	}, clientMetrics, true, nil)
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err = promtailServer.Run()
+		if err != nil {
+			err = errors.Wrap(err, "Failed to start promtail")
+		}
+	}()
+	defer promtailServer.Shutdown() // In case the test fails before the call to Shutdown below.
+
+	svr := promtailServer.server.(*pserver.PromtailServer)
+	httpListenAddr := svr.Server.HTTPListenAddr()
+	require.NotEqual(t, len(expectedConfig.String()), len(svr.PromtailConfig()))
+	require.NotEqual(t, expectedConfig.String(), svr.PromtailConfig())
+	result, err := reload(t, httpListenAddr)
+	require.Error(t, err)
+	expectedReloadResult := fmt.Sprintf("failed to reload config: Error new Config: %s\n", newConfigErr)
+	require.Equal(t, expectedReloadResult, result)
+
+	pb := &dto.Metric{}
+	err = reloadFailTotal.Write(pb)
+	require.NoError(t, err)
+	require.Equal(t, 1.0, pb.Counter.GetValue())
+
+	promtailServer.newConfig = func() (*config.Config, error) {
+		return &cfg, nil
+	}
+	result, err = reload(t, httpListenAddr)
+	require.Error(t, err)
+	require.Equal(t, fmt.Sprintf("failed to reload config: %s\n", errConfigNotChange), result)
+}
+
+func reload(t *testing.T, httpListenAddr net.Addr) (string, error) {
+	resp, err := http.Get(fmt.Sprintf("http://%s/reload", httpListenAddr))
+	if err != nil {
+		t.Fatal("Could not query reload endpoint", err)
+	}
+	if resp.StatusCode == http.StatusInternalServerError {
+		b, err := io.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatal("Error reading response body from /reload endpoint", err)
+		}
+		return string(b), errors.New("Received a 500 status code from /reload endpoint")
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.New("Received a non 200 status code from /reload endpoint")
+	}
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal("Error reading response body from /reload endpoint", err)
+	}
+	return string(b), nil
+}

clients/pkg/promtail/server/server.go

@@ -10,6 +10,7 @@ import (
 	"path"
 	"sort"
 	"strings"
+	"sync"
 	"syscall"
 	"text/template"
 
@@ -39,8 +40,10 @@ type Server interface {
 type PromtailServer struct {
 	*serverww.Server
 	log               log.Logger
+	mtx               sync.Mutex
 	tms               *targets.TargetManagers
 	externalURL       *url.URL
+	reloadCh          chan chan error
 	healthCheckTarget bool
 	promtailCfg       string
 }
@@ -51,6 +54,7 @@ type Config struct {
 	ExternalURL       string `yaml:"external_url"`
 	HealthCheckTarget *bool  `yaml:"health_check_target"`
 	Disable           bool   `yaml:"disable"`
+	Reload            bool   `yaml:"enable_runtime_reload"`
 }
 
 // RegisterFlags with prefix registers flags where every name is prefixed by
@@ -60,6 +64,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	cfg.Config.RegisterFlags(f)
 
 	f.BoolVar(&cfg.Disable, prefix+"server.disable", false, "Disable the http and grpc server.")
+	f.BoolVar(&cfg.Reload, prefix+"server.reload", false, "Enable reload via HTTP request.")
 }
 
 // RegisterFlags adds the flags required to config this to the given FlagSet
@@ -91,6 +96,7 @@ func New(cfg Config, log log.Logger, tms *targets.TargetManagers, promtailCfg st
 	serv := &PromtailServer{
 		Server:            wws,
 		log:               log,
+		reloadCh:          make(chan chan error),
 		tms:               tms,
 		externalURL:       externalURL,
 		healthCheckTarget: healthCheckTargetFlag,
@@ -103,12 +109,17 @@ func New(cfg Config, log log.Logger, tms *targets.TargetManagers, promtailCfg st
 	serv.HTTP.Path("/service-discovery").Handler(http.HandlerFunc(serv.serviceDiscovery))
 	serv.HTTP.Path("/targets").Handler(http.HandlerFunc(serv.targets))
 	serv.HTTP.Path("/config").Handler(http.HandlerFunc(serv.config))
+	if cfg.Reload {
+		serv.HTTP.Path("/reload").Handler(http.HandlerFunc(serv.reload))
+	}
 	serv.HTTP.Path("/debug/fgprof").Handler(fgprof.Handler())
 	return serv, nil
 }
 
 // serviceDiscovery serves the service discovery page.
 func (s *PromtailServer) serviceDiscovery(rw http.ResponseWriter, req *http.Request) {
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
 	var index []string
 	allTarget := s.tms.AllTargets()
 	for job := range allTarget {
@@ -187,6 +198,8 @@ func (s *PromtailServer) config(rw http.ResponseWriter, req *http.Request) {
 
 // targets serves the targets page.
 func (s *PromtailServer) targets(rw http.ResponseWriter, req *http.Request) {
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
 	executeTemplate(req.Context(), rw, templateOptions{
 		Data: struct {
 			TargetPools map[string][]target.Target
@@ -218,8 +231,36 @@ func (s *PromtailServer) targets(rw http.ResponseWriter, req *http.Request) {
 	})
 }
 
+func (s *PromtailServer) reload(rw http.ResponseWriter, req *http.Request) {
+	rc := make(chan error)
+	s.reloadCh <- rc
+	if err := <-rc; err != nil {
+		http.Error(rw, fmt.Sprintf("failed to reload config: %s", err), http.StatusInternalServerError)
+	}
+
+}
+
+// Reload returns the receive-only channel that signals configuration reload requests.
+func (s *PromtailServer) Reload() <-chan chan error {
+	return s.reloadCh
+}
+
+// Reload returns the receive-only channel that signals configuration reload requests.
+func (s *PromtailServer) PromtailConfig() string {
+	return s.promtailCfg
+}
+
+func (s *PromtailServer) ReloadServer(tms *targets.TargetManagers, promtailCfg string) {
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
+	s.tms = tms
+	s.promtailCfg = promtailCfg
+}
+
 // ready serves the ready endpoint
 func (s *PromtailServer) ready(rw http.ResponseWriter, _ *http.Request) {
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
 	if s.healthCheckTarget && !s.tms.Ready() {
 		http.Error(rw, readinessProbeFailure, http.StatusInternalServerError)
 		return

clients/pkg/promtail/targets/manager.go

@@ -41,6 +41,17 @@ const (
 	HerokuDrainConfigs   = "herokuDrainConfigs"
 )
 
+var (
+	fileMetrics        *file.Metrics
+	syslogMetrics      *syslog.Metrics
+	gcplogMetrics      *gcplog.Metrics
+	gelfMetrics        *gelf.Metrics
+	cloudflareMetrics  *cloudflare.Metrics
+	dockerMetrics      *docker.Metrics
+	journalMetrics     *journal.Metrics
+	herokuDrainMetrics *heroku.Metrics
+)
+
 type targetManager interface {
 	Ready() bool
 	Stop()
@@ -119,38 +130,28 @@ func NewTargetManagers(
 		return positionFile, nil
 	}
 
-	var (
+	if len(targetScrapeConfigs[FileScrapeConfigs]) > 0 && fileMetrics == nil {
-		fileMetrics        *file.Metrics
-		syslogMetrics      *syslog.Metrics
-		gcplogMetrics      *gcplog.Metrics
-		gelfMetrics        *gelf.Metrics
-		cloudflareMetrics  *cloudflare.Metrics
-		dockerMetrics      *docker.Metrics
-		journalMetrics     *journal.Metrics
-		herokuDrainMetrics *heroku.Metrics
-	)
-	if len(targetScrapeConfigs[FileScrapeConfigs]) > 0 {
 		fileMetrics = file.NewMetrics(reg)
 	}
-	if len(targetScrapeConfigs[SyslogScrapeConfigs]) > 0 {
+	if len(targetScrapeConfigs[SyslogScrapeConfigs]) > 0 && syslogMetrics == nil {
 		syslogMetrics = syslog.NewMetrics(reg)
 	}
-	if len(targetScrapeConfigs[GcplogScrapeConfigs]) > 0 {
+	if len(targetScrapeConfigs[GcplogScrapeConfigs]) > 0 && gcplogMetrics == nil {
 		gcplogMetrics = gcplog.NewMetrics(reg)
 	}
-	if len(targetScrapeConfigs[GelfConfigs]) > 0 {
+	if len(targetScrapeConfigs[GelfConfigs]) > 0 && gelfMetrics == nil {
 		gelfMetrics = gelf.NewMetrics(reg)
 	}
-	if len(targetScrapeConfigs[CloudflareConfigs]) > 0 {
+	if len(targetScrapeConfigs[CloudflareConfigs]) > 0 && cloudflareMetrics == nil {
 		cloudflareMetrics = cloudflare.NewMetrics(reg)
 	}
-	if len(targetScrapeConfigs[DockerConfigs]) > 0 || len(targetScrapeConfigs[DockerSDConfigs]) > 0 {
+	if (len(targetScrapeConfigs[DockerConfigs]) > 0 || len(targetScrapeConfigs[DockerSDConfigs]) > 0) && dockerMetrics == nil {
 		dockerMetrics = docker.NewMetrics(reg)
 	}
-	if len(targetScrapeConfigs[JournalScrapeConfigs]) > 0 {
+	if len(targetScrapeConfigs[JournalScrapeConfigs]) > 0 && journalMetrics == nil {
 		journalMetrics = journal.NewMetrics(reg)
 	}
-	if len(targetScrapeConfigs[HerokuDrainConfigs]) > 0 {
+	if len(targetScrapeConfigs[HerokuDrainConfigs]) > 0 && herokuDrainMetrics == nil {
 		herokuDrainMetrics = heroku.NewMetrics(reg)
 	}