Promtail: flush logs on exit (#7400)

Promtail is using the line-buffered logger (https://github.com/grafana/loki/pull/6954), and log messages were not being printed on exit

clients/cmd/promtail/main.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"reflect"
 	"sync"
-
 	// embed time zone data
 	_ "time/tzdata"
 
@@ -69,33 +68,41 @@ func (c *Config) Clone() flagext.Registerer {
 	}(*c)
 }
 
+// wrap os.Exit so that deferred functions execute before the process exits
+func exit(code int) {
+	// flush all logs that may be buffered in memory
+	util_log.Flush()
+
+	os.Exit(code)
+}
+
 func main() {
 	// Load config, merging config file and CLI flags
 	var config Config
 	args := os.Args[1:]
 	if err := cfg.DefaultUnmarshal(&config, args, flag.CommandLine); err != nil {
 		fmt.Println("Unable to parse config:", err)
-		os.Exit(1)
+		exit(1)
 	}
 	if config.checkSyntax {
 		if config.configFile == "" {
 			fmt.Println("Invalid config file")
-			os.Exit(1)
+			exit(1)
 		}
 		fmt.Println("Valid config file! No syntax issues found")
-		os.Exit(0)
+		exit(0)
 	}
 
 	// Handle -version CLI flag
 	if config.printVersion {
 		fmt.Println(version.Print("promtail"))
-		os.Exit(0)
+		exit(0)
 	}
 
 	// Init the logger which will honor the log level set in cfg.Server
 	if reflect.DeepEqual(&config.Config.ServerConfig.Config.LogLevel, &logging.Level{}) {
 		fmt.Println("Invalid log level")
-		os.Exit(1)
+		exit(1)
 	}
 	util_log.InitLogger(&config.Config.ServerConfig.Config, prometheus.DefaultRegisterer, true, false)
 
@@ -140,7 +147,7 @@ func main() {
 	p, err := promtail.New(config.Config, newConfigFunc, clientMetrics, config.dryRun)
 	if err != nil {
 		level.Error(util_log.Logger).Log("msg", "error creating promtail", "error", err)
-		os.Exit(1)
+		exit(1)
 	}
 
 	level.Info(util_log.Logger).Log("msg", "Starting Promtail", "version", version.Info())
@@ -148,6 +155,6 @@ func main() {
 
 	if err := p.Run(); err != nil {
 		level.Error(util_log.Logger).Log("msg", "error starting promtail", "error", err)
-		os.Exit(1)
+		exit(1)
 	}
 }

pkg/util/log/log.go

@@ -20,6 +20,8 @@ var (
 	// TODO: Change all components to take a non-global logger via their constructors.
 	// Prefer accepting a non-global logger as an argument.
 	Logger = log.NewNopLogger()
+
+	bufferedLogger *log.LineBufferedLogger
 )
 
 // InitLogger initialises the global gokit logger (util_log.Logger) and overrides the
@@ -37,6 +39,18 @@ func InitLogger(cfg *server.Config, reg prometheus.Registerer, buffered bool, sy
 	cfg.Log = logging.GoKit(log.With(l, "caller", log.Caller(4)))
 }
 
+type Flusher interface {
+	Flush() error
+}
+
+func Flush() error {
+	if bufferedLogger != nil {
+		return bufferedLogger.Flush()
+	}
+
+	return nil
+}
+
 // prometheusLogger exposes Prometheus counters for each of go-kit's log levels.
 type prometheusLogger struct {
 	logger              log.Logger
@@ -80,13 +94,18 @@ func newPrometheusLogger(l logging.Level, format logging.Format, reg prometheus.
 	if buffered {
 		// TODO: it's technically possible here to lose logs between the 100ms flush and the process being killed
 		// 	=> call buf.Flush() in a signal handler if this is a concern, but this is unlikely to be a problem
-		writer = log.NewLineBufferedLogger(os.Stderr, logEntries,
+
+		// retain a reference to this logger because it doesn't conform to the standard Logger interface,
+		// and we can't unwrap it to get the underlying logger when we flush on shutdown
+		bufferedLogger = log.NewLineBufferedLogger(os.Stderr, logEntries,
 			log.WithFlushPeriod(flushTimeout),
 			log.WithPrellocatedBuffer(logBufferSize),
 			log.WithFlushCallback(func(entries uint32) {
 				logFlushes.Observe(float64(entries))
 			}),
 		)
+
+		writer = bufferedLogger
 	} else {
 		writer = os.Stderr
 	}