promtail: Add support for using syslog message timestamp (#2914)

Currently promtail sets the timestamp of incoming syslog messages to the
time it was received by promtail. In some cases, it is preferable to use
the source timestamp instead.

This adds a new `use_message_timestamp` option to the syslog target
config, which allows users to opt-in to the behavior of using the
timestamp on the message, if one exists.

docs/sources/clients/promtail/configuration.md

@@ -765,6 +765,11 @@ label_structured_data: <bool>
 # Label map to add to every log message.
 labels:
   [ <labelname>: <labelvalue> ... ]
+
+# Whether promtail should pass on the timestamp from the incoming syslog message.
+# When false, or if no timestamp is present on the syslog message, Promtail will assign the current timestamp to the log when it was processed.
+# Default is false
+use_incoming_timestamp: <bool>
 ```
 
 #### Available Labels

pkg/promtail/scrapeconfig/scrapeconfig.go

@@ -158,6 +158,10 @@ type SyslogTargetConfig struct {
 
 	// Labels optionally holds labels to associate with each record read from syslog.
 	Labels model.LabelSet `yaml:"labels"`
+
+	// UseIncomingTimestamp sets the timestamp to the incoming syslog mesages
+	// timestamp if it's set.
+	UseIncomingTimestamp bool `yaml:"use_incoming_timestamp"`
 }
 
 // PushTargetConfig describes a scrape config that listens for Loki push messages.

pkg/promtail/targets/syslog/syslogtarget.go

@@ -64,8 +64,9 @@ type SyslogTarget struct {
 }
 
 type message struct {
-	labels  model.LabelSet
-	message string
+	labels    model.LabelSet
+	message   string
+	timestamp time.Time
 }
 
 // NewSyslogTarget configures a new SyslogTarget.
@@ -231,12 +232,18 @@ func (t *SyslogTarget) handleMessage(connLabels labels.Labels, msg syslog.Messag
 		filtered[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value)
 	}
 
-	t.messages <- message{filtered, *rfc5424Msg.Message}
+	var timestamp time.Time
+	if t.config.UseIncomingTimestamp && rfc5424Msg.Timestamp != nil {
+		timestamp = *rfc5424Msg.Timestamp
+	} else {
+		timestamp = time.Now()
+	}
+	t.messages <- message{filtered, *rfc5424Msg.Message, timestamp}
 }
 
 func (t *SyslogTarget) messageSender() {
 	for msg := range t.messages {
-		if err := t.handler.Handle(msg.labels, time.Now(), msg.message); err != nil {
+		if err := t.handler.Handle(msg.labels, msg.timestamp, msg.message); err != nil {
 			level.Error(t.logger).Log("msg", "error handling line", "error", err)
 		}
 		syslogEntries.Inc()