adding ability to supply timezone to timestamp pipeline stage

cmd/promtail/Dockerfile

@@ -1,5 +1,5 @@
 FROM       alpine:3.9
-RUN        apk add --update --no-cache ca-certificates
+RUN        apk add --update --no-cache ca-certificates tzdata
 ADD        promtail /usr/bin
 COPY       promtail-local-config.yaml /etc/promtail/local-config.yaml
 COPY       promtail-docker-config.yaml /etc/promtail/docker-config.yaml

cmd/promtail/Dockerfile.debug

@@ -1,5 +1,5 @@
 FROM       alpine:3.9
-RUN        apk add --update --no-cache ca-certificates
+RUN        apk add --update --no-cache ca-certificates tzdata
 ADD        promtail-debug /usr/bin
 ADD        dlv /usr/bin
 COPY       promtail-local-config.yaml /etc/promtail/local-config.yaml
@@ -14,4 +14,4 @@ RUN apk add --no-cache libc6-compat
 # Run delve, ending with -- because we pass params via kubernetes, per the docs:
 #   Pass flags to the program you are debugging using --, for example:`
 #   dlv exec ./hello -- server --config conf/config.toml`
-ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/usr/bin/promtail-debug", "--"]
+ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/usr/bin/promtail-debug", "--"]

docs/logentry/processing-log-lines.md

@@ -147,7 +147,7 @@ A regex stage will take the provided regex and set the named groups as data in t
     source:      ②
 ```
 
-① `expression` is **required** and needs to be a [golang RE2 regex string](https://github.com/google/re2/wiki/Syntax). Every capture group `(re)` will be set into the `extracted` map, every capture group **must be named:** `(?P<name>re)`, the name will be used as the key in the map.
+① `expression` is **required** and needs to be a [golang RE2 regex string](https://github.com/google/re2/wiki/Syntax). Every capture group `(re)` will be set into the `extracted` map, every capture group **must be named:** `(?P<name>re)`, the name will be used as the key in the map.  
 ② `source` is optional and contains the name of key in the `extracted` map containing the data to parse. If omitted, the regex stage will parse the log `entry`.
 
 ##### Example (without source):
@@ -209,8 +209,8 @@ A json stage will take the provided [JMESPath expressions](http://jmespath.org/)
     source:             ③
 ```
 
-① `expressions` is a required yaml object containing key/value pairs of JMESPath expressions
-② `key: expression` where `key` will be the key in the `extracted` map, and the value will be the evaluated JMESPath expression.
+① `expressions` is a required yaml object containing key/value pairs of JMESPath expressions  
+② `key: expression` where `key` will be the key in the `extracted` map, and the value will be the evaluated JMESPath expression.  
 ③ `source` is optional and contains the name of key in the `extracted` map containing the json to parse. If omitted, the json stage will parse the log `entry`.
 
 This stage uses the Go JSON unmarshaller, which means non string types like numbers or booleans will be unmarshalled into those types.  The `extracted` map will accept non-string values and this stage will keep primitive types as they are unmarshalled (e.g. bool or float64).  Downstream stages will need to perform correct type conversion of these values as necessary.
@@ -366,10 +366,12 @@ A timestamp stage will parse data from the `extracted` map and set the `time` va
 - timestamp:
     source:   ①
     format:   ②
+    location: ③
 ```
 
 ① `source` is **required** and is the key name to data in the `extracted` map.  
-② `format` is **required** and is the input to Go's [time.parse](https://golang.org/pkg/time/#Parse) function.
+② `format` is **required** and is the input to Go's [time.parse](https://golang.org/pkg/time/#Parse) function.  
+③ `location` is **optional** and is an IANA Timezone Database string, see the [go docs](https://golang.org/pkg/time/#LoadLocation) for more info
 
 Several of Go's pre-defined format's can be used by their name:
 

pkg/logentry/stages/extensions.go

@@ -26,6 +26,7 @@ func NewDocker(logger log.Logger, registerer prometheus.Registerer) (Stage, erro
 			StageTypeTimestamp: TimestampConfig{
 				"timestamp",
 				RFC3339Nano,
+				nil,
 			}},
 		PipelineStage{
 			StageTypeOutput: OutputConfig{
@@ -52,6 +53,7 @@ func NewCRI(logger log.Logger, registerer prometheus.Registerer) (Stage, error)
 			StageTypeTimestamp: TimestampConfig{
 				"time",
 				RFC3339Nano,
+				nil,
 			},
 		},
 		PipelineStage{

pkg/logentry/stages/timestamp.go

@@ -2,6 +2,7 @@ package stages
 
 import (
 	"errors"
+	"fmt"
 	"reflect"
 	"time"
 
@@ -15,6 +16,7 @@ const (
 	ErrEmptyTimestampStageConfig = "timestamp stage config cannot be empty"
 	ErrTimestampSourceRequired   = "timestamp source value is required if timestamp is specified"
 	ErrTimestampFormatRequired   = "timestamp format is required"
+	ErrInvalidLocation           = "invalid location specified: %v"
 
 	Unix   = "Unix"
 	UnixMs = "UnixMs"
@@ -23,8 +25,9 @@ const (
 
 // TimestampConfig configures timestamp extraction
 type TimestampConfig struct {
-	Source string `mapstructure:"source"`
-	Format string `mapstructure:"format"`
+	Source   string  `mapstructure:"source"`
+	Format   string  `mapstructure:"format"`
+	Location *string `mapstructure:"location"`
 }
 
 // parser can convert the time string into a time.Time value
@@ -41,7 +44,15 @@ func validateTimestampConfig(cfg *TimestampConfig) (parser, error) {
 	if cfg.Format == "" {
 		return nil, errors.New(ErrTimestampFormatRequired)
 	}
-	return convertDateLayout(cfg.Format), nil
+	var loc *time.Location
+	var err error
+	if cfg.Location != nil {
+		loc, err = time.LoadLocation(*cfg.Location)
+		if err != nil {
+			return nil, fmt.Errorf(ErrInvalidLocation, err)
+		}
+	}
+	return convertDateLayout(cfg.Format, loc), nil
 
 }
 

pkg/logentry/stages/timestamp_test.go

@@ -1,6 +1,8 @@
 package stages
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -43,9 +45,17 @@ func TestTimestampPipeline(t *testing.T) {
 	assert.Equal(t, time.Date(2012, 11, 01, 22, 8, 41, 0, time.FixedZone("", -4*60*60)).Unix(), ts.Unix())
 }
 
+var (
+	invalidLocationString = "America/Canada"
+	validLocationString   = "America/New_York"
+	validLocation, _      = time.LoadLocation(validLocationString)
+)
+
 func TestTimestampValidation(t *testing.T) {
 	tests := map[string]struct {
-		config       *TimestampConfig
+		config *TimestampConfig
+		// Note the error text validation is a little loosey as it only validates with strings.HasPrefix
+		// this is to work around different errors related to timezone loading on different systems
 		err          error
 		testString   string
 		expectedTime time.Time
@@ -64,6 +74,14 @@ func TestTimestampValidation(t *testing.T) {
 			},
 			err: errors.New(ErrTimestampFormatRequired),
 		},
+		"invalid location": {
+			config: &TimestampConfig{
+				Source:   "source1",
+				Format:   "2006-01-02",
+				Location: &invalidLocationString,
+			},
+			err: fmt.Errorf(ErrInvalidLocation, ""),
+		},
 		"standard format": {
 			config: &TimestampConfig{
 				Source: "source1",
@@ -91,6 +109,16 @@ func TestTimestampValidation(t *testing.T) {
 			testString:   "Jul 15 01:02:03",
 			expectedTime: time.Date(time.Now().Year(), 7, 15, 1, 2, 3, 0, time.UTC),
 		},
+		"custom format with location": {
+			config: &TimestampConfig{
+				Source:   "source1",
+				Format:   "2006-01-02 15:04:05",
+				Location: &validLocationString,
+			},
+			err:          nil,
+			testString:   "2009-07-01 03:30:20",
+			expectedTime: time.Date(2009, 7, 1, 3, 30, 20, 0, validLocation),
+		},
 		"unix_ms": {
 			config: &TimestampConfig{
 				Source: "source1",
@@ -110,7 +138,7 @@ func TestTimestampValidation(t *testing.T) {
 				t.Errorf("validateOutputConfig() expected error = %v, actual error = %v", test.err, err)
 				return
 			}
-			if (err != nil) && (err.Error() != test.err.Error()) {
+			if (err != nil) && !strings.HasPrefix(err.Error(), test.err.Error()) {
 				t.Errorf("validateOutputConfig() expected error = %v, actual error = %v", test.err, err)
 				return
 			}
@@ -176,6 +204,18 @@ func TestTimestampStage_Process(t *testing.T) {
 			},
 			time.Date(2019, 7, 9, 21, 48, 36, 123, time.UTC),
 		},
+		"with location success": {
+			TimestampConfig{
+				Source:   "ts",
+				Format:   "2006-01-02 15:04:05",
+				Location: &validLocationString,
+			},
+			map[string]interface{}{
+				"somethigelse": "notimportant",
+				"ts":           "2019-07-22 20:29:32",
+			},
+			time.Date(2019, 7, 22, 20, 29, 32, 0, validLocation),
+		},
 	}
 	for name, test := range tests {
 		test := test

pkg/logentry/stages/util.go

@@ -12,7 +12,7 @@ const (
 )
 
 // convertDateLayout converts pre-defined date format layout into date format
-func convertDateLayout(predef string) parser {
+func convertDateLayout(predef string, location *time.Location) parser {
 	switch predef {
 	case "ANSIC":
 		return func(t string) (time.Time, error) {
@@ -81,10 +81,13 @@ func convertDateLayout(predef string) parser {
 	default:
 		if !strings.Contains(predef, "2006") {
 			return func(t string) (time.Time, error) {
-				return parseTimestampWithoutYear(predef, t, time.Now())
+				return parseTimestampWithoutYear(predef, location, t, time.Now())
 			}
 		}
 		return func(t string) (time.Time, error) {
+			if location != nil {
+				return time.ParseInLocation(predef, t, location)
+			}
 			return time.Parse(predef, t)
 		}
 	}
@@ -93,8 +96,14 @@ func convertDateLayout(predef string) parser {
 // parseTimestampWithoutYear parses the input timestamp without the year component,
 // assuming the timestamp is related to a point in time close to "now", and correctly
 // handling the edge cases around new year's eve
-func parseTimestampWithoutYear(layout string, timestamp string, now time.Time) (time.Time, error) {
-	parsedTime, err := time.Parse(layout, timestamp)
+func parseTimestampWithoutYear(layout string, location *time.Location, timestamp string, now time.Time) (time.Time, error) {
+	var parsedTime time.Time
+	var err error
+	if location != nil {
+		parsedTime, err = time.ParseInLocation(layout, timestamp, location)
+	} else {
+		parsedTime, err = time.Parse(layout, timestamp)
+	}
 	if err != nil {
 		return parsedTime, err
 	}

pkg/logentry/stages/util_test.go

@@ -67,24 +67,37 @@ func TestGetString(t *testing.T) {
 	assert.Equal(t, "1562723913000", s64_1)
 }
 
+var (
+	location, _ = time.LoadLocation("America/New_York")
+)
+
 func TestConvertDateLayout(t *testing.T) {
 	t.Parallel()
 
 	tests := map[string]struct {
 		layout    string
+		location  *time.Location
 		timestamp string
 		expected  time.Time
 	}{
 		"custom layout with year": {
 			"2006 Jan 02 15:04:05",
+			nil,
 			"2019 Jul 15 01:02:03",
 			time.Date(2019, 7, 15, 1, 2, 3, 0, time.UTC),
 		},
 		"custom layout without year": {
 			"Jan 02 15:04:05",
+			nil,
 			"Jul 15 01:02:03",
 			time.Date(time.Now().Year(), 7, 15, 1, 2, 3, 0, time.UTC),
 		},
+		"custom layout with year and location": {
+			"Jan 02 15:04:05",
+			location,
+			"Jul 15 01:02:03",
+			time.Date(time.Now().Year(), 7, 15, 1, 2, 3, 0, location),
+		},
 	}
 
 	for testName, testData := range tests {
@@ -93,7 +106,7 @@ func TestConvertDateLayout(t *testing.T) {
 		t.Run(testName, func(t *testing.T) {
 			t.Parallel()
 
-			parser := convertDateLayout(testData.layout)
+			parser := convertDateLayout(testData.layout, testData.location)
 			parsed, err := parser(testData.timestamp)
 			if err != nil {
 				t.Errorf("convertDateLayout() parser returned an unexpected error = %v", err)
@@ -110,6 +123,7 @@ func TestParseTimestampWithoutYear(t *testing.T) {
 
 	tests := map[string]struct {
 		layout    string
+		location  *time.Location
 		timestamp string
 		now       time.Time
 		expected  time.Time
@@ -117,13 +131,31 @@ func TestParseTimestampWithoutYear(t *testing.T) {
 	}{
 		"parse timestamp within current year": {
 			"Jan 02 15:04:05",
+			nil,
 			"Jul 15 01:02:03",
 			time.Date(2019, 7, 14, 0, 0, 0, 0, time.UTC),
 			time.Date(2019, 7, 15, 1, 2, 3, 0, time.UTC),
 			nil,
 		},
+		"parse timestamp with location DST": {
+			"Jan 02 15:04:05",
+			location,
+			"Jul 15 01:02:03",
+			time.Date(2019, 7, 14, 0, 0, 0, 0, time.UTC),
+			time.Date(2019, 7, 15, 1, 2, 3, 0, location),
+			nil,
+		},
+		"parse timestamp with location non DST": {
+			"Jan 02 15:04:05",
+			location,
+			"Jan 15 01:02:03",
+			time.Date(2019, 7, 14, 0, 0, 0, 0, time.UTC),
+			time.Date(2019, 1, 15, 1, 2, 3, 0, location),
+			nil,
+		},
 		"parse timestamp on 31th Dec and today is 1st Jan": {
 			"Jan 02 15:04:05",
+			nil,
 			"Dec 31 23:59:59",
 			time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC),
 			time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC),
@@ -131,6 +163,7 @@ func TestParseTimestampWithoutYear(t *testing.T) {
 		},
 		"parse timestamp on 1st Jan and today is 31st Dec": {
 			"Jan 02 15:04:05",
+			nil,
 			"Jan 01 01:02:03",
 			time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC),
 			time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC),
@@ -138,6 +171,7 @@ func TestParseTimestampWithoutYear(t *testing.T) {
 		},
 		"error if the input layout actually includes the year component": {
 			"2006 Jan 02 15:04:05",
+			nil,
 			"2019 Jan 01 01:02:03",
 			time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC),
 			time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC),
@@ -151,7 +185,7 @@ func TestParseTimestampWithoutYear(t *testing.T) {
 		t.Run(testName, func(t *testing.T) {
 			t.Parallel()
 
-			parsed, err := parseTimestampWithoutYear(testData.layout, testData.timestamp, testData.now)
+			parsed, err := parseTimestampWithoutYear(testData.layout, testData.location, testData.timestamp, testData.now)
 			if ((err != nil) != (testData.err != nil)) || (err != nil && testData.err != nil && err.Error() != testData.err.Error()) {
 				t.Errorf("parseTimestampWithoutYear() expected error = %v, actual error = %v", testData.err, err)
 				return