loki-canary: Add support for client-side TLS certs for Loki connection (#6310)

* Add support for client-side TLS certs to loki-canary

* fix lint errors, make the case of using TLS but not client certs more clear, check that -tls is set if client certs are used

* move changelog to canary section instead of main

* wrap httpClient errors and simplify dialer creation for websocket

CHANGELOG.md

@@ -99,6 +99,7 @@
 * [5711](https://github.com/grafana/loki/pull/5711) **MichelHollands**: Update fluent-bit output name
 
 #### Loki Canary
+* [6310](https://github.com/grafana/loki/pull/6310) **chodges15**: Add support for client-side TLS certs in loki-canary for Loki connection
 * [5568](https://github.com/grafana/loki/pull/5568) **afayngelerindbx**: canary: Adds locking to prevent multiple concurrent invocations of `confirmMissing` from clobbering each other
 ### Notes
 

cmd/loki-canary/main.go

@@ -3,15 +3,18 @@ package main
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	"os"
-	"os/signal"
 	"strconv"
 	"sync"
 	"syscall"
 	"time"
 
+	"crypto/tls"
+	"net/http"
+	"os/signal"
+
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/version"
 
 	"github.com/grafana/loki/pkg/canary/comparator"
@@ -36,7 +39,10 @@ func main() {
 	sValue := flag.String("streamvalue", "stdout", "The unique stream value for this instance of loki-canary to use in the log selector")
 	port := flag.Int("port", 3500, "Port which loki-canary should expose metrics")
 	addr := flag.String("addr", "", "The Loki server URL:Port, e.g. loki:3100")
-	tls := flag.Bool("tls", false, "Does the loki connection use TLS?")
+	useTLS := flag.Bool("tls", false, "Does the loki connection use TLS?")
+	certFile := flag.String("cert-file", "", "Client PEM encoded X.509 certificate for optional use with TLS connection to Loki")
+	keyFile := flag.String("key-file", "", "Client PEM encoded X.509 key for optional use with TLS connection to Loki")
+	caFile := flag.String("ca-file", "", "Client certificate authority for optional use with TLS connection to Loki")
 	user := flag.String("user", "", "Loki username.")
 	pass := flag.String("pass", "", "Loki password.")
 	tenantID := flag.String("tenant-id", "", "Tenant ID to be set in X-Scope-OrgID header.")
@@ -83,6 +89,26 @@ func main() {
 		os.Exit(1)
 	}
 
+	var tlsConfig *tls.Config
+	tc := config.TLSConfig{}
+	if *certFile != "" || *keyFile != "" || *caFile != "" {
+		if !*useTLS {
+			_, _ = fmt.Fprintf(os.Stderr, "Must set --tls when specifying client certs\n")
+			os.Exit(1)
+		}
+		tc.CAFile = *caFile
+		tc.CertFile = *certFile
+		tc.KeyFile = *keyFile
+		tc.InsecureSkipVerify = false
+
+		var err error
+		tlsConfig, err = config.NewTLSConfig(&tc)
+		if err != nil {
+			_, _ = fmt.Fprintf(os.Stderr, "TLS configuration error: %s\n", err.Error())
+			os.Exit(1)
+		}
+	}
+
 	sentChan := make(chan time.Time)
 	receivedChan := make(chan time.Time)
 
@@ -94,7 +120,7 @@ func main() {
 		defer c.lock.Unlock()
 
 		c.writer = writer.NewWriter(os.Stdout, sentChan, *interval, *outOfOrderMin, *outOfOrderMax, *outOfOrderPercentage, *size)
-		c.reader = reader.NewReader(os.Stderr, receivedChan, *tls, *addr, *user, *pass, *tenantID, *queryTimeout, *lName, *lVal, *sName, *sValue, *interval)
+		c.reader = reader.NewReader(os.Stderr, receivedChan, *useTLS, tlsConfig, *caFile, *addr, *user, *pass, *tenantID, *queryTimeout, *lName, *lVal, *sName, *sValue, *interval)
 		c.comparator = comparator.NewComparator(os.Stderr, *wait, *maxWait, *pruneInterval, *spotCheckInterval, *spotCheckMax, *spotCheckQueryRate, *spotCheckWait, *metricTestInterval, *metricTestQueryRange, *interval, *buckets, sentChan, receivedChan, c.reader, true)
 	}
 

pkg/canary/reader/reader.go

@@ -2,6 +2,7 @@ package reader
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/base64"
 	"fmt"
 	"io"
@@ -21,6 +22,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/common/config"
 
 	"github.com/grafana/loki/pkg/loghttp"
 	"github.com/grafana/loki/pkg/logqlmodel"
@@ -48,32 +50,36 @@ type LokiReader interface {
 }
 
 type Reader struct {
-	header       http.Header
+	header          http.Header
-	tls          bool
+	tls             bool
-	addr         string
+	clientTLSConfig *tls.Config
-	user         string
+	caFile          string
-	pass         string
+	addr            string
-	tenantID     string
+	user            string
-	queryTimeout time.Duration
+	pass            string
-	sName        string
+	tenantID        string
-	sValue       string
+	queryTimeout    time.Duration
-	lName        string
+	sName           string
-	lVal         string
+	sValue          string
-	backoff      *backoff.Backoff
+	lName           string
-	nextQuery    time.Time
+	lVal            string
-	backoffMtx   sync.RWMutex
+	backoff         *backoff.Backoff
-	interval     time.Duration
+	nextQuery       time.Time
-	conn         *websocket.Conn
+	backoffMtx      sync.RWMutex
-	w            io.Writer
+	interval        time.Duration
-	recv         chan time.Time
+	conn            *websocket.Conn
-	quit         chan struct{}
+	w               io.Writer
-	shuttingDown bool
+	recv            chan time.Time
-	done         chan struct{}
+	quit            chan struct{}
+	shuttingDown    bool
+	done            chan struct{}
 }
 
 func NewReader(writer io.Writer,
 	receivedChan chan time.Time,
 	tls bool,
+	tlsConfig *tls.Config,
+	caFile string,
 	address string,
 	user string,
 	pass string,
@@ -102,25 +108,27 @@ func NewReader(writer io.Writer,
 	bkoff := backoff.New(context.Background(), bkcfg)
 
 	rd := Reader{
-		header:       h,
+		header:          h,
-		tls:          tls,
+		tls:             tls,
-		addr:         address,
+		clientTLSConfig: tlsConfig,
-		user:         user,
+		caFile:          caFile,
-		pass:         pass,
+		addr:            address,
-		tenantID:     tenantID,
+		user:            user,
-		queryTimeout: queryTimeout,
+		pass:            pass,
-		sName:        streamName,
+		tenantID:        tenantID,
-		sValue:       streamValue,
+		queryTimeout:    queryTimeout,
-		lName:        labelName,
+		sName:           streamName,
-		lVal:         labelVal,
+		sValue:          streamValue,
-		nextQuery:    next,
+		lName:           labelName,
-		backoff:      bkoff,
+		lVal:            labelVal,
-		interval:     interval,
+		nextQuery:       next,
-		w:            writer,
+		backoff:         bkoff,
-		recv:         receivedChan,
+		interval:        interval,
-		quit:         make(chan struct{}),
+		w:               writer,
-		done:         make(chan struct{}),
+		recv:            receivedChan,
-		shuttingDown: false,
+		quit:            make(chan struct{}),
+		done:            make(chan struct{}),
+		shuttingDown:    false,
 	}
 
 	go rd.run()
@@ -189,7 +197,11 @@ func (r *Reader) QueryCountOverTime(queryRange string) (float64, error) {
 	}
 	req.Header.Set("User-Agent", userAgent)
 
-	resp, err := http.DefaultClient.Do(req)
+	httpClient, err := r.httpClient()
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to create httpClient when querying Loki for count of logs over time")
+	}
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return 0, err
 	}
@@ -280,10 +292,14 @@ func (r *Reader) Query(start time.Time, end time.Time) ([]time.Time, error) {
 	}
 	req.Header.Set("User-Agent", userAgent)
 
-	resp, err := http.DefaultClient.Do(req)
+	httpClient, err := r.httpClient()
 	if err != nil {
 		return nil, err
 	}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create httpClient when issuing Loki query")
+	}
 	defer func() {
 		if err := resp.Body.Close(); err != nil {
 			log.Println("error closing body", err)
@@ -329,7 +345,7 @@ func (r *Reader) Query(start time.Time, end time.Time) ([]time.Time, error) {
 	return tss, nil
 }
 
-// run uses the established websocket connection to tail logs from Loki and
+// run uses the established websocket connection to tail logs from Loki
 func (r *Reader) run() {
 	r.closeAndReconnect()
 
@@ -421,7 +437,8 @@ func (r *Reader) closeAndReconnect() {
 
 		fmt.Fprintf(r.w, "Connecting to loki at %v, querying for label '%v' with value '%v'\n", u.String(), r.lName, r.lVal)
 
-		c, _, err := websocket.DefaultDialer.Dial(u.String(), r.header)
+		dialer := r.webSocketDialer()
+		c, _, err := dialer.Dial(u.String(), r.header)
 		if err != nil {
 			fmt.Fprintf(r.w, "failed to connect to %s with err %s\n", u.String(), err)
 			<-time.After(10 * time.Second)
@@ -442,6 +459,35 @@ func (r *Reader) closeAndReconnect() {
 	}
 }
 
+// httpClient uses the config in Reader to return a http client.
+// http.DefaultClient will be returned in the case that the connection to Loki is http or TLS without client certs.
+// For the mTLS case, return a http.Client configured to use the client side certificates.
+func (r *Reader) httpClient() (*http.Client, error) {
+	if r.clientTLSConfig == nil {
+		return http.DefaultClient, nil
+	}
+	rt, err := config.NewTLSRoundTripper(r.clientTLSConfig, r.caFile, func(tls *tls.Config) (http.RoundTripper, error) {
+		return &http.Transport{TLSClientConfig: tls}, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &http.Client{
+		Transport: rt,
+	}, nil
+}
+
+// webSocketDialer creates a dialer for the web socket connection to Loki
+// websocket.DefaultDialer will be returned in the case that the connection to Loki is http or TLS without client certs.
+// For the mTLS case, return a websocket.Dialer configured to use client side certificates.
+func (r *Reader) webSocketDialer() *websocket.Dialer {
+	return &websocket.Dialer{
+		Proxy:            http.ProxyFromEnvironment,
+		TLSClientConfig:  r.clientTLSConfig,
+		HandshakeTimeout: 45 * time.Second,
+	}
+}
+
 func parseResponse(entry *loghttp.Entry) (*time.Time, error) {
 	sp := strings.Split(entry.Line, " ")
 	if len(sp) != 2 {