pkg/storage/chunk/aws: Add s3.http.ca-file option (#4211)

* pkg/storage/chunk/aws: Fix insecure-skip-verify documentation

The documentation claimed that the `s3.http.insecure-skip-verify` option needed
to be set to `false` in order to skip verifying certificates, but it needs to
be set to `true`.

Signed-off-by: Benoît Knecht <bknecht@protonmail.ch>

* pkg/storage/chunk/aws: Add s3.http.ca-file option

This option lets users set a custom CA file for connections to S3, in case they
use a local S3 instance with an internal PKI.

Signed-off-by: Benoît Knecht <bknecht@protonmail.ch>

docs/sources/configuration/_index.md

@@ -522,10 +522,15 @@ storage:
       # CLI flag: -ruler.storage.s3.http.response-header-timeout
       [response_header_timeout: <duration> | default = 0s]
 
-      # Set to false to skip verifying the certificate chain and hostname.
+      # Set to true to skip verifying the certificate chain and hostname.
       # CLI flag: -ruler.storage.s3.http.insecure-skip-verify
       [insecure_skip_verify: <boolean> | default = false]
 
+      # Path to the trusted CA file that signed the SSL certificate of the S3
+      # endpoint.
+      # CLI flag: -ruler.storage.s3.http.ca-file
+      [ca_file: <string> | default = ""]
+
   swift:
     # Openstack authentication URL.
     # CLI flag: -ruler.storage.swift.auth-url
@@ -1190,10 +1195,15 @@ aws:
     # CLI flag: -s3.http.response-header-timeout
     [response_header_timeout: <duration> | default = 0s]
 
-    # Set to false to skip verifying the certificate chain and hostname.
+    # Set to true to skip verifying the certificate chain and hostname.
     # CLI flag: -s3.http.insecure-skip-verify
     [insecure_skip_verify: <boolean> | default = false]
 
+    # Path to the trusted CA file that signed the SSL certificate of the S3
+    # endpoint.
+    # CLI flag: -s3.http.ca-file
+    [ca_file: <string> | default = ""]
+
   # Configure the DynamoDB connection
   dynamodb:
     # URL for DynamoDB with escaped Key and Secret encoded. If only region is specified as a

pkg/storage/chunk/aws/s3_storage_client.go

@@ -3,12 +3,14 @@ package aws
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"flag"
 	"fmt"
 	"hash/fnv"
 	"io"
 	"net"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -82,6 +84,7 @@ type HTTPConfig struct {
 	IdleConnTimeout       time.Duration `yaml:"idle_conn_timeout"`
 	ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout"`
 	InsecureSkipVerify    bool          `yaml:"insecure_skip_verify"`
+	CAFile                string        `yaml:"ca_file"`
 }
 
 // RegisterFlags adds the flags required to config this to the given FlagSet
@@ -109,7 +112,8 @@ func (cfg *S3Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 
 	f.DurationVar(&cfg.HTTPConfig.IdleConnTimeout, prefix+"s3.http.idle-conn-timeout", 90*time.Second, "The maximum amount of time an idle connection will be held open.")
 	f.DurationVar(&cfg.HTTPConfig.ResponseHeaderTimeout, prefix+"s3.http.response-header-timeout", 0, "If non-zero, specifies the amount of time to wait for a server's response headers after fully writing the request.")
-	f.BoolVar(&cfg.HTTPConfig.InsecureSkipVerify, prefix+"s3.http.insecure-skip-verify", false, "Set to false to skip verifying the certificate chain and hostname.")
+	f.BoolVar(&cfg.HTTPConfig.InsecureSkipVerify, prefix+"s3.http.insecure-skip-verify", false, "Set to true to skip verifying the certificate chain and hostname.")
+	f.StringVar(&cfg.HTTPConfig.CAFile, prefix+"s3.http.ca-file", "", "Path to the trusted CA file that signed the SSL certificate of the S3 endpoint.")
 	f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", ")))
 }
 
@@ -235,6 +239,19 @@ func buildS3Config(cfg S3Config) (*aws.Config, []string, error) {
 		s3Config = s3Config.WithCredentials(creds)
 	}
 
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: cfg.HTTPConfig.InsecureSkipVerify,
+	}
+
+	if cfg.HTTPConfig.CAFile != "" {
+		tlsConfig.RootCAs = x509.NewCertPool()
+		data, err := os.ReadFile(cfg.HTTPConfig.CAFile)
+		if err != nil {
+			return nil, nil, err
+		}
+		tlsConfig.RootCAs.AppendCertsFromPEM(data)
+	}
+
 	// While extending S3 configuration this http config was copied in order to
 	// to maintain backwards compatibility with previous versions of Cortex while providing
 	// more flexible configuration of the http client
@@ -252,7 +269,7 @@ func buildS3Config(cfg S3Config) (*aws.Config, []string, error) {
 		TLSHandshakeTimeout:   3 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
 		ResponseHeaderTimeout: cfg.HTTPConfig.ResponseHeaderTimeout,
-		TLSClientConfig:       &tls.Config{InsecureSkipVerify: cfg.HTTPConfig.InsecureSkipVerify},
+		TLSClientConfig:       tlsConfig,
 	})
 
 	if cfg.Inject != nil {