Loki Canary: Enable insecure-skip-verify for all requests (#8869)

**What this PR does / why we need it**:

This supports cases where the cert cannot be trusted but we are not
using mTLS, i.e. `loki-canary -tls=true -insecure=true`

**Checklist**
- [x] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- [ ] Documentation added
- [ ] Tests updated
- [ ] `CHANGELOG.md` updated
- [ ] Changes that require user attention or interaction to upgrade are
documented in `docs/sources/upgrading/_index.md`

cmd/loki-canary/main.go

@@ -111,7 +111,9 @@ func main() {
 	}
 
 	var tlsConfig *tls.Config
-	tc := config.TLSConfig{}
+	tc := config.TLSConfig{
+		InsecureSkipVerify: *insecureSkipVerify,
+	}
 	if *certFile != "" || *keyFile != "" || *caFile != "" {
 		if !*useTLS {
 			_, _ = fmt.Fprintf(os.Stderr, "Must set --tls when specifying client certs\n")
@@ -120,8 +122,8 @@ func main() {
 		tc.CAFile = *caFile
 		tc.CertFile = *certFile
 		tc.KeyFile = *keyFile
-		tc.InsecureSkipVerify = *insecureSkipVerify
-
+	}
+	if *useTLS {
 		var err error
 		tlsConfig, err = config.NewTLSConfig(&tc)
 		if err != nil {

pkg/canary/reader/reader.go

@@ -97,15 +97,18 @@ func NewReader(writer io.Writer,
 
 	// http.DefaultClient will be used in the case that the connection to Loki is http or TLS without client certs.
 	httpClient := http.DefaultClient
-	if tlsConfig != nil {
+	if tlsConfig != nil && (certFile != "" || keyFile != "" || caFile != "") {
 		// For the mTLS case, use a http.Client configured with the client side certificates.
 		rt, err := config.NewTLSRoundTripper(tlsConfig, caFile, certFile, keyFile, func(tls *tls.Config) (http.RoundTripper, error) {
 			return &http.Transport{TLSClientConfig: tls}, nil
 		})
 		if err != nil {
-			return nil, errors.Wrapf(err, "Failed to create HTTPS transport with TLS config")
+			return nil, errors.Wrapf(err, "Failed to create HTTPS transport with mTLS config")
 		}
 		httpClient = &http.Client{Transport: rt}
+	} else if tlsConfig != nil {
+		// non-mutual TLS
+		httpClient = &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}}
 	}
 	if user != "" {
 		h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(user+":"+pass)))