mirror of https://github.com/grafana/loki
[helm] Add a loki canary test to the helm chart (#7229)
**What this PR does / why we need it**: Add a helm chart test that utilizes the Loki canary to confirm the Loki cluster is functioning correctly. **Which issue(s) this PR fixes**: Fixes #7228 Co-authored-by: Karsten Jeschkies <k@jeschkies.xyz>pull/7510/head helm-loki-3.3.0
Trevor Whitney
4 years ago
committed by
GitHub
parent
9bc6e85604
commit
6c53113be1
@ -1,45 +1,53 @@ |
||||
{ self, nixpkgs, system }: |
||||
let buildVars = import ./build-vars.nix; |
||||
in { |
||||
overlay = final: prev: rec { |
||||
loki = |
||||
let |
||||
# self.rev is only set on a clean git tree |
||||
gitRevision = if (self ? rev) then self.rev else "dirty"; |
||||
shortGitRevsion = with prev.lib; |
||||
if (self ? rev) then |
||||
(strings.concatStrings |
||||
(lists.take 8 (strings.stringToCharacters gitRevision))) |
||||
else |
||||
"dirty"; |
||||
in |
||||
{ |
||||
overlay = final: prev: |
||||
let |
||||
# self.rev is only set on a clean git tree |
||||
gitRevision = if (self ? rev) then self.rev else "dirty"; |
||||
shortGitRevsion = with prev.lib; |
||||
if (self ? rev) then |
||||
(strings.concatStrings |
||||
(lists.take 8 (strings.stringToCharacters gitRevision))) |
||||
else |
||||
"dirty"; |
||||
|
||||
# the image tag script is hard coded to take only 7 characters |
||||
imageTagVersion = with prev.lib; |
||||
if (self ? rev) then |
||||
(strings.concatStrings |
||||
(lists.take 8 (strings.stringToCharacters gitRevision))) |
||||
else |
||||
"dirty"; |
||||
# the image tag script is hard coded to take only 7 characters |
||||
imageTagVersion = with prev.lib; |
||||
if (self ? rev) then |
||||
(strings.concatStrings |
||||
(lists.take 8 (strings.stringToCharacters gitRevision))) |
||||
else |
||||
"dirty"; |
||||
|
||||
imageTag = |
||||
if (self ? rev) then |
||||
"${buildVars.gitBranch}-${imageTagVersion}" |
||||
else |
||||
"${buildVars.gitBranch}-${imageTagVersion}-WIP"; |
||||
in |
||||
prev.callPackage ./loki.nix { |
||||
imageTag = |
||||
if (self ? rev) then |
||||
"${buildVars.gitBranch}-${imageTagVersion}" |
||||
else |
||||
"${buildVars.gitBranch}-${imageTagVersion}-WIP"; |
||||
|
||||
loki-helm-test = prev.callPackage ../production/helm/loki/src/helm-test { |
||||
inherit (prev) pkgs lib buildGoModule dockerTools; |
||||
rev = gitRevision; |
||||
}; |
||||
in |
||||
{ |
||||
inherit (loki-helm-test) loki-helm-test loki-helm-test-docker; |
||||
|
||||
loki = prev.callPackage ./loki.nix { |
||||
inherit imageTag; |
||||
inherit (buildVars) gitBranch; |
||||
version = shortGitRevsion; |
||||
pkgs = prev; |
||||
}; |
||||
|
||||
faillint = prev.callPackage ./faillint.nix { |
||||
inherit (prev) lib buildGoModule fetchFromGitHub; |
||||
}; |
||||
faillint = prev.callPackage ./faillint.nix { |
||||
inherit (prev) lib buildGoModule fetchFromGitHub; |
||||
}; |
||||
|
||||
chart-releaser = prev.callPackage ./chart-releaser.nix { |
||||
inherit (prev) pkgs lib buildGoModule fetchFromGitHub; |
||||
chart-releaser = prev.callPackage ./chart-releaser.nix { |
||||
inherit (prev) pkgs lib buildGoModule fetchFromGitHub; |
||||
}; |
||||
}; |
||||
}; |
||||
} |
||||
|
||||
@ -1,27 +0,0 @@ |
||||
read: |
||||
replicas: 1 |
||||
persistence: |
||||
enabled: true |
||||
size: 100Mi |
||||
|
||||
write: |
||||
replicas: 1 |
||||
persistence: |
||||
enabled: true |
||||
size: 100Mi |
||||
|
||||
loki: |
||||
commonConfig: |
||||
replication_factor: 1 |
||||
|
||||
gateway: |
||||
nginxConfig: |
||||
httpSnippet: |- |
||||
client_max_body_size 100M; |
||||
serverSnippet: |- |
||||
client_max_body_size 100M; |
||||
|
||||
basicAuth: |
||||
enabled: true |
||||
username: user |
||||
password: pass |
||||
@ -0,0 +1,13 @@ |
||||
FROM golang:1.18.5 as build |
||||
|
||||
# build via Makefile target helm-test-image in root |
||||
# Makefile. Building from this directory will not be |
||||
# able to access source needed in rest of repo. |
||||
COPY . /src/loki |
||||
WORKDIR /src/loki |
||||
RUN make clean && make BUILD_IN_CONTAINER=false helm-test |
||||
|
||||
FROM alpine:3.16.2 |
||||
RUN apk add --update --no-cache ca-certificates=20220614-r0 |
||||
COPY --from=build /src/loki/production/helm/loki/src/helm-test/helm-test /usr/bin/helm-test |
||||
ENTRYPOINT [ "/usr/bin/helm-test" ] |
||||
@ -0,0 +1,7 @@ |
||||
# Loki Helm Test |
||||
|
||||
This folder contains a collection of go tests that test if a Loki canary is running correctly. It's primary use it to test that the helm chart is working correctly by using metrics from the Loki canary. In the helm chart, the template for this test is only available if you are running both the Loki canary and have self monitoring enabled (as the Loki canary's logs need to be in Loki for it to work). However, the tests in this folder can be run against any running Loki canary using `go test`. |
||||
|
||||
## Instructions |
||||
|
||||
Run `go test .` from this directory, or use the Docker image published at `grafana/loki-helm-test`. |
||||
@ -0,0 +1,105 @@ |
||||
//go:build helm_test
|
||||
// +build helm_test
|
||||
|
||||
package test |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"fmt" |
||||
"os" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/prometheus/client_golang/api" |
||||
v1 "github.com/prometheus/client_golang/api/prometheus/v1" |
||||
"github.com/prometheus/common/model" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestCanary(t *testing.T) { |
||||
totalEntriesQuery := "sum(loki_canary_entries_total)" |
||||
totalEntriesMissingQuery := "sum(loki_canary_missing_entries_total)" |
||||
|
||||
timeout := getEnv("CANARY_TEST_TIMEOUT", "1m") |
||||
timeoutDuration, err := time.ParseDuration(timeout) |
||||
require.NoError(t, err, "Failed to parse timeout. Please set CANARY_TEST_TIMEOUT to a valid duration.") |
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) |
||||
|
||||
t.Cleanup(func() { |
||||
cancel() |
||||
}) |
||||
|
||||
t.Run("Canary should have entries", func(t *testing.T) { |
||||
client := newClient(t) |
||||
|
||||
eventually(t, func() error { |
||||
result, _, err := client.Query(ctx, totalEntriesQuery, time.Now(), v1.WithTimeout(timeoutDuration)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return testResult(t, result, totalEntriesQuery, func(v model.SampleValue) bool { |
||||
return v > 0 |
||||
}, fmt.Sprintf("Expected %s to be greater than 0", totalEntriesQuery)) |
||||
}, timeoutDuration, "Expected Loki Canary to have entries") |
||||
}) |
||||
|
||||
t.Run("Canary should not have missed any entries", func(t *testing.T) { |
||||
client := newClient(t) |
||||
|
||||
eventually(t, func() error { |
||||
result, _, err := client.Query(ctx, totalEntriesMissingQuery, time.Now(), v1.WithTimeout(timeoutDuration)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return testResult(t, result, totalEntriesMissingQuery, func(v model.SampleValue) bool { |
||||
return v == 0 |
||||
}, fmt.Sprintf("Expected %s to equal 0", totalEntriesMissingQuery)) |
||||
}, timeoutDuration, "Expected Loki Canary to not have any missing entries") |
||||
}) |
||||
} |
||||
|
||||
func getEnv(key, fallback string) string { |
||||
if value, ok := os.LookupEnv(key); ok { |
||||
return value |
||||
} |
||||
return fallback |
||||
} |
||||
|
||||
func testResult(t *testing.T, result model.Value, query string, test func(model.SampleValue) bool, msg string) error { |
||||
if v, ok := result.(model.Vector); ok { |
||||
for _, s := range v { |
||||
t.Logf("%s => %v\n", query, s.Value) |
||||
if !test(s.Value) { |
||||
return errors.New(msg) |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
return fmt.Errorf("unexpected Prometheus result type: %v ", result.Type()) |
||||
} |
||||
|
||||
func newClient(t *testing.T) v1.API { |
||||
address := os.Getenv("CANARY_PROMETHEUS_ADDRESS") |
||||
require.NotEmpty(t, address, "CANARY_PROMETHEUS_ADDRESS must be set to a valid prometheus address") |
||||
|
||||
client, err := api.NewClient(api.Config{ |
||||
Address: address, |
||||
}) |
||||
require.NoError(t, err, "Failed to create Loki Canary client") |
||||
|
||||
return v1.NewAPI(client) |
||||
} |
||||
|
||||
func eventually(t *testing.T, test func() error, timeoutDuration time.Duration , msg string) { |
||||
require.Eventually(t, func() bool { |
||||
queryError := test() |
||||
if queryError != nil { |
||||
t.Logf("Query failed\n%+v\n", queryError) |
||||
} |
||||
return queryError == nil |
||||
}, timeoutDuration, 1*time.Second, msg) |
||||
} |
||||
@ -0,0 +1,27 @@ |
||||
{ pkgs, lib, buildGoModule, dockerTools, rev }: |
||||
rec { |
||||
loki-helm-test = buildGoModule rec { |
||||
pname = "loki-helm-test"; |
||||
version = "0.1.0"; |
||||
|
||||
src = ./../../../../..; |
||||
vendorSha256 = null; |
||||
|
||||
buildPhase = '' |
||||
runHook preBuild |
||||
go test --tags=helm_test -c -o $out/bin/helm-test ./production/helm/loki/src/helm-test |
||||
runHook postBuild |
||||
''; |
||||
|
||||
doCheck = false; |
||||
}; |
||||
|
||||
# by default, uses the nix hash as the tag, which can be retrieved with: |
||||
# basename "$(readlink result)" | cut -d - -f 1 |
||||
loki-helm-test-docker = dockerTools.buildImage { |
||||
name = "grafana/loki-helm-test"; |
||||
config = { |
||||
Entrypoint = [ "${loki-helm-test}/bin/helm-test" ]; |
||||
}; |
||||
}; |
||||
} |
||||
@ -0,0 +1,7 @@ |
||||
{{/* |
||||
Docker image name for loki helm test |
||||
*/}} |
||||
{{- define "loki.helm-test-image" -}} |
||||
{{- $dict := dict "service" .Values.test.image "global" .Values.global.image "defaultVersion" "latest" -}} |
||||
{{- include "loki.baseImage" $dict -}} |
||||
{{- end -}} |
||||
@ -0,0 +1,33 @@ |
||||
{{- with .Values.test }} |
||||
{{- if and .enabled $.Values.monitoring.selfMonitoring.enabled $.Values.monitoring.selfMonitoring.lokiCanary.enabled }} |
||||
--- |
||||
apiVersion: v1 |
||||
kind: Pod |
||||
metadata: |
||||
name: "{{ include "loki.name" $ }}-helm-test" |
||||
labels: |
||||
{{- include "loki.labels" $ | nindent 4 }} |
||||
{{- with .labels }} |
||||
{{- toYaml . | nindent 4 }} |
||||
{{- end }} |
||||
annotations: |
||||
{{- with .annotations }} |
||||
{{- toYaml . | nindent 4 }} |
||||
{{- end }} |
||||
"helm.sh/hook": test |
||||
spec: |
||||
containers: |
||||
- name: loki-helm-test |
||||
image: {{ include "loki.helm-test-image" $ }} |
||||
env: |
||||
- name: CANARY_PROMETHEUS_ADDRESS |
||||
value: "{{ .prometheusAddress }}" |
||||
{{- with .timeout }} |
||||
- name: CANARY_TEST_TIMEOUT |
||||
value: "{{ . }}" |
||||
{{- end }} |
||||
args: |
||||
- -test.v |
||||
restartPolicy: Never |
||||
{{- end }} |
||||
{{- end }} |
||||
@ -0,0 +1,24 @@ |
||||
{{- if .Values.config }} |
||||
{{- fail "Top level 'config' is not allowed. Most common configuration sections are exposed under the `loki` section. If you need to override the whole config, provide the configuration as a string that can contain template expressions under `loki.config`. Alternatively, you can provide the configuration as an external secret." }} |
||||
{{- end }} |
||||
|
||||
{{ with .Values.monitoring.selfMonitoring}} |
||||
|
||||
{{- if and (not .enabled) .lokiCanary.enabled }} |
||||
{{- fail "Loki Canary requires self monitoring to also be enabled"}} |
||||
{{- end }} |
||||
|
||||
{{- if and (not .enabled) $.Values.test.enabled }} |
||||
{{- fail "Helm test requires self monitoring to be enabled"}} |
||||
{{- end }} |
||||
|
||||
{{- if and (not .lokiCanary.enabled) $.Values.test.enabled }} |
||||
{{- fail "Helm test requires the Loki Canary to be enabled"}} |
||||
{{- end }} |
||||
|
||||
{{- end}} |
||||
|
||||
{{- if and .Values.test.enabled (not .Values.test.prometheusAddress) }} |
||||
{{- fail "Helm test requires a prometheusAddress for an instance scraping the Loki canary's metrics"}} |
||||
{{- end }} |
||||
|
||||
Loading…
Reference in new issue