From 7e4e34e4c8da860f08db3c3d89eeb8d8d77d82dd Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Fri, 12 Dec 2025 10:46:01 -0700 Subject: [PATCH] fix: bump helm deps, publish loki-helm-test w/ release (#19939) Signed-off-by: Trevor Whitney Co-authored-by: nicolevanderhoeven <36070553+nicolevanderhoeven@users.noreply.github.com> --- .github/release-workflows.jsonnet | 1 + .github/workflows/helm-ci.yml | 70 ++++++++++++++++- .github/workflows/minor-release-pr.yml | 81 ++++++++++++++++++++ .github/workflows/patch-release-pr.yml | 81 ++++++++++++++++++++ docs/sources/setup/install/helm/reference.md | 4 +- production/helm/loki/README.md | 2 +- production/helm/loki/values.yaml | 4 +- 7 files changed, 234 insertions(+), 9 deletions(-) diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet index 0713fb6685..84c4ecbabb 100644 --- a/.github/release-workflows.jsonnet +++ b/.github/release-workflows.jsonnet @@ -31,6 +31,7 @@ local imageJobs = { promtail: build.image('promtail', 'clients/cmd/promtail', platform=platforms.all), querytee: build.image('loki-query-tee', 'cmd/querytee', platform=[r.forPlatform('linux/amd64'), r.forPlatform('linux/arm64')]), 'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=[r.forPlatform('linux/amd64'), r.forPlatform('linux/arm64')]), + 'loki-helm-test': build.image('loki-helm-test', 'production/helm/loki/src/helm-test', platform=platforms.all), }; local weeklyImageJobs = { diff --git a/.github/workflows/helm-ci.yml b/.github/workflows/helm-ci.yml index 8682f3050d..1355aa7f26 100644 --- a/.github/workflows/helm-ci.yml +++ b/.github/workflows/helm-ci.yml @@ -8,7 +8,7 @@ on: env: CT_CONFIGFILE: production/helm/ct.yaml - INTEGRARION_TESTS_DIR: production/helm/loki/test/integration + INTEGRATION_TESTS_DIR: production/helm/loki/test/integration jobs: # Temporarily disabled linting because this step doesn't easily support passing GO tags yet. @@ -103,7 +103,7 @@ jobs: - name: List tests id: list_tests - working-directory: "${{ env.INTEGRARION_TESTS_DIR }}" + working-directory: "${{ env.INTEGRATION_TESTS_DIR }}" env: LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} run: | @@ -153,7 +153,69 @@ jobs: start: false - name: Run test - run: helm-chart-toolbox/tools/helm-test/helm-test "${TEST_DIRECTORY}" + run: | + # Enable verbose output only if runner debug mode is enabled + if [ "${{ runner.debug }}" == "1" ]; then + set -x + echo "=== Debug mode enabled - verbose output active ===" + fi + + echo "=== Starting Helm Test ===" + echo "Test directory: ${TEST_DIRECTORY}" + helm-chart-toolbox/tools/helm-test/helm-test "${TEST_DIRECTORY}" || { + EXIT_CODE=$? + echo "=== Test failed with exit code: $EXIT_CODE ===" + echo "=== Attempting to gather debug information ===" + + # Try to get cluster info if kubectl is available + if command -v kubectl &> /dev/null; then + echo "=== Kubernetes Cluster Debug Info ===" + echo "--- Cluster nodes ---" + kubectl get nodes -o wide || true + echo "--- All namespaces ---" + kubectl get namespaces || true + echo "--- Helm releases ---" + helm list -A || true + echo "--- Helm release status (if exists) ---" + helm status loki -n loki || echo "Helm release 'loki' not found or failed" + echo "--- All resources in loki namespace ---" + kubectl get all -n loki || true + echo "--- Pods in loki namespace ---" + kubectl get pods -n loki -o wide || true + echo "--- Pod status details ---" + kubectl get pods -n loki -o json | jq -r '.items[] | "\(.metadata.name): phase=\(.status.phase), reason=\(.status.reason // "none"), message=\(.status.message // "none")"' || true + echo "--- Events in loki namespace (last 100) ---" + kubectl get events -n loki --sort-by='.lastTimestamp' | tail -100 || true + echo "--- Failed/Error pods details ---" + kubectl get pods -n loki -o json | jq -r '.items[] | select(.status.phase != "Running" and .status.phase != "Succeeded") | "\(.metadata.name): \(.status.phase) - \(.status.reason // "no reason")"' || true + echo "--- Describe failed pods ---" + for pod in $(kubectl get pods -n loki -o json | jq -r '.items[] | select(.status.phase != "Running" and .status.phase != "Succeeded") | .metadata.name'); do + echo "=== Describe pod: $pod ===" + kubectl describe pod -n loki "$pod" || true + done + echo "--- Pod logs for failed pods ---" + for pod in $(kubectl get pods -n loki -o json | jq -r '.items[] | select(.status.phase != "Running" and .status.phase != "Succeeded") | .metadata.name'); do + echo "=== Logs for pod: $pod ===" + kubectl logs -n loki "$pod" --tail=200 || true + echo "=== Previous logs for pod: $pod ===" + kubectl logs -n loki "$pod" --previous --tail=200 || true + done + + # Only show all pod logs if debug mode is enabled (can be very verbose) + if [ "${{ runner.debug }}" == "1" ]; then + echo "--- All pod logs (last 50 lines each) ---" + for pod in $(kubectl get pods -n loki -o json | jq -r '.items[] | .metadata.name'); do + echo "=== Logs for pod: $pod ===" + kubectl logs -n loki "$pod" --tail=50 || true + done + fi + fi + + exit $EXIT_CODE + } env: - TEST_DIRECTORY: "source/${{ env.INTEGRARION_TESTS_DIR }}/${{ matrix.test }}" + TEST_DIRECTORY: "source/${{ env.INTEGRATION_TESTS_DIR }}/${{ matrix.test }}" DELETE_CLUSTER: true + # Enable verbose output for helm-test only if runner debug mode is enabled + DEBUG: ${{ runner.debug == '1' && 'true' || 'false' }} + VERBOSE: ${{ runner.debug == '1' && 'true' || 'false' }} diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index 4a2dc94c3a..ca206df710 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -37,6 +37,7 @@ jobs: - "loki-canary" - "loki-canary-boringcrypto" - "loki-docker-driver" + - "loki-helm-test" - "promtail" - "querytee" permissions: @@ -831,6 +832,86 @@ jobs: - arch: "linux/arm64" runs_on: - "github-hosted-ubuntu-arm64-small" + loki-helm-test: + needs: + - "version" + permissions: + contents: "write" + id-token: "write" + pull-requests: "write" + runs-on: "${{ matrix.runs_on }}" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + persist-credentials: false + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + persist-credentials: false + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + package-manager-cache: false + - id: "fetch_gcs_credentials" + name: "fetch gcs credentials from vault" + uses: "grafana/shared-workflows/actions/get-vault-secrets@28361cdb22223e5f1e34358c86c20908e7248760" + with: + repo_secrets: "GCS_SERVICE_ACCOUNT_KEY=gcs:service-account-key" + - name: "auth gcs" + uses: "google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f" + with: + credentials_json: "${{ env.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Docker buildx" + uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" + - id: "platform" + name: "Parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.arch }}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.arch }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" + uses: "docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1" + with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" + context: "release" + file: "release/production/helm/loki/src/helm-test/Dockerfile" + outputs: "type=docker,dest=release/images/loki-helm-test-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.arch }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-helm-test:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Upload artifacts" + uses: "google-github-actions/upload-cloud-storage@386ab77f37fdf51c0e38b3d229fad286861cc0d0" + with: + destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/images" + path: "release/images/loki-helm-test-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + include: + - arch: "linux/amd64" + runs_on: + - "github-hosted-ubuntu-x64-small" + - arch: "linux/arm64" + runs_on: + - "github-hosted-ubuntu-arm64-small" + - arch: "linux/arm" + runs_on: + - "github-hosted-ubuntu-arm64-small" promtail: needs: - "version" diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index c30f3f472c..81fe22ec39 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -37,6 +37,7 @@ jobs: - "loki-canary" - "loki-canary-boringcrypto" - "loki-docker-driver" + - "loki-helm-test" - "promtail" - "querytee" permissions: @@ -831,6 +832,86 @@ jobs: - arch: "linux/arm64" runs_on: - "github-hosted-ubuntu-arm64-small" + loki-helm-test: + needs: + - "version" + permissions: + contents: "write" + id-token: "write" + pull-requests: "write" + runs-on: "${{ matrix.runs_on }}" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + persist-credentials: false + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + persist-credentials: false + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + package-manager-cache: false + - id: "fetch_gcs_credentials" + name: "fetch gcs credentials from vault" + uses: "grafana/shared-workflows/actions/get-vault-secrets@28361cdb22223e5f1e34358c86c20908e7248760" + with: + repo_secrets: "GCS_SERVICE_ACCOUNT_KEY=gcs:service-account-key" + - name: "auth gcs" + uses: "google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f" + with: + credentials_json: "${{ env.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Docker buildx" + uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" + - id: "platform" + name: "Parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.arch }}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.arch }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" + uses: "docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1" + with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" + context: "release" + file: "release/production/helm/loki/src/helm-test/Dockerfile" + outputs: "type=docker,dest=release/images/loki-helm-test-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.arch }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-helm-test:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Upload artifacts" + uses: "google-github-actions/upload-cloud-storage@386ab77f37fdf51c0e38b3d229fad286861cc0d0" + with: + destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/images" + path: "release/images/loki-helm-test-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + include: + - arch: "linux/amd64" + runs_on: + - "github-hosted-ubuntu-x64-small" + - arch: "linux/arm64" + runs_on: + - "github-hosted-ubuntu-arm64-small" + - arch: "linux/arm" + runs_on: + - "github-hosted-ubuntu-arm64-small" promtail: needs: - "version" diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md index af4b62d455..21fc5417ba 100644 --- a/docs/sources/setup/install/helm/reference.md +++ b/docs/sources/setup/install/helm/reference.md @@ -7982,7 +7982,7 @@ true string 
-"v0.15.3"
+"v0.15.4"
@@ -12515,7 +12515,7 @@ false string Docker image tag 
-"1.30.10"
+"1.30.9"
diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index 7ea99534ac..c88368a35d 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -16,7 +16,7 @@ Helm chart for Grafana Loki and Grafana Enterprise Logs supporting monolithic, s |------------|------|---------| | https://charts.min.io/ | minio(minio) | 5.4.0 | | https://grafana.github.io/helm-charts | grafana-agent-operator(grafana-agent-operator) | 0.5.2 | -| https://grafana.github.io/helm-charts | rollout_operator(rollout-operator) | 0.37.1 | +| https://grafana.github.io/helm-charts | rollout_operator(rollout-operator) | 0.38.2 | Find more information in the Loki Helm Chart [documentation](https://grafana.com/docs/loki/latest/setup/install/helm/). diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 2591a91500..f0d1722969 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -3591,7 +3591,7 @@ memcachedExporter: enabled: true image: repository: prom/memcached-exporter - tag: v0.15.3 + tag: v0.15.4 pullPolicy: IfNotPresent resources: requests: {} @@ -4079,7 +4079,7 @@ sidecar: # -- The Docker registry and image for the k8s sidecar repository: docker.io/kiwigrid/k8s-sidecar # -- Docker image tag - tag: 1.30.10 + tag: 1.30.9 # -- Docker image sha. If empty, no sha will be used sha: "" # -- Docker image pull policy