CI: Add nightly prerelease builds (#74119)

* Add nightly prerelease builds * Fix duplicated pipeline names * Fix misnamed dependencies * Fix misnamed dependencies * Fix string formatting * Add option to specify bucket to RGM pipeline * Fix trigger? * Comment out cron triggers * Fix windows bucket for nightly * Fix versioning for windows and verify pipelines * Use grafana/grafana-build:dev-209553c * Fix version on windows steps * Fix version on windows steps * Fix windows .zip path * Fix windows .zip path * Remove windows builds from nightly for now * Remove verify release pipeline from nightly * Add docstring to rgm_release * Revert changes to get_windows_steps * Simplify changes to rgm.star * Use grafana/grafana-build:dev-f5a15d4 * Add rgm copy step * Use grafana/grafana-build:dev-d88be0f * Fix destination variable * Escape copy destination environment variable * Add -r flag to rgm copy command * Add dependency to rgm-copy step * Add dist volume * Use absolute path for dist volume * Move dist folder to drone workspace * Delegate drone workspace path to grafana-build * Use grafana/grafana-build:dev-66149b8 * Lower folder depth * Use grafana/grafana-build:dev-7355791 * Add rgm-nightly-publish pipeline * Merge imports on rgm.star * Fix rgm_copy to allow copying to local destination * Use grafana/grafana-build:dev-36ec1e2 * Use grafana/grafana-build:dev-634d8dc * Use grafana/grafana-build:dev-7a93728 * Use grafana/grafana-build:dev-5e36725 * Use grafana/grafana-build:dev-f5ebe1f * Fix copy source for nightly builds * Fix drone build number on rgm-copy step * Use grafana/grafana-build:dev-637583f * Use grafana/grafana-build:dev-f2cc524 * Allow tag trigger on grafana/grafana for testing * Use grafana/grafana-build:dev-c71d4b7 * Use grafana/grafana-build:dev-63beac8 * Use grafana/grafana-build:dev-224a0dd * Add environment variables for package publishing * Revert unintentional change to dataquery.cue * Add package publish step to nightly pipeline * Use GCS path for package publish * Pre-evaluate drone workspace on packages path * Use hardcoded drone workspace path * Remove unused env from publish packages step * Use grafana/grafana-build:dev-657ea6a * Use grafana/grafana-build:dev-1a9beec * Use grafana/grafana-build:dev-f0053c8 * Use grafana/grafana-build:main * Use grafana/grafana-build:dev-ae5182f * Use grafana/grafana-build:dev-ec3ec36 * Use grafana/grafana-build:dev-5e160d8 * Use grafana/grafana-build:dev-142d2dc * Use grafana/grafana-build:dev-db6bff1 * Use grafana/grafana-build:main * Change nightly trigger to cron
2 years ago · 55781b486c
parent 1b4c2fc948
commit 55781b486c
3 changed files with 773 additions and 178 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -2767,6 +2767,80 @@ volumes:
    path: /var/run/docker.sock
  name: docker
 ---
+clone:
+  retries: 3
+depends_on:
+- main-test-backend
+- main-test-frontend
+image_pull_secrets:
+- dockerconfigjson
+kind: pipeline
+name: rgm-main-prerelease
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - export GRAFANA_DIR=$$(pwd)
+  - cd /src && ./scripts/drone_publish_main.sh
+  environment:
+    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
+      from_secret: dagger_token
+    CDN_DESTINATION:
+      from_secret: rgm_cdn_destination
+    DESTINATION:
+      from_secret: destination
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOWNLOADS_DESTINATION:
+      from_secret: rgm_downloads_destination
+    GCOM_API_KEY:
+      from_secret: grafana_api_key_dev
+    GCP_KEY_BASE64:
+      from_secret: gcp_key_base64
+    GITHUB_TOKEN:
+      from_secret: github_token
+    GO_VERSION: 1.20.8
+    GPG_PASSPHRASE:
+      from_secret: packages_gpg_passphrase
+    GPG_PRIVATE_KEY:
+      from_secret: packages_gpg_private_key
+    GPG_PUBLIC_KEY:
+      from_secret: packages_gpg_public_key
+    NPM_TOKEN:
+      from_secret: npm_token
+    PACKAGES_DESTINATION: gs://grafana-packages-testing
+    STORYBOOK_DESTINATION:
+      from_secret: rgm_storybook_destination
+  image: grafana/grafana-build:main
+  name: rgm-build
+  pull: always
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+trigger:
+  branch: main
+  event:
+  - push
+  paths:
+    exclude:
+    - '*.md'
+    - docs/**
+    - packages/**/*.md
+    - latest.json
+  repo:
+  - grafana/grafana
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
 clone:
  retries: 3
 depends_on: []
@ -2805,9 +2879,6 @@ trigger:
    - refs/tags/*-cloud*
    include:
    - refs/tags/v*
-  repo:
-    exclude:
-    - grafana/grafana
 type: docker
 volumes:
 - host:
@ -2863,9 +2934,6 @@ trigger:
    - refs/tags/*-cloud*
    include:
    - refs/tags/v*
-  repo:
-    exclude:
-    - grafana/grafana
 type: docker
 volumes:
 - host:
@ -2943,9 +3011,6 @@ trigger:
    - refs/tags/*-cloud*
    include:
    - refs/tags/v*
-  repo:
-    exclude:
-    - grafana/grafana
 type: docker
 volumes:
 - host:
@ -2955,12 +3020,12 @@ volumes:
 clone:
  retries: 3
 depends_on:
- main-test-backend
- main-test-frontend
+- release-test-backend
+- release-test-frontend
 image_pull_secrets:
 - dockerconfigjson
 kind: pipeline
-name: rgm-main-prerelease
+name: rgm-tag-prerelease
 node:
  type: no-parallel
 platform:
@ -2970,12 +3035,22 @@ services: []
 steps:
 - commands:
  - export GRAFANA_DIR=$$(pwd)
-  - cd /src && ./scripts/drone_publish_main.sh
+  - cd /src && ./scripts/drone_publish_tag_grafana.sh
  environment:
    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
      from_secret: dagger_token
+    CDN_DESTINATION:
+      from_secret: rgm_cdn_destination
    DESTINATION:
      from_secret: destination
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOWNLOADS_DESTINATION:
+      from_secret: rgm_downloads_destination
+    GCOM_API_KEY:
+      from_secret: grafana_api_key_dev
    GCP_KEY_BASE64:
      from_secret: gcp_key_base64
    GITHUB_TOKEN:
@ -2987,7 +3062,11 @@ steps:
      from_secret: packages_gpg_private_key
    GPG_PUBLIC_KEY:
      from_secret: packages_gpg_public_key
-  failure: ignore
+    NPM_TOKEN:
+      from_secret: npm_token
+    PACKAGES_DESTINATION: gs://grafana-packages-testing
+    STORYBOOK_DESTINATION:
+      from_secret: rgm_storybook_destination
  image: grafana/grafana-build:main
  name: rgm-build
  pull: always
@ -2995,17 +3074,125 @@ steps:
  - name: docker
    path: /var/run/docker.sock
 trigger:
-  branch: main
  event:
-  - push
-  paths:
    exclude:
-    - '*.md'
-    - docs/**
-    - packages/**/*.md
-    - latest.json
-  repo:
-  - grafana/grafana
+    - promote
+  ref:
+    exclude:
+    - refs/tags/*-cloud*
+    include:
+    - refs/tags/v*
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+clone:
+  retries: 3
+depends_on:
+- rgm-tag-prerelease
+image_pull_secrets:
+- dockerconfigjson
+kind: pipeline
+name: rgm-tag-prerelease-windows
+platform:
+  arch: amd64
+  os: windows
+  version: "1809"
+services: []
+steps:
+- commands:
+  - echo $env:DRONE_RUNNER_NAME
+  failure: ignore
+  image: mcr.microsoft.com/windows:1809
+  name: identify-runner
+- commands:
+  - $$ProgressPreference = "SilentlyContinue"
+  - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/windows/grabpl.exe
+    -OutFile grabpl.exe
+  failure: ignore
+  image: grafana/ci-wix:0.1.1
+  name: windows-init
+- commands:
+  - $$gcpKey = $$env:GCP_KEY
+  - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey))
+    > gcpkey.json'
+  - dos2unix gcpkey.json
+  - gcloud auth activate-service-account --key-file=gcpkey.json
+  - rm gcpkey.json
+  - cp C:\App\nssm-2.24.zip .
+  - .\grabpl.exe windows-installer --target gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/grafana-${DRONE_TAG:1}.windows-amd64.zip
+    --edition oss ${DRONE_TAG}
+  - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]
+  - gsutil cp $$fname gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/
+  - gsutil cp "$$fname.sha256" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/
+  depends_on:
+  - windows-init
+  environment:
+    GCP_KEY:
+      from_secret: gcp_grafanauploads_base64
+    GITHUB_TOKEN:
+      from_secret: github_token
+    PRERELEASE_BUCKET:
+      from_secret: prerelease_bucket
+  failure: ignore
+  image: grafana/ci-wix:0.1.1
+  name: build-windows-installer
+trigger:
+  event:
+    exclude:
+    - promote
+  ref:
+    exclude:
+    - refs/tags/*-cloud*
+    include:
+    - refs/tags/v*
+type: docker
+volumes:
+- host:
+    path: //./pipe/docker_engine/
+  name: docker
+---
+clone:
+  retries: 3
+depends_on:
+- rgm-tag-prerelease
+- rgm-tag-prerelease-windows
+image_pull_secrets:
+- dockerconfigjson
+kind: pipeline
+name: rgm-tag-verify-prerelease-assets
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - apt-get update && apt-get install -yq gettext
+  - printenv GCP_KEY | base64 -d > /tmp/key.json
+  - gcloud auth activate-service-account --key-file=/tmp/key.json
+  - ./scripts/list-release-artifacts.sh ${DRONE_TAG} | xargs -n1 gsutil stat >> /tmp/stat.log
+  - '! cat /tmp/stat.log | grep "No URLs matched"'
+  depends_on:
+  - clone
+  environment:
+    BUCKET: grafana-prerelease
+    GCP_KEY:
+      from_secret: gcp_key_base64
+  image: google/cloud-sdk:431.0.0
+  name: gsutil-stat
+trigger:
+  event:
+    exclude:
+    - promote
+  ref:
+    exclude:
+    - refs/tags/*-cloud*
+    include:
+    - refs/tags/v*
 type: docker
 volumes:
 - host:
@ -3020,7 +3207,7 @@ depends_on:
 image_pull_secrets:
 - dockerconfigjson
 kind: pipeline
-name: rgm-tag-prerelease
+name: rgm-version-branch-prerelease
 node:
  type: no-parallel
 platform:
@ -3034,8 +3221,18 @@ steps:
  environment:
    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
      from_secret: dagger_token
+    CDN_DESTINATION:
+      from_secret: rgm_cdn_destination
    DESTINATION:
      from_secret: destination
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOWNLOADS_DESTINATION:
+      from_secret: rgm_downloads_destination
+    GCOM_API_KEY:
+      from_secret: grafana_api_key_dev
    GCP_KEY_BASE64:
      from_secret: gcp_key_base64
    GITHUB_TOKEN:
@ -3047,6 +3244,11 @@ steps:
      from_secret: packages_gpg_private_key
    GPG_PUBLIC_KEY:
      from_secret: packages_gpg_public_key
+    NPM_TOKEN:
+      from_secret: npm_token
+    PACKAGES_DESTINATION: gs://grafana-packages-testing
+    STORYBOOK_DESTINATION:
+      from_secret: rgm_storybook_destination
  image: grafana/grafana-build:main
  name: rgm-build
  pull: always
@ -3054,17 +3256,8 @@ steps:
  - name: docker
    path: /var/run/docker.sock
 trigger:
-  event:
-    exclude:
-    - promote
  ref:
-    exclude:
-    - refs/tags/*-cloud*
-    include:
-    - refs/tags/v*
-  repo:
-    exclude:
-    - grafana/grafana
+  - refs/heads/v[0-9]*
 type: docker
 volumes:
 - host:
@ -3074,81 +3267,178 @@ volumes:
 clone:
  retries: 3
 depends_on:
- rgm-tag-prerelease
+- rgm-version-branch-prerelease
 image_pull_secrets:
 - dockerconfigjson
 kind: pipeline
-name: rgm-tag-prerelease-windows
+name: rgm-prerelease-verify-prerelease-assets
+node:
+  type: no-parallel
 platform:
  arch: amd64
-  os: windows
-  version: "1809"
+  os: linux
 services: []
 steps:
 - commands:
-  - echo $env:DRONE_RUNNER_NAME
-  failure: ignore
-  image: mcr.microsoft.com/windows:1809
-  name: identify-runner
+  - apt-get update && apt-get install -yq gettext
+  - printenv GCP_KEY | base64 -d > /tmp/key.json
+  - gcloud auth activate-service-account --key-file=/tmp/key.json
+  - ./scripts/list-release-artifacts.sh ${DRONE_TAG} | xargs -n1 gsutil stat >> /tmp/stat.log
+  - '! cat /tmp/stat.log | grep "No URLs matched"'
+  depends_on:
+  - clone
+  environment:
+    BUCKET: grafana-prerelease
+    GCP_KEY:
+      from_secret: gcp_key_base64
+  image: google/cloud-sdk:431.0.0
+  name: gsutil-stat
+trigger:
+  ref:
+  - refs/heads/v[0-9]*
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+clone:
+  retries: 3
+depends_on: []
+environment:
+  EDITION: oss
+image_pull_secrets:
+- dockerconfigjson
+kind: pipeline
+name: nightly-test-frontend
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.18.3
+  name: identify-runner
+- commands:
+  - yarn install --immutable
+  depends_on: []
+  image: node:18.12.0-alpine
+  name: yarn-install
+- commands:
+  - apk add --update git bash
+  - yarn betterer ci
+  depends_on:
+  - yarn-install
+  image: node:18.12.0-alpine
+  name: betterer-frontend
+- commands:
+  - yarn run ci:test-frontend
+  depends_on:
+  - yarn-install
+  environment:
+    TEST_MAX_WORKERS: 50%
+  image: node:18.12.0-alpine
+  name: test-frontend
+trigger:
+  cron:
+    include:
+    - nightly
+  event:
+    include:
+    - cron
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+clone:
+  retries: 3
+depends_on: []
+environment:
+  EDITION: oss
+image_pull_secrets:
+- dockerconfigjson
+kind: pipeline
+name: nightly-test-backend
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.18.3
+  name: identify-runner
+- commands:
+  - '# It is required that code generated from Thema/CUE be committed and in sync
+    with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - apk add --update make
+  - CODEGEN_VERIFY=1 make gen-cue
+  depends_on: []
+  image: golang:1.20.8-alpine
+  name: verify-gen-cue
+- commands:
+  - '# It is required that generated jsonnet is committed and in sync with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - apk add --update make
+  - CODEGEN_VERIFY=1 make gen-jsonnet
+  depends_on: []
+  image: golang:1.20.8-alpine
+  name: verify-gen-jsonnet
+- commands:
+  - apk add --update make
+  - make gen-go
+  depends_on:
+  - verify-gen-cue
+  image: golang:1.20.8-alpine
+  name: wire-install
 - commands:
-  - $$ProgressPreference = "SilentlyContinue"
-  - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/windows/grabpl.exe
-    -OutFile grabpl.exe
-  failure: ignore
-  image: grafana/ci-wix:0.1.1
-  name: windows-init
+  - apk add --update build-base shared-mime-info shared-mime-info-lang
+  - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...
+  depends_on:
+  - wire-install
+  image: golang:1.20.8-alpine
+  name: test-backend
 - commands:
-  - $$gcpKey = $$env:GCP_KEY
-  - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey))
-    > gcpkey.json'
-  - dos2unix gcpkey.json
-  - gcloud auth activate-service-account --key-file=gcpkey.json
-  - rm gcpkey.json
-  - cp C:\App\nssm-2.24.zip .
-  - .\grabpl.exe windows-installer --target gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/grafana-${DRONE_TAG:1}.windows-amd64.zip
-    --edition oss ${DRONE_TAG}
-  - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]
-  - gsutil cp $$fname gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/
-  - gsutil cp "$$fname.sha256" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/
+  - apk add --update build-base
+  - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
-  - windows-init
-  environment:
-    GCP_KEY:
-      from_secret: gcp_grafanauploads_base64
-    GITHUB_TOKEN:
-      from_secret: github_token
-    PRERELEASE_BUCKET:
-      from_secret: prerelease_bucket
-  failure: ignore
-  image: grafana/ci-wix:0.1.1
-  name: build-windows-installer
+  - wire-install
+  image: golang:1.20.8-alpine
+  name: test-backend-integration
 trigger:
+  cron:
+    include:
+    - nightly
  event:
-    exclude:
-    - promote
-  ref:
-    exclude:
-    - refs/tags/*-cloud*
    include:
-    - refs/tags/v*
-  repo:
-    exclude:
-    - grafana/grafana
+    - cron
 type: docker
 volumes:
 - host:
-    path: //./pipe/docker_engine/
+    path: /var/run/docker.sock
  name: docker
 ---
 clone:
  retries: 3
 depends_on:
- release-test-backend
- release-test-frontend
+- nightly-test-backend
+- nightly-test-frontend
 image_pull_secrets:
 - dockerconfigjson
 kind: pipeline
-name: rgm-version-branch-prerelease
+name: rgm-nightly-build
 node:
  type: no-parallel
 platform:
@ -3158,12 +3448,22 @@ services: []
 steps:
 - commands:
  - export GRAFANA_DIR=$$(pwd)
-  - cd /src && ./scripts/drone_publish_tag_grafana.sh
+  - cd /src && ./scripts/drone_build_nightly_grafana.sh
  environment:
    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
      from_secret: dagger_token
+    CDN_DESTINATION:
+      from_secret: rgm_cdn_destination
    DESTINATION:
      from_secret: destination
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOWNLOADS_DESTINATION:
+      from_secret: rgm_downloads_destination
+    GCOM_API_KEY:
+      from_secret: grafana_api_key_dev
    GCP_KEY_BASE64:
      from_secret: gcp_key_base64
    GITHUB_TOKEN:
@ -3175,63 +3475,63 @@ steps:
      from_secret: packages_gpg_private_key
    GPG_PUBLIC_KEY:
      from_secret: packages_gpg_public_key
+    NPM_TOKEN:
+      from_secret: npm_token
+    PACKAGES_DESTINATION: gs://grafana-packages-testing
+    STORYBOOK_DESTINATION:
+      from_secret: rgm_storybook_destination
  image: grafana/grafana-build:main
  name: rgm-build
  pull: always
  volumes:
  - name: docker
    path: /var/run/docker.sock
-trigger:
-  ref:
-  - refs/heads/v[0-9]*
-type: docker
-volumes:
- host:
-    path: /var/run/docker.sock
-  name: docker
---
-clone:
-  retries: 3
-depends_on:
- rgm-tag-prerelease
- rgm-tag-prerelease-windows
-image_pull_secrets:
- dockerconfigjson
-kind: pipeline
-name: rgm-tag-verify-prerelease-assets
-node:
-  type: no-parallel
-platform:
-  arch: amd64
-  os: linux
-services: []
-steps:
 - commands:
-  - apt-get update && apt-get install -yq gettext
-  - printenv GCP_KEY | base64 -d > /tmp/key.json
+  - mkdir -p $${DESTINATION}/$${DRONE_BUILD_EVENT}
+  - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json
  - gcloud auth activate-service-account --key-file=/tmp/key.json
-  - ./scripts/list-release-artifacts.sh ${DRONE_TAG} | xargs -n1 gsutil stat >> /tmp/stat.log
-  - '! cat /tmp/stat.log | grep "No URLs matched"'
+  - gcloud storage cp -r $${DRONE_WORKSPACE}/dist/* $${DESTINATION}/$${DRONE_BUILD_EVENT}
  depends_on:
-  - clone
+  - rgm-build
  environment:
-    BUCKET: grafana-prerelease
-    GCP_KEY:
+    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
+      from_secret: dagger_token
+    CDN_DESTINATION:
+      from_secret: rgm_cdn_destination
+    DESTINATION:
+      from_secret: destination
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOWNLOADS_DESTINATION:
+      from_secret: rgm_downloads_destination
+    GCOM_API_KEY:
+      from_secret: grafana_api_key_dev
+    GCP_KEY_BASE64:
      from_secret: gcp_key_base64
-  image: google/cloud-sdk:431.0.0
-  name: gsutil-stat
+    GITHUB_TOKEN:
+      from_secret: github_token
+    GPG_PASSPHRASE:
+      from_secret: packages_gpg_passphrase
+    GPG_PRIVATE_KEY:
+      from_secret: packages_gpg_private_key
+    GPG_PUBLIC_KEY:
+      from_secret: packages_gpg_public_key
+    NPM_TOKEN:
+      from_secret: npm_token
+    PACKAGES_DESTINATION: gs://grafana-packages-testing
+    STORYBOOK_DESTINATION:
+      from_secret: rgm_storybook_destination
+  image: google/cloud-sdk:alpine
+  name: rgm-copy
 trigger:
+  cron:
+    include:
+    - nightly
  event:
-    exclude:
-    - promote
-  ref:
-    exclude:
-    - refs/tags/*-cloud*
    include:
-    - refs/tags/v*
-  repo:
-    exclude:
-    - grafana/grafana
+    - cron
 type: docker
 volumes:
 - host:
@ -3241,11 +3541,11 @@ volumes:
 clone:
  retries: 3
 depends_on:
- rgm-version-branch-prerelease
+- rgm-nightly-build
 image_pull_secrets:
 - dockerconfigjson
 kind: pipeline
-name: rgm-prerelease-verify-prerelease-assets
+name: rgm-nightly-publish
 node:
  type: no-parallel
 platform:
@ -3254,22 +3554,132 @@ platform:
 services: []
 steps:
 - commands:
-  - apt-get update && apt-get install -yq gettext
-  - printenv GCP_KEY | base64 -d > /tmp/key.json
+  - mkdir -p $${DRONE_WORKSPACE}/dist
+  - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json
  - gcloud auth activate-service-account --key-file=/tmp/key.json
-  - ./scripts/list-release-artifacts.sh ${DRONE_TAG} | xargs -n1 gsutil stat >> /tmp/stat.log
-  - '! cat /tmp/stat.log | grep "No URLs matched"'
+  - gcloud storage cp -r $${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_*
+    $${DRONE_WORKSPACE}/dist
+  environment:
+    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
+      from_secret: dagger_token
+    CDN_DESTINATION:
+      from_secret: rgm_cdn_destination
+    DESTINATION:
+      from_secret: destination
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOWNLOADS_DESTINATION:
+      from_secret: rgm_downloads_destination
+    GCOM_API_KEY:
+      from_secret: grafana_api_key_dev
+    GCP_KEY_BASE64:
+      from_secret: gcp_key_base64
+    GITHUB_TOKEN:
+      from_secret: github_token
+    GPG_PASSPHRASE:
+      from_secret: packages_gpg_passphrase
+    GPG_PRIVATE_KEY:
+      from_secret: packages_gpg_private_key
+    GPG_PUBLIC_KEY:
+      from_secret: packages_gpg_public_key
+    NPM_TOKEN:
+      from_secret: npm_token
+    PACKAGES_DESTINATION: gs://grafana-packages-testing
+    STORYBOOK_DESTINATION:
+      from_secret: rgm_storybook_destination
+  image: google/cloud-sdk:alpine
+  name: rgm-copy
+- commands:
+  - export GRAFANA_DIR=$$(pwd)
+  - cd /src && ./scripts/drone_publish_nightly_grafana.sh
  depends_on:
-  - clone
+  - rgm-copy
  environment:
-    BUCKET: grafana-prerelease
-    GCP_KEY:
+    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
+      from_secret: dagger_token
+    CDN_DESTINATION:
+      from_secret: rgm_cdn_destination
+    DESTINATION:
+      from_secret: destination
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOWNLOADS_DESTINATION:
+      from_secret: rgm_downloads_destination
+    GCOM_API_KEY:
+      from_secret: grafana_api_key_dev
+    GCP_KEY_BASE64:
      from_secret: gcp_key_base64
-  image: google/cloud-sdk:431.0.0
-  name: gsutil-stat
+    GITHUB_TOKEN:
+      from_secret: github_token
+    GO_VERSION: 1.20.8
+    GPG_PASSPHRASE:
+      from_secret: packages_gpg_passphrase
+    GPG_PRIVATE_KEY:
+      from_secret: packages_gpg_private_key
+    GPG_PUBLIC_KEY:
+      from_secret: packages_gpg_public_key
+    NPM_TOKEN:
+      from_secret: npm_token
+    PACKAGES_DESTINATION: gs://grafana-packages-testing
+    STORYBOOK_DESTINATION:
+      from_secret: rgm_storybook_destination
+  image: grafana/grafana-build:main
+  name: rgm-publish
+  pull: always
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+- depends_on:
+  - rgm-publish
+  image: us.gcr.io/kubernetes-dev/package-publish:latest
+  name: publish-deb
+  privileged: true
+  settings:
+    access_key_id:
+      from_secret: packages_access_key_id
+    gpg_passphrase:
+      from_secret: packages_gpg_passphrase
+    gpg_private_key:
+      from_secret: packages_gpg_private_key
+    gpg_public_key:
+      from_secret: packages_gpg_public_key
+    package_path: file:///drone/src/dist/*.deb
+    secret_access_key:
+      from_secret: packages_secret_access_key
+    service_account_json:
+      from_secret: packages_service_account
+    target_bucket: grafana-packages-testing
+- depends_on:
+  - rgm-publish
+  image: us.gcr.io/kubernetes-dev/package-publish:latest
+  name: publish-rpm
+  privileged: true
+  settings:
+    access_key_id:
+      from_secret: packages_access_key_id
+    gpg_passphrase:
+      from_secret: packages_gpg_passphrase
+    gpg_private_key:
+      from_secret: packages_gpg_private_key
+    gpg_public_key:
+      from_secret: packages_gpg_public_key
+    package_path: file:///drone/src/dist/*.rpm
+    secret_access_key:
+      from_secret: packages_secret_access_key
+    service_account_json:
+      from_secret: packages_service_account
+    target_bucket: grafana-packages-testing
 trigger:
-  ref:
-  - refs/heads/v[0-9]*
+  cron:
+    include:
+    - nightly
+  event:
+    include:
+    - cron
 type: docker
 volumes:
 - host:
@ -4059,6 +4469,12 @@ get:
 kind: secret
 name: grafana_api_key
 ---
+get:
+  name: api_key_dev
+  path: infra/data/ci/grafana-release-eng/grafanacom
+kind: secret
+name: grafana_api_key_dev
+---
 get:
  name: .dockerconfigjson
  path: secret/data/common/gcr
@ -4185,6 +4601,24 @@ get:
 kind: secret
 name: destination
 ---
+get:
+  name: storybook_destination
+  path: infra/data/ci/grafana-release-eng/rgm
+kind: secret
+name: rgm_storybook_destination
+---
+get:
+  name: cdn_destination
+  path: infra/data/ci/grafana-release-eng/rgm
+kind: secret
+name: rgm_cdn_destination
+---
+get:
+  name: downloads_destination
+  path: infra/data/ci/grafana-release-eng/rgm
+kind: secret
+name: rgm_downloads_destination
+---
 get:
  name: dagger_token
  path: infra/data/ci/grafana-release-eng/rgm
@ -4216,6 +4650,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: 852af171d897f0a2cc0b03375fa8dfeacc65c2df7113c5efd0e21b03195dd7af
+hmac: 07868df8d2431c82e8c46fad453e7fbb0edbe0a4b0796158a80387b4595418d0

 ...
--- a/scripts/drone/rgm.star
+++ b/scripts/drone/rgm.star
@ -24,10 +24,15 @@ load(
    "scripts/drone/steps/lib_windows.star",
    "get_windows_steps",
 )
+load(
+    "scripts/drone/utils/images.star",
+    "images",
+)
 load(
    "scripts/drone/utils/utils.star",
    "ignore_failure",
    "pipeline",
+    "with_deps",
 )
 load(
    "scripts/drone/variables.star",
@ -36,29 +41,16 @@ load(
 load(
    "scripts/drone/vault.star",
    "from_secret",
+    "npm_token",
+    "rgm_cdn_destination",
    "rgm_dagger_token",
    "rgm_destination",
+    "rgm_downloads_destination",
    "rgm_gcp_key_base64",
    "rgm_github_token",
+    "rgm_storybook_destination",
 )

-def rgm_env_secrets(env):
-    """Adds the rgm secret ENV variables to the given env arg
-
-    Args:
-      env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them).
-    Returns:
-        Drone step.
-    """
-    env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64)
-    env["DESTINATION"] = from_secret(rgm_destination)
-    env["GITHUB_TOKEN"] = from_secret(rgm_github_token)
-    env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token)
-    env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key")
-    env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key")
-    env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase")
-    return env
-
 docs_paths = {
    "exclude": [
        "*.md",
@ -69,11 +61,6 @@ docs_paths = {
 }

 tag_trigger = {
-    "repo": {
-        "exclude": [
-            "grafana/grafana",
-        ],
-    },
    "event": {
        "exclude": [
            "promote",
@ -89,22 +76,61 @@ tag_trigger = {
    },
 }

+nightly_trigger = {
+    "event": {
+        "include": [
+            "cron",
+        ],
+    },
+    "cron": {
+        "include": [
+            "nightly",
+        ],
+    },
+}
+
 version_branch_trigger = {"ref": ["refs/heads/v[0-9]*"]}

-def rgm_build(script = "drone_publish_main.sh", canFail = True):
+def rgm_env_secrets(env):
+    """Adds the rgm secret ENV variables to the given env arg
+
+    Args:
+      env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them).
+    Returns:
+        Drone step.
+    """
+    env["DESTINATION"] = from_secret(rgm_destination)
+    env["STORYBOOK_DESTINATION"] = from_secret(rgm_storybook_destination)
+    env["CDN_DESTINATION"] = from_secret(rgm_cdn_destination)
+    env["DOWNLOADS_DESTINATION"] = from_secret(rgm_downloads_destination)
+    env["PACKAGES_DESTINATION"] = "gs://grafana-packages-testing"
+
+    env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64)
+    env["GITHUB_TOKEN"] = from_secret(rgm_github_token)
+    env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token)
+    env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key")
+    env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key")
+    env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase")
+    env["DOCKER_USERNAME"] = from_secret("docker_username")
+    env["DOCKER_PASSWORD"] = from_secret("docker_password")
+    env["NPM_TOKEN"] = from_secret(npm_token)
+    env["GCOM_API_KEY"] = from_secret("grafana_api_key_dev")
+    return env
+
+def rgm_run(name, script):
    """Returns a pipeline that does a full build & package of Grafana.

    Args:
+      name: The name of the pipeline step.
      script: The script in the container to run.
-      canFail: if true, then this pipeline can fail while the entire build will still succeed.
    Returns:
        Drone step.
    """
    env = {
        "GO_VERSION": golang_version,
    }
-    rgm_build_step = {
-        "name": "rgm-build",
+    rgm_run_step = {
+        "name": name,
        "image": "grafana/grafana-build:main",
        "pull": "always",
        "commands": [
@ -117,14 +143,72 @@ def rgm_build(script = "drone_publish_main.sh", canFail = True):
        "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
    }

-    if canFail:
-        rgm_build_step["failure"] = "ignore"
+    return [
+        rgm_run_step,
+    ]
+
+def rgm_copy(src, dst):
+    """Copies file from/to GCS.
+
+    Args:
+      src: source of the files.
+      dst: destination of the files.
+
+    Returns:
+      Drone steps.
+    """
+    commands = [
+        "printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json",
+        "gcloud auth activate-service-account --key-file=/tmp/key.json",
+        "gcloud storage cp -r {} {}".format(src, dst),
+    ]
+
+    if not dst.startswith("gs://"):
+        commands.insert(0, "mkdir -p {}".format(dst))
+
+    rgm_copy_step = {
+        "name": "rgm-copy",
+        "image": "google/cloud-sdk:alpine",
+        "commands": commands,
+        "environment": rgm_env_secrets({}),
+    }

    return [
-        rgm_build_step,
+        rgm_copy_step,
    ]

+def rgm_publish_packages(bucket = "grafana-packages"):
+    """Publish deb and rpm packages.
+
+    Args:
+      bucket: target bucket to publish the packages.
+
+    Returns:
+      Drone steps.
+    """
+    steps = []
+    for package_manager in ["deb", "rpm"]:
+        steps.append({
+            "name": "publish-{}".format(package_manager),
+            # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image
+            "image": images["package_publish"],
+            "privileged": True,
+            "settings": {
+                "access_key_id": from_secret("packages_access_key_id"),
+                "secret_access_key": from_secret("packages_secret_access_key"),
+                "service_account_json": from_secret("packages_service_account"),
+                "target_bucket": bucket,
+                "gpg_passphrase": from_secret("packages_gpg_passphrase"),
+                "gpg_public_key": from_secret("packages_gpg_public_key"),
+                "gpg_private_key": from_secret("packages_gpg_private_key"),
+                "package_path": "file:///drone/src/dist/*.{}".format(package_manager),
+            },
+        })
+
+    return steps
+
 def rgm_main():
+    # Runs a package / build process (with some distros) when commits are merged to main
    trigger = {
        "event": [
            "push",
@ -139,15 +223,16 @@ def rgm_main():
    return pipeline(
        name = "rgm-main-prerelease",
        trigger = trigger,
-        steps = rgm_build(canFail = True),
+        steps = rgm_run("rgm-build", "drone_publish_main.sh"),
        depends_on = ["main-test-backend", "main-test-frontend"],
    )

 def rgm_tag():
+    # Runs a package / build process (with all distros) when a tag is made
    return pipeline(
        name = "rgm-tag-prerelease",
        trigger = tag_trigger,
-        steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False),
+        steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"),
        depends_on = ["release-test-backend", "release-test-frontend"],
    )

@ -166,22 +251,61 @@ def rgm_tag_windows():
    )

 def rgm_version_branch():
+    # Runs a package / build proces (with all distros) when a commit lands on a version branch
    return pipeline(
        name = "rgm-version-branch-prerelease",
        trigger = version_branch_trigger,
-        steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False),
+        steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"),
        depends_on = ["release-test-backend", "release-test-frontend"],
    )

-def rgm():
+def rgm_nightly_build():
+    src = "$${DRONE_WORKSPACE}/dist/*"
+    dst = "$${DESTINATION}/$${DRONE_BUILD_EVENT}"
+
+    copy_steps = with_deps(rgm_copy(src, dst), ["rgm-build"])
+
+    return pipeline(
+        name = "rgm-nightly-build",
+        trigger = nightly_trigger,
+        steps = rgm_run("rgm-build", "drone_build_nightly_grafana.sh") + copy_steps,
+        depends_on = ["nightly-test-backend", "nightly-test-frontend"],
+    )
+
+def rgm_nightly_publish():
+    """Nightly publish pipeline.
+
+    Returns:
+      Drone pipeline.
+    """
+    src = "$${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_*"
+    dst = "$${DRONE_WORKSPACE}/dist"
+
+    publish_steps = with_deps(rgm_run("rgm-publish", "drone_publish_nightly_grafana.sh"), ["rgm-copy"])
+    package_steps = with_deps(rgm_publish_packages("grafana-packages-testing"), ["rgm-publish"])
+
+    return pipeline(
+        name = "rgm-nightly-publish",
+        trigger = nightly_trigger,
+        steps = rgm_copy(src, dst) + publish_steps + package_steps,
+        depends_on = ["rgm-nightly-build"],
+    )
+
+def rgm_nightly_pipeline():
+    return [
+        test_frontend(nightly_trigger, "nightly"),
+        test_backend(nightly_trigger, "nightly"),
+        rgm_nightly_build(),
+        rgm_nightly_publish(),
+    ]
+
+def rgm_tag_pipeline():
    return [
        whats_new_checker_pipeline(tag_trigger),
        test_frontend(tag_trigger, "release"),
        test_backend(tag_trigger, "release"),
-        rgm_main(),  # Runs a package / build process (with some distros) when commits are merged to main
-        rgm_tag(),  # Runs a package / build process (with all distros) when a tag is made
+        rgm_tag(),
        rgm_tag_windows(),
-        rgm_version_branch(),  # Runs a package / build proces (with all distros) when a commit lands on a version branch
        verify_release_pipeline(
            trigger = tag_trigger,
            name = "rgm-tag-verify-prerelease-assets",
@ -191,6 +315,11 @@ def rgm():
                "rgm-tag-prerelease-windows",
            ],
        ),
+    ]
+
+def rgm_version_branch_pipeline():
+    return [
+        rgm_version_branch(),
        verify_release_pipeline(
            trigger = version_branch_trigger,
            name = "rgm-prerelease-verify-prerelease-assets",
@ -200,3 +329,16 @@ def rgm():
            ],
        ),
    ]
+
+def rgm_main_pipeline():
+    return [
+        rgm_main(),
+    ]
+
+def rgm():
+    return (
+        rgm_main_pipeline() +
+        rgm_tag_pipeline() +
+        rgm_version_branch_pipeline() +
+        rgm_nightly_pipeline()
+    )
--- a/scripts/drone/vault.star
+++ b/scripts/drone/vault.star
@ -14,6 +14,9 @@ azure_tenant = "azure_tenant"

 rgm_gcp_key_base64 = "gcp_key_base64"
 rgm_destination = "destination"
+rgm_storybook_destination = "rgm_storybook_destination"
+rgm_cdn_destination = "rgm_cdn_destination"
+rgm_downloads_destination = "rgm_downloads_destination"
 rgm_github_token = "github_token"
 rgm_dagger_token = "dagger_token"

@ -40,6 +43,7 @@ def secrets():
        vault_secret(gcp_grafanauploads, "infra/data/ci/grafana-release-eng/grafanauploads", "credentials.json"),
        vault_secret(gcp_grafanauploads_base64, "infra/data/ci/grafana-release-eng/grafanauploads", "credentials_base64"),
        vault_secret("grafana_api_key", "infra/data/ci/grafana-release-eng/grafanacom", "api_key"),
+        vault_secret("grafana_api_key_dev", "infra/data/ci/grafana-release-eng/grafanacom", "api_key_dev"),
        vault_secret(pull_secret, "secret/data/common/gcr", ".dockerconfigjson"),
        vault_secret("github_token", "infra/data/ci/github/grafanabot", "pat"),
        vault_secret(drone_token, "infra/data/ci/drone", "machine-user-token"),
@ -122,6 +126,21 @@ def secrets():
            "infra/data/ci/grafana-release-eng/rgm",
            "destination_prod",
        ),
+        vault_secret(
+            rgm_storybook_destination,
+            "infra/data/ci/grafana-release-eng/rgm",
+            "storybook_destination",
+        ),
+        vault_secret(
+            rgm_cdn_destination,
+            "infra/data/ci/grafana-release-eng/rgm",
+            "cdn_destination",
+        ),
+        vault_secret(
+            rgm_downloads_destination,
+            "infra/data/ci/grafana-release-eng/rgm",
+            "downloads_destination",
+        ),
        vault_secret(
            rgm_dagger_token,
            "infra/data/ci/grafana-release-eng/rgm",