diff --git a/.drone.star b/.drone.star index ce31860e844..a14146a7c14 100644 --- a/.drone.star +++ b/.drone.star @@ -7,8 +7,9 @@ This module returns a Drone configuration including pipelines and secrets. """ -load("scripts/drone/events/pr.star", "pr_pipelines") +load("scripts/drone/events/cron.star", "cronjobs") load("scripts/drone/events/main.star", "main_pipelines") +load("scripts/drone/events/pr.star", "pr_pipelines") load( "scripts/drone/events/release.star", "integration_test_pipelines", @@ -17,24 +18,22 @@ load( "publish_packages_pipeline", ) load( - "scripts/drone/rgm.star", - "rgm", + "scripts/drone/pipelines/ci_images.star", + "publish_ci_build_container_image_pipeline", + "publish_ci_windows_test_image_pipeline", ) load( "scripts/drone/pipelines/publish_images.star", "publish_image_pipelines_public", ) -load( - "scripts/drone/pipelines/ci_images.star", - "publish_ci_build_container_image_pipeline", - "publish_ci_windows_test_image_pipeline", -) load( "scripts/drone/pipelines/windows.star", "windows_test_backend", ) -load("scripts/drone/version.star", "version_branch_pipelines") -load("scripts/drone/events/cron.star", "cronjobs") +load( + "scripts/drone/rgm.star", + "rgm", +) load("scripts/drone/vault.star", "secrets") def main(_ctx): @@ -50,7 +49,6 @@ def main(_ctx): "event": ["promote"], "target": ["test-windows"], }, "oss", "testing")] + - version_branch_pipelines() + integration_test_pipelines() + publish_ci_windows_test_image_pipeline() + publish_ci_build_container_image_pipeline() + diff --git a/.drone.yml b/.drone.yml index 59f35d00be3..fa04d70a982 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,7 +24,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - ./bin/build verify-drone @@ -74,13 +74,14 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - - ./bin/build verify-starlark . + - go install github.com/bazelbuild/buildtools/buildifier@latest + - buildifier --lint=warn -mode=check -r . depends_on: - compile-build-cmd - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: lint-starlark trigger: event: @@ -118,24 +119,20 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.17.1 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -153,7 +150,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - yarn run ci:test-frontend @@ -161,7 +158,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: event: @@ -198,6 +195,7 @@ platform: services: [] steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -215,7 +213,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME @@ -224,7 +222,7 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - yarn run prettier:check @@ -234,9 +232,10 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-frontend - commands: + - apk add --update git - |- yarn run i18n:extract || (echo " Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false) @@ -248,7 +247,7 @@ steps: - yarn run i18n:compile depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: verify-i18n trigger: event: @@ -285,6 +284,7 @@ platform: services: [] steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -302,55 +302,53 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.17.1 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.6 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: test-backend-integration trigger: event: @@ -399,9 +397,10 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -419,25 +418,26 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: + - apk add --update make - make gen-go depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: - - apt-get update && apt-get install make + - apk add --update make build-base - make lint-go depends_on: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.6 + image: golang:1.21.1-alpine name: lint-backend - commands: - go run scripts/modowners/modowners.go check go.mod - image: golang:1.20.6 + image: golang:1.21.1-alpine name: validate-modfile trigger: event: @@ -493,35 +493,38 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" @@ -544,7 +547,7 @@ steps: from_secret: github_token TEST_TAG: v0.0.0-test failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: trigger-test-release when: branch: main @@ -568,63 +571,39 @@ steps: token: from_secret: drone_token - commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} + - apk add --update jq bash + - yarn packages:build - yarn packages:pack - ./scripts/validate-npm-packages.sh depends_on: - - compile-build-cmd - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-frontend-packages - commands: - - ./bin/build build-plugins --jobs 8 --edition oss + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --yarn-cache=$$YARN_CACHE_FOLDER + --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD > packages.txt depends_on: - - compile-build-cmd - yarn-install - environment: null - image: grafana/build-container:1.7.5 - name: build-plugins -- commands: - - . scripts/build/gpg-test-vars.sh && ./bin/build package --jobs 8 --edition oss - --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: null - image: grafana/build-container:1.7.5 - name: package + image: grafana/grafana-build:main + name: rgm-package + volumes: + - name: docker + path: /var/run/docker.sock - commands: - - ./scripts/grafana-server/start-server + - apk add --update tar bash + - mkdir grafana + - tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana + - cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages + - rgm-package detach: true environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 + GF_APP_MODE: development + GF_SERVER_HTTP_PORT: "3001" + GF_SERVER_ROUTER_LOGGING: "1" + image: alpine:3.17.1 name: grafana-server - commands: - ./bin/build e2e-tests --port 3001 --suite dashboards-suite @@ -719,23 +698,16 @@ steps: - yarn storybook:build - ./bin/build verify-storybook depends_on: - - build-frontend + - rgm-package - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-storybook when: paths: include: - packages/grafana-ui/** -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci-pr.conf.js @@ -750,36 +722,22 @@ steps: image: grafana/docker-puppeteer:1.1.0 name: test-a11y-frontend - commands: - - ./bin/build build-docker --edition oss -archs amd64 - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --ubuntu -archs amd64 + - /src/grafana-build docker --package=$(cat packages.txt | grep tar.gz | grep -v + docker | grep -v sha256) --ubuntu-base=ubuntu:22.10 --alpine-base=alpine:3.17.1 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu + - rgm-package + image: grafana/grafana-build:main + name: rgm-build-docker volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -868,6 +826,7 @@ services: name: memcached steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -885,7 +844,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - mkdir -p bin @@ -898,7 +857,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -909,28 +868,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -938,16 +903,20 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache @@ -956,15 +925,19 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: mysql-5.7-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root -prootpass - go clean -testcache @@ -973,30 +946,41 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-8.0 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: mysql-8.0-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: memcached-integration-tests trigger: event: @@ -1051,11 +1035,12 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - pip3 install codespell - codespell -I .codespellignore docs/ - image: grafana/build-container:1.7.5 + image: python:3.8 name: codespell - commands: - yarn run prettier:checkDocs @@ -1063,7 +1048,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-docs - commands: - mkdir -p /hugo/content/docs/grafana/latest @@ -1076,9 +1061,10 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue trigger: event: @@ -1118,13 +1104,11 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - - ./bin/build shellcheck - depends_on: - - compile-build-cmd - image: grafana/build-container:1.7.5 + - shellcheck -e SC1071 -e SC2162 scripts/**/*.sh + image: koalaman/shellcheck:stable name: shellcheck trigger: event: @@ -1199,6 +1183,7 @@ services: name: memcached steps: - commands: + - apk add --update curl jq bash - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ../grafana-enterprise - cd ../grafana-enterprise @@ -1213,48 +1198,53 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: - clone-enterprise - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: - clone-enterprise - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: + - apk add --update build-base - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: sqlite-benchmark-integration-tests - commands: + - apk add --update build-base - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: @@ -1263,9 +1253,10 @@ steps: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: postgres-benchmark-integration-tests - commands: + - apk add --update build-base - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: @@ -1273,9 +1264,10 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: mysql-5.7-benchmark-integration-tests - commands: + - apk add --update build-base - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: @@ -1283,7 +1275,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: mysql-8.0-benchmark-integration-tests trigger: event: @@ -1328,11 +1320,12 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - pip3 install codespell - codespell -I .codespellignore docs/ - image: grafana/build-container:1.7.5 + image: python:3.8 name: codespell - commands: - yarn run prettier:checkDocs @@ -1340,7 +1333,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-docs - commands: - mkdir -p /hugo/content/docs/grafana/latest @@ -1353,9 +1346,10 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue trigger: branch: main @@ -1395,22 +1389,17 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.17.1 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -1418,7 +1407,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: branch: main @@ -1460,7 +1449,7 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - yarn run prettier:check @@ -1470,9 +1459,10 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-frontend - commands: + - apk add --update git - |- yarn run i18n:extract || (echo " Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false) @@ -1484,7 +1474,7 @@ steps: - yarn run i18n:compile depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: verify-i18n trigger: branch: main @@ -1523,49 +1513,47 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.17.1 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.6 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: test-backend-integration trigger: branch: main @@ -1609,25 +1597,26 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: + - apk add --update make - make gen-go depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: - - apt-get update && apt-get install make + - apk add --update make build-base - make lint-go depends_on: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.6 + image: golang:1.21.1-alpine name: lint-backend - commands: - go run scripts/modowners/modowners.go check go.mod - image: golang:1.20.6 + image: golang:1.21.1-alpine name: validate-modfile - commands: - ./bin/build verify-drone @@ -1683,103 +1672,73 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} + - apk add --update jq bash + - yarn packages:build - yarn packages:pack - ./scripts/validate-npm-packages.sh depends_on: - - compile-build-cmd - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-frontend-packages - commands: - - ./bin/build build-plugins --jobs 8 --edition oss + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --yarn-cache=$$YARN_CACHE_FOLDER + --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD > packages.txt depends_on: - - compile-build-cmd - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: build-plugins -- commands: - - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: package + image: grafana/grafana-build:main + name: rgm-package + volumes: + - name: docker + path: /var/run/docker.sock - commands: - - ./scripts/grafana-server/start-server + - apk add --update tar bash + - mkdir grafana + - tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana + - cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages + - rgm-package detach: true environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 + GF_APP_MODE: development + GF_SERVER_HTTP_PORT: "3001" + GF_SERVER_ROUTER_LOGGING: "1" + image: alpine:3.17.1 name: grafana-server - commands: - ./bin/build e2e-tests --port 3001 --suite dashboards-suite @@ -1874,23 +1833,16 @@ steps: - yarn storybook:build - ./bin/build verify-storybook depends_on: - - build-frontend + - rgm-package - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-storybook when: paths: include: - packages/grafana-ui/** -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json @@ -1933,42 +1885,28 @@ steps: GRAFANA_MISC_STATS_API_KEY: from_secret: grafana_misc_stats_api_key failure: ignore - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: publish-frontend-metrics when: repo: - grafana/grafana - commands: - - ./bin/build build-docker --edition oss - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --ubuntu + - /src/grafana-build docker --package=$(cat packages.txt | grep tar.gz | grep -v + docker | grep -v sha256) --ubuntu-base=ubuntu:22.10 --alpine-base=alpine:3.17.1 + --tag-format='{{ .version }}-{{ .arch }}' --ubuntu-tag-format='{{ .version }}-ubuntu-{{ + .arch }}' > docker.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu + - rgm-package + image: grafana/grafana-build:main + name: rgm-build-docker volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -1993,8 +1931,7 @@ steps: - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -2023,10 +1960,11 @@ steps: - end-to-end-tests-panels-suite - end-to-end-tests-smoke-tests-suite - end-to-end-tests-various-suite + - build-frontend-packages environment: NPM_TOKEN: from_secret: npm_token - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: release-canary-npm-packages when: paths: @@ -2149,7 +2087,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -2160,28 +2098,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -2189,16 +2133,20 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache @@ -2207,15 +2155,19 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: mysql-5.7-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root -prootpass - go clean -testcache @@ -2224,30 +2176,41 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-8.0 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: mysql-8.0-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: memcached-integration-tests trigger: branch: main @@ -2304,29 +2267,6 @@ steps: -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp "$$fname" gs://grafana-downloads/oss/main/ - - gsutil cp "$$fname.sha256" gs://grafana-downloads/oss/main/ - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer trigger: branch: main event: @@ -2492,7 +2432,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - ./bin/build artifacts docker fetch --edition oss @@ -2588,7 +2528,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} @@ -2657,12 +2597,12 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - ./bin/build artifacts npm retrieve --tag ${DRONE_TAG} @@ -2686,7 +2626,7 @@ steps: NPM_TOKEN: from_secret: npm_token failure: ignore - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: release-npm-packages trigger: event: @@ -2722,7 +2662,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - depends_on: - compile-build-cmd @@ -2810,13 +2750,13 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - ./bin/build whatsnew-checker depends_on: - compile-build-cmd - image: golang:1.20.6 + image: golang:1.21.1-alpine name: whats-new-checker trigger: event: @@ -2853,22 +2793,17 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.17.1 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -2876,7 +2811,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: event: @@ -2913,49 +2848,47 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.17.1 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.6 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: test-backend-integration trigger: event: @@ -3000,6 +2933,7 @@ steps: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token + GO_VERSION: 1.21.1 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: @@ -3058,6 +2992,7 @@ steps: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token + GO_VERSION: 1.21.1 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: @@ -3121,7 +3056,7 @@ steps: - .\grabpl.exe windows-installer --target gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/grafana-${DRONE_TAG:1}.windows-amd64.zip --edition oss ${DRONE_TAG} - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp "$$fname" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ + - gsutil cp $$fname gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ - gsutil cp "$$fname.sha256" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ depends_on: - windows-init @@ -3153,6 +3088,55 @@ volumes: clone: retries: 3 depends_on: +- release-test-backend +- release-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-version-branch-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_tag_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + DESTINATION: + from_secret: destination + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.21.1 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + image: grafana/grafana-build:main + name: rgm-build + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: - rgm-tag-prerelease - rgm-tag-prerelease-windows image_pull_secrets: @@ -3195,6 +3179,44 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- rgm-version-branch-prerelease +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-prerelease-verify-prerelease-assets +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - apt-get update && apt-get install -yq gettext + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - ./scripts/list-release-artifacts.sh ${DRONE_TAG} | xargs -n1 gsutil stat >> /tmp/stat.log + - '! cat /tmp/stat.log | grep "No URLs matched"' + depends_on: + - clone + environment: + BUCKET: grafana-prerelease + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: gsutil-stat +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: disable: true depends_on: [] @@ -3221,20 +3243,20 @@ steps: - commands: [] depends_on: - clone - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.21.1-windowsservercore-1809 name: windows-init - commands: - go install github.com/google/wire/cmd/wire@v0.5.0 - wire gen -tags oss ./pkg/server depends_on: - windows-init - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.21.1-windowsservercore-1809 name: wire-install - commands: - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.21.1-windowsservercore-1809 name: test-backend trigger: event: @@ -3255,414 +3277,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-build-e2e-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - yarn install --immutable - depends_on: [] - image: grafana/build-container:1.7.5 - name: yarn-install -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.6 - name: compile-build-cmd -- commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - - yarn packages:pack - - ./scripts/validate-npm-packages.sh - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition oss - depends_on: - - compile-build-cmd - - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: build-plugins -- commands: - - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: package -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker -- commands: - - ./bin/build build-docker --edition oss --shouldSave - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - detach: true - environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 - name: grafana-server -- commands: - - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:12.15.0 - name: end-to-end-tests-dashboards-suite -- commands: - - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:12.15.0 - name: end-to-end-tests-smoke-tests-suite -- commands: - - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:12.15.0 - name: end-to-end-tests-panels-suite -- commands: - - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:12.15.0 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - failure: ignore - image: google/cloud-sdk:431.0.0 - name: e2e-tests-artifacts-upload - when: - status: - - success - - failure -- commands: - - yarn storybook:build - - ./bin/build verify-storybook - depends_on: - - build-frontend - - build-frontend-packages - environment: - NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 - name: build-storybook - when: - paths: - include: - - packages/grafana-ui/** -- commands: - - ./bin/build upload-cdn --edition oss - depends_on: - - grafana-server - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets - when: - repo: - - grafana/grafana -- commands: - - ./bin/build upload-packages --edition oss - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages - when: - repo: - - grafana/grafana -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql57 - temp: - medium: memory -- name: mysql80 - temp: - medium: memory ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-test-frontend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - yarn install --immutable - depends_on: [] - image: grafana/build-container:1.7.5 - name: yarn-install -- commands: - - yarn betterer ci - depends_on: - - yarn-install - image: grafana/build-container:1.7.5 - name: betterer-frontend -- commands: - - yarn run ci:test-frontend - depends_on: - - yarn-install - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 - name: test-frontend -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-test-backend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.6 - name: compile-build-cmd -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.5 - name: test-backend -- commands: - - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - image: grafana/build-container:1.7.5 - name: test-backend-integration -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-integration-tests +name: integration-tests node: type: no-parallel platform: @@ -3725,261 +3340,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql57:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root - -prootpass - - go clean -testcache - - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 - name: mysql-5.7-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql80:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root - -prootpass - - go clean -testcache - - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 - name: mysql-8.0-integration-tests -- commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s - - go clean -testcache - - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... - depends_on: - - wire-install - environment: - REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 - name: redis-integration-tests -- commands: - - dockerize -wait tcp://memcached:11211 -timeout 120s - - go clean -testcache - - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... - depends_on: - - wire-install - environment: - MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 - name: memcached-integration-tests -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql57 - temp: - medium: memory -- name: mysql80 - temp: - medium: memory ---- -clone: - retries: 3 -depends_on: -- release-branch-build-e2e-publish -- release-branch-test-frontend -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres - commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: integration-tests -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- commands: - - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci - environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.7.39 - name: mysql57 - volumes: - - name: mysql57 - path: /var/lib/mysql -- commands: - - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password - environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:8.0.32 - name: mysql80 - volumes: - - name: mysql80 - path: /var/lib/mysql -- environment: {} - image: redis:6.2.11-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -3987,16 +3375,20 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache @@ -4005,15 +3397,19 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: mysql-5.7-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root -prootpass - go clean -testcache @@ -4022,30 +3418,41 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-8.0 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: mysql-8.0-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.21.1-alpine name: memcached-integration-tests trigger: event: @@ -4450,13 +3857,16 @@ steps: - name: config path: /root/.docker/ - commands: + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine/git:2.40.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.21.1-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM node:18.12.0-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/build-container:1.7.5 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.17.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ubuntu:22.10 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.6 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM python:3.8 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32 @@ -4468,6 +3878,8 @@ steps: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:dbd975af06 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:12.15.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM jwilder/dockerize:0.6.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM koalaman/shellcheck:stable depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -4478,13 +3890,16 @@ steps: - name: config path: /root/.docker/ - commands: + - trivy --exit-code 1 --severity HIGH,CRITICAL alpine/git:2.40.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.21.1-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL node:18.12.0-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0 - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/build-container:1.7.5 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3 - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.17.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL ubuntu:22.10 - trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8 - - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.6 - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack + - trivy --exit-code 1 --severity HIGH,CRITICAL python:3.8 - trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39 - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32 @@ -4496,6 +3911,8 @@ steps: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:dbd975af06 - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:12.15.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL jwilder/dockerize:0.6.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL koalaman/shellcheck:stable depends_on: - authenticate-gcr environment: @@ -4541,7 +3958,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.6 + image: golang:1.21.1-alpine name: compile-build-cmd - commands: - ./bin/build publish grafana-com --edition oss @@ -4746,6 +4163,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 43be5c3799e498e245d555510ce5f2cc9664a857e77fe0995792feb949b8cf62 +hmac: 5ccdc60b94e3d992613fc1e978057a285d1858ea7db4f9daf87213f91212f1f3 ... diff --git a/Makefile b/Makefile index a6b123eefe6..b6fc8ddff2e 100644 --- a/Makefile +++ b/Makefile @@ -291,7 +291,7 @@ scripts/drone/TAGS: $(shell find scripts/drone -name '*.star') etags --lang none --regex="/def \(\w+\)[^:]+:/\1/" --regex="/\s*\(\w+\) =/\1/" $^ -o $@ format-drone: - buildifier -r scripts/drone + buildifier --lint=fix -r scripts/drone help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/scripts/drone/TAGS b/scripts/drone/TAGS deleted file mode 100644 index 1c9f9da89a5..00000000000 --- a/scripts/drone/TAGS +++ /dev/null @@ -1,628 +0,0 @@ - -events/release.star,6652 -ver_mode =ver_mode64,1602 -release_trigger =release_trigger65,1623 -def store_npm_packages_step():store_npm_packages_step74,1752 -def retrieve_npm_packages_step():retrieve_npm_packages_step90,2193 -def release_npm_packages_step():release_npm_packages_step107,2663 -def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):oss_pipelines123,3076 - environment =environment135,3492 - edition =edition136,3529 - services =services137,3549 - volumes =volumes138,3609 - package_steps =package_steps139,3659 - publish_steps =publish_steps140,3682 - should_publish =should_publish141,3705 - should_upload =should_upload142,3748 - init_steps =init_steps143,3818 - build_steps =build_steps152,4033 - integration_test_steps =integration_test_steps159,4342 - build_storybook =build_storybook182,5254 - publish_step =publish_step190,5674 - store_npm_step =store_npm_step191,5758 - windows_package_steps =windows_package_steps196,5957 - windows_pipeline =windows_pipeline198,6044 - name =name199,6077 - edition =edition200,6127 - trigger =trigger201,6154 - steps =steps202,6181 - platform =platform203,6256 - depends_on =depends_on204,6286 - environment =environment207,6393 - pipelines =pipelines209,6434 - name =name211,6470 - edition =edition212,6550 - trigger =trigger213,6581 - services =services214,6612 - steps =steps215,6639 - environment =environment216,6717 - volumes =volumes217,6756 - name =name225,6970 - edition =edition226,7038 - trigger =trigger227,7073 - services =services228,7108 - steps =steps229,7145 - environment =environment230,7329 - volumes =volumes231,7372 - deps =deps234,7433 -def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):enterprise_pipelines247,7856 - environment =environment259,8284 - edition =edition260,8328 - services =services261,8355 - volumes =volumes262,8415 - package_steps =package_steps263,8465 - publish_steps =publish_steps264,8488 - should_publish =should_publish265,8511 - should_upload =should_upload266,8554 - include_enterprise =include_enterprise267,8624 - edition2 =edition2268,8673 - init_steps =init_steps269,8702 - build_steps =build_steps277,8909 - integration_test_steps =integration_test_steps284,9218 - build_storybook =build_storybook312,10299 - publish_step =publish_step324,10892 - store_npm_step =store_npm_step325,10976 - windows_package_steps =windows_package_steps330,11175 - step =step333,11284 - deps_on_clone_enterprise_step =deps_on_clone_enterprise_step337,11418 - windows_pipeline =windows_pipeline347,11746 - name =name348,11779 - edition =edition349,11836 - trigger =trigger350,11863 - steps =steps351,11890 - platform =platform352,11965 - depends_on =depends_on353,11995 - environment =environment356,12109 - pipelines =pipelines358,12150 - name =name360,12186 - edition =edition361,12273 - trigger =trigger362,12304 - services =services363,12335 - steps =steps364,12362 - environment =environment365,12440 - volumes =volumes366,12479 - name =name374,12711 - edition =edition375,12786 - trigger =trigger376,12821 - services =services377,12856 - steps =steps378,12893 - environment =environment379,13213 - volumes =volumes380,13256 - deps =deps383,13317 -def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger):enterprise2_pipelines397,13769 - environment =environment412,14364 - edition =edition415,14424 - volumes =volumes416,14451 - package_steps =package_steps417,14501 - publish_steps =publish_steps418,14524 - should_publish =should_publish419,14547 - should_upload =should_upload420,14590 - include_enterprise =include_enterprise421,14660 - edition2 =edition2422,14709 - init_steps =init_steps423,14738 - build_steps =build_steps431,14945 - fetch_images =fetch_images442,15355 - upload_cdn =upload_cdn444,15497 - step =step458,16187 - deps_on_clone_enterprise_step =deps_on_clone_enterprise_step462,16321 - pipelines =pipelines472,16608 - name =name474,16644 - edition =edition475,16742 - trigger =trigger476,16773 - services =services477,16804 - steps =steps478,16831 - volumes =volumes479,16909 - environment =environment480,16940 -def publish_artifacts_step(mode):publish_artifacts_step486,17019 - security =security487,17053 - security =security489,17098 -def publish_artifacts_pipelines(mode):publish_artifacts_pipelines501,17538 - trigger =trigger502,17577 - steps =steps506,17655 - name =name512,17768 - trigger =trigger513,17820 - steps =steps514,17847 - edition =edition515,17870 - environment =environment516,17895 -def publish_packages_pipeline():publish_packages_pipeline519,17945 - trigger =trigger526,18162 - oss_steps =oss_steps530,18244 - enterprise_steps =enterprise_steps538,18560 - deps =deps545,18903 - name =name552,19062 - trigger =trigger553,19101 - steps =steps554,19128 - edition =edition555,19155 - depends_on =depends_on556,19180 - environment =environment557,19207 - name =name559,19266 - trigger =trigger560,19312 - steps =steps561,19339 - edition =edition562,19373 - depends_on =depends_on563,19398 - environment =environment564,19425 -def publish_npm_pipelines(mode):publish_npm_pipelines567,19482 - trigger =trigger568,19515 - steps =steps572,19593 - name =name580,19772 - trigger =trigger581,19827 - steps =steps582,19854 - edition =edition583,19877 - environment =environment584,19902 -def artifacts_page_pipeline():artifacts_page_pipeline587,19952 - trigger =trigger588,19983 - name =name593,20087 - trigger =trigger594,20128 - steps =steps595,20155 - edition =edition596,20220 - environment =environment597,20245 -def get_e2e_suffix():get_e2e_suffix600,20295 - -events/cron.star,1016 -aquasec_trivy_image =aquasec_trivy_image8,209 -def cronjobs(edition):cronjobs10,255 - grafana_com_nightly_pipeline =grafana_com_nightly_pipeline11,278 - cronName =cronName12,332 - name =name13,374 - steps =steps14,412 -def cron_job_pipeline(cronName, name, steps):cron_job_pipeline24,773 -def scan_docker_image_pipeline(edition, tag):scan_docker_image_pipeline43,1175 - edition =edition55,1530 - edition =edition57,1579 - docker_image =docker_image59,1608 - cronName =cronName62,1695 - name =name63,1725 - steps =steps64,1775 -def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):scan_docker_image_unkown_low_medium_vulnerabilities_step71,2047 -def scan_docker_image_high_critical_vulnerabilities_step(docker_image):scan_docker_image_high_critical_vulnerabilities_step80,2353 -def slack_job_failed_step(channel, image):slack_job_failed_step89,2646 -def post_to_grafana_com_step():post_to_grafana_com_step103,3069 - -events/main.star,633 -ver_mode =ver_mode49,966 -trigger =trigger50,984 -def main_pipelines(edition):main_pipelines62,1168 - drone_change_trigger =drone_change_trigger63,1197 - pipelines =pipelines79,1513 - name =name89,1951 - slack_channel =slack_channel90,1994 - trigger =trigger91,2045 - template =template92,2089 - secret =secret93,2135 - name =name97,2276 - slack_channel =slack_channel98,2310 - trigger =trigger99,2366 - depends_on =depends_on100,2425 - template =template101,2563 - secret =secret102,2604 - -events/pr.star,252 -ver_mode =ver_mode48,997 -trigger =trigger49,1013 -def pr_pipelines(edition):pr_pipelines62,1198 -def get_pr_trigger(include_paths = None, exclude_paths = None):get_pr_trigger76,2396 - paths_ex =paths_ex91,3080 - paths_in =paths_in92,3115 - -services/services.star,225 -def integration_test_services_volumes():integration_test_services_volumes5,79 -def integration_test_services(edition):integration_test_services14,292 - services =services15,332 -def ldap_service():ldap_service59,1616 - -utils/utils.star,561 -failure_template =failure_template11,191 -drone_change_template =drone_change_template12,509 - services =services19,932 - platform =platform20,955 - depends_on =depends_on21,983 - environment =environment22,1008 - volumes =volumes23,1036 - platform_conf =platform_conf50,2166 - platform_conf =platform_conf62,2534 - pipeline =pipeline70,2713 -def notify_pipeline(name, slack_channel, trigger, depends_on = [], template = None, secret = None):notify_pipeline105,3545 - trigger =trigger106,3645 - -pipelines/trigger_downstream.star,440 -trigger =trigger14,249 -def enterprise_downstream_pipeline(edition, ver_mode):enterprise_downstream_pipeline26,433 - environment =environment27,488 - steps =steps28,527 - deps =deps29,587 - name =name31,672 - edition =edition32,714 - trigger =trigger33,741 - services =services34,768 - steps =steps35,791 - depends_on =depends_on36,814 - environment =environment37,841 - -pipelines/verify_starlark.star,323 -def verify_starlark(trigger, ver_mode):verify_starlark17,305 - environment =environment18,345 - steps =steps19,382 - name =name26,546 - edition =edition27,600 - trigger =trigger28,625 - services =services29,652 - steps =steps30,675 - environment =environment31,698 - -pipelines/build.star,508 -def build_e2e(trigger, ver_mode, edition):build_e2e39,936 - environment =environment50,1096 - variants =variants51,1135 - init_steps =init_steps52,1219 - build_steps =build_steps61,1491 - publish_suffix =publish_suffix107,4049 - publish_suffix =publish_suffix109,4100 - name =name112,4158 - edition =edition113,4224 - environment =environment114,4249 - services =services115,4284 - steps =steps116,4307 - trigger =trigger117,4349 - -pipelines/shellcheck.star,386 -trigger =trigger15,235 -def shellcheck_step():shellcheck_step31,483 -def shellcheck_pipeline():shellcheck_pipeline43,725 - environment =environment44,752 - steps =steps45,789 - name =name50,886 - edition =edition51,918 - trigger =trigger52,943 - services =services53,970 - steps =steps54,993 - environment =environment55,1016 - -pipelines/verify_drone.star,317 -def verify_drone(trigger, ver_mode):verify_drone17,293 - environment =environment18,330 - steps =steps19,367 - name =name26,528 - edition =edition27,579 - trigger =trigger28,604 - services =services29,631 - steps =steps30,654 - environment =environment31,677 - -pipelines/test_backend.star,474 -def test_backend(trigger, ver_mode, edition = "oss"):test_backend23,463 - environment =environment35,882 - init_steps =init_steps36,921 - test_steps =test_steps46,1291 - pipeline_name =pipeline_name51,1387 - pipeline_name =pipeline_name53,1492 - name =name55,1584 - edition =edition56,1614 - trigger =trigger57,1641 - services =services58,1668 - steps =steps59,1691 - environment =environment60,1732 - -pipelines/lint_frontend.star,415 -def lint_frontend_pipeline(trigger, ver_mode):lint_frontend_pipeline16,260 - environment =environment26,546 - yarn_step =yarn_step27,583 - init_steps =init_steps29,660 - test_steps =test_steps33,736 - name =name37,812 - edition =edition38,864 - trigger =trigger39,889 - services =services40,916 - steps =steps41,939 - environment =environment42,980 - -pipelines/docs.star,494 -docs_paths =docs_paths19,383 -def docs_pipelines(edition, ver_mode, trigger):docs_pipelines28,511 - environment =environment29,559 - steps =steps30,598 - name =name40,815 - edition =edition41,858 - trigger =trigger42,885 - services =services43,912 - steps =steps44,935 - environment =environment45,958 -def lint_docs():lint_docs48,1000 -def trigger_docs_main():trigger_docs_main63,1328 -def trigger_docs_pr():trigger_docs_pr72,1478 - -pipelines/test_frontend.star,476 -def test_frontend(trigger, ver_mode, edition = "oss"):test_frontend20,374 - environment =environment32,794 - init_steps =init_steps33,833 - test_steps =test_steps41,1102 - pipeline_name =pipeline_name45,1205 - pipeline_name =pipeline_name47,1311 - name =name49,1404 - edition =edition50,1434 - trigger =trigger51,1461 - services =services52,1488 - steps =steps53,1511 - environment =environment54,1552 - -pipelines/integration_tests.star,483 -def integration_tests(trigger, ver_mode, edition):integration_tests26,542 - environment =environment37,900 - services =services38,939 - volumes =volumes39,989 - init_steps =init_steps40,1039 - test_steps =test_steps48,1282 - name =name54,1412 - edition =edition55,1468 - trigger =trigger56,1493 - services =services57,1520 - steps =steps58,1549 - environment =environment59,1590 - volumes =volumes60,1625 - -pipelines/windows.star,954 -def windows(trigger, edition, ver_mode):windows17,339 - environment =environment29,798 - init_cmds =init_cmds30,837 - steps =steps38,1205 - bucket =bucket49,1497 - ver_part =ver_part51,1590 - dir =dir52,1628 - dir =dir54,1670 - bucket =bucket55,1695 - build_no =build_no56,1736 - ver_part =ver_part57,1780 - installer_commands =installer_commands58,1842 - committish =committish100,3763 - committish =committish102,3846 - committish =committish104,3906 - download_grabpl_step_cmds =download_grabpl_step_cmds107,4057 - clone_cmds =clone_cmds113,4363 - name =name146,5711 - edition =edition147,5742 - trigger =trigger148,5769 - steps =steps149,5830 - depends_on =depends_on150,5889 - platform =platform151,6007 - environment =environment152,6037 - -pipelines/lint_backend.star,418 -def lint_backend_pipeline(trigger, ver_mode):lint_backend_pipeline18,306 - environment =environment28,590 - wire_step =wire_step29,627 - init_steps =init_steps31,704 - test_steps =test_steps36,809 - name =name43,959 - edition =edition44,1010 - trigger =trigger45,1035 - services =services46,1062 - steps =steps47,1085 - environment =environment48,1126 - -pipelines/publish_images.star,998 -def publish_image_steps(edition, mode, docker_repo):publish_image_steps17,303 - additional_docker_repo =additional_docker_repo31,922 - additional_docker_repo =additional_docker_repo33,979 - steps =steps34,1034 -def publish_image_pipelines_public():publish_image_pipelines_public45,1369 - mode =mode51,1521 - trigger =trigger52,1541 - name =name57,1641 - trigger =trigger58,1694 - steps =steps59,1721 - edition =edition60,1813 - environment =environment61,1835 - name =name63,1894 - trigger =trigger64,1954 - steps =steps65,1981 - edition =edition66,2091 - environment =environment67,2113 -def publish_image_pipelines_security():publish_image_pipelines_security70,2170 - mode =mode71,2210 - trigger =trigger72,2232 - name =name77,2332 - trigger =trigger78,2392 - steps =steps79,2419 - edition =edition80,2529 - environment =environment81,2551 - -steps/lib.star,8579 -grabpl_version =grabpl_version7,181 -build_image =build_image8,208 -publish_image =publish_image9,254 -deploy_docker_image =deploy_docker_image10,304 -alpine_image =alpine_image11,380 -curl_image =curl_image12,411 -windows_image =windows_image13,452 -wix_image =wix_image14,501 -go_image =go_image15,536 -disable_tests =disable_tests17,564 -trigger_oss =trigger_oss18,586 -def slack_step(channel, template, secret):slack_step24,653 -def yarn_install_step(edition = "oss"):yarn_install_step35,918 - deps =deps36,958 - deps =deps38,1004 -def wire_install_step():wire_install_step48,1222 -def identify_runner_step(platform = "linux"):identify_runner_step60,1454 -def clone_enterprise_step(ver_mode):clone_enterprise_step78,1916 - committish =committish87,2193 - committish =committish89,2268 - committish =committish91,2317 -def init_enterprise_step(ver_mode):init_enterprise_step105,2747 - source_commit =source_commit115,3098 - source_commit =source_commit117,3151 - environment =environment118,3191 - token =token121,3280 - environment =environment123,3369 - token =token126,3458 - environment =environment128,3518 - token =token129,3543 -def download_grabpl_step(platform = "linux"):download_grabpl_step148,4147 -def lint_drone_step():lint_drone_step173,4973 -def lint_starlark_step():lint_starlark_step185,5216 -def enterprise_downstream_step(edition, ver_mode):enterprise_downstream_step206,6000 - repo =repo219,6482 - step =step225,6623 -def lint_backend_step():lint_backend_step247,7248 -def benchmark_ldap_step():benchmark_ldap_step265,7713 -def build_storybook_step(edition, ver_mode):build_storybook_step278,8087 -def store_storybook_step(edition, ver_mode, trigger = None):store_storybook_step300,8743 - commands =commands314,9202 - commands =commands323,9521 - step =step325,9593 - when_cond =when_cond338,10125 - step =step346,10330 -def e2e_tests_artifacts(edition):e2e_tests_artifacts349,10391 -def upload_cdn_step(edition, ver_mode, trigger = None):upload_cdn_step386,12378 - deps =deps397,12763 - step =step407,12970 - step =step420,13423 -def build_backend_step(edition, ver_mode, variants = None):build_backend_step423,13482 - variants_str =variants_str437,14070 - variants_str =variants_str439,14109 - cmds =cmds443,14256 - build_no =build_no449,14418 - cmds =cmds450,14461 -def build_frontend_step(edition, ver_mode):build_frontend_step468,14906 - build_no =build_no478,15246 - cmds =cmds482,15356 - cmds =cmds487,15505 -def build_frontend_package_step(edition, ver_mode):build_frontend_package_step505,15960 - build_no =build_no515,16312 - cmds =cmds519,16422 - cmds =cmds524,16580 -def build_plugins_step(edition, ver_mode):build_plugins_step542,17053 - env =env544,17121 - env =env548,17220 -def test_backend_step():test_backend_step563,17607 -def test_backend_integration_step():test_backend_integration_step575,17880 -def betterer_frontend_step(edition = "oss"):betterer_frontend_step587,18187 - deps =deps596,18427 -def test_frontend_step(edition = "oss"):test_frontend_step609,18728 - deps =deps618,18962 -def lint_frontend_step():lint_frontend_step634,19343 -def test_a11y_frontend_step(ver_mode, edition, port = 3001):test_a11y_frontend_step652,19793 - commands =commands664,20279 - failure =failure667,20345 - failure =failure672,20483 -def frontend_metrics_step(edition, trigger = None):frontend_metrics_step693,21146 - step =step706,21507 - step =step721,22007 -def codespell_step():codespell_step724,22066 -def package_step(edition, ver_mode, variants = None):package_step736,22468 - deps =deps750,23006 - variants_str =variants_str757,23167 - variants_str =variants_str759,23206 - sign_args =sign_args762,23332 - env =env763,23362 - test_args =test_args769,23628 - sign_args =sign_args771,23661 - env =env772,23684 - test_args =test_args773,23703 - cmds =cmds777,23829 - build_no =build_no784,24036 - cmds =cmds785,24079 -def grafana_server_step(edition, port = 3001):grafana_server_step798,24459 - package_file_pfx =package_file_pfx808,24729 - package_file_pfx =package_file_pfx810,24788 - package_file_pfx =package_file_pfx812,24889 - environment =environment814,24938 -def e2e_tests_step(suite, edition, port = 3001, tries = None):e2e_tests_step837,25554 - cmd =cmd838,25617 -def cloud_plugins_e2e_tests_step(suite, edition, cloud, trigger = None):cloud_plugins_e2e_tests_step856,26186 - environment =environment869,26649 - when =when870,26670 - when =when872,26700 - environment =environment874,26748 - when =when882,27129 - branch =branch888,27345 - step =step889,27401 - step =step901,27822 -def build_docs_website_step():build_docs_website_step904,27874 -def copy_packages_for_docker_step(edition = None):copy_packages_for_docker_step916,28272 -def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False):build_docker_images_step929,28622 - cmd =cmd943,29193 - ubuntu_sfx =ubuntu_sfx947,29307 - ubuntu_sfx =ubuntu_sfx949,29342 - environment =environment955,29468 -def fetch_images_step(edition):fetch_images_step979,30079 -def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):publish_images_step997,30745 - name =name1013,31562 - docker_repo =docker_repo1014,31585 - mode =mode1016,31663 - mode =mode1018,31709 - environment =environment1020,31728 - cmd =cmd1026,31912 - deps =deps1029,32041 - deps =deps1032,32147 - name =name1035,32250 - docker_repo =docker_repo1036,32273 - cmd =cmd1038,32459 - step =step1040,32565 - step =step1052,32929 -def postgres_integration_tests_step():postgres_integration_tests_step1056,32989 - cmds =cmds1057,33028 -def mysql_integration_tests_step():mysql_integration_tests_step1079,33850 - cmds =cmds1080,33886 -def redis_integration_tests_step():redis_integration_tests_step1100,34629 -def memcached_integration_tests_step():memcached_integration_tests_step1114,35026 -def release_canary_npm_packages_step(edition, trigger = None):release_canary_npm_packages_step1128,35435 - step =step1141,35805 - step =step1153,36143 -def enterprise2_suffix(edition):enterprise2_suffix1156,36202 -def upload_packages_step(edition, ver_mode, trigger = None):upload_packages_step1161,36320 - deps =deps1176,36816 - step =step1184,37036 - step =step1195,37471 -def publish_grafanacom_step(edition, ver_mode):publish_grafanacom_step1198,37530 - cmd =cmd1211,38044 - build_no =build_no1215,38188 - cmd =cmd1216,38231 -def publish_linux_packages_step(edition, package_manager = "deb"):publish_linux_packages_step1239,38866 -def get_windows_steps(edition, ver_mode):get_windows_steps1261,39989 - init_cmds =init_cmds1270,40281 - steps =steps1278,40649 - bucket =bucket1289,40941 - ver_part =ver_part1291,41034 - dir =dir1292,41072 - dir =dir1294,41114 - bucket =bucket1295,41139 - build_no =build_no1296,41180 - ver_part =ver_part1297,41224 - installer_commands =installer_commands1298,41286 - committish =committish1340,43207 - committish =committish1342,43290 - committish =committish1344,43350 - download_grabpl_step_cmds =download_grabpl_step_cmds1347,43501 - clone_cmds =clone_cmds1353,43807 -def verify_gen_cue_step(edition):verify_gen_cue_step1387,45152 - deps =deps1388,45186 -def verify_gen_jsonnet_step(edition):verify_gen_jsonnet_step1402,45694 - deps =deps1403,45732 -def trigger_test_release():trigger_test_release1417,46236 -def artifacts_page_step():artifacts_page_step1451,47731 -def end_to_end_tests_deps():end_to_end_tests_deps1466,48058 -def compile_build_cmd(edition = "oss"):compile_build_cmd1476,48321 - dependencies =dependencies1477,48361 - dependencies =dependencies1479,48432 -def get_trigger_storybook(ver_mode):get_trigger_storybook1492,48780 - trigger_storybook =trigger_storybook1500,49031 - trigger_storybook =trigger_storybook1502,49088 - trigger_storybook =trigger_storybook1506,49168 - -vault.star,444 -pull_secret =pull_secret4,87 -github_token =github_token5,120 -drone_token =drone_token6,150 -prerelease_bucket =prerelease_bucket7,178 -gcp_upload_artifacts_key =gcp_upload_artifacts_key8,218 -azure_sp_app_id =azure_sp_app_id9,272 -azure_sp_app_pw =azure_sp_app_pw10,308 -azure_tenant =azure_tenant11,344 -def from_secret(secret):from_secret13,375 -def vault_secret(name, path, key):vault_secret18,451 -def secrets():secrets28,633 - -version.star,116 -ver_mode =ver_mode12,197 -trigger =trigger13,225 -def version_branch_pipelines():version_branch_pipelines15,268 diff --git a/scripts/drone/events/cron.star b/scripts/drone/events/cron.star index d34ff464451..8a7641c88a2 100644 --- a/scripts/drone/events/cron.star +++ b/scripts/drone/events/cron.star @@ -2,7 +2,6 @@ This module provides functions for cronjob pipelines and steps used within. """ -load("scripts/drone/vault.star", "from_secret") load( "scripts/drone/steps/lib.star", "compile_build_cmd", @@ -11,6 +10,7 @@ load( "scripts/drone/utils/images.star", "images", ) +load("scripts/drone/vault.star", "from_secret") aquasec_trivy_image = "aquasec/trivy:0.21.0" @@ -160,7 +160,7 @@ def scan_docker_image_high_critical_vulnerabilities_step(docker_image): def slack_job_failed_step(channel, image): return { "name": "slack-notify-failure", - "image": images["plugins_slack_image"], + "image": images["plugins_slack"], "settings": { "webhook": from_secret("slack_webhook_backend"), "channel": channel, @@ -174,7 +174,7 @@ def slack_job_failed_step(channel, image): def post_to_grafana_com_step(): return { "name": "post-to-grafana-com", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GRAFANA_COM_API_KEY": from_secret("grafana_api_key"), "GCP_KEY": from_secret("gcp_key"), diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index b37b1893229..9f20d886481 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -3,10 +3,8 @@ This module returns all the pipelines used in the event of pushes to the main br """ load( - "scripts/drone/utils/utils.star", - "drone_change_template", - "failure_template", - "notify_pipeline", + "scripts/drone/pipelines/build.star", + "build_e2e", ) load( "scripts/drone/pipelines/docs.star", @@ -14,36 +12,38 @@ load( "trigger_docs_main", ) load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", + "scripts/drone/pipelines/integration_tests.star", + "integration_tests", ) load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", + "scripts/drone/pipelines/lint_backend.star", + "lint_backend_pipeline", ) load( - "scripts/drone/pipelines/integration_tests.star", - "integration_tests", + "scripts/drone/pipelines/lint_frontend.star", + "lint_frontend_pipeline", ) load( - "scripts/drone/pipelines/build.star", - "build_e2e", + "scripts/drone/pipelines/test_backend.star", + "test_backend", ) load( - "scripts/drone/pipelines/windows.star", - "windows", + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", ) load( "scripts/drone/pipelines/trigger_downstream.star", "enterprise_downstream_pipeline", ) load( - "scripts/drone/pipelines/lint_backend.star", - "lint_backend_pipeline", + "scripts/drone/pipelines/windows.star", + "windows", ) load( - "scripts/drone/pipelines/lint_frontend.star", - "lint_frontend_pipeline", + "scripts/drone/utils/utils.star", + "drone_change_template", + "failure_template", + "notify_pipeline", ) ver_mode = "main" diff --git a/scripts/drone/events/pr.star b/scripts/drone/events/pr.star index 5acf2cd2bb6..cbd425efcf6 100644 --- a/scripts/drone/events/pr.star +++ b/scripts/drone/events/pr.star @@ -4,37 +4,21 @@ It also includes a function generating a PR trigger from a list of included and """ load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) -load( - "scripts/drone/pipelines/integration_tests.star", - "integration_tests", + "scripts/drone/pipelines/benchmarks.star", + "integration_benchmarks", ) load( "scripts/drone/pipelines/build.star", "build_e2e", ) -load( - "scripts/drone/pipelines/verify_drone.star", - "verify_drone", -) -load( - "scripts/drone/pipelines/verify_starlark.star", - "verify_starlark", -) load( "scripts/drone/pipelines/docs.star", "docs_pipelines", "trigger_docs_pr", ) load( - "scripts/drone/pipelines/shellcheck.star", - "shellcheck_pipeline", + "scripts/drone/pipelines/integration_tests.star", + "integration_tests", ) load( "scripts/drone/pipelines/lint_backend.star", @@ -45,8 +29,24 @@ load( "lint_frontend_pipeline", ) load( - "scripts/drone/pipelines/benchmarks.star", - "integration_benchmarks", + "scripts/drone/pipelines/shellcheck.star", + "shellcheck_pipeline", +) +load( + "scripts/drone/pipelines/test_backend.star", + "test_backend", +) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) +load( + "scripts/drone/pipelines/verify_drone.star", + "verify_drone", +) +load( + "scripts/drone/pipelines/verify_starlark.star", + "verify_starlark", ) ver_mode = "pr" diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 295e4a922fa..ecf03e38b5f 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -2,55 +2,35 @@ This module returns all the pipelines used in the event of a release along with supporting functions. """ +load( + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) load( "scripts/drone/steps/lib.star", - "build_backend_step", - "build_docker_images_step", - "build_frontend_package_step", - "build_frontend_step", - "build_plugins_step", - "build_storybook_step", "compile_build_cmd", - "copy_packages_for_docker_step", "download_grabpl_step", - "e2e_tests_artifacts", - "e2e_tests_step", - "get_windows_steps", - "grafana_server_step", "identify_runner_step", - "memcached_integration_tests_step", - "mysql_integration_tests_step", - "package_step", - "postgres_integration_tests_step", + "memcached_integration_tests_steps", + "mysql_integration_tests_steps", + "postgres_integration_tests_steps", "publish_grafanacom_step", "publish_linux_packages_step", - "redis_integration_tests_step", - "store_storybook_step", - "trigger_oss", - "upload_cdn_step", - "upload_packages_step", + "redis_integration_tests_steps", "verify_gen_cue_step", "verify_gen_jsonnet_step", "wire_install_step", "yarn_install_step", ) load( - "scripts/drone/services/services.star", - "integration_test_services", - "integration_test_services_volumes", + "scripts/drone/utils/images.star", + "images", ) load( "scripts/drone/utils/utils.star", "pipeline", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) load( "scripts/drone/vault.star", "from_secret", @@ -59,16 +39,9 @@ load( "prerelease_bucket", "rgm_gcp_key_base64", ) -load( - "scripts/drone/utils/images.star", - "images", -) -load( - "scripts/drone/pipelines/whats_new_checker.star", - "whats_new_checker_pipeline", -) ver_mode = "release" + release_trigger = { "event": { "exclude": [ @@ -85,25 +58,10 @@ release_trigger = { }, } -def store_npm_packages_step(): - return { - "name": "store-npm-packages", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - "build-frontend-packages", - ], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - }, - "commands": ["./bin/build artifacts npm store --tag ${DRONE_TAG}"], - } - def retrieve_npm_packages_step(): return { "name": "retrieve-npm-packages", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "compile-build-cmd", "yarn-install", @@ -119,7 +77,7 @@ def retrieve_npm_packages_step(): def release_npm_packages_step(): return { "name": "release-npm-packages", - "image": images["build_image"], + "image": images["go"], "depends_on": [ "compile-build-cmd", "retrieve-npm-packages", @@ -131,146 +89,10 @@ def release_npm_packages_step(): "commands": ["./bin/build artifacts npm release --tag ${DRONE_TAG}"], } -def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): - """Generates all pipelines used for Grafana OSS. - - Args: - ver_mode: controls which steps are included in the pipeline. - Defaults to 'release'. - trigger: controls which events can trigger the pipeline execution. - Defaults to tag events for tags with a 'v' prefix. - - Returns: - List of Drone pipelines. - """ - - environment = {"EDITION": "oss"} - - services = integration_test_services() - volumes = integration_test_services_volumes() - - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - verify_gen_cue_step(), - wire_install_step(), - yarn_install_step(), - compile_build_cmd(), - ] - - build_steps = [ - build_backend_step(ver_mode = ver_mode), - build_frontend_step(ver_mode = ver_mode), - build_frontend_package_step(ver_mode = ver_mode), - build_plugins_step(ver_mode = ver_mode), - package_step(ver_mode = ver_mode), - copy_packages_for_docker_step(), - build_docker_images_step(publish = True), - build_docker_images_step( - publish = True, - ubuntu = True, - ), - grafana_server_step(), - e2e_tests_step("dashboards-suite", tries = 3), - e2e_tests_step("smoke-tests-suite", tries = 3), - e2e_tests_step("panels-suite", tries = 3), - e2e_tests_step("various-suite", tries = 3), - e2e_tests_artifacts(), - build_storybook_step(ver_mode = ver_mode), - ] - - publish_steps = [] - - if ver_mode in ( - "release", - "release-branch", - ): - publish_steps.extend( - [ - upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss), - upload_packages_step( - ver_mode = ver_mode, - trigger = trigger_oss, - ), - ], - ) - - if ver_mode in ("release",): - publish_steps.extend( - [ - store_storybook_step(ver_mode = ver_mode), - store_npm_packages_step(), - ], - ) - - integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step("mysql57", "5.7"), - mysql_integration_tests_step("mysql80", "8.0"), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] - - pipelines = [] - - # We don't need to run integration tests at release time since they have - # been run multiple times before: - if ver_mode in ("release"): - pipelines.append(whats_new_checker_pipeline(release_trigger)) - integration_test_steps = [] - volumes = [] - - windows_pipeline_dependencies = [ - "{}-build-e2e-publish".format(ver_mode), - "{}-test-frontend".format(ver_mode), - ] - pipelines.extend([ - pipeline( - name = "{}-build-e2e-publish".format(ver_mode), - trigger = trigger, - services = [], - steps = init_steps + build_steps + publish_steps, - environment = environment, - volumes = volumes, - ), - test_frontend(trigger, ver_mode), - test_backend(trigger, ver_mode), - ]) - - if ver_mode not in ("release"): - pipelines.append(pipeline( - name = "{}-integration-tests".format(ver_mode), - trigger = trigger, - services = services, - steps = [ - download_grabpl_step(), - identify_runner_step(), - verify_gen_cue_step(), - verify_gen_jsonnet_step(), - wire_install_step(), - ] + - integration_test_steps, - environment = environment, - volumes = volumes, - )) - - windows_pipeline = pipeline( - name = "{}-windows".format(ver_mode), - trigger = trigger, - steps = get_windows_steps(ver_mode = ver_mode), - platform = "windows", - depends_on = windows_pipeline_dependencies, - environment = environment, - ) - - pipelines.append(windows_pipeline) - - return pipelines - def publish_artifacts_step(): return { "name": "publish-artifacts", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -284,7 +106,7 @@ def publish_artifacts_step(): def publish_static_assets_step(): return { "name": "publish-static-assets", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -299,7 +121,7 @@ def publish_static_assets_step(): def publish_storybook_step(): return { "name": "publish-storybook", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -411,13 +233,11 @@ def integration_test_pipelines(): } pipelines = [] volumes = integration_test_services_volumes() - integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step("mysql57", "5.7"), - mysql_integration_tests_step("mysql80", "8.0"), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] + integration_test_steps = postgres_integration_tests_steps() + \ + mysql_integration_tests_steps("mysql57", "5.7") + \ + mysql_integration_tests_steps("mysql80", "8.0") + \ + redis_integration_tests_steps() + \ + memcached_integration_tests_steps() pipelines.append(pipeline( name = "integration-tests", @@ -456,7 +276,7 @@ def verify_release_pipeline( step = { "name": "gsutil-stat", "depends_on": ["clone"], - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "BUCKET": bucket, "GCP_KEY": gcp_key, diff --git a/scripts/drone/pipelines/benchmarks.star b/scripts/drone/pipelines/benchmarks.star index 5e1fc6a3ec1..ae5a9611675 100644 --- a/scripts/drone/pipelines/benchmarks.star +++ b/scripts/drone/pipelines/benchmarks.star @@ -2,6 +2,11 @@ This module returns the pipeline used for integration benchmarks. """ +load( + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) load( "scripts/drone/steps/lib.star", "compile_build_cmd", @@ -11,11 +16,6 @@ load( "verify_gen_jsonnet_step", "wire_install_step", ) -load( - "scripts/drone/services/services.star", - "integration_test_services", - "integration_test_services_volumes", -) load( "scripts/drone/utils/utils.star", "pipeline", @@ -52,22 +52,20 @@ def integration_benchmarks(prefix): wire_install_step(), ] - benchmark_steps = [ - integration_benchmarks_step("sqlite"), - integration_benchmarks_step("postgres", { - "PGPASSWORD": "grafanatest", - "GRAFANA_TEST_DB": "postgres", - "POSTGRES_HOST": "postgres", - }), - integration_benchmarks_step("mysql-5.7", { - "GRAFANA_TEST_DB": "mysql", - "MYSQL_HOST": "mysql57", - }), - integration_benchmarks_step("mysql-8.0", { - "GRAFANA_TEST_DB": "mysql", - "MYSQL_HOST": "mysql80", - }), - ] + benchmark_steps = integration_benchmarks_step("sqlite") + \ + integration_benchmarks_step("postgres", { + "PGPASSWORD": "grafanatest", + "GRAFANA_TEST_DB": "postgres", + "POSTGRES_HOST": "postgres", + }) + \ + integration_benchmarks_step("mysql-5.7", { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql57", + }) + \ + integration_benchmarks_step("mysql-8.0", { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql80", + }) return pipeline( name = "{}-integration-benchmarks".format(prefix), diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index e4b88414148..68113348fca 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -2,15 +2,10 @@ load( "scripts/drone/steps/lib.star", - "build_backend_step", - "build_docker_images_step", "build_frontend_package_step", - "build_frontend_step", - "build_plugins_step", "build_storybook_step", "cloud_plugins_e2e_tests_step", "compile_build_cmd", - "copy_packages_for_docker_step", "download_grabpl_step", "e2e_tests_artifacts", "e2e_tests_step", @@ -18,7 +13,6 @@ load( "frontend_metrics_step", "grafana_server_step", "identify_runner_step", - "package_step", "publish_images_step", "release_canary_npm_packages_step", "store_storybook_step", @@ -32,6 +26,15 @@ load( "wire_install_step", "yarn_install_step", ) +load( + "scripts/drone/steps/rgm.star", + "rgm_build_docker_step", + "rgm_package_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -72,11 +75,8 @@ def build_e2e(trigger, ver_mode): build_steps.extend( [ - build_backend_step(ver_mode = ver_mode), - build_frontend_step(ver_mode = ver_mode), - build_frontend_package_step(ver_mode = ver_mode), - build_plugins_step(ver_mode = ver_mode), - package_step(ver_mode = ver_mode), + build_frontend_package_step(), + rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"), grafana_server_step(), e2e_tests_step("dashboards-suite"), e2e_tests_step("smoke-tests-suite"), @@ -89,7 +89,6 @@ def build_e2e(trigger, ver_mode): ), e2e_tests_artifacts(), build_storybook_step(ver_mode = ver_mode), - copy_packages_for_docker_step(), test_a11y_frontend_step(ver_mode = ver_mode), ], ) @@ -99,13 +98,7 @@ def build_e2e(trigger, ver_mode): [ store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode), frontend_metrics_step(trigger = trigger_oss), - build_docker_images_step( - publish = False, - ), - build_docker_images_step( - publish = False, - ubuntu = True, - ), + rgm_build_docker_step("packages.txt", images["ubuntu"], images["alpine"]), publish_images_step( docker_repo = "grafana", trigger = trigger_oss, @@ -130,16 +123,12 @@ def build_e2e(trigger, ver_mode): elif ver_mode == "pr": build_steps.extend( [ - build_docker_images_step( - archs = [ - "amd64", - ], - ), - build_docker_images_step( - archs = [ - "amd64", - ], - ubuntu = True, + rgm_build_docker_step( + "packages.txt", + images["ubuntu"], + images["alpine"], + tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", + ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", ), publish_images_step( docker_repo = "grafana", diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star index 65df476754b..e8180d6d0ef 100644 --- a/scripts/drone/pipelines/ci_images.star +++ b/scripts/drone/pipelines/ci_images.star @@ -3,21 +3,21 @@ This module contains steps and pipelines relating to creating CI Docker images. """ load( - "scripts/drone/utils/utils.star", - "pipeline", + "scripts/drone/utils/images.star", + "images", ) load( - "scripts/drone/vault.star", - "from_secret", - "gcp_download_build_container_assets_key", + "scripts/drone/utils/utils.star", + "pipeline", ) load( "scripts/drone/utils/windows_images.star", "windows_images", ) load( - "scripts/drone/utils/images.star", - "images", + "scripts/drone/vault.star", + "from_secret", + "gcp_download_build_container_assets_key", ) def publish_ci_windows_test_image_pipeline(): @@ -32,7 +32,7 @@ def publish_ci_windows_test_image_pipeline(): steps = [ { "name": "clone", - "image": windows_images["wix_image"], + "image": windows_images["wix"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -43,7 +43,7 @@ def publish_ci_windows_test_image_pipeline(): }, { "name": "build-and-publish", - "image": windows_images["windows_server_core_image"], + "image": windows_images["windows_server_core"], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), "DOCKER_PASSWORD": from_secret("docker_password"), @@ -81,14 +81,14 @@ def publish_ci_build_container_image_pipeline(): steps = [ { "name": "validate-version", - "image": images["alpine_image"], + "image": images["alpine"], "commands": [ "if [ -z \"${BUILD_CONTAINER_VERSION}\" ]; then echo Missing BUILD_CONTAINER_VERSION; false; fi", ], }, { "name": "download-macos-sdk", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "GCP_KEY": from_secret(gcp_download_build_container_assets_key), }, @@ -100,7 +100,7 @@ def publish_ci_build_container_image_pipeline(): }, { "name": "build-and-publish", # Consider splitting the build and the upload task. - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index cdf944e43ec..536749b9e04 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -50,7 +50,7 @@ def docs_pipelines(ver_mode, trigger): def lint_docs(): return { "name": "lint-docs", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star index c72456b45cf..454331f5069 100644 --- a/scripts/drone/pipelines/integration_tests.star +++ b/scripts/drone/pipelines/integration_tests.star @@ -2,25 +2,25 @@ This module returns the pipeline used for integration tests. """ +load( + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) load( "scripts/drone/steps/lib.star", "compile_build_cmd", "download_grabpl_step", "enterprise_setup_step", "identify_runner_step", - "memcached_integration_tests_step", - "mysql_integration_tests_step", - "postgres_integration_tests_step", - "redis_integration_tests_step", + "memcached_integration_tests_steps", + "mysql_integration_tests_steps", + "postgres_integration_tests_steps", + "redis_integration_tests_steps", "verify_gen_cue_step", "verify_gen_jsonnet_step", "wire_install_step", ) -load( - "scripts/drone/services/services.star", - "integration_test_services", - "integration_test_services_volumes", -) load( "scripts/drone/utils/utils.star", "pipeline", @@ -60,13 +60,12 @@ def integration_tests(trigger, prefix, ver_mode = "pr"): wire_install_step(), ] - test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step("mysql57", "5.7"), - mysql_integration_tests_step("mysql80", "8.0"), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] + # test_steps = [a, b] + [c, d] + [e, f]... + test_steps = postgres_integration_tests_steps() + \ + mysql_integration_tests_steps("mysql57", "5.7") + \ + mysql_integration_tests_steps("mysql80", "8.0") + \ + redis_integration_tests_steps() + \ + memcached_integration_tests_steps() return pipeline( name = "{}-integration-tests".format(prefix), diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index 5b8d481094c..6b7e2182ad5 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -3,14 +3,14 @@ This module returns a Drone step and pipeline for linting with shellcheck. """ load("scripts/drone/steps/lib.star", "compile_build_cmd") -load( - "scripts/drone/utils/utils.star", - "pipeline", -) load( "scripts/drone/utils/images.star", "images", ) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) trigger = { "event": [ @@ -29,12 +29,9 @@ trigger = { def shellcheck_step(): return { "name": "shellcheck", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - ], + "image": images["shellcheck"], "commands": [ - "./bin/build shellcheck", + "shellcheck -e SC1071 -e SC2162 scripts/**/*.sh", ], } diff --git a/scripts/drone/pipelines/test_backend.star b/scripts/drone/pipelines/test_backend.star index ad1394196f3..c5bb47da11e 100644 --- a/scripts/drone/pipelines/test_backend.star +++ b/scripts/drone/pipelines/test_backend.star @@ -4,7 +4,6 @@ This module returns the pipeline used for testing backend code. load( "scripts/drone/steps/lib.star", - "compile_build_cmd", "enterprise_setup_step", "identify_runner_step", "test_backend_integration_step", @@ -41,7 +40,6 @@ def test_backend(trigger, ver_mode): steps += [ identify_runner_step(), - compile_build_cmd(), verify_step, verify_jsonnet_step, wire_install_step(), diff --git a/scripts/drone/pipelines/test_frontend.star b/scripts/drone/pipelines/test_frontend.star index e9e353720bd..9a01f101c58 100644 --- a/scripts/drone/pipelines/test_frontend.star +++ b/scripts/drone/pipelines/test_frontend.star @@ -5,7 +5,6 @@ This module returns the pipeline used for testing backend code. load( "scripts/drone/steps/lib.star", "betterer_frontend_step", - "download_grabpl_step", "enterprise_setup_step", "identify_runner_step", "test_frontend_step", @@ -30,7 +29,6 @@ def test_frontend(trigger, ver_mode): steps = [ identify_runner_step(), - download_grabpl_step(), yarn_install_step(), betterer_frontend_step(), ] diff --git a/scripts/drone/pipelines/whats_new_checker.star b/scripts/drone/pipelines/whats_new_checker.star index e15a8d7a291..2c56be573ce 100644 --- a/scripts/drone/pipelines/whats_new_checker.star +++ b/scripts/drone/pipelines/whats_new_checker.star @@ -2,14 +2,14 @@ This module contains logic for checking if the package.json whats new url matches with the in-flight tag. """ -load( - "scripts/drone/utils/images.star", - "images", -) load( "scripts/drone/steps/lib.star", "compile_build_cmd", ) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -18,7 +18,7 @@ load( def whats_new_checker_step(): return { "name": "whats-new-checker", - "image": images["go_image"], + "image": images["go"], "depends_on": [ "compile-build-cmd", ], diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star index fd5b99c9bf0..59c12a19536 100644 --- a/scripts/drone/pipelines/windows.star +++ b/scripts/drone/pipelines/windows.star @@ -3,15 +3,15 @@ This module returns the pipeline used for building Grafana on Windows. """ load( - "scripts/drone/utils/utils.star", - "pipeline", + "scripts/drone/steps/lib_windows.star", + "clone_step_windows", + "get_windows_steps", + "test_backend_step_windows", + "wire_install_step_windows", ) load( - "scripts/drone/steps/lib.star", - "get_windows_steps", - "windows_clone_step", - "windows_test_backend_step", - "windows_wire_install_step", + "scripts/drone/utils/utils.star", + "pipeline", ) load( "scripts/drone/utils/windows_images.star", @@ -30,19 +30,19 @@ def windows_test_backend(trigger, edition, ver_mode): """ environment = {"EDITION": edition} steps = [ - windows_clone_step(), + clone_step_windows(), ] steps.extend([{ "name": "windows-init", - "image": windows_images["windows_go_image"], + "image": windows_images["go"], "depends_on": ["clone"], "commands": [], }]) steps.extend([ - windows_wire_install_step(edition), - windows_test_backend_step(), + wire_install_step_windows(edition), + test_backend_step_windows(), ]) pl = pipeline( name = "{}-test-backend-windows".format(ver_mode), diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index 0721f76b32b..f91d85bdd03 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -4,31 +4,35 @@ rgm uses 'github.com/grafana/grafana-build' to build Grafana on the following ev * A tag that begins with a 'v' """ -load( - "scripts/drone/steps/lib.star", - "get_windows_steps", -) -load( - "scripts/drone/utils/utils.star", - "ignore_failure", - "pipeline", -) load( "scripts/drone/events/release.star", "verify_release_pipeline", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) load( "scripts/drone/pipelines/test_backend.star", "test_backend", ) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) load( "scripts/drone/pipelines/whats_new_checker.star", "whats_new_checker_pipeline", ) +load( + "scripts/drone/steps/lib_windows.star", + "get_windows_steps", +) +load( + "scripts/drone/utils/utils.star", + "ignore_failure", + "pipeline", +) +load( + "scripts/drone/variables.star", + "golang_version", +) load( "scripts/drone/vault.star", "from_secret", @@ -38,15 +42,22 @@ load( "rgm_github_token", ) -rgm_env_secrets = { - "GCP_KEY_BASE64": from_secret(rgm_gcp_key_base64), - "DESTINATION": from_secret(rgm_destination), - "GITHUB_TOKEN": from_secret(rgm_github_token), - "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN": from_secret(rgm_dagger_token), - "GPG_PRIVATE_KEY": from_secret("packages_gpg_private_key"), - "GPG_PUBLIC_KEY": from_secret("packages_gpg_public_key"), - "GPG_PASSPHRASE": from_secret("packages_gpg_passphrase"), -} +def rgm_env_secrets(env): + """Adds the rgm secret ENV variables to the given env arg + + Args: + env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them). + Returns: + Drone step. + """ + env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64) + env["DESTINATION"] = from_secret(rgm_destination) + env["GITHUB_TOKEN"] = from_secret(rgm_github_token) + env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token) + env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key") + env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key") + env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase") + return env docs_paths = { "exclude": [ @@ -73,7 +84,20 @@ tag_trigger = { }, } +version_branch_trigger = {"ref": ["refs/heads/v[0-9]*"]} + def rgm_build(script = "drone_publish_main.sh", canFail = True): + """Returns a pipeline that does a full build & package of Grafana. + + Args: + script: The script in the container to run. + canFail: if true, then this pipeline can fail while the entire build will still succeed. + Returns: + Drone step. + """ + env = { + "GO_VERSION": golang_version, + } rgm_build_step = { "name": "rgm-build", "image": "grafana/grafana-build:main", @@ -81,11 +105,12 @@ def rgm_build(script = "drone_publish_main.sh", canFail = True): "export GRAFANA_DIR=$$(pwd)", "cd /src && ./scripts/{}".format(script), ], - "environment": rgm_env_secrets, + "environment": rgm_env_secrets(env), # The docker socket is a requirement for running dagger programs # In the future we should find a way to use dagger without mounting the docker socket. "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } + if canFail: rgm_build_step["failure"] = "ignore" @@ -120,7 +145,7 @@ def rgm_tag(): depends_on = ["release-test-backend", "release-test-frontend"], ) -def rgm_windows(): +def rgm_tag_windows(): return pipeline( name = "rgm-tag-prerelease-windows", trigger = tag_trigger, @@ -134,14 +159,23 @@ def rgm_windows(): platform = "windows", ) +def rgm_version_branch(): + return pipeline( + name = "rgm-version-branch-prerelease", + trigger = version_branch_trigger, + steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False), + depends_on = ["release-test-backend", "release-test-frontend"], + ) + def rgm(): return [ whats_new_checker_pipeline(tag_trigger), test_frontend(tag_trigger, "release"), test_backend(tag_trigger, "release"), - rgm_main(), - rgm_tag(), - rgm_windows(), + rgm_main(), # Runs a package / build process (with some distros) when commits are merged to main + rgm_tag(), # Runs a package / build process (with all distros) when a tag is made + rgm_tag_windows(), + rgm_version_branch(), # Runs a package / build proces (with all distros) when a commit lands on a version branch verify_release_pipeline( trigger = tag_trigger, name = "rgm-tag-verify-prerelease-assets", @@ -151,4 +185,12 @@ def rgm(): "rgm-tag-prerelease-windows", ], ), + verify_release_pipeline( + trigger = version_branch_trigger, + name = "rgm-prerelease-verify-prerelease-assets", + bucket = "grafana-prerelease", + depends_on = [ + "rgm-version-branch-prerelease", + ], + ), ] diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index ae2d939cf4b..62bbab4c774 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -18,7 +18,7 @@ def integration_test_services(): services = [ { "name": "postgres", - "image": images["postgres_alpine_image"], + "image": images["postgres_alpine"], "environment": { "POSTGRES_USER": "grafanatest", "POSTGRES_PASSWORD": "grafanatest", @@ -31,7 +31,7 @@ def integration_test_services(): }, { "name": "mysql57", - "image": images["mysql5_image"], + "image": images["mysql5"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", @@ -43,7 +43,7 @@ def integration_test_services(): }, { "name": "mysql80", - "image": images["mysql8_image"], + "image": images["mysql8"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", @@ -55,12 +55,12 @@ def integration_test_services(): }, { "name": "redis", - "image": images["redis_alpine_image"], + "image": images["redis_alpine"], "environment": {}, }, { "name": "memcached", - "image": images["memcached_alpine_image"], + "image": images["memcached_alpine"], "environment": {}, }, ] @@ -70,7 +70,7 @@ def integration_test_services(): def ldap_service(): return { "name": "ldap", - "image": images["openldap_image"], + "image": images["openldap"], "environment": { "LDAP_ADMIN_PASSWORD": "grafana", "LDAP_DOMAIN": "grafana.org", diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 6e0d4453877..dc07aadf00f 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -2,6 +2,18 @@ This module is a library of Drone steps and other pipeline components. """ +load( + "scripts/drone/steps/rgm.star", + "rgm_build_backend_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/variables.star", + "grabpl_version", +) load( "scripts/drone/vault.star", "from_secret", @@ -11,16 +23,6 @@ load( "npm_token", "prerelease_bucket", ) -load( - "scripts/drone/utils/images.star", - "images", -) -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) - -grabpl_version = "v3.0.41" trigger_oss = { "repo": [ @@ -28,21 +30,10 @@ trigger_oss = { ], } -def slack_step(channel, template, secret): - return { - "name": "slack", - "image": images["plugins_slack_image"], - "settings": { - "webhook": from_secret(secret), - "channel": channel, - "template": template, - }, - } - def yarn_install_step(): return { "name": "yarn-install", - "image": images["build_image"], + "image": images["node"], "commands": [ "yarn install --immutable", ], @@ -52,8 +43,9 @@ def yarn_install_step(): def wire_install_step(): return { "name": "wire-install", - "image": images["build_image"], + "image": images["go"], "commands": [ + "apk add --update make", "make gen-go", ], "depends_on": [ @@ -61,37 +53,15 @@ def wire_install_step(): ], } -def windows_wire_install_step(edition): +def identify_runner_step(): return { - "name": "wire-install", - "image": windows_images["windows_go_image"], + "name": "identify-runner", + "image": images["alpine"], "commands": [ - "go install github.com/google/wire/cmd/wire@v0.5.0", - "wire gen -tags {} ./pkg/server".format(edition), - ], - "depends_on": [ - "windows-init", + "echo $DRONE_RUNNER_NAME", ], } -def identify_runner_step(platform = "linux"): - if platform == "linux": - return { - "name": "identify-runner", - "image": images["alpine_image"], - "commands": [ - "echo $DRONE_RUNNER_NAME", - ], - } - else: - return { - "name": "identify-runner", - "image": windows_images["1809_image"], - "commands": [ - "echo $env:DRONE_RUNNER_NAME", - ], - } - def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True, isPromote = False): """Setup the enterprise source into the ./grafana-enterprise directory. @@ -135,11 +105,12 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai step = { "name": "clone-enterprise", - "image": images["build_image"], + "image": images["git"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, "commands": [ + "apk add --update curl jq bash", ] + check + [ 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location, "cd {}".format(location), @@ -152,22 +123,10 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai return step -def download_grabpl_step(platform = "linux"): - if platform == "windows": - return { - "name": "grabpl", - "image": windows_images["wix_image"], - "commands": [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ], - } - +def download_grabpl_step(): return { "name": "grabpl", - "image": images["curl_image"], + "image": images["curl"], "commands": [ "mkdir -p bin", "curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format( @@ -180,7 +139,7 @@ def download_grabpl_step(platform = "linux"): def lint_drone_step(): return { "name": "lint-drone", - "image": images["curl_image"], + "image": images["curl"], "commands": [ "./bin/build verify-drone", ], @@ -192,9 +151,10 @@ def lint_drone_step(): def lint_starlark_step(): return { "name": "lint-starlark", - "image": images["build_image"], + "image": images["go"], "commands": [ - "./bin/build verify-starlark .", + "go install github.com/bazelbuild/buildtools/buildifier@latest", + "buildifier --lint=warn -mode=check -r .", ], "depends_on": [ "compile-build-cmd", @@ -219,7 +179,7 @@ def enterprise_downstream_step(ver_mode): step = { "name": "trigger-enterprise-downstream", - "image": images["drone_downstream_image"], + "image": images["drone_downstream"], "settings": { "server": "https://drone.grafana.net", "token": from_secret("drone_token"), @@ -242,8 +202,7 @@ def enterprise_downstream_step(ver_mode): def lint_backend_step(): return { "name": "lint-backend", - # TODO: build_image or go_image? - "image": images["go_image"], + "image": images["go"], "environment": { # We need CGO because of go-sqlite3 "CGO_ENABLED": "1", @@ -252,7 +211,7 @@ def lint_backend_step(): "wire-install", ], "commands": [ - "apt-get update && apt-get install make", + "apk add --update make build-base", # Don't use Make since it will re-download the linters "make lint-go", ], @@ -261,32 +220,28 @@ def lint_backend_step(): def validate_modfile_step(): return { "name": "validate-modfile", - "image": images["go_image"], + "image": images["go"], "commands": [ "go run scripts/modowners/modowners.go check go.mod", ], } -def benchmark_ldap_step(): +def dockerize_step(name, hostname, port): return { - "name": "benchmark-ldap", - "image": images["build_image"], - "environment": { - "LDAP_HOSTNAME": "ldap", - }, + "name": name, + "image": images["dockerize"], "commands": [ - "dockerize -wait tcp://ldap:389 -timeout 120s", - 'go test -benchmem -run=^$ ./pkg/extensions/ldapsync -bench "^(Benchmark50Users)$"', + "dockerize -wait tcp://{}:{} -timeout 120s".format(hostname, port), ], } def build_storybook_step(ver_mode): return { "name": "build-storybook", - "image": images["build_image"], + "image": images["node"], "depends_on": [ # Best to ensure that this step doesn't mess with what's getting built and packaged - "build-frontend", + "rgm-package", "build-frontend-packages", ], "environment": { @@ -327,7 +282,7 @@ def store_storybook_step(ver_mode, trigger = None): step = { "name": "store-storybook", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "build-storybook", ] + @@ -357,7 +312,7 @@ def store_storybook_step(ver_mode, trigger = None): def e2e_tests_artifacts(): return { "name": "e2e-tests-artifacts-upload", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "depends_on": [ "end-to-end-tests-dashboards-suite", "end-to-end-tests-panels-suite", @@ -405,7 +360,7 @@ def upload_cdn_step(ver_mode, trigger = None): step = { "name": "upload-cdn-assets", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "grafana-server", ], @@ -421,75 +376,27 @@ def upload_cdn_step(ver_mode, trigger = None): step = dict(step, when = trigger) return step -def build_backend_step(ver_mode, variants = None): +def build_backend_step(distros = "linux/amd64,linux/arm64"): """Build the backend code using the Grafana build tool. Args: - ver_mode: if ver_mode != 'release', pass the DRONE_BUILD_NUMBER environment - variable as the value for the --build-id option. - TODO: is this option actually used by the build-backend subcommand? - variants: a list of variants be passed to the build-backend subcommand - using the --variants option. - Defaults to None. + distros: a list of distributes to be built. For a full list, see `go tool dist list`. Returns: Drone step. """ - variants_str = "" - if variants: - variants_str = " --variants {}".format(",".join(variants)) - # TODO: Convert number of jobs to percentage - if ver_mode == "release": - cmds = [ - "./bin/build build-backend --jobs 8 --edition oss ${DRONE_TAG}", - ] - else: - build_no = "${DRONE_BUILD_NUMBER}" - cmds = [ - "./bin/build build-backend --jobs 8 --edition oss --build-id {}{}".format( - build_no, - variants_str, - ), - ] + return rgm_build_backend_step(distros) - return { - "name": "build-backend", - "image": images["build_image"], - "depends_on": [ - "wire-install", - "compile-build-cmd", - ], - "commands": cmds, - } - -def build_frontend_step(ver_mode): - """Build the frontend code using the Grafana build tool. - - Args: - ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. +def build_frontend_step(): + """Build the frontend code to ensure it's compilable Returns: Drone step. """ - build_no = "${DRONE_BUILD_NUMBER}" - - # TODO: Use percentage for num jobs - if ver_mode == "release": - cmds = [ - "./bin/build build-frontend --jobs 8 " + - "--edition oss ${DRONE_TAG}", - ] - else: - cmds = [ - "./bin/build build-frontend --jobs 8 --edition oss" + - "--build-id {}".format(build_no), - ] - return { "name": "build-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, @@ -497,43 +404,33 @@ def build_frontend_step(ver_mode): "compile-build-cmd", "yarn-install", ], - "commands": cmds, + "commands": [ + "yarn build", + ], } -def build_frontend_package_step(ver_mode): +def build_frontend_package_step(): """Build the frontend packages using the Grafana build tool. - Args: - ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. - Returns: Drone step. """ - build_no = "${DRONE_BUILD_NUMBER}" - # TODO: Use percentage for num jobs - if ver_mode == "release": - cmds = [ - "./bin/build build-frontend-packages --jobs 8 " + - "--edition oss ${DRONE_TAG}", - ] - else: - cmds = [ - "./bin/build build-frontend-packages --jobs 8 --edition oss" + - "--build-id {}".format(build_no), - "yarn packages:pack", - "./scripts/validate-npm-packages.sh", - ] + cmds = [ + "apk add --update jq bash", # bash is needed for the validate-npm-packages.sh script since it has a 'bash' + # shebang. + "yarn packages:build", + "yarn packages:pack", + "./scripts/validate-npm-packages.sh", + ] return { "name": "build-frontend-packages", - "image": images["build_image"], + "image": images["node"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, "depends_on": [ - "compile-build-cmd", "yarn-install", ], "commands": cmds, @@ -548,42 +445,41 @@ def build_plugins_step(ver_mode): env = None return { "name": "build-plugins", - "image": images["build_image"], + "image": images["node"], "environment": env, "depends_on": [ - "compile-build-cmd", "yarn-install", ], "commands": [ - # TODO: Use percentage for num jobs - "./bin/build build-plugins --jobs 8 --edition oss", + "apk add --update findutils", # Replaces the busybox 'find' with the GNU one. + "yarn plugins:build", ], } -def test_backend_step(image = images["build_image"]): +def test_backend_step(): return { "name": "test-backend", - "image": image, + "image": images["go"], "depends_on": [ "wire-install", ], "commands": [ + # shared-mime-info and shared-mime-info-lang is used for exactly 1 test for the + # mime.TypeByExtension function. + "apk add --update build-base shared-mime-info shared-mime-info-lang", "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", ], } -def windows_test_backend_step(): - step = test_backend_step(image = windows_images["windows_go_image"]) - return step - def test_backend_integration_step(): return { "name": "test-backend-integration", - "image": images["build_image"], + "image": images["go"], "depends_on": [ "wire-install", ], "commands": [ + "apk add --update build-base", "go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", ], } @@ -597,11 +493,12 @@ def betterer_frontend_step(): return { "name": "betterer-frontend", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], "commands": [ + "apk add --update git bash", "yarn betterer ci", ], } @@ -615,7 +512,7 @@ def test_frontend_step(): return { "name": "test-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -630,7 +527,7 @@ def test_frontend_step(): def lint_frontend_step(): return { "name": "lint-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -649,11 +546,12 @@ def verify_i18n_step(): uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again." return { "name": "verify-i18n", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], "commands": [ + "apk add --update git", "yarn run i18n:extract || (echo \"{}\" && false)".format(extract_error_message), # Verify that translation extraction has been committed ''' @@ -701,7 +599,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001): return { "name": "test-a11y-frontend", # TODO which image should be used? - "image": images["docker_puppeteer_image"], + "image": images["docker_puppeteer"], "depends_on": [ "grafana-server", ], @@ -726,7 +624,7 @@ def frontend_metrics_step(trigger = None): """ step = { "name": "publish-frontend-metrics", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "test-a11y-frontend", ], @@ -745,92 +643,38 @@ def frontend_metrics_step(trigger = None): def codespell_step(): return { "name": "codespell", - "image": images["build_image"], + "image": images["python"], "commands": [ + "pip3 install codespell", "codespell -I .codespellignore docs/", ], } -def package_step(ver_mode): - """Packages Grafana with the Grafana build tool. - - Args: - ver_mode: controls whether the packages are signed for a release. - If ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. - - Returns: - Drone step. - """ - deps = [ - "build-plugins", - "build-backend", - "build-frontend", - "build-frontend-packages", - ] - - if ver_mode in ("main", "release", "release-branch"): - sign_args = " --sign" - env = { - "GRAFANA_API_KEY": from_secret("grafana_api_key"), - "GPG_PRIV_KEY": from_secret("packages_gpg_private_key"), - "GPG_PUB_KEY": from_secret("packages_gpg_public_key"), - "GPG_KEY_PASSWORD": from_secret("packages_gpg_passphrase"), - } - test_args = "" - else: - sign_args = "" - env = None - - # TODO: env vars no longer needed by build if not signing - test_args = ". scripts/build/gpg-test-vars.sh && " - - # TODO: Use percentage for jobs - if ver_mode == "release": - cmds = [ - "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + - "{} $${{DRONE_TAG}}".format(sign_args), - ] - else: - build_no = "${DRONE_BUILD_NUMBER}" - cmds = [ - "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + - "--build-id {}{}".format(build_no, sign_args), - ] - - return { - "name": "package", - "image": images["build_image"], - "depends_on": deps, - "environment": env, - "commands": cmds, - } - -def grafana_server_step(port = 3001): +def grafana_server_step(): """Runs the grafana-server binary as a service. - Args: - port: port to listen on. - Defaults to 3001. - Returns: Drone step. """ - environment = {"PORT": port, "ARCH": "linux-amd64"} + environment = { + "GF_SERVER_HTTP_PORT": "3001", + "GF_SERVER_ROUTER_LOGGING": "1", + "GF_APP_MODE": "development", + } return { "name": "grafana-server", - "image": images["build_image"], + "image": images["alpine"], "detach": True, "depends_on": [ - "build-plugins", - "build-backend", - "build-frontend", - "build-frontend-packages", + "rgm-package", ], "environment": environment, "commands": [ - "./scripts/grafana-server/start-server", + "apk add --update tar bash", + "mkdir grafana", + "tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana", + "cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server", ], } @@ -840,7 +684,7 @@ def e2e_tests_step(suite, port = 3001, tries = None): cmd += " --tries {}".format(tries) return { "name": "end-to-end-tests-{}".format(suite), - "image": images["cypress_image"], + "image": images["cypress"], "depends_on": [ "grafana-server", ], @@ -905,7 +749,7 @@ def build_docs_website_step(): return { "name": "build-docs-website", # Use latest revision here, since we want to catch if it breaks - "image": images["docs_image"], + "image": images["docs"], "commands": [ "mkdir -p /hugo/content/docs/grafana/latest", "cp -r docs/sources/* /hugo/content/docs/grafana/latest/", @@ -913,65 +757,10 @@ def build_docs_website_step(): ], } -def copy_packages_for_docker_step(): - return { - "name": "copy-packages-for-docker", - "image": images["build_image"], - "depends_on": [ - "package", - ], - "commands": [ - "ls dist/*.tar.gz*", - "cp dist/*.tar.gz* packaging/docker/", - ], - } - -def build_docker_images_step(archs = None, ubuntu = False, publish = False): - """Build Docker images using the Grafana build tool. - - Args: - archs: a list of architectures to build the image for. - Defaults to None. - ubuntu: controls whether the final image is built from an Ubuntu base image. - Defaults to False. - publish: controls whether the built image is saved to a pre-release repository. - Defaults to False. - - Returns: - Drone step. - """ - cmd = "./bin/build build-docker --edition oss" - if publish: - cmd += " --shouldSave" - - ubuntu_sfx = "" - if ubuntu: - ubuntu_sfx = "-ubuntu" - cmd += " --ubuntu" - - if archs: - cmd += " -archs {}".format(",".join(archs)) - - environment = { - "GCP_KEY": from_secret(gcp_grafanauploads), - } - - return { - "name": "build-docker-images" + ubuntu_sfx, - "image": images["cloudsdk_image"], - "depends_on": [ - "copy-packages-for-docker", - "compile-build-cmd", - ], - "commands": [cmd], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], - "environment": environment, - } - def fetch_images_step(): return { "name": "fetch-images", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads), "DOCKER_USER": from_secret("docker_username"), @@ -1012,7 +801,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): docker_repo, ) - deps = ["build-docker-images", "build-docker-images-ubuntu"] + deps = ["rgm-build-docker"] if ver_mode == "release": deps = ["fetch-images"] cmd += " --version-tag ${DRONE_TAG}" @@ -1028,7 +817,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): step = { "name": "publish-images-{}".format(name), - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": environment, "commands": [cmd], "depends_on": deps, @@ -1041,18 +830,47 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): return step -def integration_tests_step(name, cmds, environment = None): +def integration_tests_steps(name, cmds, hostname = None, port = None, environment = None): + """Integration test steps + + Args: + name: the name of the step. + cmds: the commands to run to perform the integration tests. + hostname: the hostname where the remote server is available. + port: the port where the remote server is available. + environment: Any extra environment variables needed to run the integration tests. + + Returns: + A list of drone steps. If a hostname / port were provided, then a step to wait for the remove server to be + available is also returned. + """ + dockerize_name = "wait-for-{}".format(name) + + depends = [ + "wire-install", + ] + step = { "name": "{}-integration-tests".format(name), - "image": images["build_image"], - "depends_on": ["wire-install"], - "commands": cmds, + "image": images["go"], + "depends_on": depends, + "commands": [ + "apk add --update build-base", + ] + cmds, } if environment: step["environment"] = environment - return step + if hostname == None: + return [step] + + depends = depends.append(dockerize_name) + + return [ + dockerize_step(dockerize_name, hostname, port), + step, + ] def integration_benchmarks_step(name, environment = None): cmds = [ @@ -1060,13 +878,11 @@ def integration_benchmarks_step(name, environment = None): "go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES}", ] - return integration_tests_step("{}-benchmark".format(name), cmds, environment) + return integration_tests_steps("{}-benchmark".format(name), cmds, environment = environment) -def postgres_integration_tests_step(): +def postgres_integration_tests_steps(): cmds = [ - "apt-get update", - "apt-get install -yq postgresql-client", - "dockerize -wait tcp://postgres:5432 -timeout 120s", + "apk add --update postgresql-client", "psql -p 5432 -h postgres -U grafanatest -d grafanatest -f " + "devenv/docker/blocks/postgres_tests/setup.sql", "go clean -testcache", @@ -1079,13 +895,11 @@ def postgres_integration_tests_step(): "POSTGRES_HOST": "postgres", } - return integration_tests_step("postgres", cmds, environment) + return integration_tests_steps("postgres", cmds, "postgres", "5432", environment) -def mysql_integration_tests_step(hostname, version): +def mysql_integration_tests_steps(hostname, version): cmds = [ - "apt-get update", - "apt-get install -yq default-mysql-client", - "dockerize -wait tcp://{}:3306 -timeout 120s".format(hostname), + "apk add --update mysql-client", "cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h {} -P 3306 -u root -prootpass".format(hostname), "go clean -testcache", "go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", @@ -1096,11 +910,10 @@ def mysql_integration_tests_step(hostname, version): "MYSQL_HOST": hostname, } - return integration_tests_step("mysql-{}".format(version), cmds, environment) + return integration_tests_steps("mysql-{}".format(version), cmds, hostname, "3306", environment) -def redis_integration_tests_step(): +def redis_integration_tests_steps(): cmds = [ - "dockerize -wait tcp://redis:6379/0 -timeout 120s", "go clean -testcache", "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...", ] @@ -1109,11 +922,10 @@ def redis_integration_tests_step(): "REDIS_URL": "redis://redis:6379/0", } - return integration_tests_step("redis", cmds, environment) + return integration_tests_steps("redis", cmds, "redis", "6379", environment = environment) -def memcached_integration_tests_step(): +def memcached_integration_tests_steps(): cmds = [ - "dockerize -wait tcp://memcached:11211 -timeout 120s", "go clean -testcache", "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...", ] @@ -1122,7 +934,7 @@ def memcached_integration_tests_step(): "MEMCACHED_HOSTS": "memcached:11211", } - return integration_tests_step("memcached", cmds, environment) + return integration_tests_steps("memcached", cmds, "memcached", "11211", environment) def release_canary_npm_packages_step(trigger = None): """Releases canary NPM packages. @@ -1136,8 +948,8 @@ def release_canary_npm_packages_step(trigger = None): """ step = { "name": "release-canary-npm-packages", - "image": images["build_image"], - "depends_on": end_to_end_tests_deps(), + "image": images["node"], + "depends_on": end_to_end_tests_deps() + ["build-frontend-packages"], "environment": { "NPM_TOKEN": from_secret(npm_token), }, @@ -1173,7 +985,7 @@ def upload_packages_step(ver_mode, trigger = None): """ step = { "name": "upload-packages", - "image": images["publish_image"], + "image": images["publish"], "depends_on": end_to_end_tests_deps(), "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), @@ -1211,7 +1023,7 @@ def publish_grafanacom_step(ver_mode): return { "name": "publish-grafanacom", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "publish-linux-packages-deb", "publish-linux-packages-rpm", @@ -1229,7 +1041,7 @@ def publish_linux_packages_step(package_manager = "deb"): return { "name": "publish-linux-packages-{}".format(package_manager), # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image - "image": images["package_publish_image"], + "image": images["package_publish"], "depends_on": ["compile-build-cmd"], "privileged": True, "settings": { @@ -1247,122 +1059,15 @@ def publish_linux_packages_step(package_manager = "deb"): }, } -def windows_clone_step(): - return { - "name": "clone", - "image": windows_images["wix_image"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', - "git checkout -f $$env:DRONE_COMMIT", - ], - } - -def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): - """Generate the list of Windows steps. - - Args: - ver_mode: used to differentiate steps for different version modes. - bucket: used to override prerelease bucket. - - Returns: - List of Drone steps. - """ - steps = [ - identify_runner_step("windows"), - ] - - init_cmds = [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ] - - steps.extend( - [ - { - "name": "windows-init", - "image": windows_images["wix_image"], - "commands": init_cmds, - }, - ], - ) - - if ver_mode in ( - "release", - "release-branch", - "main", - ): - gcp_bucket = "{}/artifacts/downloads".format(bucket) - if ver_mode == "release": - ver_part = "${DRONE_TAG}" - dir = "release" - else: - dir = "main" - gcp_bucket = "grafana-downloads" - build_no = "DRONE_BUILD_NUMBER" - ver_part = "--build-id $$env:{}".format(build_no) - installer_commands = [ - "$$gcpKey = $$env:GCP_KEY", - "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", - # gcloud fails to read the file unless converted with dos2unix - "dos2unix gcpkey.json", - "gcloud auth activate-service-account --key-file=gcpkey.json", - "rm gcpkey.json", - "cp C:\\App\\nssm-2.24.zip .", - ] - - if ver_mode == "release": - version = "${DRONE_TAG:1}" - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --target {} --edition oss {}".format("gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), ver_part), - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'gsutil cp "$$fname" gs://{}/{}/oss/{}/'.format(gcp_bucket, ver_part, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format(gcp_bucket, ver_part, dir), - ], - ) - if ver_mode in ("main"): - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER", - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'gsutil cp "$$fname" gs://{}/oss/{}/'.format(gcp_bucket, dir), - 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( - gcp_bucket, - dir, - ), - ], - ) - steps.append( - { - "name": "build-windows-installer", - "image": windows_images["wix_image"], - "depends_on": [ - "windows-init", - ], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": installer_commands, - }, - ) - - return steps - def verify_gen_cue_step(): return { "name": "verify-gen-cue", - "image": images["build_image"], + "image": images["go"], "depends_on": [], "commands": [ "# It is required that code generated from Thema/CUE be committed and in sync with its inputs.", "# The following command will fail if running code generators produces any diff in output.", + "apk add --update make", "CODEGEN_VERIFY=1 make gen-cue", ], } @@ -1370,11 +1075,12 @@ def verify_gen_cue_step(): def verify_gen_jsonnet_step(): return { "name": "verify-gen-jsonnet", - "image": images["build_image"], + "image": images["go"], "depends_on": [], "commands": [ "# It is required that generated jsonnet is committed and in sync with its inputs.", "# The following command will fail if running code generators produces any diff in output.", + "apk add --update make", "CODEGEN_VERIFY=1 make gen-jsonnet", ], } @@ -1382,7 +1088,7 @@ def verify_gen_jsonnet_step(): def trigger_test_release(): return { "name": "trigger-test-release", - "image": images["build_image"], + "image": images["git"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), "TEST_TAG": "v0.0.0-test", @@ -1426,7 +1132,7 @@ def compile_build_cmd(): return { "name": "compile-build-cmd", - "image": images["go_image"], + "image": images["go"], "commands": [ "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", ], @@ -1457,3 +1163,14 @@ def get_trigger_storybook(ver_mode): }, } return trigger_storybook + +def slack_step(channel, template, secret): + return { + "name": "slack", + "image": images["plugins_slack"], + "settings": { + "webhook": from_secret(secret), + "channel": channel, + "template": template, + }, + } diff --git a/scripts/drone/steps/lib_windows.star b/scripts/drone/steps/lib_windows.star new file mode 100644 index 00000000000..f9b11bcf85e --- /dev/null +++ b/scripts/drone/steps/lib_windows.star @@ -0,0 +1,187 @@ +""" +This module is a library of Drone steps that exclusively run on windows machines. +""" + +load( + "scripts/drone/utils/windows_images.star", + "windows_images", +) +load( + "scripts/drone/variables.star", + "grabpl_version", +) +load( + "scripts/drone/vault.star", + "from_secret", + "gcp_grafanauploads_base64", + "prerelease_bucket", +) + +def identify_runner_step_windows(): + return { + "name": "identify-runner", + "image": windows_images["1809"], + "commands": [ + "echo $env:DRONE_RUNNER_NAME", + ], + } + +def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): + """Generate the list of Windows steps. + + Args: + ver_mode: used to differentiate steps for different version modes. + bucket: used to override prerelease bucket. + + Returns: + List of Drone steps. + """ + steps = [ + identify_runner_step_windows(), + ] + + init_cmds = [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ] + + steps.extend( + [ + { + "name": "windows-init", + "image": windows_images["wix"], + "commands": init_cmds, + }, + ], + ) + + if ver_mode in ( + "release", + "release-branch", + ): + gcp_bucket = "{}/artifacts/downloads".format(bucket) + if ver_mode == "release": + ver_part = "${DRONE_TAG}" + dir = "release" + else: + dir = "main" + gcp_bucket = "grafana-downloads" + build_no = "DRONE_BUILD_NUMBER" + ver_part = "--build-id $$env:{}".format(build_no) + installer_commands = [ + "$$gcpKey = $$env:GCP_KEY", + "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", + # gcloud fails to read the file unless converted with dos2unix + "dos2unix gcpkey.json", + "gcloud auth activate-service-account --key-file=gcpkey.json", + "rm gcpkey.json", + "cp C:\\App\\nssm-2.24.zip .", + ] + + if ver_mode in ("release",): + version = "${DRONE_TAG:1}" + installer_commands.extend( + [ + ".\\grabpl.exe windows-installer --target {} --edition oss {}".format( + "gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), + ver_part, + ), + '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', + ], + ) + if ver_mode == "main": + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/oss/{}/".format(gcp_bucket, dir), + 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( + gcp_bucket, + dir, + ), + ], + ) + else: + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/{}/oss/{}/".format( + gcp_bucket, + ver_part, + dir, + ), + 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format( + gcp_bucket, + ver_part, + dir, + ), + ], + ) + steps.append( + { + "name": "build-windows-installer", + "image": windows_images["wix"], + "depends_on": [ + "windows-init", + ], + "environment": { + "GCP_KEY": from_secret(gcp_grafanauploads_base64), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": installer_commands, + }, + ) + + return steps + +def download_grabpl_step_windows(): + return { + "name": "grabpl", + "image": windows_images["wix"], + "commands": [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ], + } + +def test_backend_step_windows(): + # TODO: This is mostly a duplicate of "test_backend_step" in lib.star; but this file can't import that one, + # otherwise it creates an import cycle. + return { + "name": "test-backend", + "image": windows_images["go"], + "depends_on": [ + "wire-install", + ], + "commands": [ + "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", + ], + } + +def clone_step_windows(): + return { + "name": "clone", + "image": windows_images["wix"], + "environment": { + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": [ + 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', + "git checkout -f $$env:DRONE_COMMIT", + ], + } + +def wire_install_step_windows(edition): + return { + "name": "wire-install", + "image": windows_images["go"], + "commands": [ + "go install github.com/google/wire/cmd/wire@v0.5.0", + "wire gen -tags {} ./pkg/server".format(edition), + ], + "depends_on": [ + "windows-init", + ], + } diff --git a/scripts/drone/steps/rgm.star b/scripts/drone/steps/rgm.star new file mode 100644 index 00000000000..62b3c26161b --- /dev/null +++ b/scripts/drone/steps/rgm.star @@ -0,0 +1,49 @@ +""" +Individual steps that use 'grafana-build' to replace existing individual steps. +These aren't used in releases. +""" + +# rgm_package_step will create a tar.gz for use in e2e tests or other PR testing related activities.. +def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"): + return { + "name": "rgm-package", + "image": "grafana/grafana-build:main", + "depends_on": ["yarn-install"], + "commands": [ + "/src/grafana-build package --distro={} ".format(distros) + + "--yarn-cache=$$YARN_CACHE_FOLDER " + + "--build-id=$$DRONE_BUILD_NUMBER " + + "--grafana-dir=$$PWD > {}".format(file), + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +# rgm_build_backend will create compile the grafana backend for various platforms. It's preferred to use +# 'rgm_package_step' if you creating a "usable" artifact. This should really only be used to verify that the code is +# compilable. +def rgm_build_backend_step(distros = "linux/amd64,linux/arm64"): + return { + "name": "rgm-package", + "image": "grafana/grafana-build:main", + "commands": [ + "/src/grafana-build build --distro={} --grafana-dir=$$PWD".format(distros), + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +def rgm_build_docker_step(packages, ubuntu, alpine, depends_on = ["rgm-package"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"): + return { + "name": "rgm-build-docker", + "image": "grafana/grafana-build:main", + "commands": [ + "/src/grafana-build docker " + + "--package=$(cat {} | grep tar.gz | grep -v docker | grep -v sha256) ".format(packages) + + "--ubuntu-base={} ".format(ubuntu) + + "--alpine-base={} ".format(alpine) + + "--tag-format='{}' ".format(tag_format) + + "--ubuntu-tag-format='{}' > {}".format(ubuntu_tag_format, file), + "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i", + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "depends_on": depends_on, + } diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index 5cec9c9b006..9c38d3c06e8 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -2,23 +2,34 @@ This module contains all the docker images that are used to build test and publish Grafana. """ +load( + "scripts/drone/variables.star", + "golang_version", + "nodejs_version", +) + images = { - "cloudsdk_image": "google/cloud-sdk:431.0.0", - "build_image": "grafana/build-container:1.7.5", - "publish_image": "grafana/grafana-ci-deploy:1.3.3", - "alpine_image": "alpine:3.17.1", - "curl_image": "byrnedo/alpine-curl:0.1.8", - "go_image": "golang:1.20.6", - "plugins_slack_image": "plugins/slack", - "postgres_alpine_image": "postgres:12.3-alpine", - "mysql5_image": "mysql:5.7.39", - "mysql8_image": "mysql:8.0.32", - "redis_alpine_image": "redis:6.2.11-alpine", - "memcached_alpine_image": "memcached:1.6.9-alpine", - "package_publish_image": "us.gcr.io/kubernetes-dev/package-publish:latest", - "openldap_image": "osixia/openldap:1.4.0", - "drone_downstream_image": "grafana/drone-downstream", - "docker_puppeteer_image": "grafana/docker-puppeteer:1.1.0", - "docs_image": "grafana/docs-base:dbd975af06", - "cypress_image": "cypress/included:12.15.0", + "git": "alpine/git:2.40.1", + "go": "golang:{}-alpine".format(golang_version), + "node": "node:{}-alpine".format(nodejs_version), + "cloudsdk": "google/cloud-sdk:431.0.0", + "publish": "grafana/grafana-ci-deploy:1.3.3", + "alpine": "alpine:3.17.1", + "ubuntu": "ubuntu:22.10", + "curl": "byrnedo/alpine-curl:0.1.8", + "plugins_slack": "plugins/slack", + "python": "python:3.8", + "postgres_alpine": "postgres:12.3-alpine", + "mysql5": "mysql:5.7.39", + "mysql8": "mysql:8.0.32", + "redis_alpine": "redis:6.2.11-alpine", + "memcached_alpine": "memcached:1.6.9-alpine", + "package_publish": "us.gcr.io/kubernetes-dev/package-publish:latest", + "openldap": "osixia/openldap:1.4.0", + "drone_downstream": "grafana/drone-downstream", + "docker_puppeteer": "grafana/docker-puppeteer:1.1.0", + "docs": "grafana/docs-base:dbd975af06", + "cypress": "cypress/included:12.15.0", + "dockerize": "jwilder/dockerize:0.6.1", + "shellcheck": "koalaman/shellcheck:stable", } diff --git a/scripts/drone/utils/windows_images.star b/scripts/drone/utils/windows_images.star index 2a605727ba8..a25c637a11e 100644 --- a/scripts/drone/utils/windows_images.star +++ b/scripts/drone/utils/windows_images.star @@ -4,9 +4,14 @@ All the windows images needed to be in a different file than the other images, s by trivy. Related issue: https://github.com/aquasecurity/trivy/issues/1392 """ +load( + "scripts/drone/variables.star", + "golang_version", +) + windows_images = { - "1809_image": "mcr.microsoft.com/windows:1809", - "wix_image": "grafana/ci-wix:0.1.1", - "windows_server_core_image": "docker:windowsservercore-1809", - "windows_go_image": "grafana/grafana-ci-windows-test:0.1.0", + "1809": "mcr.microsoft.com/windows:1809", + "wix": "grafana/ci-wix:0.1.1", + "windows_server_core": "docker:windowsservercore-1809", + "go": "golang:{}-windowsservercore-1809".format(golang_version), } diff --git a/scripts/drone/variables.star b/scripts/drone/variables.star new file mode 100644 index 00000000000..ac3fc09961b --- /dev/null +++ b/scripts/drone/variables.star @@ -0,0 +1,9 @@ +""" +global variables +""" + +grabpl_version = "v3.0.41" +golang_version = "1.21.1" + +# nodejs_version should match what's in ".nvmrc", but without the v prefix. +nodejs_version = "18.12.0" diff --git a/scripts/drone/version.star b/scripts/drone/version.star deleted file mode 100644 index 140cd4d35d8..00000000000 --- a/scripts/drone/version.star +++ /dev/null @@ -1,16 +0,0 @@ -""" -This module returns the pipeline used for version branches. -""" - -load( - "scripts/drone/events/release.star", - "oss_pipelines", -) - -ver_mode = "release-branch" -trigger = {"ref": ["refs/heads/v[0-9]*"]} - -def version_branch_pipelines(): - return ( - oss_pipelines(ver_mode = ver_mode, trigger = trigger) - )