From 90892050d094dcaef81211ae54b1eebb2b6bc15d Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Thu, 14 Apr 2022 14:56:14 +0300 Subject: [PATCH] CI: Remove `initialize` step (#47622) * Remove initialize step from pr.star * Remove initialize step from main.star * Remove initialize step from release.star * Reorder dependencies * Re-add identify runner step * Add enterprise steps * Remove is_downstream variable from starlark logic (#47715) * Make init steps depend on init-enterprise for enterprise pipelines * Exclude shellcheck from enterprise pipelines * Add init steps to enterprise integration tests pipeline * Add dependency for windows enterprise step * Update grabpl version --- .drone.yml | 647 ++++++++++++++++----------- scripts/drone/pipelines/docs.star | 17 +- scripts/drone/pipelines/main.star | 52 ++- scripts/drone/pipelines/pr.star | 20 +- scripts/drone/pipelines/release.star | 73 ++- scripts/drone/steps/lib.star | 345 +++++++------- 6 files changed, 645 insertions(+), 509 deletions(-) diff --git a/.drone.yml b/.drone.yml index 4fc307c5c50..5ef3316da98 100644 --- a/.drone.yml +++ b/.drone.yml @@ -9,22 +9,32 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version - commands: - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + image: grafana/build-container:1.5.3 + name: wire-install +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install - commands: - ./bin/grabpl verify-drone depends_on: @@ -41,20 +51,18 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - depends_on: - - initialize image: grafana/build-container:1.5.3 name: codespell - commands: - ./bin/grabpl shellcheck depends_on: - - initialize + - grabpl image: grafana/build-container:1.5.3 name: shellcheck - commands: - ./bin/grabpl lint-backend --edition oss depends_on: - - initialize + - wire-install environment: CGO_ENABLED: "1" image: grafana/build-container:1.5.3 @@ -65,7 +73,7 @@ steps: - yarn run i18n:compile - yarn run typecheck depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -73,19 +81,19 @@ steps: - commands: - ./bin/grabpl test-backend --edition oss depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend - commands: - ./bin/grabpl integration-tests --edition oss depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-integration - commands: - yarn run ci:test-frontend depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -112,34 +120,46 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version - commands: - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + image: grafana/build-container:1.5.3 + name: wire-install +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install - commands: - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --variants linux-amd64,linux-amd64-musl,darwin-amd64,windows-amd64,armv6 depends_on: - - initialize + - gen-version + - wire-install environment: {} image: grafana/build-container:1.5.3 name: build-backend - commands: - ./bin/grabpl build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -147,7 +167,8 @@ steps: - commands: - ./bin/grabpl build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -155,7 +176,8 @@ steps: - commands: - ./bin/grabpl build-plugins --jobs 8 --edition oss depends_on: - - initialize + - gen-version + - yarn-install environment: null image: grafana/build-container:1.5.3 name: build-plugins @@ -356,10 +378,14 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - apt-get update - apt-get install -yq postgresql-client @@ -420,7 +446,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -429,11 +455,17 @@ steps: image: alpine:3.15 name: identify-runner - commands: - - make gen-go - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install - commands: - |- echo -e "unknwon @@ -444,14 +476,12 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - depends_on: - - initialize image: grafana/build-container:1.5.3 name: codespell - commands: - yarn run prettier:checkDocs depends_on: - - initialize + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -459,7 +489,8 @@ steps: - commands: - ./bin/grabpl build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -503,7 +534,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -512,11 +543,17 @@ steps: image: alpine:3.15 name: identify-runner - commands: - - make gen-go - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install - commands: - |- echo -e "unknwon @@ -527,14 +564,12 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - depends_on: - - initialize image: grafana/build-container:1.5.3 name: codespell - commands: - yarn run prettier:checkDocs depends_on: - - initialize + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -542,7 +577,8 @@ steps: - commands: - ./bin/grabpl build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -581,22 +617,32 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version - commands: - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + image: grafana/build-container:1.5.3 + name: wire-install +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install - commands: - ./bin/grabpl verify-drone depends_on: @@ -613,20 +659,18 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - depends_on: - - initialize image: grafana/build-container:1.5.3 name: codespell - commands: - ./bin/grabpl shellcheck depends_on: - - initialize + - grabpl image: grafana/build-container:1.5.3 name: shellcheck - commands: - ./bin/grabpl lint-backend --edition oss depends_on: - - initialize + - wire-install environment: CGO_ENABLED: "1" image: grafana/build-container:1.5.3 @@ -637,7 +681,7 @@ steps: - yarn run i18n:compile - yarn run typecheck depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -645,19 +689,19 @@ steps: - commands: - ./bin/grabpl test-backend --edition oss depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend - commands: - ./bin/grabpl integration-tests --edition oss depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-integration - commands: - yarn run ci:test-frontend depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -682,22 +726,32 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version - commands: - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + image: grafana/build-container:1.5.3 + name: wire-install +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install - commands: - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1 @@ -739,14 +793,16 @@ steps: - commands: - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - wire-install environment: {} image: grafana/build-container:1.5.3 name: build-backend - commands: - ./bin/grabpl build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -754,7 +810,8 @@ steps: - commands: - ./bin/grabpl build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -762,7 +819,8 @@ steps: - commands: - ./bin/grabpl build-plugins --jobs 8 --edition oss --sign --signing-admin depends_on: - - initialize + - gen-version + - yarn-install environment: GRAFANA_API_KEY: from_secret: grafana_api_key @@ -1109,10 +1167,14 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - apt-get update - apt-get install -yq postgresql-client @@ -1177,10 +1239,10 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 - name: initialize + name: windows-init - commands: - $$gcpKey = $$env:GCP_KEY - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) @@ -1196,7 +1258,7 @@ steps: - gsutil cp $$fname gs://grafana-downloads/oss/main/ - gsutil cp "$$fname.sha256" gs://grafana-downloads/oss/main/ depends_on: - - initialize + - windows-init environment: GCP_KEY: from_secret: gcp_key @@ -1264,7 +1326,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1272,10 +1334,6 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.15 name: identify-runner -- commands: - - make gen-go - image: grafana/build-container:1.5.3 - name: initialize - commands: - ./bin/grabpl store-packages --edition oss --gcp-key /tmp/gcpkey.json --build-id ${DRONE_BUILD_NUMBER} @@ -1346,28 +1404,38 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner + - ./bin/grabpl gen-version ${DRONE_TAG} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version - commands: - make gen-go - - ./bin/grabpl verify-version ${DRONE_TAG} - - ./bin/grabpl gen-version ${DRONE_TAG} + image: grafana/build-container:1.5.3 + name: wire-install +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install - commands: - ./bin/grabpl build-backend --jobs 8 --edition oss --github-token $${GITHUB_TOKEN} ${DRONE_TAG} depends_on: - - initialize + - gen-version + - wire-install environment: GITHUB_TOKEN: from_secret: github_token @@ -1377,7 +1445,8 @@ steps: - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --edition oss ${DRONE_TAG} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -1386,7 +1455,8 @@ steps: - ./bin/grabpl build-frontend-packages --jobs 8 --github-token $${GITHUB_TOKEN} --edition oss ${DRONE_TAG} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -1394,7 +1464,8 @@ steps: - commands: - ./bin/grabpl build-plugins --jobs 8 --edition oss --sign --signing-admin depends_on: - - initialize + - gen-version + - yarn-install environment: GRAFANA_API_KEY: from_secret: grafana_api_key @@ -1648,23 +1719,38 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner + - ./bin/grabpl gen-version ${DRONE_TAG} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version - commands: - make gen-go - - ./bin/grabpl verify-version ${DRONE_TAG} - - ./bin/grabpl gen-version ${DRONE_TAG} + image: grafana/build-container:1.5.3 + name: wire-install +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install +- commands: + - ./bin/grabpl shellcheck + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: shellcheck - commands: - |- echo -e "unknwon @@ -1675,20 +1761,12 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - depends_on: - - initialize image: grafana/build-container:1.5.3 name: codespell -- commands: - - ./bin/grabpl shellcheck - depends_on: - - initialize - image: grafana/build-container:1.5.3 - name: shellcheck - commands: - ./bin/grabpl lint-backend --edition oss depends_on: - - initialize + - wire-install environment: CGO_ENABLED: "1" image: grafana/build-container:1.5.3 @@ -1699,7 +1777,7 @@ steps: - yarn run i18n:compile - yarn run typecheck depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -1707,19 +1785,19 @@ steps: - commands: - ./bin/grabpl test-backend --edition oss depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend - commands: - ./bin/grabpl integration-tests --edition oss depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-integration - commands: - yarn run ci:test-frontend depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -1771,7 +1849,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1779,13 +1857,6 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.15 name: identify-runner -- commands: - - make gen-go - - ./bin/grabpl verify-version ${DRONE_TAG} - - ./bin/grabpl gen-version ${DRONE_TAG} - - yarn install --immutable - image: grafana/build-container:1.5.3 - name: initialize - commands: - apt-get update - apt-get install -yq postgresql-client @@ -1855,10 +1926,10 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 - name: initialize + name: windows-init - commands: - $$gcpKey = $$env:GCP_KEY - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) @@ -1873,7 +1944,7 @@ steps: - gsutil cp $$fname gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/oss/release/ - gsutil cp "$$fname.sha256" gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/oss/release/ depends_on: - - initialize + - windows-init environment: GCP_KEY: from_secret: gcp_key @@ -1914,7 +1985,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1940,22 +2011,37 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl verify-version ${DRONE_TAG} - - ./bin/grabpl gen-version ${DRONE_TAG} - - yarn install --immutable depends_on: - clone-enterprise environment: GITHUB_TOKEN: from_secret: github_token image: grafana/build-container:1.5.3 - name: initialize + name: init-enterprise +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: yarn-install +- commands: + - ./bin/grabpl gen-version ${DRONE_TAG} + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: gen-version - commands: - ./bin/grabpl build-backend --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} ${DRONE_TAG} depends_on: - - initialize + - gen-version + - wire-install environment: GITHUB_TOKEN: from_secret: github_token @@ -1965,7 +2051,8 @@ steps: - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --edition enterprise ${DRONE_TAG} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -1974,7 +2061,8 @@ steps: - ./bin/grabpl build-frontend-packages --jobs 8 --github-token $${GITHUB_TOKEN} --edition enterprise ${DRONE_TAG} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -1982,7 +2070,8 @@ steps: - commands: - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --sign --signing-admin depends_on: - - initialize + - gen-version + - yarn-install environment: GRAFANA_API_KEY: from_secret: grafana_api_key @@ -2011,7 +2100,8 @@ steps: - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} ${DRONE_TAG} depends_on: - - initialize + - gen-version + - wire-install environment: GITHUB_TOKEN: from_secret: github_token @@ -2268,7 +2358,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2294,17 +2384,31 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl verify-version ${DRONE_TAG} - - ./bin/grabpl gen-version ${DRONE_TAG} - - yarn install --immutable depends_on: - clone-enterprise environment: GITHUB_TOKEN: from_secret: github_token image: grafana/build-container:1.5.3 - name: initialize + name: init-enterprise +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: yarn-install +- commands: + - ./bin/grabpl gen-version ${DRONE_TAG} + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: gen-version - commands: - |- echo -e "unknwon @@ -2315,20 +2419,12 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - depends_on: - - initialize image: grafana/build-container:1.5.3 name: codespell -- commands: - - ./bin/grabpl shellcheck - depends_on: - - initialize - image: grafana/build-container:1.5.3 - name: shellcheck - commands: - ./bin/grabpl lint-backend --edition enterprise depends_on: - - initialize + - wire-install environment: CGO_ENABLED: "1" image: grafana/build-container:1.5.3 @@ -2339,7 +2435,7 @@ steps: - yarn run i18n:compile - yarn run typecheck depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -2347,19 +2443,19 @@ steps: - commands: - ./bin/grabpl test-backend --edition enterprise depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend - commands: - ./bin/grabpl integration-tests --edition enterprise depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-integration - commands: - yarn run ci:test-frontend depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -2367,7 +2463,7 @@ steps: - commands: - ./bin/grabpl lint-backend --edition enterprise2 depends_on: - - initialize + - wire-install environment: CGO_ENABLED: "1" image: grafana/build-container:1.5.3 @@ -2375,13 +2471,13 @@ steps: - commands: - ./bin/grabpl test-backend --edition enterprise2 depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-enterprise2 - commands: - ./bin/grabpl integration-tests --edition enterprise2 depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-integration-enterprise2 trigger: @@ -2441,7 +2537,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2467,17 +2563,13 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl verify-version ${DRONE_TAG} - - ./bin/grabpl gen-version ${DRONE_TAG} - - yarn install --immutable depends_on: - clone-enterprise environment: GITHUB_TOKEN: from_secret: github_token image: grafana/build-container:1.5.3 - name: initialize + name: init-enterprise - commands: - apt-get update - apt-get install -yq postgresql-client @@ -2486,7 +2578,7 @@ steps: - go clean -testcache - ./bin/grabpl integration-tests --database postgres depends_on: - - initialize + - init-enterprise environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest @@ -2502,7 +2594,7 @@ steps: - go clean -testcache - ./bin/grabpl integration-tests --database mysql depends_on: - - initialize + - init-enterprise environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql @@ -2512,7 +2604,7 @@ steps: - dockerize -wait tcp://redis:6379/0 -timeout 120s - ./bin/grabpl integration-tests depends_on: - - initialize + - init-enterprise environment: REDIS_URL: redis://redis:6379/0 image: grafana/build-container:1.5.3 @@ -2521,7 +2613,7 @@ steps: - dockerize -wait tcp://memcached:11211 -timeout 120s - ./bin/grabpl integration-tests depends_on: - - initialize + - init-enterprise environment: MEMCACHED_HOSTS: memcached:11211 image: grafana/build-container:1.5.3 @@ -2569,7 +2661,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -2592,7 +2684,7 @@ steps: GITHUB_TOKEN: from_secret: github_token image: grafana/ci-wix:0.1.1 - name: initialize + name: windows-init - commands: - $$gcpKey = $$env:GCP_KEY - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) @@ -2607,7 +2699,7 @@ steps: - gsutil cp $$fname gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/enterprise/release/ - gsutil cp "$$fname.sha256" gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/enterprise/release/ depends_on: - - initialize + - windows-init environment: GCP_KEY: from_secret: gcp_key @@ -2644,7 +2736,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2722,7 +2814,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2783,7 +2875,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2862,7 +2954,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2924,7 +3016,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2962,7 +3054,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2998,27 +3090,16 @@ platform: os: linux services: [] steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner -- commands: - - make gen-go - - ./bin/grabpl verify-version ${DRONE_TAG} - - ./bin/grabpl gen-version ${DRONE_TAG} - - yarn install --immutable - image: grafana/build-container:1.5.3 - name: initialize - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - ./bin/grabpl artifacts npm retrieve --tag v${TAG} depends_on: - - initialize + - grabpl environment: GCP_KEY: from_secret: gcp_key @@ -3061,7 +3142,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3122,33 +3203,45 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version - commands: - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + image: grafana/build-container:1.5.3 + name: wire-install +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install - commands: - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - wire-install environment: {} image: grafana/build-container:1.5.3 name: build-backend - commands: - ./bin/grabpl build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -3156,7 +3249,8 @@ steps: - commands: - ./bin/grabpl build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -3164,7 +3258,8 @@ steps: - commands: - ./bin/grabpl build-plugins --jobs 8 --edition oss --sign --signing-admin depends_on: - - initialize + - gen-version + - yarn-install environment: GRAFANA_API_KEY: from_secret: grafana_api_key @@ -3387,22 +3482,38 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: gen-version - commands: - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + image: grafana/build-container:1.5.3 + name: wire-install +- commands: - yarn install --immutable + depends_on: + - grabpl image: grafana/build-container:1.5.3 - name: initialize + name: yarn-install +- commands: + - ./bin/grabpl shellcheck + depends_on: + - grabpl + image: grafana/build-container:1.5.3 + name: shellcheck - commands: - |- echo -e "unknwon @@ -3413,20 +3524,12 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - depends_on: - - initialize image: grafana/build-container:1.5.3 name: codespell -- commands: - - ./bin/grabpl shellcheck - depends_on: - - initialize - image: grafana/build-container:1.5.3 - name: shellcheck - commands: - ./bin/grabpl lint-backend --edition oss depends_on: - - initialize + - wire-install environment: CGO_ENABLED: "1" image: grafana/build-container:1.5.3 @@ -3437,7 +3540,7 @@ steps: - yarn run i18n:compile - yarn run typecheck depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -3445,19 +3548,19 @@ steps: - commands: - ./bin/grabpl test-backend --edition oss depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend - commands: - ./bin/grabpl integration-tests --edition oss depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-integration - commands: - yarn run ci:test-frontend depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -3503,7 +3606,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3511,12 +3614,6 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.15 name: identify-runner -- commands: - - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} - - yarn install --immutable - image: grafana/build-container:1.5.3 - name: initialize - commands: - apt-get update - apt-get install -yq postgresql-client @@ -3580,10 +3677,10 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 - name: initialize + name: windows-init - commands: - $$gcpKey = $$env:GCP_KEY - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) @@ -3593,7 +3690,7 @@ steps: - rm gcpkey.json - cp C:\App\nssm-2.24.zip . depends_on: - - initialize + - windows-init environment: GCP_KEY: from_secret: gcp_key @@ -3628,7 +3725,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3653,25 +3750,42 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} - - yarn install --immutable depends_on: - clone-enterprise environment: {} image: grafana/build-container:1.5.3 - name: initialize + name: init-enterprise +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: yarn-install +- commands: + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: gen-version - commands: - ./bin/grabpl build-backend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - wire-install environment: {} image: grafana/build-container:1.5.3 name: build-backend - commands: - ./bin/grabpl build-frontend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -3680,7 +3794,8 @@ steps: - ./bin/grabpl build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - gen-version + - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 image: grafana/build-container:1.5.3 @@ -3688,7 +3803,8 @@ steps: - commands: - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --sign --signing-admin depends_on: - - initialize + - gen-version + - yarn-install environment: GRAFANA_API_KEY: from_secret: grafana_api_key @@ -3717,7 +3833,8 @@ steps: - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} --variants linux-amd64 depends_on: - - initialize + - gen-version + - wire-install environment: {} image: grafana/build-container:1.5.3 name: build-backend-enterprise2 @@ -3969,7 +4086,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3994,14 +4111,29 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} - - yarn install --immutable depends_on: - clone-enterprise environment: {} image: grafana/build-container:1.5.3 - name: initialize + name: init-enterprise +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: yarn-install +- commands: + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - init-enterprise + image: grafana/build-container:1.5.3 + name: gen-version - commands: - |- echo -e "unknwon @@ -4012,20 +4144,12 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - depends_on: - - initialize image: grafana/build-container:1.5.3 name: codespell -- commands: - - ./bin/grabpl shellcheck - depends_on: - - initialize - image: grafana/build-container:1.5.3 - name: shellcheck - commands: - ./bin/grabpl lint-backend --edition enterprise depends_on: - - initialize + - wire-install environment: CGO_ENABLED: "1" image: grafana/build-container:1.5.3 @@ -4036,7 +4160,7 @@ steps: - yarn run i18n:compile - yarn run typecheck depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -4044,19 +4168,19 @@ steps: - commands: - ./bin/grabpl test-backend --edition enterprise depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend - commands: - ./bin/grabpl integration-tests --edition enterprise depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-integration - commands: - yarn run ci:test-frontend depends_on: - - initialize + - yarn-install environment: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.5.3 @@ -4064,7 +4188,7 @@ steps: - commands: - ./bin/grabpl lint-backend --edition enterprise2 depends_on: - - initialize + - wire-install environment: CGO_ENABLED: "1" image: grafana/build-container:1.5.3 @@ -4072,13 +4196,13 @@ steps: - commands: - ./bin/grabpl test-backend --edition enterprise2 depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-enterprise2 - commands: - ./bin/grabpl integration-tests --edition enterprise2 depends_on: - - initialize + - wire-install image: grafana/build-container:1.5.3 name: test-backend-integration-enterprise2 trigger: @@ -4132,7 +4256,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4157,14 +4281,11 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} - - yarn install --immutable depends_on: - clone-enterprise environment: {} image: grafana/build-container:1.5.3 - name: initialize + name: init-enterprise - commands: - apt-get update - apt-get install -yq postgresql-client @@ -4173,7 +4294,7 @@ steps: - go clean -testcache - ./bin/grabpl integration-tests --database postgres depends_on: - - initialize + - init-enterprise environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest @@ -4189,7 +4310,7 @@ steps: - go clean -testcache - ./bin/grabpl integration-tests --database mysql depends_on: - - initialize + - init-enterprise environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql @@ -4199,7 +4320,7 @@ steps: - dockerize -wait tcp://redis:6379/0 -timeout 120s - ./bin/grabpl integration-tests depends_on: - - initialize + - init-enterprise environment: REDIS_URL: redis://redis:6379/0 image: grafana/build-container:1.5.3 @@ -4208,7 +4329,7 @@ steps: - dockerize -wait tcp://memcached:11211 -timeout 120s - ./bin/grabpl integration-tests depends_on: - - initialize + - init-enterprise environment: MEMCACHED_HOSTS: memcached:11211 image: grafana/build-container:1.5.3 @@ -4250,7 +4371,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.34/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.9.35/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -4273,7 +4394,7 @@ steps: GITHUB_TOKEN: from_secret: github_token image: grafana/ci-wix:0.1.1 - name: initialize + name: windows-init - commands: - $$gcpKey = $$env:GCP_KEY - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) @@ -4283,7 +4404,7 @@ steps: - rm gcpkey.json - cp C:\App\nssm-2.24.zip . depends_on: - - initialize + - windows-init environment: GCP_KEY: from_secret: gcp_key @@ -4447,6 +4568,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: 48fc25b65552d02f3d75f3ad1e24a9854c0485bebac8d4e062cfa8605e0a75ba +hmac: 6b2cdb67f637bdd527ba527741fd7d0af7e61b0700487be912821b93d8d16ca1 ... diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index fc6ce20cfa7..ababa390740 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -1,7 +1,9 @@ load( 'scripts/drone/steps/lib.star', 'build_image', - 'initialize_step', + 'yarn_install_step', + 'identify_runner_step', + 'gen_version_step', 'download_grabpl_step', 'lint_frontend_step', 'codespell_step', @@ -25,16 +27,17 @@ load( def docs_pipelines(edition, ver_mode, trigger): - steps = [download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) - - # Insert remaining steps - steps.extend([ + steps = [ + download_grabpl_step(), + identify_runner_step(), + gen_version_step(ver_mode), + yarn_install_step(), codespell_step(), lint_docs(), build_frontend_package_step(edition=edition, ver_mode=ver_mode), build_frontend_docs_step(edition=edition), build_docs_website_step(), - ]) + ] return pipeline( name='{}-docs'.format(ver_mode), edition=edition, trigger=trigger, services=[], steps=steps, @@ -45,7 +48,7 @@ def lint_docs(): 'name': 'lint-docs', 'image': build_image, 'depends_on': [ - 'initialize', + 'yarn-install', ], 'environment': { 'NODE_OPTIONS': '--max_old_space_size=8192', diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star index fb3f671cdcb..e86e337593f 100644 --- a/scripts/drone/pipelines/main.star +++ b/scripts/drone/pipelines/main.star @@ -2,7 +2,10 @@ load( 'scripts/drone/steps/lib.star', 'download_grabpl_step', 'build_image', - 'initialize_step', + 'identify_runner_step', + 'gen_version_step', + 'wire_install_step', + 'yarn_install_step', 'lint_drone_step', 'lint_backend_step', 'lint_frontend_step', @@ -67,10 +70,16 @@ load('scripts/drone/vault.star', 'from_secret') ver_mode = 'main' -def get_steps(edition, is_downstream=False): +def get_steps(edition): services = integration_test_services(edition) - publish = edition != 'enterprise' or is_downstream include_enterprise2 = edition == 'enterprise' + init_steps = [ + identify_runner_step(), + download_grabpl_step(), + gen_version_step(ver_mode), + wire_install_step(), + yarn_install_step(), + ] test_steps = [ lint_drone_step(), codespell_step(), @@ -84,9 +93,9 @@ def get_steps(edition, is_downstream=False): build_steps = [ trigger_test_release(), enterprise_downstream_step(edition=edition), - build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), - build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), - build_frontend_package_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), + build_backend_step(edition=edition, ver_mode=ver_mode), + build_frontend_step(edition=edition, ver_mode=ver_mode), + build_frontend_package_step(edition=edition, ver_mode=ver_mode), build_plugins_step(edition=edition, sign=True), validate_scuemata_step(), ensure_cuetsified_step(), @@ -106,12 +115,12 @@ def get_steps(edition, is_downstream=False): test_backend_integration_step(edition=edition2), ]) build_steps.extend([ - build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64'], is_downstream=is_downstream), + build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64']), ]) # Insert remaining steps build_steps.extend([ - package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream), + package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2), grafana_server_step(edition=edition), e2e_tests_step('dashboards-suite', edition=edition), e2e_tests_step('smoke-tests-suite', edition=edition), @@ -134,27 +143,27 @@ def get_steps(edition, is_downstream=False): build_steps.extend([ release_canary_npm_packages_step(edition, trigger=trigger_oss), - upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream, trigger=trigger_oss), + upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss) ]) if include_enterprise2: edition2 = 'enterprise2' build_steps.extend([ - package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-amd64'], is_downstream=is_downstream), - upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream), + package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-amd64']), + upload_packages_step(edition=edition2, ver_mode=ver_mode), upload_cdn_step(edition=edition2, ver_mode=ver_mode) ]) - windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream) - if edition == 'enterprise' and not is_downstream: + windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode) + if edition == 'enterprise': store_steps = [] else: store_steps = [ - store_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), + store_packages_step(edition=edition, ver_mode=ver_mode), ] - return test_steps, build_steps, integration_test_steps, windows_steps, store_steps + return init_steps, test_steps, build_steps, integration_test_steps, windows_steps, store_steps def trigger_test_release(): return { @@ -210,7 +219,7 @@ def main_pipelines(edition): ], }, } - test_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition) + init_steps, test_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition) if edition == 'enterprise': services.append(ldap_service()) @@ -218,27 +227,26 @@ def main_pipelines(edition): pipelines = [docs_pipelines(edition, ver_mode, trigger), pipeline( name='main-test', edition=edition, trigger=trigger, services=[], - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + test_steps, + steps=init_steps + test_steps, volumes=[], ), pipeline( name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[], - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps, + steps=init_steps + build_steps, volumes=volumes, ), pipeline( name='main-integration-tests', edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step()] + integration_test_steps, + steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps, volumes=volumes, ), pipeline( name='windows-main', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), - steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps, + steps=[identify_runner_step('windows')] + windows_steps, depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows', ), notify_pipeline( name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, template=drone_change_template, secret='drone-changes-webhook', ), pipeline( name='publish-main', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, - install_deps=False) + store_steps, + steps=[download_grabpl_step(), identify_runner_step(),] + store_steps, depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', ], ), notify_pipeline( name='notify-main', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']), diff --git a/scripts/drone/pipelines/pr.star b/scripts/drone/pipelines/pr.star index 1f768f1f5a2..d14425cc84e 100644 --- a/scripts/drone/pipelines/pr.star +++ b/scripts/drone/pipelines/pr.star @@ -1,7 +1,10 @@ load( 'scripts/drone/steps/lib.star', 'download_grabpl_step', - 'initialize_step', + 'gen_version_step', + 'yarn_install_step', + 'wire_install_step', + 'identify_runner_step', 'lint_drone_step', 'lint_backend_step', 'lint_frontend_step', @@ -59,6 +62,13 @@ def pr_pipelines(edition): volumes = integration_test_services_volumes() variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64', 'armv6',] include_enterprise2 = edition == 'enterprise' + init_steps = [ + identify_runner_step(), + download_grabpl_step(), + gen_version_step(ver_mode), + wire_install_step(), + yarn_install_step(), + ] test_steps = [ lint_drone_step(), codespell_step(), @@ -132,14 +142,12 @@ def pr_pipelines(edition): return [ pipeline( - name='pr-test', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) - + test_steps, + name='pr-test', edition=edition, trigger=trigger, services=[], steps=init_steps + test_steps, ), pipeline( - name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) - + build_steps, + name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=init_steps + build_steps, ), pipeline( name='pr-integration-tests', edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step()] + integration_test_steps, + steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps, volumes=volumes, ), docs_pipelines(edition, ver_mode, trigger_docs()) ] diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index 0e10c9892fa..cb7a8e9cb6b 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -1,10 +1,15 @@ load( 'scripts/drone/steps/lib.star', 'disable_tests', + 'clone_enterprise_step', 'download_grabpl_step', - 'initialize_step', + 'gen_version_step', + 'yarn_install_step', + 'wire_install_step', + 'init_enterprise_step', 'lint_drone_step', 'build_image', + 'identify_runner_step', 'publish_image', 'lint_backend_step', 'lint_frontend_step', @@ -77,7 +82,7 @@ def retrieve_npm_packages_step(): 'name': 'retrieve-npm-packages', 'image': publish_image, 'depends_on': [ - 'initialize', + 'grabpl', ], 'environment': { 'GCP_KEY': from_secret('gcp_key'), @@ -150,16 +155,26 @@ def get_steps(edition, ver_mode): should_upload = should_publish or ver_mode in ('release-branch',) include_enterprise2 = edition == 'enterprise' edition2 = 'enterprise2' + init_steps = [ + identify_runner_step(), + download_grabpl_step(), + gen_version_step(ver_mode), + wire_install_step(), + yarn_install_step(), + ] + + test_steps = [] + if edition != 'enterprise': + test_steps.extend([shellcheck_step()]) - test_steps = [ + test_steps.extend([ codespell_step(), - shellcheck_step(), lint_backend_step(edition=edition), lint_frontend_step(), test_backend_step(edition=edition), test_backend_integration_step(edition=edition), test_frontend_step(), - ] + ]) build_steps = [ build_backend_step(edition=edition, ver_mode=ver_mode), @@ -233,16 +248,16 @@ def get_steps(edition, ver_mode): if step: publish_steps.append(step) - return test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps + return init_steps, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps def get_oss_pipelines(trigger, ver_mode): edition = 'oss' services = integration_test_services(edition=edition) volumes = integration_test_services_volumes() - test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) + init_steps, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) windows_pipeline = pipeline( name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger, - steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps, + steps=[identify_runner_step('windows')] + windows_package_steps, platform='windows', depends_on=[ 'oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), ], @@ -250,8 +265,7 @@ def get_oss_pipelines(trigger, ver_mode): pipelines = [ pipeline( name='oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), edition=edition, trigger=trigger, services=[], - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + - build_steps + package_steps + publish_steps, + steps=init_steps + build_steps + package_steps + publish_steps, volumes=volumes, ), ] @@ -259,14 +273,12 @@ def get_oss_pipelines(trigger, ver_mode): pipelines.extend([ pipeline( name='oss-test-{}'.format(ver_mode), edition=edition, trigger=trigger, services=[], - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + - test_steps, + steps=init_steps + test_steps, volumes=[], ), pipeline( name='oss-integration-tests-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + - integration_test_steps, + steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps, volumes=volumes, ) ]) @@ -286,10 +298,28 @@ def get_enterprise_pipelines(trigger, ver_mode): edition = 'enterprise' services = integration_test_services(edition=edition) volumes = integration_test_services_volumes() - test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) + deps_on_clone_enterprise_step = { + 'depends_on': [ + 'init-enterprise', + ] + } + _, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) + init_steps = [ + download_grabpl_step(), + identify_runner_step(), + clone_enterprise_step(ver_mode), + init_enterprise_step(ver_mode) + ] + for step in [wire_install_step(), yarn_install_step(), gen_version_step(ver_mode)]: + step.update(deps_on_clone_enterprise_step) + init_steps.extend([step]) + + for step in integration_test_steps: + step.update(deps_on_clone_enterprise_step) + windows_pipeline = pipeline( name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger, - steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps, + steps=[identify_runner_step('windows')] + windows_package_steps, platform='windows', depends_on=[ 'enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), ], @@ -297,8 +327,7 @@ def get_enterprise_pipelines(trigger, ver_mode): pipelines = [ pipeline( name='enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), edition=edition, trigger=trigger, services=[], - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + - build_steps + package_steps + publish_steps, + steps=init_steps + build_steps + package_steps + publish_steps, volumes=volumes, ), ] @@ -306,14 +335,12 @@ def get_enterprise_pipelines(trigger, ver_mode): pipelines.extend([ pipeline( name='enterprise-test-{}'.format(ver_mode), edition=edition, trigger=trigger, services=[], - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + - test_steps, + steps=init_steps + test_steps, volumes=[], ), pipeline( name='enterprise-integration-tests-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + - integration_test_steps, + steps=[download_grabpl_step(), identify_runner_step(), clone_enterprise_step(ver_mode), init_enterprise_step(ver_mode),] + integration_test_steps, volumes=volumes, ), ]) @@ -401,7 +428,7 @@ def publish_npm_pipelines(mode): ] return [pipeline( - name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = initialize_step(edition='oss', platform='linux', ver_mode='release') + steps, edition="all" + name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = steps, edition="all" )] def release_pipelines(ver_mode='release', trigger=None, environment=None): diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 3ba6b73a9d2..44e4c6ae5b7 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -1,6 +1,6 @@ load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket') -grabpl_version = 'v2.9.34' +grabpl_version = 'v2.9.35' build_image = 'grafana/build-container:1.5.3' publish_image = 'grafana/grafana-ci-deploy:1.3.1' deploy_docker_image = 'us.gcr.io/kubernetes-dev/drone/plugins/deploy-image' @@ -18,6 +18,7 @@ trigger_oss = { } } + def slack_step(channel, template, secret): return { 'name': 'slack', @@ -30,116 +31,73 @@ def slack_step(channel, template, secret): } -def initialize_step(edition, platform, ver_mode, is_downstream=False, install_deps=True): - if platform == 'windows': - return [ - { - 'name': 'identify-runner', - 'image': windows_image, - 'commands': [ - 'echo $env:DRONE_RUNNER_NAME', - ], - }, - ] - - common_cmds = [ - # Generate Go code, will install Wire - # TODO: Install Wire in Docker image instead - 'make gen-go', - ] - +def gen_version_step(ver_mode): if ver_mode == 'release': args = '${DRONE_TAG}' - common_cmds.append('./bin/grabpl verify-version ${DRONE_TAG}') else: - if not is_downstream: - build_no = '${DRONE_BUILD_NUMBER}' - else: - build_no = '$${SOURCE_BUILD_NUMBER}' + build_no = '${DRONE_BUILD_NUMBER}' args = '--build-id {}'.format(build_no) + return { + 'name': 'gen-version', + 'image': build_image, + 'depends_on': [ + 'grabpl', + ], + 'commands': [ + './bin/grabpl gen-version {}'.format(args), + ], + } - identify_runner = identify_runner_step(platform) - if install_deps: - common_cmds.extend([ - './bin/grabpl gen-version {}'.format(args), +def yarn_install_step(): + return { + 'name': 'yarn-install', + 'image': build_image, + 'commands': [ 'yarn install --immutable', - ]) - if edition in ('enterprise', 'enterprise2'): - source_commit = '' - if ver_mode == 'release': - committish = '${DRONE_TAG}' - source_commit = ' ${DRONE_TAG}' - environment = { - 'GITHUB_TOKEN': from_secret(github_token), - } - token = "--github-token $${GITHUB_TOKEN}" - elif ver_mode == 'release-branch': - committish = '${DRONE_BRANCH}' - environment = {} - token = "" - else: - environment = {} - if is_downstream: - source_commit = ' $${SOURCE_COMMIT}' - committish = '${DRONE_COMMIT}' - token = "" - steps = [ - identify_runner, - clone_enterprise(committish), - { - 'name': 'initialize', - 'image': build_image, - 'depends_on': [ - 'clone-enterprise', - ], - 'environment': environment, - 'commands': [ - 'mv bin/grabpl /tmp/', - 'rmdir bin', - 'mv grafana-enterprise /tmp/', - '/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}'.format(token, source_commit), - 'mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json', - 'mkdir bin', - 'mv /tmp/grabpl bin/' - ] + common_cmds, - }, - ] - - return steps + ], + 'depends_on': [ + 'grabpl', + ], + } - steps = [ - identify_runner, - { - 'name': 'initialize', - 'image': build_image, - 'commands': common_cmds, - }, - ] - return steps +def wire_install_step(): + return { + 'name': 'wire-install', + 'image': build_image, + 'commands': [ + 'make gen-go', + ], + } -def identify_runner_step(platform): - if platform == 'linux': +def identify_runner_step(platform='linux'): + if platform == 'windows': return { 'name': 'identify-runner', - 'image': alpine_image, + 'image': windows_image, 'commands': [ - 'echo $DRONE_RUNNER_NAME', + 'echo $env:DRONE_RUNNER_NAME', ], } else: return { 'name': 'identify-runner', - 'image': windows_image, + 'image': alpine_image, 'commands': [ - 'echo $env:DRONE_RUNNER_NAME', + 'echo $DRONE_RUNNER_NAME', ], } -def clone_enterprise(committish): +def clone_enterprise_step(ver_mode): + if ver_mode == 'release': + committish = '${DRONE_TAG}' + elif ver_mode == 'release-branch': + committish = '${DRONE_BRANCH}' + else: + committish = '${DRONE_COMMIT}' return { 'name': 'clone-enterprise', 'image': build_image, @@ -153,8 +111,51 @@ def clone_enterprise(committish): ], } +def init_enterprise_step(ver_mode): + source_commit = '' + if ver_mode == 'release': + source_commit = ' ${DRONE_TAG}' + environment = { + 'GITHUB_TOKEN': from_secret(github_token), + } + token = "--github-token $${GITHUB_TOKEN}" + elif ver_mode == 'release-branch': + environment = {} + token = "" + else: + environment = {} + token = "" + return { + 'name': 'init-enterprise', + 'image': build_image, + 'depends_on': [ + 'clone-enterprise', + ], + 'environment': environment, + 'commands': [ + 'mv bin/grabpl /tmp/', + 'rmdir bin', + 'mv grafana-enterprise /tmp/', + '/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}'.format(token, source_commit), + 'mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json', + 'mkdir bin', + 'mv /tmp/grabpl bin/' + ], + } + + +def download_grabpl_step(platform="linux"): + if platform == 'windows': + return { + 'name': 'grabpl', + 'image': wix_image, + 'commands': [ + '$$ProgressPreference = "SilentlyContinue"', + 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( + grabpl_version), + ] + } -def download_grabpl_step(): return { 'name': 'grabpl', 'image': curl_image, @@ -211,7 +212,7 @@ def lint_backend_step(edition): 'CGO_ENABLED': '1', }, 'depends_on': [ - 'initialize', + 'wire-install', ], 'commands': [ # Don't use Make since it will re-download the linters @@ -224,9 +225,6 @@ def benchmark_ldap_step(): return { 'name': 'benchmark-ldap', 'image': build_image, - 'depends_on': [ - 'initialize', - ], 'environment': { 'LDAP_HOSTNAME': 'ldap', }, @@ -263,7 +261,6 @@ def store_storybook_step(edition, ver_mode, trigger=None): if edition in ('enterprise', 'enterprise2'): return None - commands = [] if ver_mode == 'release': commands.extend([ @@ -273,12 +270,12 @@ def store_storybook_step(edition, ver_mode, trigger=None): else: # main pipelines should deploy storybook to grafana-storybook/canary public bucket - commands = ['./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook',] + commands = ['./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook', ] step = { 'name': 'store-storybook', 'image': publish_image, - 'depends_on': ['build-storybook',] + end_to_end_tests_deps(edition), + 'depends_on': ['build-storybook', ] + end_to_end_tests_deps(edition), 'environment': { 'GCP_KEY': from_secret('gcp_key'), 'PRERELEASE_BUCKET': from_secret(prerelease_bucket) @@ -289,6 +286,7 @@ def store_storybook_step(edition, ver_mode, trigger=None): step.update(trigger) return step + def e2e_tests_artifacts(edition): return { 'name': 'e2e-tests-artifacts-upload' + enterprise2_suffix(edition), @@ -363,7 +361,7 @@ def upload_cdn_step(edition, ver_mode, trigger=None): return step -def build_backend_step(edition, ver_mode, variants=None, is_downstream=False): +def build_backend_step(edition, ver_mode, variants=None): variants_str = '' if variants: variants_str = ' --variants {}'.format(','.join(variants)) @@ -379,10 +377,7 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False): ), ] else: - if not is_downstream: - build_no = '${DRONE_BUILD_NUMBER}' - else: - build_no = '$${SOURCE_BUILD_NUMBER}' + build_no = '${DRONE_BUILD_NUMBER}' env = {} cmds = [ './bin/grabpl build-backend --jobs 8 --edition {} --build-id {}{}'.format( @@ -393,19 +388,17 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False): return { 'name': 'build-backend' + enterprise2_suffix(edition), 'image': build_image, + 'environment': env, 'depends_on': [ - 'initialize', + 'gen-version', + 'wire-install', ], - 'environment': env, 'commands': cmds, } -def build_frontend_step(edition, ver_mode, is_downstream=False): - if not is_downstream: - build_no = '${DRONE_BUILD_NUMBER}' - else: - build_no = '$${SOURCE_BUILD_NUMBER}' +def build_frontend_step(edition, ver_mode): + build_no = '${DRONE_BUILD_NUMBER}' # TODO: Use percentage for num jobs if ver_mode == 'release': @@ -422,42 +415,42 @@ def build_frontend_step(edition, ver_mode, is_downstream=False): return { 'name': 'build-frontend', 'image': build_image, - 'depends_on': [ - 'initialize', - ], 'environment': { 'NODE_OPTIONS': '--max_old_space_size=8192', }, + 'depends_on': [ + 'gen-version', + 'yarn-install', + ], 'commands': cmds, } -def build_frontend_package_step(edition, ver_mode, is_downstream=False): - if not is_downstream: - build_no = '${DRONE_BUILD_NUMBER}' - else: - build_no = '$${SOURCE_BUILD_NUMBER}' + +def build_frontend_package_step(edition, ver_mode): + build_no = '${DRONE_BUILD_NUMBER}' # TODO: Use percentage for num jobs if ver_mode == 'release': cmds = [ './bin/grabpl build-frontend-packages --jobs 8 --github-token $${GITHUB_TOKEN} ' + \ '--edition {} ${{DRONE_TAG}}'.format(edition), - ] + ] else: cmds = [ './bin/grabpl build-frontend-packages --jobs 8 --edition {} '.format(edition) + \ '--build-id {}'.format(build_no), - ] + ] return { 'name': 'build-frontend-packages', 'image': build_image, - 'depends_on': [ - 'initialize', - ], 'environment': { 'NODE_OPTIONS': '--max_old_space_size=8192', }, + 'depends_on': [ + 'gen-version', + 'yarn-install', + ], 'commands': cmds, } @@ -487,10 +480,11 @@ def build_plugins_step(edition, sign=False): return { 'name': 'build-plugins', 'image': build_image, + 'environment': env, 'depends_on': [ - 'initialize', + 'gen-version', + 'yarn-install', ], - 'environment': env, 'commands': [ # TODO: Use percentage for num jobs './bin/grabpl build-plugins --jobs 8 --edition {}{}'.format(edition, sign_args), @@ -503,7 +497,7 @@ def test_backend_step(edition): 'name': 'test-backend' + enterprise2_suffix(edition), 'image': build_image, 'depends_on': [ - 'initialize', + 'wire-install', ], 'commands': [ './bin/grabpl test-backend --edition {}'.format(edition), @@ -516,7 +510,7 @@ def test_backend_integration_step(edition): 'name': 'test-backend-integration' + enterprise2_suffix(edition), 'image': build_image, 'depends_on': [ - 'initialize', + 'wire-install', ], 'commands': [ './bin/grabpl integration-tests --edition {}'.format(edition), @@ -528,12 +522,12 @@ def test_frontend_step(): return { 'name': 'test-frontend', 'image': build_image, - 'depends_on': [ - 'initialize', - ], 'environment': { 'TEST_MAX_WORKERS': '50%', }, + 'depends_on': [ + 'yarn-install', + ], 'commands': [ 'yarn run ci:test-frontend', ], @@ -544,16 +538,16 @@ def lint_frontend_step(): return { 'name': 'lint-frontend', 'image': build_image, - 'depends_on': [ - 'initialize', - ], 'environment': { 'TEST_MAX_WORKERS': '50%', }, + 'depends_on': [ + 'yarn-install', + ], 'commands': [ 'yarn run prettier:check', 'yarn run lint', - 'yarn run i18n:compile', # TODO: right place for this? + 'yarn run i18n:compile', # TODO: right place for this? 'yarn run typecheck', ], } @@ -617,9 +611,6 @@ def codespell_step(): return { 'name': 'codespell', 'image': build_image, - 'depends_on': [ - 'initialize', - ], 'commands': [ # Important: all words have to be in lowercase, and separated by "\n". 'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt', @@ -634,7 +625,7 @@ def shellcheck_step(): 'name': 'shellcheck', 'image': build_image, 'depends_on': [ - 'initialize', + 'grabpl', ], 'commands': [ './bin/grabpl shellcheck', @@ -642,7 +633,7 @@ def shellcheck_step(): } -def package_step(edition, ver_mode, include_enterprise2=False, variants=None, is_downstream=False): +def package_step(edition, ver_mode, include_enterprise2=False, variants=None): deps = [ 'build-plugins', 'build-backend', @@ -683,10 +674,7 @@ def package_step(edition, ver_mode, include_enterprise2=False, variants=None, is ), ] else: - if not is_downstream: - build_no = '${DRONE_BUILD_NUMBER}' - else: - build_no = '$${SOURCE_BUILD_NUMBER}' + build_no = '${DRONE_BUILD_NUMBER}' cmds = [ '{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + \ '--build-id {}{}{}'.format(build_no, variants_str, sign_args), @@ -731,6 +719,7 @@ def grafana_server_step(edition, port=3001): ], } + def e2e_tests_step(suite, edition, port=3001, tries=None): cmd = './bin/grabpl e2e-tests --port {} --suite {}'.format(port, suite) if tries: @@ -810,13 +799,15 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis }, } + def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None): if mode == 'security': mode = '--{} '.format(mode) else: mode = '' - cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(mode, docker_repo) + cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format( + mode, docker_repo) if ver_mode == 'release': deps = ['fetch-images-{}'.format(edition)] @@ -847,10 +838,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None): def postgres_integration_tests_step(edition, ver_mode): deps = [] - if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'): - deps.extend(['initialize']) - else: - deps.extend(['grabpl']) + deps.extend(['grabpl']) return { 'name': 'postgres-integration-tests', 'image': build_image, @@ -875,10 +863,7 @@ def postgres_integration_tests_step(edition, ver_mode): def mysql_integration_tests_step(edition, ver_mode): deps = [] - if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'): - deps.extend(['initialize']) - else: - deps.extend(['grabpl']) + deps.extend(['grabpl']) return { 'name': 'mysql-integration-tests', 'image': build_image, @@ -901,10 +886,7 @@ def mysql_integration_tests_step(edition, ver_mode): def redis_integration_tests_step(edition, ver_mode): deps = [] - if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'): - deps.extend(['initialize']) - else: - deps.extend(['grabpl']) + deps.extend(['grabpl']) return { 'name': 'redis-integration-tests', 'image': build_image, @@ -921,10 +903,7 @@ def redis_integration_tests_step(edition, ver_mode): def memcached_integration_tests_step(edition, ver_mode): deps = [] - if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'): - deps.extend(['initialize']) - else: - deps.extend(['grabpl']) + deps.extend(['grabpl']) return { 'name': 'memcached-integration-tests', 'image': build_image, @@ -965,15 +944,16 @@ def enterprise2_suffix(edition): return '' -def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None): - if ver_mode == 'main' and edition in ('enterprise', 'enterprise2') and not is_downstream: +def upload_packages_step(edition, ver_mode, trigger=None): + if ver_mode == 'main' and edition in ('enterprise', 'enterprise2'): return None if ver_mode == 'release': packages_bucket = '$${{PRERELEASE_BUCKET}}/artifacts/downloads{}'.format(enterprise2_suffix(edition)) cmd = './bin/grabpl upload-packages --edition {} --packages-bucket {}'.format(edition, packages_bucket) elif edition == 'enterprise2': - cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads-enterprise2'.format(edition) + cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads-enterprise2'.format( + edition) else: cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads'.format(edition) @@ -981,7 +961,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None): if edition in 'enterprise2' or not end_to_end_tests_deps(edition): deps.extend([ 'package' + enterprise2_suffix(edition), - ]) + ]) else: deps.extend(end_to_end_tests_deps(edition)) @@ -1000,16 +980,13 @@ def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None): return step -def store_packages_step(edition, ver_mode, is_downstream=False): +def store_packages_step(edition, ver_mode): if ver_mode == 'release': cmd = './bin/grabpl store-packages --edition {} --packages-bucket grafana-downloads --gcp-key /tmp/gcpkey.json ${{DRONE_TAG}}'.format( edition, ) elif ver_mode == 'main': - if not is_downstream: - build_no = '${DRONE_BUILD_NUMBER}' - else: - build_no = '$${SOURCE_BUILD_NUMBER}' + build_no = '${DRONE_BUILD_NUMBER}' cmd = './bin/grabpl store-packages --edition {} --gcp-key /tmp/gcpkey.json --build-id {}'.format( edition, build_no, ) @@ -1035,12 +1012,7 @@ def store_packages_step(edition, ver_mode, is_downstream=False): } -def get_windows_steps(edition, ver_mode, is_downstream=False): - if not is_downstream: - source_commit = '' - else: - source_commit = ' $$env:SOURCE_COMMIT' - +def get_windows_steps(edition, ver_mode): init_cmds = [] sfx = '' if edition in ('enterprise', 'enterprise2'): @@ -1053,12 +1025,12 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): ]) steps = [ { - 'name': 'initialize', + 'name': 'windows-init', 'image': wix_image, 'commands': init_cmds, }, ] - if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2') or is_downstream)) or ver_mode in ( + if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in ( 'release', 'release-branch', ): bucket_part = '' @@ -1070,10 +1042,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): dir = 'main' bucket = 'grafana-downloads' bucket_part = ' --packages-bucket {}'.format(bucket) - if not is_downstream: - build_no = 'DRONE_BUILD_NUMBER' - else: - build_no = 'SOURCE_BUILD_NUMBER' + build_no = 'DRONE_BUILD_NUMBER' ver_part = '--build-id $$env:{}'.format(build_no) installer_commands = [ '$$gcpKey = $$env:GCP_KEY', @@ -1084,7 +1053,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): 'rm gcpkey.json', 'cp C:\\App\\nssm-2.24.zip .', ] - if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2') or is_downstream)) or ver_mode in ( + if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in ( 'release', ): installer_commands.extend([ @@ -1105,15 +1074,15 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): steps.append({ 'name': 'build-windows-installer', 'image': wix_image, + 'depends_on': [ + 'windows-init', + ], 'environment': { 'GCP_KEY': from_secret('gcp_key'), 'PRERELEASE_BUCKET': from_secret(prerelease_bucket), 'GITHUB_TOKEN': from_secret('github_token') }, 'commands': installer_commands, - 'depends_on': [ - 'initialize', - ], }) if edition in ('enterprise', 'enterprise2'): @@ -1132,11 +1101,10 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): clone_cmds = [ 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"', ] - if not is_downstream: - clone_cmds.extend([ - 'cd grafana-enterprise', - 'git checkout {}'.format(committish), - ]) + clone_cmds.extend([ + 'cd grafana-enterprise', + 'git checkout {}'.format(committish), + ]) steps.insert(0, { 'name': 'clone', 'image': wix_image, @@ -1154,7 +1122,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): 'rm -r -force grafana-enterprise', 'cp grabpl.exe C:\\App\\grabpl.exe', 'rm -force grabpl.exe', - 'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise{}'.format(source_commit), + 'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise', 'cp C:\\App\\grabpl.exe grabpl.exe', ]) if 'environment' in steps[1]: @@ -1194,6 +1162,7 @@ def ensure_cuetsified_step(): ], } + def end_to_end_tests_deps(edition): if disable_tests: return []