From bd386df617fe6ec4d4ba4986ea30b309faf55d15 Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Wed, 6 Apr 2022 10:46:05 +0300 Subject: [PATCH] CI: Run publishing steps only on OSS repo for main/version branches (#47315) * Convert steps to run on OSS repo only * Exclude versioned branches from publishing artifacts * Change trigger -> when * Add trigger to upload_* steps * Add conditions to remaining steps * Exclude release steps * Bring back exclusion for release builds --- .drone.yml | 63 ++++++++++++++++------------ scripts/drone/pipelines/main.star | 29 ++++++------- scripts/drone/pipelines/release.star | 11 +++-- scripts/drone/steps/lib.star | 51 ++++++++++++++++------ scripts/drone/version.star | 3 -- 5 files changed, 94 insertions(+), 63 deletions(-) diff --git a/.drone.yml b/.drone.yml index 9d4c9f1550a..3fcabcb84e0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -565,8 +565,6 @@ trigger: branch: main event: - push - repo: - - grafana/grafana type: docker volumes: - host: @@ -668,8 +666,6 @@ trigger: branch: main event: - push - repo: - - grafana/grafana type: docker volumes: - host: @@ -727,6 +723,8 @@ steps: paths: include: - .drone.yml + repo: + - grafana/grafana - image: grafana/drone-downstream name: trigger-enterprise-downstream settings: @@ -917,6 +915,9 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 name: store-storybook + when: + repo: + - grafana/grafana - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json @@ -940,6 +941,9 @@ steps: failure: ignore image: grafana/build-container:1.5.3 name: publish-frontend-metrics + when: + repo: + - grafana/grafana - commands: - ls dist/*.tar.gz* - cp dist/*.tar.gz* packaging/docker/ @@ -989,6 +993,9 @@ steps: volumes: - name: docker path: /var/run/docker.sock + when: + repo: + - grafana/grafana - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana-oss --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 @@ -1007,6 +1014,9 @@ steps: volumes: - name: docker path: /var/run/docker.sock + when: + repo: + - grafana/grafana - commands: - ./scripts/circle-release-canary-packages.sh depends_on: @@ -1019,6 +1029,9 @@ steps: from_secret: npm_token image: grafana/build-container:1.5.3 name: release-canary-npm-packages + when: + repo: + - grafana/grafana - commands: - ./bin/grabpl upload-packages --edition oss --packages-bucket grafana-downloads depends_on: @@ -1033,6 +1046,9 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 name: upload-packages + when: + repo: + - grafana/grafana - commands: - ./bin/grabpl upload-cdn --edition oss --src-bucket "grafana-static-assets" depends_on: @@ -1044,12 +1060,13 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 name: upload-cdn-assets + when: + repo: + - grafana/grafana trigger: branch: main event: - push - repo: - - grafana/grafana type: docker volumes: - host: @@ -1132,8 +1149,6 @@ trigger: branch: main event: - push - repo: - - grafana/grafana type: docker volumes: - host: @@ -1319,8 +1334,6 @@ trigger: branch: main event: - push - repo: - - grafana/grafana status: - failure type: docker @@ -3322,6 +3335,9 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 name: upload-cdn-assets + when: + repo: + - grafana/grafana - commands: - ./bin/grabpl upload-packages --edition oss --packages-bucket grafana-downloads depends_on: @@ -3336,11 +3352,12 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 name: upload-packages + when: + repo: + - grafana/grafana trigger: ref: - refs/heads/v[0-9]* - repo: - - grafana/grafana type: docker volumes: - host: @@ -3441,8 +3458,6 @@ steps: trigger: ref: - refs/heads/v[0-9]* - repo: - - grafana/grafana type: docker volumes: - host: @@ -3528,8 +3543,6 @@ steps: trigger: ref: - refs/heads/v[0-9]* - repo: - - grafana/grafana type: docker volumes: - host: @@ -3586,8 +3599,6 @@ steps: trigger: ref: - refs/heads/v[0-9]* - repo: - - grafana/grafana type: docker volumes: - host: @@ -3859,6 +3870,9 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 name: upload-cdn-assets + when: + repo: + - grafana/grafana - commands: - ./bin/grabpl upload-packages --edition enterprise --packages-bucket grafana-downloads depends_on: @@ -3870,6 +3884,9 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 name: upload-packages + when: + repo: + - grafana/grafana - commands: - ./bin/grabpl package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} --variants linux-amd64 --sign @@ -3917,8 +3934,6 @@ steps: trigger: ref: - refs/heads/v[0-9]* - repo: - - grafana/grafana type: docker volumes: - host: @@ -4062,8 +4077,6 @@ steps: trigger: ref: - refs/heads/v[0-9]* - repo: - - grafana/grafana type: docker volumes: - host: @@ -4196,8 +4209,6 @@ steps: trigger: ref: - refs/heads/v[0-9]* - repo: - - grafana/grafana type: docker volumes: - host: @@ -4278,8 +4289,6 @@ steps: trigger: ref: - refs/heads/v[0-9]* - repo: - - grafana/grafana type: docker volumes: - host: @@ -4431,6 +4440,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: 430843c058dc4f40a3f881e13fc61ab12f5b6dc823f99067e56ab49fc1bdd459 +hmac: 61a04d046f8fcafcd1b3e5ba483ec7c0d87f502ee77829a9ed12e8d181d3f120 ... diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star index f937358fe1b..91562878bab 100644 --- a/scripts/drone/pipelines/main.star +++ b/scripts/drone/pipelines/main.star @@ -38,7 +38,8 @@ load( 'upload_cdn_step', 'validate_scuemata_step', 'ensure_cuetsified_step', - 'test_a11y_frontend_step' + 'test_a11y_frontend_step', + 'trigger_oss' ) load( @@ -118,23 +119,23 @@ def get_steps(edition, is_downstream=False): e2e_tests_step('various-suite', edition=edition), e2e_tests_artifacts(edition=edition), build_storybook_step(edition=edition, ver_mode=ver_mode), - store_storybook_step(edition=edition, ver_mode=ver_mode), + store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), test_a11y_frontend_step(ver_mode=ver_mode, edition=edition), - frontend_metrics_step(edition=edition), + frontend_metrics_step(edition=edition, trigger=trigger_oss), copy_packages_for_docker_step(), build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False), build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False), - publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', ubuntu=False), - publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', ubuntu=True) + publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss), + publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss) ]) if include_enterprise2: integration_test_steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)]) build_steps.extend([ - release_canary_npm_packages_step(edition), - upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), - upload_cdn_step(edition=edition, ver_mode=ver_mode) + release_canary_npm_packages_step(edition, trigger=trigger_oss), + upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream, trigger=trigger_oss), + upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss) ]) if include_enterprise2: @@ -180,7 +181,10 @@ def trigger_test_release(): 'include': [ '.drone.yml', ] - } + }, + 'repo': [ + 'grafana/grafana', + ] } } @@ -190,9 +194,6 @@ def main_pipelines(edition): trigger = { 'event': ['push',], 'branch': 'main', - 'repo': [ - 'grafana/grafana', - ], } drone_change_trigger = { 'event': ['push',], @@ -233,7 +234,7 @@ def main_pipelines(edition): volumes=volumes, ), pipeline( - name='windows-main', edition=edition, trigger=trigger, + name='windows-main', edition=edition, trigger=dict(trigger, repo = ['grafana/grafana']), steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps, depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows', ), notify_pipeline( @@ -242,7 +243,7 @@ def main_pipelines(edition): ] if edition != 'enterprise': pipelines.append(pipeline( - name='publish-main', edition=edition, trigger=trigger, + name='publish-main', edition=edition, trigger=dict(trigger, repo = ['grafana/grafana']), steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, install_deps=False) + store_steps, depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main',], )) diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index 963b6236c50..37dc3becc49 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -4,7 +4,6 @@ load( 'download_grabpl_step', 'initialize_step', 'lint_drone_step', - 'test_release_ver', 'build_image', 'publish_image', 'lint_backend_step', @@ -31,14 +30,14 @@ load( 'memcached_integration_tests_step', 'get_windows_steps', 'benchmark_ldap_step', - 'frontend_metrics_step', 'store_storybook_step', 'upload_packages_step', 'store_packages_step', 'upload_cdn_step', 'validate_scuemata_step', 'ensure_cuetsified_step', - 'publish_images_step' + 'publish_images_step', + 'trigger_oss' ) load( @@ -228,8 +227,8 @@ def get_steps(edition, ver_mode): integration_test_steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)]) if should_upload: - publish_steps.append(upload_cdn_step(edition=edition, ver_mode=ver_mode)) - publish_steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode)) + publish_steps.append(upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)) + publish_steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)) if should_publish: publish_step = store_storybook_step(edition=edition, ver_mode=ver_mode) build_npm_step = build_npm_packages_step(edition=edition, ver_mode=ver_mode) @@ -433,7 +432,7 @@ def release_pipelines(ver_mode='release', trigger=None, environment=None): }, 'ref': ['refs/tags/v*',], 'repo': { - 'exclude': ['grafana/grafana'], + 'exclude': ['grafana/grafana'], }, } diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 4592cca28f7..4c9ea442ec1 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -8,9 +8,15 @@ alpine_image = 'alpine:3.15' curl_image = 'byrnedo/alpine-curl:0.1.8' windows_image = 'mcr.microsoft.com/windows:1809' wix_image = 'grafana/ci-wix:0.1.1' -test_release_ver = 'v7.3.0-test' disable_tests = False +trigger_oss = { + 'when': { + 'repo': [ + 'grafana/grafana', + ] + } +} def slack_step(channel, template, secret): return { @@ -253,7 +259,7 @@ def build_storybook_step(edition, ver_mode): } -def store_storybook_step(edition, ver_mode): +def store_storybook_step(edition, ver_mode, trigger=None): if edition in ('enterprise', 'enterprise2'): return None @@ -272,7 +278,7 @@ def store_storybook_step(edition, ver_mode): for c in channels ]) - return { + step = { 'name': 'store-storybook', 'image': publish_image, 'depends_on': ['build-storybook',] + end_to_end_tests_deps(edition), @@ -282,6 +288,9 @@ def store_storybook_step(edition, ver_mode): }, 'commands': commands, } + if trigger and ver_mode in ("release-branch", "main"): + step.update(trigger) + return step def e2e_tests_artifacts(edition): return { @@ -322,7 +331,7 @@ def e2e_tests_artifacts(edition): } -def upload_cdn_step(edition, ver_mode): +def upload_cdn_step(edition, ver_mode, trigger=None): src_dir = '' if ver_mode == "release": bucket = "$${PRERELEASE_BUCKET}" @@ -340,7 +349,7 @@ def upload_cdn_step(edition, ver_mode): 'grafana-server', ]) - return { + step = { 'name': 'upload-cdn-assets' + enterprise2_suffix(edition), 'image': publish_image, 'depends_on': deps, @@ -352,6 +361,9 @@ def upload_cdn_step(edition, ver_mode): './bin/grabpl upload-cdn --edition {} --src-bucket "{}"{}'.format(edition, bucket, src_dir), ], } + if trigger and ver_mode in ("release-branch", "main"): + step.update(trigger) + return step def build_backend_step(edition, ver_mode, variants=None, is_downstream=False): @@ -581,11 +593,11 @@ def test_a11y_frontend_step(ver_mode, edition, port=3001): } -def frontend_metrics_step(edition): +def frontend_metrics_step(edition, trigger=None): if edition in ('enterprise', 'enterprise2'): return None - return { + step = { 'name': 'publish-frontend-metrics', 'image': build_image, 'depends_on': [ @@ -599,6 +611,9 @@ def frontend_metrics_step(edition): './scripts/ci-frontend-metrics.sh | ./bin/grabpl publish-metrics $${GRAFANA_MISC_STATS_API_KEY}', ], } + if trigger: + step.update(trigger) + return step def codespell_step(): @@ -798,7 +813,7 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis }, } -def publish_images_step(edition, ver_mode, mode, docker_repo, ubuntu=False): +def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None): if mode == 'security': mode = '--{} '.format(mode) else: @@ -812,7 +827,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, ubuntu=False): else: deps = ['build-docker-images', 'build-docker-images-ubuntu'] - return { + step = { 'name': 'publish-images-{}'.format(docker_repo), 'image': 'google/cloud-sdk', 'environment': { @@ -827,6 +842,10 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, ubuntu=False): 'path': '/var/run/docker.sock' }], } + if trigger and ver_mode in ("release-branch", "main"): + step.update(trigger) + + return step def postgres_integration_tests_step(edition, ver_mode): @@ -923,11 +942,11 @@ def memcached_integration_tests_step(edition, ver_mode): } -def release_canary_npm_packages_step(edition): +def release_canary_npm_packages_step(edition, trigger=None): if edition in ('enterprise', 'enterprise2'): return None - return { + step = { 'name': 'release-canary-npm-packages', 'image': build_image, 'depends_on': end_to_end_tests_deps(edition), @@ -938,6 +957,9 @@ def release_canary_npm_packages_step(edition): './scripts/circle-release-canary-packages.sh', ], } + if trigger: + step.update(trigger) + return step def enterprise2_suffix(edition): @@ -946,7 +968,7 @@ def enterprise2_suffix(edition): return '' -def upload_packages_step(edition, ver_mode, is_downstream=False): +def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None): if ver_mode == 'main' and edition in ('enterprise', 'enterprise2') and not is_downstream: return None @@ -966,7 +988,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False): else: deps.extend(end_to_end_tests_deps(edition)) - return { + step = { 'name': 'upload-packages' + enterprise2_suffix(edition), 'image': publish_image, 'depends_on': deps, @@ -976,6 +998,9 @@ def upload_packages_step(edition, ver_mode, is_downstream=False): }, 'commands': [cmd, ], } + if trigger and ver_mode in ("release-branch", "main"): + step.update(trigger) + return step def store_packages_step(edition, ver_mode, is_downstream=False): diff --git a/scripts/drone/version.star b/scripts/drone/version.star index 983620aada8..9a5b39d41de 100644 --- a/scripts/drone/version.star +++ b/scripts/drone/version.star @@ -8,7 +8,4 @@ ver_mode = 'release-branch' def version_branch_pipelines(): return release_pipelines(ver_mode=ver_mode, trigger={ 'ref': ['refs/heads/v[0-9]*'], - 'repo': [ - 'grafana/grafana', - ], })