CI: Refactor/Reorder `*.star` files (#52384)

* Refactor test pipelines

* Move pr, main and release starlark files under scripts/drone/event

* Move cron.star to scripts/drone/events

* Further unused code removal

* Introducing build.star

* Extract verify_drone.star

* Extract windows.star

* Extract publish.star

* Remove unused code

* Fix formatting in main.star

.drone.star

@@ -4,12 +4,12 @@
 # 3. Run `make drone`
 # More information about this process here: https://github.com/grafana/deployment_tools/blob/master/docs/infrastructure/drone/signing.md
 
-load('scripts/drone/pipelines/pr.star', 'pr_pipelines')
-load('scripts/drone/pipelines/main.star', 'main_pipelines')
+load('scripts/drone/events/pr.star', 'pr_pipelines')
+load('scripts/drone/events/main.star', 'main_pipelines')
 load('scripts/drone/pipelines/docs.star', 'docs_pipelines')
-load('scripts/drone/pipelines/release.star', 'release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline')
+load('scripts/drone/events/release.star', 'release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline')
 load('scripts/drone/version.star', 'version_branch_pipelines')
-load('scripts/drone/pipelines/cron.star', 'cronjobs')
+load('scripts/drone/events/cron.star', 'cronjobs')
 load('scripts/drone/vault.star', 'secrets')
 
 def main(ctx):

.drone.yml

@@ -413,6 +413,13 @@ steps:
     NODE_OPTIONS: --max_old_space_size=4096
   image: grafana/build-container:1.5.8
   name: build-storybook
+- commands:
+  - ls dist/*.tar.gz*
+  - cp dist/*.tar.gz* packaging/docker/
+  depends_on:
+  - package
+  image: grafana/build-container:1.5.8
+  name: copy-packages-for-docker
 - commands:
   - yarn wait-on http://$HOST:$PORT
   - pa11y-ci --config .pa11yci-pr.conf.js
@@ -426,13 +433,6 @@ steps:
   failure: always
   image: grafana/docker-puppeteer:1.0.0
   name: test-a11y-frontend
-- commands:
-  - ls dist/*.tar.gz*
-  - cp dist/*.tar.gz* packaging/docker/
-  depends_on:
-  - package
-  image: grafana/build-container:1.5.8
-  name: copy-packages-for-docker
 - commands:
   - ./bin/grabpl build-docker --edition oss -archs amd64
   depends_on:
@@ -850,12 +850,6 @@ steps:
   - verify-gen-cue
   image: grafana/build-container:1.5.8
   name: wire-install
-- commands:
-  - ./bin/grabpl verify-drone
-  depends_on:
-  - grabpl
-  image: byrnedo/alpine-curl:0.1.8
-  name: lint-drone
 - commands:
   - |-
     echo -e "unknwon
@@ -894,6 +888,12 @@ steps:
   - wire-install
   image: grafana/build-container:1.5.8
   name: test-backend-integration
+- commands:
+  - ./bin/grabpl verify-drone
+  depends_on:
+  - grabpl
+  image: byrnedo/alpine-curl:0.1.8
+  name: lint-drone
 trigger:
   branch: main
   event:
@@ -1133,6 +1133,26 @@ steps:
     NODE_OPTIONS: --max_old_space_size=4096
   image: grafana/build-container:1.5.8
   name: build-storybook
+- commands:
+  - ls dist/*.tar.gz*
+  - cp dist/*.tar.gz* packaging/docker/
+  depends_on:
+  - package
+  image: grafana/build-container:1.5.8
+  name: copy-packages-for-docker
+- commands:
+  - yarn wait-on http://$HOST:$PORT
+  - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json
+  depends_on:
+  - grafana-server
+  environment:
+    GRAFANA_MISC_STATS_API_KEY:
+      from_secret: grafana_misc_stats_api_key
+    HOST: grafana-server
+    PORT: 3001
+  failure: ignore
+  image: grafana/docker-puppeteer:1.0.0
+  name: test-a11y-frontend
 - commands:
   - ./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook
   depends_on:
@@ -1151,19 +1171,6 @@ steps:
   when:
     repo:
     - grafana/grafana
-- commands:
-  - yarn wait-on http://$HOST:$PORT
-  - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json
-  depends_on:
-  - grafana-server
-  environment:
-    GRAFANA_MISC_STATS_API_KEY:
-      from_secret: grafana_misc_stats_api_key
-    HOST: grafana-server
-    PORT: 3001
-  failure: ignore
-  image: grafana/docker-puppeteer:1.0.0
-  name: test-a11y-frontend
 - commands:
   - ./scripts/ci-frontend-metrics.sh | ./bin/grabpl publish-metrics $${GRAFANA_MISC_STATS_API_KEY}
   depends_on:
@@ -1177,13 +1184,6 @@ steps:
   when:
     repo:
     - grafana/grafana
-- commands:
-  - ls dist/*.tar.gz*
-  - cp dist/*.tar.gz* packaging/docker/
-  depends_on:
-  - package
-  image: grafana/build-container:1.5.8
-  name: copy-packages-for-docker
 - commands:
   - ./bin/grabpl build-docker --edition oss
   depends_on:
@@ -4874,6 +4874,6 @@ kind: secret
 name: gcp_upload_artifacts_key
 ---
 kind: signature
-hmac: d7d85528bb22e0a126fb5fb04ec3bdde3001ab0d5a9a88a34db3cd9985eeb9fa
+hmac: fdbc2724cf738ea9a2cd2be46f70d13928c96cc29179a79b7108650af40740ff
 
 ...

scripts/drone/events/main.star

@@ -0,0 +1,88 @@
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+    'notify_pipeline',
+    'failure_template',
+    'drone_change_template',
+)
+
+load(
+    'scripts/drone/pipelines/docs.star',
+    'docs_pipelines',
+)
+
+load(
+    'scripts/drone/pipelines/test_frontend.star',
+    'test_frontend',
+)
+
+load(
+    'scripts/drone/pipelines/test_backend.star',
+    'test_backend',
+)
+
+load(
+    'scripts/drone/pipelines/integration_tests.star',
+    'integration_tests',
+)
+
+load(
+    'scripts/drone/pipelines/build.star',
+    'build_e2e',
+)
+
+load(
+    'scripts/drone/pipelines/windows.star',
+    'windows',
+)
+
+load(
+    'scripts/drone/pipelines/publish.star',
+    'publish',
+)
+
+load('scripts/drone/vault.star', 'from_secret')
+
+
+ver_mode = 'main'
+trigger = {
+    'event': ['push',],
+    'branch': 'main',
+}
+
+def main_pipelines(edition):
+    drone_change_trigger = {
+        'event': ['push',],
+        'branch': 'main',
+        'repo': [
+            'grafana/grafana',
+        ],
+        'paths': {
+            'include': [
+                '.drone.yml',
+            ],
+            'exclude': [
+                'exclude',
+            ],
+        },
+    }
+
+    pipelines = [
+        docs_pipelines(edition, ver_mode, trigger),
+        test_frontend(trigger, ver_mode),
+        test_backend(trigger, ver_mode),
+        build_e2e(trigger, ver_mode, edition),
+        integration_tests(trigger, ver_mode, edition),
+        windows(trigger, edition, ver_mode),
+    notify_pipeline(
+        name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
+        template=drone_change_template, secret='drone-changes-webhook',
+    ),
+    publish(trigger, ver_mode, edition),
+    notify_pipeline(
+        name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),
+        depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'],
+        template=failure_template, secret='slack_webhook'
+    )]
+
+    return pipelines

scripts/drone/events/pr.star

@@ -0,0 +1,81 @@
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+)
+
+load(
+    'scripts/drone/pipelines/test_frontend.star',
+    'test_frontend',
+)
+
+load(
+    'scripts/drone/pipelines/test_backend.star',
+    'test_backend',
+)
+
+load(
+    'scripts/drone/pipelines/integration_tests.star',
+    'integration_tests',
+)
+
+load(
+    'scripts/drone/pipelines/build.star',
+    'build_e2e',
+)
+
+load(
+    'scripts/drone/pipelines/verify_drone.star',
+    'verify_drone',
+)
+
+load(
+    'scripts/drone/pipelines/docs.star',
+    'docs_pipelines',
+    'trigger_docs',
+)
+
+ver_mode = 'pr'
+trigger = {
+    'event': [
+        'pull_request',
+    ],
+    'paths': {
+        'exclude': [
+            '*.md',
+            'docs/**',
+            'latest.json',
+        ],
+    },
+}
+
+
+def pr_pipelines(edition):
+    return [
+        verify_drone(get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), ver_mode),
+        test_frontend(get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), ver_mode),
+        test_backend(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), ver_mode),
+        build_e2e(trigger, ver_mode, edition),
+        integration_tests(trigger, ver_mode, edition),
+        docs_pipelines(edition, ver_mode, trigger_docs())
+    ]
+
+
+def get_pr_trigger(include_paths=None, exclude_paths=None):
+    paths_ex = ['docs/**', '*.md']
+    paths_in = []
+    if include_paths:
+        for path in include_paths:
+            paths_in.extend([path])
+    if exclude_paths:
+        for path in exclude_paths:
+            paths_ex.extend([path])
+    return {
+        'event': [
+            'pull_request',
+        ],
+        'paths': {
+            'exclude': paths_ex,
+            'include': paths_in,
+        },
+    }
+

scripts/drone/pipelines/build.star

@@ -0,0 +1,107 @@
+load(
+    'scripts/drone/steps/lib.star',
+    'download_grabpl_step',
+        'build_image',
+        'identify_runner_step',
+        'gen_version_step',
+        'wire_install_step',
+        'yarn_install_step',
+        'build_backend_step',
+        'build_frontend_step',
+        'build_frontend_package_step',
+        'build_plugins_step',
+        'package_step',
+        'grafana_server_step',
+        'e2e_tests_step',
+        'e2e_tests_artifacts',
+        'build_storybook_step',
+        'copy_packages_for_docker_step',
+        'build_docker_images_step',
+        'publish_images_step',
+        'postgres_integration_tests_step',
+        'mysql_integration_tests_step',
+        'redis_integration_tests_step',
+        'memcached_integration_tests_step',
+        'get_windows_steps',
+        'benchmark_ldap_step',
+        'enterprise_downstream_step',
+        'frontend_metrics_step',
+        'store_storybook_step',
+        'release_canary_npm_packages_step',
+        'upload_packages_step',
+        'store_packages_step',
+        'upload_cdn_step',
+        'verify_gen_cue_step',
+        'test_a11y_frontend_step',
+        'trigger_oss',
+        'betterer_frontend_step',
+        'trigger_test_release'
+)
+
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+)
+
+def build_e2e(trigger, ver_mode, edition):
+    variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',]
+    init_steps = [
+        identify_runner_step(),
+        download_grabpl_step(),
+        gen_version_step(ver_mode),
+        verify_gen_cue_step(edition="oss"),
+        wire_install_step(),
+        yarn_install_step(),
+    ]
+    build_steps = []
+    if ver_mode == 'main':
+        build_steps.extend([trigger_test_release()])
+    build_steps.extend([
+        enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
+        build_backend_step(edition=edition, ver_mode=ver_mode),
+        build_frontend_step(edition=edition, ver_mode=ver_mode),
+        build_frontend_package_step(edition=edition, ver_mode=ver_mode),
+        build_plugins_step(edition=edition, ver_mode=ver_mode),
+   ])
+    if ver_mode == 'main':
+        build_steps.extend([package_step(edition=edition, ver_mode=ver_mode)])
+    elif ver_mode == 'pr':
+        build_steps.extend([package_step(edition=edition, ver_mode=ver_mode, variants=variants)])
+
+    build_steps.extend([
+        grafana_server_step(edition=edition),
+        e2e_tests_step('dashboards-suite', edition=edition),
+        e2e_tests_step('smoke-tests-suite', edition=edition),
+        e2e_tests_step('panels-suite', edition=edition),
+        e2e_tests_step('various-suite', edition=edition),
+        e2e_tests_artifacts(edition=edition),
+        build_storybook_step(edition=edition, ver_mode=ver_mode),
+        copy_packages_for_docker_step(),
+        test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
+    ])
+    if ver_mode == 'main':
+        build_steps.extend([
+            store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
+            frontend_metrics_step(edition=edition, trigger=trigger_oss)
+        ])
+
+    if ver_mode == 'main':
+        build_steps.extend([
+            build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False),
+            build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False),
+            publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss),
+            publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss),
+            release_canary_npm_packages_step(edition, trigger=trigger_oss),
+            upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
+            upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
+        ])
+    elif ver_mode == 'pr':
+        build_steps.extend([build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ])])
+
+    publish_suffix = ''
+    if ver_mode == 'main':
+        publish_suffix = '-publish'
+
+    return pipeline(
+        name='{}-build-e2e{}'.format(ver_mode, publish_suffix), edition="oss", trigger=trigger, services=[], steps=init_steps + build_steps,
+    )

scripts/drone/pipelines/integration_tests.star

@@ -0,0 +1,40 @@
+load(
+    'scripts/drone/steps/lib.star',
+    'identify_runner_step',
+    'download_grabpl_step',
+    'verify_gen_cue_step',
+    'wire_install_step',
+    'postgres_integration_tests_step',
+    'mysql_integration_tests_step',
+)
+
+load(
+    'scripts/drone/services/services.star',
+    'integration_test_services',
+    'integration_test_services_volumes',
+    'ldap_service',
+)
+
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+)
+
+def integration_tests(trigger, ver_mode, edition):
+    services = integration_test_services(edition)
+    volumes = integration_test_services_volumes()
+    init_steps = [
+        download_grabpl_step(),
+        identify_runner_step(),
+        verify_gen_cue_step(edition="oss"),
+        wire_install_step(),
+    ]
+    test_steps = [
+        postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
+        mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
+    ]
+
+    return pipeline(
+        name='{}-integration-tests'.format(ver_mode), edition="oss", trigger=trigger, services=services, steps=init_steps + test_steps,
+        volumes=volumes
+    )

scripts/drone/pipelines/main.star

@@ -1,243 +0,0 @@
-load(
-    'scripts/drone/steps/lib.star',
-    'download_grabpl_step',
-    'build_image',
-    'identify_runner_step',
-    'gen_version_step',
-    'wire_install_step',
-    'yarn_install_step',
-    'lint_drone_step',
-    'lint_backend_step',
-    'lint_frontend_step',
-    'codespell_step',
-    'shellcheck_step',
-    'test_backend_step',
-    'test_backend_integration_step',
-    'test_frontend_step',
-    'build_backend_step',
-    'build_frontend_step',
-    'build_frontend_package_step',
-    'build_plugins_step',
-    'package_step',
-    'grafana_server_step',
-    'e2e_tests_step',
-    'e2e_tests_artifacts',
-    'build_storybook_step',
-    'copy_packages_for_docker_step',
-    'build_docker_images_step',
-    'publish_images_step',
-    'postgres_integration_tests_step',
-    'mysql_integration_tests_step',
-    'redis_integration_tests_step',
-    'memcached_integration_tests_step',
-    'get_windows_steps',
-    'benchmark_ldap_step',
-    'enterprise_downstream_step',
-    'frontend_metrics_step',
-    'store_storybook_step',
-    'release_canary_npm_packages_step',
-    'upload_packages_step',
-    'store_packages_step',
-    'upload_cdn_step',
-    'verify_gen_cue_step',
-    'test_a11y_frontend_step',
-    'trigger_oss',
-    'betterer_frontend_step'
-)
-
-load(
-    'scripts/drone/services/services.star',
-    'integration_test_services',
-    'integration_test_services_volumes',
-    'ldap_service',
-)
-
-load(
-    'scripts/drone/utils/utils.star',
-    'pipeline',
-    'notify_pipeline',
-    'failure_template',
-    'drone_change_template',
-)
-
-load(
-    'scripts/drone/pipelines/docs.star',
-    'docs_pipelines',
-)
-
-load('scripts/drone/vault.star', 'from_secret')
-
-
-ver_mode = 'main'
-trigger = {
-    'event': ['push',],
-    'branch': 'main',
-}
-
-def main_test_frontend():
-    init_steps = [
-        identify_runner_step(),
-        download_grabpl_step(),
-        gen_version_step(ver_mode),
-        yarn_install_step(),
-    ]
-    test_steps = [
-        lint_frontend_step(),
-        betterer_frontend_step(),
-        test_frontend_step(),
-    ]
-    return pipeline(
-        name='main-test-frontend', edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
-    )
-
-def main_test_backend():
-    init_steps = [
-        identify_runner_step(),
-        download_grabpl_step(),
-        gen_version_step(ver_mode),
-        verify_gen_cue_step(edition="oss"),
-        wire_install_step(),
-    ]
-    test_steps = [
-        lint_drone_step(),
-        codespell_step(),
-        shellcheck_step(),
-        lint_backend_step(edition="oss"),
-        test_backend_step(edition="oss"),
-        test_backend_integration_step(edition="oss"),
-    ]
-    return pipeline(
-        name='main-test-backend', edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
-    )
-
-
-def get_steps(edition):
-    init_steps = [
-        identify_runner_step(),
-        download_grabpl_step(),
-        gen_version_step(ver_mode),
-        verify_gen_cue_step(edition="oss"),
-        wire_install_step(),
-        yarn_install_step(),
-    ]
-    build_steps = [
-        trigger_test_release(),
-        enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
-        build_backend_step(edition=edition, ver_mode=ver_mode),
-        build_frontend_step(edition=edition, ver_mode=ver_mode),
-        build_frontend_package_step(edition=edition, ver_mode=ver_mode),
-        build_plugins_step(edition=edition, ver_mode=ver_mode),
-    ]
-    integration_test_steps = [
-        postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
-        mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
-    ]
-
-    # Insert remaining steps
-    build_steps.extend([
-        package_step(edition=edition, ver_mode=ver_mode),
-        grafana_server_step(edition=edition),
-        e2e_tests_step('dashboards-suite', edition=edition),
-        e2e_tests_step('smoke-tests-suite', edition=edition),
-        e2e_tests_step('panels-suite', edition=edition),
-        e2e_tests_step('various-suite', edition=edition),
-        e2e_tests_artifacts(edition=edition),
-        build_storybook_step(edition=edition, ver_mode=ver_mode),
-        store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
-        test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
-        frontend_metrics_step(edition=edition, trigger=trigger_oss),
-        copy_packages_for_docker_step(),
-        build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False),
-        build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False),
-        publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss),
-        publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss)
-    ])
-
-    build_steps.extend([
-        release_canary_npm_packages_step(edition, trigger=trigger_oss),
-        upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
-        upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
-    ])
-
-    windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
-    store_steps = [store_packages_step(edition=edition, ver_mode=ver_mode),]
-
-    return init_steps, build_steps, integration_test_steps, windows_steps, store_steps
-
-def trigger_test_release():
-    return {
-        'name': 'trigger-test-release',
-        'image': build_image,
-        'environment': {
-            'GITHUB_TOKEN': from_secret('github_token'),
-            'DOWNSTREAM_REPO': from_secret('downstream'),
-            'TEST_TAG': 'v0.0.0-test',
-        },
-        'commands': [
-            'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1',
-            'cd grafana-enterprise',
-            'git fetch origin "refs/tags/*:refs/tags/*"',
-            'git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} && git push origin $${TEST_TAG}',
-            'cd -',
-            'git fetch origin "refs/tags/*:refs/tags/*"',
-            'git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git',
-            'git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet',
-        ],
-        'failure': 'ignore',
-        'when': {
-            'paths': {
-                'include': [
-                    '.drone.yml',
-                ]
-            },
-            'repo': [
-                'grafana/grafana',
-            ]
-        }
-    }
-
-def main_pipelines(edition):
-    services = integration_test_services(edition)
-    volumes = integration_test_services_volumes()
-    drone_change_trigger = {
-        'event': ['push',],
-        'branch': 'main',
-        'repo': [
-            'grafana/grafana',
-        ],
-        'paths': {
-            'include': [
-                '.drone.yml',
-            ],
-            'exclude': [
-                'exclude',
-            ],
-        },
-    }
-    init_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition)
-
-    pipelines = [docs_pipelines(edition, ver_mode, trigger), main_test_frontend(), main_test_backend(), pipeline(
-        name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[],
-        steps=init_steps + build_steps,
-    ), pipeline(
-        name='main-integration-tests', edition=edition, trigger=trigger, services=services,
-        steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), ] + integration_test_steps,
-        volumes=volumes,
-    ), pipeline(
-        name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
-        steps=[identify_runner_step('windows')] + windows_steps,
-        depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
-    ), notify_pipeline(
-        name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
-        template=drone_change_template, secret='drone-changes-webhook',
-    ), pipeline(
-        name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
-        steps=[download_grabpl_step(), gen_version_step(ver_mode), identify_runner_step(),] + store_steps,
-        depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ],
-    ), notify_pipeline(
-        name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),
-        depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'],
-        template=failure_template, secret='slack_webhook'
-    )]
-
-    return pipelines

scripts/drone/pipelines/pr.star

@@ -1,192 +0,0 @@
-load(
-    'scripts/drone/steps/lib.star',
-    'download_grabpl_step',
-    'gen_version_step',
-    'yarn_install_step',
-    'wire_install_step',
-    'identify_runner_step',
-    'lint_drone_step',
-    'lint_backend_step',
-    'lint_frontend_step',
-    'codespell_step',
-    'shellcheck_step',
-    'build_backend_step',
-    'build_frontend_step',
-    'build_frontend_package_step',
-    'build_plugins_step',
-    'test_backend_step',
-    'test_backend_integration_step',
-    'test_frontend_step',
-    'package_step',
-    'grafana_server_step',
-    'e2e_tests_step',
-    'e2e_tests_artifacts',
-    'build_storybook_step',
-    'copy_packages_for_docker_step',
-    'build_docker_images_step',
-    'postgres_integration_tests_step',
-    'mysql_integration_tests_step',
-    'redis_integration_tests_step',
-    'memcached_integration_tests_step',
-    'benchmark_ldap_step',
-    'verify_gen_cue_step',
-    'test_a11y_frontend_step',
-    'enterprise_downstream_step',
-    'betterer_frontend_step',
-)
-
-load(
-    'scripts/drone/services/services.star',
-    'integration_test_services',
-    'integration_test_services_volumes',
-    'ldap_service',
-)
-
-load(
-    'scripts/drone/utils/utils.star',
-    'notify_pipeline',
-    'pipeline',
-    'failure_template',
-    'drone_change_template',
-)
-
-load(
-    'scripts/drone/pipelines/docs.star',
-    'docs_pipelines',
-    'trigger_docs',
-)
-
-ver_mode = 'pr'
-trigger = {
-    'event': [
-        'pull_request',
-    ],
-    'paths': {
-        'exclude': [
-            '*.md',
-            'docs/**',
-            'latest.json',
-        ],
-    },
-}
-
-def pr_verify_drone():
-    steps = [
-        identify_runner_step(),
-        download_grabpl_step(),
-        lint_drone_step(),
-    ]
-    return pipeline(
-        name='pr-verify-drone', edition="oss", trigger=get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), services=[], steps=steps,
-    )
-
-
-def pr_test_frontend():
-    init_steps = [
-        identify_runner_step(),
-        download_grabpl_step(),
-        gen_version_step(ver_mode),
-        yarn_install_step(),
-    ]
-    test_steps = [
-        lint_frontend_step(),
-        betterer_frontend_step(),
-        test_frontend_step(),
-    ]
-    return pipeline(
-        name='pr-test-frontend', edition="oss", trigger=get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), services=[], steps=init_steps + test_steps,
-    )
-
-
-def pr_test_backend():
-    init_steps = [
-        identify_runner_step(),
-        download_grabpl_step(),
-        gen_version_step(ver_mode),
-        verify_gen_cue_step(edition="oss"),
-        wire_install_step(),
-    ]
-    test_steps = [
-        codespell_step(),
-        shellcheck_step(),
-        lint_backend_step(edition="oss"),
-        test_backend_step(edition="oss"),
-        test_backend_integration_step(edition="oss"),
-    ]
-    return pipeline(
-        name='pr-test-backend', edition="oss", trigger=get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), services=[], steps=init_steps + test_steps,
-    )
-
-
-def pr_pipelines(edition):
-    services = integration_test_services(edition)
-    volumes = integration_test_services_volumes()
-    variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',]
-    init_steps = [
-        identify_runner_step(),
-        download_grabpl_step(),
-        gen_version_step(ver_mode),
-        verify_gen_cue_step(edition="oss"),
-        wire_install_step(),
-        yarn_install_step(),
-    ]
-    build_steps = [
-        enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
-        build_backend_step(edition=edition, ver_mode=ver_mode),
-        build_frontend_step(edition=edition, ver_mode=ver_mode),
-        build_frontend_package_step(edition=edition, ver_mode=ver_mode),
-        build_plugins_step(edition=edition, ver_mode=ver_mode),
-    ]
-    integration_test_steps = [
-        postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
-        mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
-    ]
-
-    # Insert remaining build_steps
-    build_steps.extend([
-        package_step(edition=edition, ver_mode=ver_mode, variants=variants),
-        grafana_server_step(edition=edition),
-        e2e_tests_step('dashboards-suite', edition=edition),
-        e2e_tests_step('smoke-tests-suite', edition=edition),
-        e2e_tests_step('panels-suite', edition=edition),
-        e2e_tests_step('various-suite', edition=edition),
-        e2e_tests_artifacts(edition=edition),
-        build_storybook_step(edition=edition, ver_mode=ver_mode),
-        test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
-        copy_packages_for_docker_step(),
-        build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ]),
-    ])
-
-    return [
-        pr_verify_drone(),
-        pr_test_frontend(),
-        pr_test_backend(),
-        pipeline(
-            name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=init_steps + build_steps,
-        ), pipeline(
-            name='pr-integration-tests', edition=edition, trigger=trigger, services=services,
-            steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), ] + integration_test_steps,
-            volumes=volumes,
-        ), docs_pipelines(edition, ver_mode, trigger_docs())
-    ]
-
-
-def get_pr_trigger(include_paths=None, exclude_paths=None):
-    paths_ex = ['docs/**', '*.md']
-    paths_in = []
-    if include_paths:
-        for path in include_paths:
-            paths_in.extend([path])
-    if exclude_paths:
-        for path in exclude_paths:
-            paths_ex.extend([path])
-    return {
-        'event': [
-            'pull_request',
-        ],
-        'paths': {
-            'exclude': paths_ex,
-            'include': paths_in,
-        },
-    }
-

scripts/drone/pipelines/publish.star

@@ -0,0 +1,25 @@
+load(
+    'scripts/drone/steps/lib.star',
+    'identify_runner_step',
+    'gen_version_step',
+    'download_grabpl_step',
+    'store_packages_step'
+)
+
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+)
+
+def publish(trigger, ver_mode, edition):
+    steps = [
+        download_grabpl_step(),
+        gen_version_step(ver_mode),
+        identify_runner_step(),
+        store_packages_step(edition=edition, ver_mode=ver_mode),
+    ]
+    return pipeline(
+                   name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
+                   steps=steps,
+                   depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ],
+               )

scripts/drone/pipelines/test_backend.star

@@ -0,0 +1,41 @@
+load(
+    'scripts/drone/steps/lib.star',
+    'identify_runner_step',
+    'download_grabpl_step',
+    'gen_version_step',
+    'wire_install_step',
+    'codespell_step',
+    'shellcheck_step',
+    'lint_backend_step',
+    'lint_drone_step',
+    'test_backend_step',
+    'test_backend_integration_step',
+    'verify_gen_cue_step',
+)
+
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+)
+
+def test_backend(trigger, ver_mode):
+    init_steps = [
+        identify_runner_step(),
+        download_grabpl_step(),
+        gen_version_step(ver_mode),
+        verify_gen_cue_step(edition="oss"),
+        wire_install_step(),
+    ]
+    test_steps = [
+        codespell_step(),
+        shellcheck_step(),
+        lint_backend_step(edition="oss"),
+        test_backend_step(edition="oss"),
+        test_backend_integration_step(edition="oss"),
+    ]
+    if ver_mode == 'main':
+        test_steps.extend([lint_drone_step()])
+
+    return pipeline(
+        name='{}-test-backend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
+    )

scripts/drone/pipelines/test_frontend.star

@@ -0,0 +1,31 @@
+load(
+    'scripts/drone/steps/lib.star',
+    'identify_runner_step',
+    'download_grabpl_step',
+    'gen_version_step',
+    'yarn_install_step',
+    'lint_frontend_step',
+    'betterer_frontend_step',
+    'test_frontend_step',
+)
+
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+)
+
+def test_frontend(trigger, ver_mode):
+    init_steps = [
+        identify_runner_step(),
+        download_grabpl_step(),
+        gen_version_step(ver_mode),
+        yarn_install_step(),
+    ]
+    test_steps = [
+        lint_frontend_step(),
+        betterer_frontend_step(),
+        test_frontend_step(),
+    ]
+    return pipeline(
+        name='{}-test-frontend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
+    )

scripts/drone/pipelines/verify_drone.star

@@ -0,0 +1,21 @@
+load(
+    'scripts/drone/steps/lib.star',
+    'identify_runner_step',
+    'download_grabpl_step',
+    'lint_drone_step',
+)
+
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+)
+
+def verify_drone(trigger, ver_mode):
+    steps = [
+        identify_runner_step(),
+        download_grabpl_step(),
+        lint_drone_step(),
+    ]
+    return pipeline(
+        name='{}-verify-drone'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=steps,
+    )

scripts/drone/pipelines/windows.star

@@ -0,0 +1,137 @@
+load(
+    'scripts/drone/steps/lib.star',
+    'grabpl_version',
+    'wix_image',
+    'identify_runner_step',
+)
+
+load(
+    'scripts/drone/utils/utils.star',
+    'pipeline',
+)
+
+load('scripts/drone/vault.star', 'from_secret', 'prerelease_bucket', 'github_token')
+
+def windows(trigger, edition, ver_mode):
+    init_cmds = []
+    sfx = ''
+    if edition in ('enterprise', 'enterprise2'):
+        sfx = '-{}'.format(edition)
+    else:
+        init_cmds.extend([
+            '$$ProgressPreference = "SilentlyContinue"',
+            'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
+                grabpl_version),
+        ])
+    steps = [
+        {
+            'name': 'windows-init',
+            'image': wix_image,
+            'commands': init_cmds,
+        },
+    ]
+    if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
+        'release', 'release-branch',
+    ):
+        bucket_part = ''
+        bucket = '%PRERELEASE_BUCKET%/artifacts/downloads'
+        if ver_mode == 'release':
+            ver_part = '${DRONE_TAG}'
+            dir = 'release'
+        else:
+            dir = 'main'
+            bucket = 'grafana-downloads'
+            bucket_part = ' --packages-bucket {}'.format(bucket)
+            build_no = 'DRONE_BUILD_NUMBER'
+            ver_part = '--build-id $$env:{}'.format(build_no)
+        installer_commands = [
+            '$$gcpKey = $$env:GCP_KEY',
+            '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json',
+            # gcloud fails to read the file unless converted with dos2unix
+            'dos2unix gcpkey.json',
+            'gcloud auth activate-service-account --key-file=gcpkey.json',
+            'rm gcpkey.json',
+            'cp C:\\App\\nssm-2.24.zip .',
+        ]
+        if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
+            'release',
+        ):
+            installer_commands.extend([
+                '.\\grabpl.exe gen-version {}'.format(ver_part),
+                '.\\grabpl.exe windows-installer --edition {}{} {}'.format(edition, bucket_part, ver_part),
+                '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
+            ])
+            if ver_mode == 'main':
+                installer_commands.extend([
+                    'gsutil cp $$fname gs://{}/{}/{}/'.format(bucket, edition, dir),
+                    'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(bucket, edition, dir),
+                ])
+            else:
+                installer_commands.extend([
+                    'gsutil cp $$fname gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir),
+                    'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir),
+                ])
+        steps.append({
+            'name': 'build-windows-installer',
+            'image': wix_image,
+            'depends_on': [
+                'windows-init',
+            ],
+            'environment': {
+                'GCP_KEY': from_secret('gcp_key'),
+                'PRERELEASE_BUCKET': from_secret(prerelease_bucket),
+                'GITHUB_TOKEN': from_secret('github_token')
+            },
+            'commands': installer_commands,
+        })
+
+    if edition in ('enterprise', 'enterprise2'):
+        if ver_mode == 'release':
+            committish = '${DRONE_TAG}'
+        elif ver_mode == 'release-branch':
+            committish = '$$env:DRONE_BRANCH'
+        else:
+            committish = '$$env:DRONE_COMMIT'
+        # For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
+        download_grabpl_step_cmds = [
+            '$$ProgressPreference = "SilentlyContinue"',
+            'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
+                grabpl_version),
+        ]
+        clone_cmds = [
+            'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
+        ]
+        clone_cmds.extend([
+            'cd grafana-enterprise',
+            'git checkout {}'.format(committish),
+        ])
+        steps.insert(0, {
+            'name': 'clone',
+            'image': wix_image,
+            'environment': {
+                'GITHUB_TOKEN': from_secret(github_token),
+            },
+            'commands': download_grabpl_step_cmds + clone_cmds,
+        })
+        steps[1]['depends_on'] = [
+            'clone',
+        ]
+        steps[1]['commands'].extend([
+            # Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
+            'cp -r grafana-enterprise C:\\App\\grafana-enterprise',
+            'rm -r -force grafana-enterprise',
+            'cp grabpl.exe C:\\App\\grabpl.exe',
+            'rm -force grabpl.exe',
+            'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise',
+            'cp C:\\App\\grabpl.exe grabpl.exe',
+        ])
+        if 'environment' in steps[1]:
+            steps[1]['environment'] + {'GITHUB_TOKEN': from_secret(github_token)}
+        else:
+            steps[1]['environment'] = {'GITHUB_TOKEN': from_secret(github_token)}
+
+    return pipeline(
+        name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
+        steps=[identify_runner_step('windows')] + steps,
+        depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
+    )

scripts/drone/steps/lib.star

@@ -1173,6 +1173,38 @@ def verify_gen_cue_step(edition):
         ],
     }
 
+def trigger_test_release():
+    return {
+        'name': 'trigger-test-release',
+        'image': build_image,
+        'environment': {
+            'GITHUB_TOKEN': from_secret('github_token'),
+            'DOWNSTREAM_REPO': from_secret('downstream'),
+            'TEST_TAG': 'v0.0.0-test',
+        },
+        'commands': [
+            'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1',
+            'cd grafana-enterprise',
+            'git fetch origin "refs/tags/*:refs/tags/*"',
+            'git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} && git push origin $${TEST_TAG}',
+            'cd -',
+            'git fetch origin "refs/tags/*:refs/tags/*"',
+            'git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git',
+            'git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet',
+        ],
+        'failure': 'ignore',
+        'when': {
+            'paths': {
+                'include': [
+                    '.drone.yml',
+                ]
+            },
+            'repo': [
+                'grafana/grafana',
+            ]
+        }
+    }
+
 def artifacts_page_step():
     return {
         'name': 'artifacts-page',

scripts/drone/version.star

@@ -1,5 +1,5 @@
 load(
-    'scripts/drone/pipelines/release.star',
+    'scripts/drone/events/release.star',
     'release_pipelines',
 )