Split pipelines on main builds (#43324)

.drone.yml

@@ -408,33 +408,13 @@ volumes:
 ---
 depends_on: []
 kind: pipeline
-name: build-main
+name: main-test
 node:
   type: no-parallel
 platform:
   arch: amd64
   os: linux
-services:
+services: []
-- environment:
-    PGDATA: /var/lib/postgresql/data/pgdata
-    POSTGRES_DB: grafanatest
-    POSTGRES_PASSWORD: grafanatest
-    POSTGRES_USER: grafanatest
-  image: postgres:12.3-alpine
-  name: postgres
-  volumes:
-  - name: postgres
-    path: /var/lib/postgresql/data/pgdata
-- environment:
-    MYSQL_DATABASE: grafana_tests
-    MYSQL_PASSWORD: password
-    MYSQL_ROOT_PASSWORD: rootpass
-    MYSQL_USER: grafana
-  image: mysql:5.6.48
-  name: mysql
-  volumes:
-  - name: mysql
-    path: /var/lib/mysql
 steps:
 - commands:
   - mkdir -p bin
@@ -458,17 +438,6 @@ steps:
   - grabpl
   image: byrnedo/alpine-curl:0.1.8
   name: lint-drone
-- image: grafana/drone-downstream
-  name: trigger-enterprise-downstream
-  settings:
-    params:
-    - SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}
-    - SOURCE_COMMIT=${DRONE_COMMIT}
-    repositories:
-    - grafana/grafana-enterprise@main
-    server: https://drone.grafana.net
-    token:
-      from_secret: drone_token
 - commands:
   - |-
     echo -e "unknwon
@@ -528,36 +497,55 @@ steps:
     TEST_MAX_WORKERS: 50%
   image: grafana/build-container:1.4.8
   name: test-frontend
+trigger:
+  branch: main
+  event:
+  - push
+type: docker
+volumes:
+- name: cypress_cache
+  temp: {}
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+depends_on: []
+kind: pipeline
+name: main-build-e2e-publish
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
 - commands:
-  - apt-get update
+  - mkdir -p bin
-  - apt-get install -yq postgresql-client
+  - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.5/grabpl
-  - dockerize -wait tcp://postgres:5432 -timeout 120s
+  - chmod +x bin/grabpl
-  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
+  image: byrnedo/alpine-curl:0.1.8
-  - go clean -testcache
+  name: grabpl
-  - ./bin/grabpl integration-tests --database postgres
-  depends_on:
-  - grabpl
-  environment:
-    GRAFANA_TEST_DB: postgres
-    PGPASSWORD: grafanatest
-    POSTGRES_HOST: postgres
-  image: grafana/build-container:1.4.8
-  name: postgres-integration-tests
 - commands:
-  - apt-get update
+  - echo $DRONE_RUNNER_NAME
-  - apt-get install -yq default-mysql-client
+  image: alpine:3.14.3
-  - dockerize -wait tcp://mysql:3306 -timeout 120s
+  name: identify-runner
-  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
+- commands:
-    -prootpass
+  - make gen-go
-  - go clean -testcache
+  - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER}
-  - ./bin/grabpl integration-tests --database mysql
+  - yarn install --immutable
-  depends_on:
-  - grabpl
-  environment:
-    GRAFANA_TEST_DB: mysql
-    MYSQL_HOST: mysql
   image: grafana/build-container:1.4.8
-  name: mysql-integration-tests
+  name: initialize
+- image: grafana/drone-downstream
+  name: trigger-enterprise-downstream
+  settings:
+    params:
+    - SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}
+    - SOURCE_COMMIT=${DRONE_COMMIT}
+    repositories:
+    - grafana/grafana-enterprise@main
+    server: https://drone.grafana.net
+    token:
+      from_secret: drone_token
 - commands:
   - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER}
     --no-pull-enterprise
@@ -834,8 +822,84 @@ volumes:
   temp:
     medium: memory
 ---
+depends_on: []
+kind: pipeline
+name: main-integration-tests
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - mkdir -p bin
+  - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.5/grabpl
+  - chmod +x bin/grabpl
+  image: byrnedo/alpine-curl:0.1.8
+  name: grabpl
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.14.3
+  name: identify-runner
+- commands:
+  - make gen-go
+  - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER}
+  - yarn install --immutable
+  image: grafana/build-container:1.4.8
+  name: initialize
+- commands:
+  - apt-get update
+  - apt-get install -yq postgresql-client
+  - dockerize -wait tcp://postgres:5432 -timeout 120s
+  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
+  - go clean -testcache
+  - ./bin/grabpl integration-tests --database postgres
+  depends_on:
+  - grabpl
+  environment:
+    GRAFANA_TEST_DB: postgres
+    PGPASSWORD: grafanatest
+    POSTGRES_HOST: postgres
+  image: grafana/build-container:1.4.8
+  name: postgres-integration-tests
+- commands:
+  - apt-get update
+  - apt-get install -yq default-mysql-client
+  - dockerize -wait tcp://mysql:3306 -timeout 120s
+  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
+    -prootpass
+  - go clean -testcache
+  - ./bin/grabpl integration-tests --database mysql
+  depends_on:
+  - grabpl
+  environment:
+    GRAFANA_TEST_DB: mysql
+    MYSQL_HOST: mysql
+  image: grafana/build-container:1.4.8
+  name: mysql-integration-tests
+trigger:
+  branch: main
+  event:
+  - push
+type: docker
+volumes:
+- name: cypress_cache
+  temp: {}
+- host:
+    path: /var/run/docker.sock
+  name: docker
+- name: postgres
+  temp:
+    medium: memory
+- name: mysql
+  temp:
+    medium: memory
+---
 depends_on:
-- build-main
+- main-test
+- main-build-e2e-publish
+- main-integration-tests
 kind: pipeline
 name: windows-main
 platform:
@@ -919,7 +983,9 @@ trigger:
 type: docker
 ---
 depends_on:
-- build-main
+- main-test
+- main-build-e2e-publish
+- main-integration-tests
 - windows-main
 kind: pipeline
 name: publish-main
@@ -976,7 +1042,9 @@ volumes:
   name: docker
 ---
 depends_on:
-- build-main
+- main-test
+- main-build-e2e-publish
+- main-integration-tests
 - windows-main
 - publish-main
 kind: pipeline
@@ -4221,6 +4289,6 @@ kind: secret
 name: prerelease_bucket
 ---
 kind: signature
-hmac: 4a5d47ae5989a19e3a5f204853c7c61f7af67e0f712138b92afc4770f61d4223
+hmac: f551ccae0072c77a25a540bcc61f49946d972a60d67a2987b40cfdf3e6c87e04
 
 ...

scripts/drone/pipelines/main.star

@@ -57,11 +57,11 @@ load(
 ver_mode = 'main'
 
 def get_steps(edition, is_downstream=False):
+    services = integration_test_services(edition)
     publish = edition != 'enterprise' or is_downstream
     include_enterprise2 = edition == 'enterprise'
-    steps = [
+    test_steps = [
         lint_drone_step(),
-        enterprise_downstream_step(edition=edition),
         codespell_step(),
         shellcheck_step(),
         lint_backend_step(edition=edition),
@@ -69,26 +69,35 @@ def get_steps(edition, is_downstream=False):
         test_backend_step(edition=edition),
         test_backend_integration_step(edition=edition),
         test_frontend_step(),
-        postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
+    ]
-        mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
+    build_steps = [
+        enterprise_downstream_step(edition=edition),
         build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
         build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
         build_plugins_step(edition=edition, sign=True),
         validate_scuemata_step(),
         ensure_cuetsified_step(),
     ]
+    integration_test_steps = [
+        postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
+        mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
+    ]
 
     if include_enterprise2:
         edition2 = 'enterprise2'
-        steps.extend([
+        build_steps.append(benchmark_ldap_step())
+        services.append(ldap_service())
+        test_steps.extend([
             lint_backend_step(edition=edition2),
             test_backend_step(edition=edition2),
             test_backend_integration_step(edition=edition2),
+        ])
+        build_steps.extend([
             build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream),
         ])
 
     # Insert remaining steps
-    steps.extend([
+    build_steps.extend([
         package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream),
         install_cypress_step(),
         e2e_tests_server_step(edition=edition),
@@ -107,9 +116,9 @@ def get_steps(edition, is_downstream=False):
     ])
 
     if include_enterprise2:
-      steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)])
+      integration_test_steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)])
 
-    steps.extend([
+    build_steps.extend([
         release_canary_npm_packages_step(edition),
         upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
         upload_cdn_step(edition=edition, ver_mode=ver_mode)
@@ -117,7 +126,7 @@ def get_steps(edition, is_downstream=False):
 
     if include_enterprise2:
         edition2 = 'enterprise2'
-        steps.extend([
+        build_steps.extend([
             package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64'], is_downstream=is_downstream),
             upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream),
             upload_cdn_step(edition=edition2, ver_mode=ver_mode)
@@ -131,7 +140,7 @@ def get_steps(edition, is_downstream=False):
             publish_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
         ]
 
-    return steps, windows_steps, publish_steps
+    return test_steps, build_steps, integration_test_steps, windows_steps, publish_steps
 
 def main_pipelines(edition):
     services = integration_test_services(edition)
@@ -152,22 +161,32 @@ def main_pipelines(edition):
             ],
         },
     }
-    steps, windows_steps, publish_steps = get_steps(edition=edition)
+    test_steps, build_steps, integration_test_steps, windows_steps, publish_steps = get_steps(edition=edition)
 
     if edition == 'enterprise':
-        steps.append(benchmark_ldap_step())
         services.append(ldap_service())
+        integration_test_steps.append(benchmark_ldap_step())
 
     pipelines = [
         pipeline(
-            name='build-main', edition=edition, trigger=trigger, services=services,
+            name='main-test', edition=edition, trigger=trigger, services=[],
-            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + test_steps,
+            volumes=[],
+        ),
+        pipeline(
+            name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[],
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps,
+            volumes=volumes,
+        ),
+        pipeline(
+            name='main-integration-tests', edition=edition, trigger=trigger, services=[],
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps,
             volumes=volumes,
         ),
         pipeline(
             name='windows-main', edition=edition, trigger=trigger,
             steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
-            depends_on=['build-main'], platform='windows',
+            depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
         ), notify_pipeline(
             name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, template=drone_change_template, secret='drone-changes-webhook',
         ),
@@ -176,22 +195,22 @@ def main_pipelines(edition):
         pipelines.append(pipeline(
             name='publish-main', edition=edition, trigger=trigger,
             steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, install_deps=False) + publish_steps,
-            depends_on=['build-main', 'windows-main',],
+            depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main',],
         ))
 
         pipelines.append(notify_pipeline(
             name='notify-main', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status = ['failure']),
-            depends_on=['build-main', 'windows-main', 'publish-main'], template=failure_template, secret='slack_webhook'
+            depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', 'publish-main'], template=failure_template, secret='slack_webhook'
         ))
     else:
         # Add downstream enterprise pipelines triggerable from OSS builds
         trigger = {
             'event': ['custom',],
         }
-        steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
+        test_steps, build_steps, integration_test_steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
         pipelines.append(pipeline(
             name='build-main-downstream', edition=edition, trigger=trigger, services=services,
-            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + test_steps + build_steps + integration_test_steps,
             volumes=volumes,
         ))
         pipelines.append(pipeline(