Split pipelines on main builds (#43324)

3 years ago · 2fd2fd353e
parent aa47cac69f
commit 2fd2fd353e
2 changed files with 170 additions and 83 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -408,33 +408,13 @@ volumes:
 ---
 depends_on: []
 kind: pipeline
-name: build-main
+name: main-test
 node:
  type: no-parallel
 platform:
  arch: amd64
  os: linux
-services:
- environment:
-    PGDATA: /var/lib/postgresql/data/pgdata
-    POSTGRES_DB: grafanatest
-    POSTGRES_PASSWORD: grafanatest
-    POSTGRES_USER: grafanatest
-  image: postgres:12.3-alpine
-  name: postgres
-  volumes:
-  - name: postgres
-    path: /var/lib/postgresql/data/pgdata
- environment:
-    MYSQL_DATABASE: grafana_tests
-    MYSQL_PASSWORD: password
-    MYSQL_ROOT_PASSWORD: rootpass
-    MYSQL_USER: grafana
-  image: mysql:5.6.48
-  name: mysql
-  volumes:
-  - name: mysql
-    path: /var/lib/mysql
+services: []
 steps:
 - commands:
  - mkdir -p bin
@ -458,17 +438,6 @@ steps:
  - grabpl
  image: byrnedo/alpine-curl:0.1.8
  name: lint-drone
- image: grafana/drone-downstream
-  name: trigger-enterprise-downstream
-  settings:
-    params:
-    - SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}
-    - SOURCE_COMMIT=${DRONE_COMMIT}
-    repositories:
-    - grafana/grafana-enterprise@main
-    server: https://drone.grafana.net
-    token:
-      from_secret: drone_token
 - commands:
  - |-
    echo -e "unknwon
@ -528,36 +497,55 @@ steps:
    TEST_MAX_WORKERS: 50%
  image: grafana/build-container:1.4.8
  name: test-frontend
+trigger:
+  branch: main
+  event:
+  - push
+type: docker
+volumes:
+- name: cypress_cache
+  temp: {}
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+depends_on: []
+kind: pipeline
+name: main-build-e2e-publish
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
 - commands:
-  - apt-get update
-  - apt-get install -yq postgresql-client
-  - dockerize -wait tcp://postgres:5432 -timeout 120s
-  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
-  - go clean -testcache
-  - ./bin/grabpl integration-tests --database postgres
-  depends_on:
-  - grabpl
-  environment:
-    GRAFANA_TEST_DB: postgres
-    PGPASSWORD: grafanatest
-    POSTGRES_HOST: postgres
-  image: grafana/build-container:1.4.8
-  name: postgres-integration-tests
+  - mkdir -p bin
+  - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.5/grabpl
+  - chmod +x bin/grabpl
+  image: byrnedo/alpine-curl:0.1.8
+  name: grabpl
 - commands:
-  - apt-get update
-  - apt-get install -yq default-mysql-client
-  - dockerize -wait tcp://mysql:3306 -timeout 120s
-  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
-    -prootpass
-  - go clean -testcache
-  - ./bin/grabpl integration-tests --database mysql
-  depends_on:
-  - grabpl
-  environment:
-    GRAFANA_TEST_DB: mysql
-    MYSQL_HOST: mysql
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.14.3
+  name: identify-runner
+- commands:
+  - make gen-go
+  - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER}
+  - yarn install --immutable
  image: grafana/build-container:1.4.8
-  name: mysql-integration-tests
+  name: initialize
+- image: grafana/drone-downstream
+  name: trigger-enterprise-downstream
+  settings:
+    params:
+    - SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}
+    - SOURCE_COMMIT=${DRONE_COMMIT}
+    repositories:
+    - grafana/grafana-enterprise@main
+    server: https://drone.grafana.net
+    token:
+      from_secret: drone_token
 - commands:
  - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER}
    --no-pull-enterprise
@ -834,8 +822,84 @@ volumes:
  temp:
    medium: memory
 ---
+depends_on: []
+kind: pipeline
+name: main-integration-tests
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - mkdir -p bin
+  - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.5/grabpl
+  - chmod +x bin/grabpl
+  image: byrnedo/alpine-curl:0.1.8
+  name: grabpl
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.14.3
+  name: identify-runner
+- commands:
+  - make gen-go
+  - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER}
+  - yarn install --immutable
+  image: grafana/build-container:1.4.8
+  name: initialize
+- commands:
+  - apt-get update
+  - apt-get install -yq postgresql-client
+  - dockerize -wait tcp://postgres:5432 -timeout 120s
+  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
+  - go clean -testcache
+  - ./bin/grabpl integration-tests --database postgres
+  depends_on:
+  - grabpl
+  environment:
+    GRAFANA_TEST_DB: postgres
+    PGPASSWORD: grafanatest
+    POSTGRES_HOST: postgres
+  image: grafana/build-container:1.4.8
+  name: postgres-integration-tests
+- commands:
+  - apt-get update
+  - apt-get install -yq default-mysql-client
+  - dockerize -wait tcp://mysql:3306 -timeout 120s
+  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
+    -prootpass
+  - go clean -testcache
+  - ./bin/grabpl integration-tests --database mysql
+  depends_on:
+  - grabpl
+  environment:
+    GRAFANA_TEST_DB: mysql
+    MYSQL_HOST: mysql
+  image: grafana/build-container:1.4.8
+  name: mysql-integration-tests
+trigger:
+  branch: main
+  event:
+  - push
+type: docker
+volumes:
+- name: cypress_cache
+  temp: {}
+- host:
+    path: /var/run/docker.sock
+  name: docker
+- name: postgres
+  temp:
+    medium: memory
+- name: mysql
+  temp:
+    medium: memory
+---
 depends_on:
- build-main
+- main-test
+- main-build-e2e-publish
+- main-integration-tests
 kind: pipeline
 name: windows-main
 platform:
@ -919,7 +983,9 @@ trigger:
 type: docker
 ---
 depends_on:
- build-main
+- main-test
+- main-build-e2e-publish
+- main-integration-tests
 - windows-main
 kind: pipeline
 name: publish-main
@ -976,7 +1042,9 @@ volumes:
  name: docker
 ---
 depends_on:
- build-main
+- main-test
+- main-build-e2e-publish
+- main-integration-tests
 - windows-main
 - publish-main
 kind: pipeline
@ -4221,6 +4289,6 @@ kind: secret
 name: prerelease_bucket
 ---
 kind: signature
-hmac: 4a5d47ae5989a19e3a5f204853c7c61f7af67e0f712138b92afc4770f61d4223
+hmac: f551ccae0072c77a25a540bcc61f49946d972a60d67a2987b40cfdf3e6c87e04

 ...
--- a/scripts/drone/pipelines/main.star
+++ b/scripts/drone/pipelines/main.star
@ -57,11 +57,11 @@ load(
 ver_mode = 'main'

 def get_steps(edition, is_downstream=False):
+    services = integration_test_services(edition)
    publish = edition != 'enterprise' or is_downstream
    include_enterprise2 = edition == 'enterprise'
-    steps = [
+    test_steps = [
        lint_drone_step(),
-        enterprise_downstream_step(edition=edition),
        codespell_step(),
        shellcheck_step(),
        lint_backend_step(edition=edition),
@ -69,26 +69,35 @@ def get_steps(edition, is_downstream=False):
        test_backend_step(edition=edition),
        test_backend_integration_step(edition=edition),
        test_frontend_step(),
-        postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
-        mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
+    ]
+    build_steps = [
+        enterprise_downstream_step(edition=edition),
        build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
        build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
        build_plugins_step(edition=edition, sign=True),
        validate_scuemata_step(),
        ensure_cuetsified_step(),
    ]
+    integration_test_steps = [
+        postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
+        mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
+    ]

    if include_enterprise2:
        edition2 = 'enterprise2'
-        steps.extend([
+        build_steps.append(benchmark_ldap_step())
+        services.append(ldap_service())
+        test_steps.extend([
            lint_backend_step(edition=edition2),
            test_backend_step(edition=edition2),
            test_backend_integration_step(edition=edition2),
+        ])
+        build_steps.extend([
            build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream),
        ])

    # Insert remaining steps
-    steps.extend([
+    build_steps.extend([
        package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream),
        install_cypress_step(),
        e2e_tests_server_step(edition=edition),
@ -107,9 +116,9 @@ def get_steps(edition, is_downstream=False):
    ])

    if include_enterprise2:
-      steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)])
+      integration_test_steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)])

-    steps.extend([
+    build_steps.extend([
        release_canary_npm_packages_step(edition),
        upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
        upload_cdn_step(edition=edition, ver_mode=ver_mode)
@ -117,7 +126,7 @@ def get_steps(edition, is_downstream=False):

    if include_enterprise2:
        edition2 = 'enterprise2'
-        steps.extend([
+        build_steps.extend([
            package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64'], is_downstream=is_downstream),
            upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream),
            upload_cdn_step(edition=edition2, ver_mode=ver_mode)
@ -131,7 +140,7 @@ def get_steps(edition, is_downstream=False):
            publish_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
        ]

-    return steps, windows_steps, publish_steps
+    return test_steps, build_steps, integration_test_steps, windows_steps, publish_steps

 def main_pipelines(edition):
    services = integration_test_services(edition)
@ -152,22 +161,32 @@ def main_pipelines(edition):
            ],
        },
    }
-    steps, windows_steps, publish_steps = get_steps(edition=edition)
+    test_steps, build_steps, integration_test_steps, windows_steps, publish_steps = get_steps(edition=edition)

    if edition == 'enterprise':
-        steps.append(benchmark_ldap_step())
        services.append(ldap_service())
+        integration_test_steps.append(benchmark_ldap_step())

    pipelines = [
        pipeline(
-            name='build-main', edition=edition, trigger=trigger, services=services,
-            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
+            name='main-test', edition=edition, trigger=trigger, services=[],
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + test_steps,
+            volumes=[],
+        ),
+        pipeline(
+            name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[],
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps,
+            volumes=volumes,
+        ),
+        pipeline(
+            name='main-integration-tests', edition=edition, trigger=trigger, services=[],
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps,
            volumes=volumes,
        ),
        pipeline(
            name='windows-main', edition=edition, trigger=trigger,
            steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
-            depends_on=['build-main'], platform='windows',
+            depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
        ), notify_pipeline(
            name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, template=drone_change_template, secret='drone-changes-webhook',
        ),
@ -176,22 +195,22 @@ def main_pipelines(edition):
        pipelines.append(pipeline(
            name='publish-main', edition=edition, trigger=trigger,
            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, install_deps=False) + publish_steps,
-            depends_on=['build-main', 'windows-main',],
+            depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main',],
        ))

        pipelines.append(notify_pipeline(
            name='notify-main', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status = ['failure']),
-            depends_on=['build-main', 'windows-main', 'publish-main'], template=failure_template, secret='slack_webhook'
+            depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', 'publish-main'], template=failure_template, secret='slack_webhook'
        ))
    else:
        # Add downstream enterprise pipelines triggerable from OSS builds
        trigger = {
            'event': ['custom',],
        }
-        steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
+        test_steps, build_steps, integration_test_steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
        pipelines.append(pipeline(
            name='build-main-downstream', edition=edition, trigger=trigger, services=services,
-            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + test_steps + build_steps + integration_test_steps,
            volumes=volumes,
        ))
        pipelines.append(pipeline(