diff --git a/.drone.yml b/.drone.yml index c0bb0112b38..e4a9abcdf92 100644 --- a/.drone.yml +++ b/.drone.yml @@ -408,33 +408,13 @@ volumes: --- depends_on: [] kind: pipeline -name: build-main +name: main-test node: type: no-parallel platform: arch: amd64 os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql +services: [] steps: - commands: - mkdir -p bin @@ -458,17 +438,6 @@ steps: - grabpl image: byrnedo/alpine-curl:0.1.8 name: lint-drone -- image: grafana/drone-downstream - name: trigger-enterprise-downstream - settings: - params: - - SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER} - - SOURCE_COMMIT=${DRONE_COMMIT} - repositories: - - grafana/grafana-enterprise@main - server: https://drone.grafana.net - token: - from_secret: drone_token - commands: - |- echo -e "unknwon @@ -528,36 +497,55 @@ steps: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.4.8 name: test-frontend +trigger: + branch: main + event: + - push +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: main-build-e2e-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: - commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - ./bin/grabpl integration-tests --database postgres - depends_on: - - grabpl - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.4.8 - name: postgres-integration-tests + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root - -prootpass - - go clean -testcache - - ./bin/grabpl integration-tests --database mysql - depends_on: - - grabpl - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql + - echo $DRONE_RUNNER_NAME + image: alpine:3.14.3 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + - yarn install --immutable image: grafana/build-container:1.4.8 - name: mysql-integration-tests + name: initialize +- image: grafana/drone-downstream + name: trigger-enterprise-downstream + settings: + params: + - SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER} + - SOURCE_COMMIT=${DRONE_COMMIT} + repositories: + - grafana/grafana-enterprise@main + server: https://drone.grafana.net + token: + from_secret: drone_token - commands: - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise @@ -834,8 +822,84 @@ volumes: temp: medium: memory --- +depends_on: [] +kind: pipeline +name: main-integration-tests +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.14.3 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + - yarn install --immutable + image: grafana/build-container:1.4.8 + name: initialize +- commands: + - apt-get update + - apt-get install -yq postgresql-client + - dockerize -wait tcp://postgres:5432 -timeout 120s + - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql + - go clean -testcache + - ./bin/grabpl integration-tests --database postgres + depends_on: + - grabpl + environment: + GRAFANA_TEST_DB: postgres + PGPASSWORD: grafanatest + POSTGRES_HOST: postgres + image: grafana/build-container:1.4.8 + name: postgres-integration-tests +- commands: + - apt-get update + - apt-get install -yq default-mysql-client + - dockerize -wait tcp://mysql:3306 -timeout 120s + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root + -prootpass + - go clean -testcache + - ./bin/grabpl integration-tests --database mysql + depends_on: + - grabpl + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql + image: grafana/build-container:1.4.8 + name: mysql-integration-tests +trigger: + branch: main + event: + - push +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- depends_on: -- build-main +- main-test +- main-build-e2e-publish +- main-integration-tests kind: pipeline name: windows-main platform: @@ -919,7 +983,9 @@ trigger: type: docker --- depends_on: -- build-main +- main-test +- main-build-e2e-publish +- main-integration-tests - windows-main kind: pipeline name: publish-main @@ -976,7 +1042,9 @@ volumes: name: docker --- depends_on: -- build-main +- main-test +- main-build-e2e-publish +- main-integration-tests - windows-main - publish-main kind: pipeline @@ -4221,6 +4289,6 @@ kind: secret name: prerelease_bucket --- kind: signature -hmac: 4a5d47ae5989a19e3a5f204853c7c61f7af67e0f712138b92afc4770f61d4223 +hmac: f551ccae0072c77a25a540bcc61f49946d972a60d67a2987b40cfdf3e6c87e04 ... diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star index 1b8b418c240..7b2e1dec89a 100644 --- a/scripts/drone/pipelines/main.star +++ b/scripts/drone/pipelines/main.star @@ -57,11 +57,11 @@ load( ver_mode = 'main' def get_steps(edition, is_downstream=False): + services = integration_test_services(edition) publish = edition != 'enterprise' or is_downstream include_enterprise2 = edition == 'enterprise' - steps = [ + test_steps = [ lint_drone_step(), - enterprise_downstream_step(edition=edition), codespell_step(), shellcheck_step(), lint_backend_step(edition=edition), @@ -69,26 +69,35 @@ def get_steps(edition, is_downstream=False): test_backend_step(edition=edition), test_backend_integration_step(edition=edition), test_frontend_step(), - postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), - mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), + ] + build_steps = [ + enterprise_downstream_step(edition=edition), build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), build_plugins_step(edition=edition, sign=True), validate_scuemata_step(), ensure_cuetsified_step(), ] + integration_test_steps = [ + postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), + mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), + ] if include_enterprise2: edition2 = 'enterprise2' - steps.extend([ + build_steps.append(benchmark_ldap_step()) + services.append(ldap_service()) + test_steps.extend([ lint_backend_step(edition=edition2), test_backend_step(edition=edition2), test_backend_integration_step(edition=edition2), + ]) + build_steps.extend([ build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream), ]) # Insert remaining steps - steps.extend([ + build_steps.extend([ package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream), install_cypress_step(), e2e_tests_server_step(edition=edition), @@ -107,9 +116,9 @@ def get_steps(edition, is_downstream=False): ]) if include_enterprise2: - steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)]) + integration_test_steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)]) - steps.extend([ + build_steps.extend([ release_canary_npm_packages_step(edition), upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), upload_cdn_step(edition=edition, ver_mode=ver_mode) @@ -117,7 +126,7 @@ def get_steps(edition, is_downstream=False): if include_enterprise2: edition2 = 'enterprise2' - steps.extend([ + build_steps.extend([ package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64'], is_downstream=is_downstream), upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream), upload_cdn_step(edition=edition2, ver_mode=ver_mode) @@ -131,7 +140,7 @@ def get_steps(edition, is_downstream=False): publish_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), ] - return steps, windows_steps, publish_steps + return test_steps, build_steps, integration_test_steps, windows_steps, publish_steps def main_pipelines(edition): services = integration_test_services(edition) @@ -152,22 +161,32 @@ def main_pipelines(edition): ], }, } - steps, windows_steps, publish_steps = get_steps(edition=edition) + test_steps, build_steps, integration_test_steps, windows_steps, publish_steps = get_steps(edition=edition) if edition == 'enterprise': - steps.append(benchmark_ldap_step()) services.append(ldap_service()) + integration_test_steps.append(benchmark_ldap_step()) pipelines = [ pipeline( - name='build-main', edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps, + name='main-test', edition=edition, trigger=trigger, services=[], + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + test_steps, + volumes=[], + ), + pipeline( + name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[], + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps, + volumes=volumes, + ), + pipeline( + name='main-integration-tests', edition=edition, trigger=trigger, services=[], + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps, volumes=volumes, ), pipeline( name='windows-main', edition=edition, trigger=trigger, steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps, - depends_on=['build-main'], platform='windows', + depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows', ), notify_pipeline( name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, template=drone_change_template, secret='drone-changes-webhook', ), @@ -176,22 +195,22 @@ def main_pipelines(edition): pipelines.append(pipeline( name='publish-main', edition=edition, trigger=trigger, steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, install_deps=False) + publish_steps, - depends_on=['build-main', 'windows-main',], + depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main',], )) pipelines.append(notify_pipeline( name='notify-main', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status = ['failure']), - depends_on=['build-main', 'windows-main', 'publish-main'], template=failure_template, secret='slack_webhook' + depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', 'publish-main'], template=failure_template, secret='slack_webhook' )) else: # Add downstream enterprise pipelines triggerable from OSS builds trigger = { 'event': ['custom',], } - steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True) + test_steps, build_steps, integration_test_steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True) pipelines.append(pipeline( name='build-main-downstream', edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps, + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + test_steps + build_steps + integration_test_steps, volumes=volumes, )) pipelines.append(pipeline(