apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Build: Speed up integration tests (#43045)

Browse Source 
Changes so that postgres/mysql services in Drone maps their data volumes 
to tmpfs volumes (memory) to improve performance.
pull/43153/head
Marcus Efraimsson 4 years ago committed by GitHub
parent 6724c42e84
commit 5bec46ce23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 136 additions and 3 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 106
      .drone.yml
  2. 4
      scripts/drone/pipelines/main.star
  3. 6
      scripts/drone/pipelines/pr.star
  4. 5
      scripts/drone/pipelines/release.star
  5. 15
      scripts/drone/services/services.star
  6. 3
      scripts/drone/utils/utils.star

106
.drone.yml
Unescape View File

@ -322,11 +322,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@ -334,6 +338,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@ -391,6 +398,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on: []
kind: pipeline
@ -402,11 +415,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@ -414,6 +431,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@ -805,6 +825,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- build-main
@ -985,11 +1011,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@ -997,6 +1027,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@ -1357,6 +1390,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- oss-build-release
@ -1429,11 +1468,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@ -1441,6 +1484,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
- environment: {}
image: redis:6.2.1-alpine
name: redis
@ -1883,6 +1929,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
clone:
disable: true
@ -1975,11 +2027,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@ -1987,6 +2043,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@ -2324,6 +2383,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- oss-build-test-release
@ -2394,11 +2459,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@ -2406,6 +2475,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
- environment: {}
image: redis:6.2.1-alpine
name: redis
@ -2844,6 +2916,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
clone:
disable: true
@ -3045,11 +3123,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@ -3057,6 +3139,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@ -3376,6 +3461,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- oss-build-release-branch
@ -3441,11 +3532,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@ -3453,6 +3548,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
- environment: {}
image: redis:6.2.1-alpine
name: redis
@ -3895,6 +3993,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
clone:
disable: true
@ -4109,6 +4213,6 @@ kind: secret
name: prerelease_bucket
---
kind: signature
hmac: f5a6e496e7c71454eebf1a74ee83030fa3bab8dd721a2c228e8de2fdbc1e47de
hmac: 095d741ce6f28b818cbf059f4cf19d5403a4e451fabfcad111c8674124e10db9
...

4
scripts/drone/pipelines/main.star
Unescape View File

@ -42,6 +42,7 @@ load(
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
@ -134,6 +135,7 @@ def get_steps(edition, is_downstream=False):
def main_pipelines(edition):
services = integration_test_services(edition)
volumes = integration_test_services_volumes()
trigger = {
'event': ['push',],
'branch': 'main',
@ -160,6 +162,7 @@ def main_pipelines(edition):
pipeline(
name='build-main', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
volumes=volumes,
),
pipeline(
name='windows-main', edition=edition, trigger=trigger,
@ -189,6 +192,7 @@ def main_pipelines(edition):
pipelines.append(pipeline(
name='build-main-downstream', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps,
volumes=volumes,
))
pipelines.append(pipeline(
name='windows-main-downstream', edition=edition, trigger=trigger,

6
scripts/drone/pipelines/pr.star
Unescape View File

@ -35,6 +35,7 @@ load(
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
@ -50,6 +51,7 @@ ver_mode = 'pr'
def pr_pipelines(edition):
services = integration_test_services(edition)
volumes = integration_test_services_volumes()
variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64', 'armv6',]
include_enterprise2 = edition == 'enterprise'
test_steps = [
@ -125,6 +127,8 @@ def pr_pipelines(edition):
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+ build_steps,
), pipeline(
name='pr-integration-tests', edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps,
name='pr-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps,
volumes=volumes,
),
]

5
scripts/drone/pipelines/release.star
Unescape View File

@ -41,6 +41,7 @@ load(
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
@ -205,12 +206,14 @@ def get_steps(edition, ver_mode):
def get_oss_pipelines(trigger, ver_mode):
edition = 'oss'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
build_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
return [
pipeline(
name='oss-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
build_steps + package_steps + publish_steps,
volumes=volumes,
),
pipeline(
name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
@ -222,12 +225,14 @@ def get_oss_pipelines(trigger, ver_mode):
def get_enterprise_pipelines(trigger, ver_mode):
edition = 'enterprise'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
build_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
return [
pipeline(
name='enterprise-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
build_steps + package_steps + publish_steps,
volumes=volumes,
),
pipeline(
name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,

15
scripts/drone/services/services.star
Unescape View File

@ -1,3 +1,9 @@
def integration_test_services_volumes():
return [
{ 'name': 'postgres', 'temp': { 'medium': 'memory' } },
{ 'name': 'mysql', 'temp': { 'medium': 'memory' }
}]
def integration_test_services(edition):
services = [
{
@ -7,7 +13,12 @@ def integration_test_services(edition):
'POSTGRES_USER': 'grafanatest',
'POSTGRES_PASSWORD': 'grafanatest',
'POSTGRES_DB': 'grafanatest',
'PGDATA': '/var/lib/postgresql/data/pgdata',
},
'volumes': [{
'name': 'postgres',
'path': '/var/lib/postgresql/data/pgdata'
}],
},
{
'name': 'mysql',
@ -18,6 +29,10 @@ def integration_test_services(edition):
'MYSQL_USER': 'grafana',
'MYSQL_PASSWORD': 'password',
},
'volumes': [{
'name': 'mysql',
'path': '/var/lib/mysql'
}]
},
]

3
scripts/drone/utils/utils.star
Unescape View File

@ -10,7 +10,7 @@ failure_template = 'Build {{build.number}} failed for commit: <https://github.co
drone_change_template = '`.drone.yml` and `starlark` files have been changed on the OSS repo, by: {{build.author}}. \nBranch: <https://github.com/{{ repo.owner }}/{{ repo.name }}/commits/{{ build.branch }}|{{ build.branch }}>\nCommit hash: <https://github.com/{{repo.owner}}/{{repo.name}}/commit/{{build.commit}}|{{ truncate build.commit 8 }}>'
def pipeline(
name, edition, trigger, steps, services=[], platform='linux', depends_on=[],
name, edition, trigger, steps, services=[], platform='linux', depends_on=[], volumes=[],
):
if platform != 'windows':
platform_conf = {
@ -51,6 +51,7 @@ def pipeline(
}],
'depends_on': depends_on,
}
pipeline['volumes'].extend(volumes)
pipeline.update(platform_conf)
if edition in ('enterprise', 'enterprise2'):

Write Preview
Loading…
Cancel
Save