apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Security Scans: Add trivy scans to every docker image used for building/testing/publishing (#69911)

Browse Source 
* Created images.star

* Fix typo

* Add cronjobs for build-images
pull/69926/head
Dimitris Sotirakis 3 years ago committed by GitHub
parent d363741d39
commit 2cda971796
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 289 additions and 123 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 84
      .drone.yml
  2. 71
      scripts/drone/events/cron.star
  3. 21
      scripts/drone/events/release.star
  4. 7
      scripts/drone/pipelines/aws_marketplace.star
  5. 12
      scripts/drone/pipelines/ci_images.star
  6. 7
      scripts/drone/pipelines/docs.star
  7. 7
      scripts/drone/pipelines/github.star
  8. 8
      scripts/drone/pipelines/shellcheck.star
  9. 7
      scripts/drone/pipelines/windows.star
  10. 17
      scripts/drone/services/services.star
  11. 142
      scripts/drone/steps/lib.star
  12. 29
      scripts/drone/utils/images.star

84
.drone.yml
Unescape View File

@ -6925,7 +6925,7 @@ steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest
image: aquasec/trivy:0.21.0
name: scan-unkown-low-medium-vulnerabilities
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest
image: aquasec/trivy:0.21.0
@ -6955,7 +6955,7 @@ steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main
image: aquasec/trivy:0.21.0
name: scan-unkown-low-medium-vulnerabilities
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main
image: aquasec/trivy:0.21.0
@ -6985,7 +6985,7 @@ steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu
image: aquasec/trivy:0.21.0
name: scan-unkown-low-medium-vulnerabilities
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu
image: aquasec/trivy:0.21.0
@ -7016,7 +7016,7 @@ steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu
image: aquasec/trivy:0.21.0
name: scan-unkown-low-medium-vulnerabilities
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu
image: aquasec/trivy:0.21.0
@ -7039,6 +7039,80 @@ type: docker
clone:
retries: 3
kind: pipeline
name: scan-build-test-and-publish-docker-images
platform:
arch: amd64
os: linux
steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/build-container:1.7.4
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.17.1
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mcr.microsoft.com/windows:1809
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/ci-wix:0.1.1
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.4
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-windows-test:0.1.0
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM memcached:1.6.9-alpine
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM docker:windowsservercore-1809
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us.gcr.io/kubernetes-dev/package-publish:latest
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM osixia/openldap:1.4.0
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/drone-downstream
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:latest
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest
image: aquasec/trivy:0.21.0
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/build-container:1.7.4
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3
- trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.17.1
- trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8
- trivy --exit-code 1 --severity HIGH,CRITICAL mcr.microsoft.com/windows:1809
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/ci-wix:0.1.1
- trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.4
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-windows-test:0.1.0
- trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack
- trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine
- trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39
- trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32
- trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine
- trivy --exit-code 1 --severity HIGH,CRITICAL memcached:1.6.9-alpine
- trivy --exit-code 1 --severity HIGH,CRITICAL docker:windowsservercore-1809
- trivy --exit-code 1 --severity HIGH,CRITICAL us.gcr.io/kubernetes-dev/package-publish:latest
- trivy --exit-code 1 --severity HIGH,CRITICAL osixia/openldap:1.4.0
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/drone-downstream
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:latest
- trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
- trivy --exit-code 1 --severity HIGH,CRITICAL us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest
image: aquasec/trivy:0.21.0
name: scan-high-critical-vulnerabilities
- image: plugins/slack
name: slack-notify-failure
settings:
channel: grafana-backend-ops
template: 'Nightly docker image scan job for build-images failed: {{build.link}}'
webhook:
from_secret: slack_webhook_backend
when:
status: failure
trigger:
cron: nightly
event: cron
type: docker
---
clone:
retries: 3
kind: pipeline
name: grafana-com-nightly
platform:
arch: amd64
@ -7242,6 +7316,6 @@ kind: secret
name: delivery-bot-app-private-key
---
kind: signature
hmac: caf02d8ce78dd5c2fd6d841d0661702e74571c07939c6af602ac11d1790b31b8
hmac: 44f95e0e3d9eb3fc8891e94f7205c18e85adacab25b69906d3c5212875baa383
...

71
scripts/drone/events/cron.star
Unescape View File

@ -6,7 +6,10 @@ load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"publish_image",
)
load(
"scripts/drone/utils/images.star",
"images",
)
aquasec_trivy_image = "aquasec/trivy:0.21.0"
@ -17,6 +20,7 @@ def cronjobs():
scan_docker_image_pipeline("main"),
scan_docker_image_pipeline("latest-ubuntu"),
scan_docker_image_pipeline("main-ubuntu"),
scan_build_test_publish_docker_image_pipeline(),
grafana_com_nightly_pipeline(),
]
@ -54,34 +58,77 @@ def scan_docker_image_pipeline(tag):
cronName = "nightly",
name = "scan-" + docker_image + "-image",
steps = [
scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image),
scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image),
scan_docker_image_high_critical_vulnerabilities_step(docker_image),
slack_job_failed_step("grafana-backend-ops", docker_image),
],
)
def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):
def scan_build_test_publish_docker_image_pipeline():
"""Generates a cronjob pipeline for nightly scans of grafana Docker images.
Returns:
Drone cronjob pipeline.
"""
return cron_job_pipeline(
cronName = "nightly",
name = "scan-build-test-and-publish-docker-images",
steps = [
scan_docker_image_unknown_low_medium_vulnerabilities_step("all"),
scan_docker_image_high_critical_vulnerabilities_step("all"),
slack_job_failed_step("grafana-backend-ops", "build-images"),
],
)
def scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image):
"""Generates a step for scans of Grafana Docker images.
Args:
docker_image: determines which image is scanned.
Returns:
Drone cronjob step .
"""
cmds = []
if docker_image == "all":
for key in images:
cmds = cmds + ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + images[key]]
else:
cmds = ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image]
return {
"name": "scan-unkown-low-medium-vulnerabilities",
"name": "scan-unknown-low-medium-vulnerabilities",
"image": aquasec_trivy_image,
"commands": [
"trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image,
],
"commands": cmds,
}
def scan_docker_image_high_critical_vulnerabilities_step(docker_image):
"""Generates a step for scans of Grafana Docker images.
Args:
docker_image: determines which image is scanned.
Returns:
Drone cronjob step .
"""
cmds = []
if docker_image == "all":
for key in images:
cmds = cmds + ["trivy --exit-code 1 --severity HIGH,CRITICAL " + images[key]]
else:
cmds = ["trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image]
return {
"name": "scan-high-critical-vulnerabilities",
"image": aquasec_trivy_image,
"commands": [
"trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image,
],
"commands": cmds,
}
def slack_job_failed_step(channel, image):
return {
"name": "slack-notify-failure",
"image": "plugins/slack",
"image": images["plugins_slack_image"],
"settings": {
"webhook": from_secret("slack_webhook_backend"),
"channel": channel,
@ -95,7 +142,7 @@ def slack_job_failed_step(channel, image):
def post_to_grafana_com_step():
return {
"name": "post-to-grafana-com",
"image": publish_image,
"image": images["publish_image"],
"environment": {
"GRAFANA_COM_API_KEY": from_secret("grafana_api_key"),
"GCP_KEY": from_secret("gcp_key"),

21
scripts/drone/events/release.star
Unescape View File

@ -9,11 +9,9 @@ load(
"build_docker_images_step",
"build_frontend_package_step",
"build_frontend_step",
"build_image",
"build_plugins_step",
"build_storybook_step",
"clone_enterprise_step",
"cloudsdk_image",
"compile_build_cmd",
"copy_packages_for_docker_step",
"download_grabpl_step",
@ -29,7 +27,6 @@ load(
"package_step",
"postgres_integration_tests_step",
"publish_grafanacom_step",
"publish_image",
"publish_images_step",
"publish_linux_packages_step",
"redis_integration_tests_step",
@ -63,6 +60,10 @@ load(
"test_backend_enterprise",
)
load("scripts/drone/vault.star", "from_secret", "prerelease_bucket")
load(
"scripts/drone/utils/images.star",
"images",
)
ver_mode = "release"
release_trigger = {
@ -84,7 +85,7 @@ release_trigger = {
def store_npm_packages_step():
return {
"name": "store-npm-packages",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
"build-frontend-packages",
@ -99,7 +100,7 @@ def store_npm_packages_step():
def retrieve_npm_packages_step():
return {
"name": "retrieve-npm-packages",
"image": publish_image,
"image": images["publish_image"],
"depends_on": [
"compile-build-cmd",
"yarn-install",
@ -115,7 +116,7 @@ def retrieve_npm_packages_step():
def release_npm_packages_step():
return {
"name": "release-npm-packages",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
"retrieve-npm-packages",
@ -554,7 +555,7 @@ def publish_artifacts_step(mode):
security = "--security "
return {
"name": "publish-artifacts",
"image": publish_image,
"image": images["publish_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@ -572,7 +573,7 @@ def publish_artifacts_step(mode):
def publish_static_assets_step():
return {
"name": "publish-static-assets",
"image": publish_image,
"image": images["publish_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@ -587,7 +588,7 @@ def publish_static_assets_step():
def publish_storybook_step():
return {
"name": "publish-storybook",
"image": publish_image,
"image": images["publish_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@ -817,7 +818,7 @@ def verify_release_pipeline(
step = {
"name": "gsutil-stat",
"depends_on": ["clone"],
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"environment": {
"BUCKET": bucket,
"GCP_KEY": gcp_key,

7
scripts/drone/pipelines/aws_marketplace.star
Unescape View File

@ -5,18 +5,21 @@ This module contains steps and pipelines publishing to AWS Marketplace.
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"publish_image",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_aws_marketplace_step():
return {
"name": "publish-aws-marketplace",
"image": publish_image,
"image": images["publish_image"],
"commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"],
"depends_on": ["compile-build-cmd"],
"environment": {

12
scripts/drone/pipelines/ci_images.star
Unescape View File

@ -2,10 +2,6 @@
This module contains steps and pipelines relating to creating CI Docker images.
"""
load(
"scripts/drone/steps/lib.star",
"wix_image",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
@ -14,6 +10,10 @@ load(
"scripts/drone/vault.star",
"from_secret",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_ci_windows_test_image_pipeline():
trigger = {
@ -28,7 +28,7 @@ def publish_ci_windows_test_image_pipeline():
steps = [
{
"name": "clone",
"image": wix_image,
"image": images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -39,7 +39,7 @@ def publish_ci_windows_test_image_pipeline():
},
{
"name": "build-and-publish",
"image": "docker:windowsservercore-1809",
"image": images["windows_server_core_image"],
"environment": {
"DOCKER_USERNAME": from_secret("docker_username"),
"DOCKER_PASSWORD": from_secret("docker_password"),

7
scripts/drone/pipelines/docs.star
Unescape View File

@ -5,11 +5,14 @@ This module returns all the pipelines used in the event of documentation changes
load(
"scripts/drone/steps/lib.star",
"build_docs_website_step",
"build_image",
"codespell_step",
"identify_runner_step",
"yarn_install_step",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
@ -46,7 +49,7 @@ def docs_pipelines(ver_mode, trigger):
def lint_docs():
return {
"name": "lint-docs",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"yarn-install",
],

7
scripts/drone/pipelines/github.star
Unescape View File

@ -6,18 +6,21 @@ load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"fetch_images_step",
"publish_image",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_github_step():
return {
"name": "publish-github",
"image": publish_image,
"image": images["publish_image"],
"commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"],
"depends_on": ["fetch-images-enterprise2"],
"environment": {

8
scripts/drone/pipelines/shellcheck.star
Unescape View File

@ -2,11 +2,15 @@
This module returns a Drone step and pipeline for linting with shellcheck.
"""
load("scripts/drone/steps/lib.star", "build_image", "compile_build_cmd")
load("scripts/drone/steps/lib.star", "compile_build_cmd")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
trigger = {
"event": [
@ -25,7 +29,7 @@ trigger = {
def shellcheck_step():
return {
"name": "shellcheck",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
],

7
scripts/drone/pipelines/windows.star
Unescape View File

@ -10,11 +10,14 @@ load(
"scripts/drone/steps/lib.star",
"get_windows_steps",
"windows_clone_step",
"windows_go_image",
"windows_init_enterprise_steps",
"windows_test_backend_step",
"windows_wire_install_step",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def windows_test_backend(trigger, edition, ver_mode):
""" Generates a pipeline that runs backend tests on Windows
@ -36,7 +39,7 @@ def windows_test_backend(trigger, edition, ver_mode):
else:
steps.extend([{
"name": "windows-init",
"image": windows_go_image,
"image": images["windows_go_image"],
"depends_on": ["clone"],
"commands": [],
}])

17
scripts/drone/services/services.star
Unescape View File

@ -2,6 +2,11 @@
This module has functions for Drone services to be used in pipelines.
"""
load(
"scripts/drone/utils/images.star",
"images",
)
def integration_test_services_volumes():
return [
{"name": "postgres", "temp": {"medium": "memory"}},
@ -13,7 +18,7 @@ def integration_test_services():
services = [
{
"name": "postgres",
"image": "postgres:12.3-alpine",
"image": images["postgres_alpine_image"],
"environment": {
"POSTGRES_USER": "grafanatest",
"POSTGRES_PASSWORD": "grafanatest",
@ -26,7 +31,7 @@ def integration_test_services():
},
{
"name": "mysql57",
"image": "mysql:5.7.39",
"image": images["mysql5_image"],
"environment": {
"MYSQL_ROOT_PASSWORD": "rootpass",
"MYSQL_DATABASE": "grafana_tests",
@ -37,7 +42,7 @@ def integration_test_services():
},
{
"name": "mysql80",
"image": "mysql:8.0.32",
"image": images["mysql8_image"],
"environment": {
"MYSQL_ROOT_PASSWORD": "rootpass",
"MYSQL_DATABASE": "grafana_tests",
@ -49,12 +54,12 @@ def integration_test_services():
},
{
"name": "redis",
"image": "redis:6.2.11-alpine",
"image": images["redis_alpine_image"],
"environment": {},
},
{
"name": "memcached",
"image": "memcached:1.6.9-alpine",
"image": images["memcached_alpine_image"],
"environment": {},
},
]
@ -64,7 +69,7 @@ def integration_test_services():
def ldap_service():
return {
"name": "ldap",
"image": "osixia/openldap:1.4.0",
"image": images["openldap_image"],
"environment": {
"LDAP_ADMIN_PASSWORD": "grafana",
"LDAP_DOMAIN": "grafana.org",

142
scripts/drone/steps/lib.star
Unescape View File

@ -7,18 +7,12 @@ load(
"from_secret",
"prerelease_bucket",
)
load(
"scripts/drone/utils/images.star",
"images",
)
grabpl_version = "v3.0.38"
cloudsdk_image = "google/cloud-sdk:431.0.0"
build_image = "grafana/build-container:1.7.4"
publish_image = "grafana/grafana-ci-deploy:1.3.3"
deploy_docker_image = "us.gcr.io/kubernetes-dev/drone/plugins/deploy-image"
alpine_image = "alpine:3.17.1"
curl_image = "byrnedo/alpine-curl:0.1.8"
windows_image = "mcr.microsoft.com/windows:1809"
wix_image = "grafana/ci-wix:0.1.1"
go_image = "golang:1.20.4"
windows_go_image = "grafana/grafana-ci-windows-test:0.1.0"
trigger_oss = {
"repo": [
@ -29,7 +23,7 @@ trigger_oss = {
def slack_step(channel, template, secret):
return {
"name": "slack",
"image": "plugins/slack",
"image": images["plugins_slack_image"],
"settings": {
"webhook": from_secret(secret),
"channel": channel,
@ -40,7 +34,7 @@ def slack_step(channel, template, secret):
def yarn_install_step():
return {
"name": "yarn-install",
"image": build_image,
"image": images["build_image"],
"commands": [
"yarn install --immutable",
],
@ -50,7 +44,7 @@ def yarn_install_step():
def wire_install_step():
return {
"name": "wire-install",
"image": build_image,
"image": images["build_image"],
"commands": [
"make gen-go",
],
@ -62,7 +56,7 @@ def wire_install_step():
def windows_wire_install_step(edition):
return {
"name": "wire-install",
"image": windows_go_image,
"image": images["windows_go_image"],
"commands": [
"go install github.com/google/wire/cmd/wire@v0.5.0",
"wire gen -tags {} ./pkg/server".format(edition),
@ -76,7 +70,7 @@ def identify_runner_step(platform = "linux"):
if platform == "linux":
return {
"name": "identify-runner",
"image": alpine_image,
"image": images["alpine_image"],
"commands": [
"echo $DRONE_RUNNER_NAME",
],
@ -84,7 +78,7 @@ def identify_runner_step(platform = "linux"):
else:
return {
"name": "identify-runner",
"image": windows_image,
"image": images["windows_image"],
"commands": [
"echo $env:DRONE_RUNNER_NAME",
],
@ -111,7 +105,7 @@ def clone_enterprise_step(source = "${DRONE_COMMIT}"):
"""
step = {
"name": "clone-enterprise",
"image": build_image,
"image": images["build_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -137,7 +131,7 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai
"""
step = {
"name": "clone-enterprise",
"image": build_image,
"image": images["build_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -183,7 +177,7 @@ def init_enterprise_step(ver_mode):
token = ""
return {
"name": "init-enterprise",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"clone-enterprise",
"grabpl",
@ -241,7 +235,7 @@ def windows_init_enterprise_steps(ver_mode):
download_grabpl_step(platform = "windows"),
{
"name": "clone",
"image": wix_image,
"image": images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -249,7 +243,7 @@ def windows_init_enterprise_steps(ver_mode):
},
{
"name": "windows-init",
"image": wix_image,
"image": images["wix_image"],
"commands": init_cmds,
"depends_on": ["clone"],
"environment": {"GITHUB_TOKEN": from_secret("github_token")},
@ -262,7 +256,7 @@ def download_grabpl_step(platform = "linux"):
if platform == "windows":
return {
"name": "grabpl",
"image": wix_image,
"image": images["wix_image"],
"commands": [
'$$ProgressPreference = "SilentlyContinue"',
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format(
@ -273,7 +267,7 @@ def download_grabpl_step(platform = "linux"):
return {
"name": "grabpl",
"image": curl_image,
"image": images["curl_image"],
"commands": [
"mkdir -p bin",
"curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format(
@ -286,7 +280,7 @@ def download_grabpl_step(platform = "linux"):
def lint_drone_step():
return {
"name": "lint-drone",
"image": curl_image,
"image": images["curl_image"],
"commands": [
"./bin/build verify-drone",
],
@ -298,7 +292,7 @@ def lint_drone_step():
def lint_starlark_step():
return {
"name": "lint-starlark",
"image": build_image,
"image": images["build_image"],
"commands": [
"./bin/build verify-starlark .",
],
@ -325,7 +319,7 @@ def enterprise_downstream_step(ver_mode):
step = {
"name": "trigger-enterprise-downstream",
"image": "grafana/drone-downstream",
"image": images["drone_downstream_image"],
"settings": {
"server": "https://drone.grafana.net",
"token": from_secret("drone_token"),
@ -349,7 +343,7 @@ def lint_backend_step():
return {
"name": "lint-backend",
# TODO: build_image or go_image?
"image": go_image,
"image": images["go_image"],
"environment": {
# We need CGO because of go-sqlite3
"CGO_ENABLED": "1",
@ -367,7 +361,7 @@ def lint_backend_step():
def validate_modfile_step():
return {
"name": "validate-modfile",
"image": go_image,
"image": images["go_image"],
"failure": "ignore",
"commands": [
"go run scripts/modowners/modowners.go check go.mod",
@ -377,7 +371,7 @@ def validate_modfile_step():
def benchmark_ldap_step():
return {
"name": "benchmark-ldap",
"image": build_image,
"image": images["build_image"],
"environment": {
"LDAP_HOSTNAME": "ldap",
},
@ -390,7 +384,7 @@ def benchmark_ldap_step():
def build_storybook_step(ver_mode):
return {
"name": "build-storybook",
"image": build_image,
"image": images["build_image"],
"depends_on": [
# Best to ensure that this step doesn't mess with what's getting built and packaged
"build-frontend",
@ -434,7 +428,7 @@ def store_storybook_step(ver_mode, trigger = None):
step = {
"name": "store-storybook",
"image": publish_image,
"image": images["publish_image"],
"depends_on": [
"build-storybook",
] +
@ -464,7 +458,7 @@ def store_storybook_step(ver_mode, trigger = None):
def e2e_tests_artifacts():
return {
"name": "e2e-tests-artifacts-upload",
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"depends_on": [
"end-to-end-tests-dashboards-suite",
"end-to-end-tests-panels-suite",
@ -526,7 +520,7 @@ def upload_cdn_step(edition, ver_mode, trigger = None):
step = {
"name": "upload-cdn-assets" + enterprise2_suffix(edition),
"image": publish_image,
"image": images["publish_image"],
"depends_on": deps,
"environment": {
"GCP_KEY": from_secret("gcp_key"),
@ -578,7 +572,7 @@ def build_backend_step(edition, ver_mode, variants = None):
return {
"name": "build-backend" + enterprise2_suffix(edition),
"image": build_image,
"image": images["build_image"],
"depends_on": [
"wire-install",
"compile-build-cmd",
@ -613,7 +607,7 @@ def build_frontend_step(edition, ver_mode):
return {
"name": "build-frontend",
"image": build_image,
"image": images["build_image"],
"environment": {
"NODE_OPTIONS": "--max_old_space_size=8192",
},
@ -653,7 +647,7 @@ def build_frontend_package_step(edition, ver_mode):
return {
"name": "build-frontend-packages",
"image": build_image,
"image": images["build_image"],
"environment": {
"NODE_OPTIONS": "--max_old_space_size=8192",
},
@ -673,7 +667,7 @@ def build_plugins_step(edition, ver_mode):
env = None
return {
"name": "build-plugins",
"image": build_image,
"image": images["build_image"],
"environment": env,
"depends_on": [
"compile-build-cmd",
@ -685,7 +679,7 @@ def build_plugins_step(edition, ver_mode):
],
}
def test_backend_step(image = build_image):
def test_backend_step(image = images["build_image"]):
return {
"name": "test-backend",
"image": image,
@ -698,13 +692,13 @@ def test_backend_step(image = build_image):
}
def windows_test_backend_step():
step = test_backend_step(image = windows_go_image)
step = test_backend_step(image = images["windows_go_image"])
return step
def test_backend_integration_step():
return {
"name": "test-backend-integration",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"wire-install",
],
@ -729,7 +723,7 @@ def betterer_frontend_step(edition = "oss"):
deps.extend(["yarn-install"])
return {
"name": "betterer-frontend",
"image": build_image,
"image": images["build_image"],
"depends_on": deps,
"commands": [
"yarn betterer ci",
@ -752,7 +746,7 @@ def test_frontend_step(edition = "oss"):
deps.extend(["yarn-install"])
return {
"name": "test-frontend",
"image": build_image,
"image": images["build_image"],
"environment": {
"TEST_MAX_WORKERS": "50%",
},
@ -765,7 +759,7 @@ def test_frontend_step(edition = "oss"):
def lint_frontend_step():
return {
"name": "lint-frontend",
"image": build_image,
"image": images["build_image"],
"environment": {
"TEST_MAX_WORKERS": "50%",
},
@ -784,7 +778,7 @@ def verify_i18n_step():
uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again."
return {
"name": "verify-i18n",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"yarn-install",
],
@ -836,7 +830,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001):
return {
"name": "test-a11y-frontend",
# TODO which image should be used?
"image": "grafana/docker-puppeteer:1.1.0",
"image": images["docker_puppeteer_image"],
"depends_on": [
"grafana-server",
],
@ -861,7 +855,7 @@ def frontend_metrics_step(trigger = None):
"""
step = {
"name": "publish-frontend-metrics",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"test-a11y-frontend",
],
@ -880,7 +874,7 @@ def frontend_metrics_step(trigger = None):
def codespell_step():
return {
"name": "codespell",
"image": build_image,
"image": images["build_image"],
"commands": [
# Important: all words have to be in lowercase, and separated by "\n".
'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt',
@ -939,7 +933,7 @@ def package_step(edition, ver_mode):
return {
"name": "package" + enterprise2_suffix(edition),
"image": build_image,
"image": images["build_image"],
"depends_on": deps,
"environment": env,
"commands": cmds,
@ -962,7 +956,7 @@ def grafana_server_step(edition, port = 3001):
return {
"name": "grafana-server",
"image": build_image,
"image": images["build_image"],
"detach": True,
"depends_on": [
"build-plugins",
@ -982,7 +976,7 @@ def e2e_tests_step(suite, port = 3001, tries = None):
cmd += " --tries {}".format(tries)
return {
"name": "end-to-end-tests-{}".format(suite),
"image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97",
"image": images["cypress_image"],
"depends_on": [
"grafana-server",
],
@ -1034,7 +1028,7 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None):
branch = "${DRONE_SOURCE_BRANCH}".replace("/", "-")
step = {
"name": "end-to-end-tests-{}-{}".format(suite, cloud),
"image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest",
"image": images["cloud_datasources_e2e_image"],
"depends_on": [
"grafana-server",
],
@ -1048,7 +1042,7 @@ def build_docs_website_step():
return {
"name": "build-docs-website",
# Use latest revision here, since we want to catch if it breaks
"image": "grafana/docs-base:latest",
"image": images["docs_image"],
"commands": [
"mkdir -p /hugo/content/docs/grafana/latest",
"cp -r docs/sources/* /hugo/content/docs/grafana/latest/",
@ -1059,7 +1053,7 @@ def build_docs_website_step():
def copy_packages_for_docker_step(edition = None):
return {
"name": "copy-packages-for-docker",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"package" + enterprise2_suffix(edition),
],
@ -1107,7 +1101,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
return {
"name": "build-docker-images" + ubuntu_sfx,
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"depends_on": [
"copy-packages-for-docker",
"compile-build-cmd",
@ -1120,7 +1114,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
def fetch_images_step(edition):
return {
"name": "fetch-images-{}".format(edition),
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"DOCKER_USER": from_secret("docker_username"),
@ -1199,7 +1193,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):
step = {
"name": "publish-images-{}".format(name),
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"environment": environment,
"commands": [cmd],
"depends_on": deps,
@ -1224,7 +1218,7 @@ def postgres_integration_tests_step():
]
return {
"name": "postgres-integration-tests",
"image": build_image,
"image": images["build_image"],
"depends_on": ["wire-install"],
"environment": {
"PGPASSWORD": "grafanatest",
@ -1245,7 +1239,7 @@ def mysql_integration_tests_step(hostname, version):
]
return {
"name": "mysql-{}-integration-tests".format(version),
"image": build_image,
"image": images["build_image"],
"depends_on": ["wire-install"],
"environment": {
"GRAFANA_TEST_DB": "mysql",
@ -1257,7 +1251,7 @@ def mysql_integration_tests_step(hostname, version):
def redis_integration_tests_step():
return {
"name": "redis-integration-tests",
"image": build_image,
"image": images["build_image"],
"depends_on": ["wire-install"],
"environment": {
"REDIS_URL": "redis://redis:6379/0",
@ -1272,7 +1266,7 @@ def redis_integration_tests_step():
def memcached_integration_tests_step():
return {
"name": "memcached-integration-tests",
"image": build_image,
"image": images["build_image"],
"depends_on": ["wire-install"],
"environment": {
"MEMCACHED_HOSTS": "memcached:11211",
@ -1296,7 +1290,7 @@ def release_canary_npm_packages_step(trigger = None):
"""
step = {
"name": "release-canary-npm-packages",
"image": build_image,
"image": images["build_image"],
"depends_on": end_to_end_tests_deps(),
"environment": {
"NPM_TOKEN": from_secret("npm_token"),
@ -1339,7 +1333,7 @@ def upload_packages_step(edition, ver_mode, trigger = None):
"""
step = {
"name": "upload-packages" + enterprise2_suffix(edition),
"image": publish_image,
"image": images["publish_image"],
"depends_on": end_to_end_tests_deps(),
"environment": {
"GCP_KEY": from_secret("gcp_key"),
@ -1381,7 +1375,7 @@ def publish_grafanacom_step(edition, ver_mode):
return {
"name": "publish-grafanacom-{}".format(edition),
"image": publish_image,
"image": images["publish_image"],
"depends_on": [
"publish-linux-packages-deb",
"publish-linux-packages-rpm",
@ -1399,7 +1393,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"):
return {
"name": "publish-linux-packages-{}".format(package_manager),
# See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image
"image": "us.gcr.io/kubernetes-dev/package-publish:latest",
"image": images["package_publish_image"],
"depends_on": ["compile-build-cmd"],
"privileged": True,
"settings": {
@ -1421,7 +1415,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"):
def windows_clone_step():
return {
"name": "clone",
"image": wix_image,
"image": images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -1481,7 +1475,7 @@ def get_windows_steps(edition, ver_mode):
[
{
"name": "clone",
"image": wix_image,
"image": images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -1489,7 +1483,7 @@ def get_windows_steps(edition, ver_mode):
},
{
"name": "windows-init",
"image": wix_image,
"image": images["wix_image"],
"commands": init_cmds,
"depends_on": ["clone"],
"environment": {"GITHUB_TOKEN": from_secret("github_token")},
@ -1508,7 +1502,7 @@ def get_windows_steps(edition, ver_mode):
[
{
"name": "windows-init",
"image": wix_image,
"image": images["wix_image"],
"commands": init_cmds,
},
],
@ -1583,7 +1577,7 @@ def get_windows_steps(edition, ver_mode):
steps.append(
{
"name": "build-windows-installer",
"image": wix_image,
"image": images["wix_image"],
"depends_on": [
"windows-init",
],
@ -1601,7 +1595,7 @@ def get_windows_steps(edition, ver_mode):
def verify_gen_cue_step():
return {
"name": "verify-gen-cue",
"image": build_image,
"image": images["build_image"],
"depends_on": [],
"commands": [
"# It is required that code generated from Thema/CUE be committed and in sync with its inputs.",
@ -1613,7 +1607,7 @@ def verify_gen_cue_step():
def verify_gen_jsonnet_step():
return {
"name": "verify-gen-jsonnet",
"image": build_image,
"image": images["build_image"],
"depends_on": [],
"commands": [
"# It is required that generated jsonnet is committed and in sync with its inputs.",
@ -1625,7 +1619,7 @@ def verify_gen_jsonnet_step():
def trigger_test_release():
return {
"name": "trigger-test-release",
"image": build_image,
"image": images["build_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token_pr"),
"TEST_TAG": "v0.0.0-test",
@ -1659,7 +1653,7 @@ def trigger_test_release():
def artifacts_page_step():
return {
"name": "artifacts-page",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
],
@ -1687,7 +1681,7 @@ def compile_build_cmd(edition = "oss"):
]
return {
"name": "compile-build-cmd",
"image": go_image,
"image": images["go_image"],
"commands": [
"go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd",
],

29
scripts/drone/utils/images.star
Unescape View File

@ -0,0 +1,29 @@
"""
This module contains all the docker images that are used to build test and publish Grafana.
"""
images = {
"cloudsdk_image": "google/cloud-sdk:431.0.0",
"build_image": "grafana/build-container:1.7.4",
"publish_image": "grafana/grafana-ci-deploy:1.3.3",
"alpine_image": "alpine:3.17.1",
"curl_image": "byrnedo/alpine-curl:0.1.8",
"windows_image": "mcr.microsoft.com/windows:1809",
"wix_image": "grafana/ci-wix:0.1.1",
"go_image": "golang:1.20.4",
"windows_go_image": "grafana/grafana-ci-windows-test:0.1.0",
"plugins_slack_image": "plugins/slack",
"postgres_alpine_image": "postgres:12.3-alpine",
"mysql5_image": "mysql:5.7.39",
"mysql8_image": "mysql:8.0.32",
"redis_alpine_image": "redis:6.2.11-alpine",
"memcached_alpine_image": "memcached:1.6.9-alpine",
"windows_server_core_image": "docker:windowsservercore-1809",
"package_publish_image": "us.gcr.io/kubernetes-dev/package-publish:latest",
"openldap_image": "osixia/openldap:1.4.0",
"drone_downstream_image": "grafana/drone-downstream",
"docker_puppeteer_image": "grafana/docker-puppeteer:1.1.0",
"docs_image": "grafana/docs-base:latest",
"cypress_image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97",
"cloud_datasources_e2e_image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest",
}
Write Preview
Loading…
Cancel
Save