mirror of https://github.com/grafana/grafana
[v9.4.x] CI: Update CI/CD tooling and pipelines from main (#76881)
* CI: Update CI/CD tooling and pipelines from main (#76814)
* CI: Update CI/CD tooling and pipelines from main
* Update Makefile
* Comment out validate_openapi_spec_step
* Update broken frontend tests
* Fix validate-npm-packages regex to work without suffix
* Fix cypress image version
(cherry picked from commit 03ecb1db39)
* Fix path for ./pkg/kindsys/report.go on Makefile
* Re-add ./pkg/cmd/grafana-cli/runner to make gen-go
pull/77015/head
Guilherme Caulada
2 years ago
committed by
GitHub
parent
6365037e69
commit
2b54a169b2
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,80 @@ |
|||||||
|
""" |
||||||
|
This module returns the pipeline used for integration benchmarks. |
||||||
|
""" |
||||||
|
|
||||||
|
load( |
||||||
|
"scripts/drone/services/services.star", |
||||||
|
"integration_test_services", |
||||||
|
"integration_test_services_volumes", |
||||||
|
) |
||||||
|
load( |
||||||
|
"scripts/drone/steps/lib.star", |
||||||
|
"compile_build_cmd", |
||||||
|
"enterprise_setup_step", |
||||||
|
"integration_benchmarks_step", |
||||||
|
"verify_gen_cue_step", |
||||||
|
"verify_gen_jsonnet_step", |
||||||
|
"wire_install_step", |
||||||
|
) |
||||||
|
load( |
||||||
|
"scripts/drone/utils/utils.star", |
||||||
|
"pipeline", |
||||||
|
) |
||||||
|
|
||||||
|
def integration_benchmarks(prefix): |
||||||
|
"""Generate a pipeline for integration tests. |
||||||
|
|
||||||
|
Args: |
||||||
|
prefix: used in the naming of the pipeline. |
||||||
|
Returns: |
||||||
|
Drone pipeline. |
||||||
|
""" |
||||||
|
environment = {"EDITION": "oss"} |
||||||
|
|
||||||
|
services = integration_test_services() |
||||||
|
volumes = integration_test_services_volumes() |
||||||
|
|
||||||
|
# In pull requests, attempt to clone grafana enterprise. |
||||||
|
init_steps = [enterprise_setup_step(isPromote = True)] |
||||||
|
|
||||||
|
verify_step = verify_gen_cue_step() |
||||||
|
verify_jsonnet_step = verify_gen_jsonnet_step() |
||||||
|
|
||||||
|
# Ensure that verif_gen_cue happens after we clone enterprise |
||||||
|
# At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on. |
||||||
|
verify_step["depends_on"].append("clone-enterprise") |
||||||
|
verify_jsonnet_step["depends_on"].append("clone-enterprise") |
||||||
|
|
||||||
|
init_steps += [ |
||||||
|
compile_build_cmd(), |
||||||
|
verify_step, |
||||||
|
verify_jsonnet_step, |
||||||
|
wire_install_step(), |
||||||
|
] |
||||||
|
|
||||||
|
benchmark_steps = integration_benchmarks_step("sqlite") + \ |
||||||
|
integration_benchmarks_step("postgres", { |
||||||
|
"PGPASSWORD": "grafanatest", |
||||||
|
"GRAFANA_TEST_DB": "postgres", |
||||||
|
"POSTGRES_HOST": "postgres", |
||||||
|
}) + \ |
||||||
|
integration_benchmarks_step("mysql-5.7", { |
||||||
|
"GRAFANA_TEST_DB": "mysql", |
||||||
|
"MYSQL_HOST": "mysql57", |
||||||
|
}) + \ |
||||||
|
integration_benchmarks_step("mysql-8.0", { |
||||||
|
"GRAFANA_TEST_DB": "mysql", |
||||||
|
"MYSQL_HOST": "mysql80", |
||||||
|
}) |
||||||
|
|
||||||
|
return pipeline( |
||||||
|
name = "{}-integration-benchmarks".format(prefix), |
||||||
|
trigger = { |
||||||
|
"event": ["promote"], |
||||||
|
"target": ["gobenchmarks"], |
||||||
|
}, |
||||||
|
environment = environment, |
||||||
|
services = services, |
||||||
|
volumes = volumes, |
||||||
|
steps = init_steps + benchmark_steps, |
||||||
|
) |
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,187 @@ |
|||||||
|
""" |
||||||
|
This module is a library of Drone steps that exclusively run on windows machines. |
||||||
|
""" |
||||||
|
|
||||||
|
load( |
||||||
|
"scripts/drone/utils/windows_images.star", |
||||||
|
"windows_images", |
||||||
|
) |
||||||
|
load( |
||||||
|
"scripts/drone/variables.star", |
||||||
|
"grabpl_version", |
||||||
|
) |
||||||
|
load( |
||||||
|
"scripts/drone/vault.star", |
||||||
|
"from_secret", |
||||||
|
"gcp_grafanauploads_base64", |
||||||
|
"prerelease_bucket", |
||||||
|
) |
||||||
|
|
||||||
|
def identify_runner_step_windows(): |
||||||
|
return { |
||||||
|
"name": "identify-runner", |
||||||
|
"image": windows_images["1809"], |
||||||
|
"commands": [ |
||||||
|
"echo $env:DRONE_RUNNER_NAME", |
||||||
|
], |
||||||
|
} |
||||||
|
|
||||||
|
def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): |
||||||
|
"""Generate the list of Windows steps. |
||||||
|
|
||||||
|
Args: |
||||||
|
ver_mode: used to differentiate steps for different version modes. |
||||||
|
bucket: used to override prerelease bucket. |
||||||
|
|
||||||
|
Returns: |
||||||
|
List of Drone steps. |
||||||
|
""" |
||||||
|
steps = [ |
||||||
|
identify_runner_step_windows(), |
||||||
|
] |
||||||
|
|
||||||
|
init_cmds = [ |
||||||
|
'$$ProgressPreference = "SilentlyContinue"', |
||||||
|
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( |
||||||
|
grabpl_version, |
||||||
|
), |
||||||
|
] |
||||||
|
|
||||||
|
steps.extend( |
||||||
|
[ |
||||||
|
{ |
||||||
|
"name": "windows-init", |
||||||
|
"image": windows_images["wix"], |
||||||
|
"commands": init_cmds, |
||||||
|
}, |
||||||
|
], |
||||||
|
) |
||||||
|
|
||||||
|
if ver_mode in ( |
||||||
|
"release", |
||||||
|
"release-branch", |
||||||
|
): |
||||||
|
gcp_bucket = "{}/artifacts/downloads".format(bucket) |
||||||
|
if ver_mode == "release": |
||||||
|
ver_part = "${DRONE_TAG}" |
||||||
|
dir = "release" |
||||||
|
else: |
||||||
|
dir = "main" |
||||||
|
gcp_bucket = "grafana-downloads" |
||||||
|
build_no = "DRONE_BUILD_NUMBER" |
||||||
|
ver_part = "--build-id $$env:{}".format(build_no) |
||||||
|
installer_commands = [ |
||||||
|
"$$gcpKey = $$env:GCP_KEY", |
||||||
|
"[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", |
||||||
|
# gcloud fails to read the file unless converted with dos2unix |
||||||
|
"dos2unix gcpkey.json", |
||||||
|
"gcloud auth activate-service-account --key-file=gcpkey.json", |
||||||
|
"rm gcpkey.json", |
||||||
|
"cp C:\\App\\nssm-2.24.zip .", |
||||||
|
] |
||||||
|
|
||||||
|
if ver_mode in ("release",): |
||||||
|
version = "${DRONE_TAG:1}" |
||||||
|
installer_commands.extend( |
||||||
|
[ |
||||||
|
".\\grabpl.exe windows-installer --target {} --edition oss {}".format( |
||||||
|
"gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), |
||||||
|
ver_part, |
||||||
|
), |
||||||
|
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', |
||||||
|
], |
||||||
|
) |
||||||
|
if ver_mode == "main": |
||||||
|
installer_commands.extend( |
||||||
|
[ |
||||||
|
"gsutil cp $$fname gs://{}/oss/{}/".format(gcp_bucket, dir), |
||||||
|
'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( |
||||||
|
gcp_bucket, |
||||||
|
dir, |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
else: |
||||||
|
installer_commands.extend( |
||||||
|
[ |
||||||
|
"gsutil cp $$fname gs://{}/{}/oss/{}/".format( |
||||||
|
gcp_bucket, |
||||||
|
ver_part, |
||||||
|
dir, |
||||||
|
), |
||||||
|
'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format( |
||||||
|
gcp_bucket, |
||||||
|
ver_part, |
||||||
|
dir, |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
steps.append( |
||||||
|
{ |
||||||
|
"name": "build-windows-installer", |
||||||
|
"image": windows_images["wix"], |
||||||
|
"depends_on": [ |
||||||
|
"windows-init", |
||||||
|
], |
||||||
|
"environment": { |
||||||
|
"GCP_KEY": from_secret(gcp_grafanauploads_base64), |
||||||
|
"PRERELEASE_BUCKET": from_secret(prerelease_bucket), |
||||||
|
"GITHUB_TOKEN": from_secret("github_token"), |
||||||
|
}, |
||||||
|
"commands": installer_commands, |
||||||
|
}, |
||||||
|
) |
||||||
|
|
||||||
|
return steps |
||||||
|
|
||||||
|
def download_grabpl_step_windows(): |
||||||
|
return { |
||||||
|
"name": "grabpl", |
||||||
|
"image": windows_images["wix"], |
||||||
|
"commands": [ |
||||||
|
'$$ProgressPreference = "SilentlyContinue"', |
||||||
|
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( |
||||||
|
grabpl_version, |
||||||
|
), |
||||||
|
], |
||||||
|
} |
||||||
|
|
||||||
|
def test_backend_step_windows(): |
||||||
|
# TODO: This is mostly a duplicate of "test_backend_step" in lib.star; but this file can't import that one, |
||||||
|
# otherwise it creates an import cycle. |
||||||
|
return { |
||||||
|
"name": "test-backend", |
||||||
|
"image": windows_images["go"], |
||||||
|
"depends_on": [ |
||||||
|
"wire-install", |
||||||
|
], |
||||||
|
"commands": [ |
||||||
|
"go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", |
||||||
|
], |
||||||
|
} |
||||||
|
|
||||||
|
def clone_step_windows(): |
||||||
|
return { |
||||||
|
"name": "clone", |
||||||
|
"image": windows_images["wix"], |
||||||
|
"environment": { |
||||||
|
"GITHUB_TOKEN": from_secret("github_token"), |
||||||
|
}, |
||||||
|
"commands": [ |
||||||
|
'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', |
||||||
|
"git checkout -f $$env:DRONE_COMMIT", |
||||||
|
], |
||||||
|
} |
||||||
|
|
||||||
|
def wire_install_step_windows(edition): |
||||||
|
return { |
||||||
|
"name": "wire-install", |
||||||
|
"image": windows_images["go"], |
||||||
|
"commands": [ |
||||||
|
"go install github.com/google/wire/cmd/wire@v0.5.0", |
||||||
|
"wire gen -tags {} ./pkg/server".format(edition), |
||||||
|
], |
||||||
|
"depends_on": [ |
||||||
|
"windows-init", |
||||||
|
], |
||||||
|
} |
||||||
@ -0,0 +1,61 @@ |
|||||||
|
""" |
||||||
|
Individual steps that use 'grafana-build' to replace existing individual steps. |
||||||
|
These aren't used in releases. |
||||||
|
""" |
||||||
|
|
||||||
|
load( |
||||||
|
"scripts/drone/variables.star", |
||||||
|
"golang_version", |
||||||
|
) |
||||||
|
|
||||||
|
# rgm_package_step will create a tar.gz for use in e2e tests or other PR testing related activities.. |
||||||
|
def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"): |
||||||
|
return { |
||||||
|
"name": "rgm-package", |
||||||
|
"image": "grafana/grafana-build:main", |
||||||
|
"pull": "always", |
||||||
|
"depends_on": ["yarn-install"], |
||||||
|
"commands": [ |
||||||
|
"/src/grafana-build package --distro={} ".format(distros) + |
||||||
|
"--go-version={} ".format(golang_version) + |
||||||
|
"--yarn-cache=$$YARN_CACHE_FOLDER " + |
||||||
|
"--build-id=$$DRONE_BUILD_NUMBER " + |
||||||
|
"--grafana-dir=$$PWD > {}".format(file), |
||||||
|
], |
||||||
|
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], |
||||||
|
} |
||||||
|
|
||||||
|
# rgm_build_backend will create compile the grafana backend for various platforms. It's preferred to use |
||||||
|
# 'rgm_package_step' if you creating a "usable" artifact. This should really only be used to verify that the code is |
||||||
|
# compilable. |
||||||
|
def rgm_build_backend_step(distros = "linux/amd64,linux/arm64"): |
||||||
|
return { |
||||||
|
"name": "rgm-package", |
||||||
|
"image": "grafana/grafana-build:main", |
||||||
|
"pull": "always", |
||||||
|
"commands": [ |
||||||
|
"/src/grafana-build build " + |
||||||
|
"--go-version={} ".format(golang_version) + |
||||||
|
"--distro={} --grafana-dir=$$PWD".format(distros), |
||||||
|
], |
||||||
|
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], |
||||||
|
} |
||||||
|
|
||||||
|
def rgm_build_docker_step(packages, ubuntu, alpine, depends_on = ["rgm-package"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"): |
||||||
|
return { |
||||||
|
"name": "rgm-build-docker", |
||||||
|
"image": "grafana/grafana-build:main", |
||||||
|
"pull": "always", |
||||||
|
"commands": [ |
||||||
|
"docker run --privileged --rm tonistiigi/binfmt --install all", |
||||||
|
"/src/grafana-build docker " + |
||||||
|
"$(cat {} | grep tar.gz | grep -v docker | grep -v sha256 | awk '{{print \"--package=\" $0}}') ".format(packages) + |
||||||
|
"--ubuntu-base={} ".format(ubuntu) + |
||||||
|
"--alpine-base={} ".format(alpine) + |
||||||
|
"--tag-format='{}' ".format(tag_format) + |
||||||
|
"--ubuntu-tag-format='{}' > {}".format(ubuntu_tag_format, file), |
||||||
|
"find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i", |
||||||
|
], |
||||||
|
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], |
||||||
|
"depends_on": depends_on, |
||||||
|
} |
||||||
@ -0,0 +1,9 @@ |
|||||||
|
""" |
||||||
|
global variables |
||||||
|
""" |
||||||
|
|
||||||
|
grabpl_version = "v3.0.42" |
||||||
|
golang_version = "1.20.10" |
||||||
|
|
||||||
|
# nodejs_version should match what's in ".nvmrc", but without the v prefix. |
||||||
|
nodejs_version = "18.12.0" |
||||||
@ -1,16 +0,0 @@ |
|||||||
""" |
|
||||||
This module returns the pipeline used for version branches. |
|
||||||
""" |
|
||||||
|
|
||||||
load( |
|
||||||
"scripts/drone/events/release.star", |
|
||||||
"oss_pipelines", |
|
||||||
) |
|
||||||
|
|
||||||
ver_mode = "release-branch" |
|
||||||
trigger = {"ref": ["refs/heads/v[0-9]*"]} |
|
||||||
|
|
||||||
def version_branch_pipelines(): |
|
||||||
return ( |
|
||||||
oss_pipelines(ver_mode = ver_mode, trigger = trigger) |
|
||||||
) |
|
||||||
@ -0,0 +1,80 @@ |
|||||||
|
#!/bin/bash |
||||||
|
|
||||||
|
# This script is used to validate the npm packages that are published to npmjs.org are in the correct format. |
||||||
|
# It won't catch things like malformed JS or Types but it will assert that the package has |
||||||
|
# the correct files and package.json properties. |
||||||
|
ARTIFACTS_DIR="./npm-artifacts" |
||||||
|
|
||||||
|
for file in "$ARTIFACTS_DIR"/*.tgz; do |
||||||
|
echo "🔍 Checking NPM package: $file" |
||||||
|
# get filename then strip everything after package name. |
||||||
|
dir_name=$(basename "$file" .tgz | sed -E 's/@([a-zA-Z0-9-]+)-[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9-]+)?/\1/') |
||||||
|
mkdir -p "./npm-artifacts/$dir_name" |
||||||
|
tar -xzf "$file" -C "./npm-artifacts/$dir_name" --strip-components=1 |
||||||
|
|
||||||
|
# Make sure the tar wasn't empty |
||||||
|
if [ ! -d "./npm-artifacts/$dir_name" ]; then |
||||||
|
echo -e "❌ Failed: Empty package $dir_name.\n" |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
|
||||||
|
# Navigate inside the new extracted directory |
||||||
|
pushd "./npm-artifacts/$dir_name" || exit |
||||||
|
|
||||||
|
# Check for required files |
||||||
|
check_files=("package.json" "README.md" "CHANGELOG.md" "LICENSE_APACHE2") |
||||||
|
for check_file in "${check_files[@]}"; do |
||||||
|
if [ ! -f "$check_file" ]; then |
||||||
|
echo -e "❌ Failed: Missing required file $check_file in package $dir_name.\n" |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
done |
||||||
|
|
||||||
|
# @grafana/toolkit structure is different to the other packages |
||||||
|
if [[ "$dir_name" == "grafana-toolkit" ]]; then |
||||||
|
if [ ! -d bin ] || [ ! -f bin/grafana-toolkit.js ]; then |
||||||
|
echo -e "❌ Failed: Missing 'bin' directory or required files in package $dir_name.\n" |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
|
||||||
|
echo -e "✅ Passed: package checks for $file.\n" |
||||||
|
popd || exit |
||||||
|
continue |
||||||
|
fi |
||||||
|
|
||||||
|
# Assert commonjs builds |
||||||
|
if [ ! -d dist ] || [ ! -f dist/index.js ] || [ ! -f dist/index.d.ts ]; then |
||||||
|
echo -e "❌ Failed: Missing 'dist' directory or required commonjs files in package $dir_name.\n" |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
|
||||||
|
if [ "$(jq -r '.main' package.json)" != "dist/index.js" ] || \ |
||||||
|
[ "$(jq -r '.types' package.json)" != "dist/index.d.ts" ]; then |
||||||
|
echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n" |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
|
||||||
|
# Assert esm builds |
||||||
|
esm_packages=("grafana-data" "grafana-ui" "grafana-runtime" "grafana-e2e-selectors" "grafana-schema") |
||||||
|
for esm_package in "${esm_packages[@]}"; do |
||||||
|
if [[ "$dir_name" == "$esm_package" ]]; then |
||||||
|
if [ ! -d dist/esm ] || [ ! -f dist/esm/index.js ]; then |
||||||
|
echo -e "❌ Failed: Missing 'dist/esm' directory or required esm files in package $dir_name.\n" |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
|
||||||
|
if [ "$(jq -r '.module' package.json)" != "dist/esm/index.js" ]; then |
||||||
|
echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n" |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
fi |
||||||
|
done |
||||||
|
|
||||||
|
echo -e "✅ Passed: package checks for $file.\n" |
||||||
|
popd || exit |
||||||
|
|
||||||
|
done |
||||||
|
|
||||||
|
echo "🚀 All NPM package checks passed! 🚀" |
||||||
|
rm -rf "${ARTIFACTS_DIR:?}/"*/ |
||||||
|
exit 0 |
||||||
Loading…
Reference in new issue