mirror of https://github.com/grafana/grafana
CI: Lint starlark files with `buildifier` (#59157)
* Add verify-starlark build action that returns an error for starlark files with lint Relies on `buildifier` tool. Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add verify_starlark_step to PR pipeline Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Manually fetch buildifier in curl_image until a new build_image is created Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Format with buildifier Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Remove all unused variables retaining one unused function Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use snake_case for variable Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Replace deprecated dictionary concatenation with .update() method Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Start adding docstrings for all modules and functions Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Prefer os.WriteFile as ioutil.WriteFile has been deprecated since go 1.16 Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Attempt to document the behavior of the init_enterprise_step Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document test_backend pipeline Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document enterprise_downstream_step Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document the pipeline utility function Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document publish_images_step Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document publish_images_steps Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document enterprise2_pipelines function Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add tags table for Starlark files. Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document test_frontend Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document windows function Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add docstrings to verifystarlark functions Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Refactor error handling to be more clear and document complex behavior Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Split errors into execution errors and verification errors Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document all other library functions Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add local variables to TAGS Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add blank line between all Args and Returns sections Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Fix new linting errors Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Lint new Starlark files Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Correct buildifier binary mv Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document the need to set nofile ulimit to at least 2048 Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Update build-container to include buildifier Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Ensure buildifier binary is executable Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Fix valid content test Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Simply return execution error Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Only check files rather than fixing them Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use updated build-container with executable buildifier Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Test that context cancellation stops execution Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Simplify error handling Return execution errors that short circuit WalkDir rather than separately tracking that error. Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Remove fetching of buildifier binary now that it is in the build-container Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use build image in verify-starlark step Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use semver tag The image is the same but uses a semver tag to make it clearer that this is a forward upgrade from the old version. Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use node 18 image with buildifier Signed-off-by: Jack Baldry <jack.baldry@grafana.com> --------- Signed-off-by: Jack Baldry <jack.baldry@grafana.com>pull/62446/head
Jack Baldry
2 years ago
committed by
GitHub
parent
907e2a840e
commit
8379a5338c
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,142 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"fmt" |
||||
"io/fs" |
||||
"os/exec" |
||||
"path/filepath" |
||||
"strings" |
||||
|
||||
"github.com/urfave/cli/v2" |
||||
) |
||||
|
||||
func mapSlice[I any, O any](a []I, f func(I) O) []O { |
||||
o := make([]O, len(a)) |
||||
for i, e := range a { |
||||
o[i] = f(e) |
||||
} |
||||
return o |
||||
} |
||||
|
||||
// VerifyStarlark is the CLI Action for verifying Starlark files in a workspace.
|
||||
// It expects a single context argument which is the path to the workspace.
|
||||
// The actual verification procedure can return multiple errors which are
|
||||
// joined together to be one holistic error for the action.
|
||||
func VerifyStarlark(c *cli.Context) error { |
||||
if c.NArg() != 1 { |
||||
var message string |
||||
if c.NArg() == 0 { |
||||
message = "ERROR: missing required argument <workspace path>" |
||||
} |
||||
if c.NArg() > 1 { |
||||
message = "ERROR: too many arguments" |
||||
} |
||||
|
||||
if err := cli.ShowSubcommandHelp(c); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return cli.Exit(message, 1) |
||||
} |
||||
|
||||
workspace := c.Args().Get(0) |
||||
verificationErrs, executionErr := verifyStarlark(c.Context, workspace, buildifierLintCommand) |
||||
if executionErr != nil { |
||||
return executionErr |
||||
} |
||||
|
||||
if len(verificationErrs) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
noun := "file" |
||||
if len(verificationErrs) > 1 { |
||||
noun += "s" |
||||
} |
||||
|
||||
return fmt.Errorf("verification failed for %d %s:\n%s", |
||||
len(verificationErrs), |
||||
noun, |
||||
strings.Join( |
||||
mapSlice(verificationErrs, func(e error) string { return e.Error() }), |
||||
"\n", |
||||
)) |
||||
} |
||||
|
||||
type commandFunc = func(path string) (command string, args []string) |
||||
|
||||
func buildifierLintCommand(path string) (string, []string) { |
||||
return "buildifier", []string{"-lint", "warn", "-mode", "check", path} |
||||
} |
||||
|
||||
// verifyStarlark walks all directories starting at provided workspace path and
|
||||
// verifies any Starlark files it finds.
|
||||
// Starlark files are assumed to end with the .star extension.
|
||||
// The verification relies on linting frovided by the 'buildifier' binary which
|
||||
// must be in the PATH.
|
||||
// A slice of verification errors are returned, one for each file that failed verification.
|
||||
// If any execution of the `buildifier` command fails, this is returned separately.
|
||||
// commandFn is executed on every Starlark file to determine the command and arguments to be executed.
|
||||
// The caller is trusted and it is the callers responsibility to ensure that the resulting command is safe to execute.
|
||||
func verifyStarlark(ctx context.Context, workspace string, commandFn commandFunc) ([]error, error) { |
||||
var verificationErrs []error |
||||
|
||||
// All errors from filepath.WalkDir are filtered by the fs.WalkDirFunc.
|
||||
// Lstat or ReadDir errors are reported as verificationErrors.
|
||||
// If any execution of the `buildifier` command fails or if the context is cancelled,
|
||||
// it is reported as an error and any verification of subsequent files is skipped.
|
||||
err := filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { |
||||
// Skip verification of the file or files within the directory if there is an error
|
||||
// returned by Lstat or ReadDir.
|
||||
if err != nil { |
||||
verificationErrs = append(verificationErrs, err) |
||||
return nil |
||||
} |
||||
|
||||
if d.IsDir() { |
||||
return nil |
||||
} |
||||
|
||||
if filepath.Ext(path) == ".star" { |
||||
command, args := commandFn(path) |
||||
// The caller is trusted.
|
||||
//nolint:gosec
|
||||
cmd := exec.CommandContext(ctx, command, args...) |
||||
cmd.Dir = workspace |
||||
|
||||
_, err = cmd.Output() |
||||
if err == nil { // No error, early return.
|
||||
return nil |
||||
} |
||||
|
||||
// The error returned from cmd.Output() is never wrapped.
|
||||
//nolint:errorlint
|
||||
if err, ok := err.(*exec.ExitError); ok { |
||||
switch err.ExitCode() { |
||||
// Case comments are informed by the output of `buildifier --help`
|
||||
case 1: // syntax errors in input
|
||||
verificationErrs = append(verificationErrs, errors.New(string(err.Stderr))) |
||||
return nil |
||||
case 2: // usage errors: invoked incorrectly
|
||||
return fmt.Errorf("command %q: %s", cmd, err.Stderr) |
||||
case 3: // unexpected runtime errors: file I/O problems or internal bugs
|
||||
return fmt.Errorf("command %q: %s", cmd, err.Stderr) |
||||
case 4: // check mode failed (reformat is needed)
|
||||
verificationErrs = append(verificationErrs, errors.New(string(err.Stderr))) |
||||
return nil |
||||
default: |
||||
return fmt.Errorf("command %q: %s", cmd, err.Stderr) |
||||
} |
||||
} |
||||
|
||||
// Error was not an exit error from the command.
|
||||
return fmt.Errorf("command %q: %v", cmd, err) |
||||
} |
||||
|
||||
return nil |
||||
}) |
||||
|
||||
return verificationErrs, err |
||||
} |
||||
@ -0,0 +1,135 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"os" |
||||
"path/filepath" |
||||
"strings" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestVerifyStarlark(t *testing.T) { |
||||
t.Run("execution errors", func(t *testing.T) { |
||||
t.Run("invalid usage", func(t *testing.T) { |
||||
ctx := context.Background() |
||||
workspace := t.TempDir() |
||||
err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm) |
||||
if err != nil { |
||||
t.Fatalf(err.Error()) |
||||
} |
||||
|
||||
_, executionErr := verifyStarlark(ctx, workspace, func(string) (string, []string) { return "buildifier", []string{"--invalid"} }) |
||||
if executionErr == nil { |
||||
t.Fatalf("Expected execution error but got none") |
||||
} |
||||
}) |
||||
|
||||
t.Run("context cancellation", func(t *testing.T) { |
||||
ctx, cancel := context.WithCancel(context.Background()) |
||||
workspace := t.TempDir() |
||||
err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm) |
||||
if err != nil { |
||||
t.Fatalf(err.Error()) |
||||
} |
||||
err = os.WriteFile(filepath.Join(workspace, "other-ignored.star"), []byte{}, os.ModePerm) |
||||
if err != nil { |
||||
t.Fatalf(err.Error()) |
||||
} |
||||
cancel() |
||||
|
||||
_, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) |
||||
if executionErr == nil { |
||||
t.Fatalf("Expected execution error but got none") |
||||
} |
||||
}) |
||||
}) |
||||
|
||||
t.Run("verification errors", func(t *testing.T) { |
||||
t.Run("a single file with lint", func(t *testing.T) { |
||||
ctx := context.Background() |
||||
workspace := t.TempDir() |
||||
|
||||
invalidContent := []byte(`load("scripts/drone/other.star", "function") |
||||
|
||||
function()`) |
||||
err := os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm) |
||||
if err != nil { |
||||
t.Fatalf(err.Error()) |
||||
} |
||||
|
||||
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) |
||||
if executionErr != nil { |
||||
t.Fatalf("Unexpected execution error: %v", executionErr) |
||||
} |
||||
if len(verificationErrs) == 0 { |
||||
t.Fatalf(`"has-lint.star" requires linting but the verifyStarlark function provided no linting error`) |
||||
} |
||||
if len(verificationErrs) > 1 { |
||||
t.Fatalf(`verifyStarlark returned multiple errors for the "has-lint.star" file but only one was expected: %v`, verificationErrs) |
||||
} |
||||
if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") { |
||||
t.Fatalf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) |
||||
} |
||||
}) |
||||
|
||||
t.Run("no files with lint", func(t *testing.T) { |
||||
ctx := context.Background() |
||||
workspace := t.TempDir() |
||||
|
||||
content := []byte(`""" |
||||
This module does nothing. |
||||
""" |
||||
|
||||
load("scripts/drone/other.star", "function") |
||||
|
||||
function() |
||||
`) |
||||
require.NoError(t, os.WriteFile(filepath.Join(workspace, "no-lint.star"), content, os.ModePerm)) |
||||
|
||||
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) |
||||
if executionErr != nil { |
||||
t.Fatalf("Unexpected execution error: %v", executionErr) |
||||
} |
||||
if len(verificationErrs) != 0 { |
||||
t.Log(`"no-lint.star" has no lint but the verifyStarlark function provided at least one error`) |
||||
for _, err := range verificationErrs { |
||||
t.Log(err) |
||||
} |
||||
t.FailNow() |
||||
} |
||||
}) |
||||
|
||||
t.Run("multiple files with lint", func(t *testing.T) { |
||||
ctx := context.Background() |
||||
workspace := t.TempDir() |
||||
|
||||
invalidContent := []byte(`load("scripts/drone/other.star", "function") |
||||
|
||||
function()`) |
||||
require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm)) |
||||
require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint2.star"), invalidContent, os.ModePerm)) |
||||
|
||||
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) |
||||
if executionErr != nil { |
||||
t.Fatalf("Unexpected execution error: %v", executionErr) |
||||
} |
||||
if len(verificationErrs) == 0 { |
||||
t.Fatalf(`Two files require linting but the verifyStarlark function provided no linting error`) |
||||
} |
||||
if len(verificationErrs) == 1 { |
||||
t.Fatalf(`Two files require linting but the verifyStarlark function provided only one linting error: %v`, verificationErrs[0]) |
||||
} |
||||
if len(verificationErrs) > 2 { |
||||
t.Fatalf(`verifyStarlark returned more errors than expected: %v`, verificationErrs) |
||||
} |
||||
if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") { |
||||
t.Errorf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) |
||||
} |
||||
if !strings.Contains(verificationErrs[1].Error(), "has-lint2.star:1: module-docstring: The file has no module docstring.") { |
||||
t.Fatalf(`"has-lint2.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) |
||||
} |
||||
}) |
||||
}) |
||||
} |
||||
@ -1,38 +1,43 @@ |
||||
""" |
||||
This module contains steps and pipelines publishing to AWS Marketplace. |
||||
""" |
||||
|
||||
load( |
||||
'scripts/drone/steps/lib.star', |
||||
'download_grabpl_step', |
||||
'publish_images_step', |
||||
'compile_build_cmd', |
||||
'fetch_images_step', |
||||
'publish_image', |
||||
"scripts/drone/steps/lib.star", |
||||
"compile_build_cmd", |
||||
"fetch_images_step", |
||||
"publish_image", |
||||
) |
||||
|
||||
load('scripts/drone/vault.star', 'from_secret') |
||||
|
||||
load("scripts/drone/vault.star", "from_secret") |
||||
load( |
||||
'scripts/drone/utils/utils.star', |
||||
'pipeline', |
||||
"scripts/drone/utils/utils.star", |
||||
"pipeline", |
||||
) |
||||
|
||||
def publish_aws_marketplace_step(): |
||||
return { |
||||
'name': 'publish-aws-marketplace', |
||||
'image': publish_image, |
||||
'commands': ['./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e'], |
||||
'depends_on': ['fetch-images-enterprise'], |
||||
'environment': { |
||||
'AWS_REGION': from_secret('aws_region'), |
||||
'AWS_ACCESS_KEY_ID': from_secret('aws_access_key_id'), |
||||
'AWS_SECRET_ACCESS_KEY': from_secret('aws_secret_access_key'), |
||||
"name": "publish-aws-marketplace", |
||||
"image": publish_image, |
||||
"commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"], |
||||
"depends_on": ["fetch-images-enterprise"], |
||||
"environment": { |
||||
"AWS_REGION": from_secret("aws_region"), |
||||
"AWS_ACCESS_KEY_ID": from_secret("aws_access_key_id"), |
||||
"AWS_SECRET_ACCESS_KEY": from_secret("aws_secret_access_key"), |
||||
}, |
||||
'volumes': [{'name': 'docker', 'path': '/var/run/docker.sock'}], |
||||
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], |
||||
} |
||||
|
||||
def publish_aws_marketplace_pipeline(mode): |
||||
trigger = { |
||||
'event': ['promote'], |
||||
'target': [mode], |
||||
"event": ["promote"], |
||||
"target": [mode], |
||||
} |
||||
return [pipeline( |
||||
name='publish-aws-marketplace-{}'.format(mode), trigger=trigger, steps=[compile_build_cmd(), fetch_images_step('enterprise'), publish_aws_marketplace_step()], edition="", depends_on = ['publish-docker-enterprise-public'], environment = {'EDITION': 'enterprise2'} |
||||
),] |
||||
name = "publish-aws-marketplace-{}".format(mode), |
||||
trigger = trigger, |
||||
steps = [compile_build_cmd(), fetch_images_step("enterprise"), publish_aws_marketplace_step()], |
||||
edition = "", |
||||
depends_on = ["publish-docker-enterprise-public"], |
||||
environment = {"EDITION": "enterprise2"}, |
||||
)] |
||||
|
||||
@ -1,36 +1,40 @@ |
||||
""" |
||||
This module contains steps and pipelines relating to GitHub. |
||||
""" |
||||
|
||||
load( |
||||
'scripts/drone/steps/lib.star', |
||||
'download_grabpl_step', |
||||
'publish_images_step', |
||||
'compile_build_cmd', |
||||
'fetch_images_step', |
||||
'publish_image', |
||||
"scripts/drone/steps/lib.star", |
||||
"compile_build_cmd", |
||||
"fetch_images_step", |
||||
"publish_image", |
||||
) |
||||
|
||||
load('scripts/drone/vault.star', 'from_secret') |
||||
|
||||
load("scripts/drone/vault.star", "from_secret") |
||||
load( |
||||
'scripts/drone/utils/utils.star', |
||||
'pipeline', |
||||
"scripts/drone/utils/utils.star", |
||||
"pipeline", |
||||
) |
||||
|
||||
def publish_github_step(): |
||||
return { |
||||
'name': 'publish-github', |
||||
'image': publish_image, |
||||
'commands': ['./bin/build publish github --repo $${GH_REGISTRY} --create'], |
||||
'depends_on': ['fetch-images-enterprise2'], |
||||
'environment': { |
||||
'GH_TOKEN': from_secret('github_token'), |
||||
'GH_REGISTRY': from_secret('gh_registry'), |
||||
"name": "publish-github", |
||||
"image": publish_image, |
||||
"commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"], |
||||
"depends_on": ["fetch-images-enterprise2"], |
||||
"environment": { |
||||
"GH_TOKEN": from_secret("github_token"), |
||||
"GH_REGISTRY": from_secret("gh_registry"), |
||||
}, |
||||
} |
||||
|
||||
def publish_github_pipeline(mode): |
||||
trigger = { |
||||
'event': ['promote'], |
||||
'target': [mode], |
||||
"event": ["promote"], |
||||
"target": [mode], |
||||
} |
||||
return [pipeline( |
||||
name='publish-github-{}'.format(mode), trigger=trigger, steps=[compile_build_cmd(), fetch_images_step('enterprise2'), publish_github_step()], edition="", environment = {'EDITION': 'enterprise2'} |
||||
),] |
||||
name = "publish-github-{}".format(mode), |
||||
trigger = trigger, |
||||
steps = [compile_build_cmd(), fetch_images_step("enterprise2"), publish_github_step()], |
||||
edition = "", |
||||
environment = {"EDITION": "enterprise2"}, |
||||
)] |
||||
|
||||
@ -1,75 +1,97 @@ |
||||
""" |
||||
This module returns the pipeline used for publishing Docker images and its steps. |
||||
""" |
||||
|
||||
load( |
||||
'scripts/drone/steps/lib.star', |
||||
'identify_runner_step', |
||||
'download_grabpl_step', |
||||
'publish_images_step', |
||||
'compile_build_cmd', |
||||
'fetch_images_step', |
||||
"scripts/drone/steps/lib.star", |
||||
"compile_build_cmd", |
||||
"download_grabpl_step", |
||||
"fetch_images_step", |
||||
"identify_runner_step", |
||||
"publish_images_step", |
||||
) |
||||
|
||||
load( |
||||
'scripts/drone/utils/utils.star', |
||||
'pipeline', |
||||
"scripts/drone/utils/utils.star", |
||||
"pipeline", |
||||
) |
||||
|
||||
|
||||
def publish_image_steps(edition, mode, docker_repo): |
||||
"""Generates the steps used for publising Docker images using grabpl. |
||||
|
||||
Args: |
||||
edition: controls which version of an image is fetched in the case of a release. |
||||
It also controls which publishing implementation is used. |
||||
If edition == 'oss', it additionally publishes the grafana/grafana-oss repository. |
||||
mode: uses to control the publishing of security images when mode == 'security'. |
||||
docker_repo: the Docker image name. |
||||
It is combined with the 'grafana/' library prefix. |
||||
|
||||
Returns: |
||||
List of Drone steps. |
||||
""" |
||||
steps = [ |
||||
identify_runner_step(), |
||||
download_grabpl_step(), |
||||
compile_build_cmd(), |
||||
fetch_images_step(edition), |
||||
publish_images_step(edition, 'release', mode, docker_repo), |
||||
publish_images_step(edition, "release", mode, docker_repo), |
||||
] |
||||
|
||||
if edition == 'oss': |
||||
if edition == "oss": |
||||
steps.append( |
||||
publish_images_step(edition, 'release', mode, 'grafana/grafana-oss') |
||||
publish_images_step(edition, "release", mode, "grafana/grafana-oss"), |
||||
) |
||||
|
||||
return steps |
||||
|
||||
|
||||
def publish_image_pipelines_public(): |
||||
mode = 'public' |
||||
"""Generates the pipeline used for publising public Docker images. |
||||
|
||||
Returns: |
||||
Drone pipeline |
||||
""" |
||||
mode = "public" |
||||
trigger = { |
||||
'event': ['promote'], |
||||
'target': [mode], |
||||
"event": ["promote"], |
||||
"target": [mode], |
||||
} |
||||
return [ |
||||
pipeline( |
||||
name='publish-docker-oss-{}'.format(mode), |
||||
name = "publish-docker-oss-{}".format(mode), |
||||
trigger = trigger, |
||||
steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana'), |
||||
steps = publish_image_steps(edition = "oss", mode = mode, docker_repo = "grafana"), |
||||
edition = "", |
||||
environment={'EDITION': 'oss'}, |
||||
environment = {"EDITION": "oss"}, |
||||
), |
||||
pipeline( |
||||
name='publish-docker-enterprise-{}'.format(mode), |
||||
name = "publish-docker-enterprise-{}".format(mode), |
||||
trigger = trigger, |
||||
steps = publish_image_steps( |
||||
edition='enterprise', mode=mode, docker_repo='grafana-enterprise' |
||||
edition = "enterprise", |
||||
mode = mode, |
||||
docker_repo = "grafana-enterprise", |
||||
), |
||||
edition = "", |
||||
environment={'EDITION': 'enterprise'}, |
||||
environment = {"EDITION": "enterprise"}, |
||||
), |
||||
] |
||||
|
||||
|
||||
def publish_image_pipelines_security(): |
||||
mode = 'security' |
||||
mode = "security" |
||||
trigger = { |
||||
'event': ['promote'], |
||||
'target': [mode], |
||||
"event": ["promote"], |
||||
"target": [mode], |
||||
} |
||||
return [ |
||||
pipeline( |
||||
name='publish-docker-enterprise-{}'.format(mode), |
||||
name = "publish-docker-enterprise-{}".format(mode), |
||||
trigger = trigger, |
||||
steps = publish_image_steps( |
||||
edition='enterprise', mode=mode, docker_repo='grafana-enterprise' |
||||
edition = "enterprise", |
||||
mode = mode, |
||||
docker_repo = "grafana-enterprise", |
||||
), |
||||
edition = "", |
||||
environment={'EDITION': 'enterprise'}, |
||||
environment = {"EDITION": "enterprise"}, |
||||
), |
||||
] |
||||
|
||||
@ -0,0 +1,32 @@ |
||||
""" |
||||
This module returns a Drone pipeline that verifies all Starlark files are linted. |
||||
""" |
||||
|
||||
load( |
||||
"scripts/drone/steps/lib.star", |
||||
"compile_build_cmd", |
||||
"download_grabpl_step", |
||||
"identify_runner_step", |
||||
"lint_starlark_step", |
||||
) |
||||
load( |
||||
"scripts/drone/utils/utils.star", |
||||
"pipeline", |
||||
) |
||||
|
||||
def verify_starlark(trigger, ver_mode): |
||||
environment = {"EDITION": "oss"} |
||||
steps = [ |
||||
identify_runner_step(), |
||||
download_grabpl_step(), |
||||
compile_build_cmd(), |
||||
lint_starlark_step(), |
||||
] |
||||
return pipeline( |
||||
name = "{}-verify-starlark".format(ver_mode), |
||||
edition = "oss", |
||||
trigger = trigger, |
||||
services = [], |
||||
steps = steps, |
||||
environment = environment, |
||||
) |
||||
@ -1,28 +1,41 @@ |
||||
""" |
||||
This module returns the pipeline used for building Grafana on Windows. |
||||
""" |
||||
|
||||
load( |
||||
'scripts/drone/steps/lib.star', |
||||
'get_windows_steps', |
||||
"scripts/drone/utils/utils.star", |
||||
"pipeline", |
||||
) |
||||
|
||||
load( |
||||
'scripts/drone/utils/utils.star', |
||||
'pipeline', |
||||
"scripts/drone/steps/lib.star", |
||||
"get_windows_steps", |
||||
) |
||||
|
||||
|
||||
def windows(trigger, edition, ver_mode): |
||||
environment = {'EDITION': edition} |
||||
"""Generates the pipeline used for building Grafana on Windows. |
||||
|
||||
Args: |
||||
trigger: a Drone trigger for the pipeline. |
||||
edition: controls whether enterprise code is included in the pipeline steps. |
||||
ver_mode: controls whether a pre-release or actual release pipeline is generated. |
||||
Also indirectly controls which version of enterprise code is used. |
||||
|
||||
Returns: |
||||
Drone pipeline. |
||||
""" |
||||
environment = {"EDITION": edition} |
||||
|
||||
return pipeline( |
||||
name='main-windows', |
||||
name = "main-windows", |
||||
edition = edition, |
||||
trigger=dict(trigger, repo=['grafana/grafana']), |
||||
trigger = dict(trigger, repo = ["grafana/grafana"]), |
||||
steps = get_windows_steps(edition, ver_mode), |
||||
depends_on = [ |
||||
'main-test-frontend', |
||||
'main-test-backend', |
||||
'main-build-e2e-publish', |
||||
'main-integration-tests', |
||||
"main-test-frontend", |
||||
"main-test-backend", |
||||
"main-build-e2e-publish", |
||||
"main-integration-tests", |
||||
], |
||||
platform='windows', |
||||
platform = "windows", |
||||
environment = environment, |
||||
) |
||||
|
||||
@ -1,64 +1,66 @@ |
||||
""" |
||||
This module has functions for Drone services to be used in pipelines. |
||||
""" |
||||
|
||||
def integration_test_services_volumes(): |
||||
return [ |
||||
{'name': 'postgres', 'temp': {'medium': 'memory'}}, |
||||
{'name': 'mysql', 'temp': {'medium': 'memory'}}, |
||||
{"name": "postgres", "temp": {"medium": "memory"}}, |
||||
{"name": "mysql", "temp": {"medium": "memory"}}, |
||||
] |
||||
|
||||
|
||||
def integration_test_services(edition): |
||||
services = [ |
||||
{ |
||||
'name': 'postgres', |
||||
'image': 'postgres:12.3-alpine', |
||||
'environment': { |
||||
'POSTGRES_USER': 'grafanatest', |
||||
'POSTGRES_PASSWORD': 'grafanatest', |
||||
'POSTGRES_DB': 'grafanatest', |
||||
'PGDATA': '/var/lib/postgresql/data/pgdata', |
||||
"name": "postgres", |
||||
"image": "postgres:12.3-alpine", |
||||
"environment": { |
||||
"POSTGRES_USER": "grafanatest", |
||||
"POSTGRES_PASSWORD": "grafanatest", |
||||
"POSTGRES_DB": "grafanatest", |
||||
"PGDATA": "/var/lib/postgresql/data/pgdata", |
||||
}, |
||||
'volumes': [ |
||||
{'name': 'postgres', 'path': '/var/lib/postgresql/data/pgdata'} |
||||
"volumes": [ |
||||
{"name": "postgres", "path": "/var/lib/postgresql/data/pgdata"}, |
||||
], |
||||
}, |
||||
{ |
||||
'name': 'mysql', |
||||
'image': 'mysql:5.7.39', |
||||
'environment': { |
||||
'MYSQL_ROOT_PASSWORD': 'rootpass', |
||||
'MYSQL_DATABASE': 'grafana_tests', |
||||
'MYSQL_USER': 'grafana', |
||||
'MYSQL_PASSWORD': 'password', |
||||
"name": "mysql", |
||||
"image": "mysql:5.7.39", |
||||
"environment": { |
||||
"MYSQL_ROOT_PASSWORD": "rootpass", |
||||
"MYSQL_DATABASE": "grafana_tests", |
||||
"MYSQL_USER": "grafana", |
||||
"MYSQL_PASSWORD": "password", |
||||
}, |
||||
'volumes': [{'name': 'mysql', 'path': '/var/lib/mysql'}], |
||||
"volumes": [{"name": "mysql", "path": "/var/lib/mysql"}], |
||||
}, |
||||
] |
||||
|
||||
if edition in ('enterprise', 'enterprise2'): |
||||
if edition in ("enterprise", "enterprise2"): |
||||
services.extend( |
||||
[ |
||||
{ |
||||
'name': 'redis', |
||||
'image': 'redis:6.2.1-alpine', |
||||
'environment': {}, |
||||
"name": "redis", |
||||
"image": "redis:6.2.1-alpine", |
||||
"environment": {}, |
||||
}, |
||||
{ |
||||
'name': 'memcached', |
||||
'image': 'memcached:1.6.9-alpine', |
||||
'environment': {}, |
||||
"name": "memcached", |
||||
"image": "memcached:1.6.9-alpine", |
||||
"environment": {}, |
||||
}, |
||||
] |
||||
], |
||||
) |
||||
|
||||
return services |
||||
|
||||
|
||||
def ldap_service(): |
||||
return { |
||||
'name': 'ldap', |
||||
'image': 'osixia/openldap:1.4.0', |
||||
'environment': { |
||||
'LDAP_ADMIN_PASSWORD': 'grafana', |
||||
'LDAP_DOMAIN': 'grafana.org', |
||||
'SLAPD_ADDITIONAL_MODULES': 'memberof', |
||||
"name": "ldap", |
||||
"image": "osixia/openldap:1.4.0", |
||||
"environment": { |
||||
"LDAP_ADMIN_PASSWORD": "grafana", |
||||
"LDAP_DOMAIN": "grafana.org", |
||||
"SLAPD_ADDITIONAL_MODULES": "memberof", |
||||
}, |
||||
} |
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,97 +1,97 @@ |
||||
pull_secret = 'dockerconfigjson' |
||||
drone_token = 'drone_token' |
||||
prerelease_bucket = 'prerelease_bucket' |
||||
gcp_upload_artifacts_key = 'gcp_upload_artifacts_key' |
||||
azure_sp_app_id = 'azure_sp_app_id' |
||||
azure_sp_app_pw = 'azure_sp_app_pw' |
||||
azure_tenant = 'azure_tenant' |
||||
|
||||
""" |
||||
This module returns functions for generating Drone secrets fetched from Vault. |
||||
""" |
||||
pull_secret = "dockerconfigjson" |
||||
drone_token = "drone_token" |
||||
prerelease_bucket = "prerelease_bucket" |
||||
gcp_upload_artifacts_key = "gcp_upload_artifacts_key" |
||||
azure_sp_app_id = "azure_sp_app_id" |
||||
azure_sp_app_pw = "azure_sp_app_pw" |
||||
azure_tenant = "azure_tenant" |
||||
|
||||
def from_secret(secret): |
||||
return {'from_secret': secret} |
||||
|
||||
return {"from_secret": secret} |
||||
|
||||
def vault_secret(name, path, key): |
||||
return { |
||||
'kind': 'secret', |
||||
'name': name, |
||||
'get': { |
||||
'path': path, |
||||
'name': key, |
||||
"kind": "secret", |
||||
"name": name, |
||||
"get": { |
||||
"path": path, |
||||
"name": key, |
||||
}, |
||||
} |
||||
|
||||
|
||||
def secrets(): |
||||
return [ |
||||
vault_secret(pull_secret, 'secret/data/common/gcr', '.dockerconfigjson'), |
||||
vault_secret('github_token', 'infra/data/ci/github/grafanabot', 'pat'), |
||||
vault_secret(drone_token, 'infra/data/ci/drone', 'machine-user-token'), |
||||
vault_secret(prerelease_bucket, 'infra/data/ci/grafana/prerelease', 'bucket'), |
||||
vault_secret(pull_secret, "secret/data/common/gcr", ".dockerconfigjson"), |
||||
vault_secret("github_token", "infra/data/ci/github/grafanabot", "pat"), |
||||
vault_secret(drone_token, "infra/data/ci/drone", "machine-user-token"), |
||||
vault_secret(prerelease_bucket, "infra/data/ci/grafana/prerelease", "bucket"), |
||||
vault_secret( |
||||
gcp_upload_artifacts_key, |
||||
'infra/data/ci/grafana/releng/artifacts-uploader-service-account', |
||||
'credentials.json', |
||||
"infra/data/ci/grafana/releng/artifacts-uploader-service-account", |
||||
"credentials.json", |
||||
), |
||||
vault_secret( |
||||
azure_sp_app_id, |
||||
'infra/data/ci/datasources/cpp-azure-resourcemanager-credentials', |
||||
'application_id', |
||||
"infra/data/ci/datasources/cpp-azure-resourcemanager-credentials", |
||||
"application_id", |
||||
), |
||||
vault_secret( |
||||
azure_sp_app_pw, |
||||
'infra/data/ci/datasources/cpp-azure-resourcemanager-credentials', |
||||
'application_secret', |
||||
"infra/data/ci/datasources/cpp-azure-resourcemanager-credentials", |
||||
"application_secret", |
||||
), |
||||
vault_secret( |
||||
azure_tenant, |
||||
'infra/data/ci/datasources/cpp-azure-resourcemanager-credentials', |
||||
'tenant_id', |
||||
"infra/data/ci/datasources/cpp-azure-resourcemanager-credentials", |
||||
"tenant_id", |
||||
), |
||||
# Package publishing |
||||
vault_secret( |
||||
'packages_gpg_public_key', |
||||
'infra/data/ci/packages-publish/gpg', |
||||
'public-key-b64', |
||||
"packages_gpg_public_key", |
||||
"infra/data/ci/packages-publish/gpg", |
||||
"public-key-b64", |
||||
), |
||||
vault_secret( |
||||
'packages_gpg_private_key', |
||||
'infra/data/ci/packages-publish/gpg', |
||||
'private-key-b64', |
||||
"packages_gpg_private_key", |
||||
"infra/data/ci/packages-publish/gpg", |
||||
"private-key-b64", |
||||
), |
||||
vault_secret( |
||||
'packages_gpg_passphrase', |
||||
'infra/data/ci/packages-publish/gpg', |
||||
'passphrase', |
||||
"packages_gpg_passphrase", |
||||
"infra/data/ci/packages-publish/gpg", |
||||
"passphrase", |
||||
), |
||||
vault_secret( |
||||
'packages_service_account', |
||||
'infra/data/ci/packages-publish/service-account', |
||||
'credentials.json', |
||||
"packages_service_account", |
||||
"infra/data/ci/packages-publish/service-account", |
||||
"credentials.json", |
||||
), |
||||
vault_secret( |
||||
'packages_access_key_id', |
||||
'infra/data/ci/packages-publish/bucket-credentials', |
||||
'AccessID', |
||||
"packages_access_key_id", |
||||
"infra/data/ci/packages-publish/bucket-credentials", |
||||
"AccessID", |
||||
), |
||||
vault_secret( |
||||
'packages_secret_access_key', |
||||
'infra/data/ci/packages-publish/bucket-credentials', |
||||
'Secret', |
||||
"packages_secret_access_key", |
||||
"infra/data/ci/packages-publish/bucket-credentials", |
||||
"Secret", |
||||
), |
||||
vault_secret( |
||||
'aws_region', |
||||
'secret/data/common/aws-marketplace', |
||||
'aws_region', |
||||
"aws_region", |
||||
"secret/data/common/aws-marketplace", |
||||
"aws_region", |
||||
), |
||||
vault_secret( |
||||
'aws_access_key_id', |
||||
'secret/data/common/aws-marketplace', |
||||
'aws_access_key_id', |
||||
"aws_access_key_id", |
||||
"secret/data/common/aws-marketplace", |
||||
"aws_access_key_id", |
||||
), |
||||
vault_secret( |
||||
'aws_secret_access_key', |
||||
'secret/data/common/aws-marketplace', |
||||
'aws_secret_access_key', |
||||
"aws_secret_access_key", |
||||
"secret/data/common/aws-marketplace", |
||||
"aws_secret_access_key", |
||||
), |
||||
] |
||||
|
||||
@ -1,17 +1,20 @@ |
||||
""" |
||||
This module returns the pipeline used for version branches. |
||||
""" |
||||
|
||||
load( |
||||
'scripts/drone/events/release.star', |
||||
'oss_pipelines', |
||||
'enterprise_pipelines', |
||||
'enterprise2_pipelines', |
||||
"scripts/drone/events/release.star", |
||||
"enterprise2_pipelines", |
||||
"enterprise_pipelines", |
||||
"oss_pipelines", |
||||
) |
||||
|
||||
ver_mode = 'release-branch' |
||||
trigger = {'ref': ['refs/heads/v[0-9]*']} |
||||
|
||||
ver_mode = "release-branch" |
||||
trigger = {"ref": ["refs/heads/v[0-9]*"]} |
||||
|
||||
def version_branch_pipelines(): |
||||
return ( |
||||
oss_pipelines(ver_mode=ver_mode, trigger=trigger) |
||||
+ enterprise_pipelines(ver_mode=ver_mode, trigger=trigger) |
||||
+ enterprise2_pipelines(ver_mode=ver_mode, trigger=trigger) |
||||
oss_pipelines(ver_mode = ver_mode, trigger = trigger) + |
||||
enterprise_pipelines(ver_mode = ver_mode, trigger = trigger) + |
||||
enterprise2_pipelines(ver_mode = ver_mode, trigger = trigger) |
||||
) |
||||
|
||||
Loading…
Reference in new issue