From d784d370d34dfbbfca3c28d6f06228e57b036fc1 Mon Sep 17 00:00:00 2001 From: Kevin Minehart <5140827+kminehart@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:57:07 -0500 Subject: [PATCH] [v11.3.x] CI: Support more version formats in publishing (#94750) CI: Support more version formats in publishing (#94575) * cleanup dead code * add tests and rewrite publish grafanacom steps to reuse * add pkg/build tests; don't upload CDN assets on grafana releases (cherry picked from commit 7a2edd35d59e3aa3b72767d54c4559b48f6c244d) --- .drone.yml | 86 +- pkg/build/cmd.go | 8 + pkg/build/cmd/argcount_wrapper.go | 14 - pkg/build/cmd/buildbackend.go | 68 -- pkg/build/cmd/builddocker.go | 51 - pkg/build/cmd/buildfrontend.go | 39 - pkg/build/cmd/buildfrontendpackages.go | 38 - pkg/build/cmd/buildinternalplugins.go | 53 - pkg/build/cmd/enterprisecheck.go | 133 -- pkg/build/cmd/enterprisecheck_test.go | 69 -- pkg/build/cmd/exportversion.go | 32 - pkg/build/cmd/fetchimages.go | 114 +- pkg/build/cmd/fetchimages_test.go | 48 + pkg/build/cmd/flags.go | 22 - pkg/build/cmd/grafanacom.go | 115 +- pkg/build/cmd/grafanacom_test.go | 225 ++++ pkg/build/cmd/main.go | 144 +-- pkg/build/cmd/package.go | 80 -- pkg/build/cmd/publishimages_enterprise2.go | 101 -- pkg/build/cmd/shellcheck.go | 42 - pkg/build/cmd/uploadpackages.go | 24 +- pkg/build/cmd/verifystarlark.go | 142 --- pkg/build/cmd/verifystarlark_test.go | 137 --- pkg/build/compilers/install.go | 50 - pkg/build/config/revision.go | 69 -- pkg/build/cryptoutil/md5.go | 35 - pkg/build/docker/build.go | 182 --- pkg/build/docker/init.go | 34 - pkg/build/docker/push.go | 62 - pkg/build/env/lookup.go | 18 - pkg/build/env/lookup_test.go | 43 - pkg/build/errutil/group.go | 61 - pkg/build/executil/exec.go | 46 - pkg/build/frontend/build.go | 56 - pkg/build/frontend/config.go | 42 - pkg/build/frontend/config_test.go | 118 -- pkg/build/fsutil/copy_recursive.go | 65 - pkg/build/fsutil/createtemp.go | 43 - pkg/build/fsutil/createtemp_test.go | 48 - pkg/build/gcom/url.go | 72 ++ pkg/build/gcom/url_test.go | 367 ++++++ pkg/build/git/git.go | 90 -- pkg/build/git/git_checks_test.go | 56 - pkg/build/git/git_issues_test.go | 135 --- pkg/build/git/git_test.go | 57 - pkg/build/go.sum | 2 - pkg/build/golangutils/build.go | 124 -- pkg/build/golangutils/doc.go | 2 - pkg/build/gpg/gpg.go | 73 -- pkg/build/gpg/import.go | 73 -- pkg/build/grafana/build.go | 129 -- pkg/build/grafana/variant.go | 159 --- pkg/build/lerna/lerna.go | 50 - pkg/build/packaging/artifacts.go | 173 ++- pkg/build/packaging/docs.go | 2 - pkg/build/packaging/errors.go | 1 - pkg/build/packaging/grafana.go | 1133 ------------------ pkg/build/packaging/grafana_test.go | 23 - pkg/build/plugins/build.go | 66 - pkg/build/plugins/download.go | 118 -- pkg/build/plugins/manifest.go | 204 ---- pkg/build/plugins/zip.go | 64 - pkg/build/stringutil/contains.go | 10 - pkg/build/syncutil/pool.go | 43 - pkg/build/validation/validation.go | 14 - pkg/build/versions/parse.go | 31 + scripts/drone/events/release.star | 20 +- scripts/drone/pipelines/publish_images.star | 41 +- scripts/drone/pipelines/shellcheck.star | 2 - scripts/drone/pipelines/verify_starlark.star | 2 - scripts/drone/steps/lib.star | 4 +- 71 files changed, 1052 insertions(+), 5045 deletions(-) delete mode 100644 pkg/build/cmd/buildbackend.go delete mode 100644 pkg/build/cmd/builddocker.go delete mode 100644 pkg/build/cmd/buildfrontend.go delete mode 100644 pkg/build/cmd/buildfrontendpackages.go delete mode 100644 pkg/build/cmd/buildinternalplugins.go delete mode 100644 pkg/build/cmd/enterprisecheck.go delete mode 100644 pkg/build/cmd/enterprisecheck_test.go delete mode 100644 pkg/build/cmd/exportversion.go create mode 100644 pkg/build/cmd/fetchimages_test.go delete mode 100644 pkg/build/cmd/package.go delete mode 100644 pkg/build/cmd/publishimages_enterprise2.go delete mode 100644 pkg/build/cmd/shellcheck.go delete mode 100644 pkg/build/cmd/verifystarlark.go delete mode 100644 pkg/build/cmd/verifystarlark_test.go delete mode 100644 pkg/build/compilers/install.go delete mode 100644 pkg/build/config/revision.go delete mode 100644 pkg/build/cryptoutil/md5.go delete mode 100644 pkg/build/docker/build.go delete mode 100644 pkg/build/docker/init.go delete mode 100644 pkg/build/docker/push.go delete mode 100644 pkg/build/env/lookup.go delete mode 100644 pkg/build/env/lookup_test.go delete mode 100644 pkg/build/errutil/group.go delete mode 100644 pkg/build/executil/exec.go delete mode 100644 pkg/build/frontend/build.go delete mode 100644 pkg/build/frontend/config.go delete mode 100644 pkg/build/frontend/config_test.go delete mode 100644 pkg/build/fsutil/copy_recursive.go delete mode 100644 pkg/build/fsutil/createtemp.go delete mode 100644 pkg/build/fsutil/createtemp_test.go create mode 100644 pkg/build/gcom/url.go create mode 100644 pkg/build/gcom/url_test.go delete mode 100644 pkg/build/git/git_checks_test.go delete mode 100644 pkg/build/git/git_issues_test.go delete mode 100644 pkg/build/git/git_test.go delete mode 100644 pkg/build/golangutils/build.go delete mode 100644 pkg/build/golangutils/doc.go delete mode 100644 pkg/build/gpg/gpg.go delete mode 100644 pkg/build/gpg/import.go delete mode 100644 pkg/build/grafana/build.go delete mode 100644 pkg/build/packaging/docs.go delete mode 100644 pkg/build/packaging/errors.go delete mode 100644 pkg/build/packaging/grafana.go delete mode 100644 pkg/build/packaging/grafana_test.go delete mode 100644 pkg/build/plugins/build.go delete mode 100644 pkg/build/plugins/download.go delete mode 100644 pkg/build/plugins/manifest.go delete mode 100644 pkg/build/plugins/zip.go delete mode 100644 pkg/build/stringutil/contains.go delete mode 100644 pkg/build/syncutil/pool.go create mode 100644 pkg/build/versions/parse.go diff --git a/.drone.yml b/.drone.yml index 48d7046793b..4d7826a65a7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -71,18 +71,10 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.20.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.23.1-alpine - name: compile-build-cmd - commands: - go install github.com/bazelbuild/buildtools/buildifier@latest - buildifier --lint=warn -mode=check -r . - depends_on: - - compile-build-cmd + depends_on: [] image: golang:1.23.1-alpine name: lint-starlark trigger: @@ -1278,13 +1270,6 @@ platform: os: linux services: [] steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.23.1-alpine - name: compile-build-cmd - commands: - apt-get update -yq && apt-get install shellcheck - shellcheck -e SC1071 -e SC2162 scripts/**/*.sh @@ -3434,31 +3419,32 @@ steps: - |2- bash -c ' + IMAGE_TAG=$(echo "$${TAG}" | sed -e "s/+/-/g") debug= if [[ -n $${DRY_RUN} ]]; then debug=echo; fi docker login -u $${DOCKER_USER} -p $${DOCKER_PASSWORD} # Push the grafana-image-tags images - $$debug docker push grafana/grafana-image-tags:$${TAG}-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Create the grafana manifests - $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${TAG}-amd64 grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7 + $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 - $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Push the grafana manifests - $$debug docker manifest push grafana/grafana:$${TAG} - $$debug docker manifest push grafana/grafana:$${TAG}-ubuntu + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG} + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG}-ubuntu # if LATEST is set, then also create & push latest if [[ -n $${LATEST} ]]; then - $$debug docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${TAG}-amd64 grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 $$debug docker manifest push grafana/grafana:latest $$debug docker manifest push grafana/grafana:latest-ubuntu @@ -3565,31 +3551,32 @@ steps: - |2- bash -c ' + IMAGE_TAG=$(echo "$${TAG}" | sed -e "s/+/-/g") debug= if [[ -n $${DRY_RUN} ]]; then debug=echo; fi docker login -u $${DOCKER_USER} -p $${DOCKER_PASSWORD} # Push the grafana-image-tags images - $$debug docker push grafana/grafana-image-tags:$${TAG}-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Create the grafana manifests - $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${TAG}-amd64 grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7 + $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 - $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Push the grafana manifests - $$debug docker manifest push grafana/grafana:$${TAG} - $$debug docker manifest push grafana/grafana:$${TAG}-ubuntu + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG} + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG}-ubuntu # if LATEST is set, then also create & push latest if [[ -n $${LATEST} ]]; then - $$debug docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${TAG}-amd64 grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 $$debug docker manifest push grafana/grafana:latest $$debug docker manifest push grafana/grafana:latest-ubuntu @@ -3681,7 +3668,8 @@ steps: image: golang:1.23.1-alpine name: compile-build-cmd - commands: - - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} + - ./bin/build artifacts packages --artifacts-editions=oss --tag $${DRONE_TAG} --src-bucket + $${PRERELEASE_BUCKET} depends_on: - compile-build-cmd environment: @@ -3691,19 +3679,6 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts -- commands: - - ./bin/build artifacts static-assets --tag ${DRONE_TAG} --static-asset-editions=grafana-oss - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - STATIC_ASSET_EDITIONS: - from_secret: static_asset_editions - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-static-assets - commands: - ./bin/build artifacts storybook --tag ${DRONE_TAG} depends_on: @@ -3723,7 +3698,6 @@ steps: -f latest=$${LATEST} --repo=grafana/grafana release-pr.yml depends_on: - publish-artifacts - - publish-static-assets environment: GH_CLI_URL: https://github.com/cli/cli/releases/download/v2.50.0/gh_2.50.0_linux_amd64.tar.gz GITHUB_TOKEN: @@ -6013,6 +5987,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: e618274ea7a8bfbf3d5e151d459348aa9382fe63fe7fef76c997db3cba74779f +hmac: dc30a3a00ee542fb289da36ef6db4274684db4533c472f7f903468919d1046ac ... diff --git a/pkg/build/cmd.go b/pkg/build/cmd.go index fa98316e9c2..7b775ad0f44 100644 --- a/pkg/build/cmd.go +++ b/pkg/build/cmd.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" "time" + + "github.com/urfave/cli/v2" ) const ( @@ -30,6 +32,12 @@ func logError(message string, err error) int { return 1 } +func RunCmdCLI(c *cli.Context) error { + os.Exit(RunCmd()) + + return nil +} + // RunCmd runs the build command and returns the exit code func RunCmd() int { opts := BuildOptsFromFlags() diff --git a/pkg/build/cmd/argcount_wrapper.go b/pkg/build/cmd/argcount_wrapper.go index 690695cd350..394283e163d 100644 --- a/pkg/build/cmd/argcount_wrapper.go +++ b/pkg/build/cmd/argcount_wrapper.go @@ -2,20 +2,6 @@ package main import "github.com/urfave/cli/v2" -// ArgCountWrapper will cause the action to fail if there were not exactly `num` args provided. -func ArgCountWrapper(num int, action cli.ActionFunc) cli.ActionFunc { - return func(ctx *cli.Context) error { - if ctx.NArg() != num { - if err := cli.ShowSubcommandHelp(ctx); err != nil { - return cli.Exit(err.Error(), 1) - } - return cli.Exit("", 1) - } - - return action(ctx) - } -} - // ArgCountWrapper will cause the action to fail if there were more than `num` args provided. func MaxArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc { return func(ctx *cli.Context) error { diff --git a/pkg/build/cmd/buildbackend.go b/pkg/build/cmd/buildbackend.go deleted file mode 100644 index 40c4f90388a..00000000000 --- a/pkg/build/cmd/buildbackend.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "fmt" - "log" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/compilers" - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/grafana" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func BuildBackend(ctx *cli.Context) error { - metadata, err := config.GenerateMetadata(ctx) - if err != nil { - return err - } - version := metadata.GrafanaVersion - - var ( - edition = config.Edition(ctx.String("edition")) - cfg = config.Config{ - NumWorkers: ctx.Int("jobs"), - } - ) - - buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return fmt.Errorf("could not get version / package info for mode '%s': %w", metadata.ReleaseMode.Mode, err) - } - - const grafanaDir = "." - - log.Printf("Building Grafana back-end, version %q, %s edition, variants [%v]", - version, edition, buildConfig.Variants) - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - if err := compilers.Install(); err != nil { - return cli.Exit(err.Error(), 1) - } - - g, _ := errutil.GroupWithContext(ctx.Context) - for _, variant := range buildConfig.Variants { - variant := variant - - opts := grafana.BuildVariantOpts{ - Variant: variant, - Edition: edition, - Version: version, - GrafanaDir: grafanaDir, - } - - p.Schedule(g.Wrap(func() error { - return grafana.BuildVariant(ctx.Context, opts) - })) - } - if err := g.Wait(); err != nil { - return cli.Exit(err.Error(), 1) - } - - log.Println("Successfully built back-end binaries!") - return nil -} diff --git a/pkg/build/cmd/builddocker.go b/pkg/build/cmd/builddocker.go deleted file mode 100644 index f623057ee60..00000000000 --- a/pkg/build/cmd/builddocker.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "log" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/docker" - "github.com/grafana/grafana/pkg/build/gcloud" -) - -func BuildDocker(c *cli.Context) error { - if err := docker.Init(); err != nil { - return err - } - - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - useUbuntu := c.Bool("ubuntu") - buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return err - } - - shouldSave := buildConfig.Docker.ShouldSave - if shouldSave { - if err := gcloud.ActivateServiceAccount(); err != nil { - return err - } - } - - edition := config.Edition(c.String("edition")) - - version := metadata.GrafanaVersion - - log.Printf("Building Docker images, version %s, %s edition, Ubuntu based: %v...", version, edition, - useUbuntu) - - for _, arch := range buildConfig.Docker.Architectures { - if _, err := docker.BuildImage(version, arch, ".", useUbuntu, shouldSave, edition, metadata.ReleaseMode.Mode); err != nil { - return cli.Exit(err.Error(), 1) - } - } - - log.Println("Successfully built Docker images!") - return nil -} diff --git a/pkg/build/cmd/buildfrontend.go b/pkg/build/cmd/buildfrontend.go deleted file mode 100644 index ef0fbc9a5fd..00000000000 --- a/pkg/build/cmd/buildfrontend.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "log" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/frontend" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func BuildFrontend(c *cli.Context) error { - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - cfg, mode, err := frontend.GetConfig(c, metadata) - if err != nil { - return err - } - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - g, _ := errutil.GroupWithContext(c.Context) - if err := frontend.Build(mode, frontend.GrafanaDir, p, g); err != nil { - return err - } - if err := g.Wait(); err != nil { - return err - } - - log.Println("Successfully built Grafana front-end!") - - return nil -} diff --git a/pkg/build/cmd/buildfrontendpackages.go b/pkg/build/cmd/buildfrontendpackages.go deleted file mode 100644 index b1fef3dfcee..00000000000 --- a/pkg/build/cmd/buildfrontendpackages.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -import ( - "log" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/frontend" - "github.com/grafana/grafana/pkg/build/syncutil" - "github.com/urfave/cli/v2" -) - -func BuildFrontendPackages(c *cli.Context) error { - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - cfg, mode, err := frontend.GetConfig(c, metadata) - if err != nil { - return err - } - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - g, _ := errutil.GroupWithContext(c.Context) - if err := frontend.BuildFrontendPackages(cfg.PackageVersion, mode, frontend.GrafanaDir, p, g); err != nil { - return cli.Exit(err.Error(), 1) - } - if err := g.Wait(); err != nil { - return cli.Exit(err.Error(), 1) - } - - log.Println("Successfully built Grafana front-end packages!") - - return nil -} diff --git a/pkg/build/cmd/buildinternalplugins.go b/pkg/build/cmd/buildinternalplugins.go deleted file mode 100644 index 15a1397158d..00000000000 --- a/pkg/build/cmd/buildinternalplugins.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "context" - "log" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/plugins" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func BuildInternalPlugins(c *cli.Context) error { - cfg := config.Config{ - NumWorkers: c.Int("jobs"), - } - - const grafanaDir = "." - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return err - } - - log.Println("Building internal Grafana plug-ins...") - - ctx := context.Background() - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - var g *errutil.Group - g, ctx = errutil.GroupWithContext(ctx) - if err := plugins.Build(ctx, grafanaDir, p, g, buildConfig); err != nil { - return cli.Exit(err.Error(), 1) - } - if err := g.Wait(); err != nil { - return cli.Exit(err.Error(), 1) - } - - if err := plugins.Download(ctx, grafanaDir, p); err != nil { - return cli.Exit(err.Error(), 1) - } - - log.Println("Successfully built Grafana plug-ins!") - - return nil -} diff --git a/pkg/build/cmd/enterprisecheck.go b/pkg/build/cmd/enterprisecheck.go deleted file mode 100644 index 3fb13f6c893..00000000000 --- a/pkg/build/cmd/enterprisecheck.go +++ /dev/null @@ -1,133 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "strconv" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/env" - "github.com/grafana/grafana/pkg/build/git" -) - -// checkOpts are options used to create a new GitHub check for the enterprise downstream test. -type checkOpts struct { - SHA string - URL string - Branch string - PR int -} - -func getCheckOpts(args []string) (*checkOpts, error) { - branch, ok := env.Lookup("DRONE_SOURCE_BRANCH", args) - if !ok { - return nil, cli.Exit("Unable to retrieve build source branch", 1) - } - - var ( - rgx = git.PRCheckRegexp() - matches = rgx.FindStringSubmatch(branch) - ) - - sha, ok := env.Lookup("SOURCE_COMMIT", args) - if !ok { - if matches == nil || len(matches) <= 1 { - return nil, cli.Exit("Unable to retrieve source commit", 1) - } - sha = matches[2] - } - - url, ok := env.Lookup("DRONE_BUILD_LINK", args) - if !ok { - return nil, cli.Exit(`missing environment variable "DRONE_BUILD_LINK"`, 1) - } - - prStr, ok := env.Lookup("OSS_PULL_REQUEST", args) - if !ok { - if matches == nil || len(matches) <= 1 { - return nil, cli.Exit("Unable to retrieve PR number", 1) - } - - prStr = matches[1] - } - - pr, err := strconv.Atoi(prStr) - if err != nil { - return nil, err - } - - return &checkOpts{ - Branch: branch, - PR: pr, - SHA: sha, - URL: url, - }, nil -} - -// EnterpriseCheckBegin creates the GitHub check and signals the beginning of the downstream build / test process -func EnterpriseCheckBegin(c *cli.Context) error { - var ( - ctx = c.Context - client = git.NewGitHubClient(ctx, c.String("github-token")) - ) - - opts, err := getCheckOpts(os.Environ()) - if err != nil { - return err - } - - if _, err = git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, "pending"); err != nil { - return err - } - - return nil -} - -func EnterpriseCheckSuccess(c *cli.Context) error { - return completeEnterpriseCheck(c, true) -} - -func EnterpriseCheckFail(c *cli.Context) error { - return completeEnterpriseCheck(c, false) -} - -func completeEnterpriseCheck(c *cli.Context, success bool) error { - var ( - ctx = c.Context - client = git.NewGitHubClient(ctx, c.String("github-token")) - ) - - // Update the pull request labels - opts, err := getCheckOpts(os.Environ()) - if err != nil { - return err - } - - status := "failure" - if success { - status = "success" - } - - // Update the GitHub check... - if _, err := git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, status); err != nil { - return err - } - - // Delete branch if needed - log.Printf("Checking branch '%s' against '%s'", git.PRCheckRegexp().String(), opts.Branch) - if git.PRCheckRegexp().MatchString(opts.Branch) { - log.Println("Deleting branch", opts.Branch) - if err := git.DeleteEnterpriseBranch(ctx, client.Git, opts.Branch); err != nil { - return fmt.Errorf("error deleting enterprise branch: %w", err) - } - } - - label := "enterprise-failed" - if success { - label = "enterprise-ok" - } - - return git.AddLabelToPR(ctx, client.Issues, opts.PR, label) -} diff --git a/pkg/build/cmd/enterprisecheck_test.go b/pkg/build/cmd/enterprisecheck_test.go deleted file mode 100644 index 0eeb5bd5741..00000000000 --- a/pkg/build/cmd/enterprisecheck_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestGetCheckOpts(t *testing.T) { - t.Run("it should return the checkOpts if the correct environment variables are set", func(t *testing.T) { - args := []string{ - "SOURCE_COMMIT=1234", - "DRONE_SOURCE_BRANCH=test", - "DRONE_BUILD_LINK=http://example.com", - "OSS_PULL_REQUEST=1", - } - - opts, err := getCheckOpts(args) - require.NoError(t, err) - require.Equal(t, opts.SHA, "1234") - require.Equal(t, opts.URL, "http://example.com") - }) - t.Run("it should return an error if SOURCE_COMMIT is not set", func(t *testing.T) { - args := []string{ - "DRONE_BUILD_LINK=http://example.com", - "DRONE_SOURCE_BRANCH=test", - "DRONE_BUILD_LINK=http://example.com", - "OSS_PULL_REQUEST=1", - } - - opts, err := getCheckOpts(args) - require.Nil(t, opts) - require.Error(t, err) - }) - t.Run("it should return an error if DRONE_BUILD_LINK is not set", func(t *testing.T) { - args := []string{ - "SOURCE_COMMIT=1234", - "DRONE_SOURCE_BRANCH=test", - "OSS_PULL_REQUEST=1", - } - - opts, err := getCheckOpts(args) - require.Nil(t, opts) - require.Error(t, err) - }) - t.Run("it should return an error if OSS_PULL_REQUEST is not set", func(t *testing.T) { - args := []string{ - "SOURCE_COMMIT=1234", - "DRONE_SOURCE_BRANCH=test", - "DRONE_BUILD_LINK=http://example.com", - } - - opts, err := getCheckOpts(args) - require.Nil(t, opts) - require.Error(t, err) - }) - t.Run("it should return an error if OSS_PULL_REQUEST is not an integer", func(t *testing.T) { - args := []string{ - "SOURCE_COMMIT=1234", - "DRONE_SOURCE_BRANCH=test", - "DRONE_BUILD_LINK=http://example.com", - "OSS_PULL_REQUEST=http://example.com", - } - - opts, err := getCheckOpts(args) - require.Nil(t, opts) - require.Error(t, err) - }) -} diff --git a/pkg/build/cmd/exportversion.go b/pkg/build/cmd/exportversion.go deleted file mode 100644 index c4c4744ebef..00000000000 --- a/pkg/build/cmd/exportversion.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" -) - -func ExportVersion(c *cli.Context) error { - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - const distDir = "dist" - if err := os.RemoveAll(distDir); err != nil { - return err - } - if err := os.Mkdir(distDir, 0750); err != nil { - return err - } - - // nolint:gosec - if err := os.WriteFile(filepath.Join(distDir, "grafana.version"), []byte(metadata.GrafanaVersion), 0664); err != nil { - return err - } - - return nil -} diff --git a/pkg/build/cmd/fetchimages.go b/pkg/build/cmd/fetchimages.go index b131cc9a73d..26260aedb19 100644 --- a/pkg/build/cmd/fetchimages.go +++ b/pkg/build/cmd/fetchimages.go @@ -4,7 +4,7 @@ import ( "fmt" "log" "os/exec" - "strings" + "path/filepath" "github.com/urfave/cli/v2" @@ -18,6 +18,25 @@ const ( ubuntu = "ubuntu" ) +// GetImageFiles returns the list of image (.img, but should be .tar because they are tar archives) files that are +// created in the 'tag' process and stored in the prerelease bucket, waiting to be released. +func GetImageFiles(grafana string, version string, architectures []config.Architecture) []string { + bases := []string{alpine, ubuntu} + images := []string{} + for _, base := range bases { + for _, arch := range architectures { + image := fmt.Sprintf("%s-%s-%s.img", grafana, version, arch) + if base == "ubuntu" { + image = fmt.Sprintf("%s-%s-ubuntu-%s.img", grafana, version, arch) + } + + images = append(images, image) + } + } + + return images +} + func FetchImages(c *cli.Context) error { if c.NArg() > 0 { if err := cli.ShowSubcommandHelp(c); err != nil { @@ -44,74 +63,65 @@ func FetchImages(c *cli.Context) error { Tag: metadata.GrafanaVersion, } - edition := fmt.Sprintf("-%s", cfg.Edition) - - err = gcloud.ActivateServiceAccount() - if err != nil { - return err + grafana := "grafana" + if cfg.Edition == "enterprise" { + grafana = "grafana-enterprise" } - - var basesStr []string - for _, base := range cfg.Distribution { - switch base { - case alpine: - basesStr = append(basesStr, "") - case ubuntu: - basesStr = append(basesStr, "-ubuntu") - default: - return fmt.Errorf("unrecognized base %q", base) - } + if cfg.Edition == "enterprise2" { + grafana = "grafana-enterprise2" + } + if cfg.Edition == "grafana" || cfg.Edition == "oss" { + grafana = "grafana-oss" } - err = downloadFromGCS(cfg, basesStr, edition) - if err != nil { + baseURL := fmt.Sprintf("gs://%s/%s/", cfg.Bucket, cfg.Tag) + images := GetImageFiles(grafana, cfg.Tag, cfg.Archs) + + log.Printf("Fetching images [%v]", images) + + if err := gcloud.ActivateServiceAccount(); err != nil { return err } - - err = loadImages(cfg, basesStr, edition) - if err != nil { + if err := DownloadImages(baseURL, images, "."); err != nil { + return err + } + if err := LoadImages(images, "."); err != nil { return err } return nil } -func loadImages(cfg docker.Config, basesStr []string, edition string) error { - log.Println("Loading fetched image files to local docker registry...") - log.Printf("Number of images to be loaded: %d\n", len(basesStr)*len(cfg.Archs)) - for _, base := range basesStr { - for _, arch := range cfg.Archs { - imageFilename := fmt.Sprintf("grafana%s-%s%s-%s.img", edition, cfg.Tag, base, arch) - log.Printf("image file name: %s\n", imageFilename) - //nolint:gosec - cmd := exec.Command("docker", "load", "-i", imageFilename) - cmd.Dir = "." - out, err := cmd.CombinedOutput() - if err != nil { - log.Printf("out: %s\n", out) - return fmt.Errorf("error loading image: %q", err) - } - log.Printf("Successfully loaded %s!\n %s\n", fmt.Sprintf("grafana%s-%s%s-%s", edition, cfg.Tag, base, arch), out) +// LoadImages uses the `docker load -i` command to load the image tar file into the docker daemon so that it can be +// tagged and pushed. +func LoadImages(images []string, source string) error { + p := filepath.Clean(source) + for _, image := range images { + image := filepath.Join(p, image) + log.Println("Loading image", image) + //nolint:gosec + cmd := exec.Command("docker", "load", "-i", image) + cmd.Dir = "." + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("out: %s\n", out) + return fmt.Errorf("error loading image: %q", err) } + log.Println("Loaded image", image) } log.Println("Images successfully loaded!") return nil } -func downloadFromGCS(cfg docker.Config, basesStr []string, edition string) error { - log.Printf("Downloading Docker images from GCS bucket: %s\n", cfg.Bucket) - - for _, base := range basesStr { - for _, arch := range cfg.Archs { - src := fmt.Sprintf("gs://%s/%s/grafana%s-%s%s-%s.img", cfg.Bucket, cfg.Tag, edition, cfg.Tag, base, arch) - args := strings.Split(fmt.Sprintf("-m cp -r %s .", src), " ") - //nolint:gosec - cmd := exec.Command("gsutil", args...) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to download: %w\n%s", err, out) - } +func DownloadImages(baseURL string, images []string, destination string) error { + for _, image := range images { + p := baseURL + image + log.Println("Downloading image", p) + //nolint:gosec + cmd := exec.Command("gsutil", "-m", "cp", "-r", p, destination) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to download: %w\n%s", err, out) } } - log.Printf("Successfully fetched image files from %s bucket!\n", cfg.Bucket) return nil } diff --git a/pkg/build/cmd/fetchimages_test.go b/pkg/build/cmd/fetchimages_test.go new file mode 100644 index 00000000000..9452b011a66 --- /dev/null +++ b/pkg/build/cmd/fetchimages_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "testing" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/stretchr/testify/require" +) + +func TestGetImageFiles(t *testing.T) { + var ( + architectures = []config.Architecture{ + config.ArchAMD64, + config.ArchARM64, + config.ArchARMv7, + } + ) + + t.Run("1.2.3", func(t *testing.T) { + expect := []string{ + "grafana-oss-1.2.3-amd64.img", + "grafana-oss-1.2.3-arm64.img", + "grafana-oss-1.2.3-armv7.img", + "grafana-oss-1.2.3-ubuntu-amd64.img", + "grafana-oss-1.2.3-ubuntu-arm64.img", + "grafana-oss-1.2.3-ubuntu-armv7.img", + } + + res := GetImageFiles("grafana-oss", "1.2.3", architectures) + + require.Equal(t, expect, res) + }) + + t.Run("1.2.3+example-01", func(t *testing.T) { + expect := []string{ + "grafana-oss-1.2.3+example-01-amd64.img", + "grafana-oss-1.2.3+example-01-arm64.img", + "grafana-oss-1.2.3+example-01-armv7.img", + "grafana-oss-1.2.3+example-01-ubuntu-amd64.img", + "grafana-oss-1.2.3+example-01-ubuntu-arm64.img", + "grafana-oss-1.2.3+example-01-ubuntu-armv7.img", + } + + res := GetImageFiles("grafana-oss", "1.2.3+example-01", architectures) + + require.Equal(t, expect, res) + }) +} diff --git a/pkg/build/cmd/flags.go b/pkg/build/cmd/flags.go index c9020fe2403..7ab2eb6f993 100644 --- a/pkg/build/cmd/flags.go +++ b/pkg/build/cmd/flags.go @@ -16,37 +16,15 @@ var ( Usage: "The edition of Grafana to build (oss or enterprise)", Value: "oss", } - variantsFlag = cli.StringFlag{ - Name: "variants", - Usage: "Comma-separated list of variants to build", - } triesFlag = cli.IntFlag{ Name: "tries", Usage: "Specify number of tries before failing", Value: 1, } - noInstallDepsFlag = cli.BoolFlag{ - Name: "no-install-deps", - Usage: "Don't install dependencies", - } - signingAdminFlag = cli.BoolFlag{ - Name: "signing-admin", - Usage: "Use manifest signing admin API endpoint?", - } - signFlag = cli.BoolFlag{ - Name: "sign", - Usage: "Enable plug-in signing (you must set GRAFANA_API_KEY)", - } dryRunFlag = cli.BoolFlag{ Name: "dry-run", Usage: "Only simulate actions", } - gitHubTokenFlag = cli.StringFlag{ - Name: "github-token", - Value: "", - EnvVars: []string{"GITHUB_TOKEN"}, - Usage: "GitHub token", - } tagFlag = cli.StringFlag{ Name: "tag", Usage: "Grafana version tag", diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go index 0627effcefa..305aa1195e8 100644 --- a/pkg/build/cmd/grafanacom.go +++ b/pkg/build/cmd/grafanacom.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/gcloud" "github.com/grafana/grafana/pkg/build/gcloud/storage" + "github.com/grafana/grafana/pkg/build/gcom" "github.com/grafana/grafana/pkg/build/packaging" ) @@ -125,6 +126,48 @@ func getReleaseURLs() (string, string, error) { return pconf.Grafana.WhatsNewURL, pconf.Grafana.ReleaseNotesURL, nil } +func Builds(baseURL *url.URL, grafana, version string, packages []packaging.BuildArtifact) ([]GCOMPackage, error) { + builds := make([]GCOMPackage, len(packages)) + for i, v := range packages { + var ( + os = v.Distro + arch = v.Arch + ) + + if v.Distro == "windows" { + os = "win" + if v.Ext == "msi" { + os = "win-installer" + } + } + + if v.Distro == "rhel" { + if arch == "aarch64" { + arch = "arm64" + } + } + + if v.Distro == "deb" { + if arch == "armhf" { + arch = "armv7" + if v.RaspberryPi { + log.Println(v.Distro, arch, "raspberrypi == true") + arch = "armv6" + } + } + } + + u := gcom.GetURL(baseURL, version, grafana, v.Distro, v.Arch, v.Ext, v.Musl, v.RaspberryPi) + builds[i] = GCOMPackage{ + OS: os, + URL: u.String(), + Arch: arch, + } + } + + return builds, nil +} + // publishPackages publishes packages to grafana.com. func publishPackages(cfg packaging.PublishConfig) error { log.Printf("Publishing Grafana packages, version %s, %s edition, %s mode, dryRun: %v, simulating: %v...\n", @@ -133,14 +176,17 @@ func publishPackages(cfg packaging.PublishConfig) error { versionStr := fmt.Sprintf("v%s", cfg.Version) log.Printf("Creating release %s at grafana.com...\n", versionStr) - var sfx string - var pth string + var ( + pth string + grafana = "grafana" + ) + switch cfg.Edition { case config.EditionOSS: pth = "oss" case config.EditionEnterprise: + grafana = "grafana-enterprise" pth = "enterprise" - sfx = packaging.EnterpriseSfx default: return fmt.Errorf("unrecognized edition %q", cfg.Edition) } @@ -152,28 +198,19 @@ func publishPackages(cfg packaging.PublishConfig) error { pth = path.Join(pth, packaging.ReleaseFolder) } - product := fmt.Sprintf("grafana%s", sfx) - pth = path.Join(pth, product) - baseArchiveURL := fmt.Sprintf("https://dl.grafana.com/%s", pth) - - builds := make([]buildRepr, len(packaging.ArtifactConfigs)) - for i, ba := range packaging.ArtifactConfigs { - u := ba.GetURL(baseArchiveURL, cfg) - - sha256, err := getSHA256(u) - if err != nil { - return err - } + pth = path.Join(pth) + baseArchiveURL := &url.URL{ + Scheme: "https", + Host: "dl.grafana.com", + Path: pth, + } - builds[i] = buildRepr{ - OS: ba.Os, - URL: u, - SHA256: string(sha256), - Arch: ba.Arch, - } + builds, err := Builds(baseArchiveURL, grafana, cfg.Version, packaging.ArtifactConfigs) + if err != nil { + return err } - r := releaseRepr{ + r := Release{ Version: cfg.Version, ReleaseDate: time.Now().UTC(), Builds: builds, @@ -195,6 +232,15 @@ func publishPackages(cfg packaging.PublishConfig) error { return err } + for i, v := range r.Builds { + sha, err := getSHA256(v.URL) + if err != nil { + return err + } + + r.Builds[i].SHA256 = string(sha) + } + for _, b := range r.Builds { if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages", cfg.Version), b, fmt.Sprintf("create build %s %s", b.OS, b.Arch)); err != nil { @@ -211,6 +257,7 @@ func publishPackages(cfg packaging.PublishConfig) error { func getSHA256(u string) ([]byte, error) { shaURL := fmt.Sprintf("%s.sha256", u) + // nolint:gosec resp, err := http.Get(shaURL) if err != nil { @@ -232,7 +279,7 @@ func getSHA256(u string) ([]byte, error) { return sha256, nil } -func postRequest(cfg packaging.PublishConfig, pth string, obj any, descr string) error { +func postRequest(cfg packaging.PublishConfig, pth string, body any, descr string) error { var sfx string switch cfg.Edition { case config.EditionOSS: @@ -243,7 +290,7 @@ func postRequest(cfg packaging.PublishConfig, pth string, obj any, descr string) } product := fmt.Sprintf("grafana%s", sfx) - jsonB, err := json.Marshal(obj) + jsonB, err := json.Marshal(body) if err != nil { return fmt.Errorf("failed to JSON encode release: %w", err) } @@ -303,20 +350,20 @@ func constructURL(product string, pth string) (string, error) { return u.String(), err } -type buildRepr struct { +type GCOMPackage struct { OS string `json:"os"` URL string `json:"url"` SHA256 string `json:"sha256"` Arch string `json:"arch"` } -type releaseRepr struct { - Version string `json:"version"` - ReleaseDate time.Time `json:"releaseDate"` - Stable bool `json:"stable"` - Beta bool `json:"beta"` - Nightly bool `json:"nightly"` - WhatsNewURL string `json:"whatsNewUrl"` - ReleaseNotesURL string `json:"releaseNotesUrl"` - Builds []buildRepr `json:"-"` +type Release struct { + Version string `json:"version"` + ReleaseDate time.Time `json:"releaseDate"` + Stable bool `json:"stable"` + Beta bool `json:"beta"` + Nightly bool `json:"nightly"` + WhatsNewURL string `json:"whatsNewUrl"` + ReleaseNotesURL string `json:"releaseNotesUrl"` + Builds []GCOMPackage `json:"-"` } diff --git a/pkg/build/cmd/grafanacom_test.go b/pkg/build/cmd/grafanacom_test.go index bf90874b1c9..9f4fbd7c981 100644 --- a/pkg/build/cmd/grafanacom_test.go +++ b/pkg/build/cmd/grafanacom_test.go @@ -1,7 +1,14 @@ package main import ( + "fmt" + "net/url" + "path" "testing" + + "github.com/grafana/grafana/pkg/build/packaging" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_constructURL(t *testing.T) { @@ -33,3 +40,221 @@ func Test_constructURL(t *testing.T) { }) } } + +func TestBuilds(t *testing.T) { + baseURL := &url.URL{ + Scheme: "https", + Host: "dl.example.com", + Path: path.Join("oss", "release"), + } + + version := "1.2.3" + grafana := "grafana" + packages := []packaging.BuildArtifact{ + { + Distro: "deb", + Arch: "arm64", + Ext: "deb", + }, + { + Distro: "rhel", + Arch: "aarch64", + Ext: "rpm", + }, + { + Distro: "linux", + Arch: "arm64", + Ext: "tar.gz", + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + RaspberryPi: true, + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + }, + { + Distro: "linux", + Arch: "armv7", + Ext: "tar.gz", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "zip", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "msi", + }, + } + + expect := []GCOMPackage{ + { + URL: "https://dl.example.com/oss/release/grafana_1.2.3_arm64.deb", + OS: "deb", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3-1.aarch64.rpm", + OS: "rhel", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3.linux-arm64.tar.gz", + OS: "linux", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-rpi_1.2.3_armhf.deb", + OS: "deb", + Arch: "armv6", + }, + { + URL: "https://dl.example.com/oss/release/grafana_1.2.3_armhf.deb", + OS: "deb", + Arch: "armv7", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3.linux-armv7.tar.gz", + OS: "linux", + Arch: "armv7", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3.windows-amd64.zip", + OS: "win", + Arch: "amd64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3.windows-amd64.msi", + OS: "win-installer", + Arch: "amd64", + }, + } + + builds, err := Builds(baseURL, grafana, version, packages) + require.NoError(t, err) + require.Equal(t, len(expect), len(builds)) + + for i := range builds { + t.Run(fmt.Sprintf("[%d/%d] %s", i+1, len(builds), expect[i].URL), func(t *testing.T) { + assert.Equal(t, expect[i].URL, builds[i].URL) + assert.Equal(t, expect[i].OS, builds[i].OS) + assert.Equal(t, expect[i].Arch, builds[i].Arch) + }) + } +} + +func TestBuildsWithPlus(t *testing.T) { + baseURL := &url.URL{ + Scheme: "https", + Host: "dl.example.com", + Path: path.Join("oss", "release"), + } + + version := "1.2.3+example-01" + grafana := "grafana" + packages := []packaging.BuildArtifact{ + { + Distro: "deb", + Arch: "arm64", + Ext: "deb", + }, + { + Distro: "rhel", + Arch: "aarch64", + Ext: "rpm", + }, + { + Distro: "linux", + Arch: "arm64", + Ext: "tar.gz", + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + RaspberryPi: true, + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + }, + { + Distro: "linux", + Arch: "armv7", + Ext: "tar.gz", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "zip", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "msi", + }, + } + + expect := []GCOMPackage{ + { + URL: "https://dl.example.com/oss/release/grafana_1.2.3+example~01_arm64.deb", + OS: "deb", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example~01-1.aarch64.rpm", + OS: "rhel", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example-01.linux-arm64.tar.gz", + OS: "linux", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-rpi_1.2.3+example~01_armhf.deb", + OS: "deb", + Arch: "armv6", + }, + { + URL: "https://dl.example.com/oss/release/grafana_1.2.3+example~01_armhf.deb", + OS: "deb", + Arch: "armv7", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example-01.linux-armv7.tar.gz", + OS: "linux", + Arch: "armv7", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example-01.windows-amd64.zip", + OS: "win", + Arch: "amd64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example-01.windows-amd64.msi", + OS: "win-installer", + Arch: "amd64", + }, + } + + builds, err := Builds(baseURL, grafana, version, packages) + require.NoError(t, err) + require.Equal(t, len(expect), len(builds)) + + for i := range builds { + t.Run(fmt.Sprintf("[%d/%d] %s", i+1, len(builds), expect[i].URL), func(t *testing.T) { + assert.Equal(t, expect[i].URL, builds[i].URL) + assert.Equal(t, expect[i].OS, builds[i].OS) + assert.Equal(t, expect[i].Arch, builds[i].Arch) + }) + } +} diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index dff9db8f8b0..d207abe020b 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -3,11 +3,9 @@ package main import ( "log" "os" - "strings" + "github.com/grafana/grafana/pkg/build" "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/docker" ) var additionalCommands []*cli.Command = make([]*cli.Command, 0, 5) @@ -21,28 +19,8 @@ func main() { app := cli.NewApp() app.Commands = cli.Commands{ { - Name: "build-backend", - Usage: "Build one or more variants of back-end binaries", - ArgsUsage: "[version]", - Action: MaxArgCountWrapper(1, BuildBackend), - Flags: []cli.Flag{ - &jobsFlag, - &variantsFlag, - &editionFlag, - &buildIDFlag, - }, - }, - { - Name: "build-frontend-packages", - Usage: "Build front-end packages", - ArgsUsage: "[version]", - Action: BuildFrontendPackages, - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &buildIDFlag, - &noInstallDepsFlag, - }, + Name: "build", + Action: build.RunCmdCLI, }, { Name: "e2e-tests", @@ -71,44 +49,11 @@ func main() { }, }, }, - { - Name: "build-frontend", - Usage: "Build front-end artifacts", - ArgsUsage: "[version]", - Action: MaxArgCountWrapper(1, BuildFrontend), - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &buildIDFlag, - }, - }, { Name: "whatsnew-checker", Usage: "Checks whatsNewUrl in package.json for differences between the tag and the docs version", Action: WhatsNewChecker, }, - { - Name: "build-docker", - Usage: "Build Grafana Docker images", - Action: MaxArgCountWrapper(1, BuildDocker), - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &cli.BoolFlag{ - Name: "ubuntu", - Usage: "Use Ubuntu base image", - }, - &cli.BoolFlag{ - Name: "shouldSave", - Usage: "Should save docker image to tarball", - }, - &cli.StringFlag{ - Name: "archs", - Value: strings.Join(docker.AllArchs, ","), - Usage: "Comma separated architectures to build", - }, - }, - }, { Name: "upload-cdn", Usage: "Upload public/* to a cdn bucket", @@ -117,23 +62,6 @@ func main() { &editionFlag, }, }, - { - Name: "shellcheck", - Usage: "Run shellcheck on shell scripts", - Action: Shellcheck, - }, - { - Name: "build-plugins", - Usage: "Build internal plug-ins", - Action: MaxArgCountWrapper(1, BuildInternalPlugins), - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &signingAdminFlag, - &signFlag, - &noInstallDepsFlag, - }, - }, { Name: "publish-metrics", Usage: "Publish a set of metrics from stdin", @@ -145,30 +73,6 @@ func main() { Usage: "Verify Drone configuration", Action: VerifyDrone, }, - { - Name: "verify-starlark", - Usage: "Verify Starlark configuration", - ArgsUsage: "", - Action: VerifyStarlark, - }, - { - Name: "export-version", - Usage: "Exports version in dist/grafana.version", - Action: ExportVersion, - }, - { - Name: "package", - Usage: "Package one or more Grafana variants", - ArgsUsage: "[version]", - Action: MaxArgCountWrapper(1, Package), - Flags: []cli.Flag{ - &jobsFlag, - &variantsFlag, - &editionFlag, - &buildIDFlag, - &signFlag, - }, - }, { Name: "store-storybook", Usage: "Stores storybook to GCS buckets", @@ -279,18 +183,6 @@ func main() { &editionFlag, }, }, - { - Name: "publish-enterprise2", - Usage: "Handle Grafana Enterprise2 Docker images", - ArgsUsage: "[version]", - Action: Enterprise2, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "dockerhub-repo", - Usage: "DockerHub repo to push images", - }, - }, - }, }, }, { @@ -399,36 +291,6 @@ func main() { }, }, }, - { - Name: "enterprise-check", - Usage: "Commands for testing against Grafana Enterprise", - Subcommands: cli.Commands{ - { - Name: "begin", - Usage: "Creates the GitHub check in a pull request and begins the tests", - Action: EnterpriseCheckBegin, - Flags: []cli.Flag{ - &gitHubTokenFlag, - }, - }, - { - Name: "success", - Usage: "Updates the GitHub check in a pull request to show a successful build and updates the pull request labels", - Action: EnterpriseCheckSuccess, - Flags: []cli.Flag{ - &gitHubTokenFlag, - }, - }, - { - Name: "fail", - Usage: "Updates the GitHub check in a pull request to show a failed build and updates the pull request labels", - Action: EnterpriseCheckFail, - Flags: []cli.Flag{ - &gitHubTokenFlag, - }, - }, - }, - }, } app.Commands = append(app.Commands, additionalCommands...) diff --git a/pkg/build/cmd/package.go b/pkg/build/cmd/package.go deleted file mode 100644 index 6b961512ec1..00000000000 --- a/pkg/build/cmd/package.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "context" - "log" - "strings" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/gpg" - "github.com/grafana/grafana/pkg/build/packaging" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func Package(c *cli.Context) error { - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - edition := config.Edition(c.String("edition")) - - releaseMode, err := metadata.GetReleaseMode() - if err != nil { - return cli.Exit(err.Error(), 1) - } - - releaseModeConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return cli.Exit(err.Error(), 1) - } - - cfg := config.Config{ - NumWorkers: c.Int("jobs"), - SignPackages: c.Bool("sign"), - } - - ctx := context.Background() - - variants := []config.Variant{} - variantStrs := strings.Split(c.String("variants"), ",") - if c.String("variants") != "" { - for _, varStr := range variantStrs { - if varStr == "" { - continue - } - variants = append(variants, config.Variant(varStr)) - } - } else { - variants = releaseModeConfig.Variants - } - - if len(variants) == 0 { - variants = config.AllVariants - } - - log.Printf("Packaging Grafana version %q, version mode %s, %s edition, variants %s", metadata.GrafanaVersion, releaseMode.Mode, - edition, strings.Join(variantStrs, ",")) - - if cfg.SignPackages { - if err := gpg.LoadGPGKeys(&cfg); err != nil { - return cli.Exit(err, 1) - } - defer gpg.RemoveGPGFiles(cfg) - if err := gpg.Import(cfg); err != nil { - return cli.Exit(err, 1) - } - } - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - if err := packaging.PackageGrafana(ctx, metadata.GrafanaVersion, ".", cfg, edition, variants, releaseModeConfig.PluginSignature.Sign, p); err != nil { - return cli.Exit(err, 1) - } - - log.Println("Successfully packaged Grafana!") - return nil -} diff --git a/pkg/build/cmd/publishimages_enterprise2.go b/pkg/build/cmd/publishimages_enterprise2.go deleted file mode 100644 index 94cd4b57c7d..00000000000 --- a/pkg/build/cmd/publishimages_enterprise2.go +++ /dev/null @@ -1,101 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/docker" - "github.com/grafana/grafana/pkg/build/gcloud" -) - -func Enterprise2(c *cli.Context) error { - if c.NArg() > 0 { - if err := cli.ShowSubcommandHelp(c); err != nil { - return cli.Exit(err.Error(), 1) - } - return cli.Exit("", 1) - } - - if err := gcloud.ActivateServiceAccount(); err != nil { - return fmt.Errorf("couldn't activate service account, err: %w", err) - } - - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return err - } - - cfg := docker.Config{ - Archs: buildConfig.Docker.Architectures, - Distribution: buildConfig.Docker.Distribution, - DockerHubRepo: c.String("dockerhub-repo"), - Tag: metadata.GrafanaVersion, - } - - err = dockerLoginEnterprise2() - if err != nil { - return err - } - - var distributionStr []string - for _, distribution := range cfg.Distribution { - switch distribution { - case alpine: - distributionStr = append(distributionStr, "") - case ubuntu: - distributionStr = append(distributionStr, "-ubuntu") - default: - return fmt.Errorf("unrecognized distribution %q", distribution) - } - } - - for _, distribution := range distributionStr { - var imageFileNames []string - for _, arch := range cfg.Archs { - imageFilename := fmt.Sprintf("%s:%s%s-%s", cfg.DockerHubRepo, cfg.Tag, distribution, arch) - err := docker.PushImage(imageFilename) - if err != nil { - return err - } - imageFileNames = append(imageFileNames, imageFilename) - } - manifest := fmt.Sprintf("%s:%s%s", cfg.DockerHubRepo, cfg.Tag, distribution) - args := []string{"manifest", "create", manifest} - args = append(args, imageFileNames...) - - //nolint:gosec - cmd := exec.Command("docker", args...) - cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled") - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to create Docker manifest: %w\n%s", err, output) - } - - err = docker.PushManifest(manifest) - if err != nil { - return err - } - } - - return nil -} - -func dockerLoginEnterprise2() error { - log.Println("Docker login...") - cmd := exec.Command("gcloud", "auth", "configure-docker") - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("error logging in to DockerHub: %s %q", out, err) - } - - log.Println("Successful login!") - return nil -} diff --git a/pkg/build/cmd/shellcheck.go b/pkg/build/cmd/shellcheck.go deleted file mode 100644 index e18b1d85258..00000000000 --- a/pkg/build/cmd/shellcheck.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/urfave/cli/v2" -) - -func Shellcheck(c *cli.Context) error { - log.Println("Running shellcheck...") - - fpaths := []string{} - if err := filepath.Walk("scripts", func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if strings.HasSuffix(path, ".sh") { - fpaths = append(fpaths, path) - } - - return nil - }); err != nil { - return fmt.Errorf("couldn't traverse scripts/: %w", err) - } - - log.Printf("Running shellcheck on %s", strings.Join(fpaths, ",")) - args := append([]string{"-e", "SC1071", "-e", "SC2162"}, fpaths...) - //nolint:gosec - cmd := exec.Command("shellcheck", args...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("shellcheck failed: %s", output) - } - - log.Println("Successfully ran shellcheck!") - return nil -} diff --git a/pkg/build/cmd/uploadpackages.go b/pkg/build/cmd/uploadpackages.go index da0dd6a607a..5801192c62c 100644 --- a/pkg/build/cmd/uploadpackages.go +++ b/pkg/build/cmd/uploadpackages.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "github.com/urfave/cli/v2" @@ -14,9 +15,28 @@ import ( "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/droneutil" "github.com/grafana/grafana/pkg/build/gcloud" - "github.com/grafana/grafana/pkg/build/packaging" ) +// PackageRegexp returns a regexp for matching packages corresponding to a certain Grafana edition. +func PackageRegexp(edition config.Edition) *regexp.Regexp { + var sfx string + switch edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = "-enterprise" + case config.EditionEnterprise2: + sfx = "-enterprise2" + default: + panic(fmt.Sprintf("unrecognized edition %q", edition)) + } + rePkg, err := regexp.Compile(fmt.Sprintf(`^grafana%s(?:-rpi)?[-_][^-_]+.*$`, sfx)) + if err != nil { + panic(fmt.Sprintf("Failed to compile regexp: %s", err)) + } + + return rePkg +} + const releaseFolder = "release" const mainFolder = "main" const releaseBranchFolder = "prerelease" @@ -181,7 +201,7 @@ func uploadPackages(cfg uploadConfig) error { return fmt.Errorf("failed to list packages: %w", err) } fpaths := []string{} - rePkg := packaging.PackageRegexp(cfg.edition) + rePkg := PackageRegexp(cfg.edition) for _, fpath := range matches { fname := filepath.Base(fpath) if strings.Contains(fname, "latest") || !rePkg.MatchString(fname) { diff --git a/pkg/build/cmd/verifystarlark.go b/pkg/build/cmd/verifystarlark.go deleted file mode 100644 index ff33a77a3af..00000000000 --- a/pkg/build/cmd/verifystarlark.go +++ /dev/null @@ -1,142 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "io/fs" - "os/exec" - "path/filepath" - "strings" - - "github.com/urfave/cli/v2" -) - -func mapSlice[I any, O any](a []I, f func(I) O) []O { - o := make([]O, len(a)) - for i, e := range a { - o[i] = f(e) - } - return o -} - -// VerifyStarlark is the CLI Action for verifying Starlark files in a workspace. -// It expects a single context argument which is the path to the workspace. -// The actual verification procedure can return multiple errors which are -// joined together to be one holistic error for the action. -func VerifyStarlark(c *cli.Context) error { - if c.NArg() != 1 { - var message string - if c.NArg() == 0 { - message = "ERROR: missing required argument " - } - if c.NArg() > 1 { - message = "ERROR: too many arguments" - } - - if err := cli.ShowSubcommandHelp(c); err != nil { - return err - } - - return cli.Exit(message, 1) - } - - workspace := c.Args().Get(0) - verificationErrs, executionErr := verifyStarlark(c.Context, workspace, buildifierLintCommand) - if executionErr != nil { - return executionErr - } - - if len(verificationErrs) == 0 { - return nil - } - - noun := "file" - if len(verificationErrs) > 1 { - noun += "s" - } - - return fmt.Errorf("verification failed for %d %s:\n%s", - len(verificationErrs), - noun, - strings.Join( - mapSlice(verificationErrs, func(e error) string { return e.Error() }), - "\n", - )) -} - -type commandFunc = func(path string) (command string, args []string) - -func buildifierLintCommand(path string) (string, []string) { - return "buildifier", []string{"-lint", "warn", "-mode", "check", path} -} - -// verifyStarlark walks all directories starting at provided workspace path and -// verifies any Starlark files it finds. -// Starlark files are assumed to end with the .star extension. -// The verification relies on linting frovided by the 'buildifier' binary which -// must be in the PATH. -// A slice of verification errors are returned, one for each file that failed verification. -// If any execution of the `buildifier` command fails, this is returned separately. -// commandFn is executed on every Starlark file to determine the command and arguments to be executed. -// The caller is trusted and it is the callers responsibility to ensure that the resulting command is safe to execute. -func verifyStarlark(ctx context.Context, workspace string, commandFn commandFunc) ([]error, error) { - var verificationErrs []error - - // All errors from filepath.WalkDir are filtered by the fs.WalkDirFunc. - // Lstat or ReadDir errors are reported as verificationErrors. - // If any execution of the `buildifier` command fails or if the context is cancelled, - // it is reported as an error and any verification of subsequent files is skipped. - err := filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { - // Skip verification of the file or files within the directory if there is an error - // returned by Lstat or ReadDir. - if err != nil { - verificationErrs = append(verificationErrs, err) - return nil - } - - if d.IsDir() { - return nil - } - - if filepath.Ext(path) == ".star" { - command, args := commandFn(path) - // The caller is trusted. - //nolint:gosec - cmd := exec.CommandContext(ctx, command, args...) - cmd.Dir = workspace - - _, err = cmd.Output() - if err == nil { // No error, early return. - return nil - } - - // The error returned from cmd.Output() is never wrapped. - //nolint:errorlint - if err, ok := err.(*exec.ExitError); ok { - switch err.ExitCode() { - // Case comments are informed by the output of `buildifier --help` - case 1: // syntax errors in input - verificationErrs = append(verificationErrs, errors.New(string(err.Stderr))) - return nil - case 2: // usage errors: invoked incorrectly - return fmt.Errorf("command %q: %s", cmd, err.Stderr) - case 3: // unexpected runtime errors: file I/O problems or internal bugs - return fmt.Errorf("command %q: %s", cmd, err.Stderr) - case 4: // check mode failed (reformat is needed) - verificationErrs = append(verificationErrs, errors.New(string(err.Stderr))) - return nil - default: - return fmt.Errorf("command %q: %s", cmd, err.Stderr) - } - } - - // Error was not an exit error from the command. - return fmt.Errorf("command %q: %v", cmd, err) - } - - return nil - }) - - return verificationErrs, err -} diff --git a/pkg/build/cmd/verifystarlark_test.go b/pkg/build/cmd/verifystarlark_test.go deleted file mode 100644 index 1121ca5bd64..00000000000 --- a/pkg/build/cmd/verifystarlark_test.go +++ /dev/null @@ -1,137 +0,0 @@ -//go:build requires_buildifier - -package main - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestVerifyStarlark(t *testing.T) { - t.Run("execution errors", func(t *testing.T) { - t.Run("invalid usage", func(t *testing.T) { - ctx := context.Background() - workspace := t.TempDir() - err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm) - if err != nil { - t.Fatalf(err.Error()) - } - - _, executionErr := verifyStarlark(ctx, workspace, func(string) (string, []string) { return "buildifier", []string{"--invalid"} }) - if executionErr == nil { - t.Fatalf("Expected execution error but got none") - } - }) - - t.Run("context cancellation", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - workspace := t.TempDir() - err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm) - if err != nil { - t.Fatalf(err.Error()) - } - err = os.WriteFile(filepath.Join(workspace, "other-ignored.star"), []byte{}, os.ModePerm) - if err != nil { - t.Fatalf(err.Error()) - } - cancel() - - _, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) - if executionErr == nil { - t.Fatalf("Expected execution error but got none") - } - }) - }) - - t.Run("verification errors", func(t *testing.T) { - t.Run("a single file with lint", func(t *testing.T) { - ctx := context.Background() - workspace := t.TempDir() - - invalidContent := []byte(`load("scripts/drone/other.star", "function") - -function()`) - err := os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm) - if err != nil { - t.Fatalf(err.Error()) - } - - verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) - if executionErr != nil { - t.Fatalf("Unexpected execution error: %v", executionErr) - } - if len(verificationErrs) == 0 { - t.Fatalf(`"has-lint.star" requires linting but the verifyStarlark function provided no linting error`) - } - if len(verificationErrs) > 1 { - t.Fatalf(`verifyStarlark returned multiple errors for the "has-lint.star" file but only one was expected: %v`, verificationErrs) - } - if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") { - t.Fatalf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) - } - }) - - t.Run("no files with lint", func(t *testing.T) { - ctx := context.Background() - workspace := t.TempDir() - - content := []byte(`""" -This module does nothing. -""" - -load("scripts/drone/other.star", "function") - -function() -`) - require.NoError(t, os.WriteFile(filepath.Join(workspace, "no-lint.star"), content, os.ModePerm)) - - verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) - if executionErr != nil { - t.Fatalf("Unexpected execution error: %v", executionErr) - } - if len(verificationErrs) != 0 { - t.Log(`"no-lint.star" has no lint but the verifyStarlark function provided at least one error`) - for _, err := range verificationErrs { - t.Log(err) - } - t.FailNow() - } - }) - - t.Run("multiple files with lint", func(t *testing.T) { - ctx := context.Background() - workspace := t.TempDir() - - invalidContent := []byte(`load("scripts/drone/other.star", "function") - -function()`) - require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint2.star"), invalidContent, os.ModePerm)) - - verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) - if executionErr != nil { - t.Fatalf("Unexpected execution error: %v", executionErr) - } - if len(verificationErrs) == 0 { - t.Fatalf(`Two files require linting but the verifyStarlark function provided no linting error`) - } - if len(verificationErrs) == 1 { - t.Fatalf(`Two files require linting but the verifyStarlark function provided only one linting error: %v`, verificationErrs[0]) - } - if len(verificationErrs) > 2 { - t.Fatalf(`verifyStarlark returned more errors than expected: %v`, verificationErrs) - } - if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") { - t.Errorf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) - } - if !strings.Contains(verificationErrs[1].Error(), "has-lint2.star:1: module-docstring: The file has no module docstring.") { - t.Fatalf(`"has-lint2.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) - } - }) - }) -} diff --git a/pkg/build/compilers/install.go b/pkg/build/compilers/install.go deleted file mode 100644 index 9a45ba48c0b..00000000000 --- a/pkg/build/compilers/install.go +++ /dev/null @@ -1,50 +0,0 @@ -package compilers - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" -) - -const ( - ArmV6 = "/opt/rpi-tools/arm-bcm2708/arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc" - Armv7 = "arm-linux-gnueabihf-gcc" - Armv7Musl = "/tmp/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc" - Arm64 = "aarch64-linux-gnu-gcc" - Arm64Musl = "/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc" - Osx64 = "/tmp/osxcross/target/bin/o64-clang" - Win64 = "x86_64-w64-mingw32-gcc" - LinuxX64 = "/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc" - LinuxX64Musl = "/tmp/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc" -) - -func Install() error { - // From the os.TempDir documentation: - // On Unix systems, it returns $TMPDIR if non-empty, - // else /tmp. On Windows, it uses GetTempPath, - // returning the first non-empty value from %TMP%, %TEMP%, %USERPROFILE%, - // or the Windows directory. On Plan 9, it returns /tmp. - tmp := os.TempDir() - - var ( - centosArchive = "x86_64-centos6-linux-gnu.tar.xz" - osxArchive = "osxcross.tar.xz" - ) - - for _, fname := range []string{centosArchive, osxArchive} { - path := filepath.Join(tmp, fname) - if _, err := os.Stat(path); err != nil { - return fmt.Errorf("stat error: %w", err) - } - // Ignore gosec G204 as this function is only used in the build process. - //nolint:gosec - cmd := exec.Command("tar", "xfJ", fname) - cmd.Dir = tmp - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to unpack %q: %q, %w", fname, output, err) - } - } - - return nil -} diff --git a/pkg/build/config/revision.go b/pkg/build/config/revision.go deleted file mode 100644 index 70be21cdf18..00000000000 --- a/pkg/build/config/revision.go +++ /dev/null @@ -1,69 +0,0 @@ -package config - -import ( - "context" - "fmt" - "log" - "strconv" - "time" - - "github.com/grafana/grafana/pkg/build/executil" -) - -type Revision struct { - Timestamp int64 - SHA256 string - EnterpriseCommit string - Branch string -} - -func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) { - out, err := executil.OutputAt(ctx, dir, "git", "show", "-s", "--format=%ct") - if err != nil { - return time.Now().Unix(), nil - } - - stamp, err := strconv.ParseInt(out, 10, 64) - if err != nil { - return 0, fmt.Errorf("failed to parse output from git show: %q", out) - } - - return stamp, nil -} - -// GrafanaRevision uses git commands to get information about the checked out Grafana code located at 'grafanaDir'. -// This could maybe be a more generic "Describe" function in the "git" package. -func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) { - stamp, err := GrafanaTimestamp(ctx, grafanaDir) - if err != nil { - return Revision{}, err - } - - sha, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--short", "HEAD") - if err != nil { - return Revision{}, err - } - - enterpriseCommit, err := executil.OutputAt(ctx, grafanaDir, "git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD") - if err != nil { - enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "..", "rev-parse", "--short", "HEAD") - if err != nil { - enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "/tmp/grafana-enterprise", "rev-parse", "--short", "HEAD") - if err != nil { - log.Println("Could not get enterprise commit. Error:", err) - } - } - } - - branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD") - if err != nil { - return Revision{}, err - } - - return Revision{ - SHA256: sha, - EnterpriseCommit: enterpriseCommit, - Branch: branch, - Timestamp: stamp, - }, nil -} diff --git a/pkg/build/cryptoutil/md5.go b/pkg/build/cryptoutil/md5.go deleted file mode 100644 index fa4b4fcc5ec..00000000000 --- a/pkg/build/cryptoutil/md5.go +++ /dev/null @@ -1,35 +0,0 @@ -package cryptoutil - -import ( - "crypto/md5" - "fmt" - "io" - "log" - "os" -) - -func MD5File(fpath string) error { - // Ignore gosec G304 as this function is only used in the build process. - //nolint:gosec - fd, err := os.Open(fpath) - if err != nil { - return err - } - defer func() { - if err := fd.Close(); err != nil { - log.Printf("error closing file at '%s': %s", fpath, err.Error()) - } - }() - - h := md5.New() // nolint:gosec - if _, err = io.Copy(h, fd); err != nil { - return err - } - - // nolint:gosec - if err := os.WriteFile(fpath+".md5", []byte(fmt.Sprintf("%x\n", h.Sum(nil))), 0664); err != nil { - return err - } - - return nil -} diff --git a/pkg/build/docker/build.go b/pkg/build/docker/build.go deleted file mode 100644 index 8731ab7a247..00000000000 --- a/pkg/build/docker/build.go +++ /dev/null @@ -1,182 +0,0 @@ -package docker - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/grafana/grafana/pkg/build/config" -) - -// verifyArchive verifies the integrity of an archive file. -func verifyArchive(archive string) error { - log.Printf("Verifying checksum of %q", archive) - - //nolint:gosec - shaB, err := os.ReadFile(archive + ".sha256") - if err != nil { - return err - } - - exp := strings.TrimSpace(string(shaB)) - - //nolint:gosec - f, err := os.Open(archive) - if err != nil { - return err - } - - defer func() { - if err := f.Close(); err != nil { - log.Println("error closing file:", err) - } - }() - - h := sha256.New() - _, err = io.Copy(h, f) - if err != nil { - return err - } - - chksum := hex.EncodeToString(h.Sum(nil)) - if chksum != exp { - return fmt.Errorf("archive checksum is different than expected: %q", archive) - } - - log.Printf("Archive %q has expected checksum: %s", archive, exp) - - return nil -} - -// BuildImage builds a Docker image. -// The image tag is returned. -func BuildImage(version string, arch config.Architecture, grafanaDir string, useUbuntu, shouldSave bool, edition config.Edition, mode config.VersionMode) ([]string, error) { - var baseArch string - - switch arch { - case "amd64": - case "armv7": - baseArch = "arm32v7/" - case "arm64": - baseArch = "arm64v8/" - default: - return []string{}, fmt.Errorf("unrecognized architecture %q", arch) - } - - libc := "-musl" - baseImage := fmt.Sprintf("%salpine:3.18.5", baseArch) - tagSuffix := "" - if useUbuntu { - libc = "" - baseImage = fmt.Sprintf("%subuntu:22.04", baseArch) - tagSuffix = "-ubuntu" - } - - var editionStr string - var dockerRepo string - var additionalDockerRepo string - var tags []string - var imageFileBase string - var dockerEnterprise2Repo string - if repo, ok := os.LookupEnv("DOCKER_ENTERPRISE2_REPO"); ok { - dockerEnterprise2Repo = repo - } - - switch edition { - case config.EditionOSS: - dockerRepo = "grafana/grafana-image-tags" - additionalDockerRepo = "grafana/grafana-oss-image-tags" - imageFileBase = "grafana-oss" - case config.EditionEnterprise: - dockerRepo = "grafana/grafana-enterprise-image-tags" - imageFileBase = "grafana-enterprise" - editionStr = "-enterprise" - case config.EditionEnterprise2: - dockerRepo = dockerEnterprise2Repo - imageFileBase = "grafana-enterprise2" - editionStr = "-enterprise2" - default: - return []string{}, fmt.Errorf("unrecognized edition %s", edition) - } - - buildDir := filepath.Join(grafanaDir, "packaging/docker") - // For example: grafana-8.5.0-52819pre.linux-amd64-musl.tar.gz - archive := fmt.Sprintf("grafana%s-%s.linux-%s%s.tar.gz", editionStr, version, arch, libc) - if err := verifyArchive(filepath.Join(buildDir, archive)); err != nil { - return []string{}, err - } - - tag := fmt.Sprintf("%s:%s%s-%s", dockerRepo, version, tagSuffix, arch) - tags = append(tags, tag) - - args := []string{ - "build", - "-q", - "--build-arg", fmt.Sprintf("BASE_IMAGE=%s", baseImage), - "--build-arg", fmt.Sprintf("GRAFANA_TGZ=%s", archive), - "--build-arg", "GO_SRC=tgz-builder", - "--build-arg", "JS_SRC=tgz-builder", - "--build-arg", "RUN_SH=./run.sh", - "--tag", tag, - "--no-cache", - "--file", "../../Dockerfile", - ".", - "--label", fmt.Sprintf("mode=%s", string(mode)), - } - - //nolint:gosec - cmd := exec.Command("docker", args...) - cmd.Dir = buildDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled", "DOCKER_BUILDKIT=1") - log.Printf("Running Docker: DOCKER_CLI_EXPERIMENTAL=enabled DOCKER_BUILDKIT=1 %s", cmd) - if err := cmd.Run(); err != nil { - return []string{}, fmt.Errorf("building Docker image failed: %w", err) - } - if shouldSave { - imageFile := fmt.Sprintf("%s-%s%s-%s.img", imageFileBase, version, tagSuffix, arch) - //nolint:gosec - cmd = exec.Command("docker", "save", tag, "-o", imageFile) - cmd.Dir = buildDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - log.Printf("Running Docker: %s", cmd) - if err := cmd.Run(); err != nil { - return []string{}, fmt.Errorf("saving Docker image failed: %w", err) - } - gcsURL := fmt.Sprintf("gs://grafana-prerelease/artifacts/docker/%s/%s", version, imageFile) - //nolint:gosec - cmd = exec.Command("gsutil", "-q", "cp", imageFile, gcsURL) - cmd.Dir = buildDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - log.Printf("Running gsutil: %s", cmd) - if err := cmd.Run(); err != nil { - return []string{}, fmt.Errorf("storing Docker image failed: %w", err) - } - log.Printf("Docker image %s stored to grafana-prerelease GCS bucket", imageFile) - } - if additionalDockerRepo != "" { - additionalTag := fmt.Sprintf("%s:%s%s-%s", additionalDockerRepo, version, tagSuffix, arch) - - //nolint:gosec - cmd = exec.Command("docker", "tag", tag, additionalTag) - cmd.Dir = buildDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - log.Printf("Running Docker: %s", cmd) - if err := cmd.Run(); err != nil { - return []string{}, fmt.Errorf("tagging Docker image failed: %w", err) - } - tags = append(tags, additionalTag) - } - - return tags, nil -} diff --git a/pkg/build/docker/init.go b/pkg/build/docker/init.go deleted file mode 100644 index 94fe236aa52..00000000000 --- a/pkg/build/docker/init.go +++ /dev/null @@ -1,34 +0,0 @@ -package docker - -import ( - "fmt" - "log" - "os" - "os/exec" -) - -// AllArchs is a list of all supported Docker image architectures. -var AllArchs = []string{"amd64", "arm64"} - -// emulatorImage is the docker image used as the cross-platform emulator -var emulatorImage = "tonistiigi/binfmt:qemu-v7.0.0" - -// Init initializes the OS for Docker image building. -func Init() error { - // Necessary for cross-platform builds - if err := os.Setenv("DOCKER_BUILDKIT", "1"); err != nil { - log.Println("error setting DOCKER_BUILDKIT environment variable:", err) - } - - // Enable execution of Docker images for other architectures - //nolint:gosec - cmd := exec.Command("docker", "run", "--privileged", "--rm", - emulatorImage, "--install", "all") - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to enable execution of cross-platform Docker images: %w\n%s", err, output) - } - log.Println("emulators have been installed successfully!") - - return nil -} diff --git a/pkg/build/docker/push.go b/pkg/build/docker/push.go deleted file mode 100644 index da87ef2cb16..00000000000 --- a/pkg/build/docker/push.go +++ /dev/null @@ -1,62 +0,0 @@ -package docker - -import ( - "fmt" - "log" - "os" - "os/exec" - "time" -) - -const ( - tries = 3 - sleepTime = 30 -) - -func PushImage(newImage string) error { - var err error - for i := 0; i < tries; i++ { - log.Printf("push attempt #%d...", i+1) - var out []byte - cmd := exec.Command("docker", "push", newImage) - cmd.Dir = "." - out, err = cmd.CombinedOutput() - if err != nil { - log.Printf("output: %s", out) - log.Printf("sleep for %d, before retrying...", sleepTime) - time.Sleep(sleepTime * time.Second) - } else { - log.Printf("Successfully pushed %s!", newImage) - break - } - } - if err != nil { - return fmt.Errorf("error pushing images to DockerHub: %q", err) - } - return nil -} - -func PushManifest(manifest string) error { - log.Printf("Pushing Docker manifest %s...", manifest) - - var err error - for i := 0; i < tries; i++ { - log.Printf("push attempt #%d...", i+1) - var out []byte - cmd := exec.Command("docker", "manifest", "push", manifest) - cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled") - out, err = cmd.CombinedOutput() - if err != nil { - log.Printf("output: %s", out) - log.Printf("sleep for %d, before retrying...", sleepTime) - time.Sleep(sleepTime * time.Second) - } else { - log.Printf("Successful manifest push! %s", string(out)) - break - } - } - if err != nil { - return fmt.Errorf("failed to push manifest, err: %w", err) - } - return nil -} diff --git a/pkg/build/env/lookup.go b/pkg/build/env/lookup.go deleted file mode 100644 index 993b7259e14..00000000000 --- a/pkg/build/env/lookup.go +++ /dev/null @@ -1,18 +0,0 @@ -package env - -import ( - "strings" -) - -// Lookup is the equivalent of os.LookupEnv, only you are able to provide the list of environment variables. -// To use this as os.LookupEnv would be used, simply call -// `env.Lookup("ENVIRONMENT_VARIABLE", os.Environ())` -func Lookup(name string, vars []string) (string, bool) { - for _, v := range vars { - if strings.HasPrefix(v, name) { - return strings.TrimPrefix(v, name+"="), true - } - } - - return "", false -} diff --git a/pkg/build/env/lookup_test.go b/pkg/build/env/lookup_test.go deleted file mode 100644 index ccd44b11e8c..00000000000 --- a/pkg/build/env/lookup_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package env_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/build/env" -) - -func TestLookup(t *testing.T) { - values := []string{"ENV_1=a", "ENV_2=b", "ENV_3=c", "ENV_4_TEST="} - - { - v, ok := env.Lookup("ENV_1", values) - require.Equal(t, v, "a") - require.True(t, ok) - } - - { - v, ok := env.Lookup("ENV_2", values) - require.Equal(t, v, "b") - require.True(t, ok) - } - - { - v, ok := env.Lookup("ENV_3", values) - require.Equal(t, v, "c") - require.True(t, ok) - } - - { - v, ok := env.Lookup("ENV_4_TEST", values) - require.Equal(t, v, "") - require.True(t, ok) - } - - { - v, ok := env.Lookup("NOT_THERE", values) - require.Equal(t, v, "") - require.False(t, ok) - } -} diff --git a/pkg/build/errutil/group.go b/pkg/build/errutil/group.go deleted file mode 100644 index 0844616e579..00000000000 --- a/pkg/build/errutil/group.go +++ /dev/null @@ -1,61 +0,0 @@ -package errutil - -import ( - "context" - "log" - "sync" -) - -type Group struct { - cancel func() - wg sync.WaitGroup - errOnce sync.Once - err error -} - -func GroupWithContext(ctx context.Context) (*Group, context.Context) { - ctx, cancel := context.WithCancel(ctx) - return &Group{cancel: cancel}, ctx -} - -// Wait waits for any wrapped goroutines to finish and returns any error having occurred in one of them. -func (g *Group) Wait() error { - log.Println("Waiting on Group") - g.wg.Wait() - if g.cancel != nil { - log.Println("Group canceling its context after waiting") - g.cancel() - } - return g.err -} - -// Cancel cancels the associated context. -func (g *Group) Cancel() { - log.Println("Group's Cancel method being called") - g.cancel() -} - -// Wrap wraps a function to be executed in a goroutine. -func (g *Group) Wrap(f func() error) func() { - g.wg.Add(1) - return func() { - defer g.wg.Done() - - if err := f(); err != nil { - g.errOnce.Do(func() { - log.Printf("An error occurred in Group: %s", err) - g.err = err - if g.cancel != nil { - log.Println("Group canceling its context due to error") - g.cancel() - } - }) - } - } -} - -// Go wraps the provided function and executes it in a goroutine. -func (g *Group) Go(f func() error) { - wrapped := g.Wrap(f) - go wrapped() -} diff --git a/pkg/build/executil/exec.go b/pkg/build/executil/exec.go deleted file mode 100644 index e46f568a997..00000000000 --- a/pkg/build/executil/exec.go +++ /dev/null @@ -1,46 +0,0 @@ -package executil - -import ( - "context" - "fmt" - "os/exec" - "strings" -) - -func RunAt(ctx context.Context, dir, cmd string, args ...string) error { - // Ignore gosec G204 as this function is only used in the build process. - //nolint:gosec - c := exec.CommandContext(ctx, cmd, args...) - c.Dir = dir - - b, err := c.CombinedOutput() - - if err != nil { - return fmt.Errorf("%w. '%s %v': %s", err, cmd, args, string(b)) - } - - return nil -} - -func Run(ctx context.Context, cmd string, args ...string) error { - return RunAt(ctx, ".", cmd, args...) -} - -func OutputAt(ctx context.Context, dir, cmd string, args ...string) (string, error) { - // Ignore gosec G204 as this function is only used in the build process. - //nolint:gosec - c := exec.CommandContext(ctx, cmd, args...) - c.Dir = dir - - b, err := c.CombinedOutput() - - if err != nil { - return "", err - } - - return strings.TrimSpace(string(b)), nil -} - -func Output(ctx context.Context, cmd string, args ...string) (string, error) { - return OutputAt(ctx, ".", cmd, args...) -} diff --git a/pkg/build/frontend/build.go b/pkg/build/frontend/build.go deleted file mode 100644 index c4259c25e57..00000000000 --- a/pkg/build/frontend/build.go +++ /dev/null @@ -1,56 +0,0 @@ -package frontend - -import ( - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/lerna" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func BuildFrontendPackages(version string, edition config.Edition, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group) error { - p.Schedule(g.Wrap(func() error { - if err := lerna.BuildFrontendPackages(version, edition, grafanaDir); err != nil { - return fmt.Errorf("failed to build %s frontend packages: %v", edition, err) - } - - log.Printf("Finished building %s frontend packages", string(edition)) - return nil - })) - - return nil -} - -// Build builds the Grafana front-end -func Build(edition config.Edition, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group) error { - log.Printf("Building %s frontend in %q", edition, grafanaDir) - grafanaDir, err := filepath.Abs(grafanaDir) - if err != nil { - return err - } - - for _, dpath := range []string{"tmp", "public_gen", "public/build"} { - dpath = filepath.Join(grafanaDir, dpath) - if err := os.RemoveAll(dpath); err != nil { - return fmt.Errorf("failed to remove %q: %w", dpath, err) - } - } - - p.Schedule(g.Wrap(func() error { - cmd := exec.Command("yarn", "run", "build") - cmd.Dir = grafanaDir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to build %s frontend with webpack: %s", edition, output) - } - - log.Printf("Finished building %s frontend", edition) - return nil - })) - - return nil -} diff --git a/pkg/build/frontend/config.go b/pkg/build/frontend/config.go deleted file mode 100644 index 318168de1d4..00000000000 --- a/pkg/build/frontend/config.go +++ /dev/null @@ -1,42 +0,0 @@ -package frontend - -import ( - "fmt" - - "github.com/blang/semver/v4" - "github.com/grafana/grafana/pkg/build/config" - "github.com/urfave/cli/v2" -) - -const GrafanaDir = "." - -func GetConfig(c *cli.Context, metadata config.Metadata) (config.Config, config.Edition, error) { - cfg := config.Config{ - NumWorkers: c.Int("jobs"), - GitHubToken: c.String("github-token"), - } - - mode := config.Edition(c.String("edition")) - - if metadata.ReleaseMode.Mode == config.TagMode && !metadata.ReleaseMode.IsTest { - packageJSONVersion, err := config.GetPackageJSONVersion(GrafanaDir) - if err != nil { - return config.Config{}, "", err - } - semverGrafanaVersion, err := semver.Parse(metadata.GrafanaVersion) - if err != nil { - return config.Config{}, "", err - } - semverPackageJSONVersion, err := semver.Parse(packageJSONVersion) - if err != nil { - return config.Config{}, "", err - } - // Check if the semver digits of the tag are not equal - if semverGrafanaVersion.FinalizeVersion() != semverPackageJSONVersion.FinalizeVersion() { - return config.Config{}, "", cli.Exit(fmt.Errorf("package.json version and input tag version differ %s != %s.\nPlease update package.json", packageJSONVersion, metadata.GrafanaVersion), 1) - } - } - - cfg.PackageVersion = metadata.GrafanaVersion - return cfg, mode, nil -} diff --git a/pkg/build/frontend/config_test.go b/pkg/build/frontend/config_test.go deleted file mode 100644 index 770df3be7a6..00000000000 --- a/pkg/build/frontend/config_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package frontend - -import ( - "encoding/json" - "flag" - "os" - "testing" - - "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" -) - -const ( - jobs = "jobs" - githubToken = "github-token" - buildID = "build-id" -) - -type packageJson struct { - Version string `json:"version"` -} - -type flagObj struct { - name string - value string -} - -var app = cli.NewApp() - -func TestGetConfig(t *testing.T) { - tests := []struct { - ctx *cli.Context - name string - packageJsonVersion string - metadata config.Metadata - wantErr bool - }{ - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), - name: "package.json matches tag", - packageJsonVersion: "10.0.0", - metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, - wantErr: false, - }, - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), - name: "custom tag, package.json doesn't match", - packageJsonVersion: "10.0.0", - metadata: config.Metadata{GrafanaVersion: "10.0.0-abcd123pre", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, - wantErr: false, - }, - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), - name: "package.json doesn't match tag", - packageJsonVersion: "10.1.0", - metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, - wantErr: true, - }, - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), - name: "test tag event, check should be skipped", - packageJsonVersion: "10.1.0", - metadata: config.Metadata{GrafanaVersion: "10.1.0-test", ReleaseMode: config.ReleaseMode{Mode: config.TagMode, IsTest: true}}, - wantErr: false, - }, - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}, flagObj{name: buildID, value: "12345"}), nil), - name: "non-tag event", - packageJsonVersion: "10.1.0-pre", - metadata: config.Metadata{GrafanaVersion: "10.1.0-12345pre", ReleaseMode: config.ReleaseMode{Mode: config.PullRequestMode}}, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := createTempPackageJson(t, tt.packageJsonVersion) - require.NoError(t, err) - - got, _, err := GetConfig(tt.ctx, tt.metadata) - if !tt.wantErr { - require.Equal(t, got.PackageVersion, tt.metadata.GrafanaVersion) - } - - if tt.wantErr { - require.Equal(t, got.PackageVersion, "") - require.Error(t, err) - } - }) - } -} - -func setFlags(t *testing.T, flagSet *flag.FlagSet, flags ...flagObj) *flag.FlagSet { - t.Helper() - for _, f := range flags { - if f.name != "" { - flagSet.StringVar(&f.name, f.name, f.value, "") - } - } - return flagSet -} - -func createTempPackageJson(t *testing.T, version string) error { - t.Helper() - - data := packageJson{Version: version} - file, _ := json.MarshalIndent(data, "", " ") - - err := os.WriteFile("package.json", file, 0644) - require.NoError(t, err) - - t.Cleanup(func() { - err := os.RemoveAll("package.json") - require.NoError(t, err) - }) - return nil -} diff --git a/pkg/build/fsutil/copy_recursive.go b/pkg/build/fsutil/copy_recursive.go deleted file mode 100644 index daf60a5f5e9..00000000000 --- a/pkg/build/fsutil/copy_recursive.go +++ /dev/null @@ -1,65 +0,0 @@ -package fsutil - -import ( - "fmt" - "os" - "path/filepath" -) - -// CopyRecursive copies files and directories recursively. -func CopyRecursive(src, dst string) error { - sfi, err := os.Stat(src) - if err != nil { - return err - } - if !sfi.IsDir() { - return CopyFile(src, dst) - } - - if _, err := os.Stat(dst); os.IsNotExist(err) { - if err := os.MkdirAll(dst, sfi.Mode()); err != nil { - return fmt.Errorf("failed to create directory %q: %s", dst, err) - } - } - - entries, err := os.ReadDir(src) - if err != nil { - return err - } - for _, entry := range entries { - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - srcFi, err := os.Stat(srcPath) - if err != nil { - return err - } - - switch srcFi.Mode() & os.ModeType { - case os.ModeDir: - if err := CopyRecursive(srcPath, dstPath); err != nil { - return err - } - case os.ModeSymlink: - link, err := os.Readlink(srcPath) - if err != nil { - return err - } - if err := os.Symlink(link, dstPath); err != nil { - return err - } - default: - if err := CopyFile(srcPath, dstPath); err != nil { - return err - } - } - - if srcFi.Mode()&os.ModeSymlink != 0 { - if err := os.Chmod(dstPath, srcFi.Mode()); err != nil { - return err - } - } - } - - return nil -} diff --git a/pkg/build/fsutil/createtemp.go b/pkg/build/fsutil/createtemp.go deleted file mode 100644 index 21720a9a3d8..00000000000 --- a/pkg/build/fsutil/createtemp.go +++ /dev/null @@ -1,43 +0,0 @@ -package fsutil - -import ( - "fmt" - "os" -) - -// CreateTempFile generates a temp filepath, based on the provided suffix. -// A typical generated path looks like /var/folders/abcd/abcdefg/A/1137975807. -func CreateTempFile(sfx string) (string, error) { - var suffix string - if sfx != "" { - suffix = fmt.Sprintf("*-%s", sfx) - } else { - suffix = sfx - } - f, err := os.CreateTemp("", suffix) - if err != nil { - return "", err - } - if err := f.Close(); err != nil { - return "", err - } - - return f.Name(), nil -} - -// CreateTempDir generates a temp directory, based on the provided suffix. -// A typical generated path looks like /var/folders/abcd/abcdefg/A/1137975807/. -func CreateTempDir(sfx string) (string, error) { - var suffix string - if sfx != "" { - suffix = fmt.Sprintf("*-%s", sfx) - } else { - suffix = sfx - } - dir, err := os.MkdirTemp("", suffix) - if err != nil { - return "", err - } - - return dir, nil -} diff --git a/pkg/build/fsutil/createtemp_test.go b/pkg/build/fsutil/createtemp_test.go deleted file mode 100644 index 640585f4ea5..00000000000 --- a/pkg/build/fsutil/createtemp_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package fsutil - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCreateTempFile(t *testing.T) { - t.Run("empty suffix, expects pattern like: /var/folders/abcd/abcdefg/A/1137975807", func(t *testing.T) { - filePath, err := CreateTempFile("") - require.NoError(t, err) - - pathParts := strings.Split(filePath, "/") - require.Greater(t, len(pathParts), 1) - require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 1) - }) - - t.Run("non-empty suffix, expects /var/folders/abcd/abcdefg/A/1137975807-foobar", func(t *testing.T) { - filePath, err := CreateTempFile("foobar") - require.NoError(t, err) - - pathParts := strings.Split(filePath, "/") - require.Greater(t, len(pathParts), 1) - require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 2) - }) -} - -func TestCreateTempDir(t *testing.T) { - t.Run("empty suffix, expects pattern like: /var/folders/abcd/abcdefg/A/1137975807/", func(t *testing.T) { - filePath, err := CreateTempFile("") - require.NoError(t, err) - - pathParts := strings.Split(filePath, "/") - require.Greater(t, len(pathParts), 1) - require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 1) - }) - - t.Run("non-empty suffix, expects /var/folders/abcd/abcdefg/A/1137975807-foobar/", func(t *testing.T) { - filePath, err := CreateTempFile("foobar") - require.NoError(t, err) - - pathParts := strings.Split(filePath, "/") - require.Greater(t, len(pathParts), 1) - require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 2) - }) -} diff --git a/pkg/build/gcom/url.go b/pkg/build/gcom/url.go new file mode 100644 index 00000000000..e85c1039796 --- /dev/null +++ b/pkg/build/gcom/url.go @@ -0,0 +1,72 @@ +package gcom + +import ( + "fmt" + "net/url" + "path" + "strings" + + "github.com/grafana/grafana/pkg/build/versions" +) + +func PackageName(grafana, distro, arch, version, ext string, musl bool, raspberryPi bool) string { + v := versions.ParseSemver(version) + + if raspberryPi { + grafana += "-rpi" + } + + versionString := strings.Join([]string{v.Major, v.Minor, v.Patch}, ".") + fmt.Println("Version string:", versionString) + if distro == "deb" { + if v.BuildMetadata != "" { + versionString += "+" + strings.ReplaceAll(v.BuildMetadata, "-", "~") + } + + if v.Prerelease != "" { + versionString += "~" + v.Prerelease + } + + return strings.Join([]string{grafana, versionString, arch}, "_") + "." + ext + } + + if distro == "rhel" { + if v.BuildMetadata != "" { + versionString += "+" + strings.ReplaceAll(v.BuildMetadata, "-", "~") + } + + if v.Prerelease != "" { + versionString += "~" + v.Prerelease + } + + versionString += "-1" + + // Notable difference between our deb naming and our RPM naming: the file ends with `.arch.ext`, not + // `_arch.ext`. + return strings.Join([]string{grafana, versionString}, "-") + "." + arch + "." + ext + } + + if v.Prerelease != "" { + versionString += "-" + v.Prerelease + } + + if v.BuildMetadata != "" { + versionString += "+" + v.BuildMetadata + } + + if musl { + arch += "-musl" + } + + // grafana-enterprise-1.2.3+example-01.linux-amd64.tar.gz + return fmt.Sprintf("%s-%s.%s-%s.%s", grafana, versionString, distro, arch, ext) +} + +func GetURL(baseURL *url.URL, version, grafana, distro, arch, ext string, musl, raspberryPi bool) *url.URL { + packageName := PackageName(grafana, distro, arch, version, ext, musl, raspberryPi) + return &url.URL{ + Host: baseURL.Host, + Scheme: baseURL.Scheme, + Path: path.Join(baseURL.Path, packageName), + } +} diff --git a/pkg/build/gcom/url_test.go b/pkg/build/gcom/url_test.go new file mode 100644 index 00000000000..680a4326fd1 --- /dev/null +++ b/pkg/build/gcom/url_test.go @@ -0,0 +1,367 @@ +package gcom_test + +import ( + "fmt" + "testing" + + "github.com/grafana/grafana/pkg/build/gcom" + "github.com/stretchr/testify/require" +) + +func TestPackageName(t *testing.T) { + type args struct { + Distro string + Arch string + Version string + Ext string + Musl bool + RaspberryPi bool + + Expect string + } + + cases := []args{ + { + RaspberryPi: true, + Distro: "deb", + Arch: "armhf", + Version: "1.2.3", + Ext: "deb", + Expect: "grafana-rpi_1.2.3_armhf.deb", + }, + { + Distro: "deb", + Arch: "arm64", + Version: "1.2.3", + Ext: "deb", + Expect: "grafana_1.2.3_arm64.deb", + }, + { + Distro: "rhel", + Arch: "aarch64", + Version: "1.2.3", + Ext: "rpm", + Expect: "grafana-1.2.3-1.aarch64.rpm", + }, + { + Distro: "rhel", + Arch: "aarch64", + Ext: "rpm.sha256", + Version: "1.2.3", + Expect: "grafana-1.2.3-1.aarch64.rpm.sha256", + }, + { + Distro: "rhel", + Ext: "rpm", + Version: "1.2.3", + Arch: "x86_64", + Expect: "grafana-1.2.3-1.x86_64.rpm", + }, + { + Distro: "rhel", + Ext: "rpm.sha256", + Version: "1.2.3", + Arch: "x86_64", + Expect: "grafana-1.2.3-1.x86_64.rpm.sha256", + }, + { + Distro: "darwin", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.darwin-amd64.tar.gz", + }, + { + Distro: "darwin", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.darwin-amd64.tar.gz.sha256", + }, + { + Distro: "darwin", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.darwin-arm64-musl.tar.gz", + Musl: true, + }, + { + Distro: "darwin", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.darwin-arm64-musl.tar.gz.sha256", + Musl: true, + }, + { + Distro: "darwin", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.darwin-arm64.tar.gz", + }, + { + Distro: "darwin", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.darwin-arm64.tar.gz.sha256", + }, + { + Distro: "linux", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.linux-amd64-musl.tar.gz", + Musl: true, + }, + { + Distro: "linux", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.linux-amd64-musl.tar.gz.sha256", + Musl: true, + }, + { + Distro: "linux", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.linux-amd64.tar.gz", + }, + { + Distro: "linux", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.linux-amd64.tar.gz.sha256", + }, + { + Distro: "linux", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.linux-arm64-musl.tar.gz", + Musl: true, + }, + { + Distro: "linux", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.linux-arm64-musl.tar.gz.sha256", + Musl: true, + }, + { + Distro: "linux", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.linux-arm64.tar.gz", + }, + { + Ext: "tar.gz.sha256", + Version: "1.2.3", + Distro: "linux", + Arch: "arm64", + Expect: "grafana-1.2.3.linux-arm64.tar.gz.sha256", + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "linux", + Arch: "armv6", + Expect: "grafana-1.2.3.linux-armv6.tar.gz", + }, + { + Ext: "tar.gz.sha256", + Version: "1.2.3", + Distro: "linux", + Arch: "armv6", + Expect: "grafana-1.2.3.linux-armv6.tar.gz.sha256", + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "linux", + Arch: "armv7", + Expect: "grafana-1.2.3.linux-armv7-musl.tar.gz", + Musl: true, + }, + { + Ext: "tar.gz.sha256", + Version: "1.2.3", + Distro: "linux", + Arch: "armv7", + Expect: "grafana-1.2.3.linux-armv7-musl.tar.gz.sha256", + Musl: true, + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "linux", + Arch: "armv7", + Expect: "grafana-1.2.3.linux-armv7.tar.gz", + }, + { + Ext: "tar.gz.sha256", + Version: "1.2.3", + Distro: "linux", + Arch: "armv7", + Expect: "grafana-1.2.3.linux-armv7.tar.gz.sha256", + }, + { + Version: "1.2.3", + Arch: "amd64", + Ext: "exe", + Distro: "windows", + Expect: "grafana-1.2.3.windows-amd64.exe", + }, + { + Version: "1.2.3", + Arch: "amd64", + Distro: "windows", + Ext: "exe.sha256", + Expect: "grafana-1.2.3.windows-amd64.exe.sha256", + }, + { + Version: "1.2.3", + Arch: "amd64", + Distro: "windows", + Ext: "msi", + Expect: "grafana-1.2.3.windows-amd64.msi", + }, + { + Version: "1.2.3", + Arch: "amd64", + Distro: "windows", + Ext: "msi.sha256", + Expect: "grafana-1.2.3.windows-amd64.msi.sha256", + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "windows", + Expect: "grafana-1.2.3.windows-amd64.tar.gz", + Arch: "amd64", + }, + { + Version: "1.2.3", + Distro: "windows", + Arch: "amd64", + Ext: "tar.gz.sha256", + Expect: "grafana-1.2.3.windows-amd64.tar.gz.sha256", + }, + { + Version: "1.2.3", + Distro: "windows", + Expect: "grafana-1.2.3.windows-amd64.zip", + Ext: "zip", + Arch: "amd64", + }, + { + Version: "1.2.3", + Distro: "windows", + Expect: "grafana-1.2.3.windows-amd64.zip.sha256", + Ext: "zip.sha256", + Arch: "amd64", + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "windows", + Arch: "arm64", + Expect: "grafana-1.2.3.windows-arm64-musl.tar.gz", + Musl: true, + }, + { + Version: "1.2.3", + Ext: "tar.gz.sha256", + Distro: "windows", + Arch: "arm64", + Expect: "grafana-1.2.3.windows-arm64-musl.tar.gz.sha256", + Musl: true, + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "windows", + Arch: "arm64", + Expect: "grafana-1.2.3.windows-arm64.tar.gz", + }, + { + Version: "1.2.3", + Ext: "tar.gz.sha256", + Distro: "windows", + Arch: "arm64", + Expect: "grafana-1.2.3.windows-arm64.tar.gz.sha256", + }, + { + RaspberryPi: true, + Version: "1.2.3", + Ext: "deb", + Arch: "armhf", + Distro: "deb", + Expect: "grafana-rpi_1.2.3_armhf.deb", + }, + { + RaspberryPi: true, + Version: "1.2.3", + Ext: "deb.sha256", + Distro: "deb", + Arch: "armhf", + Expect: "grafana-rpi_1.2.3_armhf.deb.sha256", + }, + { + Version: "1.2.3", + Ext: "deb", + Distro: "deb", + Expect: "grafana_1.2.3_amd64.deb", + Arch: "amd64", + }, + { + Version: "1.2.3", + Ext: "deb.sha256", + Distro: "deb", + Expect: "grafana_1.2.3_amd64.deb.sha256", + Arch: "amd64", + }, + { + Version: "1.2.3", + Ext: "deb", + Arch: "arm64", + Distro: "deb", + Expect: "grafana_1.2.3_arm64.deb", + }, + { + Version: "1.2.3", + Ext: "deb.sha256", + Arch: "arm64", + Distro: "deb", + Expect: "grafana_1.2.3_arm64.deb.sha256", + }, + { + Version: "1.2.3", + Ext: "deb", + Distro: "deb", + Arch: "armhf", + Expect: "grafana_1.2.3_armhf.deb", + }, + { + Version: "1.2.3", + Ext: "deb.sha256", + Arch: "armhf", + Distro: "deb", + Expect: "grafana_1.2.3_armhf.deb.sha256", + }, + } + + for i, v := range cases { + t.Run(fmt.Sprintf("[%d / %d] %s", i+1, len(cases), v.Expect), func(t *testing.T) { + n := gcom.PackageName("grafana", v.Distro, v.Arch, v.Version, v.Ext, v.Musl, v.RaspberryPi) + require.Equal(t, v.Expect, n) + }) + } +} diff --git a/pkg/build/git/git.go b/pkg/build/git/git.go index 9fbcc346e92..2977cf13775 100644 --- a/pkg/build/git/git.go +++ b/pkg/build/git/git.go @@ -4,13 +4,9 @@ import ( "context" "errors" "fmt" - "net/http" "regexp" "github.com/google/go-github/v45/github" - "golang.org/x/oauth2" - - "github.com/grafana/grafana/pkg/build/stringutil" ) const ( @@ -47,19 +43,6 @@ type StatusesService interface { CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) } -// NewGitHubClient creates a new Client using the provided GitHub token if not empty. -func NewGitHubClient(ctx context.Context, token string) *github.Client { - var tc *http.Client - if token != "" { - ts := oauth2.StaticTokenSource(&oauth2.Token{ - AccessToken: token, - }) - tc = oauth2.NewClient(ctx, ts) - } - - return github.NewClient(tc) -} - func PRCheckRegexp() *regexp.Regexp { reBranch, err := regexp.Compile(`^prc-([0-9]+)-([A-Za-z0-9]+)\/(.+)$`) if err != nil { @@ -68,76 +51,3 @@ func PRCheckRegexp() *regexp.Regexp { return reBranch } - -func AddLabelToPR(ctx context.Context, client LabelsService, prID int, newLabel string) error { - // Check existing labels - labels, _, err := client.ListLabelsByIssue(ctx, RepoOwner, OSSRepo, prID, nil) - if err != nil { - return err - } - - duplicate := false - for _, label := range labels { - if *label.Name == newLabel { - duplicate = true - continue - } - - // Delete existing "enterprise-xx" labels - if stringutil.Contains(EnterpriseCheckLabels, *label.Name) { - _, err := client.RemoveLabelForIssue(ctx, RepoOwner, OSSRepo, prID, *label.Name) - if err != nil { - return err - } - } - } - - if duplicate { - return nil - } - - _, _, err = client.AddLabelsToIssue(ctx, RepoOwner, OSSRepo, prID, []string{newLabel}) - if err != nil { - return err - } - - return nil -} - -func DeleteEnterpriseBranch(ctx context.Context, client GitService, branchName string) error { - ref := "heads/" + branchName - if _, err := client.DeleteRef(ctx, RepoOwner, EnterpriseRepo, ref); err != nil { - return err - } - - return nil -} - -// CreateEnterpriseStatus sets the status on a commit for the enterprise build check. -func CreateEnterpriseStatus(ctx context.Context, client StatusesService, sha, link, status string) (*github.RepoStatus, error) { - check, _, err := client.CreateStatus(ctx, RepoOwner, OSSRepo, sha, &github.RepoStatus{ - Context: github.String(EnterpriseCheckName), - Description: github.String(EnterpriseCheckDescription), - TargetURL: github.String(link), - State: github.String(status), - }) - - if err != nil { - return nil, err - } - - return check, nil -} - -func CreateEnterpriseBuildFailedComment(ctx context.Context, client CommentService, link string, prID int) error { - body := fmt.Sprintf("Drone build failed: %s", link) - - _, _, err := client.CreateComment(ctx, RepoOwner, OSSRepo, prID, &github.IssueComment{ - Body: &body, - }) - if err != nil { - return err - } - - return nil -} diff --git a/pkg/build/git/git_checks_test.go b/pkg/build/git/git_checks_test.go deleted file mode 100644 index c9bf0c98d0b..00000000000 --- a/pkg/build/git/git_checks_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package git_test - -import ( - "context" - "errors" - "testing" - - "github.com/google/go-github/v45/github" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/build/git" -) - -type TestChecksService struct { - CreateCheckRunError error -} - -func (s *TestChecksService) CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) { - if s.CreateCheckRunError != nil { - return nil, nil, s.CreateCheckRunError - } - - return &github.RepoStatus{ - ID: github.Int64(1), - URL: status.URL, - }, nil, nil -} - -func TestCreateEnterpriseRepoStatus(t *testing.T) { - t.Run("It should create a repo status", func(t *testing.T) { - var ( - ctx = context.Background() - client = &TestChecksService{} - link = "http://example.com" - sha = "1234" - ) - - _, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success") - - require.NoError(t, err) - }) - t.Run("It should return an error if GitHub fails to create the status", func(t *testing.T) { - var ( - ctx = context.Background() - createCheckError = errors.New("create check run error") - client = &TestChecksService{ - CreateCheckRunError: createCheckError, - } - link = "http://example.com" - sha = "1234" - ) - - _, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success") - require.ErrorIs(t, err, createCheckError) - }) -} diff --git a/pkg/build/git/git_issues_test.go b/pkg/build/git/git_issues_test.go deleted file mode 100644 index dab9fddde77..00000000000 --- a/pkg/build/git/git_issues_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package git_test - -import ( - "context" - "errors" - "testing" - - "github.com/google/go-github/v45/github" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/build/git" -) - -type TestLabelsService struct { - Labels []*github.Label - ListLabelsError error - RemoveLabelError error - AddLabelsError error -} - -func (s *TestLabelsService) ListLabelsByIssue(ctx context.Context, owner string, repo string, number int, opts *github.ListOptions) ([]*github.Label, *github.Response, error) { - if s.ListLabelsError != nil { - return nil, nil, s.ListLabelsError - } - - labels := s.Labels - if labels == nil { - labels = []*github.Label{} - } - - return labels, nil, nil -} - -func (s *TestLabelsService) RemoveLabelForIssue(ctx context.Context, owner string, repo string, number int, label string) (*github.Response, error) { - if s.RemoveLabelError != nil { - return nil, s.RemoveLabelError - } - - return &github.Response{}, nil -} - -func (s *TestLabelsService) AddLabelsToIssue(ctx context.Context, owner string, repo string, number int, labels []string) ([]*github.Label, *github.Response, error) { - if s.AddLabelsError != nil { - return nil, nil, s.AddLabelsError - } - - l := make([]*github.Label, len(labels)) - for i, v := range labels { - l[i] = &github.Label{ - Name: github.String(v), - } - } - - return l, nil, nil -} - -func TestAddLabelToPR(t *testing.T) { - t.Run("It should add a label to a pull request", func(t *testing.T) { - var ( - ctx = context.Background() - client = &TestLabelsService{} - pr = 20 - label = "test-label" - ) - - require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) - }) - t.Run("It should not return an error if the label already exists", func(t *testing.T) { - var ( - ctx = context.Background() - client = &TestLabelsService{ - Labels: []*github.Label{ - { - Name: github.String("test-label"), - }, - }, - } - pr = 20 - label = "test-label" - ) - - require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) - }) - - t.Run("It should return an error if GitHub returns an error when listing labels", func(t *testing.T) { - var ( - ctx = context.Background() - listLabelsError = errors.New("list labels error") - client = &TestLabelsService{ - ListLabelsError: listLabelsError, - Labels: []*github.Label{}, - } - pr = 20 - label = "test-label" - ) - - require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), listLabelsError) - }) - - t.Run("It should not return an error if there are existing enterprise-check labels.", func(t *testing.T) { - var ( - ctx = context.Background() - client = &TestLabelsService{ - Labels: []*github.Label{ - { - Name: github.String("enterprise-failed"), - }, - }, - } - pr = 20 - label = "test-label" - ) - - require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) - }) - - t.Run("It should return an error if GitHub returns an error when removing existing enterprise-check labels", func(t *testing.T) { - var ( - ctx = context.Background() - removeLabelError = errors.New("remove label error") - client = &TestLabelsService{ - RemoveLabelError: removeLabelError, - Labels: []*github.Label{ - { - Name: github.String("enterprise-failed"), - }, - }, - } - pr = 20 - label = "test-label" - ) - - require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), removeLabelError) - }) -} diff --git a/pkg/build/git/git_test.go b/pkg/build/git/git_test.go deleted file mode 100644 index 3b47ce08dd0..00000000000 --- a/pkg/build/git/git_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package git_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/grafana/grafana/pkg/build/git" -) - -func TestPRCheckRegexp(t *testing.T) { - type match struct { - String string - Commit string - Branch string - PR string - } - - var ( - shouldMatch = []match{ - { - String: "prc-1-a1b2c3d4/branch-name", - Branch: "branch-name", - Commit: "a1b2c3d4", - PR: "1", - }, - { - String: "prc-111-a1b2c3d4/branch/name", - Branch: "branch/name", - Commit: "a1b2c3d4", - PR: "111", - }, - { - String: "prc-102930122-a1b2c3d4/branch-name", - Branch: "branch-name", - Commit: "a1b2c3d4", - PR: "102930122", - }, - } - - shouldNotMatch = []string{"prc-a/branch", "km/test", "test", "prc", "prc/test", "price"} - ) - - regex := git.PRCheckRegexp() - - for _, v := range shouldMatch { - assert.Truef(t, regex.MatchString(v.String), "regex '%s' should match %s", regex.String(), v) - m := regex.FindStringSubmatch(v.String) - assert.Equal(t, m[1], v.PR) - assert.Equal(t, m[2], v.Commit) - assert.Equal(t, m[3], v.Branch) - } - - for _, v := range shouldNotMatch { - assert.False(t, regex.MatchString(v), "regex '%s' should not match %s", regex.String(), v) - } -} diff --git a/pkg/build/go.sum b/pkg/build/go.sum index e348659ac5e..01378fb3e17 100644 --- a/pkg/build/go.sum +++ b/pkg/build/go.sum @@ -39,8 +39,6 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8= diff --git a/pkg/build/golangutils/build.go b/pkg/build/golangutils/build.go deleted file mode 100644 index 14da56a7765..00000000000 --- a/pkg/build/golangutils/build.go +++ /dev/null @@ -1,124 +0,0 @@ -package golangutils - -import ( - "context" - "fmt" - "io" - "os/exec" - "strings" - - "github.com/grafana/grafana/pkg/build/config" -) - -type BuildOpts struct { - // Package refers to the path to the `main` package containing `func main` - Package string - - // Output is used as the -o argument in the go build command - Output string - - // Workdir should define some place in the module where the package path resolves. - // Go commands need to be ran inside a the Go module directory. - Workdir string - - GoOS config.OS - GoArch config.Architecture - GoArm string - Go386 string - CC string - LibC string - - CGoEnabled bool - CGoCFlags string - - // LdFlags are joined by a space character and provided to the -ldflags argument. - // A valid element here would be `-X 'main.version=1.0.0'`. - LdFlags []string - - Stdout io.ReadWriter - Stderr io.ReadWriter - Stdin io.ReadWriter - - // ExtraEnv allows consumers to provide extra env args that are not defined above. - // A single element should be formatted using like so: {NAME}={VALUE}. Example: GOOS=linux. - ExtraEnv []string - - // ExtraArgs allows consumers to provide extra arguments that are not defined above. - // Flag names and values should be two separate elements. - // These flags will be appended to the command arguments before the package path in "go build". - ExtraArgs []string -} - -// Env constructs a list of key/value pairs for setting a build command's environment. -// Should we consider using something to unmarshal the struct to env? -func (opts BuildOpts) Env() []string { - env := []string{} - if opts.CGoEnabled { - env = append(env, "CGO_ENABLED=1") - } - - if opts.GoOS != "" { - env = append(env, fmt.Sprintf("GOOS=%s", opts.GoOS)) - } - - if opts.GoArch != "" { - env = append(env, fmt.Sprintf("GOARCH=%s", opts.GoArch)) - } - - if opts.CC != "" { - env = append(env, fmt.Sprintf("CC=%s", opts.CC)) - } - - if opts.CGoCFlags != "" { - env = append(env, fmt.Sprintf("CGO_CFLAGS=%s", opts.CGoCFlags)) - } - - if opts.GoArm != "" { - env = append(env, fmt.Sprintf("GOARM=%s", opts.GoArm)) - } - - if opts.ExtraEnv != nil { - return append(opts.ExtraEnv, env...) - } - - return env -} - -// Args constructs a list of flags and values for use with the exec.Command type when running "go build". -func (opts BuildOpts) Args() []string { - args := []string{} - - if opts.LdFlags != nil { - args = append(args, "-ldflags", strings.Join(opts.LdFlags, " ")) - } - - if opts.Output != "" { - args = append(args, "-o", opts.Output) - } - - if opts.ExtraArgs != nil { - args = append(args, opts.ExtraArgs...) - } - - args = append(args, opts.Package) - - return args -} - -// Build runs the go build process in the current shell given the opts. -// This function will panic if no Stdout/Stderr/Stdin is provided in the opts. -func RunBuild(ctx context.Context, opts BuildOpts) error { - env := opts.Env() - args := append([]string{"build"}, opts.Args()...) - // Ignore gosec G304 as this function is only used in the build process. - //nolint:gosec - cmd := exec.CommandContext(ctx, "go", args...) - cmd.Env = env - - cmd.Stdout = opts.Stdout - cmd.Stderr = opts.Stderr - cmd.Stdin = opts.Stdin - cmd.Dir = opts.Workdir - - return cmd.Run() -} diff --git a/pkg/build/golangutils/doc.go b/pkg/build/golangutils/doc.go deleted file mode 100644 index 2cb33af05e1..00000000000 --- a/pkg/build/golangutils/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package golangutils holds utility functions, wrappers, and types for building Go binaries for Grafana. -package golangutils diff --git a/pkg/build/gpg/gpg.go b/pkg/build/gpg/gpg.go deleted file mode 100644 index 59f96722014..00000000000 --- a/pkg/build/gpg/gpg.go +++ /dev/null @@ -1,73 +0,0 @@ -package gpg - -import ( - "encoding/base64" - "fmt" - "log" - "os" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/fsutil" -) - -// LoadGPGKeys loads GPG key pair and password from the environment and writes them to corresponding files. -// -// The passed config's GPG fields also get updated. Make sure to call RemoveGPGFiles at application exit. -func LoadGPGKeys(cfg *config.Config) error { - var err error - cfg.GPGPrivateKey, err = fsutil.CreateTempFile("priv.key") - if err != nil { - return err - } - cfg.GPGPublicKey, err = fsutil.CreateTempFile("pub.key") - if err != nil { - return err - } - cfg.GPGPassPath, err = fsutil.CreateTempFile("") - if err != nil { - return err - } - - gpgPrivKey := os.Getenv("GPG_PRIV_KEY") - if gpgPrivKey == "" { - return fmt.Errorf("$GPG_PRIV_KEY must be defined") - } - gpgPubKey := os.Getenv("GPG_PUB_KEY") - if gpgPubKey == "" { - return fmt.Errorf("$GPG_PUB_KEY must be defined") - } - gpgPass := os.Getenv("GPG_KEY_PASSWORD") - if gpgPass == "" { - return fmt.Errorf("$GPG_KEY_PASSWORD must be defined") - } - - gpgPrivKeyB, err := base64.StdEncoding.DecodeString(gpgPrivKey) - if err != nil { - return fmt.Errorf("couldn't decode $GPG_PRIV_KEY: %w", err) - } - gpgPubKeyB, err := base64.StdEncoding.DecodeString(gpgPubKey) - if err != nil { - return fmt.Errorf("couldn't decode $GPG_PUB_KEY: %w", err) - } - - if err := os.WriteFile(cfg.GPGPrivateKey, append(gpgPrivKeyB, '\n'), 0400); err != nil { - return fmt.Errorf("failed to write GPG private key file: %w", err) - } - if err := os.WriteFile(cfg.GPGPublicKey, append(gpgPubKeyB, '\n'), 0400); err != nil { - return fmt.Errorf("failed to write GPG public key file: %w", err) - } - if err := os.WriteFile(cfg.GPGPassPath, []byte(gpgPass+"\n"), 0400); err != nil { - return fmt.Errorf("failed to write GPG password file: %w", err) - } - - return nil -} - -// RemoveGPGFiles removes configured GPG files. -func RemoveGPGFiles(cfg config.Config) { - for _, fpath := range []string{cfg.GPGPrivateKey, cfg.GPGPublicKey, cfg.GPGPassPath} { - if err := os.Remove(fpath); err != nil { - log.Printf("failed to remove %q", fpath) - } - } -} diff --git a/pkg/build/gpg/import.go b/pkg/build/gpg/import.go deleted file mode 100644 index 4542a64d8c9..00000000000 --- a/pkg/build/gpg/import.go +++ /dev/null @@ -1,73 +0,0 @@ -package gpg - -import ( - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/fsutil" -) - -// writeRpmMacros writes ~/.rpmmacros. -func writeRpmMacros(homeDir, gpgPassPath string) error { - fpath := filepath.Join(homeDir, ".rpmmacros") - content := fmt.Sprintf(`%%_signature gpg -%%_gpg_path %s/.gnupg -%%_gpg_name Grafana -%%_gpgbin /usr/bin/gpg -%%__gpg_sign_cmd %%{__gpg} gpg --batch --yes --pinentry-mode loopback --no-armor --passphrase-file %s --no-secmem-warning -u "%%{_gpg_name}" -sbo %%{__signature_filename} %%{__plaintext_filename} -`, homeDir, gpgPassPath) - //nolint:gosec - if err := os.WriteFile(fpath, []byte(content), 0600); err != nil { - return fmt.Errorf("failed to write %q: %w", fpath, err) - } - - return nil -} - -// Import imports the GPG package signing key. -// ~/.rpmmacros also gets written. -func Import(cfg config.Config) error { - exists, err := fsutil.Exists(cfg.GPGPrivateKey) - if err != nil { - return err - } - if !exists { - return fmt.Errorf("GPG private key file doesn't exist: %q", cfg.GPGPrivateKey) - } - - log.Printf("Importing GPG key %q...", cfg.GPGPrivateKey) - // nolint:gosec - cmd := exec.Command("gpg", "--batch", "--yes", "--no-tty", "--allow-secret-key-import", "--import", - cfg.GPGPrivateKey) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to import private key: %s", output) - } - - homeDir, err := os.UserHomeDir() - if err != nil { - return err - } - - if err := writeRpmMacros(homeDir, cfg.GPGPassPath); err != nil { - return err - } - - pubKeysPath := filepath.Join(homeDir, ".rpmdb", "pubkeys") - if err := os.MkdirAll(pubKeysPath, 0700); err != nil { - return fmt.Errorf("failed to make %s: %w", pubKeysPath, err) - } - gpgPub, err := os.ReadFile(cfg.GPGPublicKey) - if err != nil { - return err - } - //nolint:gosec - if err := os.WriteFile(filepath.Join(homeDir, ".rpmdb", "pubkeys", "grafana.key"), gpgPub, 0400); err != nil { - return fmt.Errorf("failed to write pub key to ~/.rpmdb: %w", err) - } - - return nil -} diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go deleted file mode 100644 index 586b2d22839..00000000000 --- a/pkg/build/grafana/build.go +++ /dev/null @@ -1,129 +0,0 @@ -package grafana - -import ( - "context" - "fmt" - "log" - "os" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/cryptoutil" - "github.com/grafana/grafana/pkg/build/golangutils" -) - -var binaries = []string{"grafana", "grafana-server", "grafana-cli"} - -const ( - SuffixEnterprise2 = "-enterprise2" -) - -const ( - ExtensionExe = ".exe" -) - -func GrafanaLDFlags(version string, r config.Revision) []string { - cmd := []string{ - "-w", - fmt.Sprintf("-X main.version=%s", version), - fmt.Sprintf("-X main.commit=%s", r.SHA256), - fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp), - fmt.Sprintf("-X main.buildBranch=%s", r.Branch), - } - - if r.EnterpriseCommit != "" { - cmd = append(cmd, fmt.Sprintf("-X main.enterpriseCommit=%s", r.EnterpriseCommit)) - } - - return cmd -} - -// BinaryFolder returns the path to where the Grafana binary is build given the provided arguments. -func BinaryFolder(edition config.Edition, args BuildArgs) string { - sfx := "" - if edition == config.EditionEnterprise2 { - sfx = SuffixEnterprise2 - } - - arch := string(args.GoArch) - if args.GoArch == config.ArchARM { - arch = string(args.GoArch) + "v" + args.GoArm - } - - format := fmt.Sprintf("%s-%s", args.GoOS, arch) - if args.LibC != "" { - format += fmt.Sprintf("-%s", args.LibC) - } - format += sfx - - if args.GoOS == config.OSWindows { - format += ExtensionExe - } - - return format -} - -func GrafanaDescriptor(opts golangutils.BuildOpts) string { - libcPart := "" - if opts.LibC != "" { - libcPart = fmt.Sprintf("/%s", opts.LibC) - } - arch := string(opts.GoArch) - if opts.GoArch == config.ArchARM { - arch = string(opts.GoArch) + "v" + opts.GoArm - } - - return fmt.Sprintf("%s/%s%s", opts.GoOS, arch, libcPart) -} - -// BuildGrafanaBinary builds a certain binary according to certain parameters. -func BuildGrafanaBinary(ctx context.Context, name, version string, args BuildArgs, edition config.Edition) error { - opts := args.BuildOpts - opts.ExtraEnv = os.Environ() - - revision, err := config.GrafanaRevision(ctx, opts.Workdir) - if err != nil { - return err - } - - folder := BinaryFolder(edition, args) - - if opts.GoOS == config.OSWindows { - name += ExtensionExe - } - - binary := filepath.Join(opts.Workdir, "bin", folder, name) - opts.Output = binary - - if err := os.RemoveAll(binary); err != nil { - return fmt.Errorf("failed to remove %q: %w", binary, err) - } - - if err := os.RemoveAll(binary + ".md5"); err != nil { - return fmt.Errorf("failed to remove %q: %w", binary+".md5", err) - } - - descriptor := GrafanaDescriptor(opts) - - log.Printf("Building %q for %s", binary, descriptor) - - opts.LdFlags = append(args.LdFlags, GrafanaLDFlags(version, revision)...) - - if edition == config.EditionEnterprise2 { - opts.ExtraArgs = []string{"-tags=pro"} - } - - log.Printf("Running command 'go %s'", opts.Args()) - - if err := golangutils.RunBuild(ctx, opts); err != nil { - return err - } - - // Create an MD5 checksum of the binary, to be included in the archive for - // automatic upgrades. - if err := cryptoutil.MD5File(binary); err != nil { - return err - } - - return nil -} diff --git a/pkg/build/grafana/variant.go b/pkg/build/grafana/variant.go index 6ccd4abb8a4..89654722cf2 100644 --- a/pkg/build/grafana/variant.go +++ b/pkg/build/grafana/variant.go @@ -1,160 +1 @@ package grafana - -import ( - "bytes" - "context" - "fmt" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/compilers" - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/golangutils" -) - -// BuildArgs represent the build parameters that define the "go build" behavior of a single variant. -// These arguments are applied as environment variables and arguments to the "go build" command. -type BuildArgs struct { - golangutils.BuildOpts - DebArch config.Architecture - RPMArch config.Architecture -} - -type BuildVariantOpts struct { - Variant config.Variant - Edition config.Edition - - Version string - GrafanaDir string -} - -// BuildVariant builds a certain variant of the grafana-server and grafana-cli binaries sequentially. -func BuildVariant(ctx context.Context, opts BuildVariantOpts) error { - grafanaDir, err := filepath.Abs(opts.GrafanaDir) - if err != nil { - return err - } - - var ( - args = VariantBuildArgs(opts.Variant) - ) - - for _, binary := range binaries { - // Note that for Golang cmd paths we must use the relative path and the Linux file separators (/) even for Windows users. - var ( - pkg = fmt.Sprintf("./pkg/cmd/%s", binary) - stdout = bytes.NewBuffer(nil) - stderr = bytes.NewBuffer(nil) - ) - - args.Workdir = grafanaDir - args.Stdout = stdout - args.Stderr = stderr - args.Package = pkg - - if err := BuildGrafanaBinary(ctx, binary, opts.Version, args, opts.Edition); err != nil { - return fmt.Errorf("failed to build %s for %s: %w\nstdout: %s\nstderr: %s", pkg, opts.Variant, err, stdout.String(), stderr.String()) - } - } - - return nil -} - -var ldFlagsStatic = []string{"-linkmode=external", "-extldflags=-static"} - -var variantArgs = map[config.Variant]BuildArgs{ - config.VariantArmV6: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - CGoEnabled: true, - GoArch: config.ArchARM, - GoArm: "6", - CC: compilers.ArmV6, - }, - DebArch: config.ArchARMHF, - }, - config.VariantArmV7: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - CGoEnabled: true, - GoArch: config.ArchARM, - GoArm: "7", - CC: compilers.Armv7, - }, - DebArch: config.ArchARMHF, - RPMArch: config.ArchARMHFP, - }, - config.VariantArmV7Musl: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - CGoEnabled: true, - GoArch: config.ArchARM, - GoArm: "7", - LibC: config.LibCMusl, - CC: compilers.Armv7Musl, - LdFlags: ldFlagsStatic, - }, - }, - config.VariantArm64: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - CGoEnabled: true, - GoArch: config.ArchARM64, - CC: compilers.Arm64, - }, - DebArch: config.ArchARM64, - RPMArch: "aarch64", - }, - config.VariantArm64Musl: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - GoArch: config.ArchARM64, - CGoEnabled: true, - CC: compilers.Arm64Musl, - LibC: config.LibCMusl, - LdFlags: ldFlagsStatic, - }, - }, - config.VariantDarwinAmd64: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSDarwin, - CGoEnabled: true, - GoArch: config.ArchAMD64, - CC: compilers.Osx64, - }, - }, - config.VariantWindowsAmd64: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSWindows, - GoArch: config.ArchAMD64, - CC: compilers.Win64, - CGoEnabled: true, - CGoCFlags: "-D_WIN32_WINNT=0x0601", - }, - }, - config.VariantLinuxAmd64: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - GoArch: config.ArchAMD64, - CC: compilers.LinuxX64, - }, - DebArch: config.ArchAMD64, - RPMArch: config.ArchAMD64, - }, - config.VariantLinuxAmd64Musl: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - GoArch: config.ArchAMD64, - CC: compilers.LinuxX64Musl, - LibC: config.LibCMusl, - LdFlags: ldFlagsStatic, - }, - }, -} - -func VariantBuildArgs(v config.Variant) BuildArgs { - if val, ok := variantArgs[v]; ok { - return val - } - - return BuildArgs{} -} diff --git a/pkg/build/lerna/lerna.go b/pkg/build/lerna/lerna.go index 3bd67a87b44..048c0dd374d 100644 --- a/pkg/build/lerna/lerna.go +++ b/pkg/build/lerna/lerna.go @@ -2,63 +2,13 @@ package lerna import ( "context" - "encoding/json" "fmt" "os" "os/exec" - "path/filepath" - "strings" - "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/fsutil" ) -// BuildFrontendPackages will bump the version for the package to the latest canary build -// and build the packages so they are ready for being published, used for generating docs etc. -func BuildFrontendPackages(version string, mode config.Edition, grafanaDir string) error { - err := bumpLernaVersion(version, grafanaDir) - if err != nil { - return err - } - cmd := exec.Command("yarn", "run", "packages:build") - cmd.Dir = grafanaDir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to build %s frontend packages: %s", mode, output) - } - - return nil -} - -func bumpLernaVersion(version string, grafanaDir string) error { - //nolint:gosec - cmd := exec.Command("yarn", "run", "lerna", "version", version, "--exact", "--no-git-tag-version", "--no-push", "--force-publish", "-y") - cmd.Dir = grafanaDir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to bump version for frontend packages: %s\n%s", err, output) - } - - return nil -} - -func GetLernaVersion(grafanaDir string) (string, error) { - lernaJSONPath := filepath.Join(grafanaDir, "lerna.json") - //nolint:gosec - lernaJSONB, err := os.ReadFile(lernaJSONPath) - if err != nil { - return "", fmt.Errorf("failed to read %q: %w", lernaJSONPath, err) - } - pkgObj := map[string]any{} - if err := json.Unmarshal(lernaJSONB, &pkgObj); err != nil { - return "", fmt.Errorf("failed decoding %q: %w", lernaJSONPath, err) - } - - version := pkgObj["version"].(string) - if version == "" { - return "", fmt.Errorf("failed to read version from %q", lernaJSONPath) - } - return strings.TrimSpace(version), nil -} - func PackFrontendPackages(ctx context.Context, tag, grafanaDir, artifactsDir string) error { exists, err := fsutil.Exists(artifactsDir) if err != nil { diff --git a/pkg/build/packaging/artifacts.go b/pkg/build/packaging/artifacts.go index 36c1e3258e5..1dc60f18a2b 100644 --- a/pkg/build/packaging/artifacts.go +++ b/pkg/build/packaging/artifacts.go @@ -1,9 +1,6 @@ package packaging import ( - "fmt" - "strings" - "github.com/grafana/grafana/pkg/build/config" ) @@ -12,11 +9,18 @@ const MainFolder = "main" const EnterpriseSfx = "-enterprise" const CacheSettings = "Cache-Control:public, max-age=" -type buildArtifact struct { - Os string - Arch string - urlPostfix string - packagePostfix string +type BuildArtifact struct { + // Distro can be "windows", "darwin", "deb", "rhel", or "linux" + Distro string + Arch string + // Ext is the file extension without the "." + Ext string + Musl bool + RaspberryPi bool + + // URL can be set optionally by another process + // Note: check other repos before determining this to be dead code + URL string } type PublishConfig struct { @@ -32,110 +36,91 @@ type PublishConfig struct { SimulateRelease bool } -const rhelOS = "rhel" -const debOS = "deb" - -func (t buildArtifact) GetURL(baseArchiveURL string, cfg PublishConfig) string { - rev := "" - prefix := "-" - if t.Os == debOS { - prefix = "_" - } else if t.Os == rhelOS { - rev = "-1" - } - - version := cfg.Version - verComponents := strings.Split(version, "-") - if len(verComponents) > 2 { - panic(fmt.Sprintf("Version string contains more than one hyphen: %q", version)) - } - - switch t.Os { - case debOS, rhelOS: - if len(verComponents) > 1 { - // With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this - // is considered of lower lexical value than the empty character, and this way pre-release versions are - // considered to be of a lower version than the final version (which lacks this suffix). - version = fmt.Sprintf("%s~%s", verComponents[0], verComponents[1]) - } - } - - // https://dl.grafana.com/oss/main/grafana_8.5.0~54094pre_armhf.deb: 404 Not Found - url := fmt.Sprintf("%s%s%s%s%s%s", baseArchiveURL, t.packagePostfix, prefix, version, rev, t.urlPostfix) - return url +var LinuxArtifacts = []BuildArtifact{ + { + Distro: "linux", + Arch: "arm64", + Ext: "tar.gz", + }, + { + Distro: "deb", + Arch: "amd64", + Ext: "deb", + }, + { + Distro: "rhel", + Arch: "x86_64", + Ext: "rpm", + }, + { + Distro: "linux", + Arch: "amd64", + Ext: "tar.gz", + }, } -var ArtifactConfigs = []buildArtifact{ +var DarwinArtifacts = []BuildArtifact{ { - Os: debOS, - Arch: "arm64", - urlPostfix: "_arm64.deb", + Distro: "darwin", + Arch: "amd64", + Ext: "tar.gz", }, +} + +var WindowsArtifacts = []BuildArtifact{ { - Os: rhelOS, - Arch: "arm64", - urlPostfix: ".aarch64.rpm", + Distro: "windows", + Arch: "amd64", + Ext: "zip", }, { - Os: "linux", - Arch: "arm64", - urlPostfix: ".linux-arm64.tar.gz", + Distro: "windows", + Arch: "amd64", + Ext: "msi", }, - // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved - // { - // Os: debOS, - // Arch: "armv7", - // urlPostfix: "_armhf.deb", - // }, - // { - // Os: debOS, - // Arch: "armv6", - // packagePostfix: "-rpi", - // urlPostfix: "_armhf.deb", - // }, - // { - // Os: rhelOS, - // Arch: "armv7", - // urlPostfix: ".armhfp.rpm", - // }, - // { - // Os: "linux", - // Arch: "armv6", - // urlPostfix: ".linux-armv6.tar.gz", - // }, - // { - // Os: "linux", - // Arch: "armv7", - // urlPostfix: ".linux-armv7.tar.gz", - // }, +} + +var ARMArtifacts = []BuildArtifact{ { - Os: "darwin", - Arch: "amd64", - urlPostfix: ".darwin-amd64.tar.gz", + Distro: "deb", + Arch: "arm64", + Ext: "deb", }, { - Os: "deb", - Arch: "amd64", - urlPostfix: "_amd64.deb", + Distro: "rhel", + Arch: "aarch64", + Ext: "rpm", }, { - Os: rhelOS, - Arch: "amd64", - urlPostfix: ".x86_64.rpm", + Distro: "deb", + Arch: "armhf", + Ext: "deb", + RaspberryPi: false, }, { - Os: "linux", - Arch: "amd64", - urlPostfix: ".linux-amd64.tar.gz", + Distro: "deb", + Arch: "armhf", + RaspberryPi: true, + Ext: "deb", }, { - Os: "win", - Arch: "amd64", - urlPostfix: ".windows-amd64.zip", + Distro: "linux", + Arch: "armv6", + Ext: "tar.gz", }, { - Os: "win-installer", - Arch: "amd64", - urlPostfix: ".windows-amd64.msi", + Distro: "linux", + Arch: "armv7", + Ext: "tar.gz", }, } + +func join(a []BuildArtifact, b ...[]BuildArtifact) []BuildArtifact { + for i := range b { + a = append(a, b[i]...) + } + + return a +} + +var ArtifactConfigs = join(LinuxArtifacts, DarwinArtifacts, WindowsArtifacts, ARMArtifacts) diff --git a/pkg/build/packaging/docs.go b/pkg/build/packaging/docs.go deleted file mode 100644 index a723cb3d346..00000000000 --- a/pkg/build/packaging/docs.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package packaging holds functions and types for creating the tar.gz, deb, and rpm packages of Grafana. -package packaging diff --git a/pkg/build/packaging/errors.go b/pkg/build/packaging/errors.go deleted file mode 100644 index c20b9edbfae..00000000000 --- a/pkg/build/packaging/errors.go +++ /dev/null @@ -1 +0,0 @@ -package packaging diff --git a/pkg/build/packaging/grafana.go b/pkg/build/packaging/grafana.go deleted file mode 100644 index 7022f1bf469..00000000000 --- a/pkg/build/packaging/grafana.go +++ /dev/null @@ -1,1133 +0,0 @@ -package packaging - -import ( - "archive/tar" - "archive/zip" - "compress/gzip" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/fsutil" - "github.com/grafana/grafana/pkg/build/grafana" - "github.com/grafana/grafana/pkg/build/plugins" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -var ( - ErrorNoBinaries = errors.New("no binaries found") - ErrorNoDebArch = errors.New("deb architecture not defined") - ErrorNoRPMArch = errors.New("rpm architecture not defined") -) - -const ( - maxAttempts = 3 - enterpriseSfx = "-enterprise" - enterprise2Sfx = "-enterprise2" - DefaultDebDBBucket = "grafana-aptly-db" - DefaultDebRepoBucket = "grafana-repo" - DefaultRPMRepoBucket = "grafana-repo" - DefaultTTLSeconds = "300" -) - -// PackageRegexp returns a regexp for matching packages corresponding to a certain Grafana edition. -func PackageRegexp(edition config.Edition) *regexp.Regexp { - var sfx string - switch edition { - case config.EditionOSS: - case config.EditionEnterprise: - sfx = "-enterprise" - case config.EditionEnterprise2: - sfx = "-enterprise2" - default: - panic(fmt.Sprintf("unrecognized edition %q", edition)) - } - rePkg, err := regexp.Compile(fmt.Sprintf(`^grafana%s(?:-rpi)?[-_][^-_]+.*$`, sfx)) - if err != nil { - panic(fmt.Sprintf("Failed to compile regexp: %s", err)) - } - - return rePkg -} - -// PackageGrafana packages Grafana for various variants. -func PackageGrafana( - ctx context.Context, - version string, - grafanaDir string, - cfg config.Config, - edition config.Edition, - variants []config.Variant, - shouldSign bool, - p syncutil.WorkerPool, -) error { - if err := packageGrafana(ctx, edition, version, grafanaDir, variants, shouldSign, p); err != nil { - return err - } - - if cfg.SignPackages { - if err := signRPMPackages(edition, cfg, grafanaDir); err != nil { - return err - } - } - - if err := checksumPackages(grafanaDir, edition); err != nil { - return err - } - - return nil -} - -func packageGrafana( - ctx context.Context, - edition config.Edition, - version string, - grafanaDir string, - variants []config.Variant, - shouldSign bool, - p syncutil.WorkerPool, -) error { - distDir := filepath.Join(grafanaDir, "dist") - exists, err := fsutil.Exists(distDir) - if err != nil { - return err - } - if !exists { - log.Printf("directory %s doesn't exist - creating...", distDir) - //nolint - if err := os.MkdirAll(distDir, 0o755); err != nil { - return fmt.Errorf("couldn't create dist: %w", err) - } - } - - var m pluginsManifest - manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json") - //nolint:gosec - manifestB, err := os.ReadFile(manifestPath) - if err != nil { - return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err) - } - if err := json.Unmarshal(manifestB, &m); err != nil { - return err - } - - g, ctx := errutil.GroupWithContext(ctx) - for _, v := range variants { - packageVariant(ctx, v, edition, version, grafanaDir, shouldSign, g, p, m) - } - if err := g.Wait(); err != nil { - return err - } - - return nil -} - -// packageVariant packages Grafana for a certain variant. -func packageVariant( - ctx context.Context, - v config.Variant, - edition config.Edition, - version string, - grafanaDir string, - shouldSign bool, - g *errutil.Group, - p syncutil.WorkerPool, - m pluginsManifest, -) { - p.Schedule(g.Wrap(func() error { - // We've experienced spurious packaging failures, so retry on failure. - i := 0 - for { - if err := realPackageVariant(ctx, v, edition, version, grafanaDir, m, shouldSign); err != nil { - i++ - if i < maxAttempts { - log.Printf("Packaging for variant %s, %s edition failed: %s, trying again", v, edition, err) - continue - } - - return err - } - - break - } - - return nil - })) -} - -// signRPMPackages signs the RPM packages. -func signRPMPackages(edition config.Edition, cfg config.Config, grafanaDir string) error { - log.Printf("Signing %s RPM packages...", edition) - var sfx string - switch edition { - case config.EditionOSS: - case config.EditionEnterprise: - sfx = enterpriseSfx - case config.EditionEnterprise2: - sfx = enterprise2Sfx - default: - panic(fmt.Sprintf("Unrecognized edition %s", edition)) - } - rpms, err := filepath.Glob(filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-*.rpm", sfx))) - if err != nil { - return err - } - - if len(rpms) > 0 { - rpmArgs := append([]string{"--addsign"}, rpms...) - log.Printf("Invoking rpm with args: %+v", rpmArgs) - //nolint:gosec - cmd := exec.Command("rpm", rpmArgs...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to sign RPM packages: %s", output) - } - if err := os.Remove(cfg.GPGPassPath); err != nil { - return fmt.Errorf("failed to remove %q: %w", cfg.GPGPassPath, err) - } - - log.Printf("Verifying %s RPM packages...", edition) - // The output changed between rpm versions - reOutput := regexp.MustCompile("(?:digests signatures OK)|(?:pgp.+OK)") - for _, p := range rpms { - //nolint:gosec - cmd := exec.Command("rpm", "-K", p) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to verify RPM signature: %w", err) - } - - if !reOutput.Match(output) { - return fmt.Errorf("RPM package %q not verified: %s", p, output) - } - } - } - - return nil -} - -// checksumPackages generates package checksums with SHA-256. -func checksumPackages(grafanaDir string, edition config.Edition) error { - log.Printf("Checksumming %s packages...", edition) - distDir := filepath.Join(grafanaDir, "dist") - rePkg := PackageRegexp(edition) - if err := filepath.Walk(distDir, func(fpath string, info os.FileInfo, err error) error { - if err != nil { - var pathErr *os.PathError - if errors.As(err, &pathErr) { - log.Printf("path error in walk function for file %q: %s", pathErr.Path, pathErr.Err.Error()) - return nil - } - return fmt.Errorf("walking through dist folder failed: %w", err) - } - - if info.IsDir() { - return nil - } - - fname := filepath.Base(fpath) - if strings.HasSuffix(fname, ".sha256") || strings.HasSuffix(fname, ".version") || !rePkg.MatchString(fname) { - log.Printf("Ignoring non-package %q", fpath) - return nil - } - - return shaFile(fpath) - }); err != nil { - return fmt.Errorf("checksumming packages in %q failed: %w", distDir, err) - } - - log.Printf("Successfully checksummed %s packages", edition) - return nil -} - -func shaFile(fpath string) error { - //nolint:gosec - fd, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fpath, err) - } - defer func() { - if err := fd.Close(); err != nil { - log.Println(err) - } - }() - - h := sha256.New() - _, err = io.Copy(h, fd) - if err != nil { - return fmt.Errorf("failed to read %q: %w", fpath, err) - } - - //nolint:gosec - out, err := os.Create(fpath + ".sha256") - if err != nil { - return fmt.Errorf("failed to create %q: %w", fpath+".sha256", err) - } - defer func() { - if err := out.Close(); err != nil { - log.Println("failed to close file", out.Name()) - } - }() - - if _, err = fmt.Fprintf(out, "%x\n", h.Sum(nil)); err != nil { - return fmt.Errorf("failed to write %q: %w", out.Name(), err) - } - - return nil -} - -// createPackage creates a Linux package. -func createPackage(srcDir string, options linuxPackageOptions) error { - binary := "grafana" - cliBinary := "grafana-cli" - serverBinary := "grafana-server" - - packageRoot, err := os.MkdirTemp("", "grafana-linux-pack") - if err != nil { - return fmt.Errorf("failed to create temporary directory: %w", err) - } - defer func() { - if err := os.RemoveAll(packageRoot); err != nil { - log.Println(err) - } - }() - - for _, dname := range []string{ - options.homeDir, - options.configDir, - "etc/init.d", - options.etcDefaultPath, - "usr/lib/systemd/system", - "usr/sbin", - } { - dpath := filepath.Join(packageRoot, dname) - //nolint - if err := os.MkdirAll(dpath, 0o755); err != nil { - return fmt.Errorf("failed to make directory %q: %w", dpath, err) - } - } - - if err := fsutil.CopyFile(filepath.Join(options.wrapperFilePath, binary), - filepath.Join(packageRoot, "usr", "sbin", binary)); err != nil { - return err - } - if err := fsutil.CopyFile(filepath.Join(options.wrapperFilePath, cliBinary), - filepath.Join(packageRoot, "usr", "sbin", cliBinary)); err != nil { - return err - } - if err := fsutil.CopyFile(filepath.Join(options.wrapperFilePath, serverBinary), - filepath.Join(packageRoot, "usr", "sbin", serverBinary)); err != nil { - return err - } - if err := fsutil.CopyFile(options.initdScriptSrc, filepath.Join(packageRoot, options.initdScriptFilePath)); err != nil { - return err - } - if err := fsutil.CopyFile(options.defaultFileSrc, filepath.Join(packageRoot, options.etcDefaultFilePath)); err != nil { - return err - } - if err := fsutil.CopyFile(options.systemdFileSrc, filepath.Join(packageRoot, options.systemdServiceFilePath)); err != nil { - return err - } - if err := fsutil.CopyRecursive(srcDir, filepath.Join(packageRoot, options.homeDir)); err != nil { - return err - } - - if err := executeFPM(options, packageRoot, srcDir); err != nil { - return err - } - - return nil -} -func executeFPM(options linuxPackageOptions, packageRoot, srcDir string) error { - name := "grafana" - vendor := "Grafana" - if options.edition == config.EditionEnterprise || options.edition == config.EditionEnterprise2 { - vendor += " Enterprise" - if options.edition == config.EditionEnterprise2 { - name += enterprise2Sfx - } else if options.edition == config.EditionEnterprise { - name += enterpriseSfx - } - } - - if options.goArch == config.ArchARM && options.goArm == "6" { - name += "-rpi" - } - - pkgVersion := packageVersion(options) - args := []string{ - "-s", "dir", - "--description", "Grafana", - "-C", packageRoot, - "--url", "https://grafana.com", - "--maintainer", "contact@grafana.com", - "--config-files", options.initdScriptFilePath, - "--config-files", options.etcDefaultFilePath, - "--config-files", options.systemdServiceFilePath, - "--after-install", options.postinstSrc, - "--version", pkgVersion, - "-p", "dist/", - "--name", name, - "--vendor", vendor, - "-a", string(options.packageArch), - } - if options.prermSrc != "" { - args = append(args, "--before-remove", options.prermSrc) - } - if options.edition == config.EditionEnterprise || options.edition == config.EditionEnterprise2 || options.goArch == config.ArchARMv6 { - args = append(args, "--conflicts", "grafana") - } - if options.edition == config.EditionOSS { - args = append(args, "--license", "\"AGPLv3\"") - } - switch options.packageType { - case packageTypeRpm: - args = append(args, "-t", "rpm", "--rpm-posttrans", "packaging/rpm/control/posttrans") - args = append(args, "--rpm-digest", "sha256") - case packageTypeDeb: - args = append(args, "-t", "deb", "--deb-no-default-config-files") - default: - panic(fmt.Sprintf("Unrecognized package type %d", options.packageType)) - } - for _, dep := range options.depends { - args = append(args, "--depends", dep) - } - args = append(args, ".") - - distDir := filepath.Join(options.grafanaDir, "dist") - log.Printf("Generating package in %q (source directory %q)", distDir, srcDir) - - cmdStr := "fpm" - for _, arg := range args { - if strings.Contains(arg, " ") { - arg = fmt.Sprintf("'%s'", arg) - } - cmdStr += fmt.Sprintf(" %s", arg) - } - log.Printf("Creating %s package: %s...", options.packageType, cmdStr) - const rvmPath = "/etc/profile.d/rvm.sh" - exists, err := fsutil.Exists(rvmPath) - if err != nil { - return err - } - if exists { - cmdStr = fmt.Sprintf("source %q && %s", rvmPath, cmdStr) - log.Printf("Sourcing %q before running fpm", rvmPath) - } - //nolint:gosec - cmd := exec.Command("/bin/bash", "-c", cmdStr) - cmd.Dir = options.grafanaDir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to run fpm: %s", output) - } - - return nil -} - -// copyPubDir copies public/ from grafanaDir to tmpDir. -func copyPubDir(grafanaDir, tmpDir string) error { - srcPubDir := filepath.Join(grafanaDir, "public") - tgtPubDir := filepath.Join(tmpDir, "public") - log.Printf("Copying %q to %q...", srcPubDir, tgtPubDir) - if err := fsutil.CopyRecursive(srcPubDir, tgtPubDir); err != nil { - return fmt.Errorf("failed to copy %q to %q: %w", srcPubDir, tgtPubDir, err) - } - - return nil -} - -// copyBinaries copies binaries from grafanaDir into tmpDir. -func copyBinaries(grafanaDir, tmpDir string, args grafana.BuildArgs, edition config.Edition) error { - tgtDir := filepath.Join(tmpDir, "bin") - //nolint - if err := os.MkdirAll(tgtDir, 0o755); err != nil { - return fmt.Errorf("failed to make directory %q: %w", tgtDir, err) - } - - binDir := filepath.Join(grafanaDir, "bin", grafana.BinaryFolder(edition, args)) - - files, err := os.ReadDir(binDir) - if err != nil { - return fmt.Errorf("failed to list files in %q: %w", binDir, err) - } - - if len(files) == 0 { - return fmt.Errorf("%w in %s", ErrorNoBinaries, binDir) - } - - for _, file := range files { - srcPath := filepath.Join(binDir, file.Name()) - tgtPath := filepath.Join(tgtDir, file.Name()) - - if err := fsutil.CopyFile(srcPath, tgtPath); err != nil { - return err - } - } - - return nil -} - -// copyConfFiles copies configuration files from grafanaDir into tmpDir. -func copyConfFiles(grafanaDir, tmpDir string) error { - //nolint:gosec - if err := os.MkdirAll(filepath.Join(tmpDir, "conf"), 0o755); err != nil { - return fmt.Errorf("failed to create dir %q: %w", filepath.Join(tmpDir, "conf"), err) - } - - confDir := filepath.Join(grafanaDir, "conf") - infos, err := os.ReadDir(confDir) - if err != nil { - return fmt.Errorf("failed to list files in %q: %w", confDir, err) - } - for _, info := range infos { - fpath := filepath.Join(confDir, info.Name()) - if err := fsutil.CopyRecursive(fpath, filepath.Join(tmpDir, "conf", info.Name())); err != nil { - return err - } - } - - return nil -} - -// copyPlugins copies plugins from grafanaDir into tmpDir. -func copyPlugins(ctx context.Context, v config.Variant, grafanaDir, tmpDir string, m pluginsManifest, shouldSign bool) error { - log.Printf("Copying plugins for package variant %s...", v) - - variant2Sfx := map[config.Variant]string{ - config.VariantLinuxAmd64: "linux_amd64", - config.VariantDarwinAmd64: "darwin_amd64", - config.VariantWindowsAmd64: "windows_amd64.exe", - } - - tgtDir := filepath.Join(tmpDir, "plugins-bundled") - exists, err := fsutil.Exists(tgtDir) - if err != nil { - return err - } - if !exists { - //nolint:gosec - if err := os.MkdirAll(tgtDir, 0o755); err != nil { - return err - } - } - pluginsDir := filepath.Join(grafanaDir, "plugins-bundled") - - // External plugins. - for _, pm := range m.Plugins { - srcDir := filepath.Join(pluginsDir, fmt.Sprintf("%s-%s", pm.Name, pm.Version)) - dstDir := filepath.Join(tgtDir, fmt.Sprintf("%s-%s", pm.Name, pm.Version)) - log.Printf("Copying external plugin %q to %q...", srcDir, dstDir) - - //nolint:gosec - jsonB, err := os.ReadFile(filepath.Join(srcDir, "plugin.json")) - if err != nil { - return fmt.Errorf("failed to read %q: %w", filepath.Join(srcDir, "plugin.json"), err) - } - var plugJSON map[string]any - if err := json.Unmarshal(jsonB, &plugJSON); err != nil { - return err - } - - plugExe, ok := plugJSON["executable"].(string) - var wantExe string - if ok && strings.TrimSpace(plugExe) != "" { - sfx := variant2Sfx[v] - if sfx == "" { - log.Printf("External plugin %s-%s doesn't have an executable for variant %s - ignoring", - pm.Name, pm.Version, v) - continue - } - - wantExe = fmt.Sprintf("%s_%s", plugExe, sfx) - log.Printf("The external plugin should contain an executable %q", wantExe) - exists, err := fsutil.Exists(filepath.Join(srcDir, wantExe)) - if err != nil { - return err - } - if !exists { - log.Printf("External plugin %s-%s doesn't have an executable of the right format: %q - ignoring", - pm.Name, pm.Version, wantExe) - continue - } - } - - if err := filepath.Walk(srcDir, func(pth string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - log.Printf("Handling %q", pth) - - relPath := strings.TrimPrefix(pth, srcDir) - relPath = strings.TrimPrefix(relPath, "/") - dstPath := filepath.Join(dstDir, relPath) - - if info.IsDir() { - log.Printf("Making directory %q", dstPath) - //nolint:gosec - return os.MkdirAll(dstPath, info.Mode()) - } - - if wantExe != "" { - m, err := regexp.MatchString(fmt.Sprintf(`^%s_[^/]+$`, plugExe), relPath) - if err != nil { - return err - } - if m && relPath != wantExe { - // Ignore other executable variants - log.Printf("Ignoring executable variant %q", pth) - return nil - } - } - - log.Printf("Copying %q to %q", pth, dstPath) - return fsutil.CopyFile(pth, dstPath) - }); err != nil { - return fmt.Errorf("failed to copy external plugin %q to %q: %w", srcDir, dstDir, err) - } - - if shouldSign { - if err := plugins.BuildManifest(ctx, dstDir, true); err != nil { - return fmt.Errorf("failed to generate signed manifest for external plugin %q: %w", dstDir, err) - } - } - } - - return copyInternalPlugins(pluginsDir, tmpDir) -} - -func copyInternalPlugins(pluginsDir, tmpDir string) error { - tgtDir := filepath.Join(tmpDir, "plugins-bundled", "internal") - srcDir := filepath.Join(pluginsDir, "dist") - - exists, err := fsutil.Exists(tgtDir) - if err != nil { - return err - } - if !exists { - //nolint:gosec - if err := os.MkdirAll(tgtDir, 0o755); err != nil { - return err - } - } - - // Copy over internal plugins. - fis, err := os.ReadDir(srcDir) - if err != nil { - return fmt.Errorf("failed to list internal plugins in %q: %w", srcDir, err) - } - for _, fi := range fis { - srcPath := filepath.Join(srcDir, fi.Name()) - if !fi.IsDir() { - log.Printf("Ignoring non-directory %q", srcPath) - continue - } - - dstPath := filepath.Join(tgtDir, fi.Name()) - log.Printf("Copying internal plugin %q to %q...", srcPath, dstPath) - if err := fsutil.CopyRecursive(srcPath, dstPath); err != nil { - return fmt.Errorf("failed to copy %q to %q: %w", srcPath, dstPath, err) - } - } - - return nil -} - -func realPackageVariant(ctx context.Context, v config.Variant, edition config.Edition, version, grafanaDir string, - m pluginsManifest, shouldSign bool) error { - log.Printf("Packaging Grafana %s for %s...", edition, v) - - enableDeb := false - enableRpm := false - switch v { - case config.VariantLinuxAmd64: - enableDeb = true - enableRpm = true - case config.VariantArmV6: - enableDeb = true - case config.VariantArmV7: - enableDeb = true - enableRpm = true - case config.VariantArm64: - enableDeb = true - enableRpm = true - default: - } - - tmpDir, err := os.MkdirTemp("", "") - if err != nil { - return fmt.Errorf("failed to create temporary directory: %w", err) - } - defer func() { - if err := os.RemoveAll(tmpDir); err != nil { - log.Println(err) - } - }() - - args := grafana.VariantBuildArgs(v) - - if err := copyPubDir(grafanaDir, tmpDir); err != nil { - return err - } - if err := copyBinaries(grafanaDir, tmpDir, args, edition); err != nil { - return err - } - if err := copyConfFiles(grafanaDir, tmpDir); err != nil { - return err - } - if err := copyPlugins(ctx, v, grafanaDir, tmpDir, m, shouldSign); err != nil { - return err - } - - if v == config.VariantWindowsAmd64 { - toolsDir := filepath.Join(tmpDir, "tools") - //nolint:gosec - if err := os.MkdirAll(toolsDir, 0o755); err != nil { - return fmt.Errorf("failed to create tools dir %q: %w", toolsDir, err) - } - - if err := fsutil.CopyFile("/usr/local/go/lib/time/zoneinfo.zip", - filepath.Join(tmpDir, "tools", "zoneinfo.zip")); err != nil { - return err - } - } - - if err := os.WriteFile(filepath.Join(tmpDir, "VERSION"), []byte(version), 0664); err != nil { - return fmt.Errorf("failed to write %s/VERSION: %w", tmpDir, err) - } - - if err := createArchive(tmpDir, edition, v, version, grafanaDir); err != nil { - return err - } - - if enableDeb { - if args.DebArch == "" { - return fmt.Errorf("%w for %s", ErrorNoDebArch, v) - } - - if err := createPackage(tmpDir, linuxPackageOptions{ - edition: edition, - version: version, - grafanaDir: grafanaDir, - goArch: args.GoArch, - goArm: args.GoArm, - packageType: packageTypeDeb, - packageArch: args.DebArch, - homeDir: "/usr/share/grafana", - homeBinDir: "/usr/share/grafana/bin", - binPath: "/usr/sbin", - configDir: "/etc/grafana", - etcDefaultPath: "/etc/default", - etcDefaultFilePath: "/etc/default/grafana-server", - initdScriptFilePath: "/etc/init.d/grafana-server", - systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service", - postinstSrc: filepath.Join(grafanaDir, "packaging", "deb", "control", "postinst"), - prermSrc: filepath.Join(grafanaDir, "packaging", "deb", "control", "prerm"), - initdScriptSrc: filepath.Join(grafanaDir, "packaging", "deb", "init.d", "grafana-server"), - defaultFileSrc: filepath.Join(grafanaDir, "packaging", "deb", "default", "grafana-server"), - systemdFileSrc: filepath.Join(grafanaDir, "packaging", "deb", "systemd", "grafana-server.service"), - wrapperFilePath: filepath.Join(grafanaDir, "packaging", "wrappers"), - depends: []string{"adduser", "libfontconfig1"}, - }); err != nil { - return err - } - } - - if !enableRpm { - return nil - } - - if args.RPMArch == "" { - return fmt.Errorf("%w for %s", ErrorNoRPMArch, v) - } - - if err := createPackage(tmpDir, linuxPackageOptions{ - edition: edition, - version: version, - grafanaDir: grafanaDir, - goArch: args.GoArch, - packageType: packageTypeRpm, - packageArch: args.RPMArch, - homeDir: "/usr/share/grafana", - homeBinDir: "/usr/share/grafana/bin", - binPath: "/usr/sbin", - configDir: "/etc/grafana", - etcDefaultPath: "/etc/sysconfig", - etcDefaultFilePath: "/etc/sysconfig/grafana-server", - initdScriptFilePath: "/etc/init.d/grafana-server", - systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service", - postinstSrc: filepath.Join(grafanaDir, "packaging", "rpm", "control", "postinst"), - initdScriptSrc: filepath.Join(grafanaDir, "packaging", "rpm", "init.d", "grafana-server"), - defaultFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "sysconfig", "grafana-server"), - systemdFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "systemd", "grafana-server.service"), - wrapperFilePath: filepath.Join(grafanaDir, "packaging", "wrappers"), - depends: []string{"/sbin/service", "fontconfig", "freetype"}, - }); err != nil { - return err - } - - return nil -} - -// pluginManifest has details of an external plugin package. -type pluginManifest struct { - Name string `json:"name"` - Version string `json:"version"` - Checksum string `json:"checksum"` -} - -// pluginsManifest represents a manifest of Grafana's external plugins. -type pluginsManifest struct { - Plugins []pluginManifest `json:"plugins"` -} - -// packageVersion converts a Grafana version into the corresponding package version. -func packageVersion(options linuxPackageOptions) string { - verComponents := strings.Split(options.version, "-") - if len(verComponents) > 2 { - panic(fmt.Sprintf("Version string contains more than one hyphen: %q", options.version)) - } - - switch options.packageType { - case packageTypeDeb, packageTypeRpm: - if len(verComponents) > 1 { - // With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this - // is considered of lower lexical value than the empty character, and this way pre-release versions are - // considered to be of a lower version than the final version (which lacks this suffix). - return fmt.Sprintf("%s~%s", verComponents[0], verComponents[1]) - } - - return options.version - default: - panic(fmt.Sprintf("Unrecognized package type %s", options.packageType)) - } -} - -type packageType int - -func (pt packageType) String() string { - switch pt { - case packageTypeDeb: - return "Debian" - case packageTypeRpm: - return "RPM" - default: - panic(fmt.Sprintf("Unrecognized package type %d", pt)) - } -} - -const ( - packageTypeDeb packageType = iota - packageTypeRpm -) - -type linuxPackageOptions struct { - edition config.Edition - packageType packageType - version string - grafanaDir string - goArch config.Architecture - goArm string - packageArch config.Architecture - homeDir string - homeBinDir string - binPath string - configDir string - etcDefaultPath string - etcDefaultFilePath string - initdScriptFilePath string - systemdServiceFilePath string - postinstSrc string - prermSrc string - initdScriptSrc string - defaultFileSrc string - systemdFileSrc string - wrapperFilePath string - - depends []string -} - -// createArchive makes a distribution archive. -func createArchive(srcDir string, edition config.Edition, v config.Variant, version, grafanaDir string) error { - distDir := filepath.Join(grafanaDir, "dist") - exists, err := fsutil.Exists(distDir) - if err != nil { - return err - } - if !exists { - log.Printf("directory %s doesn't exist - creating...", distDir) - //nolint:gosec - if err := os.MkdirAll(distDir, 0o755); err != nil { - return fmt.Errorf("couldn't create dist: %w", err) - } - } - sfx := "" - if edition == config.EditionEnterprise2 { - sfx = enterprise2Sfx - } else if edition == config.EditionEnterprise { - sfx = enterpriseSfx - } - if v != config.VariantWindowsAmd64 { - return createTarball(srcDir, version, string(v), sfx, grafanaDir) - } - - return createZip(srcDir, version, string(v), sfx, grafanaDir) -} - -func createZip(srcDir, version, variantStr, sfx, grafanaDir string) error { - fpath := filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-%s.%s.zip", sfx, version, variantStr)) - //nolint:gosec - tgt, err := os.Create(fpath) - if err != nil { - return fmt.Errorf("failed to create %q: %w", fpath, err) - } - defer func() { - if err := tgt.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - log.Println(err) - } - }() - - //nolint:gosec - if err := os.Chmod(fpath, 0664); err != nil { - return fmt.Errorf("failed to set permissions on %q: %w", fpath, err) - } - zipWriter := zip.NewWriter(tgt) - defer func() { - if err := zipWriter.Close(); err != nil { - log.Println(err) - } - }() - - for _, fname := range []string{"LICENSE", "README.md", "NOTICE.md"} { - fpath := filepath.Join(grafanaDir, fname) - fi, err := os.Lstat(fpath) - if err != nil { - return fmt.Errorf("couldn't stat %q: %w", fpath, err) - } - hdr, err := zip.FileInfoHeader(fi) - if err != nil { - return fmt.Errorf("failed to open zip header: %w", err) - } - // Enable compression, as it's disabled by default - hdr.Method = zip.Deflate - hdr.Name = fmt.Sprintf("grafana-%s/%s", version, fname) - w, err := zipWriter.CreateHeader(hdr) - if err != nil { - return fmt.Errorf("failed writing zip header: %w", err) - } - //nolint:gosec - src, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fname, err) - } - if _, err := io.Copy(w, src); err != nil { - if err := src.Close(); err != nil { - log.Println(err) - } - return fmt.Errorf("failed writing zip entry: %w", err) - } - if err := src.Close(); err != nil { - log.Println(err) - } - } - if err := filepath.Walk(srcDir, func(fpath string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if fpath == srcDir { - return nil - } - - hdr, err := zip.FileInfoHeader(fi) - if err != nil { - return fmt.Errorf("failed to open zip header: %s", err) - } - // Enable compression, as it's disabled by default - hdr.Method = zip.Deflate - hdr.Name = fmt.Sprintf("grafana-%s/%s", version, strings.TrimPrefix(fpath, fmt.Sprintf("%s/", srcDir))) - if fi.IsDir() { - // A trailing slash means it's a directory - if hdr.Name[len(hdr.Name)-1] != '/' { - hdr.Name += "/" - } - } - w, err := zipWriter.CreateHeader(hdr) - if err != nil { - return fmt.Errorf("failed writing zip header: %s", err) - } - if fi.IsDir() { - return nil - } - - //nolint:gosec - src, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fpath, err) - } - if _, err := io.Copy(w, src); err != nil { - if err := src.Close(); err != nil { - log.Println(err) - } - return fmt.Errorf("failed writing zip entry: %w", err) - } - if err := src.Close(); err != nil { - log.Println(err) - } - return nil - }); err != nil { - return fmt.Errorf("failed to traverse directory %q: %w", srcDir, err) - } - - if err := zipWriter.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - if err := tgt.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - - log.Printf("Successfully created %q", fpath) - return nil -} - -// nolint -func createTarball(srcDir, version, variantStr, sfx, grafanaDir string) error { - fpath := filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-%s.%s.tar.gz", sfx, version, variantStr)) - //nolint:gosec - tgt, err := os.Create(fpath) - if err != nil { - return fmt.Errorf("failed to create %q: %w", fpath, err) - } - defer func() { - if err := tgt.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - log.Println(err) - } - }() - - //nolint:gosec - if err := os.Chmod(fpath, 0664); err != nil { - return fmt.Errorf("failed to set permissions on %q: %w", fpath, err) - } - gzWriter, err := gzip.NewWriterLevel(tgt, gzip.BestCompression) - if err != nil { - return fmt.Errorf("failed to create gzip writer: %w", err) - } - defer func() { - if err := gzWriter.Close(); err != nil { - log.Println(err) - } - }() - tarWriter := tar.NewWriter(gzWriter) - defer func() { - if err := tarWriter.Close(); err != nil { - log.Println(err) - } - }() - - for _, fname := range []string{"LICENSE", "README.md", "NOTICE.md"} { - fpath := filepath.Join(grafanaDir, fname) - fi, err := os.Lstat(fpath) - if err != nil { - return fmt.Errorf("couldn't stat %q: %w", fpath, err) - } - hdr, err := tar.FileInfoHeader(fi, "") - if err != nil { - return fmt.Errorf("failed getting tar header: %w", err) - } - hdr.Name = fmt.Sprintf("grafana-%s/%s", version, fname) - if err := tarWriter.WriteHeader(hdr); err != nil { - return fmt.Errorf("failed writing tar header: %w", err) - } - //nolint:gosec - src, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fname, err) - } - if _, err := io.Copy(tarWriter, src); err != nil { - if err := src.Close(); err != nil { - log.Println(err) - } - return fmt.Errorf("failed writing tar entry: %w", err) - } - if err := src.Close(); err != nil { - log.Println(err) - } - } - if err := filepath.Walk(srcDir, func(fpath string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if fpath == srcDir { - return nil - } - - linkTgt := "" - if fi.Mode()&os.ModeSymlink != 0 { - log.Printf("reading link '%s'", fpath) - linkTgt, err = os.Readlink(fpath) - if err != nil { - return err - } - linkTgt = fmt.Sprintf("grafana-%s/%s", version, linkTgt) - } - - hdr, err := tar.FileInfoHeader(fi, linkTgt) - if err != nil { - return fmt.Errorf("failed getting tar header: %w", err) - } - hdr.Name = fmt.Sprintf("grafana-%s/%s", version, strings.TrimPrefix(fpath, fmt.Sprintf("%s/", srcDir))) - if err := tarWriter.WriteHeader(hdr); err != nil { - return fmt.Errorf("failed writing tar header: %w", err) - } - if fi.IsDir() { - return nil - } - - //nolint:gosec - src, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fpath, err) - } - if _, err := io.Copy(tarWriter, src); err != nil { - if err := src.Close(); err != nil { - log.Println(err) - } - return fmt.Errorf("failed writing tar entry: %w", err) - } - if err := src.Close(); err != nil { - log.Println(err) - } - - return nil - }); err != nil { - return fmt.Errorf("failed to traverse directory %q: %w", srcDir, err) - } - - if err := tarWriter.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - if err := gzWriter.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - if err := tgt.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - - st, err := os.Stat(fpath) - if err != nil { - return err - } - perms := st.Mode() & os.ModePerm - log.Printf("Successfully created package %q (permissions: %o)", fpath, perms) - - return nil -} diff --git a/pkg/build/packaging/grafana_test.go b/pkg/build/packaging/grafana_test.go deleted file mode 100644 index 7f45a632718..00000000000 --- a/pkg/build/packaging/grafana_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package packaging_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/packaging" -) - -func TestPackageRegexp(t *testing.T) { - t.Run("It should match enterprise2 packages", func(t *testing.T) { - rgx := packaging.PackageRegexp(config.EditionEnterprise2) - matches := []string{ - "grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz", - "grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz.sha256", - } - for _, v := range matches { - assert.Truef(t, rgx.MatchString(v), "'%s' should match regex '%s'", v, rgx.String()) - } - }) -} diff --git a/pkg/build/plugins/build.go b/pkg/build/plugins/build.go deleted file mode 100644 index da9a23cd5d1..00000000000 --- a/pkg/build/plugins/build.go +++ /dev/null @@ -1,66 +0,0 @@ -package plugins - -import ( - "context" - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/fsutil" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -type PluginSigningMode = int - -// BuildPlugins builds internal plugins. -// The built plugins are placed in plugins-bundled/dist/. -func Build(ctx context.Context, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group, verMode *config.BuildConfig) error { - log.Printf("Building plugins in %q...", grafanaDir) - - root := filepath.Join(grafanaDir, "plugins-bundled", "internal") - fis, err := os.ReadDir(root) - if err != nil { - return err - } - - for i := range fis { - fi := fis[i] - if !fi.IsDir() { - continue - } - - dpath := filepath.Join(root, fi.Name()) - - p.Schedule(g.Wrap(func() error { - log.Printf("Building plugin %q...", dpath) - - cmd := exec.Command("yarn", "build") - cmd.Dir = dpath - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("yarn build failed: %s", output) - } - - dstPath := filepath.Join("plugins-bundled", "dist", fi.Name()) - if err := fsutil.CopyRecursive(filepath.Join(dpath, "dist"), dstPath); err != nil { - return err - } - if !verMode.PluginSignature.Sign { - return nil - } - - return BuildManifest(ctx, dstPath, verMode.PluginSignature.AdminSign) - })) - } - - if err := g.Wait(); err != nil { - return err - } - - log.Printf("Built all plug-ins successfully!") - - return nil -} diff --git a/pkg/build/plugins/download.go b/pkg/build/plugins/download.go deleted file mode 100644 index ed8f00bf911..00000000000 --- a/pkg/build/plugins/download.go +++ /dev/null @@ -1,118 +0,0 @@ -package plugins - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package. -func logCloseError(closeFunc func() error) { - if err := closeFunc(); err != nil { - log.Println(err) - } -} - -// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package. -func logError(err error) { - if err != nil { - log.Println(err) - } -} - -// pluginManifest has details of an external plugin package. -type pluginManifest struct { - Name string `json:"name"` - Version string `json:"version"` - Checksum string `json:"checksum"` -} - -// pluginsManifest represents a manifest of Grafana's external plugins. -type pluginsManifest struct { - Plugins []pluginManifest `json:"plugins"` -} - -// downloadPlugins downloads Grafana plugins that should be bundled into packages. -// -// The plugin archives are downloaded into /plugins-bundled. -func Download(ctx context.Context, grafanaDir string, p syncutil.WorkerPool) error { - g, _ := errutil.GroupWithContext(ctx) - - log.Println("Downloading external plugins...") - - var m pluginsManifest - manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json") - //nolint:gosec - manifestB, err := os.ReadFile(manifestPath) - if err != nil { - return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err) - } - if err := json.Unmarshal(manifestB, &m); err != nil { - return err - } - - for i := range m.Plugins { - pm := m.Plugins[i] - p.Schedule(g.Wrap(func() error { - tgt := filepath.Join(grafanaDir, "plugins-bundled", fmt.Sprintf("%s-%s.zip", pm.Name, pm.Version)) - //nolint:gosec - out, err := os.Create(tgt) - if err != nil { - return err - } - defer logCloseError(out.Close) - - u := fmt.Sprintf("http://storage.googleapis.com/plugins-ci/plugins/%s/%s-%s.zip", pm.Name, pm.Name, - pm.Version) - log.Printf("Downloading plugin %q to %q...", u, tgt) - // nolint:gosec - resp, err := http.Get(u) - if err != nil { - return fmt.Errorf("downloading %q failed: %w", u, err) - } - defer logError(resp.Body.Close()) - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("failed to download %q, status code %d", u, resp.StatusCode) - } - - if _, err := io.Copy(out, resp.Body); err != nil { - return fmt.Errorf("downloading %q failed: %w", u, err) - } - if err := out.Close(); err != nil { - return fmt.Errorf("downloading %q failed: %w", u, err) - } - - //nolint:gosec - fd, err := os.Open(tgt) - if err != nil { - return err - } - defer logCloseError(fd.Close) - - h := sha256.New() - if _, err := io.Copy(h, fd); err != nil { - return err - } - - chksum := hex.EncodeToString(h.Sum(nil)) - if chksum != pm.Checksum { - return fmt.Errorf("plugin %q has bad checksum: %s (expected %s)", u, chksum, pm.Checksum) - } - - return Unzip(tgt, filepath.Join(grafanaDir, "plugins-bundled")) - })) - } - - return g.Wait() -} diff --git a/pkg/build/plugins/manifest.go b/pkg/build/plugins/manifest.go deleted file mode 100644 index 359e8ad77bc..00000000000 --- a/pkg/build/plugins/manifest.go +++ /dev/null @@ -1,204 +0,0 @@ -package plugins - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" -) - -type manifest struct { - Plugin string `json:"plugin"` - Version string `json:"version"` - Files map[string]string `json:"files"` -} - -func getManifest(dpath string, chksums map[string]string) (manifest, error) { - m := manifest{} - - type pluginInfo struct { - Version string `json:"version"` - } - - type plugin struct { - ID string `json:"id"` - Info pluginInfo `json:"info"` - } - - //nolint:gosec - f, err := os.Open(filepath.Join(dpath, "plugin.json")) - if err != nil { - return m, err - } - decoder := json.NewDecoder(f) - var p plugin - if err := decoder.Decode(&p); err != nil { - return m, err - } - - if p.ID == "" { - return m, fmt.Errorf("plugin.json doesn't define id") - } - if p.Info.Version == "" { - return m, fmt.Errorf("plugin.json doesn't define info.version") - } - - return manifest{ - Plugin: p.ID, - Version: p.Info.Version, - Files: chksums, - }, nil -} - -// BuildManifest requests a plugin's signed manifest file fromt he Grafana API. -// If signingAdmin is true, the manifest signing admin endpoint (without plugin ID) will be used, and requires -// an admin API key. -func BuildManifest(ctx context.Context, dpath string, signingAdmin bool) error { - log.Printf("Building manifest for plug-in at %q", dpath) - - apiKey := os.Getenv("GRAFANA_API_KEY") - if apiKey == "" { - return fmt.Errorf("GRAFANA_API_KEY must be set") - } - - manifestPath := filepath.Join(dpath, "MANIFEST.txt") - chksums, err := getChksums(dpath, manifestPath) - if err != nil { - return err - } - m, err := getManifest(dpath, chksums) - if err != nil { - return err - } - - b := bytes.NewBuffer(nil) - encoder := json.NewEncoder(b) - if err := encoder.Encode(&m); err != nil { - return err - } - jsonB := b.Bytes() - u := "https://grafana.com/api/plugins/ci/sign" - if !signingAdmin { - u = fmt.Sprintf("https://grafana.com/api/plugins/%s/ci/sign", m.Plugin) - } - log.Printf("Requesting signed manifest from Grafana API...") - req, err := http.NewRequestWithContext(ctx, "POST", u, bytes.NewReader(jsonB)) - if err != nil { - return err - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - req.Header.Add("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("failed to get signed manifest from Grafana API: %w", err) - } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Println("failed to close response body, err: %w", err) - } - }() - if resp.StatusCode != 200 { - msg, err := io.ReadAll(resp.Body) - if err != nil { - log.Printf("Failed to read response body: %s", err) - msg = []byte("") - } - return fmt.Errorf("request for signed manifest failed with status code %d: %s", resp.StatusCode, string(msg)) - } - - log.Printf("Successfully signed manifest via Grafana API, writing to %q", manifestPath) - //nolint:gosec - f, err := os.Create(manifestPath) - if err != nil { - return fmt.Errorf("failed to create %s: %w", manifestPath, err) - } - defer func() { - if err := f.Close(); err != nil { - log.Println("failed to close file, err: %w", err) - } - }() - if _, err := io.Copy(f, resp.Body); err != nil { - return fmt.Errorf("failed to write %s: %w", manifestPath, err) - } - if err := f.Close(); err != nil { - return fmt.Errorf("failed to write %s: %w", manifestPath, err) - } - - return nil -} - -func getChksums(dpath, manifestPath string) (map[string]string, error) { - manifestPath = filepath.Clean(manifestPath) - - chksums := map[string]string{} - if err := filepath.Walk(dpath, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - - if fi.IsDir() { - return nil - } - - path = filepath.Clean(path) - - // Handle symbolic links - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - finalPath, err := filepath.EvalSymlinks(path) - if err != nil { - return err - } - - log.Printf("Handling symlink %q, pointing to %q", path, finalPath) - - info, err := os.Stat(finalPath) - if err != nil { - return err - } - if info.IsDir() { - return nil - } - - if _, err := filepath.Rel(dpath, finalPath); err != nil { - return fmt.Errorf("symbolic link %q targets a file outside of the plugin directory: %q", path, finalPath) - } - - if finalPath == manifestPath { - return nil - } - } - - if path == manifestPath { - return nil - } - - h := sha256.New() - //nolint:gosec - f, err := os.Open(path) - if err != nil { - return err - } - defer logCloseError(f.Close) - if _, err := io.Copy(h, f); err != nil { - return err - } - - relPath, err := filepath.Rel(dpath, path) - if err != nil { - return err - } - chksums[relPath] = fmt.Sprintf("%x", h.Sum(nil)) - - return nil - }); err != nil { - return nil, err - } - - return chksums, nil -} diff --git a/pkg/build/plugins/zip.go b/pkg/build/plugins/zip.go deleted file mode 100644 index 73f8e8d82f6..00000000000 --- a/pkg/build/plugins/zip.go +++ /dev/null @@ -1,64 +0,0 @@ -package plugins - -import ( - "archive/zip" - "io" - "log" - "os" - "path/filepath" -) - -// Unzip unzips a plugin. -func Unzip(fpath, tgtDir string) error { - log.Printf("Unzipping plugin %q into %q...", fpath, tgtDir) - - r, err := zip.OpenReader(fpath) - if err != nil { - return err - } - defer logCloseError(r.Close) - - // Closure to address file descriptors issue with all the deferred .Close() methods - extractAndWriteFile := func(f *zip.File) error { - log.Printf("Extracting zip member %q...", f.Name) - - rc, err := f.Open() - if err != nil { - return err - } - defer logCloseError(rc.Close) - - //nolint:gosec - dstPath := filepath.Join(tgtDir, f.Name) - - if f.FileInfo().IsDir() { - return os.MkdirAll(dstPath, f.Mode()) - } - - if err := os.MkdirAll(filepath.Dir(dstPath), f.Mode()); err != nil { - return err - } - - //nolint:gosec - fd, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - defer logCloseError(fd.Close) - - // nolint:gosec - if _, err := io.Copy(fd, rc); err != nil { - return err - } - - return fd.Close() - } - - for _, f := range r.File { - if err := extractAndWriteFile(f); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/build/stringutil/contains.go b/pkg/build/stringutil/contains.go deleted file mode 100644 index b53efe70759..00000000000 --- a/pkg/build/stringutil/contains.go +++ /dev/null @@ -1,10 +0,0 @@ -package stringutil - -func Contains(arr []string, s string) bool { - for _, e := range arr { - if e == s { - return true - } - } - return false -} diff --git a/pkg/build/syncutil/pool.go b/pkg/build/syncutil/pool.go deleted file mode 100644 index 6034059d2bf..00000000000 --- a/pkg/build/syncutil/pool.go +++ /dev/null @@ -1,43 +0,0 @@ -package syncutil - -import ( - "log" - "runtime" -) - -func worker(jobs chan func()) { - for j := range jobs { - j() - } -} - -// WorkerPool represents a concurrent worker pool. -type WorkerPool struct { - NumWorkers int - jobs chan func() -} - -// NewWorkerPool constructs a new WorkerPool. -func NewWorkerPool(numWorkers int) WorkerPool { - if numWorkers <= 0 { - numWorkers = runtime.NumCPU() - } - log.Printf("Creating worker pool with %d workers", numWorkers) - jobs := make(chan func(), 100) - for i := 0; i < numWorkers; i++ { - go worker(jobs) - } - return WorkerPool{ - NumWorkers: numWorkers, - jobs: jobs, - } -} - -// Schedule schedules a job to be executed by a worker in the pool. -func (p WorkerPool) Schedule(job func()) { - p.jobs <- job -} - -func (p WorkerPool) Close() { - close(p.jobs) -} diff --git a/pkg/build/validation/validation.go b/pkg/build/validation/validation.go index 17951261114..bc01ed189b7 100644 --- a/pkg/build/validation/validation.go +++ b/pkg/build/validation/validation.go @@ -1,9 +1,5 @@ package validation -import ( - "context" -) - type ArtifactType int const ( @@ -15,13 +11,3 @@ type Artifact struct { Type ArtifactType URL string } - -// ReleaseArtifacts generates a list of release artifacts -func ReleaseArtifacts(version string) ([]Artifact, error) { - return nil, nil -} - -// VerifyRelease tests that a that, given the information, a release will completed wholly and successfully. -func VerifyRelease(ctx context.Context, version string) (bool, error) { - return false, nil -} diff --git a/pkg/build/versions/parse.go b/pkg/build/versions/parse.go new file mode 100644 index 00000000000..0a38af4329c --- /dev/null +++ b/pkg/build/versions/parse.go @@ -0,0 +1,31 @@ +package versions + +import "regexp" + +var semverRegex = regexp.MustCompile(`^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) + +type Semver struct { + Major string + Minor string + Patch string + Prerelease string + BuildMetadata string +} + +func ParseSemver(version string) Semver { + matches := semverRegex.FindStringSubmatch(version) + results := make(map[string]string) + for i, name := range semverRegex.SubexpNames() { + if i != 0 && name != "" { + results[name] = matches[i] + } + } + + return Semver{ + Major: results["major"], + Minor: results["minor"], + Patch: results["patch"], + Prerelease: results["prerelease"], + BuildMetadata: results["buildmetadata"], + } +} diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 9f7a5563e4a..22819a28ec8 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -112,22 +112,7 @@ def publish_artifacts_step(): "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), }, "commands": [ - "./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET}", - ], - "depends_on": ["compile-build-cmd"], - } - -def publish_static_assets_step(): - return { - "name": "publish-static-assets", - "image": images["publish"], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), - "STATIC_ASSET_EDITIONS": from_secret("static_asset_editions"), - }, - "commands": [ - "./bin/build artifacts static-assets --tag ${DRONE_TAG} --static-asset-editions=grafana-oss", + "./bin/build artifacts packages --artifacts-editions=oss --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET}", ], "depends_on": ["compile-build-cmd"], } @@ -163,9 +148,8 @@ def publish_artifacts_pipelines(mode): steps = [ compile_build_cmd(), publish_artifacts_step(), - publish_static_assets_step(), publish_storybook_step(), - release_pr_step(depends_on = ["publish-artifacts", "publish-static-assets"]), + release_pr_step(depends_on = ["publish-artifacts"]), ] return [ diff --git a/scripts/drone/pipelines/publish_images.star b/scripts/drone/pipelines/publish_images.star index 6fbcb7c475c..9ed847a22fb 100644 --- a/scripts/drone/pipelines/publish_images.star +++ b/scripts/drone/pipelines/publish_images.star @@ -31,43 +31,44 @@ def publish_image_public_step(): """ command = """ bash -c ' + IMAGE_TAG=$(echo "$${TAG}" | sed -e "s/+/-/g") debug= if [[ -n $${DRY_RUN} ]]; then debug=echo; fi docker login -u $${DOCKER_USER} -p $${DOCKER_PASSWORD} # Push the grafana-image-tags images - $$debug docker push grafana/grafana-image-tags:$${TAG}-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Create the grafana manifests $$debug docker manifest create grafana/grafana:${TAG} \ - grafana/grafana-image-tags:$${TAG}-amd64 \ - grafana/grafana-image-tags:$${TAG}-arm64 \ - grafana/grafana-image-tags:$${TAG}-armv7 + grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 \ + grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 \ + grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 $$debug docker manifest create grafana/grafana:${TAG}-ubuntu \ - grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 \ - grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 \ - grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 \ + grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 \ + grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Push the grafana manifests - $$debug docker manifest push grafana/grafana:$${TAG} - $$debug docker manifest push grafana/grafana:$${TAG}-ubuntu + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG} + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG}-ubuntu # if LATEST is set, then also create & push latest if [[ -n $${LATEST} ]]; then $$debug docker manifest create grafana/grafana:latest \ - grafana/grafana-image-tags:$${TAG}-amd64 \ - grafana/grafana-image-tags:$${TAG}-arm64 \ - grafana/grafana-image-tags:$${TAG}-armv7 + grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 \ + grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 \ + grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 $$debug docker manifest create grafana/grafana:latest-ubuntu \ - grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 \ - grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 \ - grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 \ + grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 \ + grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 $$debug docker manifest push grafana/grafana:latest $$debug docker manifest push grafana/grafana:latest-ubuntu diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index 7f56c553976..4b00a9314d9 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -2,7 +2,6 @@ This module returns a Drone step and pipeline for linting with shellcheck. """ -load("scripts/drone/steps/lib.star", "compile_build_cmd") load( "scripts/drone/utils/images.star", "images", @@ -39,7 +38,6 @@ def shellcheck_step(): def shellcheck_pipeline(): environment = {"EDITION": "oss"} steps = [ - compile_build_cmd(), shellcheck_step(), ] return pipeline( diff --git a/scripts/drone/pipelines/verify_starlark.star b/scripts/drone/pipelines/verify_starlark.star index 9e058893d7b..e939cd598ef 100644 --- a/scripts/drone/pipelines/verify_starlark.star +++ b/scripts/drone/pipelines/verify_starlark.star @@ -4,7 +4,6 @@ This module returns a Drone pipeline that verifies all Starlark files are linted load( "scripts/drone/steps/lib.star", - "compile_build_cmd", "identify_runner_step", "lint_starlark_step", ) @@ -17,7 +16,6 @@ def verify_starlark(trigger, ver_mode): environment = {"EDITION": "oss"} steps = [ identify_runner_step(), - compile_build_cmd(), lint_starlark_step(), ] return pipeline( diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 500b3ab0305..903d23bfb02 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -156,9 +156,7 @@ def lint_starlark_step(): "go install github.com/bazelbuild/buildtools/buildifier@latest", "buildifier --lint=warn -mode=check -r .", ], - "depends_on": [ - "compile-build-cmd", - ], + "depends_on": [], } def enterprise_downstream_step(ver_mode):