From 185d1a1530003a3fd3d84f6a9328a1fbf1935eb1 Mon Sep 17 00:00:00 2001 From: Mariell Hoversholm Date: Wed, 11 Jun 2025 08:43:06 +0200 Subject: [PATCH] E2E: Rework the runner (#105712) --- .github/CODEOWNERS | 1 - .github/workflows/pr-e2e-tests.yml | 117 +++++-- .github/workflows/run-e2e-suite.yml | 39 --- .github/zizmor.yml | 1 + e2e/main.go | 351 +++++++++++++++++++++ e2e/run-suite | 6 +- go.mod | 1 + go.sum | 2 + pkg/build/daggerbuild/arguments/grafana.go | 6 +- pkg/build/e2e/main.go | 150 +++++++-- pkg/build/e2e/run.go | 15 +- pkg/build/e2e/service.go | 40 ++- pkg/build/go.mod | 1 + pkg/build/go.sum | 2 + 14 files changed, 616 insertions(+), 116 deletions(-) delete mode 100644 .github/workflows/run-e2e-suite.yml create mode 100644 e2e/main.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7e90f5e6e6a..1c198645369 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -836,7 +836,6 @@ embed.go @grafana/grafana-as-code /.github/workflows/frontend-lint.yml @grafana/grafana-frontend-platform /.github/workflows/analytics-events-report.yml @grafana/grafana-frontend-platform /.github/workflows/pr-e2e-tests.yml @grafana/grafana-developer-enablement-squad -/.github/workflows/run-e2e-suite.yml @grafana/grafana-developer-enablement-squad /.github/workflows/skye-add-to-project.yml @grafana/grafana-frontend-platform /.github/zizmor.yml @grafana/grafana-developer-enablement-squad /.github/license_finder.yaml @bergquist diff --git a/.github/workflows/pr-e2e-tests.yml b/.github/workflows/pr-e2e-tests.yml index c88813150c5..dc7381d2cfc 100644 --- a/.github/workflows/pr-e2e-tests.yml +++ b/.github/workflows/pr-e2e-tests.yml @@ -39,39 +39,98 @@ jobs: retention-days: 1 name: ${{ steps.artifact.outputs.artifact }} path: grafana.tar.gz - e2e-matrix: + + build-e2e-runner: + name: Build E2E test runner + runs-on: ubuntu-latest permissions: contents: read - name: ${{ matrix.suite }} - strategy: - fail-fast: false - matrix: - suite: - - various-suite - - dashboards-suite - - smoke-tests-suite - - panels-suite + outputs: + artifact: ${{ steps.artifact.outputs.artifact }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: ${{ !github.event.pull_request.head.repo.fork }} + - name: Build E2E test runner + id: artifact + run: | + # We want a static binary, so we need to set CGO_ENABLED=0 + CGO_ENABLED=0 go build -o ./e2e-runner ./e2e/ + echo "artifact=e2e-runner-${{github.run_number}}" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@v4 + id: upload + with: + retention-days: 1 + name: ${{ steps.artifact.outputs.artifact }} + path: e2e-runner + + run-e2e-tests: needs: - build-grafana - uses: ./.github/workflows/run-e2e-suite.yml - with: - package: ${{ needs.build-grafana.outputs.artifact }} - suite: ${{ matrix.suite }} - e2e-matrix-old-arch: - permissions: - contents: read - name: ${{ matrix.suite }} (old arch) + - build-e2e-runner strategy: fail-fast: false matrix: - suite: - - old-arch/various-suite - - old-arch/dashboards-suite - - old-arch/smoke-tests-suite - - old-arch/panels-suite - needs: - - build-grafana - uses: ./.github/workflows/run-e2e-suite.yml - with: - package: ${{ needs.build-grafana.outputs.artifact }} - suite: ${{ matrix.suite }} + include: + - suite: various-suite + path: e2e/various-suite + - suite: dashboards-suite + path: e2e/dashboards-suite + - suite: smoke-tests-suite + path: e2e/smoke-tests-suite + - suite: panels-suite + path: e2e/panels-suite + - suite: various-suite (old arch) + path: e2e/old-arch/various-suite + flags: --flags="--env dashboardScene=false" + - suite: dashboards-suite (old arch) + path: e2e/old-arch/dashboards-suite + flags: --flags="--env dashboardScene=false" + - suite: smoke-tests-suite (old arch) + path: e2e/old-arch/smoke-tests-suite + flags: --flags="--env dashboardScene=false" + - suite: panels-suite (old arch) + path: e2e/old-arch/panels-suite + flags: --flags="--env dashboardScene=false" + name: ${{ matrix.suite }} + runs-on: ubuntu-latest-8-cores + permissions: + contents: read + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-grafana.outputs.artifact }} + - uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-e2e-runner.outputs.artifact }} + - name: chmod +x + run: chmod +x ./e2e-runner + - name: Run E2E tests + uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e + with: + verb: run + args: go run ./pkg/build/e2e --package=grafana.tar.gz + --suite=${{ matrix.path }} + ${{ matrix.flags }} + - name: Set suite name + id: set-suite-name + if: success() || failure() + env: + SUITE: ${{ matrix.path }} + run: | + echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: ${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }} + path: videos + retention-days: 1 diff --git a/.github/workflows/run-e2e-suite.yml b/.github/workflows/run-e2e-suite.yml deleted file mode 100644 index 0aa6d292abc..00000000000 --- a/.github/workflows/run-e2e-suite.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: e2e suite - -on: - workflow_call: - inputs: - package: - type: string - required: true - suite: - type: string - required: true - -jobs: - main: - runs-on: ubuntu-latest-8-cores - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package }} - - uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e - with: - verb: run - args: go run ./pkg/build/e2e --package=grafana.tar.gz --suite=${{ inputs.suite }} - - name: Set suite name - id: set-suite-name - if: success() || failure() - env: - SUITE: ${{ inputs.suite }} - run: | - echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" - - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: e2e-${{ steps.set-suite-name.outputs.suite }}-${{github.run_number}} - path: videos - retention-days: 1 diff --git a/.github/zizmor.yml b/.github/zizmor.yml index eac536c3db5..92fe26bf7b4 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -21,6 +21,7 @@ rules: - pr-frontend-unit-tests.yml - pr-test-integration.yml - publish-kinds-release.yml + - pr-e2e-tests.yml dangerous-triggers: ignore: - auto-milestone.yml diff --git a/e2e/main.go b/e2e/main.go new file mode 100644 index 00000000000..130d8d85426 --- /dev/null +++ b/e2e/main.go @@ -0,0 +1,351 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/urfave/cli/v3" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + if err := Run().Run(ctx, os.Args); err != nil { + cancel() + fmt.Println(err) + os.Exit(1) + } +} + +func Run() *cli.Command { + var suitePath string + + return &cli.Command{ + Name: "e2e", + Usage: "Run the test suite", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "command", + Usage: "Cypress command to run. 'open' can be useful for development (enum: run, open)", + Value: "run", + Validator: func(s string) error { + if s != "run" && s != "open" { + return fmt.Errorf("invalid command: %s, must be 'run' or 'open'", s) + } + return nil + }, + }, + &cli.StringFlag{ + Name: "browser", + Usage: "Browser to run tests with (e.g.: chrome, electron)", + Value: "chrome", + }, + &cli.StringFlag{ + Name: "grafana-base-url", + Usage: "Base URL for Grafana", + Value: "http://localhost:3001", + }, + &cli.BoolFlag{ + Name: "cypress-video", + Usage: "Enable Cypress video recordings", + Value: false, + }, + &cli.BoolFlag{ + Name: "smtp-plugin", + Usage: "Enable SMTP plugin", + Value: false, + }, + &cli.BoolFlag{ + Name: "benchmark-plugin", + Usage: "Enable Benchmark plugin", + Value: false, + }, + &cli.BoolFlag{ + Name: "slowmo", + Usage: "Slow down the test run", + Value: false, + }, + &cli.StringSliceFlag{ + Name: "env", + Usage: "Additional Cypress environment variables to set (format: KEY=VALUE)", + Validator: func(s []string) error { + pattern := regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*=.*`) + for _, v := range s { + if !pattern.MatchString(v) { + return fmt.Errorf("invalid environment variable format: %s, must be KEY=VALUE", v) + } + } + return nil + }, + }, + &cli.StringSliceFlag{ + Name: "parameters", + Usage: "Additional parameters to pass to the Cypress command (e.g. --headed)", + }, + &cli.DurationFlag{ + Name: "timeout", + Usage: "Timeout for the Cypress command (precision: milliseconds)", + Value: time.Second * 30, + Validator: func(d time.Duration) error { + if d < 0 { + return fmt.Errorf("timeout must be a positive duration") + } + if d.Round(time.Millisecond) != d { + return fmt.Errorf("timeout must be a whole number of milliseconds") + } + return nil + }, + }, + &cli.StringFlag{ + Name: "timezone", + Usage: "Timezone to set for the Cypress run (e.g. 'America/New_York')", + Value: "Pacific/Honolulu", + }, + + &cli.BoolFlag{ + Name: "start-grafana", + Usage: "Start and wait for Grafana before running the tests", + Value: true, + Category: "Grafana Server", + }, + &cli.StringFlag{ + Name: "license-path", + Usage: "Path to the Grafana Enterprise license file (optional; requires --start-grafana)", + Value: "", + TakesFile: true, + Category: "Grafana Server", + }, + &cli.BoolFlag{ + Name: "image-renderer", + Usage: "Install the image renderer plugin (requires --start-grafana)", + Category: "Grafana Server", + }, + + &cli.StringFlag{ + Name: "suite", + Usage: "Path to the suite to run (e.g. './e2e/dashboards-suite')", + TakesFile: true, + Required: true, + Destination: &suitePath, + }, + }, + Action: runAction, + } +} + +func runAction(ctx context.Context, c *cli.Command) error { + suitePath := c.String("suite") + suitePath, err := normalisePath(suitePath) + if err != nil { + return fmt.Errorf("failed to normalise suite path: %w", err) + } + + repoRoot, err := gitRepoRoot(ctx, suitePath) + if err != nil { + return fmt.Errorf("failed to get git repo root: %w", err) + } + + screenshotsFolder := path.Join(suitePath, "screenshots") + videosFolder := path.Join(suitePath, "videos") + fileServerFolder := path.Join(repoRoot, "e2e", "cypress") + fixturesFolder := path.Join(fileServerFolder, "fixtures") + downloadsFolder := path.Join(fileServerFolder, "downloads") + benchmarkPluginResultsFolder := path.Join(suitePath, "benchmark-results") + reporter := path.Join(repoRoot, "e2e", "log-reporter.js") + + env := map[string]string{ + "BENCHMARK_PLUGIN_ENABLED": fmt.Sprintf("%t", c.Bool("benchmark-plugin")), + "SMTP_PLUGIN_ENABLED": fmt.Sprintf("%t", c.Bool("smtp-plugin")), + "BENCHMARK_PLUGIN_RESULTS_FOLDER": benchmarkPluginResultsFolder, + "SLOWMO": "0", + "BASE_URL": c.String("grafana-base-url"), + } + for _, v := range c.StringSlice("env") { + parts := strings.SplitN(v, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid environment variable format: %s, must be KEY=VALUE", v) + } + env[parts[0]] = parts[1] + } + + cypressConfig := map[string]string{ + "screenshotsFolder": screenshotsFolder, + "fixturesFolder": fixturesFolder, + "videosFolder": videosFolder, + "downloadsFolder": downloadsFolder, + "fileServerFolder": fileServerFolder, + "reporter": reporter, + + "specPattern": path.Join(suitePath, "*.spec.ts"), + "defaultCommandTimeout": fmt.Sprintf("%d", c.Duration("timeout").Milliseconds()), + "viewportWidth": "1920", + "viewportHeight": "1080", + "trashAssetsBeforeRuns": "false", + "baseUrl": c.String("grafana-base-url"), + "video": fmt.Sprintf("%t", c.Bool("cypress-video")), + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if c.Bool("start-grafana") { + startServerPath := path.Join(repoRoot, "scripts", "grafana-server", "start-server") + waitForGrafanaPath := path.Join(repoRoot, "scripts", "grafana-server", "wait-for-grafana") + go func() { + var args []string + if c.String("license-path") != "" { + args = append(args, c.String("license-path")) + } + //nolint:gosec + cmd := exec.CommandContext(ctx, startServerPath, args...) + cmd.Dir = repoRoot + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("TZ=%s", c.String("timezone"))) + if c.Bool("image-renderer") { + cmd.Env = append(cmd.Env, "INSTALL_IMAGE_RENDERER=true") + } + cmd.Stdout = prefixGrafana(os.Stdout) + cmd.Stderr = prefixGrafana(os.Stderr) + cmd.Stdin = nil + + if err := cmd.Run(); err != nil { + fmt.Println("Error running Grafana:", err) + } + }() + + //nolint:gosec + cmd := exec.CommandContext(ctx, waitForGrafanaPath) + cmd.Dir = repoRoot + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("TZ=%s", c.String("timezone"))) + cmd.Stdout = prefixGrafana(os.Stdout) + cmd.Stderr = prefixGrafana(os.Stderr) + cmd.Stdin = nil + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to wait for Grafana: %w", err) + } + } + + args := []string{"run", "cypress", c.String("command"), + "--env", joinCypressCfg(env), + "--config", joinCypressCfg(cypressConfig), + "--browser", c.String("browser")} + args = append(args, c.StringSlice("parameters")...) + //nolint:gosec + cmd := exec.CommandContext(ctx, "yarn", args...) + cmd.Dir = repoRoot + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("TZ=%s", c.String("timezone"))) + cmd.Stdout = prefixCypress(os.Stdout) + cmd.Stderr = prefixCypress(os.Stderr) + cmd.Stdin = os.Stdin + return cmd.Run() +} + +func gitRepoRoot(ctx context.Context, dir string) (string, error) { + cmd := exec.CommandContext(ctx, "git", "rev-parse", "--show-toplevel") + cmd.Dir = dir + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get git repo root: %w", err) + } + p := strings.TrimSpace(string(out)) + p, err = normalisePath(p) + if err != nil { + return "", fmt.Errorf("failed to normalise git repo root path: %w", err) + } + return p, nil +} + +func normalisePath(p string) (string, error) { + absPath, err := filepath.Abs(p) + if err != nil { + return "", fmt.Errorf("failed to get absolute path: %w", err) + } + return path.Clean(filepath.ToSlash(absPath)), nil +} + +func joinCypressCfg(cfg map[string]string) string { + config := make([]string, 0, len(cfg)) + for k, v := range cfg { + config = append(config, fmt.Sprintf("%s=%s", k, v)) + } + return strings.Join(config, ",") +} + +const ( + resetColor = "\033[0m" + yellowColor = "\033[0;33m" + cyanColor = "\033[0;36m" +) + +func prefixCypress(w io.Writer) io.Writer { + if _, ok := os.LookupEnv("CI"); ok { + return w + } + + return newWrappingOutput(cyanColor+"Cypress: ", resetColor, w) +} + +func prefixGrafana(w io.Writer) io.Writer { + if _, ok := os.LookupEnv("CI"); ok { + return w + } + + return newWrappingOutput(yellowColor+"Grafana: ", resetColor, w) +} + +var _ io.Writer = (*wrappingOutput)(nil) + +type wrappingOutput struct { + prefix string + suffix string + mu *sync.Mutex + inner io.Writer + writtenPrefix bool +} + +func newWrappingOutput(prefix, suffix string, inner io.Writer) *wrappingOutput { + return &wrappingOutput{ + prefix: prefix, + suffix: suffix, + mu: &sync.Mutex{}, + inner: inner, + } +} + +func (p *wrappingOutput) Write(b []byte) (int, error) { + p.mu.Lock() + defer p.mu.Unlock() + + for line := range bytes.Lines(b) { + if !p.writtenPrefix { + if _, err := p.inner.Write([]byte(p.prefix)); err != nil { + return 0, err + } + p.writtenPrefix = true + } + if _, err := p.inner.Write(line); err != nil { + return 0, err + } + if bytes.HasSuffix(line, []byte("\n")) { + p.writtenPrefix = false + if _, err := p.inner.Write([]byte(p.suffix)); err != nil { + return 0, err + } + } + } + return len(b), nil +} diff --git a/e2e/run-suite b/e2e/run-suite index f4be6852580..386fbae4f0e 100755 --- a/e2e/run-suite +++ b/e2e/run-suite @@ -26,7 +26,7 @@ declare -A env=( ) testFilesForSingleSuite="*.spec.ts" -rootForEnterpriseSuite="./e2e/extensions-suite" +rootForEnterpriseSuite="./e2e/extensions" rootForOldArch="./e2e/old-arch" rootForKubernetesDashboards="./e2e/dashboards-suite" rootForSearchDashboards="./e2e/dashboards-search-suite" @@ -45,6 +45,7 @@ declare -A cypressConfig=( [trashAssetsBeforeRuns]=false [reporter]=./e2e/log-reporter.js [baseUrl]=${BASE_URL:-"http://$HOST:$PORT"} + [video]=${CYPRESS_VIDEO:-false} ) case "$1" in @@ -70,8 +71,6 @@ case "$1" in "enterprise") echo "Enterprise" env[SMTP_PLUGIN_ENABLED]=true - CLEANUP="rm -rf ./e2e/extensions-suite" - SETUP="cp -Lr ./e2e/extensions ./e2e/extensions-suite" enterpriseSuite=$(basename "${args[1]}") case "$2" in "debug") @@ -87,7 +86,6 @@ case "$1" in ;; esac cypressConfig[specPattern]=$rootForEnterpriseSuite/$enterpriseSuite/*-suite/*.spec.ts - $CLEANUP && $SETUP ;; "") ;; diff --git a/go.mod b/go.mod index a505e783ece..1d956b329fb 100644 --- a/go.mod +++ b/go.mod @@ -156,6 +156,7 @@ require ( github.com/ua-parser/uap-go v0.0.0-20250213224047-9c035f085b90 // @grafana/grafana-backend-group github.com/urfave/cli v1.22.16 // indirect; @grafana/grafana-backend-group github.com/urfave/cli/v2 v2.27.6 // @grafana/grafana-backend-group + github.com/urfave/cli/v3 v3.3.3 // @grafana/grafana-backend-group github.com/wk8/go-ordered-map v1.0.0 // @grafana/grafana-backend-group github.com/xlab/treeprint v1.2.0 // @grafana/observability-traces-and-profiling github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // @grafana/grafana-operator-experience-squad diff --git a/go.sum b/go.sum index 8933e3b7f92..e8c24ae49bc 100644 --- a/go.sum +++ b/go.sum @@ -2442,6 +2442,8 @@ github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ= github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po= github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g= github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/urfave/cli/v3 v3.3.3 h1:byCBaVdIXuLPIDm5CYZRVG6NvT7tv1ECqdU4YzlEa3I= +github.com/urfave/cli/v3 v3.3.3/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= diff --git a/pkg/build/daggerbuild/arguments/grafana.go b/pkg/build/daggerbuild/arguments/grafana.go index 00bad617e32..90c03751e4b 100644 --- a/pkg/build/daggerbuild/arguments/grafana.go +++ b/pkg/build/daggerbuild/arguments/grafana.go @@ -217,12 +217,12 @@ func enterpriseDirectory(ctx context.Context, opts *pipeline.ArgumentOpts) (any, return nil, fmt.Errorf("error initializing grafana directory: %w", err) } - src, err := cloneOrMount(ctx, opts.Client, o.EnterpriseDir, o.EnterpriseRepo, o.EnterpriseRef, o.GitHubToken) + clone, err := cloneOrMount(ctx, opts.Client, o.EnterpriseDir, o.EnterpriseRepo, o.EnterpriseRef, o.GitHubToken) if err != nil { - return nil, err + return nil, fmt.Errorf("error cloning or mounting Grafana Enterprise directory: %w", err) } - return InitializeEnterprise(opts.Client, grafanaDir.(*dagger.Directory), src), nil + return InitializeEnterprise(opts.Client, grafanaDir.(*dagger.Directory), clone), nil } var GrafanaDirectoryFlags = []cli.Flag{ diff --git a/pkg/build/e2e/main.go b/pkg/build/e2e/main.go index b97369d2331..9bc35151de8 100644 --- a/pkg/build/e2e/main.go +++ b/pkg/build/e2e/main.go @@ -2,71 +2,173 @@ package main import ( "context" - "flag" "fmt" "log" "os" + "os/signal" + "path" "dagger.io/dagger" + "github.com/urfave/cli/v3" ) func main() { - var ( - ctx = context.Background() - grafanaPath = flag.String("grafana-dir", ".", "Path to cloned grafana repo") - targzPath = flag.String("package", "grafana.tar.gz", "Path to grafana tar.gz package") - suite = flag.String("suite", "", "e2e suite name (used in arg to run-suite script)") - ) - flag.Parse() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + if err := NewApp().Run(ctx, os.Args); err != nil { + cancel() + fmt.Println(err) + os.Exit(1) + } +} + +func NewApp() *cli.Command { + return &cli.Command{ + Name: "e2e", + Usage: "Run the E2E tests for Grafana", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "suite", + Usage: "E2E test suite path (e.g. e2e/various-suite)", + Validator: mustBeDir("suite"), + TakesFile: true, + Required: true, + }, + + &cli.StringFlag{ + Name: "grafana-dir", + Usage: "Path to the grafana/grafana clone directory", + Value: ".", + Validator: mustBeDir("grafana-dir"), + TakesFile: true, + }, + &cli.StringFlag{ + Name: "package", + Usage: "Path to the grafana tar.gz package", + Value: "grafana.tar.gz", + Validator: mustBeFile("package", false), + TakesFile: true, + }, + &cli.StringFlag{ + Name: "license", + Usage: "Path to the Grafana Enterprise license file (optional)", + Validator: mustBeFile("license", true), + TakesFile: true, + }, + &cli.StringFlag{ + Name: "flags", + Usage: "Flags to pass through to the e2e runner", + }, + &cli.BoolFlag{ + Name: "image-renderer", + Usage: "Install the image renderer plugin", + Value: false, + }, + }, + Action: run, + } +} + +func run(ctx context.Context, cmd *cli.Command) error { + grafanaDir := cmd.String("grafana-dir") + suite := cmd.String("suite") + targzPath := cmd.String("package") + licensePath := cmd.String("license") + imageRenderer := cmd.Bool("image-renderer") + runnerFlags := cmd.String("flags") d, err := dagger.Connect(ctx) if err != nil { - panic(err) + return fmt.Errorf("failed to connect to Dagger: %w", err) } yarnCache := d.CacheVolume("yarn") - log.Println("grafana dir:", *grafanaPath) - log.Println("targz:", *targzPath) + log.Println("grafana dir:", grafanaDir) + log.Println("targz:", targzPath) + log.Println("license path:", licensePath) grafana := d.Host().Directory(".", dagger.HostDirectoryOpts{ - Exclude: []string{".git", "node_modules", "*.tar.gz"}, + Exclude: []string{"node_modules", "*.tar.gz"}, }) + targz := d.Host().File(targzPath) - targz := d.Host().File("grafana.tar.gz") + var license *dagger.File + if licensePath != "" { + license = d.Host().File(licensePath) + } svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{ - GrafanaDir: grafana, - GrafanaTarGz: targz, - YarnCache: yarnCache, + GrafanaDir: grafana, + GrafanaTarGz: targz, + YarnCache: yarnCache, + License: license, + InstallImageRenderer: imageRenderer, }) if err != nil { - panic(err) + return fmt.Errorf("failed to create Grafana service: %w", err) } - videosDir := fmt.Sprintf("/src/e2e/%s/videos", *suite) + videosDir := path.Join("/src", suite, "videos") // *spec.ts.mp4 - c := RunSuite(d, svc, grafana, yarnCache, *suite) + c := RunSuite(d, svc, grafana, yarnCache, suite, runnerFlags) c, err = c.Sync(ctx) if err != nil { - log.Fatalf("error running dagger: %s", err) + return fmt.Errorf("failed to run e2e test suite: %w", err) } code, err := c.ExitCode(ctx) if err != nil { - log.Fatalf("error getting exit code: %s", err) + return fmt.Errorf("failed to get exit code of e2e test suite: %w", err) } log.Println("exit code:", code) // No sync error; export the videos dir if _, err := c.Directory(videosDir).Export(ctx, "videos"); err != nil { - log.Fatalf("error getting videos: %s", err) + return fmt.Errorf("failed to export videos directory: %w", err) } if code != 0 { - log.Printf("tests failed: exit code %d", code) + return fmt.Errorf("e2e tests failed with exit code %d", code) } - os.Exit(code) + log.Println("e2e tests completed successfully") + return nil +} + +func mustBeFile(arg string, emptyOk bool) func(string) error { + return func(s string) error { + if s == "" { + if emptyOk { + return nil + } + return cli.Exit(arg+" cannot be empty", 1) + } + stat, err := os.Stat(s) + if err != nil { + return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) + } + if stat.IsDir() { + return cli.Exit(arg+" must be a file, not a directory: "+s, 1) + } + return nil + } +} + +func mustBeDir(arg string) func(string) error { + return func(s string) error { + if s == "" { + return cli.Exit(arg+" cannot be empty", 1) + } + stat, err := os.Stat(s) + if err != nil { + return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) + } + if !stat.IsDir() { + return cli.Exit(arg+" must be a directory: "+s, 1) + } + return nil + } } diff --git a/pkg/build/e2e/run.go b/pkg/build/e2e/run.go index d1577576e97..db42d397c38 100644 --- a/pkg/build/e2e/run.go +++ b/pkg/build/e2e/run.go @@ -6,17 +6,14 @@ import ( "dagger.io/dagger" ) -func RunSuite(d *dagger.Client, svc *dagger.Service, src *dagger.Directory, cache *dagger.CacheVolume, suite string) *dagger.Container { +func RunSuite(d *dagger.Client, svc *dagger.Service, src *dagger.Directory, cache *dagger.CacheVolume, suite, runnerFlags string) *dagger.Container { + command := fmt.Sprintf( + "./e2e-runner --start-grafana=false --cypress-video"+ + " --grafana-base-url http://grafana:3001 --suite %s %s", suite, runnerFlags) + return WithYarnCache(WithGrafanaFrontend(d.Container().From("cypress/included:13.1.0"), src), cache). WithWorkdir("/src"). - WithEnvVariable("HOST", "grafana"). - WithEnvVariable("PORT", "3001"). WithServiceBinding("grafana", svc). WithExec([]string{"yarn", "install", "--immutable"}). - WithExec([]string{ - "/bin/bash", "-c", - fmt.Sprintf("./e2e/run-suite %s true", suite), - }, dagger.ContainerWithExecOpts{ - Expect: dagger.ReturnTypeAny, - }) + WithExec([]string{"/bin/bash", "-c", command}, dagger.ContainerWithExecOpts{Expect: dagger.ReturnTypeAny}) } diff --git a/pkg/build/e2e/service.go b/pkg/build/e2e/service.go index 04ddbf10f7b..95de093ec18 100644 --- a/pkg/build/e2e/service.go +++ b/pkg/build/e2e/service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "os" "strings" "dagger.io/dagger" @@ -22,9 +23,11 @@ func NodeImage(version string) string { } type GrafanaServiceOpts struct { - GrafanaDir *dagger.Directory - GrafanaTarGz *dagger.File - YarnCache *dagger.CacheVolume + GrafanaDir *dagger.Directory + GrafanaTarGz *dagger.File + YarnCache *dagger.CacheVolume + License *dagger.File + InstallImageRenderer bool } func Frontend(src *dagger.Directory) *dagger.Directory { @@ -71,8 +74,9 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp WithExec([]string{"yarn", "install", "--immutable"}). WithExec([]string{"yarn", "e2e:plugin:build"}) - svc := d.Container().From("alpine"). - WithExec([]string{"apk", "add", "bash"}). + // _/ubuntu:latest sticks to latest LTS. + // We need ubuntu to support the image renderer plugin, which assumes glibc. + container := d.Container().From("ubuntu:latest"). WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz). WithExec([]string{"mkdir", "-p", "/src/grafana"}). WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}). @@ -84,8 +88,30 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp WithEnvVariable("GF_APP_MODE", "development"). WithEnvVariable("GF_SERVER_HTTP_PORT", "3001"). WithEnvVariable("GF_SERVER_ROUTER_LOGGING", "1"). - WithExposedPort(3001). - AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "-x", "scripts/grafana-server/start-server"}}) + WithExposedPort(3001) + + var licenseArg string + if opts.License != nil { + container = container.WithMountedFile("/src/license.jwt", opts.License) + licenseArg = "/src/license.jwt" + } + + if opts.InstallImageRenderer { + container = container.WithEnvVariable("INSTALL_IMAGE_RENDERER", "true"). + WithExec([]string{"apt-get", "update"}). + WithExec([]string{"apt-get", "install", "-y", "ca-certificates"}) + } + + // We add all GF_ environment variables to allow for overriding Grafana configuration. + // It is unlikely the runner has any such otherwise. + for _, env := range os.Environ() { + if strings.HasPrefix(env, "GF_") { + parts := strings.SplitN(env, "=", 2) + container = container.WithEnvVariable(parts[0], parts[1]) + } + } + + svc := container.AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "-x", "scripts/grafana-server/start-server", licenseArg}}) return svc, nil } diff --git a/pkg/build/go.mod b/pkg/build/go.mod index bcbabd1f502..ace3ff91767 100644 --- a/pkg/build/go.mod +++ b/pkg/build/go.mod @@ -82,6 +82,7 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/google/go-github/v70 v70.0.0 github.com/quasilyte/go-ruleguard/dsl v0.3.22 + github.com/urfave/cli/v3 v3.3.3 ) require ( diff --git a/pkg/build/go.sum b/pkg/build/go.sum index e0002142e22..d6baed9d369 100644 --- a/pkg/build/go.sum +++ b/pkg/build/go.sum @@ -251,6 +251,8 @@ github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ= github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po= github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g= github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/urfave/cli/v3 v3.3.3 h1:byCBaVdIXuLPIDm5CYZRVG6NvT7tv1ECqdU4YzlEa3I= +github.com/urfave/cli/v3 v3.3.3/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTdwFp0s= github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=