mirror of https://github.com/grafana/grafana
CI: Add `artifacts npm` commands from grabpl (#61908)
Horst Gutmann
3 years ago
committed by
GitHub
parent
0be920e61c
commit
e32cd6d4ff
@ -0,0 +1,88 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"fmt" |
||||
"os" |
||||
"os/exec" |
||||
"strings" |
||||
|
||||
"github.com/grafana/grafana/pkg/build/npm" |
||||
"github.com/urfave/cli/v2" |
||||
) |
||||
|
||||
func NpmRetrieveAction(c *cli.Context) error { |
||||
if c.NArg() > 0 { |
||||
if err := cli.ShowSubcommandHelp(c); err != nil { |
||||
return cli.Exit(err.Error(), 1) |
||||
} |
||||
return cli.Exit("", 1) |
||||
} |
||||
|
||||
tag := c.String("tag") |
||||
if tag == "" { |
||||
return fmt.Errorf("no tag version specified, exitting") |
||||
} |
||||
|
||||
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET")) |
||||
if prereleaseBucket == "" { |
||||
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1) |
||||
} |
||||
|
||||
err := npm.FetchNpmPackages(c.Context, tag, prereleaseBucket) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func NpmStoreAction(c *cli.Context) error { |
||||
if c.NArg() > 0 { |
||||
if err := cli.ShowSubcommandHelp(c); err != nil { |
||||
return cli.Exit(err.Error(), 1) |
||||
} |
||||
return cli.Exit("", 1) |
||||
} |
||||
|
||||
tag := c.String("tag") |
||||
if tag == "" { |
||||
return fmt.Errorf("no tag version specified, exiting") |
||||
} |
||||
|
||||
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET")) |
||||
if prereleaseBucket == "" { |
||||
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1) |
||||
} |
||||
|
||||
err := npm.StoreNpmPackages(c.Context, tag, prereleaseBucket) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func NpmReleaseAction(c *cli.Context) error { |
||||
if c.NArg() > 0 { |
||||
if err := cli.ShowSubcommandHelp(c); err != nil { |
||||
return cli.Exit(err.Error(), 1) |
||||
} |
||||
return cli.Exit("", 1) |
||||
} |
||||
|
||||
tag := c.String("tag") |
||||
if tag == "" { |
||||
return fmt.Errorf("no tag version specified, exitting") |
||||
} |
||||
|
||||
cmd := exec.Command("git", "checkout", ".") |
||||
if err := cmd.Run(); err != nil { |
||||
fmt.Println("command failed to run, err: ", err) |
||||
return err |
||||
} |
||||
|
||||
err := npm.PublishNpmPackages(c.Context, tag) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,240 @@ |
||||
package npm |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"log" |
||||
"os" |
||||
"os/exec" |
||||
"path/filepath" |
||||
"strings" |
||||
|
||||
"github.com/grafana/grafana/pkg/build/gcloud/storage" |
||||
"github.com/grafana/grafana/pkg/build/lerna" |
||||
"github.com/grafana/grafana/pkg/build/versions" |
||||
) |
||||
|
||||
const GrafanaDir = "." |
||||
const NpmArtifactDir = "./npm-artifacts" |
||||
|
||||
// TODO: could this be replaced by `yarn lerna list -p` ?
|
||||
var packages = []string{ |
||||
"@grafana/ui", |
||||
"@grafana/data", |
||||
"@grafana/toolkit", |
||||
"@grafana/runtime", |
||||
"@grafana/e2e", |
||||
"@grafana/e2e-selectors", |
||||
"@grafana/schema", |
||||
} |
||||
|
||||
// PublishNpmPackages will publish local NPM packages to NPM registry.
|
||||
func PublishNpmPackages(ctx context.Context, tag string) error { |
||||
version, err := versions.GetVersion(tag) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
log.Printf("Grafana version: %s", version.Version) |
||||
|
||||
if err := setNpmCredentials(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
npmArtifacts, err := storage.ListLocalFiles(NpmArtifactDir) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
for _, packedFile := range npmArtifacts { |
||||
// nolint:gosec
|
||||
cmd := exec.CommandContext(ctx, "npm", "publish", packedFile.FullPath, "--tag", version.Channel) |
||||
cmd.Dir = GrafanaDir |
||||
if out, err := cmd.CombinedOutput(); err != nil { |
||||
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) |
||||
} |
||||
} |
||||
|
||||
return updateTag(ctx, version, tag) |
||||
} |
||||
|
||||
// StoreNpmPackages will store local NPM packages in GCS bucket `bucketName`.
|
||||
func StoreNpmPackages(ctx context.Context, tag, bucketName string) error { |
||||
err := lerna.PackFrontendPackages(ctx, tag, GrafanaDir, NpmArtifactDir) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
gcs, err := storage.New() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
bucket := gcs.Bucket(bucketName) |
||||
bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag) |
||||
if err = gcs.CopyLocalDir(ctx, NpmArtifactDir, bucket, bucketPath, true); err != nil { |
||||
return err |
||||
} |
||||
|
||||
log.Print("Successfully stored npm packages!") |
||||
return nil |
||||
} |
||||
|
||||
// FetchNpmPackages will store NPM packages stored in GCS bucket `bucketName` on local disk in `frontend.NpmArtifactDir`.
|
||||
func FetchNpmPackages(ctx context.Context, tag, bucketName string) error { |
||||
gcs, err := storage.New() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag) |
||||
bucket := gcs.Bucket(bucketName) |
||||
err = gcs.DownloadDirectory(ctx, bucket, NpmArtifactDir, storage.FilesFilter{ |
||||
Prefix: bucketPath, |
||||
FileExts: []string{".tgz"}, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// updateTag will move next or latest npm dist-tags, if needed.
|
||||
//
|
||||
// Note: This function makes the assumption that npm dist-tags has already
|
||||
// been updated and hence why move of dist-tags not always happens:
|
||||
//
|
||||
// If stable the dist-tag latest was used.
|
||||
// If beta the dist-tag next was used.
|
||||
//
|
||||
// Scenarios:
|
||||
//
|
||||
// 1. Releasing a newer stable than the current stable
|
||||
// Latest and next is 9.1.5.
|
||||
// 9.1.6 is released, latest and next should point to 9.1.6.
|
||||
// The next dist-tag is moved to point to 9.1.6.
|
||||
//
|
||||
// 2. Releasing during an active beta period:
|
||||
// Latest and next is 9.1.6.
|
||||
// 9.2.0-beta1 is released, the latest should stay on 9.1.6, next should point to 9.2.0-beta1
|
||||
// No move of dist-tags
|
||||
// 9.1.7 is relased, the latest should point to 9.1.7, next should stay to 9.2.0-beta1
|
||||
// No move of dist-tags
|
||||
// Next week 9.2.0-beta2 is released, the latest should point to 9.1.7, next should point to 9.2.0-beta2
|
||||
// No move of dist-tags
|
||||
// In two weeks 9.2.0 stable is relased, the latest and next should point to 9.2.0.
|
||||
// The next dist-tag is moved to point to 9.2.0.
|
||||
//
|
||||
// 3. Releasing an older stable than the current stable
|
||||
// Latest and next is 9.2.0.
|
||||
// Next 9.1.8 is released, latest should point to 9.2.0, next should point to 9.2.0
|
||||
// The latest dist-tag is moved to point to 9.2.0.
|
||||
func updateTag(ctx context.Context, version *versions.Version, releaseVersion string) error { |
||||
if version.Channel != versions.Latest { |
||||
return nil |
||||
} |
||||
|
||||
latestStableVersion, err := getLatestStableVersion() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
betaVersion, err := getLatestBetaVersion() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
isLatest, err := versions.IsGreaterThanOrEqual(releaseVersion, latestStableVersion) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
isNewerThanLatestBeta, err := versions.IsGreaterThanOrEqual(releaseVersion, betaVersion) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
for _, pkg := range packages { |
||||
if !isLatest { |
||||
err = runMoveLatestNPMTagCommand(ctx, pkg, latestStableVersion) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
if isLatest && isNewerThanLatestBeta { |
||||
err = runMoveNextNPMTagCommand(ctx, pkg, version.Version) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func getLatestStableVersion() (string, error) { |
||||
return versions.GetLatestVersion(versions.LatestStableVersionURL) |
||||
} |
||||
|
||||
func getLatestBetaVersion() (string, error) { |
||||
return versions.GetLatestVersion(versions.LatestBetaVersionURL) |
||||
} |
||||
|
||||
func runMoveNextNPMTagCommand(ctx context.Context, pkg string, packageVersion string) error { |
||||
// nolint:gosec
|
||||
cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, packageVersion), "next") |
||||
if out, err := cmd.CombinedOutput(); err != nil { |
||||
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func runMoveLatestNPMTagCommand(ctx context.Context, pkg string, latestStableVersion string) error { |
||||
// nolint:gosec
|
||||
cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, latestStableVersion), "latest") |
||||
if out, err := cmd.CombinedOutput(); err != nil { |
||||
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// setNpmCredentials Creates a .npmrc file in the users home folder and writes the
|
||||
// necessary credentials to it for publishing packages to the NPM registry.
|
||||
func setNpmCredentials() error { |
||||
npmToken := strings.TrimSpace(os.Getenv("NPM_TOKEN")) |
||||
if npmToken == "" { |
||||
return fmt.Errorf("npm token is not set") |
||||
} |
||||
|
||||
homeDir, err := os.UserHomeDir() |
||||
if err != nil { |
||||
return fmt.Errorf("failed to obtain home directory, err: %q", err) |
||||
} |
||||
|
||||
npmPath := filepath.Join(homeDir, ".npmrc") |
||||
registry := []byte(fmt.Sprintf("//registry.npmjs.org/:_authToken=%s", npmToken)) |
||||
if _, err = os.Stat(npmPath); os.IsNotExist(err) { |
||||
// nolint:gosec
|
||||
f, err := os.Create(npmPath) |
||||
if err != nil { |
||||
return fmt.Errorf("couldn't create npmrc file, err: %q", err) |
||||
} |
||||
_, err = f.Write(registry) |
||||
if err != nil { |
||||
return fmt.Errorf("failed to write to file, err: %q", err) |
||||
} |
||||
defer func() { |
||||
if err := f.Close(); err != nil { |
||||
log.Printf("Failed to close file: %s", err.Error()) |
||||
} |
||||
}() |
||||
} else { |
||||
err = os.WriteFile(npmPath, registry, 0644) |
||||
if err != nil { |
||||
return fmt.Errorf("error writing to file, err: %q", err) |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,160 @@ |
||||
package versions |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"io" |
||||
"log" |
||||
"net/http" |
||||
"regexp" |
||||
"strconv" |
||||
|
||||
"github.com/Masterminds/semver/v3" |
||||
) |
||||
|
||||
var ( |
||||
reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`) |
||||
reGrafanaTagBeta = regexp.MustCompile(`^v(\d+\.\d+\.\d+-beta)`) |
||||
reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`) |
||||
) |
||||
|
||||
const ( |
||||
Latest = "latest" |
||||
Next = "next" |
||||
Test = "test" |
||||
) |
||||
|
||||
type Version struct { |
||||
Version string |
||||
Channel string |
||||
} |
||||
|
||||
type VersionFromAPI struct { |
||||
Version string `json:"version"` |
||||
} |
||||
|
||||
type LatestGcomAPI = string |
||||
|
||||
const ( |
||||
LatestStableVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/stable" |
||||
LatestBetaVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/beta" |
||||
) |
||||
|
||||
func GetLatestVersion(url LatestGcomAPI) (string, error) { |
||||
// nolint:gosec
|
||||
resp, err := http.Get(url) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
defer func() { |
||||
if err := resp.Body.Close(); err != nil { |
||||
log.Printf("Failed to close body: %s", err.Error()) |
||||
} |
||||
}() |
||||
|
||||
if resp.StatusCode != http.StatusOK { |
||||
return "", fmt.Errorf("server returned non 200 status code: %d", resp.StatusCode) |
||||
} |
||||
|
||||
body, err := io.ReadAll(resp.Body) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
|
||||
var apiResponse VersionFromAPI |
||||
err = json.Unmarshal(body, &apiResponse) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
|
||||
return apiResponse.Version, nil |
||||
} |
||||
|
||||
// IsGreaterThanOrEqual semantically checks whether newVersion is greater than or equal to stableVersion.
|
||||
func IsGreaterThanOrEqual(newVersion, stableVersion string) (bool, error) { |
||||
v1SemVer, err := semver.NewVersion(newVersion) |
||||
if err != nil { |
||||
return isGreaterThanOrEqualFourDigit(newVersion, stableVersion) |
||||
} |
||||
|
||||
v2SemVer, err := semver.NewVersion(stableVersion) |
||||
if err != nil { |
||||
return isGreaterThanOrEqualFourDigit(newVersion, stableVersion) |
||||
} |
||||
|
||||
comp := v1SemVer.Compare(v2SemVer) |
||||
switch comp { |
||||
case -1: |
||||
return false, nil |
||||
case 1, 0: |
||||
return true, nil |
||||
default: |
||||
return true, fmt.Errorf("unknown comparison value between scemantic versions, err: %q", err) |
||||
} |
||||
} |
||||
|
||||
var fourDigitRe = regexp.MustCompile(`(\d+\.\d+\.\d+)\.(\d+)`) |
||||
|
||||
func parseFourDigit(version string) (*semver.Version, int, error) { |
||||
matches := fourDigitRe.FindStringSubmatch(version) |
||||
if len(matches) < 2 { |
||||
semVer, err := semver.NewVersion(version) |
||||
if err != nil { |
||||
return nil, 0, err |
||||
} |
||||
return semVer, 0, nil |
||||
} |
||||
semVer, err := semver.NewVersion(matches[1]) |
||||
if err != nil { |
||||
return nil, 0, err |
||||
} |
||||
i, err := strconv.Atoi(matches[2]) |
||||
if err != nil { |
||||
return nil, 0, err |
||||
} |
||||
|
||||
return semVer, i, nil |
||||
} |
||||
|
||||
func isGreaterThanOrEqualFourDigit(newVersion, stableVersion string) (bool, error) { |
||||
newVersionSemVer, newVersionSemVerNo, err := parseFourDigit(newVersion) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
|
||||
stableVersionSemVer, stableVersionSemVerNo, err := parseFourDigit(stableVersion) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
|
||||
if stableVersionSemVer.Original() != newVersionSemVer.Original() { |
||||
return IsGreaterThanOrEqual(newVersionSemVer.Original(), stableVersionSemVer.Original()) |
||||
} |
||||
|
||||
return newVersionSemVerNo >= stableVersionSemVerNo, nil |
||||
} |
||||
|
||||
func GetVersion(tag string) (*Version, error) { |
||||
var version Version |
||||
switch { |
||||
case reGrafanaTag.MatchString(tag): |
||||
version = Version{ |
||||
Version: reGrafanaTag.FindStringSubmatch(tag)[1], |
||||
Channel: Latest, |
||||
} |
||||
case reGrafanaTagBeta.MatchString(tag): |
||||
version = Version{ |
||||
Version: reGrafanaTagBeta.FindStringSubmatch(tag)[1], |
||||
Channel: Next, |
||||
} |
||||
case reGrafanaTagCustom.MatchString(tag): |
||||
version = Version{ |
||||
Version: reGrafanaTagCustom.FindStringSubmatch(tag)[1], |
||||
Channel: Test, |
||||
} |
||||
default: |
||||
return nil, fmt.Errorf("%s not a supported Grafana version, exitting", tag) |
||||
} |
||||
|
||||
return &version, nil |
||||
} |
||||
@ -0,0 +1,69 @@ |
||||
package versions |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"net/http" |
||||
"net/http/httptest" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestIsGreaterThanOrEqual(t *testing.T) { |
||||
testCases := []struct { |
||||
newVersion string |
||||
stableVersion string |
||||
expected bool |
||||
}{ |
||||
{newVersion: "9.0.0", stableVersion: "8.0.0", expected: true}, |
||||
{newVersion: "6.0.0", stableVersion: "6.0.0", expected: true}, |
||||
{newVersion: "7.0.0", stableVersion: "8.0.0", expected: false}, |
||||
{newVersion: "8.5.0-beta1", stableVersion: "8.0.0", expected: true}, |
||||
{newVersion: "8.5.0", stableVersion: "8.5.0-beta1", expected: true}, |
||||
{newVersion: "9.0.0.1", stableVersion: "9.0.0", expected: true}, |
||||
{newVersion: "9.0.0.2", stableVersion: "9.0.0.1", expected: true}, |
||||
{newVersion: "9.1.0", stableVersion: "9.0.0.1", expected: true}, |
||||
{newVersion: "9.1-0-beta1", stableVersion: "9.0.0.1", expected: true}, |
||||
{newVersion: "9.0.0.1", stableVersion: "9.0.1.1", expected: false}, |
||||
{newVersion: "9.0.1.1", stableVersion: "9.0.0.1", expected: true}, |
||||
{newVersion: "9.0.0.1", stableVersion: "9.0.0.1", expected: true}, |
||||
{newVersion: "7.0.0.1", stableVersion: "8.0.0", expected: false}, |
||||
{newVersion: "9.1-0-beta1", stableVersion: "9.1-0-beta2", expected: false}, |
||||
{newVersion: "9.1-0-beta3", stableVersion: "9.1-0-beta2", expected: true}, |
||||
} |
||||
|
||||
for _, tc := range testCases { |
||||
name := fmt.Sprintf("newVersion %s greater than or equal stableVersion %s = %v", tc.newVersion, tc.stableVersion, tc.expected) |
||||
t.Run(name, func(t *testing.T) { |
||||
result, err := IsGreaterThanOrEqual(tc.newVersion, tc.stableVersion) |
||||
require.NoError(t, err) |
||||
require.Equal(t, tc.expected, result) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestGetLatestVersion(t *testing.T) { |
||||
t.Run("it returns a version", func(t *testing.T) { |
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
||||
response := VersionFromAPI{ |
||||
Version: "8.4.0", |
||||
} |
||||
jsonRes, err := json.Marshal(&response) |
||||
require.NoError(t, err) |
||||
_, err = w.Write(jsonRes) |
||||
require.NoError(t, err) |
||||
})) |
||||
version, err := GetLatestVersion(server.URL) |
||||
require.NoError(t, err) |
||||
require.Equal(t, "8.4.0", version) |
||||
}) |
||||
|
||||
t.Run("it handles non 200 responses", func(t *testing.T) { |
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
||||
w.WriteHeader(http.StatusInternalServerError) |
||||
})) |
||||
_, err := GetLatestVersion(server.URL) |
||||
require.Error(t, err) |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue