Merge pull request #13968 from xlson/configurable-release-publisher

Configurable release publisher

.circleci/config.yml

@@ -335,6 +335,9 @@ jobs:
       - run:
           name: deploy to gcp
           command: '/opt/google-cloud-sdk/bin/gsutil cp ./enterprise-dist/* gs://$GCP_BUCKET_NAME/enterprise/master'
+      - run:
+          name: Deploy to grafana.com
+          command: 'cd enterprise-dist && scripts/build/release_publisher/release_publisher -apikey ${GRAFANA_COM_API_KEY} -enterprise -from-local'
 
 
   deploy-enterprise-release:

scripts/build/publish.sh

@@ -1,4 +1,4 @@
-#/bin/sh
+#!/bin/sh
 
 # no relation to publish.go
 

scripts/build/release_publisher/externalrelease.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+)
+
+type releaseFromExternalContent struct {
+	getter     urlGetter
+	rawVersion string
+	artifactConfigurations []buildArtifact
+}
+
+func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) {
+	version := re.rawVersion[1:]
+	isBeta := strings.Contains(version, "beta")
+
+	builds := []build{}
+	for _, ba := range re.artifactConfigurations {
+		sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(baseArchiveUrl, version, isBeta)))
+		if err != nil {
+			return nil, err
+		}
+		builds = append(builds, newBuild(baseArchiveUrl, ba, version, isBeta, sha256))
+	}
+
+	r := release{
+		Version:         version,
+		ReleaseDate:     time.Now(),
+		Stable:          !isBeta,
+		Beta:            isBeta,
+		Nightly:         false,
+		WhatsNewUrl:     whatsNewUrl,
+		ReleaseNotesUrl: releaseNotesUrl,
+		Builds:          builds,
+	}
+	return &r, nil
+}
+
+type urlGetter interface {
+	getContents(url string) (string, error)
+}
+
+type getHttpContents struct{}
+
+func (getHttpContents) getContents(url string) (string, error) {
+	response, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+
+	defer response.Body.Close()
+	all, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return string(all), nil
+}

scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz.sha256

@@ -0,0 +1 @@
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip.sha256

@@ -0,0 +1 @@
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm.sha256

@@ -0,0 +1 @@
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb.sha256

@@ -0,0 +1 @@
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

scripts/build/release_publisher/localrelease.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type releaseLocalSources struct {
+	path string
+	artifactConfigurations []buildArtifact
+}
+
+func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) {
+	buildData := r.findBuilds(baseArchiveUrl)
+
+	rel := release{
+		Version:         buildData.version,
+		ReleaseDate:     time.Now(),
+		Stable:          false,
+		Beta:            false,
+		Nightly:         true,
+		WhatsNewUrl:     whatsNewUrl,
+		ReleaseNotesUrl: releaseNotesUrl,
+		Builds:          buildData.builds,
+	}
+
+	return &rel, nil
+}
+
+type buildData struct {
+	version string
+	builds []build
+}
+
+func (r releaseLocalSources) findBuilds(baseArchiveUrl string) buildData {
+	data := buildData{}
+	filepath.Walk(r.path, createBuildWalker(r.path, &data, r.artifactConfigurations, baseArchiveUrl))
+	return data
+}
+
+func createBuildWalker(path string, data *buildData, archiveTypes []buildArtifact, baseArchiveUrl string) func(path string, f os.FileInfo, err error) error {
+	return func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			log.Printf("error: %v", err)
+		}
+
+		if f.Name() == path || strings.HasSuffix(f.Name(), ".sha256") {
+			return nil
+		}
+
+		for _, archive := range archiveTypes {
+			if strings.HasSuffix(f.Name(), archive.urlPostfix) {
+				shaBytes, err := ioutil.ReadFile(path + ".sha256")
+				if err != nil {
+					log.Fatalf("Failed to read sha256 file %v", err)
+				}
+
+				version, err := grabVersion(f.Name(), archive.urlPostfix)
+				if err != nil {
+					log.Println(err)
+					continue
+				}
+				data.version = version
+				data.builds = append(data.builds, build{
+					Os:     archive.os,
+					Url:    archive.getUrl(baseArchiveUrl, version, false),
+					Sha256: string(shaBytes),
+					Arch:   archive.arch,
+				})
+				return nil
+			}
+		}
+		return nil
+	}
+
+}
+func grabVersion(name string, suffix string) (string, error) {
+	match := regexp.MustCompile(fmt.Sprintf(`grafana(-enterprise)?[-_](.*)%s`, suffix)).FindSubmatch([]byte(name))
+	if len(match) > 0 {
+		return string(match[2]), nil
+	}
+
+	return "", errors.New("No version found.")
+}

scripts/build/release_publisher/main.go

@@ -7,13 +7,13 @@ import (
 	"os"
 )
 
-var baseUri string = "https://grafana.com/api"
-
 func main() {
 	var version string
 	var whatsNewUrl string
 	var releaseNotesUrl string
 	var dryRun bool
+	var enterprise bool
+	var fromLocal bool
 	var apiKey string
 
 	flag.StringVar(&version, "version", "", "Grafana version (ex: --version v5.2.0-beta1)")
@@ -21,20 +21,55 @@ func main() {
 	flag.StringVar(&releaseNotesUrl, "rn", "", "Grafana version (ex: --rn https://community.grafana.com/t/release-notes-v5-2-x/7894)")
 	flag.StringVar(&apiKey, "apikey", "", "Grafana.com API key (ex: --apikey ABCDEF)")
 	flag.BoolVar(&dryRun, "dry-run", false, "--dry-run")
+	flag.BoolVar(&enterprise, "enterprise", false, "--enterprise")
+	flag.BoolVar(&fromLocal, "from-local", false, "--from-local")
 	flag.Parse()
 
 	if len(os.Args) == 1 {
-		fmt.Println("Usage: go run publisher.go main.go --version <v> --wn <what's new url> --rn <release notes url> --apikey <api key> --dry-run false")
-		fmt.Println("example: go run publisher.go main.go --version v5.2.0-beta2 --wn http://docs.grafana.org/guides/whats-new-in-v5-2/ --rn https://community.grafana.com/t/release-notes-v5-2-x/7894 --apikey ASDF123 --dry-run true")
+		fmt.Println("Usage: go run publisher.go main.go --version <v> --wn <what's new url> --rn <release notes url> --apikey <api key> --dry-run false --enterprise false")
+		fmt.Println("example: go run publisher.go main.go --version v5.2.0-beta2 --wn http://docs.grafana.org/guides/whats-new-in-v5-2/ --rn https://community.grafana.com/t/release-notes-v5-2-x/7894 --apikey ASDF123 --dry-run --enterprise")
 		os.Exit(1)
 	}
 
 	if dryRun {
 		log.Println("Dry-run has been enabled.")
 	}
+	var baseUrl string
+	var builder releaseBuilder
+	var product string
+
+	if fromLocal {
+		path, _ := os.Getwd()
+		builder = releaseLocalSources{
+			path: path,
+			artifactConfigurations: buildArtifactConfigurations,
+		}
+	} else {
+		builder = releaseFromExternalContent{
+			getter:     getHttpContents{},
+			rawVersion: version,
+			artifactConfigurations: buildArtifactConfigurations,
+		}
+	}
 
-	p := publisher{apiKey: apiKey}
-	if err := p.doRelease(version, whatsNewUrl, releaseNotesUrl, dryRun); err != nil {
+	if enterprise {
+		baseUrl = "https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/release/grafana-enterprise"
+		product = "grafana-enterprise"
+	} else {
+		baseUrl = "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana"
+		product = "grafana"
+	}
+
+	p := publisher{
+		apiKey:         apiKey,
+		apiUri:         "https://grafana.com/api",
+		product:        product,
+		dryRun:         dryRun,
+		enterprise:     enterprise,
+		baseArchiveUrl: baseUrl,
+		builder:        builder,
+	}
+	if err := p.doRelease(whatsNewUrl, releaseNotesUrl); err != nil {
 		log.Fatalf("error: %v", err)
 	}
 }

scripts/build/release_publisher/publisher.go

@@ -12,53 +12,47 @@ import (
 )
 
 type publisher struct {
-	apiKey string
+	apiKey         string
+	apiUri         string
+	product        string
+	dryRun         bool
+	enterprise     bool
+	baseArchiveUrl string
+	builder        releaseBuilder
 }
 
-func (p *publisher) doRelease(version string, whatsNewUrl string, releaseNotesUrl string, dryRun bool) error {
-	currentRelease, err := newRelease(version, whatsNewUrl, releaseNotesUrl, buildArtifactConfigurations, getHttpContents{})
+type releaseBuilder interface {
+	prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error)
+}
+
+func (p *publisher) doRelease(whatsNewUrl string, releaseNotesUrl string) error {
+	currentRelease, err := p.builder.prepareRelease(p.baseArchiveUrl, whatsNewUrl, releaseNotesUrl)
 	if err != nil {
 		return err
 	}
 
-	if dryRun {
-		relJson, err := json.Marshal(currentRelease)
-		if err != nil {
-			return err
-		}
-		log.Println(string(relJson))
-
-		for _, b := range currentRelease.Builds {
-			artifactJson, err := json.Marshal(b)
-			if err != nil {
-				return err
-			}
-			log.Println(string(artifactJson))
-		}
-	} else {
-		if err := p.postRelease(currentRelease); err != nil {
-			return err
-		}
+	if err := p.postRelease(currentRelease); err != nil {
+		return err
 	}
 
 	return nil
 }
 
 func (p *publisher) postRelease(r *release) error {
-	err := p.postRequest("/grafana/versions", r, fmt.Sprintf("Create Release %s", r.Version))
+	err := p.postRequest("/versions", r, fmt.Sprintf("Create Release %s", r.Version))
 	if err != nil {
 		return err
 	}
-	err = p.postRequest("/grafana/versions/"+r.Version, r, fmt.Sprintf("Update Release %s", r.Version))
+	err = p.postRequest("/versions/"+r.Version, r, fmt.Sprintf("Update Release %s", r.Version))
 	if err != nil {
 		return err
 	}
 	for _, b := range r.Builds {
-		err = p.postRequest(fmt.Sprintf("/grafana/versions/%s/packages", r.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch))
+		err = p.postRequest(fmt.Sprintf("/versions/%s/packages", r.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch))
 		if err != nil {
 			return err
 		}
-		err = p.postRequest(fmt.Sprintf("/grafana/versions/%s/packages/%s/%s", r.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch))
+		err = p.postRequest(fmt.Sprintf("/versions/%s/packages/%s/%s", r.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch))
 		if err != nil {
 			return err
 		}
@@ -67,15 +61,13 @@ func (p *publisher) postRelease(r *release) error {
 	return nil
 }
 
-const baseArhiveUrl = "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana"
-
 type buildArtifact struct {
 	os         string
 	arch       string
 	urlPostfix string
 }
 
-func (t buildArtifact) getUrl(version string, isBeta bool) string {
+func (t buildArtifact) getUrl(baseArchiveUrl, version string, isBeta bool) string {
 	prefix := "-"
 	rhelReleaseExtra := ""
 
@@ -87,7 +79,7 @@ func (t buildArtifact) getUrl(version string, isBeta bool) string {
 		rhelReleaseExtra = "-1"
 	}
 
-	url := strings.Join([]string{baseArhiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "")
+	url := strings.Join([]string{baseArchiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "")
 	return url
 }
 
@@ -149,48 +141,32 @@ var buildArtifactConfigurations = []buildArtifact{
 	},
 }
 
-func newRelease(rawVersion string, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact, getter urlGetter) (*release, error) {
-	version := rawVersion[1:]
-	now := time.Now()
-	isBeta := strings.Contains(version, "beta")
-
-	builds := []build{}
-	for _, ba := range artifactConfigurations {
-		sha256, err := getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(version, isBeta)))
-		if err != nil {
-			return nil, err
-		}
-		builds = append(builds, newBuild(ba, version, isBeta, sha256))
-	}
-
-	r := release{
-		Version:         version,
-		ReleaseDate:     time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local),
-		Stable:          !isBeta,
-		Beta:            isBeta,
-		Nightly:         false,
-		WhatsNewUrl:     whatsNewUrl,
-		ReleaseNotesUrl: releaseNotesUrl,
-		Builds:          builds,
-	}
-	return &r, nil
-}
-
-func newBuild(ba buildArtifact, version string, isBeta bool, sha256 string) build {
+func newBuild(baseArchiveUrl string, ba buildArtifact, version string, isBeta bool, sha256 string) build {
 	return build{
 		Os:     ba.os,
-		Url:    ba.getUrl(version, isBeta),
+		Url:    ba.getUrl(baseArchiveUrl, version, isBeta),
 		Sha256: sha256,
 		Arch:   ba.arch,
 	}
 }
 
+func (p *publisher) apiUrl(url string) string {
+	return fmt.Sprintf("%s/%s%s", p.apiUri, p.product, url)
+}
+
 func (p *publisher) postRequest(url string, obj interface{}, desc string) error {
 	jsonBytes, err := json.Marshal(obj)
 	if err != nil {
 		return err
 	}
-	req, err := http.NewRequest(http.MethodPost, baseUri+url, bytes.NewReader(jsonBytes))
+
+	if p.dryRun {
+		log.Println(fmt.Sprintf("POST to %s:", p.apiUrl(url)))
+		log.Println(string(jsonBytes))
+		return nil
+	}
+
+	req, err := http.NewRequest(http.MethodPost, p.apiUrl(url), bytes.NewReader(jsonBytes))
 	if err != nil {
 		return err
 	}
@@ -243,24 +219,3 @@ type build struct {
 	Sha256 string `json:"sha256"`
 	Arch   string `json:"arch"`
 }
-
-type urlGetter interface {
-	getContents(url string) (string, error)
-}
-
-type getHttpContents struct{}
-
-func (getHttpContents) getContents(url string) (string, error) {
-	response, err := http.Get(url)
-	if err != nil {
-		return "", err
-	}
-
-	defer response.Body.Close()
-	all, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		return "", err
-	}
-
-	return string(all), nil
-}

scripts/build/release_publisher/publisher_test.go

@@ -2,16 +2,24 @@ package main
 
 import "testing"
 
-func TestNewRelease(t *testing.T) {
+func TestPreparingReleaseFromRemote(t *testing.T) {
 	versionIn := "v5.2.0-beta1"
 	expectedVersion := "5.2.0-beta1"
 	whatsNewUrl := "https://whatsnews.foo/"
 	relNotesUrl := "https://relnotes.foo/"
 	expectedArch := "amd64"
 	expectedOs := "linux"
-	buildArtifacts := []buildArtifact{{expectedOs, expectedArch, ".linux-amd64.tar.gz"}}
+	buildArtifacts := []buildArtifact{{expectedOs,expectedArch, ".linux-amd64.tar.gz"}}
 
-	rel, _ := newRelease(versionIn, whatsNewUrl, relNotesUrl, buildArtifacts, mockHttpGetter{})
+	var builder releaseBuilder
+
+	builder = releaseFromExternalContent{
+		getter:     mockHttpGetter{},
+		rawVersion: versionIn,
+		artifactConfigurations: buildArtifactConfigurations,
+	}
+
+	rel, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana", whatsNewUrl, relNotesUrl)
 
 	if !rel.Beta || rel.Stable {
 		t.Errorf("%s should have been tagged as beta (not stable), but wasn't	.", versionIn)
@@ -41,3 +49,71 @@ type mockHttpGetter struct{}
 func (mockHttpGetter) getContents(url string) (string, error) {
 	return url, nil
 }
+
+
+func TestPreparingReleaseFromLocal(t *testing.T) {
+	whatsNewUrl := "https://whatsnews.foo/"
+	relNotesUrl := "https://relnotes.foo/"
+	expectedVersion := "5.4.0-123pre1"
+	expectedBuilds := 4
+
+	var builder releaseBuilder
+	testDataPath := "local_test_data"
+	builder = releaseLocalSources{
+		path:                   testDataPath,
+		artifactConfigurations: buildArtifactConfigurations,
+	}
+
+	relAll, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl)
+
+	if relAll.Stable || !relAll.Nightly {
+		t.Error("Expected a nightly release but wasn't.")
+	}
+
+	if relAll.ReleaseNotesUrl != relNotesUrl {
+		t.Errorf("expected releaseNotesUrl to be %s, but it was %s", relNotesUrl, relAll.ReleaseNotesUrl)
+	}
+	if relAll.WhatsNewUrl != whatsNewUrl {
+		t.Errorf("expected whatsNewUrl to be %s, but it was %s", whatsNewUrl, relAll.WhatsNewUrl)
+	}
+
+	if relAll.Beta {
+		t.Errorf("Expected release to be nightly, not beta.")
+	}
+
+	if relAll.Version != expectedVersion {
+		t.Errorf("Expected version=%s, but got=%s", expectedVersion, relAll.Version)
+	}
+
+	if len(relAll.Builds) != expectedBuilds {
+		t.Errorf("Expected %v builds, but was %v", expectedBuilds, len(relAll.Builds))
+	}
+
+	expectedArch := "amd64"
+	expectedOs := "win"
+
+	builder = releaseLocalSources{
+		path:                   testDataPath,
+		artifactConfigurations: []buildArtifact{{
+			os:         expectedOs,
+			arch:       expectedArch,
+			urlPostfix: ".windows-amd64.zip",
+		}},
+	}
+
+	relOne, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl)
+
+	if len(relOne.Builds) != 1 {
+		t.Errorf("Expected 1 artifact, but was %v", len(relOne.Builds))
+	}
+
+	build := relOne.Builds[0]
+
+	if build.Arch != expectedArch {
+		t.Fatalf("Expected arch to be %s, but was %s", expectedArch, build.Arch)
+	}
+
+	if build.Os != expectedOs {
+		t.Fatalf("Expected os to be %s, but was %s", expectedOs, build.Os)
+	}
+}