apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move files under pkg/build/packaging (#56435)

Browse Source
pull/56448/head^2
Dimitris Sotirakis 3 years ago committed by GitHub
parent ec7d9e196e
commit 3aacda5579
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 620 additions and 622 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 243
      pkg/build/cmd/deb.go
  2. 17
      pkg/build/cmd/grafanacom.go
  3. 9
      pkg/build/cmd/publishpackages.go
  4. 365
      pkg/build/cmd/rpm.go
  5. 2
      pkg/build/packaging/artifacts.go
  6. 241
      pkg/build/packaging/deb.go
  7. 363
      pkg/build/packaging/rpm.go
  8. 2
      pkg/build/packaging/rpm_test.go

243
pkg/build/cmd/deb.go
Unescape View File

@ -1,243 +0,0 @@
package main
import (
"fmt"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/grafana/grafana/pkg/infra/fs"
"github.com/urfave/cli/v2"
)
func writeAptlyConf(dbDir, repoDir string) error {
aptlyConf := fmt.Sprintf(`{
"rootDir": "%s",
"downloadConcurrency": 4,
"downloadSpeedLimit": 0,
"architectures": [],
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"gpgDisableSign": false,
"gpgDisableVerify": false,
"gpgProvider": "gpg2",
"downloadSourcePackages": false,
"skipLegacyPool": true,
"ppaDistributorID": "ubuntu",
"ppaCodename": "",
"skipContentsPublishing": false,
"FileSystemPublishEndpoints": {
"repo": {
"rootDir": "%s",
"linkMethod": "copy"
}
},
"S3PublishEndpoints": {},
"SwiftPublishEndpoints": {}
}
`, dbDir, repoDir)
home, err := os.UserHomeDir()
if err != nil {
return err
}
return os.WriteFile(filepath.Join(home, ".aptly.conf"), []byte(aptlyConf), 0600)
}
// downloadDebs downloads Deb packages.
func downloadDebs(cfg PublishConfig, workDir string) error {
if cfg.Bucket == "" {
panic("cfg.Bucket has to be set")
}
if !strings.HasSuffix(workDir, string(filepath.Separator)) {
workDir += string(filepath.Separator)
}
var version string
if cfg.ReleaseMode.Mode == config.TagMode {
if cfg.ReleaseMode.IsBeta {
version = strings.ReplaceAll(cfg.Version, "-", "~")
} else {
version = cfg.Version
}
}
if version == "" {
panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode))
}
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
u := fmt.Sprintf("gs://%s/%s/%s/grafana%s_%s_*.deb*", cfg.Bucket,
strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version)
log.Printf("Downloading Deb packages %q...\n", u)
args := []string{
"-m",
"cp",
u,
workDir,
}
//nolint:gosec
cmd := exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download Deb packages %q: %w\n%s", u, err, output)
}
return nil
}
// updateDebRepo updates the Debian repository with the new release.
func updateDebRepo(cfg PublishConfig, workDir string) error {
if cfg.ReleaseMode.Mode != config.TagMode {
panic(fmt.Sprintf("Unsupported version mode: %s", cfg.ReleaseMode.Mode))
}
if cfg.ReleaseMode.IsTest {
if cfg.Config.DebDBBucket == packaging.DefaultDebDBBucket {
return fmt.Errorf("in test-release mode, the default Deb DB bucket shouldn't be used")
}
if cfg.Config.DebRepoBucket == packaging.DefaultDebRepoBucket {
return fmt.Errorf("in test-release mode, the default Deb repo bucket shouldn't be used")
}
}
if err := downloadDebs(cfg, workDir); err != nil {
return err
}
repoName := "grafana"
if cfg.ReleaseMode.IsBeta {
repoName = "beta"
}
repoRoot := path.Join(os.TempDir(), "deb-repo")
defer func() {
if err := os.RemoveAll(repoRoot); err != nil {
log.Printf("Failed to remove temporary directory %q: %s\n", repoRoot, err.Error())
}
}()
dbDir := filepath.Join(repoRoot, "db")
repoDir := filepath.Join(repoRoot, "repo")
tmpDir := filepath.Join(repoRoot, "tmp")
for _, dpath := range []string{dbDir, repoDir, tmpDir} {
if err := os.MkdirAll(dpath, 0750); err != nil {
return err
}
}
if err := writeAptlyConf(dbDir, repoDir); err != nil {
return err
}
// Download the Debian repo database
u := fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition)))
log.Printf("Downloading Debian repo database from %s...\n", u)
//nolint:gosec
cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", u, dbDir)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download Debian repo database: %w\n%s", err, output)
}
if err := addPkgsToRepo(cfg, workDir, tmpDir, repoName); err != nil {
return err
}
log.Println("Updating local Debian package repository...")
// Update published local repository. This assumes that there exists already a local, published repo.
for _, tp := range []string{"stable", "beta"} {
passArg := fmt.Sprintf("-passphrase-file=%s", cfg.GPGPassPath)
//nolint:gosec
cmd := exec.Command("aptly", "publish", "update", "-batch", passArg, "-force-overwrite", tp,
"filesystem:repo:grafana")
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1)
}
}
// Update database in GCS
u = fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition)))
if cfg.DryRun {
log.Printf("Simulating upload of Debian repo database to GCS (%s)\n", u)
} else {
log.Printf("Uploading Debian repo database to GCS (%s)...\n", u)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", dbDir, u)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1)
}
}
// Update metadata and binaries in repository bucket
u = fmt.Sprintf("gs://%s/%s/deb", cfg.DebRepoBucket, strings.ToLower(string(cfg.Edition)))
grafDir := filepath.Join(repoDir, "grafana")
if cfg.DryRun {
log.Printf("Simulating upload of Debian repo resources to GCS (%s)\n", u)
} else {
log.Printf("Uploading Debian repo resources to GCS (%s)...\n", u)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", grafDir, u)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1)
}
allRepoResources := fmt.Sprintf("%s/**/*", u)
log.Printf("Setting cache ttl for Debian repo resources on GCS (%s)...\n", allRepoResources)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1)
}
}
return nil
}
func addPkgsToRepo(cfg PublishConfig, workDir, tmpDir, repoName string) error {
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unsupported edition %q", cfg.Edition)
}
log.Printf("Adding packages to Debian %q repo...\n", repoName)
// TODO: Be more specific about filename pattern
debs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s*.deb", sfx)))
if err != nil {
return err
}
for _, deb := range debs {
basename := filepath.Base(deb)
if strings.Contains(basename, "latest") {
continue
}
tgt := filepath.Join(tmpDir, basename)
if err := fs.CopyFile(deb, tgt); err != nil {
return err
}
}
// XXX: Adds too many packages in enterprise (Arve: What does this mean exactly?)
//nolint:gosec
cmd := exec.Command("aptly", "repo", "add", "-force-replace", repoName, tmpDir)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1)
}
return nil
}

17
pkg/build/cmd/grafanacom.go
Unescape View File

@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/urfave/cli/v2"
)
@ -72,7 +73,7 @@ func GrafanaCom(c *cli.Context) error {
}
// TODO: Verify config values
cfg := PublishConfig{
cfg := packaging.PublishConfig{
Config: config.Config{
Version: version,
},
@ -124,7 +125,7 @@ func getReleaseURLs() (string, string, error) {
}
// publishPackages publishes packages to grafana.com.
func publishPackages(cfg PublishConfig) error {
func publishPackages(cfg packaging.PublishConfig) error {
log.Printf("Publishing Grafana packages, version %s, %s edition, %s mode, dryRun: %v, simulating: %v...\n",
cfg.Version, cfg.Edition, cfg.ReleaseMode.Mode, cfg.DryRun, cfg.SimulateRelease)
@ -138,16 +139,16 @@ func publishPackages(cfg PublishConfig) error {
pth = "oss"
case config.EditionEnterprise:
pth = "enterprise"
sfx = EnterpriseSfx
sfx = packaging.EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
switch cfg.ReleaseMode.Mode {
case config.MainMode, config.CustomMode, config.CronjobMode:
pth = path.Join(pth, MainFolder)
pth = path.Join(pth, packaging.MainFolder)
default:
pth = path.Join(pth, ReleaseFolder)
pth = path.Join(pth, packaging.ReleaseFolder)
}
product := fmt.Sprintf("grafana%s", sfx)
@ -155,7 +156,7 @@ func publishPackages(cfg PublishConfig) error {
baseArchiveURL := fmt.Sprintf("https://dl.grafana.com/%s", pth)
var builds []buildRepr
for _, ba := range ArtifactConfigs {
for _, ba := range packaging.ArtifactConfigs {
u := ba.GetURL(baseArchiveURL, cfg)
sha256, err := getSHA256(u)
@ -230,12 +231,12 @@ func getSHA256(u string) ([]byte, error) {
return sha256, nil
}
func postRequest(cfg PublishConfig, pth string, obj interface{}, descr string) error {
func postRequest(cfg packaging.PublishConfig, pth string, obj interface{}, descr string) error {
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
sfx = packaging.EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}

9
pkg/build/cmd/publishpackages.go
Unescape View File

@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/gpg"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/urfave/cli/v2"
)
@ -43,7 +44,7 @@ func PublishPackages(c *cli.Context) error {
edition := config.Edition(c.String("edition"))
// TODO: Verify config values
cfg := PublishConfig{
cfg := packaging.PublishConfig{
Config: config.Config{
Version: metadata.GrafanaVersion,
Bucket: c.String("packages-bucket"),
@ -83,18 +84,18 @@ func PublishPackages(c *cli.Context) error {
}
// updatePkgRepos updates package manager repositories.
func updatePkgRepos(cfg PublishConfig, workDir string) error {
func updatePkgRepos(cfg packaging.PublishConfig, workDir string) error {
if err := gpg.Import(cfg.Config); err != nil {
return err
}
// If updating the Deb repo fails, still continue with the RPM repo, so we don't have to retry
// both by hand
debErr := updateDebRepo(cfg, workDir)
debErr := packaging.UpdateDebRepo(cfg, workDir)
if debErr != nil {
log.Printf("Updating Deb repo failed: %s\n", debErr)
}
rpmErr := updateRPMRepo(cfg, workDir)
rpmErr := packaging.UpdateRPMRepo(cfg, workDir)
if rpmErr != nil {
log.Printf("Updating RPM repo failed: %s\n", rpmErr)
}

365
pkg/build/cmd/rpm.go
Unescape View File

@ -1,365 +0,0 @@
package main
import (
"bytes"
"crypto"
"fmt"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/grafana/grafana/pkg/infra/fs"
"golang.org/x/crypto/openpgp"
"golang.org/x/crypto/openpgp/armor"
"golang.org/x/crypto/openpgp/packet"
)
// updateRPMRepo updates the RPM repository with the new release.
func updateRPMRepo(cfg PublishConfig, workDir string) error {
if cfg.ReleaseMode.Mode != config.TagMode {
panic(fmt.Sprintf("Unsupported version mode %s", cfg.ReleaseMode.Mode))
}
if cfg.ReleaseMode.IsTest && cfg.Config.RPMRepoBucket == packaging.DefaultRPMRepoBucket {
return fmt.Errorf("in test-release mode, the default RPM repo bucket shouldn't be used")
}
if err := downloadRPMs(cfg, workDir); err != nil {
return err
}
repoRoot := path.Join(os.TempDir(), "rpm-repo")
defer func() {
if err := os.RemoveAll(repoRoot); err != nil {
log.Printf("Failed to remove %q: %s\n", repoRoot, err.Error())
}
}()
repoName := "rpm"
if cfg.ReleaseMode.IsBeta {
repoName = "rpm-beta"
}
folderURI := fmt.Sprintf("gs://%s/%s/%s", cfg.RPMRepoBucket, strings.ToLower(string(cfg.Edition)), repoName)
// Download the RPM database
log.Printf("Downloading RPM database from GCS (%s)...\n", folderURI)
//nolint:gosec
cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", folderURI, repoRoot)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download RPM database from GCS: %w\n%s", err, output)
}
// Add the new release to the repo
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unsupported edition %q", cfg.Edition)
}
allRPMs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s-*.rpm", sfx)))
if err != nil {
return fmt.Errorf("failed to list RPMs in %q: %w", workDir, err)
}
rpms := []string{}
for _, rpm := range allRPMs {
if strings.Contains(rpm, "-latest") {
continue
}
rpms = append(rpms, rpm)
}
// XXX: What does the following comment mean?
// adds to many files for enterprise
for _, rpm := range rpms {
if err := fs.CopyFile(rpm, filepath.Join(repoRoot, filepath.Base(rpm))); err != nil {
return err
}
}
//nolint:gosec
cmd = exec.Command("createrepo", repoRoot)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create repo at %q: %w\n%s", repoRoot, err, output)
}
if err := signRPMRepo(repoRoot, cfg); err != nil {
return err
}
// Update the repo in GCS
// Sync packages first to avoid cache misses
if cfg.DryRun {
log.Printf("Simulating upload of RPMs to GCS (%s)\n", folderURI)
} else {
log.Printf("Uploading RPMs to GCS (%s)...\n", folderURI)
args := []string{"-m", "cp"}
args = append(args, rpms...)
args = append(args, folderURI)
//nolint:gosec
cmd = exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to upload RPMs to GCS: %w\n%s", err, output)
}
}
if cfg.DryRun {
log.Printf("Simulating upload of RPM repo metadata to GCS (%s)\n", folderURI)
} else {
log.Printf("Uploading RPM repo metadata to GCS (%s)...\n", folderURI)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", repoRoot, folderURI)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to upload RPM repo metadata to GCS: %w\n%s", err, output)
}
allRepoResources := fmt.Sprintf("%s/**/*", folderURI)
log.Printf("Setting cache ttl for RPM repo resources on GCS (%s)...\n", allRepoResources)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to set cache ttl for RPM repo resources on GCS: %w\n%s", err, output)
}
}
return nil
}
// downloadRPMs downloads RPM packages.
func downloadRPMs(cfg PublishConfig, workDir string) error {
if !strings.HasSuffix(workDir, string(filepath.Separator)) {
workDir += string(filepath.Separator)
}
var version string
if cfg.ReleaseMode.Mode == config.TagMode {
if cfg.ReleaseMode.IsBeta {
version = strings.ReplaceAll(cfg.Version, "-", "~")
} else {
version = cfg.Version
}
}
if version == "" {
panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode))
}
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
u := fmt.Sprintf("gs://%s/%s/%s/grafana%s-%s-*.*.rpm*", cfg.Bucket,
strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version)
log.Printf("Downloading RPM packages %q...\n", u)
args := []string{
"-m",
"cp",
u,
workDir,
}
//nolint:gosec
cmd := exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download RPM packages %q: %w\n%s", u, err, output)
}
return nil
}
func getPublicKey(cfg PublishConfig) (*packet.PublicKey, error) {
f, err := os.Open(cfg.GPGPublicKey)
if err != nil {
return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPublicKey, err)
}
defer func(f *os.File) {
err := f.Close()
if err != nil {
return
}
}(f)
block, err := armor.Decode(f)
if err != nil {
return nil, err
}
if block.Type != openpgp.PublicKeyType {
return nil, fmt.Errorf("invalid public key block type: %q", block.Type)
}
packetReader := packet.NewReader(block.Body)
pkt, err := packetReader.Next()
if err != nil {
return nil, err
}
key, ok := pkt.(*packet.PublicKey)
if !ok {
return nil, fmt.Errorf("got non-public key from packet reader: %T", pkt)
}
return key, nil
}
func getPrivateKey(cfg PublishConfig) (*packet.PrivateKey, error) {
f, err := os.Open(cfg.GPGPrivateKey)
if err != nil {
return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPrivateKey, err)
}
defer func(f *os.File) {
err := f.Close()
if err != nil {
return
}
}(f)
passphraseB, err := os.ReadFile(cfg.GPGPassPath)
if err != nil {
return nil, fmt.Errorf("failed to read %q: %w", cfg.GPGPrivateKey, err)
}
passphraseB = bytes.TrimSuffix(passphraseB, []byte("\n"))
block, err := armor.Decode(f)
if err != nil {
return nil, err
}
if block.Type != openpgp.PrivateKeyType {
return nil, fmt.Errorf("invalid private key block type: %q", block.Type)
}
packetReader := packet.NewReader(block.Body)
pkt, err := packetReader.Next()
if err != nil {
return nil, err
}
key, ok := pkt.(*packet.PrivateKey)
if !ok {
return nil, fmt.Errorf("got non-private key from packet reader: %T", pkt)
}
if err := key.Decrypt(passphraseB); err != nil {
return nil, fmt.Errorf("failed to decrypt private key: %w", err)
}
return key, nil
}
// signRPMRepo signs an RPM repository using PGP.
// The signature gets written to the file repodata/repomd.xml.asc.
func signRPMRepo(repoRoot string, cfg PublishConfig) error {
if cfg.GPGPublicKey == "" || cfg.GPGPrivateKey == "" {
return fmt.Errorf("private or public key is empty")
}
log.Printf("Signing RPM repo")
pubKey, err := getPublicKey(cfg)
if err != nil {
return err
}
privKey, err := getPrivateKey(cfg)
if err != nil {
return err
}
pcfg := packet.Config{
DefaultHash: crypto.SHA256,
DefaultCipher: packet.CipherAES256,
DefaultCompressionAlgo: packet.CompressionZLIB,
CompressionConfig: &packet.CompressionConfig{
Level: 9,
},
RSABits: 4096,
}
currentTime := pcfg.Now()
uid := packet.NewUserId("", "", "")
isPrimaryID := false
keyLifetimeSecs := uint32(86400 * 365)
signer := openpgp.Entity{
PrimaryKey: pubKey,
PrivateKey: privKey,
Identities: map[string]*openpgp.Identity{
uid.Id: {
Name: uid.Name,
UserId: uid,
SelfSignature: &packet.Signature{
CreationTime: currentTime,
SigType: packet.SigTypePositiveCert,
PubKeyAlgo: packet.PubKeyAlgoRSA,
Hash: pcfg.Hash(),
IsPrimaryId: &isPrimaryID,
FlagsValid: true,
FlagSign: true,
FlagCertify: true,
IssuerKeyId: &pubKey.KeyId,
},
},
},
Subkeys: []openpgp.Subkey{
{
PublicKey: pubKey,
PrivateKey: privKey,
Sig: &packet.Signature{
CreationTime: currentTime,
SigType: packet.SigTypeSubkeyBinding,
PubKeyAlgo: packet.PubKeyAlgoRSA,
Hash: pcfg.Hash(),
PreferredHash: []uint8{8}, // SHA-256
FlagsValid: true,
FlagEncryptStorage: true,
FlagEncryptCommunications: true,
IssuerKeyId: &pubKey.KeyId,
KeyLifetimeSecs: &keyLifetimeSecs,
},
},
},
}
// Ignore gosec G304 as this function is only used in the build process.
//nolint:gosec
freader, err := os.Open(filepath.Join(repoRoot, "repodata", "repomd.xml"))
if err != nil {
return err
}
defer func(freader *os.File) {
err := freader.Close()
if err != nil {
return
}
}(freader)
// Ignore gosec G304 as this function is only used in the build process.
//nolint:gosec
sigwriter, err := os.Create(filepath.Join(repoRoot, "repodata", "repomd.xml.asc"))
if err != nil {
return err
}
defer func(sigwriter *os.File) {
err := sigwriter.Close()
if err != nil {
return
}
}(sigwriter)
if err := openpgp.ArmoredDetachSignText(sigwriter, &signer, freader, nil); err != nil {
return fmt.Errorf("failed to write PGP signature: %w", err)
}
if err := sigwriter.Close(); err != nil {
return fmt.Errorf("failed to write PGP signature: %w", err)
}
return nil
}

2
pkg/build/cmd/artifacts.go → pkg/build/packaging/artifacts.go
Unescape View File

@ -1,4 +1,4 @@
package main
package packaging
import (
"fmt"

241
pkg/build/packaging/deb.go
Unescape View File

@ -1 +1,242 @@
package packaging
import (
"fmt"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/infra/fs"
"github.com/urfave/cli/v2"
)
func writeAptlyConf(dbDir, repoDir string) error {
aptlyConf := fmt.Sprintf(`{
"rootDir": "%s",
"downloadConcurrency": 4,
"downloadSpeedLimit": 0,
"architectures": [],
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"gpgDisableSign": false,
"gpgDisableVerify": false,
"gpgProvider": "gpg2",
"downloadSourcePackages": false,
"skipLegacyPool": true,
"ppaDistributorID": "ubuntu",
"ppaCodename": "",
"skipContentsPublishing": false,
"FileSystemPublishEndpoints": {
"repo": {
"rootDir": "%s",
"linkMethod": "copy"
}
},
"S3PublishEndpoints": {},
"SwiftPublishEndpoints": {}
}
`, dbDir, repoDir)
home, err := os.UserHomeDir()
if err != nil {
return err
}
return os.WriteFile(filepath.Join(home, ".aptly.conf"), []byte(aptlyConf), 0600)
}
// downloadDebs downloads Deb packages.
func downloadDebs(cfg PublishConfig, workDir string) error {
if cfg.Bucket == "" {
panic("cfg.Bucket has to be set")
}
if !strings.HasSuffix(workDir, string(filepath.Separator)) {
workDir += string(filepath.Separator)
}
var version string
if cfg.ReleaseMode.Mode == config.TagMode {
if cfg.ReleaseMode.IsBeta {
version = strings.ReplaceAll(cfg.Version, "-", "~")
} else {
version = cfg.Version
}
}
if version == "" {
panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode))
}
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
u := fmt.Sprintf("gs://%s/%s/%s/grafana%s_%s_*.deb*", cfg.Bucket,
strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version)
log.Printf("Downloading Deb packages %q...\n", u)
args := []string{
"-m",
"cp",
u,
workDir,
}
//nolint:gosec
cmd := exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download Deb packages %q: %w\n%s", u, err, output)
}
return nil
}
// UpdateDebRepo updates the Debian repository with the new release.
func UpdateDebRepo(cfg PublishConfig, workDir string) error {
if cfg.ReleaseMode.Mode != config.TagMode {
panic(fmt.Sprintf("Unsupported version mode: %s", cfg.ReleaseMode.Mode))
}
if cfg.ReleaseMode.IsTest {
if cfg.Config.DebDBBucket == DefaultDebDBBucket {
return fmt.Errorf("in test-release mode, the default Deb DB bucket shouldn't be used")
}
if cfg.Config.DebRepoBucket == DefaultDebRepoBucket {
return fmt.Errorf("in test-release mode, the default Deb repo bucket shouldn't be used")
}
}
if err := downloadDebs(cfg, workDir); err != nil {
return err
}
repoName := "grafana"
if cfg.ReleaseMode.IsBeta {
repoName = "beta"
}
repoRoot := path.Join(os.TempDir(), "deb-repo")
defer func() {
if err := os.RemoveAll(repoRoot); err != nil {
log.Printf("Failed to remove temporary directory %q: %s\n", repoRoot, err.Error())
}
}()
dbDir := filepath.Join(repoRoot, "db")
repoDir := filepath.Join(repoRoot, "repo")
tmpDir := filepath.Join(repoRoot, "tmp")
for _, dpath := range []string{dbDir, repoDir, tmpDir} {
if err := os.MkdirAll(dpath, 0750); err != nil {
return err
}
}
if err := writeAptlyConf(dbDir, repoDir); err != nil {
return err
}
// Download the Debian repo database
u := fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition)))
log.Printf("Downloading Debian repo database from %s...\n", u)
//nolint:gosec
cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", u, dbDir)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download Debian repo database: %w\n%s", err, output)
}
if err := addPkgsToRepo(cfg, workDir, tmpDir, repoName); err != nil {
return err
}
log.Println("Updating local Debian package repository...")
// Update published local repository. This assumes that there exists already a local, published repo.
for _, tp := range []string{"stable", "beta"} {
passArg := fmt.Sprintf("-passphrase-file=%s", cfg.GPGPassPath)
//nolint:gosec
cmd := exec.Command("aptly", "publish", "update", "-batch", passArg, "-force-overwrite", tp,
"filesystem:repo:grafana")
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1)
}
}
// Update database in GCS
u = fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition)))
if cfg.DryRun {
log.Printf("Simulating upload of Debian repo database to GCS (%s)\n", u)
} else {
log.Printf("Uploading Debian repo database to GCS (%s)...\n", u)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", dbDir, u)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1)
}
}
// Update metadata and binaries in repository bucket
u = fmt.Sprintf("gs://%s/%s/deb", cfg.DebRepoBucket, strings.ToLower(string(cfg.Edition)))
grafDir := filepath.Join(repoDir, "grafana")
if cfg.DryRun {
log.Printf("Simulating upload of Debian repo resources to GCS (%s)\n", u)
} else {
log.Printf("Uploading Debian repo resources to GCS (%s)...\n", u)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", grafDir, u)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1)
}
allRepoResources := fmt.Sprintf("%s/**/*", u)
log.Printf("Setting cache ttl for Debian repo resources on GCS (%s)...\n", allRepoResources)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1)
}
}
return nil
}
func addPkgsToRepo(cfg PublishConfig, workDir, tmpDir, repoName string) error {
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unsupported edition %q", cfg.Edition)
}
log.Printf("Adding packages to Debian %q repo...\n", repoName)
// TODO: Be more specific about filename pattern
debs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s*.deb", sfx)))
if err != nil {
return err
}
for _, deb := range debs {
basename := filepath.Base(deb)
if strings.Contains(basename, "latest") {
continue
}
tgt := filepath.Join(tmpDir, basename)
if err := fs.CopyFile(deb, tgt); err != nil {
return err
}
}
// XXX: Adds too many packages in enterprise (Arve: What does this mean exactly?)
//nolint:gosec
cmd := exec.Command("aptly", "repo", "add", "-force-replace", repoName, tmpDir)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1)
}
return nil
}

363
pkg/build/packaging/rpm.go
Unescape View File

@ -1 +1,364 @@
package packaging
import (
"bytes"
"crypto"
"fmt"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/infra/fs"
"golang.org/x/crypto/openpgp"
"golang.org/x/crypto/openpgp/armor"
"golang.org/x/crypto/openpgp/packet"
)
// UpdateRPMRepo updates the RPM repository with the new release.
func UpdateRPMRepo(cfg PublishConfig, workDir string) error {
if cfg.ReleaseMode.Mode != config.TagMode {
panic(fmt.Sprintf("Unsupported version mode %s", cfg.ReleaseMode.Mode))
}
if cfg.ReleaseMode.IsTest && cfg.Config.RPMRepoBucket == DefaultRPMRepoBucket {
return fmt.Errorf("in test-release mode, the default RPM repo bucket shouldn't be used")
}
if err := downloadRPMs(cfg, workDir); err != nil {
return err
}
repoRoot := path.Join(os.TempDir(), "rpm-repo")
defer func() {
if err := os.RemoveAll(repoRoot); err != nil {
log.Printf("Failed to remove %q: %s\n", repoRoot, err.Error())
}
}()
repoName := "rpm"
if cfg.ReleaseMode.IsBeta {
repoName = "rpm-beta"
}
folderURI := fmt.Sprintf("gs://%s/%s/%s", cfg.RPMRepoBucket, strings.ToLower(string(cfg.Edition)), repoName)
// Download the RPM database
log.Printf("Downloading RPM database from GCS (%s)...\n", folderURI)
//nolint:gosec
cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", folderURI, repoRoot)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download RPM database from GCS: %w\n%s", err, output)
}
// Add the new release to the repo
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unsupported edition %q", cfg.Edition)
}
allRPMs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s-*.rpm", sfx)))
if err != nil {
return fmt.Errorf("failed to list RPMs in %q: %w", workDir, err)
}
rpms := []string{}
for _, rpm := range allRPMs {
if strings.Contains(rpm, "-latest") {
continue
}
rpms = append(rpms, rpm)
}
// XXX: What does the following comment mean?
// adds to many files for enterprise
for _, rpm := range rpms {
if err := fs.CopyFile(rpm, filepath.Join(repoRoot, filepath.Base(rpm))); err != nil {
return err
}
}
//nolint:gosec
cmd = exec.Command("createrepo", repoRoot)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create repo at %q: %w\n%s", repoRoot, err, output)
}
if err := signRPMRepo(repoRoot, cfg); err != nil {
return err
}
// Update the repo in GCS
// Sync packages first to avoid cache misses
if cfg.DryRun {
log.Printf("Simulating upload of RPMs to GCS (%s)\n", folderURI)
} else {
log.Printf("Uploading RPMs to GCS (%s)...\n", folderURI)
args := []string{"-m", "cp"}
args = append(args, rpms...)
args = append(args, folderURI)
//nolint:gosec
cmd = exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to upload RPMs to GCS: %w\n%s", err, output)
}
}
if cfg.DryRun {
log.Printf("Simulating upload of RPM repo metadata to GCS (%s)\n", folderURI)
} else {
log.Printf("Uploading RPM repo metadata to GCS (%s)...\n", folderURI)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", repoRoot, folderURI)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to upload RPM repo metadata to GCS: %w\n%s", err, output)
}
allRepoResources := fmt.Sprintf("%s/**/*", folderURI)
log.Printf("Setting cache ttl for RPM repo resources on GCS (%s)...\n", allRepoResources)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to set cache ttl for RPM repo resources on GCS: %w\n%s", err, output)
}
}
return nil
}
// downloadRPMs downloads RPM packages.
func downloadRPMs(cfg PublishConfig, workDir string) error {
if !strings.HasSuffix(workDir, string(filepath.Separator)) {
workDir += string(filepath.Separator)
}
var version string
if cfg.ReleaseMode.Mode == config.TagMode {
if cfg.ReleaseMode.IsBeta {
version = strings.ReplaceAll(cfg.Version, "-", "~")
} else {
version = cfg.Version
}
}
if version == "" {
panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode))
}
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
u := fmt.Sprintf("gs://%s/%s/%s/grafana%s-%s-*.*.rpm*", cfg.Bucket,
strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version)
log.Printf("Downloading RPM packages %q...\n", u)
args := []string{
"-m",
"cp",
u,
workDir,
}
//nolint:gosec
cmd := exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download RPM packages %q: %w\n%s", u, err, output)
}
return nil
}
func getPublicKey(cfg PublishConfig) (*packet.PublicKey, error) {
f, err := os.Open(cfg.GPGPublicKey)
if err != nil {
return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPublicKey, err)
}
defer func(f *os.File) {
err := f.Close()
if err != nil {
return
}
}(f)
block, err := armor.Decode(f)
if err != nil {
return nil, err
}
if block.Type != openpgp.PublicKeyType {
return nil, fmt.Errorf("invalid public key block type: %q", block.Type)
}
packetReader := packet.NewReader(block.Body)
pkt, err := packetReader.Next()
if err != nil {
return nil, err
}
key, ok := pkt.(*packet.PublicKey)
if !ok {
return nil, fmt.Errorf("got non-public key from packet reader: %T", pkt)
}
return key, nil
}
func getPrivateKey(cfg PublishConfig) (*packet.PrivateKey, error) {
f, err := os.Open(cfg.GPGPrivateKey)
if err != nil {
return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPrivateKey, err)
}
defer func(f *os.File) {
err := f.Close()
if err != nil {
return
}
}(f)
passphraseB, err := os.ReadFile(cfg.GPGPassPath)
if err != nil {
return nil, fmt.Errorf("failed to read %q: %w", cfg.GPGPrivateKey, err)
}
passphraseB = bytes.TrimSuffix(passphraseB, []byte("\n"))
block, err := armor.Decode(f)
if err != nil {
return nil, err
}
if block.Type != openpgp.PrivateKeyType {
return nil, fmt.Errorf("invalid private key block type: %q", block.Type)
}
packetReader := packet.NewReader(block.Body)
pkt, err := packetReader.Next()
if err != nil {
return nil, err
}
key, ok := pkt.(*packet.PrivateKey)
if !ok {
return nil, fmt.Errorf("got non-private key from packet reader: %T", pkt)
}
if err := key.Decrypt(passphraseB); err != nil {
return nil, fmt.Errorf("failed to decrypt private key: %w", err)
}
return key, nil
}
// signRPMRepo signs an RPM repository using PGP.
// The signature gets written to the file repodata/repomd.xml.asc.
func signRPMRepo(repoRoot string, cfg PublishConfig) error {
if cfg.GPGPublicKey == "" || cfg.GPGPrivateKey == "" {
return fmt.Errorf("private or public key is empty")
}
log.Printf("Signing RPM repo")
pubKey, err := getPublicKey(cfg)
if err != nil {
return err
}
privKey, err := getPrivateKey(cfg)
if err != nil {
return err
}
pcfg := packet.Config{
DefaultHash: crypto.SHA256,
DefaultCipher: packet.CipherAES256,
DefaultCompressionAlgo: packet.CompressionZLIB,
CompressionConfig: &packet.CompressionConfig{
Level: 9,
},
RSABits: 4096,
}
currentTime := pcfg.Now()
uid := packet.NewUserId("", "", "")
isPrimaryID := false
keyLifetimeSecs := uint32(86400 * 365)
signer := openpgp.Entity{
PrimaryKey: pubKey,
PrivateKey: privKey,
Identities: map[string]*openpgp.Identity{
uid.Id: {
Name: uid.Name,
UserId: uid,
SelfSignature: &packet.Signature{
CreationTime: currentTime,
SigType: packet.SigTypePositiveCert,
PubKeyAlgo: packet.PubKeyAlgoRSA,
Hash: pcfg.Hash(),
IsPrimaryId: &isPrimaryID,
FlagsValid: true,
FlagSign: true,
FlagCertify: true,
IssuerKeyId: &pubKey.KeyId,
},
},
},
Subkeys: []openpgp.Subkey{
{
PublicKey: pubKey,
PrivateKey: privKey,
Sig: &packet.Signature{
CreationTime: currentTime,
SigType: packet.SigTypeSubkeyBinding,
PubKeyAlgo: packet.PubKeyAlgoRSA,
Hash: pcfg.Hash(),
PreferredHash: []uint8{8}, // SHA-256
FlagsValid: true,
FlagEncryptStorage: true,
FlagEncryptCommunications: true,
IssuerKeyId: &pubKey.KeyId,
KeyLifetimeSecs: &keyLifetimeSecs,
},
},
},
}
// Ignore gosec G304 as this function is only used in the build process.
//nolint:gosec
freader, err := os.Open(filepath.Join(repoRoot, "repodata", "repomd.xml"))
if err != nil {
return err
}
defer func(freader *os.File) {
err := freader.Close()
if err != nil {
return
}
}(freader)
// Ignore gosec G304 as this function is only used in the build process.
//nolint:gosec
sigwriter, err := os.Create(filepath.Join(repoRoot, "repodata", "repomd.xml.asc"))
if err != nil {
return err
}
defer func(sigwriter *os.File) {
err := sigwriter.Close()
if err != nil {
return
}
}(sigwriter)
if err := openpgp.ArmoredDetachSignText(sigwriter, &signer, freader, nil); err != nil {
return fmt.Errorf("failed to write PGP signature: %w", err)
}
if err := sigwriter.Close(); err != nil {
return fmt.Errorf("failed to write PGP signature: %w", err)
}
return nil
}

2
pkg/build/cmd/rpm_test.go → pkg/build/packaging/rpm_test.go
Unescape View File

@ -1,4 +1,4 @@
package main
package packaging
import (
"os"
Write Preview
Loading…
Cancel
Save