CloudMigrations: Store encryption key in unified secrets table (#90908)

* store encryption key in unified secrets table

* fix local dev mode

* make metadata more realistic

* fix tests

* fix sql queries against postgres

* fix stats endpoint

pkg/services/cloudmigration/api/api.go

@@ -445,6 +445,7 @@ func (cma *CloudMigrationAPI) GetSnapshot(c *contextmodel.ReqContext) response.R
 	dtoStats := SnapshotResourceStats{
 		Types:    make(map[MigrateDataType]int, len(snapshot.StatsRollup.CountsByStatus)),
 		Statuses: make(map[ItemStatus]int, len(snapshot.StatsRollup.CountsByType)),
+		Total:    snapshot.StatsRollup.Total,
 	}
 	for s, c := range snapshot.StatsRollup.CountsByStatus {
 		dtoStats.Statuses[ItemStatus(s)] = c

pkg/services/cloudmigration/api/api_test.go

@@ -471,7 +471,7 @@ func TestCloudMigrationAPI_GetSnapshot(t *testing.T) {
 			requestUrl:         "/api/cloudmigration/migration/1234/snapshot/1",
 			basicRole:          org.RoleAdmin,
 			expectedHttpResult: http.StatusOK,
-			expectedBody:       `{"uid":"fake_uid","status":"CREATING","sessionUid":"1234","created":"0001-01-01T00:00:00Z","finished":"0001-01-01T00:00:00Z","results":[],"stats":{"types":{},"statuses":{}}}`,
+			expectedBody:       `{"uid":"fake_uid","status":"CREATING","sessionUid":"1234","created":"0001-01-01T00:00:00Z","finished":"0001-01-01T00:00:00Z","results":[],"stats":{"types":{},"statuses":{},"total":0}}`,
 		},
 		{
 			desc:               "should return 403 if no used is not admin",

pkg/services/cloudmigration/api/dtos.go

@@ -307,6 +307,7 @@ type GetSnapshotResponseDTO struct {
 type SnapshotResourceStats struct {
 	Types    map[MigrateDataType]int `json:"types"`
 	Statuses map[ItemStatus]int      `json:"statuses"`
+	Total    int                     `json:"total"`
 }
 
 // swagger:parameters getShapshotList

pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go

@@ -28,6 +28,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/folder"
 	"github.com/grafana/grafana/pkg/services/gcom"
 	"github.com/grafana/grafana/pkg/services/secrets"
+	secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/util"
@@ -82,6 +83,7 @@ func ProvideService(
 	features featuremgmt.FeatureToggles,
 	db db.DB,
 	dsService datasources.DataSourceService,
+	secretsStore secretskv.SecretsKVStore,
 	secretsService secrets.Service,
 	routeRegister routing.RouteRegister,
 	prom prometheus.Registerer,
@@ -95,7 +97,7 @@ func ProvideService(
 	}
 
 	s := &Service{
-		store:            &sqlStore{db: db, secretsService: secretsService},
+		store:            &sqlStore{db: db, secretsStore: secretsStore, secretsService: secretsService},
 		log:              log.New(LogPrefix),
 		cfg:              cfg,
 		features:         features,

pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/folder"
 	"github.com/grafana/grafana/pkg/services/folder/foldertest"
 	secretsfakes "github.com/grafana/grafana/pkg/services/secrets/fakes"
+	secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/prometheus/client_golang/prometheus"
@@ -438,6 +439,7 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool) cloudmigration.Servi
 			featuremgmt.FlagDashboardRestore),
 		sqlStore,
 		dsService,
+		secretskv.NewFakeSQLSecretsKVStore(t),
 		secretsService,
 		rr,
 		prometheus.DefaultRegisterer,

pkg/services/cloudmigration/cloudmigrationimpl/xorm_store.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/services/cloudmigration"
 	"github.com/grafana/grafana/pkg/services/secrets"
+	secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/util"
 )
@@ -17,11 +18,13 @@ var _ store = (*sqlStore)(nil)
 
 type sqlStore struct {
 	db             db.DB
+	secretsStore   secretskv.SecretsKVStore
 	secretsService secrets.Service
 }
 
 const (
-	tableName = "cloud_migration_resource"
+	tableName  = "cloud_migration_resource"
+	secretType = "cloudmigration-snapshot-encryption-key"
 )
 
 func (ss *sqlStore) GetMigrationSessionByUID(ctx context.Context, uid string) (*cloudmigration.CloudMigrationSession, error) {
@@ -157,14 +160,14 @@ func (ss *sqlStore) GetMigrationStatusList(ctx context.Context, migrationUID str
 }
 
 func (ss *sqlStore) CreateSnapshot(ctx context.Context, snapshot cloudmigration.CloudMigrationSnapshot) (string, error) {
-	if err := ss.encryptKey(ctx, &snapshot); err != nil {
-		return "", err
-	}
-
 	if snapshot.UID == "" {
 		snapshot.UID = util.GenerateShortUID()
 	}
 
+	if err := ss.secretsStore.Set(ctx, secretskv.AllOrganizations, snapshot.UID, secretType, string(snapshot.EncryptionKey)); err != nil {
+		return "", err
+	}
+
 	err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
 		snapshot.Created = time.Now()
 		snapshot.Updated = time.Now()
@@ -228,8 +231,12 @@ func (ss *sqlStore) GetSnapshotByUID(ctx context.Context, uid string, resultPage
 		return nil, err
 	}
 
-	if err := ss.decryptKey(ctx, &snapshot); err != nil {
+	if secret, found, err := ss.secretsStore.Get(ctx, secretskv.AllOrganizations, snapshot.UID, secretType); err != nil {
 		return &snapshot, err
+	} else if !found {
+		return &snapshot, fmt.Errorf("encryption key not found for snapshot with UID %s", snapshot.UID)
+	} else {
+		snapshot.EncryptionKey = []byte(secret)
 	}
 
 	resources, err := ss.GetSnapshotResources(ctx, uid, resultPage, resultLimit)
@@ -259,8 +266,12 @@ func (ss *sqlStore) GetSnapshotList(ctx context.Context, query cloudmigration.Li
 		return nil, err
 	}
 	for i, snapshot := range snapshots {
-		if err := ss.decryptKey(ctx, &snapshot); err != nil {
+		if secret, found, err := ss.secretsStore.Get(ctx, secretskv.AllOrganizations, snapshot.UID, secretType); err != nil {
 			return nil, err
+		} else if !found {
+			return nil, fmt.Errorf("encryption key not found for snapshot with UID %s", snapshot.UID)
+		} else {
+			snapshot.EncryptionKey = []byte(secret)
 		}
 
 		if stats, err := ss.GetSnapshotResourceStats(ctx, snapshot.UID); err != nil {
@@ -346,14 +357,14 @@ func (ss *sqlStore) GetSnapshotResourceStats(ctx context.Context, snapshotUid st
 		} else {
 			total = int(t)
 		}
-		sess.Select("count(uid) as 'count', resource_type as 'type'").
+		sess.Select("count(uid) as \"count\", resource_type as \"type\"").
 			Table(tableName).
 			GroupBy("type").
 			Where("snapshot_uid = ?", snapshotUid)
 		if err := sess.Find(&typeCounts); err != nil {
 			return err
 		}
-		sess.Select("count(uid) as 'count', status").
+		sess.Select("count(uid) as \"count\", status").
 			Table(tableName).
 			GroupBy("status").
 			Where("snapshot_uid = ?", snapshotUid)
@@ -411,24 +422,3 @@ func (ss *sqlStore) decryptToken(ctx context.Context, cm *cloudmigration.CloudMi
 
 	return nil
 }
-
-func (ss *sqlStore) encryptKey(ctx context.Context, snapshot *cloudmigration.CloudMigrationSnapshot) error {
-	s, err := ss.secretsService.Encrypt(ctx, snapshot.EncryptionKey, secrets.WithoutScope())
-	if err != nil {
-		return fmt.Errorf("encrypting key: %w", err)
-	}
-
-	snapshot.EncryptionKey = s
-
-	return nil
-}
-
-func (ss *sqlStore) decryptKey(ctx context.Context, snapshot *cloudmigration.CloudMigrationSnapshot) error {
-	t, err := ss.secretsService.Decrypt(ctx, snapshot.EncryptionKey)
-	if err != nil {
-		return fmt.Errorf("decrypting key: %w", err)
-	}
-	snapshot.EncryptionKey = t
-
-	return nil
-}

pkg/services/cloudmigration/cloudmigrationimpl/xorm_store_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/services/cloudmigration"
 	fakeSecrets "github.com/grafana/grafana/pkg/services/secrets/fakes"
+	secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/tests/testsuite"
 	"github.com/grafana/grafana/pkg/util"
@@ -274,6 +275,7 @@ func setUpTest(t *testing.T) (*sqlstore.SQLStore, *sqlStore) {
 	s := &sqlStore{
 		db:             testDB,
 		secretsService: fakeSecrets.FakeSecretsService{},
+		secretsStore:   secretskv.NewFakeSQLSecretsKVStore(t),
 	}
 	ctx := context.Background()
 

pkg/services/cloudmigration/gmsclient/inmemory_client.go

@@ -2,6 +2,7 @@ package gmsclient
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 
@@ -51,16 +52,34 @@ func (c *memoryClientImpl) MigrateData(
 	return &result, nil
 }
 
-func (c *memoryClientImpl) StartSnapshot(context.Context, cloudmigration.CloudMigrationSession) (*cloudmigration.StartSnapshotResponse, error) {
+func (c *memoryClientImpl) StartSnapshot(_ context.Context, sess cloudmigration.CloudMigrationSession) (*cloudmigration.StartSnapshotResponse, error) {
 	publicKey, _, err := box.GenerateKey(cryptoRand.Reader)
 	if err != nil {
 		return nil, fmt.Errorf("nacl: generating public and private key: %w", err)
 	}
+
+	snapshotUid := uuid.NewString()
+
+	metadataBuffer, err := json.Marshal(struct {
+		SnapshotID string `json:"snapshotID"`
+		StackID    string `json:"stackID"`
+		Slug       string `json:"slug"`
+	}{
+		SnapshotID: snapshotUid,
+		StackID:    fmt.Sprintf("%d", sess.StackID),
+		Slug:       sess.Slug,
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("marshalling metadata: %w", err)
+	}
+
 	c.snapshot = &cloudmigration.StartSnapshotResponse{
 		EncryptionKey:        publicKey[:],
-		SnapshotID:           uuid.NewString(),
+		SnapshotID:           snapshotUid,
 		MaxItemsPerPartition: 10,
 		Algo:                 "nacl",
+		Metadata:             metadataBuffer,
 	}
 
 	return c.snapshot, nil

pkg/services/cloudmigration/model.go

@@ -37,7 +37,7 @@ type CloudMigrationSnapshot struct {
 	UID            string `xorm:"uid"`
 	SessionUID     string `xorm:"session_uid"`
 	Status         SnapshotStatus
-	EncryptionKey  []byte `xorm:"encryption_key"` // stored in the unified secrets table
+	EncryptionKey  []byte `xorm:"-"` // stored in the unified secrets table
 	LocalDir       string `xorm:"local_directory"`
 	GMSSnapshotUID string `xorm:"gms_snapshot_uid"`
 	ErrorString    string `xorm:"error_string"`

public/api-merged.json

@@ -20421,6 +20421,10 @@
             "format": "int64"
           }
         },
+        "total": {
+          "type": "integer",
+          "format": "int64"
+        },
         "types": {
           "type": "object",
           "additionalProperties": {

public/openapi3.json

@@ -10497,6 +10497,10 @@
             },
             "type": "object"
           },
+          "total": {
+            "format": "int64",
+            "type": "integer"
+          },
           "types": {
             "additionalProperties": {
               "format": "int64",