SecretsService/Chore: restructure secret storage packages (#100053)

restructure storage
pull/99901/head
Dana Axinte 4 months ago committed by GitHub
parent beb94c4ef1
commit e926493c6c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 6
      pkg/registry/apis/secret/encryption/fakes/fake_store.go
  2. 4
      pkg/registry/apis/secret/encryption/manager/helpers.go
  3. 12
      pkg/registry/apis/secret/encryption/manager/manager.go
  4. 18
      pkg/registry/apis/secret/encryption/manager/manager_test.go
  5. 4
      pkg/registry/apis/secret/secretkeeper/fakes/fake_keeper.go
  6. 6
      pkg/registry/apis/secret/secretkeeper/secretkeeper.go
  7. 6
      pkg/registry/apis/secret/secretkeeper/secretkeeper_test.go
  8. 8
      pkg/registry/apis/secret/secretkeeper/sqlkeeper/keeper.go
  9. 6
      pkg/registry/apis/secret/secretkeeper/sqlkeeper/keeper_test.go
  10. 11
      pkg/server/wire.go
  11. 2
      pkg/storage/secret/encryption/data_key_model.go
  12. 7
      pkg/storage/secret/encryption/data_key_store.go
  13. 2
      pkg/storage/secret/encryption/data_key_store_test.go
  14. 6
      pkg/storage/secret/encryption/encrypted_value_model.go
  15. 2
      pkg/storage/secret/encryption/encrypted_value_store.go
  16. 2
      pkg/storage/secret/encryption/encrypted_value_store_test.go
  17. 5
      pkg/storage/secret/metadata/keeper_model.go
  18. 2
      pkg/storage/secret/metadata/keeper_store.go
  19. 5
      pkg/storage/secret/metadata/secure_value_model.go
  20. 5
      pkg/storage/secret/metadata/secure_value_store.go
  21. 4
      pkg/storage/secret/migrator/migrator.go

@ -4,7 +4,7 @@ import (
"context"
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption"
"github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
)
type FakeEncryptionStore struct {
@ -18,7 +18,7 @@ func NewFakeEncryptionStore() FakeEncryptionStore {
func (f FakeEncryptionStore) GetDataKey(_ context.Context, id string) (*encryption.DataKey, error) {
key, ok := f.store[id]
if !ok {
return nil, secret.ErrDataKeyNotFound
return nil, encryptionstorage.ErrDataKeyNotFound
}
return key, nil
@ -31,7 +31,7 @@ func (f FakeEncryptionStore) GetCurrentDataKey(_ context.Context, label string)
}
}
return nil, secret.ErrDataKeyNotFound
return nil, encryptionstorage.ErrDataKeyNotFound
}
func (f FakeEncryptionStore) GetAllDataKeys(_ context.Context) ([]*encryption.DataKey, error) {

@ -14,7 +14,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/kmsproviders/osskmsproviders"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
)
func setupTestService(tb testing.TB) *EncryptionManager {
@ -34,7 +34,7 @@ func setupTestService(tb testing.TB) *EncryptionManager {
require.NoError(tb, err)
cfg := &setting.Cfg{Raw: raw}
store, err := secret.ProvideDataKeyStorageStorage(testDB, cfg, features)
store, err := encryptionstorage.ProvideDataKeyStorageStorage(testDB, cfg, features)
require.NoError(tb, err)
encProvider := encryptionprovider.Provider{}

@ -22,7 +22,7 @@ import (
"github.com/grafana/grafana/pkg/services/kmsproviders"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
"github.com/grafana/grafana/pkg/util"
)
@ -38,7 +38,7 @@ var (
type EncryptionManager struct {
tracer tracing.Tracer
store secret.DataKeyStorage
store encryptionstorage.DataKeyStorage
enc legacyEncryption.Internal
cfg *setting.Cfg
usageStats usagestats.Service
@ -57,7 +57,7 @@ type EncryptionManager struct {
func NewEncryptionManager(
tracer tracing.Tracer,
store secret.DataKeyStorage,
store encryptionstorage.DataKeyStorage,
kmsProvidersService kmsproviders.Service,
enc legacyEncryption.Internal,
cfg *setting.Cfg,
@ -219,7 +219,7 @@ func (s *EncryptionManager) dataKeyByLabel(ctx context.Context, namespace, label
// 1. Get data key from database.
dataKey, err := s.store.GetCurrentDataKey(ctx, namespace, label)
if err != nil {
if errors.Is(err, secret.ErrDataKeyNotFound) {
if errors.Is(err, encryptionstorage.ErrDataKeyNotFound) {
return "", nil, nil
}
return "", nil, err
@ -266,7 +266,7 @@ func (s *EncryptionManager) newDataKey(ctx context.Context, namespace string, la
// 3. Store its encrypted value into the DB.
id := util.GenerateShortUID()
dbDataKey := secret.SecretDataKey{
dbDataKey := encryptionstorage.SecretDataKey{
Active: true,
UID: id,
Namespace: namespace,
@ -474,7 +474,7 @@ func (s *EncryptionManager) Run(ctx context.Context) error {
// Look at the comments inline for further details.
// You can also take a look at the issue below for more context:
// https://github.com/grafana/grafana-enterprise/issues/4252
func (s *EncryptionManager) cacheDataKey(dataKey *secret.SecretDataKey, decrypted []byte) {
func (s *EncryptionManager) cacheDataKey(dataKey *encryptionstorage.SecretDataKey, decrypted []byte) {
// First, we cache the data key by id, because cache "by id" is
// only used by decrypt operations, so no risk of corrupting data.
entry := &dataKeyCacheEntry{

@ -21,7 +21,7 @@ import (
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
"github.com/grafana/grafana/pkg/tests/testsuite"
"github.com/grafana/grafana/pkg/util"
)
@ -103,13 +103,13 @@ func TestEncryptionService_DataKeys(t *testing.T) {
data_keys_cache_cleanup_interval = 1ns`))
require.NoError(t, err)
cfg := &setting.Cfg{Raw: raw}
store, err := secret.ProvideDataKeyStorageStorage(testDB, cfg, features)
store, err := encryptionstorage.ProvideDataKeyStorageStorage(testDB, cfg, features)
require.NoError(t, err)
ctx := context.Background()
namespace := "test-namespace"
dataKey := &secret.SecretDataKey{
dataKey := &encryptionstorage.SecretDataKey{
UID: util.GenerateShortUID(),
Label: "test1",
Active: true,
@ -120,7 +120,7 @@ func TestEncryptionService_DataKeys(t *testing.T) {
t.Run("querying for a DEK that does not exist", func(t *testing.T) {
res, err := store.GetDataKey(ctx, namespace, dataKey.UID)
assert.ErrorIs(t, secret.ErrDataKeyNotFound, err)
assert.ErrorIs(t, encryptionstorage.ErrDataKeyNotFound, err)
assert.Nil(t, res)
})
@ -146,7 +146,7 @@ func TestEncryptionService_DataKeys(t *testing.T) {
})
t.Run("creating an inactive DEK", func(t *testing.T) {
k := &secret.SecretDataKey{
k := &encryptionstorage.SecretDataKey{
UID: util.GenerateShortUID(),
Namespace: namespace,
Active: false,
@ -159,7 +159,7 @@ func TestEncryptionService_DataKeys(t *testing.T) {
require.Error(t, err)
res, err := store.GetDataKey(ctx, namespace, k.UID)
assert.Equal(t, secret.ErrDataKeyNotFound, err)
assert.Equal(t, encryptionstorage.ErrDataKeyNotFound, err)
assert.Nil(t, res)
})
@ -179,7 +179,7 @@ func TestEncryptionService_DataKeys(t *testing.T) {
require.NoError(t, err)
res, err := store.GetDataKey(ctx, namespace, dataKey.UID)
assert.Equal(t, secret.ErrDataKeyNotFound, err)
assert.Equal(t, encryptionstorage.ErrDataKeyNotFound, err)
assert.Nil(t, res)
})
}
@ -214,7 +214,7 @@ func TestEncryptionService_UseCurrentProvider(t *testing.T) {
features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
kms := newFakeKMS(osskmsproviders.ProvideService(encryptionService, cfg, features))
testDB := db.InitTestDB(t)
encryptionStore, err := secret.ProvideDataKeyStorageStorage(testDB, &setting.Cfg{}, features)
encryptionStore, err := encryptionstorage.ProvideDataKeyStorageStorage(testDB, &setting.Cfg{}, features)
require.NoError(t, err)
encryptionManager, err := NewEncryptionManager(
@ -518,7 +518,7 @@ func TestIntegration_SecretsService(t *testing.T) {
require.NoError(t, err)
cfg := &setting.Cfg{Raw: raw}
store, err := secret.ProvideDataKeyStorageStorage(testDB, cfg, features)
store, err := encryptionstorage.ProvideDataKeyStorageStorage(testDB, cfg, features)
require.NoError(t, err)
encProvider := encryptionprovider.Provider{}

@ -9,7 +9,7 @@ import (
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/manager"
keepertypes "github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper/types"
secretStorage "github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
)
type FakeKeeper struct {
@ -18,7 +18,7 @@ type FakeKeeper struct {
var _ keepertypes.Keeper = (*FakeKeeper)(nil)
func NewFakeKeeper(tracer tracing.Tracer, encryptionManager *manager.EncryptionManager, store secretStorage.EncryptedValueStorage) (*FakeKeeper, error) {
func NewFakeKeeper(tracer tracing.Tracer, encryptionManager *manager.EncryptionManager, store encryptionstorage.EncryptedValueStorage) (*FakeKeeper, error) {
return &FakeKeeper{
values: make(map[string]map[string]string),
}, nil

@ -7,7 +7,7 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/manager"
"github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper/sqlkeeper"
keepertypes "github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper/types"
secretstorage "github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
)
type Service interface {
@ -17,10 +17,10 @@ type Service interface {
type OSSKeeperService struct {
tracer tracing.Tracer
encryptionManager *manager.EncryptionManager
store secretstorage.EncryptedValueStorage
store encryptionstorage.EncryptedValueStorage
}
func ProvideService(encryptionManager *manager.EncryptionManager, store secretstorage.EncryptedValueStorage) (OSSKeeperService, error) {
func ProvideService(encryptionManager *manager.EncryptionManager, store encryptionstorage.EncryptedValueStorage) (OSSKeeperService, error) {
return OSSKeeperService{
encryptionManager: encryptionManager,
store: store,

@ -18,7 +18,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/kmsproviders/osskmsproviders"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
"github.com/grafana/grafana/pkg/tests/testsuite"
)
@ -55,10 +55,10 @@ func setupTestService(t *testing.T, config string) (OSSKeeperService, error) {
cfg := &setting.Cfg{Raw: raw}
features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
dataKeyStore, err := secret.ProvideDataKeyStorageStorage(testDB, cfg, features)
dataKeyStore, err := encryptionstorage.ProvideDataKeyStorageStorage(testDB, cfg, features)
require.NoError(t, err)
encValueStore, err := secret.ProvideEncryptedValueStorage(testDB, cfg, features)
encValueStore, err := encryptionstorage.ProvideEncryptedValueStorage(testDB, cfg, features)
require.NoError(t, err)
// Initialize the encryption manager

@ -10,18 +10,18 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption"
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/manager"
keepertypes "github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper/types"
secretStorage "github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
)
type SQLKeeper struct {
tracer tracing.Tracer
encryptionManager *manager.EncryptionManager
store secretStorage.EncryptedValueStorage
store encryptionstorage.EncryptedValueStorage
}
var _ keepertypes.Keeper = (*SQLKeeper)(nil)
func NewSQLKeeper(tracer tracing.Tracer, encryptionManager *manager.EncryptionManager, store secretStorage.EncryptedValueStorage) (*SQLKeeper, error) {
func NewSQLKeeper(tracer tracing.Tracer, encryptionManager *manager.EncryptionManager, store encryptionstorage.EncryptedValueStorage) (*SQLKeeper, error) {
return &SQLKeeper{
tracer: tracer,
encryptionManager: encryptionManager,
@ -52,7 +52,7 @@ func (s *SQLKeeper) Expose(ctx context.Context, cfg secretv0alpha1.KeeperConfig,
encryptedValue, err := s.store.Get(ctx, externalID.String())
if err != nil {
if errors.Is(err, secretStorage.ErrEncryptedValueNotFound) {
if errors.Is(err, encryptionstorage.ErrEncryptedValueNotFound) {
return "", keepertypes.ErrSecretNotFound
}
return "", fmt.Errorf("unable to get encrypted value: %w", err)

@ -18,7 +18,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/kmsproviders/osskmsproviders"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
"github.com/grafana/grafana/pkg/tests/testsuite"
)
@ -135,7 +135,7 @@ func setupTestService(t *testing.T, config string) (*SQLKeeper, error) {
features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
// Initialize the encryption manager
dataKeyStore, err := secret.ProvideDataKeyStorageStorage(testDB, cfg, features)
dataKeyStore, err := encryptionstorage.ProvideDataKeyStorageStorage(testDB, cfg, features)
require.NoError(t, err)
encProvider := encryptionprovider.Provider{}
@ -154,7 +154,7 @@ func setupTestService(t *testing.T, config string) (*SQLKeeper, error) {
require.NoError(t, err)
// Initialize encrypted value storage with a fake db
encValueStore, err := secret.ProvideEncryptedValueStorage(testDB, cfg, features)
encValueStore, err := encryptionstorage.ProvideEncryptedValueStorage(testDB, cfg, features)
require.NoError(t, err)
// Initialize the SQLKeeper

@ -157,7 +157,8 @@ import (
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/services/user/userimpl"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret"
secretencryption "github.com/grafana/grafana/pkg/storage/secret/encryption"
secretmetadata "github.com/grafana/grafana/pkg/storage/secret/metadata"
"github.com/grafana/grafana/pkg/storage/unified"
unifiedsearch "github.com/grafana/grafana/pkg/storage/unified/search"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor"
@ -275,10 +276,10 @@ var wireBasicSet = wire.NewSet(
jaeger.ProvideService,
datasourceservice.ProvideCacheService,
wire.Bind(new(datasources.CacheService), new(*datasourceservice.CacheServiceImpl)),
secret.ProvideSecureValueStorage,
secret.ProvideDataKeyStorageStorage,
secret.ProvideKeeperStorage,
secret.ProvideEncryptedValueStorage,
secretmetadata.ProvideSecureValueStorage,
secretmetadata.ProvideKeeperStorage,
secretencryption.ProvideDataKeyStorageStorage,
secretencryption.ProvideEncryptedValueStorage,
manager.NewEncryptionManager,
encryptionservice.ProvideEncryptionService,
wire.Bind(new(encryption.Internal), new(*encryptionservice.Service)),

@ -1,4 +1,4 @@
package secret
package encryption
import (
"context"
@ -12,6 +12,7 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
)
var (
@ -42,7 +43,7 @@ func ProvideDataKeyStorageStorage(db db.DB, cfg *setting.Cfg, features featuremg
return &encryptionStoreImpl{}, nil
}
if err := migrateSecretSQL(db.GetEngine(), cfg); err != nil {
if err := migrator.MigrateSecretSQL(db.GetEngine(), cfg); err != nil {
return nil, fmt.Errorf("failed to run migrations: %w", err)
}
@ -250,7 +251,7 @@ func (ss *encryptionStoreImpl) ReEncryptDataKeys(
%s.encrypted_data = updates.encrypted_data,
%s.provider = '%s',
%s.updated = '%s'
`, strings.Join(selectStatements, " UNION ALL "), TableNameDataKey, TableNameDataKey, TableNameDataKey, TableNameDataKey, TableNameDataKey, currProvider, TableNameDataKey, time.Now().UTC().Format("2006-01-02 15:04:05"))
`, strings.Join(selectStatements, " UNION ALL "), migrator.TableNameDataKey, migrator.TableNameDataKey, migrator.TableNameDataKey, migrator.TableNameDataKey, migrator.TableNameDataKey, currProvider, migrator.TableNameDataKey, time.Now().UTC().Format("2006-01-02 15:04:05"))
fmt.Println(rawSql)
if err := ss.db.WithDbSession(ctx, func(sess *db.Session) error {

@ -1,4 +1,6 @@
package secret
package encryption
import "github.com/grafana/grafana/pkg/storage/secret/migrator"
type EncryptedValue struct {
UID string `xorm:"pk 'uid'"`
@ -8,5 +10,5 @@ type EncryptedValue struct {
}
func (*EncryptedValue) TableName() string {
return TableNameEncryptedValue
return migrator.TableNameEncryptedValue
}

@ -1,4 +1,4 @@
package secret
package metadata
import (
"encoding/json"
@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/apimachinery/utils"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
@ -42,7 +43,7 @@ type keeperDB struct {
}
func (*keeperDB) TableName() string {
return TableNameKeeper
return migrator.TableNameKeeper
}
// toKubernetes maps a DB row into a Kubernetes resource (metadata + spec).

@ -1,4 +1,4 @@
package secret
package metadata
import (
"context"

@ -1,4 +1,4 @@
package secret
package metadata
import (
"encoding/json"
@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/apimachinery/utils"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
@ -34,7 +35,7 @@ type secureValueDB struct {
}
func (*secureValueDB) TableName() string {
return TableNameSecureValue
return migrator.TableNameSecureValue
}
// toKubernetes maps a DB row into a Kubernetes resource (metadata + spec).

@ -1,4 +1,4 @@
package secret
package metadata
import (
"context"
@ -12,6 +12,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
"k8s.io/apimachinery/pkg/apis/meta/internalversion"
"k8s.io/apimachinery/pkg/labels"
)
@ -22,7 +23,7 @@ func ProvideSecureValueStorage(db db.DB, cfg *setting.Cfg, features featuremgmt.
return &secureValueStorage{}, nil
}
if err := migrateSecretSQL(db.GetEngine(), cfg); err != nil {
if err := migrator.MigrateSecretSQL(db.GetEngine(), cfg); err != nil {
return nil, fmt.Errorf("failed to run migrations: %w", err)
}

@ -1,4 +1,4 @@
package secret
package migrator
import (
"fmt"
@ -16,7 +16,7 @@ const (
TableNameEncryptedValue = "secret_encrypted_value"
)
func migrateSecretSQL(engine *xorm.Engine, cfg *setting.Cfg) error {
func MigrateSecretSQL(engine *xorm.Engine, cfg *setting.Cfg) error {
mg := migrator.NewScopedMigrator(engine, cfg, "secret")
mg.AddCreateMigration()
Loading…
Cancel
Save