apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
The open and composable observability and data visualization platform. Visualize metrics, logs, and traces from multiple sources like Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
36364 Commits
2179 Branches
608 Tags
1.9 GiB
Tag: Branch: Tree: f14b45bc86
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'f14b45bc86'
${ noResults }
grafana/pkg/services/datasources/service/secrets_mig_test.go

341 lines
13 KiB
Raw Normal View History Unescape

Secrets: Improve unified secrets migration and implement compatibility flag (#50463) * Implement disableSecretsCompatibility flag * Allow secret deletion right after migration * Use dialect.Quote for secure_json_data on secret deletion Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com> * Set secure_json_data to NULL instead of empty json * Run toggles_gen_test and use generated flag variable * Add ID to delete data source secrets command on function call Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com> * Remove extra query to get datasource on secret deletion * Fix linting issues with CHANGELOG.md * Use empty json string when deleting secure json data * Implement secret migration as a background process * Refactor secret migration as a background service * Refactor migration to be inside secret store * Re-add secret deletion function removed on merge * Try using transaction to fix db lock during tests * Disable migration for pipeline debugging * Try adding sleep to fix database lock * Remove unecessary time sleep from migration * Fix merge issue, replace models with datasources * Try event listener approach * Fix merge issue, replace models with datasources * Fix linting issues with unchecked error * Remove unecessary trainling new line * Increase wait interval on background secret migration * Rename secret store migration folder for consistency * Convert background migration to blocking * Fix number of arguments on server tests * Check error value of secret migration provider * Fix linting issue with method varaible * Revert unintended change on background services * Move secret migration service provider to wire.go * Remove unecessary else from datasource service * Move transaction inside loop on secret migration * Remove unecessary GetServices function * Remove unecessary interface after method removal * Rename Run to Migrate on secret migration interface * Rename secret migrations service variable on server * Use MustBool on datasource secret migration * Revert changes to GetDataSources * Implement GetAllDataSources function * Remove DeleteDataSourceSecrets function * Move datasource secret migration to datasource service * Remove unecessary properties from datasource secret migration * Make DecryptLegacySecrets a private method * Remove context canceled check on secret migrator * Log error when fail to unmarshal datasource secret * Add necessary fields to update command on migration * Handle high availability on secret migration * Use kvstore for datasource secret migration status * Add error check for migration status set on kvstore * Remove NewSecretMigrationService from server tests * Use const for strings on datasource secrets migration * Test all cases for datasources secret migrations Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>
3 years ago
package service
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/infra/kvstore"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert"
)
func SetupTestMigrationService(t *testing.T, sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, secretsStore secretsStore.SecretsKVStore, compatibility bool) *DataSourceSecretMigrationService {
t.Helper()
cfg := &setting.Cfg{}
features := featuremgmt.WithFeatures()
if !compatibility {
features = featuremgmt.WithFeatures(featuremgmt.FlagDisableSecretsCompatibility, true)
}
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, features, acmock.New().WithDisabled(), acmock.NewMockedPermissionsService())
migService := ProvideDataSourceMigrationService(dsService, kvStore, features)
return migService
}
func TestMigrate(t *testing.T) {
t.Run("should migrate from legacy to unified without compatibility", func(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsStore := secretsStore.SetupTestService(t)
migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgId: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
Url: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.Empty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should migrate from legacy to unified with compatibility", func(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsStore := secretsStore.SetupTestService(t)
migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgId: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
Url: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was maintained for compatibility
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should replicate from unified to legacy for compatibility", func(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsStore := secretsStore.SetupTestService(t)
migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgId: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
Url: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration without compatibility
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.Empty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
// Run the migration with compatibility
migService = SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was re-added for compatibility
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should delete from legacy to remove compatibility", func(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsStore := secretsStore.SetupTestService(t)
migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgId: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
Url: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration with compatibility
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was maintained for compatibility
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
// Run the migration without compatibility
migService = SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.Empty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
})
}