Security: Add secrets service (#39418)

* Add secrets service

* Revert accidental changes in util encryption

* Make minor changes

Move functional options to models

Revert renaming types to models

* Add context

* Minor change in GetDataKey

* Use CreateDataKeyWithDBSession in CreateDataKey

* Handle empty DEK name in DeleteDataKey

* Rename defaultProvider

* Remove secrets store service

pkg/server/backgroundsvcs/background_services.go

@@ -20,6 +20,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/notifications"
 	"github.com/grafana/grafana/pkg/services/provisioning"
 	"github.com/grafana/grafana/pkg/services/rendering"
+	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/tsdb/azuremonitor"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch"
 	"github.com/grafana/grafana/pkg/tsdb/elasticsearch"
@@ -46,7 +47,7 @@ func ProvideBackgroundServiceRegistry(
 	// Need to make sure these are initialized, is there a better place to put them?
 	_ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service,
 	_ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service,
-	_ *testdatasource.TestDataPlugin, _ *plugindashboards.Service, _ *dashboardsnapshots.Service,
+	_ *testdatasource.TestDataPlugin, _ *plugindashboards.Service, _ *dashboardsnapshots.Service, _ secrets.SecretsService,
 	_ *postgres.Service, _ *mysql.Service, _ *mssql.Service, _ *grafanads.Service,
 
 ) *BackgroundServiceRegistry {

pkg/server/wire.go

@@ -49,6 +49,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/rendering"
 	"github.com/grafana/grafana/pkg/services/schemaloader"
 	"github.com/grafana/grafana/pkg/services/search"
+	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/services/shorturls"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
@@ -142,6 +143,7 @@ var wireBasicSet = wire.NewSet(
 	graphite.ProvideService,
 	prometheus.ProvideService,
 	elasticsearch.ProvideService,
+	secrets.ProvideSecretsService,
 	grafanads.ProvideService,
 	dashboardsnapshots.ProvideService,
 )

pkg/services/secrets/database.go

@@ -0,0 +1,78 @@
+package secrets
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/secrets/types"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+)
+
+const dataKeysTable = "data_keys"
+
+var logger = log.New("secrets-store")
+
+func (s *SecretsService) GetDataKey(ctx context.Context, name string) (*types.DataKey, error) {
+	dataKey := &types.DataKey{}
+	var exists bool
+
+	err := s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+		var err error
+		exists, err = sess.Table(dataKeysTable).
+			Where("name = ? AND active = ?", name, s.sqlStore.Dialect.BooleanStr(true)).
+			Get(dataKey)
+		return err
+	})
+
+	if !exists {
+		return nil, types.ErrDataKeyNotFound
+	}
+
+	if err != nil {
+		logger.Error("Failed getting data key", "err", err, "name", name)
+		return nil, fmt.Errorf("failed getting data key: %w", err)
+	}
+
+	return dataKey, nil
+}
+
+func (s *SecretsService) GetAllDataKeys(ctx context.Context) ([]*types.DataKey, error) {
+	result := make([]*types.DataKey, 0)
+	err := s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+		err := sess.Table(dataKeysTable).Find(&result)
+		return err
+	})
+	return result, err
+}
+
+func (s *SecretsService) CreateDataKey(ctx context.Context, dataKey types.DataKey) error {
+	return s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+		return s.CreateDataKeyWithDBSession(ctx, dataKey, sess)
+	})
+}
+
+func (s *SecretsService) CreateDataKeyWithDBSession(_ context.Context, dataKey types.DataKey, sess *sqlstore.DBSession) error {
+	if !dataKey.Active {
+		return fmt.Errorf("cannot insert deactivated data keys")
+	}
+
+	dataKey.Created = time.Now()
+	dataKey.Updated = dataKey.Created
+
+	_, err := sess.Table(dataKeysTable).Insert(&dataKey)
+	return err
+}
+
+func (s *SecretsService) DeleteDataKey(ctx context.Context, name string) error {
+	if len(name) == 0 {
+		return fmt.Errorf("data key name is missing")
+	}
+
+	return s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+		_, err := sess.Table(dataKeysTable).Delete(&types.DataKey{Name: name})
+
+		return err
+	})
+}

pkg/services/secrets/grafana_provider.go

@@ -0,0 +1,28 @@
+package secrets
+
+import (
+	"github.com/grafana/grafana/pkg/services/encryption"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+type grafanaProvider struct {
+	settings   setting.Provider
+	encryption encryption.Service
+}
+
+func newGrafanaProvider(settings setting.Provider, encryption encryption.Service) grafanaProvider {
+	return grafanaProvider{
+		settings:   settings,
+		encryption: encryption,
+	}
+}
+
+func (p grafanaProvider) Encrypt(blob []byte) ([]byte, error) {
+	key := p.settings.KeyValue("security", "secret_key").Value()
+	return p.encryption.Encrypt(blob, key)
+}
+
+func (p grafanaProvider) Decrypt(blob []byte) ([]byte, error) {
+	key := p.settings.KeyValue("security", "secret_key").Value()
+	return p.encryption.Decrypt(blob, key)
+}

pkg/services/secrets/secrets.go

@@ -0,0 +1,269 @@
+package secrets
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+
+	"github.com/grafana/grafana/pkg/services/encryption"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/services/secrets/types"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+const defaultProvider = "secretKey"
+
+type SecretsService struct {
+	sqlStore *sqlstore.SQLStore
+	bus      bus.Bus
+	enc      encryption.Service
+	settings setting.Provider
+
+	defaultProvider string
+	providers       map[string]Provider
+	dataKeyCache    map[string]dataKeyCacheItem
+}
+
+func ProvideSecretsService(sqlStore *sqlstore.SQLStore, bus bus.Bus, enc encryption.Service, settings setting.Provider) SecretsService {
+	providers := map[string]Provider{
+		defaultProvider: newGrafanaProvider(settings, enc),
+	}
+
+	s := SecretsService{
+		sqlStore:        sqlStore,
+		bus:             bus,
+		enc:             enc,
+		settings:        settings,
+		defaultProvider: defaultProvider,
+		providers:       providers,
+		dataKeyCache:    make(map[string]dataKeyCacheItem),
+	}
+
+	return s
+}
+
+type dataKeyCacheItem struct {
+	expiry  time.Time
+	dataKey []byte
+}
+
+type Provider interface {
+	Encrypt(blob []byte) ([]byte, error)
+	Decrypt(blob []byte) ([]byte, error)
+}
+
+var b64 = base64.RawStdEncoding
+
+type EncryptionOptions func() string
+
+// WithoutScope uses a root level data key for encryption (DEK),
+// in other words this DEK is not bound to any specific scope (not attached to any user, org, etc.).
+func WithoutScope() EncryptionOptions {
+	return func() string {
+		return "root"
+	}
+}
+
+// WithScope uses a data key for encryption bound to some specific scope (i.e., user, org, etc.).
+// Scope should look like "user:10", "org:1".
+func WithScope(scope string) EncryptionOptions {
+	return func() string {
+		return scope
+	}
+}
+
+func (s *SecretsService) Encrypt(ctx context.Context, payload []byte, opt EncryptionOptions) ([]byte, error) {
+	scope := opt()
+	keyName := fmt.Sprintf("%s/%s@%s", time.Now().Format("2006-01-02"), scope, s.defaultProvider)
+
+	dataKey, err := s.dataKey(ctx, keyName)
+	if err != nil {
+		if errors.Is(err, types.ErrDataKeyNotFound) {
+			dataKey, err = s.newDataKey(ctx, keyName, scope)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	encrypted, err := s.enc.Encrypt(payload, string(dataKey))
+	if err != nil {
+		return nil, err
+	}
+
+	prefix := make([]byte, b64.EncodedLen(len(keyName))+2)
+	b64.Encode(prefix[1:], []byte(keyName))
+	prefix[0] = '#'
+	prefix[len(prefix)-1] = '#'
+
+	blob := make([]byte, len(prefix)+len(encrypted))
+	copy(blob, prefix)
+	copy(blob[len(prefix):], encrypted)
+
+	return blob, nil
+}
+
+func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, error) {
+	if len(payload) == 0 {
+		return nil, fmt.Errorf("unable to decrypt empty payload")
+	}
+
+	var dataKey []byte
+
+	if payload[0] != '#' {
+		secretKey := s.settings.KeyValue("security", "secret_key").Value()
+		dataKey = []byte(secretKey)
+	} else {
+		payload = payload[1:]
+		endOfKey := bytes.Index(payload, []byte{'#'})
+		if endOfKey == -1 {
+			return nil, fmt.Errorf("could not find valid key in encrypted payload")
+		}
+		b64Key := payload[:endOfKey]
+		payload = payload[endOfKey+1:]
+		key := make([]byte, b64.DecodedLen(len(b64Key)))
+		_, err := b64.Decode(key, b64Key)
+		if err != nil {
+			return nil, err
+		}
+
+		dataKey, err = s.dataKey(ctx, string(key))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return s.enc.Decrypt(payload, string(dataKey))
+}
+
+func (s *SecretsService) EncryptJsonData(ctx context.Context, kv map[string]string, opt EncryptionOptions) (map[string][]byte, error) {
+	encrypted := make(map[string][]byte)
+	for key, value := range kv {
+		encryptedData, err := s.Encrypt(ctx, []byte(value), opt)
+		if err != nil {
+			return nil, err
+		}
+
+		encrypted[key] = encryptedData
+	}
+	return encrypted, nil
+}
+
+func (s *SecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) {
+	decrypted := make(map[string]string)
+	for key, data := range sjd {
+		decryptedData, err := s.Decrypt(ctx, data)
+		if err != nil {
+			return nil, err
+		}
+
+		decrypted[key] = string(decryptedData)
+	}
+	return decrypted, nil
+}
+
+func (s *SecretsService) GetDecryptedValue(ctx context.Context, sjd map[string][]byte, key, fallback string) string {
+	if value, ok := sjd[key]; ok {
+		decryptedData, err := s.Decrypt(ctx, value)
+		if err != nil {
+			return fallback
+		}
+
+		return string(decryptedData)
+	}
+
+	return fallback
+}
+
+func newRandomDataKey() ([]byte, error) {
+	rawDataKey := make([]byte, 16)
+	_, err := rand.Read(rawDataKey)
+	if err != nil {
+		return nil, err
+	}
+	return rawDataKey, nil
+}
+
+// newDataKey creates a new random DEK, caches it and returns its value
+func (s *SecretsService) newDataKey(ctx context.Context, name string, scope string) ([]byte, error) {
+	// 1. Create new DEK
+	dataKey, err := newRandomDataKey()
+	if err != nil {
+		return nil, err
+	}
+	provider, exists := s.providers[s.defaultProvider]
+	if !exists {
+		return nil, fmt.Errorf("could not find encryption provider '%s'", s.defaultProvider)
+	}
+
+	// 2. Encrypt it
+	encrypted, err := provider.Encrypt(dataKey)
+	if err != nil {
+		return nil, err
+	}
+
+	// 3. Store its encrypted value in db
+	err = s.CreateDataKey(ctx, types.DataKey{
+		Active:        true, // TODO: right now we never mark a key as deactivated
+		Name:          name,
+		Provider:      s.defaultProvider,
+		EncryptedData: encrypted,
+		Scope:         scope,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// 4. Cache its unencrypted value and return it
+	s.dataKeyCache[name] = dataKeyCacheItem{
+		expiry:  time.Now().Add(15 * time.Minute),
+		dataKey: dataKey,
+	}
+
+	return dataKey, nil
+}
+
+// dataKey looks up DEK in cache or database, and decrypts it
+func (s *SecretsService) dataKey(ctx context.Context, name string) ([]byte, error) {
+	if item, exists := s.dataKeyCache[name]; exists {
+		if item.expiry.Before(time.Now()) && !item.expiry.IsZero() {
+			delete(s.dataKeyCache, name)
+		} else {
+			return item.dataKey, nil
+		}
+	}
+
+	// 1. get encrypted data key from database
+	dataKey, err := s.GetDataKey(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	// 2. decrypt data key
+	provider, exists := s.providers[dataKey.Provider]
+	if !exists {
+		return nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
+	}
+
+	decrypted, err := provider.Decrypt(dataKey.EncryptedData)
+	if err != nil {
+		return nil, err
+	}
+
+	// 3. cache data key
+	s.dataKeyCache[name] = dataKeyCacheItem{
+		expiry:  time.Now().Add(15 * time.Minute),
+		dataKey: decrypted,
+	}
+
+	return decrypted, nil
+}

pkg/services/secrets/secrets_test.go

@@ -0,0 +1,138 @@
+package secrets
+
+import (
+	"context"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/services/secrets/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSecrets_EnvelopeEncryption(t *testing.T) {
+	svc := SetupTestService(t)
+	ctx := context.Background()
+
+	t.Run("encrypting with no entity_id should create DEK", func(t *testing.T) {
+		plaintext := []byte("very secret string")
+
+		encrypted, err := svc.Encrypt(context.Background(), plaintext, WithoutScope())
+		require.NoError(t, err)
+
+		decrypted, err := svc.Decrypt(context.Background(), encrypted)
+		require.NoError(t, err)
+		assert.Equal(t, plaintext, decrypted)
+
+		keys, err := svc.GetAllDataKeys(ctx)
+		require.NoError(t, err)
+		assert.Equal(t, len(keys), 1)
+	})
+	t.Run("encrypting another secret with no entity_id should use the same DEK", func(t *testing.T) {
+		plaintext := []byte("another very secret string")
+
+		encrypted, err := svc.Encrypt(context.Background(), plaintext, WithoutScope())
+		require.NoError(t, err)
+
+		decrypted, err := svc.Decrypt(context.Background(), encrypted)
+		require.NoError(t, err)
+		assert.Equal(t, plaintext, decrypted)
+
+		keys, err := svc.GetAllDataKeys(ctx)
+		require.NoError(t, err)
+		assert.Equal(t, len(keys), 1)
+	})
+	t.Run("encrypting with entity_id provided should create a new DEK", func(t *testing.T) {
+		plaintext := []byte("some test data")
+
+		encrypted, err := svc.Encrypt(context.Background(), plaintext, WithScope("user:100"))
+		require.NoError(t, err)
+
+		decrypted, err := svc.Decrypt(context.Background(), encrypted)
+		require.NoError(t, err)
+		assert.Equal(t, plaintext, decrypted)
+
+		keys, err := svc.GetAllDataKeys(ctx)
+		require.NoError(t, err)
+		assert.Equal(t, len(keys), 2)
+	})
+
+	t.Run("decrypting empty payload should return error", func(t *testing.T) {
+		_, err := svc.Decrypt(context.Background(), []byte(""))
+		require.Error(t, err)
+
+		assert.Equal(t, "unable to decrypt empty payload", err.Error())
+	})
+
+	t.Run("decrypting legacy secret encrypted with secret key from settings", func(t *testing.T) {
+		expected := "grafana"
+		encrypted := []byte{122, 56, 53, 113, 101, 117, 73, 89, 20, 254, 36, 112, 112, 16, 128, 232, 227, 52, 166, 108, 192, 5, 28, 125, 126, 42, 197, 190, 251, 36, 94}
+		decrypted, err := svc.Decrypt(context.Background(), encrypted)
+		require.NoError(t, err)
+		assert.Equal(t, expected, string(decrypted))
+	})
+}
+
+func TestSecretsService_DataKeys(t *testing.T) {
+	svc := SetupTestService(t)
+	ctx := context.Background()
+
+	dataKey := types.DataKey{
+		Active:        true,
+		Name:          "test1",
+		Provider:      "test",
+		EncryptedData: []byte{0x62, 0xAF, 0xA1, 0x1A},
+	}
+
+	t.Run("querying for a DEK that does not exist", func(t *testing.T) {
+		res, err := svc.GetDataKey(ctx, dataKey.Name)
+		assert.ErrorIs(t, types.ErrDataKeyNotFound, err)
+		assert.Nil(t, res)
+	})
+
+	t.Run("creating an active DEK", func(t *testing.T) {
+		err := svc.CreateDataKey(ctx, dataKey)
+		require.NoError(t, err)
+
+		res, err := svc.GetDataKey(ctx, dataKey.Name)
+		require.NoError(t, err)
+		assert.Equal(t, dataKey.EncryptedData, res.EncryptedData)
+		assert.Equal(t, dataKey.Provider, res.Provider)
+		assert.Equal(t, dataKey.Name, res.Name)
+		assert.True(t, dataKey.Active)
+	})
+
+	t.Run("creating an inactive DEK", func(t *testing.T) {
+		k := types.DataKey{
+			Active:        false,
+			Name:          "test2",
+			Provider:      "test",
+			EncryptedData: []byte{0x62, 0xAF, 0xA1, 0x1A},
+		}
+		err := svc.CreateDataKey(ctx, k)
+		require.Error(t, err)
+
+		res, err := svc.GetDataKey(ctx, k.Name)
+		assert.Equal(t, types.ErrDataKeyNotFound, err)
+		assert.Nil(t, res)
+	})
+
+	t.Run("deleting DEK when no name provided must fail", func(t *testing.T) {
+		beforeDelete, err := svc.GetAllDataKeys(ctx)
+		require.NoError(t, err)
+		err = svc.DeleteDataKey(ctx, "")
+		require.Error(t, err)
+
+		afterDelete, err := svc.GetAllDataKeys(ctx)
+		require.NoError(t, err)
+		assert.Equal(t, beforeDelete, afterDelete)
+	})
+
+	t.Run("deleting a DEK", func(t *testing.T) {
+		err := svc.DeleteDataKey(ctx, dataKey.Name)
+		require.NoError(t, err)
+
+		res, err := svc.GetDataKey(ctx, dataKey.Name)
+		assert.Equal(t, types.ErrDataKeyNotFound, err)
+		assert.Nil(t, res)
+	})
+}

pkg/services/secrets/test_helper.go

@@ -0,0 +1,32 @@
+package secrets
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/services/encryption/ossencryption"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/ini.v1"
+)
+
+func SetupTestService(t *testing.T) SecretsService {
+	t.Helper()
+	defaultKey := "SdlklWklckeLS"
+	if len(setting.SecretKey) > 0 {
+		defaultKey = setting.SecretKey
+	}
+	raw, err := ini.Load([]byte(`
+		[security]
+		secret_key = ` + defaultKey))
+	require.NoError(t, err)
+	settings := &setting.OSSImpl{Cfg: &setting.Cfg{Raw: raw}}
+
+	return ProvideSecretsService(
+		sqlstore.InitTestDB(t),
+		bus.New(),
+		ossencryption.ProvideService(),
+		settings,
+	)
+}

pkg/services/secrets/types/types.go

@@ -0,0 +1,18 @@
+package types
+
+import (
+	"errors"
+	"time"
+)
+
+var ErrDataKeyNotFound = errors.New("data key not found")
+
+type DataKey struct {
+	Active        bool
+	Name          string
+	Scope         string
+	Provider      string
+	EncryptedData []byte
+	Created       time.Time
+	Updated       time.Time
+}

pkg/services/sqlstore/migrations/migrations.go

@@ -52,6 +52,7 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
 		addLiveChannelMigrations(mg)
 	}
 	ualert.RerunDashAlertMigration(mg)
+	addSecretsMigration(mg)
 	addKVStoreMigrations(mg)
 }
 

pkg/services/sqlstore/migrations/secrets_mig.go

@@ -0,0 +1,21 @@
+package migrations
+
+import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+
+func addSecretsMigration(mg *migrator.Migrator) {
+	dataKeysV1 := migrator.Table{
+		Name: "data_keys",
+		Columns: []*migrator.Column{
+			{Name: "name", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: true},
+			{Name: "active", Type: migrator.DB_Bool},
+			{Name: "scope", Type: migrator.DB_NVarchar, Length: 30, Nullable: false},
+			{Name: "provider", Type: migrator.DB_NVarchar, Length: 50, Nullable: false},
+			{Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false},
+			{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
+			{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
+		},
+		Indices: []*migrator.Index{},
+	}
+
+	mg.AddMigration("create data_keys table", migrator.NewAddTableMigration(dataKeysV1))
+}

pkg/util/encryption.go

@@ -15,6 +15,8 @@ import (
 const saltLength = 8
 
 // Decrypt decrypts a payload with a given secret.
+// Deprecated. Do not use it.
+// Use encryption.Service instead.
 var Decrypt = func(payload []byte, secret string) ([]byte, error) {
 	if len(payload) < saltLength {
 		return nil, fmt.Errorf("unable to compute salt")
@@ -47,6 +49,8 @@ var Decrypt = func(payload []byte, secret string) ([]byte, error) {
 }
 
 // Encrypt encrypts a payload with a given secret.
+// Deprecated. Do not use it.
+// Use encryption.Service instead.
 var Encrypt = func(payload []byte, secret string) ([]byte, error) {
 	salt, err := GetRandomString(saltLength)
 	if err != nil {

pkg/util/encryption_test.go

@@ -28,7 +28,7 @@ func TestEncryption(t *testing.T) {
 		assert.Equal(t, []byte("grafana"), decrypted)
 	})
 
-	t.Run("decrypting empty payload should not fail", func(t *testing.T) {
+	t.Run("decrypting empty payload should fail", func(t *testing.T) {
 		_, err := Decrypt([]byte(""), "1234")
 		require.Error(t, err)