root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 142 Packages Projects Releases Wiki Activity

Secrets: garbage collection (#110247) 

* clean up older secret versions

* start gargbage collection worker as background service

* make gen-go

* fix typo

* make update-workspace

* undo go mod changes

* undo go work sum changes

* Update pkg/registry/apis/secret/garbagecollectionworker/worker.go

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* Update pkg/registry/apis/secret/garbagecollectionworker/worker.go

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* default gc_worker_batch_size to 1 minute

* fix typo

* fix typo

* add test to ensure cleaning up secure values is idempotent

* make gen-go

* make update-workspace

* undo go.mod and .sum changes

* undo enterprise imports

---------

Co-authored-by: Matheus Macabu <macabu.matheus@gmail.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>
This commit is contained in:
Bruno 2025-09-02 11:11:01 -03:00 committed by GitHub
parent d5eb3e291a
commit f8cd7049e8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 1069 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/registry/apis/secret/clock/clock.go Normal file
View File

@ -0,0 +1,14 @@
package clock
import "time"
type Clock struct {
}
func ProvideClock() *Clock {
return &Clock{}
}
func (c *Clock) Now() time.Time {
return time.Now()
}

7
pkg/registry/apis/secret/contracts/clock.go Normal file
View File

@ -0,0 +1,7 @@
package contracts
import "time"
type Clock interface {
Now() time.Time
}

2
pkg/registry/apis/secret/contracts/secure_value.go
View File

@ -37,6 +37,8 @@ type SecureValueMetadataStorage interface {
SetVersionToActive(ctx context.Context, namespace xkube.Namespace, name string, version int64) error
SetVersionToInactive(ctx context.Context, namespace xkube.Namespace, name string, version int64) error
SetExternalID(ctx context.Context, namespace xkube.Namespace, name string, version int64, externalID ExternalID) error
Delete(ctx context.Context, namespace xkube.Namespace, name string, version int64) error
LeaseInactiveSecureValues(ctx context.Context, maxBatchSize uint16) ([]secretv1beta1.SecureValue, error)
}
type SecureValueService interface {

117
pkg/registry/apis/secret/garbagecollectionworker/worker.go Normal file
View File

@ -0,0 +1,117 @@
package garbagecollectionworker
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/grafana/grafana-app-sdk/logging"
secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
"github.com/grafana/grafana/pkg/setting"
"golang.org/x/sync/semaphore"
)
// Secure values have the `active` flag set to false on creation and deletion.
// The `active` flag is set to true when the creation process succeeds.
// The worker deletes secure values that are inactive because the creation process failed
// or because the secure value has been deleted.
type Worker struct {
Cfg *setting.Cfg
secureValueMetadataStorage contracts.SecureValueMetadataStorage
keeperMetadataStorage contracts.KeeperMetadataStorage
keeperService contracts.KeeperService
}
func ProvideWorker(
cfg *setting.Cfg,
secureValueMetadataStorage contracts.SecureValueMetadataStorage,
keeperMetadataStorage contracts.KeeperMetadataStorage,
keeperService contracts.KeeperService) *Worker {
return &Worker{
Cfg: cfg,
secureValueMetadataStorage: secureValueMetadataStorage,
keeperMetadataStorage: keeperMetadataStorage,
keeperService: keeperService}
}
func (w *Worker) Run(ctx context.Context) error {
if !w.Cfg.SecretsManagement.GCWorkerEnabled {
return nil
}
timer := time.NewTicker(w.Cfg.SecretsManagement.GCWorkerPollInterval)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
timeoutCtx, cancel := context.WithTimeout(context.Background(), w.Cfg.SecretsManagement.GCWorkerPerSecureValueCleanupTimeout)
if _, err := w.CleanupInactiveSecureValues(timeoutCtx); err != nil {
logging.FromContext(timeoutCtx).Error("cleaning up inactive secure values", "error", err)
}
cancel()
}
}
}
func (w *Worker) CleanupInactiveSecureValues(ctx context.Context) ([]secretv1beta1.SecureValue, error) {
secureValues, err := w.secureValueMetadataStorage.LeaseInactiveSecureValues(ctx, w.Cfg.SecretsManagement.GCWorkerMaxBatchSize)
if err != nil {
return nil, fmt.Errorf("fetching inactive secure values that need to be cleaned up: %w", err)
}
if len(secureValues) == 0 {
return nil, nil
}
errs := make([]error, len(secureValues))
sema := semaphore.NewWeighted(int64(w.Cfg.SecretsManagement.GCWorkerMaxConcurrentCleanups))
wg := &sync.WaitGroup{}
wg.Add(len(secureValues))
for i, sv := range secureValues {
if err := sema.Acquire(ctx, 1); err != nil {
return nil, fmt.Errorf("acquiring semaphore: %w", err)
}
go func(i int, sv *secretv1beta1.SecureValue) {
defer sema.Release(1)
defer wg.Done()
errs[i] = w.Cleanup(ctx, sv)
}(i, &sv)
}
wg.Wait()
return secureValues, errors.Join(errs...)
}
func (w *Worker) Cleanup(ctx context.Context, sv *secretv1beta1.SecureValue) error {
keeperCfg, err := w.keeperMetadataStorage.GetKeeperConfig(ctx, sv.Namespace, sv.Spec.Keeper, contracts.ReadOpts{ForUpdate: false})
if err != nil {
return fmt.Errorf("fetching keeper config: namespace=%+v keeperName=%+v %w", sv.Namespace, sv.Spec.Keeper, err)
}
keeper, err := w.keeperService.KeeperForConfig(keeperCfg)
if err != nil {
return fmt.Errorf("getting keeper for config: namespace=%+v keeperName=%+v %w", sv.Namespace, sv.Spec.Keeper, err)
}
// Keeper deletion is idempotent
if err := keeper.Delete(ctx, keeperCfg, sv.Namespace, sv.Name, sv.Status.Version); err != nil {
return fmt.Errorf("deleting secure value from keeper: %w", err)
}
// Metadata deletion is not idempotent but not found errors are ignored
if err := w.secureValueMetadataStorage.Delete(ctx, xkube.Namespace(sv.Namespace), sv.Name, sv.Status.Version); err != nil && !errors.Is(err, contracts.ErrSecureValueNotFound) {
return fmt.Errorf("deleting secure value from metadata storage: %w", err)
}
return nil
}

247
pkg/registry/apis/secret/garbagecollectionworker/worker_test.go Normal file
View File

@ -0,0 +1,247 @@
package garbagecollectionworker_test
import (
"fmt"
"slices"
"testing"
"time"
secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/testutils"
"github.com/grafana/grafana/pkg/storage/secret/encryption"
"github.com/mitchellh/copystructure"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
"pgregory.net/rapid"
)
func TestBasic(t *testing.T) {
t.Parallel()
t.Run("when no secure values exist, there's no work to do", func(t *testing.T) {
t.Parallel()
sut := testutils.Setup(t)
ids, err := sut.GarbageCollectionWorker.CleanupInactiveSecureValues(t.Context())
require.NoError(t, err)
require.Empty(t, ids)
})
t.Run("inactive secure values are not deleted immediately because of the grace period", func(t *testing.T) {
t.Parallel()
sut := testutils.Setup(t)
sv1, err := sut.CreateSv(t.Context())
require.NoError(t, err)
_, err = sut.DeleteSv(t.Context(), sv1.Namespace, sv1.Name)
require.NoError(t, err)
// Try to fetch inactive secure values for deletion
svs, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
require.NoError(t, err)
require.Empty(t, svs)
})
t.Run("secure values are fetched for deletion and deleted from keeper", func(t *testing.T) {
sut := testutils.Setup(t)
sv, err := sut.CreateSv(t.Context())
require.NoError(t, err)
keeperCfg, err := sut.KeeperMetadataStorage.GetKeeperConfig(t.Context(), sv.Namespace, sv.Spec.Keeper, contracts.ReadOpts{ForUpdate: false})
require.NoError(t, err)
keeper, err := sut.KeeperService.KeeperForConfig(keeperCfg)
require.NoError(t, err)
// Get the secret value once to make sure it's reachable
exposedValue, err := keeper.Expose(t.Context(), keeperCfg, sv.Namespace, sv.Name, sv.Status.Version)
require.NoError(t, err)
require.NotEmpty(t, exposedValue.DangerouslyExposeAndConsumeValue())
_, err = sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
require.NoError(t, err)
// Advance time to wait for grace period
sut.Clock.AdvanceBy(10 * time.Minute)
svs, err := sut.GarbageCollectionWorker.CleanupInactiveSecureValues(t.Context())
require.NoError(t, err)
require.Equal(t, 1, len(svs))
require.Equal(t, sv.UID, svs[0].UID)
svs, err = sut.GarbageCollectionWorker.CleanupInactiveSecureValues(t.Context())
require.NoError(t, err)
require.Empty(t, svs)
// Try to get the secreet value again to make sure it's been deleted from the keeper
exposedValue, err = keeper.Expose(t.Context(), keeperCfg, sv.Namespace, sv.Name, sv.Status.Version)
require.ErrorIs(t, err, encryption.ErrEncryptedValueNotFound)
require.Empty(t, exposedValue)
})
t.Run("cleaning up secure values is idempotent", func(t *testing.T) {
t.Parallel()
sut := testutils.Setup(t)
sv, err := sut.CreateSv(t.Context())
require.NoError(t, err)
_, err = sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
require.NoError(t, err)
// Clean up the same secure value twice and ensure it succeeds
require.NoError(t, sut.GarbageCollectionWorker.Cleanup(t.Context(), sv))
require.NoError(t, sut.GarbageCollectionWorker.Cleanup(t.Context(), sv))
})
}
var (
decryptersGen = rapid.SampledFrom([]string{"svc1", "svc2", "svc3", "svc4", "svc5"})
nameGen = rapid.SampledFrom([]string{"n1", "n2", "n3", "n4", "n5"})
namespaceGen = rapid.SampledFrom([]string{"ns1", "ns2", "ns3", "ns4", "ns5"})
anySecureValueGen = rapid.Custom(func(t *rapid.T) *secretv1beta1.SecureValue {
return &secretv1beta1.SecureValue{
ObjectMeta: metav1.ObjectMeta{
Name: nameGen.Draw(t, "name"),
Namespace: namespaceGen.Draw(t, "ns"),
},
Spec: secretv1beta1.SecureValueSpec{
Description: rapid.SampledFrom([]string{"d1", "d2", "d3", "d4", "d5"}).Draw(t, "description"),
Value: ptr.To(secretv1beta1.NewExposedSecureValue(rapid.SampledFrom([]string{"v1", "v2", "v3", "v4", "v5"}).Draw(t, "value"))),
Decrypters: rapid.SliceOfDistinct(decryptersGen, func(v string) string { return v }).Draw(t, "decrypters"),
},
Status: secretv1beta1.SecureValueStatus{},
}
})
)
func TestProperty(t *testing.T) {
t.Parallel()
tt := t
rapid.Check(t, func(t *rapid.T) {
sut := testutils.Setup(tt)
model := newModel()
t.Repeat(map[string]func(*rapid.T){
"create": func(t *rapid.T) {
sv := anySecureValueGen.Draw(t, "sv")
svCopy := deepCopy(sv)
createdSv, err := sut.CreateSv(t.Context(), testutils.CreateSvWithSv(sv))
svCopy.UID = createdSv.UID
modelErr := model.create(sut.Clock.Now(), svCopy)
require.ErrorIs(t, err, modelErr)
},
"delete": func(t *rapid.T) {
if len(model.items) == 0 {
return
}
i := rapid.IntRange(0, len(model.items)-1).Draw(t, "index")
sv := model.items[i]
modelErr := model.delete(sv.Namespace, sv.Name)
_, err := sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
require.ErrorIs(t, err, modelErr)
},
"cleanup": func(t *rapid.T) {
// Taken from secureValueMetadataStorage.acquireLeases
minAge := 300 * time.Second
maxBatchSize := sut.GarbageCollectionWorker.Cfg.SecretsManagement.GCWorkerMaxBatchSize
modelDeleted, modelErr := model.cleanupInactiveSecureValues(sut.Clock.Now(), minAge, maxBatchSize)
deleted, err := sut.GarbageCollectionWorker.CleanupInactiveSecureValues(t.Context())
require.ErrorIs(t, err, modelErr)
require.Equal(t, len(modelDeleted), len(deleted), "model and impl deleted a different number of secure values")
seen := make(map[types.UID]bool, 0)
for _, v := range modelDeleted {
seen[v.UID] = true
}
for _, v := range deleted {
require.True(t, seen[v.UID], "impl deleted a secure value that the model did not")
}
},
"advanceTime": func(t *rapid.T) {
duration := time.Duration(rapid.IntRange(1, 10).Draw(t, "minutes")) * time.Minute
sut.Clock.AdvanceBy(duration)
},
})
})
}
type model struct {
items []*modelSecureValue
}
type modelSecureValue struct {
*secretv1beta1.SecureValue
active bool
created time.Time
}
func newModel() *model {
return &model{
items: make([]*modelSecureValue, 0),
}
}
func (m *model) create(now time.Time, sv *secretv1beta1.SecureValue) error {
for _, item := range m.items {
if item.active && item.Namespace == sv.Namespace && item.Name == sv.Name {
item.active = false
break
}
}
m.items = append(m.items, &modelSecureValue{SecureValue: sv, active: true, created: now})
return nil
}
func (m *model) delete(ns string, name string) error {
for _, sv := range m.items {
if sv.active && sv.Namespace == ns && sv.Name == name {
sv.active = false
return nil
}
}
return contracts.ErrSecureValueNotFound
}
func (m *model) cleanupInactiveSecureValues(now time.Time, minAge time.Duration, maxBatchSize uint16) ([]*modelSecureValue, error) {
// Using a slice to allow duplicates
toDelete := make([]*modelSecureValue, 0)
for _, sv := range m.items {
if len(toDelete) >= int(maxBatchSize) {
break
}
if !sv.active && now.Sub(sv.created) > minAge {
toDelete = append(toDelete, sv)
}
}
// PERF: The slices are always small
m.items = slices.DeleteFunc(m.items, func(v1 *modelSecureValue) bool {
return slices.ContainsFunc(toDelete, func(v2 *modelSecureValue) bool {
return v2.UID == v1.UID
})
})
return toDelete, nil
}
func deepCopy[T any](sv T) T {
copied, err := copystructure.Copy(sv)
if err != nil {
panic(fmt.Sprintf("failed to copy secure value: %v", err))
}
return copied.(T)
}

45
pkg/registry/apis/secret/testutils/testutils.go
View File

@ -3,6 +3,7 @@ package testutils
import (
"context"
"testing"
"time"
"github.com/grafana/authlib/authn"
"github.com/grafana/authlib/types"
@ -21,6 +22,7 @@ import (
cipher "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher/service"
osskmsproviders "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/kmsproviders"
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/manager"
"github.com/grafana/grafana/pkg/registry/apis/secret/garbagecollectionworker"
"github.com/grafana/grafana/pkg/registry/apis/secret/mutator"
"github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper/sqlkeeper"
"github.com/grafana/grafana/pkg/registry/apis/secret/service"
@ -32,6 +34,7 @@ import (
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/secret/database"
encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
"github.com/grafana/grafana/pkg/storage/secret/metadata"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
)
@ -71,7 +74,9 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
keeperMetadataStorage, err := metadata.ProvideKeeperMetadataStorage(database, tracer, nil)
require.NoError(t, err)
secureValueMetadataStorage, err := metadata.ProvideSecureValueMetadataStorage(database, tracer, nil)
clock := NewFakeClock()
secureValueMetadataStorage, err := metadata.ProvideSecureValueMetadataStorage(clock, database, tracer, nil)
require.NoError(t, err)
// Initialize access client + access control
@ -84,8 +89,11 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
defaultKey := "SdlklWklckeLS"
cfg := setting.NewCfg()
cfg.SecretsManagement = setting.SecretsManagerSettings{
CurrentEncryptionProvider: "secret_key.v1",
ConfiguredKMSProviders: map[string]map[string]string{"secret_key.v1": {"secret_key": defaultKey}},
CurrentEncryptionProvider: "secret_key.v1",
ConfiguredKMSProviders: map[string]map[string]string{"secret_key.v1": {"secret_key": defaultKey}},
GCWorkerEnabled: false,
GCWorkerMaxBatchSize: 2,
GCWorkerMaxConcurrentCleanups: 2,
}
store, err := encryptionstorage.ProvideDataKeyStorage(database, tracer, nil)
require.NoError(t, err)
@ -143,6 +151,12 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
consolidationService := service.ProvideConsolidationService(tracer, globalDataKeyStore, encryptedValueStorage, globalEncryptedValueStorage, encryptionManager)
garbageCollectionWorker := garbagecollectionworker.ProvideWorker(
cfg,
secureValueMetadataStorage,
keeperMetadataStorage,
keeperService)
return Sut{
SecureValueService: secureValueService,
SecureValueMetadataStorage: secureValueMetadataStorage,
@ -156,6 +170,10 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
ConsolidationService: consolidationService,
EncryptionManager: encryptionManager,
GlobalDataKeyStore: globalDataKeyStore,
GarbageCollectionWorker: garbageCollectionWorker,
Clock: clock,
KeeperService: keeperService,
KeeperMetadataStorage: keeperMetadataStorage,
}
}
@ -172,6 +190,11 @@ type Sut struct {
ConsolidationService contracts.ConsolidationService
EncryptionManager contracts.EncryptionManager
GlobalDataKeyStore contracts.GlobalDataKeyStorage
GarbageCollectionWorker *garbagecollectionworker.Worker
// The fake clock passed to implementations to make testing easier
Clock *FakeClock
KeeperService contracts.KeeperService
KeeperMetadataStorage contracts.KeeperMetadataStorage
}
type CreateSvConfig struct {
@ -327,3 +350,19 @@ func CreateX509TestDir(t *testing.T) TestCertPaths {
CA: caCertFile.Name(),
}
}
type FakeClock struct {
Current time.Time
}
func NewFakeClock() *FakeClock {
return &FakeClock{Current: time.Now()}
}
func (c *FakeClock) Now() time.Time {
return c.Current
}
func (c *FakeClock) AdvanceBy(duration time.Duration) {
c.Current = c.Current.Add(duration)
}

3
pkg/registry/backgroundsvcs/background_services.go
View File

@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/infra/usagestats/statscollector"
"github.com/grafana/grafana/pkg/registry"
apiregistry "github.com/grafana/grafana/pkg/registry/apis"
secretsgarbagecollectionworker "github.com/grafana/grafana/pkg/registry/apis/secret/garbagecollectionworker"
appregistry "github.com/grafana/grafana/pkg/registry/apps"
"github.com/grafana/grafana/pkg/services/accesscontrol/dualwrite"
"github.com/grafana/grafana/pkg/services/anonymous/anonimpl"
@ -69,6 +70,7 @@ func ProvideBackgroundServiceRegistry(
appRegistry *appregistry.Service,
pluginDashboardUpdater *plugindashboardsservice.DashboardUpdater,
dashboardServiceImpl *service.DashboardServiceImpl,
secretsGarbageCollectionWorker *secretsgarbagecollectionworker.Worker,
// Need to make sure these are initialized, is there a better place to put them?
_ dashboardsnapshots.Service,
_ serviceaccounts.Service,
@ -115,6 +117,7 @@ func ProvideBackgroundServiceRegistry(
appRegistry,
pluginDashboardUpdater,
dashboardServiceImpl,
secretsGarbageCollectionWorker,
)
}

5
pkg/server/wire.go
View File

@ -42,10 +42,12 @@ import (
"github.com/grafana/grafana/pkg/middleware/loggermw"
apiregistry "github.com/grafana/grafana/pkg/registry/apis"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
secretclock "github.com/grafana/grafana/pkg/registry/apis/secret/clock"
secretcontracts "github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
secretdecrypt "github.com/grafana/grafana/pkg/registry/apis/secret/decrypt"
cipher "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher/service"
encryptionManager "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/manager"
secretsgarbagecollectionworker "github.com/grafana/grafana/pkg/registry/apis/secret/garbagecollectionworker"
secretinline "github.com/grafana/grafana/pkg/registry/apis/secret/inline"
secretmutator "github.com/grafana/grafana/pkg/registry/apis/secret/mutator"
secretsecurevalueservice "github.com/grafana/grafana/pkg/registry/apis/secret/service"
@ -312,6 +314,7 @@ var wireBasicSet = wire.NewSet(
wire.Bind(new(secrets.Service), new(*secretsManager.SecretsService)),
secretsDatabase.ProvideSecretsStore,
wire.Bind(new(secrets.Store), new(*secretsDatabase.SecretsStoreImpl)),
secretsgarbagecollectionworker.ProvideWorker,
grafanads.ProvideService,
wire.Bind(new(dashboardsnapshots.Store), new(*dashsnapstore.DashboardSnapshotStore)),
dashsnapstore.ProvideStore,
@ -442,7 +445,9 @@ var wireBasicSet = wire.NewSet(
secretmutator.ProvideSecureValueMutator,
secretmigrator.NewWithEngine,
secretdatabase.ProvideDatabase,
secretclock.ProvideClock,
wire.Bind(new(secretcontracts.Database), new(*secretdatabase.Database)),
wire.Bind(new(secretcontracts.Clock), new(*secretclock.Clock)),
encryptionManager.ProvideEncryptionManager,
cipher.ProvideAESGCMCipherService,
// Unified storage

16
pkg/server/wire_gen.go
View File

File diff suppressed because one or more lines are too long

18
pkg/setting/setting_secrets_manager.go
View File

@ -2,6 +2,7 @@ package setting
import (
"strings"
"time"
)
const (
@ -22,6 +23,17 @@ type SecretsManagerSettings struct {
GrpcServerTLSServerName string // Server name to use for TLS verification
GrpcServerAddress string // Address for gRPC secrets server
GrpcGrafanaServiceName string // Service name to use for background grafana decryption/inline
// Used for testing. Set to false to disable the control loop.
GCWorkerEnabled bool
// Max number of inactive secure values to fetch from the database.
GCWorkerMaxBatchSize uint16
// Max number of tasks to delete secure values that can be inflight at a time.
GCWorkerMaxConcurrentCleanups uint16
// How long to wait for between fetching inactive secure values for cleanup.
GCWorkerPollInterval time.Duration
// How long to wait for the process to clean up a secure value to complete.
GCWorkerPerSecureValueCleanupTimeout time.Duration
}
func (cfg *Cfg) readSecretsManagerSettings() {
@ -35,6 +47,12 @@ func (cfg *Cfg) readSecretsManagerSettings() {
cfg.SecretsManagement.GrpcServerAddress = valueAsString(secretsMgmt, "grpc_server_address", "")
cfg.SecretsManagement.GrpcGrafanaServiceName = valueAsString(secretsMgmt, "grpc_grafana_service_name", "")
cfg.SecretsManagement.GCWorkerEnabled = secretsMgmt.Key("gc_worker_enabled").MustBool(true)
cfg.SecretsManagement.GCWorkerMaxBatchSize = uint16(secretsMgmt.Key("gc_worker_batch_size").MustUint(16))
cfg.SecretsManagement.GCWorkerMaxConcurrentCleanups = uint16(secretsMgmt.Key("gc_worker_max_concurrency").MustUint(16))
cfg.SecretsManagement.GCWorkerPollInterval = secretsMgmt.Key("gc_worker_poll_interval").MustDuration(1 * time.Minute)
cfg.SecretsManagement.GCWorkerPerSecureValueCleanupTimeout = secretsMgmt.Key("gc_worker_per_request_timeout").MustDuration(5 * time.Second)
// Extract available KMS providers from configuration sections
providers := make(map[string]map[string]string)
for _, section := range cfg.Raw.Sections() {

7
pkg/storage/secret/metadata/data/secure_value_delete.sql Normal file
View File

@ -0,0 +1,7 @@
DELETE FROM
{{ .Ident "secret_secure_value" }}
WHERE
{{ .Ident "namespace" }} = {{ .Arg .Namespace }} AND
{{ .Ident "name" }} = {{ .Arg .Name }} AND
{{ .Ident "version" }} = {{ .Arg .Version }}
;

14
pkg/storage/secret/metadata/data/secure_value_lease_inactive.sql Normal file
View File

@ -0,0 +1,14 @@
UPDATE
{{ .Ident "secret_secure_value" }}
SET
{{ .Ident "lease_token" }} = {{ .Arg .LeaseToken }},
{{ .Ident "lease_created" }} = {{ .Arg .Now }}
WHERE
{{ .Ident "guid" }} IN (SELECT {{ .Ident "guid"}}
FROM {{ .Ident "secret_secure_value" }}
WHERE
{{ .Ident "active" }} = FALSE AND
{{ .Arg .Now }} - {{ .Ident "created" }} > {{ .Arg .MinAge }} AND
{{ .Arg .Now }} - {{ .Ident "lease_created" }} > {{ .Arg .LeaseTTL }}
LIMIT {{ .Arg .MaxBatchSize }})
;

26
pkg/storage/secret/metadata/data/secure_value_list_by_lease_token.sql Normal file
View File

@ -0,0 +1,26 @@
SELECT
{{ .Ident "guid" }},
{{ .Ident "name" }},
{{ .Ident "namespace" }},
{{ .Ident "annotations" }},
{{ .Ident "labels" }},
{{ .Ident "created" }},
{{ .Ident "created_by" }},
{{ .Ident "updated" }},
{{ .Ident "updated_by" }},
{{ .Ident "description" }},
{{ .Ident "keeper" }},
{{ .Ident "decrypters" }},
{{ .Ident "ref" }},
{{ .Ident "external_id" }},
{{ .Ident "version" }},
{{ .Ident "active" }},
{{ .Ident "owner_reference_api_group" }},
{{ .Ident "owner_reference_api_version" }},
{{ .Ident "owner_reference_kind" }},
{{ .Ident "owner_reference_name" }}
FROM
{{ .Ident "secret_secure_value" }}
WHERE
{{ .Ident "lease_token" }} = {{ .Arg .LeaseToken }}
;

9
pkg/storage/secret/metadata/metrics/metrics.go
View File

@ -30,6 +30,7 @@ type StorageMetrics struct {
SecureValueMetadataListDuration *prometheus.HistogramVec
SecureValueSetExternalIDDuration *prometheus.HistogramVec
SecureValueSetStatusDuration *prometheus.HistogramVec
SecureValueDeleteDuration *prometheus.HistogramVec
DecryptDuration *prometheus.HistogramVec
}
@ -116,6 +117,13 @@ func newStorageMetrics() *StorageMetrics {
Help: "Duration of secure value set status operations",
Buckets: prometheus.DefBuckets,
}, []string{successLabel}),
SecureValueDeleteDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "secure_value_delete_duration_seconds",
Help: "Duration of secure value delete operations",
Buckets: prometheus.DefBuckets,
}, []string{successLabel}),
// Decrypt metrics
DecryptDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
@ -151,6 +159,7 @@ func NewStorageMetrics(reg prometheus.Registerer) *StorageMetrics {
m.SecureValueMetadataListDuration,
m.SecureValueSetExternalIDDuration,
m.SecureValueSetStatusDuration,
m.SecureValueDeleteDuration,
m.DecryptDuration,
)
}

39
pkg/storage/secret/metadata/query.go
View File

@ -28,6 +28,9 @@ var (
sqlSecureValueList = mustTemplate("secure_value_list.sql")
sqlSecureValueCreate = mustTemplate("secure_value_create.sql")
sqlSecureValueUpdateExternalId = mustTemplate("secure_value_updateExternalId.sql")
sqlSecureValueDelete = mustTemplate("secure_value_delete.sql")
sqlSecureValueLeaseInactive = mustTemplate("secure_value_lease_inactive.sql")
sqlSecureValueListByLeaseToken = mustTemplate("secure_value_list_by_lease_token.sql")
sqlGetLatestSecureValueVersion = mustTemplate("secure_value_get_latest_version.sql")
sqlSecureValueSetVersionToActive = mustTemplate("secure_value_set_version_to_active.sql")
@ -208,3 +211,39 @@ type updateExternalIdSecureValue struct {
func (r updateExternalIdSecureValue) Validate() error {
return nil // TODO
}
type deleteSecureValue struct {
sqltemplate.SQLTemplate
Namespace string
Name string
Version int64
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r deleteSecureValue) Validate() error {
return nil // TODO
}
type leaseInactiveSecureValues struct {
sqltemplate.SQLTemplate
LeaseToken string
MaxBatchSize uint16
MinAge int64
LeaseTTL int64
Now int64
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r leaseInactiveSecureValues) Validate() error {
return nil // TODO
}
type listSecureValuesByLeaseToken struct {
sqltemplate.SQLTemplate
LeaseToken string
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r listSecureValuesByLeaseToken) Validate() error {
return nil // TODO
}

34
pkg/storage/secret/metadata/query_test.go
View File

@ -3,6 +3,7 @@ package metadata
import (
"testing"
"text/template"
"time"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
"k8s.io/utils/ptr"
@ -238,6 +239,39 @@ func TestSecureValueQueries(t *testing.T) {
},
},
},
sqlSecureValueDelete: {
{
Name: "deleteSecureValue",
Data: &deleteSecureValue{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Namespace: "ns",
Name: "name",
Version: 1,
},
},
},
sqlSecureValueLeaseInactive: {
{
Name: "lease inactive",
Data: &leaseInactiveSecureValues{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Now: 10,
LeaseToken: "token",
LeaseTTL: int64((30 * time.Second).Seconds()),
MaxBatchSize: 10,
MinAge: int64((300 * time.Second).Seconds()),
},
},
},
sqlSecureValueListByLeaseToken: {
{
Name: "list by lease token",
Data: &listSecureValuesByLeaseToken{
SQLTemplate: mocks.NewTestingSQLTemplate(),
LeaseToken: "token",
},
},
},
},
})
}

8
pkg/storage/secret/metadata/secure_value_model.go
View File

@ -122,18 +122,18 @@ func (sv *secureValueDB) toKubernetes() (*secretv1beta1.SecureValue, error) {
}
// toCreateRow maps a Kubernetes resource into a DB row for new resources being created/inserted.
func toCreateRow(sv *secretv1beta1.SecureValue, actorUID string) (*secureValueDB, error) {
func toCreateRow(now time.Time, sv *secretv1beta1.SecureValue, actorUID string) (*secureValueDB, error) {
row, err := toRow(sv, "")
if err != nil {
return nil, fmt.Errorf("failed to convert SecureValue to secureValueDB: %w", err)
}
now := time.Now().UTC().Unix()
timestamp := now.UTC().Unix()
row.GUID = uuid.New().String()
row.Created = now
row.Created = timestamp
row.CreatedBy = actorUID
row.Updated = now
row.Updated = timestamp
row.UpdatedBy = actorUID
return row, nil

177
pkg/storage/secret/metadata/secure_value_store.go
View File

@ -6,6 +6,7 @@ import (
"strconv"
"time"
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
@ -23,11 +24,13 @@ import (
var _ contracts.SecureValueMetadataStorage = (*secureValueMetadataStorage)(nil)
func ProvideSecureValueMetadataStorage(
clock contracts.Clock,
db contracts.Database,
tracer trace.Tracer,
reg prometheus.Registerer,
) (contracts.SecureValueMetadataStorage, error) {
return &secureValueMetadataStorage{
clock: clock,
db: db,
dialect: sqltemplate.DialectForDriver(db.DriverName()),
metrics: metrics.NewStorageMetrics(reg),
@ -37,6 +40,7 @@ func ProvideSecureValueMetadataStorage(
// secureValueMetadataStorage is the actual implementation of the secure value (metadata) storage.
type secureValueMetadataStorage struct {
clock contracts.Clock
db contracts.Database
dialect sqltemplate.Dialect
metrics *metrics.StorageMetrics
@ -44,7 +48,7 @@ type secureValueMetadataStorage struct {
}
func (s *secureValueMetadataStorage) Create(ctx context.Context, sv *secretv1beta1.SecureValue, actorUID string) (_ *secretv1beta1.SecureValue, svmCreateErr error) {
start := time.Now()
start := s.clock.Now()
name := sv.GetName()
namespace := sv.GetNamespace()
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Create", trace.WithAttributes(
@ -123,7 +127,7 @@ func (s *secureValueMetadataStorage) Create(ctx context.Context, sv *secretv1bet
for {
sv.Status.Version = version
row, err = toCreateRow(sv, actorUID)
row, err = toCreateRow(s.clock.Now(), sv, actorUID)
if err != nil {
return fmt.Errorf("to create row: %w", err)
}
@ -256,7 +260,7 @@ func (s *secureValueMetadataStorage) readActiveVersion(ctx context.Context, name
}
func (s *secureValueMetadataStorage) Read(ctx context.Context, namespace xkube.Namespace, name string, opts contracts.ReadOpts) (_ *secretv1beta1.SecureValue, readErr error) {
start := time.Now()
start := s.clock.Now()
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Read", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
@ -297,7 +301,7 @@ func (s *secureValueMetadataStorage) Read(ctx context.Context, namespace xkube.N
}
func (s *secureValueMetadataStorage) List(ctx context.Context, namespace xkube.Namespace) (svList []secretv1beta1.SecureValue, listErr error) {
start := time.Now()
start := s.clock.Now()
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.List", trace.WithAttributes(
attribute.String("namespace", namespace.String()),
))
@ -400,7 +404,7 @@ func (s *secureValueMetadataStorage) SetVersionToActive(ctx context.Context, nam
return fmt.Errorf("setting secure value version to active: namespace=%+v name=%+v version=%+v %w", namespace, name, version, err)
}
// validate modified cound
// validate modified count
modifiedCount, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("fetching number of modified rows: %w", err)
@ -449,7 +453,7 @@ func (s *secureValueMetadataStorage) SetVersionToInactive(ctx context.Context, n
}
func (s *secureValueMetadataStorage) SetExternalID(ctx context.Context, namespace xkube.Namespace, name string, version int64, externalID contracts.ExternalID) (setExtIDErr error) {
start := time.Now()
start := s.clock.Now()
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.SetExternalID", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
@ -508,3 +512,164 @@ func (s *secureValueMetadataStorage) SetExternalID(ctx context.Context, namespac
return nil
}
func (s *secureValueMetadataStorage) Delete(ctx context.Context, namespace xkube.Namespace, name string, version int64) (err error) {
start := s.clock.Now()
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Delete", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
attribute.Int64("version", version),
))
defer span.End()
defer func() {
success := err == nil
args := []any{
"namespace", namespace.String(),
"name", name,
"version", strconv.FormatInt(version, 10),
"success", success,
}
if !success {
span.SetStatus(codes.Error, "SecureValueMetadataStorage.Delete failed")
span.RecordError(err)
args = append(args, "error", err)
}
logging.FromContext(ctx).Info("SecureValueMetadataStorage.Delete", args...)
s.metrics.SecureValueDeleteDuration.WithLabelValues(strconv.FormatBool(success)).Observe(time.Since(start).Seconds())
}()
req := deleteSecureValue{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
Name: name,
Version: version,
}
q, err := sqltemplate.Execute(sqlSecureValueDelete, req)
if err != nil {
return fmt.Errorf("execute template %q: %w", sqlSecureValueDelete.Name(), err)
}
res, err := s.db.ExecContext(ctx, q, req.GetArgs()...)
if err != nil {
return fmt.Errorf("deleting secure value: namespace=%+v name=%+v version=%+v %w", namespace, name, version, err)
}
modifiedCount, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("getting rows affected: %w", err)
}
// Deleting is idempotent so modifiedCunt must be in {0, 1}
if modifiedCount > 1 {
return fmt.Errorf("secureValueMetadataStorage.Delete: delete more than one secret, this is a bug, check the where condition: modifiedCount=%d", modifiedCount)
}
return nil
}
func (s *secureValueMetadataStorage) LeaseInactiveSecureValues(ctx context.Context, maxBatchSize uint16) (out []secretv1beta1.SecureValue, err error) {
start := s.clock.Now()
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.LeaseInactiveSecureValues", trace.WithAttributes(
attribute.Int("maxBatchSize", int(maxBatchSize)),
))
defer span.End()
defer func() {
success := err == nil
if !success {
span.SetStatus(codes.Error, "SecureValueMetadataStorage.LeaseInactiveSecureValues failed")
span.RecordError(err)
}
s.metrics.SecureValueDeleteDuration.WithLabelValues(strconv.FormatBool(success)).Observe(time.Since(start).Seconds())
}()
leaseToken := uuid.NewString()
if err := s.acquireLeases(ctx, leaseToken, maxBatchSize); err != nil {
return nil, fmt.Errorf("acquiring leases for inactive secure values: %w", err)
}
secureValues, err := s.listByLeaseToken(ctx, leaseToken)
if err != nil {
return nil, fmt.Errorf("fetching secure values by lease token: %w", err)
}
return secureValues, nil
}
func (s *secureValueMetadataStorage) acquireLeases(ctx context.Context, leaseToken string, maxBatchSize uint16) error {
req := leaseInactiveSecureValues{
SQLTemplate: sqltemplate.New(s.dialect),
LeaseToken: leaseToken,
MaxBatchSize: maxBatchSize,
MinAge: int64((300 * time.Second).Seconds()),
LeaseTTL: int64((30 * time.Second).Seconds()),
Now: s.clock.Now().UTC().Unix(),
}
q, err := sqltemplate.Execute(sqlSecureValueLeaseInactive, req)
if err != nil {
return fmt.Errorf("execute template %q: %w", sqlSecureValueLeaseInactive.Name(), err)
}
if _, err := s.db.ExecContext(ctx, q, req.GetArgs()...); err != nil {
return fmt.Errorf("leasing inactive secure values: %w", err)
}
return nil
}
func (s *secureValueMetadataStorage) listByLeaseToken(ctx context.Context, leaseToken string) ([]secretv1beta1.SecureValue, error) {
req := listSecureValuesByLeaseToken{
SQLTemplate: sqltemplate.New(s.dialect),
LeaseToken: leaseToken,
}
q, err := sqltemplate.Execute(sqlSecureValueListByLeaseToken, req)
if err != nil {
return nil, fmt.Errorf("execute template %q: %w", sqlSecureValueListByLeaseToken.Name(), err)
}
rows, err := s.db.QueryContext(ctx, q, req.GetArgs()...)
if err != nil {
return nil, fmt.Errorf("listing secure values: %w", err)
}
defer func() { _ = rows.Close() }()
secureValues := make([]secretv1beta1.SecureValue, 0)
for rows.Next() {
row := secureValueDB{}
err = rows.Scan(&row.GUID,
&row.Name, &row.Namespace, &row.Annotations,
&row.Labels,
&row.Created, &row.CreatedBy,
&row.Updated, &row.UpdatedBy,
&row.Description, &row.Keeper, &row.Decrypters,
&row.Ref, &row.ExternalID, &row.Version, &row.Active,
&row.OwnerReferenceAPIGroup, &row.OwnerReferenceAPIVersion, &row.OwnerReferenceKind, &row.OwnerReferenceName,
)
if err != nil {
return nil, fmt.Errorf("error reading secure value row: %w", err)
}
secureValue, err := row.toKubernetes()
if err != nil {
return nil, fmt.Errorf("convert to kubernetes object: %w", err)
}
secureValues = append(secureValues, *secureValue)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read rows error: %w", err)
}
return secureValues, nil
}

99
pkg/storage/secret/metadata/secure_value_store_test.go
View File

@ -3,13 +3,17 @@ package metadata_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
"k8s.io/utils/ptr"
"pgregory.net/rapid"
secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
"github.com/grafana/grafana/pkg/registry/apis/secret/clock"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/testutils"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/storage/secret/database"
@ -43,7 +47,7 @@ func Test_SecureValueMetadataStorage_CreateAndRead(t *testing.T) {
db := database.ProvideDatabase(testDB, tracer)
// Initialize the secure value storage
secureValueStorage, err := metadata.ProvideSecureValueMetadataStorage(db, tracer, nil)
secureValueStorage, err := metadata.ProvideSecureValueMetadataStorage(clock.ProvideClock(), db, tracer, nil)
require.NoError(t, err)
// Initialize the keeper storage
@ -142,3 +146,96 @@ func Test_SecureValueMetadataStorage_CreateAndRead(t *testing.T) {
require.Equal(t, contracts.ErrSecureValueNotFound, err)
})
}
func TestLeaseInactiveSecureValues(t *testing.T) {
t.Parallel()
t.Run("no secure value exists", func(t *testing.T) {
t.Parallel()
sut := testutils.Setup(t)
svs, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
require.NoError(t, err)
require.Empty(t, svs)
})
t.Run("secure values are not visible to other requests during lease duration", func(t *testing.T) {
sut := testutils.Setup(t)
sv, err := sut.CreateSv(t.Context())
require.NoError(t, err)
_, err = sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
require.NoError(t, err)
// Advance clock to handle grace period
sut.Clock.AdvanceBy(10 * time.Minute)
// Acquire a lease on inactive secure values
values1, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
require.NoError(t, err)
require.Equal(t, 1, len(values1))
require.Equal(t, sv.UID, values1[0].UID)
// Try to acquire a lease again
values2, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
require.NoError(t, err)
// There's only one inactive secure value and it is already leased
require.Empty(t, values2)
// Advance clock to expire lease
sut.Clock.AdvanceBy(10 * time.Minute)
values3, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
require.NoError(t, err)
// Should acquire a new lease since the previous one expired
require.Equal(t, 1, len(values3))
require.Equal(t, sv.UID, values3[0].UID)
})
}
func TestPropertySecureValueMetadataStorage(t *testing.T) {
t.Parallel()
tt := t
rapid.Check(t, func(t *rapid.T) {
sut := testutils.Setup(tt)
model := newModel()
t.Repeat(map[string]func(*rapid.T){
"create": func(t *rapid.T) {
sv := anySecureValueGen.Draw(t, "sv")
modelCreatedSv, modelErr := model.create(sut.Clock.Now(), deepCopy(sv))
createdSv, err := sut.CreateSv(t.Context(), testutils.CreateSvWithSv(deepCopy(sv)))
if err != nil || modelErr != nil {
require.ErrorIs(t, err, modelErr)
return
}
require.Equal(t, modelCreatedSv.Namespace, createdSv.Namespace)
require.Equal(t, modelCreatedSv.Name, createdSv.Name)
require.Equal(t, modelCreatedSv.Status.Version, createdSv.Status.Version)
},
"delete": func(t *rapid.T) {
ns := namespaceGen.Draw(t, "ns")
name := nameGen.Draw(t, "name")
modelSv, modelErr := model.delete(ns, name)
sv, err := sut.DeleteSv(t.Context(), ns, name)
if err != nil || modelErr != nil {
require.ErrorIs(t, err, modelErr)
return
}
require.Equal(t, modelSv.Namespace, sv.Namespace)
require.Equal(t, modelSv.Name, sv.Name)
require.Equal(t, modelSv.Status.Version, sv.Status.Version)
},
"lease": func(t *rapid.T) {
// Taken from secureValueMetadataStorage.acquireLeases
minAge := 300 * time.Second
leaseTTL := 30 * time.Second
maxBatchSize := rapid.Uint16Range(1, 10).Draw(t, "maxBatchSize")
modelSvs, modelErr := model.leaseInactiveSecureValues(sut.Clock.Now(), minAge, leaseTTL, maxBatchSize)
svs, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), maxBatchSize)
require.ErrorIs(t, err, modelErr)
require.Equal(t, len(modelSvs), len(svs))
},
"advanceTime": func(t *rapid.T) {
duration := time.Duration(rapid.IntRange(1, 10).Draw(t, "minutes")) * time.Minute
sut.Clock.AdvanceBy(duration)
},
})
})
}

57
pkg/storage/secret/metadata/secure_value_test.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"slices"
"testing"
"time"
"github.com/mitchellh/copystructure"
"github.com/stretchr/testify/require"
@ -20,7 +21,9 @@ import (
type modelSecureValue struct {
*secretv1beta1.SecureValue
active bool
active bool
created time.Time
leaseCreated time.Time
}
// A simplified model of the grafana secrets manager
@ -69,8 +72,8 @@ func (m *model) readActiveVersion(namespace, name string) *modelSecureValue {
return nil
}
func (m *model) create(sv *secretv1beta1.SecureValue, actorUID string) (*secretv1beta1.SecureValue, error) {
modelSv := &modelSecureValue{sv, false}
func (m *model) create(now time.Time, sv *secretv1beta1.SecureValue) (*secretv1beta1.SecureValue, error) {
modelSv := &modelSecureValue{SecureValue: sv, active: false, created: now}
modelSv.Status.Version = m.getNewVersionNumber(modelSv.Namespace, modelSv.Name)
modelSv.Status.ExternalID = fmt.Sprintf("%d", modelSv.Status.Version)
m.secureValues = append(m.secureValues, modelSv)
@ -78,7 +81,7 @@ func (m *model) create(sv *secretv1beta1.SecureValue, actorUID string) (*secretv
return modelSv.SecureValue, nil
}
func (m *model) update(newSecureValue *secretv1beta1.SecureValue, actorUID string) (*secretv1beta1.SecureValue, bool, error) {
func (m *model) update(now time.Time, newSecureValue *secretv1beta1.SecureValue) (*secretv1beta1.SecureValue, bool, error) {
// If the payload doesn't contain a value, get the value from current version
if newSecureValue.Spec.Value == nil {
sv := m.readActiveVersion(newSecureValue.Namespace, newSecureValue.Name)
@ -87,7 +90,7 @@ func (m *model) update(newSecureValue *secretv1beta1.SecureValue, actorUID strin
}
newSecureValue.Spec.Value = sv.Spec.Value
}
createdSv, err := m.create(newSecureValue, actorUID)
createdSv, err := m.create(now, newSecureValue)
return createdSv, true, err
}
@ -141,6 +144,22 @@ func (m *model) read(namespace, name string) (*secretv1beta1.SecureValue, error)
return modelSv.SecureValue, nil
}
func (m *model) leaseInactiveSecureValues(now time.Time, minAge, leaseTTL time.Duration, maxBatchSize uint16) ([]*modelSecureValue, error) {
out := make([]*modelSecureValue, 0)
for _, sv := range m.secureValues {
if len(out) >= int(maxBatchSize) {
break
}
if !sv.active && now.Sub(sv.created) > minAge && now.Sub(sv.leaseCreated) > leaseTTL {
sv.leaseCreated = now
out = append(out, sv)
}
}
return out, nil
}
var (
decryptersGen = rapid.SampledFrom([]string{"svc1", "svc2", "svc3", "svc4", "svc5"})
nameGen = rapid.SampledFrom([]string{"n1", "n2", "n3", "n4", "n5"})
@ -204,16 +223,17 @@ func TestModel(t *testing.T) {
t.Parallel()
m := newModel()
now := time.Now()
// Create a secure value
sv1, err := m.create(deepCopy(sv), "actor-uid")
sv1, err := m.create(now, deepCopy(sv))
require.NoError(t, err)
require.Equal(t, sv.Namespace, sv1.Namespace)
require.Equal(t, sv.Name, sv1.Name)
require.EqualValues(t, 1, sv1.Status.Version)
// Create a new version of a secure value
sv2, err := m.create(deepCopy(sv), "actor-uid")
sv2, err := m.create(now, deepCopy(sv))
require.NoError(t, err)
require.Equal(t, sv.Namespace, sv2.Namespace)
require.Equal(t, sv.Name, sv2.Name)
@ -225,11 +245,13 @@ func TestModel(t *testing.T) {
m := newModel()
sv1, err := m.create(deepCopy(sv), "actor-uid")
now := time.Now()
sv1, err := m.create(now, deepCopy(sv))
require.NoError(t, err)
// Create a new version of a secure value by updating it
sv2, _, err := m.update(deepCopy(sv1), "actor-uid")
sv2, _, err := m.update(now, deepCopy(sv1))
require.NoError(t, err)
require.Equal(t, sv.Namespace, sv2.Namespace)
require.Equal(t, sv.Name, sv2.Name)
@ -239,14 +261,14 @@ func TestModel(t *testing.T) {
sv3 := deepCopy(sv2)
sv3.Name = "i_dont_exist"
sv3.Spec.Value = nil
_, _, err = m.update(sv3, "actor-uid")
_, _, err = m.update(now, sv3)
require.ErrorIs(t, err, contracts.ErrSecureValueNotFound)
// Updating a value that doesn't exist creates a new version
sv4 := deepCopy(sv3)
sv4.Name = "i_dont_exist"
sv4.Spec.Value = ptr.To(secretv1beta1.NewExposedSecureValue("sv4"))
sv4, _, err = m.update(sv4, "actor-uid")
sv4, _, err = m.update(now, sv4)
require.NoError(t, err)
require.EqualValues(t, 1, sv4.Status.Version)
})
@ -255,8 +277,9 @@ func TestModel(t *testing.T) {
t.Parallel()
m := newModel()
now := time.Now()
sv1, err := m.create(deepCopy(sv), "actor-uid")
sv1, err := m.create(now, deepCopy(sv))
require.NoError(t, err)
// Deleting a secure value
@ -275,6 +298,7 @@ func TestModel(t *testing.T) {
t.Parallel()
m := newModel()
now := time.Now()
// No secure values exist yet
list, err := m.list(sv.Namespace)
@ -282,7 +306,7 @@ func TestModel(t *testing.T) {
require.Equal(t, 0, len(list.Items))
// Create a secure value
sv1, err := m.create(deepCopy(sv), "actor-uid")
sv1, err := m.create(now, deepCopy(sv))
require.NoError(t, err)
// 1 secure value exists and it should be returned
@ -298,6 +322,7 @@ func TestModel(t *testing.T) {
t.Parallel()
m := newModel()
now := time.Now()
// Decrypting a secure value that does not exist
result, err := m.decrypt("decrypter", "namespace", "name")
@ -308,7 +333,7 @@ func TestModel(t *testing.T) {
// Create a secure value
secret := "v1"
sv1, err := m.create(deepCopy(sv), "actor-uid")
sv1, err := m.create(now, deepCopy(sv))
require.NoError(t, err)
// Decrypt the just created secure value
@ -333,7 +358,7 @@ func TestStateMachine(t *testing.T) {
"create": func(t *rapid.T) {
sv := anySecureValueGen.Draw(t, "sv")
modelCreatedSv, modelErr := model.create(deepCopy(sv), "actor-uid")
modelCreatedSv, modelErr := model.create(sut.Clock.Now(), deepCopy(sv))
createdSv, err := sut.CreateSv(t.Context(), testutils.CreateSvWithSv(deepCopy(sv)))
if err != nil || modelErr != nil {
@ -346,7 +371,7 @@ func TestStateMachine(t *testing.T) {
},
"update": func(t *rapid.T) {
sv := updateSecureValueGen.Draw(t, "sv")
modelCreatedSv, _, modelErr := model.update(deepCopy(sv), "actor-uid")
modelCreatedSv, _, modelErr := model.update(sut.Clock.Now(), deepCopy(sv))
createdSv, err := sut.UpdateSv(t.Context(), deepCopy(sv))
if err != nil || modelErr != nil {
require.ErrorIs(t, err, modelErr)

7
pkg/storage/secret/metadata/testdata/mysql--secure_value_delete-deleteSecureValue.sql vendored Executable file
View File

@ -0,0 +1,7 @@
DELETE FROM
`secret_secure_value`
WHERE
`namespace` = 'ns' AND
`name` = 'name' AND
`version` = 1
;

14
pkg/storage/secret/metadata/testdata/mysql--secure_value_lease_inactive-lease inactive.sql vendored Executable file
View File

@ -0,0 +1,14 @@
UPDATE
`secret_secure_value`
SET
`lease_token` = 'token',
`lease_created` = 10
WHERE
`guid` IN (SELECT `guid`
FROM `secret_secure_value`
WHERE
`active` = FALSE AND
10 - `created` > 300 AND
10 - `lease_created` > 30
LIMIT 10)
;

26
pkg/storage/secret/metadata/testdata/mysql--secure_value_list_by_lease_token-list by lease token.sql vendored Executable file
View File

@ -0,0 +1,26 @@
SELECT
`guid`,
`name`,
`namespace`,
`annotations`,
`labels`,
`created`,
`created_by`,
`updated`,
`updated_by`,
`description`,
`keeper`,
`decrypters`,
`ref`,
`external_id`,
`version`,
`active`,
`owner_reference_api_group`,
`owner_reference_api_version`,
`owner_reference_kind`,
`owner_reference_name`
FROM
`secret_secure_value`
WHERE
`lease_token` = 'token'
;

7
pkg/storage/secret/metadata/testdata/postgres--secure_value_delete-deleteSecureValue.sql vendored Executable file
View File

@ -0,0 +1,7 @@
DELETE FROM
"secret_secure_value"
WHERE
"namespace" = 'ns' AND
"name" = 'name' AND
"version" = 1
;

14
pkg/storage/secret/metadata/testdata/postgres--secure_value_lease_inactive-lease inactive.sql vendored Executable file
View File

@ -0,0 +1,14 @@
UPDATE
"secret_secure_value"
SET
"lease_token" = 'token',
"lease_created" = 10
WHERE
"guid" IN (SELECT "guid"
FROM "secret_secure_value"
WHERE
"active" = FALSE AND
10 - "created" > 300 AND
10 - "lease_created" > 30
LIMIT 10)
;

26
pkg/storage/secret/metadata/testdata/postgres--secure_value_list_by_lease_token-list by lease token.sql vendored Executable file
View File

@ -0,0 +1,26 @@
SELECT
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"keeper",
"decrypters",
"ref",
"external_id",
"version",
"active",
"owner_reference_api_group",
"owner_reference_api_version",
"owner_reference_kind",
"owner_reference_name"
FROM
"secret_secure_value"
WHERE
"lease_token" = 'token'
;

7
pkg/storage/secret/metadata/testdata/sqlite--secure_value_delete-deleteSecureValue.sql vendored Executable file
View File

@ -0,0 +1,7 @@
DELETE FROM
"secret_secure_value"
WHERE
"namespace" = 'ns' AND
"name" = 'name' AND
"version" = 1
;

14
pkg/storage/secret/metadata/testdata/sqlite--secure_value_lease_inactive-lease inactive.sql vendored Executable file
View File

@ -0,0 +1,14 @@
UPDATE
"secret_secure_value"
SET
"lease_token" = 'token',
"lease_created" = 10
WHERE
"guid" IN (SELECT "guid"
FROM "secret_secure_value"
WHERE
"active" = FALSE AND
10 - "created" > 300 AND
10 - "lease_created" > 30
LIMIT 10)
;

26
pkg/storage/secret/metadata/testdata/sqlite--secure_value_list_by_lease_token-list by lease token.sql vendored Executable file
View File

@ -0,0 +1,26 @@
SELECT
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"keeper",
"decrypters",
"ref",
"external_id",
"version",
"active",
"owner_reference_api_group",
"owner_reference_api_version",
"owner_reference_kind",
"owner_reference_name"
FROM
"secret_secure_value"
WHERE
"lease_token" = 'token'
;

19
pkg/storage/secret/migrator/migrator.go
View File

@ -181,4 +181,23 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {
Length: 253, // Limit enforced by K8s.
Nullable: true,
}))
mg.AddMigration("add lease_token column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
Name: "lease_token",
Type: migrator.DB_NVarchar,
Length: 36,
Nullable: true,
}))
mg.AddMigration("add lease_token index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
Cols: []string{"lease_token"},
}))
mg.AddMigration("add lease_created column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
Name: "lease_created",
Type: migrator.DB_BigInt,
Nullable: false,
Default: "0",
}))
mg.AddMigration("add lease_created index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
Cols: []string{"lease_created"},
}))
}