Secrets: garbage collection (#110247)

* clean up older secret versions

* start gargbage collection worker as background service

* make gen-go

* fix typo

* make update-workspace

* undo go mod changes

* undo go work sum changes

* Update pkg/registry/apis/secret/garbagecollectionworker/worker.go

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* Update pkg/registry/apis/secret/garbagecollectionworker/worker.go

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* default gc_worker_batch_size to 1 minute

* fix typo

* fix typo

* add test to ensure cleaning up secure values is idempotent

* make gen-go

* make update-workspace

* undo go.mod and .sum changes

* undo enterprise imports

---------

Co-authored-by: Matheus Macabu <macabu.matheus@gmail.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

pkg/registry/apis/secret/clock/clock.go

@@ -0,0 +1,14 @@
+package clock
+
+import "time"
+
+type Clock struct {
+}
+
+func ProvideClock() *Clock {
+	return &Clock{}
+}
+
+func (c *Clock) Now() time.Time {
+	return time.Now()
+}

pkg/registry/apis/secret/contracts/clock.go

@@ -0,0 +1,7 @@
+package contracts
+
+import "time"
+
+type Clock interface {
+	Now() time.Time
+}

pkg/registry/apis/secret/contracts/secure_value.go

@@ -37,6 +37,8 @@ type SecureValueMetadataStorage interface {
 	SetVersionToActive(ctx context.Context, namespace xkube.Namespace, name string, version int64) error
 	SetVersionToInactive(ctx context.Context, namespace xkube.Namespace, name string, version int64) error
 	SetExternalID(ctx context.Context, namespace xkube.Namespace, name string, version int64, externalID ExternalID) error
+	Delete(ctx context.Context, namespace xkube.Namespace, name string, version int64) error
+	LeaseInactiveSecureValues(ctx context.Context, maxBatchSize uint16) ([]secretv1beta1.SecureValue, error)
 }
 
 type SecureValueService interface {

pkg/registry/apis/secret/garbagecollectionworker/worker.go

@@ -0,0 +1,117 @@
+package garbagecollectionworker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/grafana/grafana-app-sdk/logging"
+	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+	"github.com/grafana/grafana/pkg/setting"
+	"golang.org/x/sync/semaphore"
+)
+
+// Secure values have the `active` flag set to false on creation and deletion.
+// The `active` flag is set to true when the creation process succeeds.
+// The worker deletes secure values that are inactive because the creation process failed
+// or because the secure value has been deleted.
+type Worker struct {
+	Cfg                        *setting.Cfg
+	secureValueMetadataStorage contracts.SecureValueMetadataStorage
+	keeperMetadataStorage      contracts.KeeperMetadataStorage
+	keeperService              contracts.KeeperService
+}
+
+func ProvideWorker(
+	cfg *setting.Cfg,
+	secureValueMetadataStorage contracts.SecureValueMetadataStorage,
+	keeperMetadataStorage contracts.KeeperMetadataStorage,
+	keeperService contracts.KeeperService) *Worker {
+	return &Worker{
+		Cfg:                        cfg,
+		secureValueMetadataStorage: secureValueMetadataStorage,
+		keeperMetadataStorage:      keeperMetadataStorage,
+		keeperService:              keeperService}
+}
+
+func (w *Worker) Run(ctx context.Context) error {
+	if !w.Cfg.SecretsManagement.GCWorkerEnabled {
+		return nil
+	}
+
+	timer := time.NewTicker(w.Cfg.SecretsManagement.GCWorkerPollInterval)
+	defer timer.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+
+		case <-timer.C:
+			timeoutCtx, cancel := context.WithTimeout(context.Background(), w.Cfg.SecretsManagement.GCWorkerPerSecureValueCleanupTimeout)
+			if _, err := w.CleanupInactiveSecureValues(timeoutCtx); err != nil {
+				logging.FromContext(timeoutCtx).Error("cleaning up inactive secure values", "error", err)
+			}
+			cancel()
+		}
+	}
+}
+
+func (w *Worker) CleanupInactiveSecureValues(ctx context.Context) ([]secretv1beta1.SecureValue, error) {
+	secureValues, err := w.secureValueMetadataStorage.LeaseInactiveSecureValues(ctx, w.Cfg.SecretsManagement.GCWorkerMaxBatchSize)
+	if err != nil {
+		return nil, fmt.Errorf("fetching inactive secure values that need to be cleaned up: %w", err)
+	}
+	if len(secureValues) == 0 {
+		return nil, nil
+	}
+
+	errs := make([]error, len(secureValues))
+
+	sema := semaphore.NewWeighted(int64(w.Cfg.SecretsManagement.GCWorkerMaxConcurrentCleanups))
+	wg := &sync.WaitGroup{}
+	wg.Add(len(secureValues))
+
+	for i, sv := range secureValues {
+		if err := sema.Acquire(ctx, 1); err != nil {
+			return nil, fmt.Errorf("acquiring semaphore: %w", err)
+		}
+		go func(i int, sv *secretv1beta1.SecureValue) {
+			defer sema.Release(1)
+			defer wg.Done()
+			errs[i] = w.Cleanup(ctx, sv)
+		}(i, &sv)
+	}
+
+	wg.Wait()
+
+	return secureValues, errors.Join(errs...)
+}
+
+func (w *Worker) Cleanup(ctx context.Context, sv *secretv1beta1.SecureValue) error {
+	keeperCfg, err := w.keeperMetadataStorage.GetKeeperConfig(ctx, sv.Namespace, sv.Spec.Keeper, contracts.ReadOpts{ForUpdate: false})
+	if err != nil {
+		return fmt.Errorf("fetching keeper config: namespace=%+v keeperName=%+v %w", sv.Namespace, sv.Spec.Keeper, err)
+	}
+
+	keeper, err := w.keeperService.KeeperForConfig(keeperCfg)
+	if err != nil {
+		return fmt.Errorf("getting keeper for config: namespace=%+v keeperName=%+v %w", sv.Namespace, sv.Spec.Keeper, err)
+	}
+
+	// Keeper deletion is idempotent
+	if err := keeper.Delete(ctx, keeperCfg, sv.Namespace, sv.Name, sv.Status.Version); err != nil {
+		return fmt.Errorf("deleting secure value from keeper: %w", err)
+	}
+
+	// Metadata deletion is not idempotent but not found errors are ignored
+	if err := w.secureValueMetadataStorage.Delete(ctx, xkube.Namespace(sv.Namespace), sv.Name, sv.Status.Version); err != nil && !errors.Is(err, contracts.ErrSecureValueNotFound) {
+		return fmt.Errorf("deleting secure value from metadata storage: %w", err)
+	}
+
+	return nil
+}

pkg/registry/apis/secret/garbagecollectionworker/worker_test.go

@@ -0,0 +1,247 @@
+package garbagecollectionworker_test
+
+import (
+	"fmt"
+	"slices"
+	"testing"
+	"time"
+
+	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/testutils"
+	"github.com/grafana/grafana/pkg/storage/secret/encryption"
+	"github.com/mitchellh/copystructure"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
+	"pgregory.net/rapid"
+)
+
+func TestBasic(t *testing.T) {
+	t.Parallel()
+
+	t.Run("when no secure values exist, there's no work to do", func(t *testing.T) {
+		t.Parallel()
+		sut := testutils.Setup(t)
+		ids, err := sut.GarbageCollectionWorker.CleanupInactiveSecureValues(t.Context())
+		require.NoError(t, err)
+		require.Empty(t, ids)
+	})
+
+	t.Run("inactive secure values are not deleted immediately because of the grace period", func(t *testing.T) {
+		t.Parallel()
+		sut := testutils.Setup(t)
+
+		sv1, err := sut.CreateSv(t.Context())
+		require.NoError(t, err)
+
+		_, err = sut.DeleteSv(t.Context(), sv1.Namespace, sv1.Name)
+		require.NoError(t, err)
+
+		// Try to fetch inactive secure values for deletion
+		svs, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		require.Empty(t, svs)
+	})
+
+	t.Run("secure values are fetched for deletion and deleted from keeper", func(t *testing.T) {
+		sut := testutils.Setup(t)
+
+		sv, err := sut.CreateSv(t.Context())
+		require.NoError(t, err)
+
+		keeperCfg, err := sut.KeeperMetadataStorage.GetKeeperConfig(t.Context(), sv.Namespace, sv.Spec.Keeper, contracts.ReadOpts{ForUpdate: false})
+		require.NoError(t, err)
+
+		keeper, err := sut.KeeperService.KeeperForConfig(keeperCfg)
+		require.NoError(t, err)
+
+		// Get the secret value once to make sure it's reachable
+		exposedValue, err := keeper.Expose(t.Context(), keeperCfg, sv.Namespace, sv.Name, sv.Status.Version)
+		require.NoError(t, err)
+		require.NotEmpty(t, exposedValue.DangerouslyExposeAndConsumeValue())
+
+		_, err = sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
+		require.NoError(t, err)
+
+		// Advance time to wait for grace period
+		sut.Clock.AdvanceBy(10 * time.Minute)
+
+		svs, err := sut.GarbageCollectionWorker.CleanupInactiveSecureValues(t.Context())
+		require.NoError(t, err)
+		require.Equal(t, 1, len(svs))
+		require.Equal(t, sv.UID, svs[0].UID)
+
+		svs, err = sut.GarbageCollectionWorker.CleanupInactiveSecureValues(t.Context())
+		require.NoError(t, err)
+		require.Empty(t, svs)
+
+		// Try to get the secreet value again to make sure it's been deleted from the keeper
+		exposedValue, err = keeper.Expose(t.Context(), keeperCfg, sv.Namespace, sv.Name, sv.Status.Version)
+		require.ErrorIs(t, err, encryption.ErrEncryptedValueNotFound)
+		require.Empty(t, exposedValue)
+	})
+
+	t.Run("cleaning up secure values is idempotent", func(t *testing.T) {
+		t.Parallel()
+
+		sut := testutils.Setup(t)
+
+		sv, err := sut.CreateSv(t.Context())
+		require.NoError(t, err)
+
+		_, err = sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
+		require.NoError(t, err)
+
+		// Clean up the same secure value twice and ensure it succeeds
+		require.NoError(t, sut.GarbageCollectionWorker.Cleanup(t.Context(), sv))
+		require.NoError(t, sut.GarbageCollectionWorker.Cleanup(t.Context(), sv))
+	})
+}
+
+var (
+	decryptersGen     = rapid.SampledFrom([]string{"svc1", "svc2", "svc3", "svc4", "svc5"})
+	nameGen           = rapid.SampledFrom([]string{"n1", "n2", "n3", "n4", "n5"})
+	namespaceGen      = rapid.SampledFrom([]string{"ns1", "ns2", "ns3", "ns4", "ns5"})
+	anySecureValueGen = rapid.Custom(func(t *rapid.T) *secretv1beta1.SecureValue {
+		return &secretv1beta1.SecureValue{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      nameGen.Draw(t, "name"),
+				Namespace: namespaceGen.Draw(t, "ns"),
+			},
+			Spec: secretv1beta1.SecureValueSpec{
+				Description: rapid.SampledFrom([]string{"d1", "d2", "d3", "d4", "d5"}).Draw(t, "description"),
+				Value:       ptr.To(secretv1beta1.NewExposedSecureValue(rapid.SampledFrom([]string{"v1", "v2", "v3", "v4", "v5"}).Draw(t, "value"))),
+				Decrypters:  rapid.SliceOfDistinct(decryptersGen, func(v string) string { return v }).Draw(t, "decrypters"),
+			},
+			Status: secretv1beta1.SecureValueStatus{},
+		}
+	})
+)
+
+func TestProperty(t *testing.T) {
+	t.Parallel()
+
+	tt := t
+
+	rapid.Check(t, func(t *rapid.T) {
+		sut := testutils.Setup(tt)
+		model := newModel()
+
+		t.Repeat(map[string]func(*rapid.T){
+			"create": func(t *rapid.T) {
+				sv := anySecureValueGen.Draw(t, "sv")
+				svCopy := deepCopy(sv)
+
+				createdSv, err := sut.CreateSv(t.Context(), testutils.CreateSvWithSv(sv))
+				svCopy.UID = createdSv.UID
+				modelErr := model.create(sut.Clock.Now(), svCopy)
+				require.ErrorIs(t, err, modelErr)
+			},
+			"delete": func(t *rapid.T) {
+				if len(model.items) == 0 {
+					return
+				}
+
+				i := rapid.IntRange(0, len(model.items)-1).Draw(t, "index")
+				sv := model.items[i]
+				modelErr := model.delete(sv.Namespace, sv.Name)
+				_, err := sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
+				require.ErrorIs(t, err, modelErr)
+			},
+			"cleanup": func(t *rapid.T) {
+				// Taken from secureValueMetadataStorage.acquireLeases
+				minAge := 300 * time.Second
+				maxBatchSize := sut.GarbageCollectionWorker.Cfg.SecretsManagement.GCWorkerMaxBatchSize
+				modelDeleted, modelErr := model.cleanupInactiveSecureValues(sut.Clock.Now(), minAge, maxBatchSize)
+				deleted, err := sut.GarbageCollectionWorker.CleanupInactiveSecureValues(t.Context())
+				require.ErrorIs(t, err, modelErr)
+
+				require.Equal(t, len(modelDeleted), len(deleted), "model and impl deleted a different number of secure values")
+				seen := make(map[types.UID]bool, 0)
+				for _, v := range modelDeleted {
+					seen[v.UID] = true
+				}
+
+				for _, v := range deleted {
+					require.True(t, seen[v.UID], "impl deleted a secure value that the model did not")
+				}
+			},
+			"advanceTime": func(t *rapid.T) {
+				duration := time.Duration(rapid.IntRange(1, 10).Draw(t, "minutes")) * time.Minute
+				sut.Clock.AdvanceBy(duration)
+			},
+		})
+	})
+}
+
+type model struct {
+	items []*modelSecureValue
+}
+
+type modelSecureValue struct {
+	*secretv1beta1.SecureValue
+	active  bool
+	created time.Time
+}
+
+func newModel() *model {
+	return &model{
+		items: make([]*modelSecureValue, 0),
+	}
+}
+
+func (m *model) create(now time.Time, sv *secretv1beta1.SecureValue) error {
+	for _, item := range m.items {
+		if item.active && item.Namespace == sv.Namespace && item.Name == sv.Name {
+			item.active = false
+			break
+		}
+	}
+	m.items = append(m.items, &modelSecureValue{SecureValue: sv, active: true, created: now})
+	return nil
+}
+
+func (m *model) delete(ns string, name string) error {
+	for _, sv := range m.items {
+		if sv.active && sv.Namespace == ns && sv.Name == name {
+			sv.active = false
+			return nil
+		}
+	}
+
+	return contracts.ErrSecureValueNotFound
+}
+
+func (m *model) cleanupInactiveSecureValues(now time.Time, minAge time.Duration, maxBatchSize uint16) ([]*modelSecureValue, error) {
+	// Using a slice to allow duplicates
+	toDelete := make([]*modelSecureValue, 0)
+
+	for _, sv := range m.items {
+		if len(toDelete) >= int(maxBatchSize) {
+			break
+		}
+
+		if !sv.active && now.Sub(sv.created) > minAge {
+			toDelete = append(toDelete, sv)
+		}
+	}
+
+	// PERF: The slices are always small
+	m.items = slices.DeleteFunc(m.items, func(v1 *modelSecureValue) bool {
+		return slices.ContainsFunc(toDelete, func(v2 *modelSecureValue) bool {
+			return v2.UID == v1.UID
+		})
+	})
+
+	return toDelete, nil
+}
+
+func deepCopy[T any](sv T) T {
+	copied, err := copystructure.Copy(sv)
+	if err != nil {
+		panic(fmt.Sprintf("failed to copy secure value: %v", err))
+	}
+	return copied.(T)
+}

pkg/registry/apis/secret/testutils/testutils.go

@@ -3,6 +3,7 @@ package testutils
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/grafana/authlib/authn"
 	"github.com/grafana/authlib/types"
@@ -21,6 +22,7 @@ import (
 	cipher "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher/service"
 	osskmsproviders "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/kmsproviders"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/manager"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/garbagecollectionworker"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/mutator"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper/sqlkeeper"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/service"
@@ -32,6 +34,7 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/storage/secret/database"
 	encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
+
 	"github.com/grafana/grafana/pkg/storage/secret/metadata"
 	"github.com/grafana/grafana/pkg/storage/secret/migrator"
 )
@@ -71,7 +74,9 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 	keeperMetadataStorage, err := metadata.ProvideKeeperMetadataStorage(database, tracer, nil)
 	require.NoError(t, err)
 
-	secureValueMetadataStorage, err := metadata.ProvideSecureValueMetadataStorage(database, tracer, nil)
+	clock := NewFakeClock()
+
+	secureValueMetadataStorage, err := metadata.ProvideSecureValueMetadataStorage(clock, database, tracer, nil)
 	require.NoError(t, err)
 
 	// Initialize access client + access control
@@ -84,8 +89,11 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 	defaultKey := "SdlklWklckeLS"
 	cfg := setting.NewCfg()
 	cfg.SecretsManagement = setting.SecretsManagerSettings{
-		CurrentEncryptionProvider: "secret_key.v1",
+		CurrentEncryptionProvider:     "secret_key.v1",
-		ConfiguredKMSProviders:    map[string]map[string]string{"secret_key.v1": {"secret_key": defaultKey}},
+		ConfiguredKMSProviders:        map[string]map[string]string{"secret_key.v1": {"secret_key": defaultKey}},
+		GCWorkerEnabled:               false,
+		GCWorkerMaxBatchSize:          2,
+		GCWorkerMaxConcurrentCleanups: 2,
 	}
 	store, err := encryptionstorage.ProvideDataKeyStorage(database, tracer, nil)
 	require.NoError(t, err)
@@ -143,6 +151,12 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 
 	consolidationService := service.ProvideConsolidationService(tracer, globalDataKeyStore, encryptedValueStorage, globalEncryptedValueStorage, encryptionManager)
 
+	garbageCollectionWorker := garbagecollectionworker.ProvideWorker(
+		cfg,
+		secureValueMetadataStorage,
+		keeperMetadataStorage,
+		keeperService)
+
 	return Sut{
 		SecureValueService:          secureValueService,
 		SecureValueMetadataStorage:  secureValueMetadataStorage,
@@ -156,6 +170,10 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 		ConsolidationService:        consolidationService,
 		EncryptionManager:           encryptionManager,
 		GlobalDataKeyStore:          globalDataKeyStore,
+		GarbageCollectionWorker:     garbageCollectionWorker,
+		Clock:                       clock,
+		KeeperService:               keeperService,
+		KeeperMetadataStorage:       keeperMetadataStorage,
 	}
 }
 
@@ -172,6 +190,11 @@ type Sut struct {
 	ConsolidationService        contracts.ConsolidationService
 	EncryptionManager           contracts.EncryptionManager
 	GlobalDataKeyStore          contracts.GlobalDataKeyStorage
+	GarbageCollectionWorker     *garbagecollectionworker.Worker
+	// The fake clock passed to implementations to make testing easier
+	Clock                 *FakeClock
+	KeeperService         contracts.KeeperService
+	KeeperMetadataStorage contracts.KeeperMetadataStorage
 }
 
 type CreateSvConfig struct {
@@ -327,3 +350,19 @@ func CreateX509TestDir(t *testing.T) TestCertPaths {
 		CA:         caCertFile.Name(),
 	}
 }
+
+type FakeClock struct {
+	Current time.Time
+}
+
+func NewFakeClock() *FakeClock {
+	return &FakeClock{Current: time.Now()}
+}
+
+func (c *FakeClock) Now() time.Time {
+	return c.Current
+}
+
+func (c *FakeClock) AdvanceBy(duration time.Duration) {
+	c.Current = c.Current.Add(duration)
+}

pkg/registry/backgroundsvcs/background_services.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/usagestats/statscollector"
 	"github.com/grafana/grafana/pkg/registry"
 	apiregistry "github.com/grafana/grafana/pkg/registry/apis"
+	secretsgarbagecollectionworker "github.com/grafana/grafana/pkg/registry/apis/secret/garbagecollectionworker"
 	appregistry "github.com/grafana/grafana/pkg/registry/apps"
 	"github.com/grafana/grafana/pkg/services/accesscontrol/dualwrite"
 	"github.com/grafana/grafana/pkg/services/anonymous/anonimpl"
@@ -69,6 +70,7 @@ func ProvideBackgroundServiceRegistry(
 	appRegistry *appregistry.Service,
 	pluginDashboardUpdater *plugindashboardsservice.DashboardUpdater,
 	dashboardServiceImpl *service.DashboardServiceImpl,
+	secretsGarbageCollectionWorker *secretsgarbagecollectionworker.Worker,
 	// Need to make sure these are initialized, is there a better place to put them?
 	_ dashboardsnapshots.Service,
 	_ serviceaccounts.Service,
@@ -115,6 +117,7 @@ func ProvideBackgroundServiceRegistry(
 		appRegistry,
 		pluginDashboardUpdater,
 		dashboardServiceImpl,
+		secretsGarbageCollectionWorker,
 	)
 }
 

pkg/server/wire.go

@@ -42,10 +42,12 @@ import (
 	"github.com/grafana/grafana/pkg/middleware/loggermw"
 	apiregistry "github.com/grafana/grafana/pkg/registry/apis"
 	"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
+	secretclock "github.com/grafana/grafana/pkg/registry/apis/secret/clock"
 	secretcontracts "github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
 	secretdecrypt "github.com/grafana/grafana/pkg/registry/apis/secret/decrypt"
 	cipher "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher/service"
 	encryptionManager "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/manager"
+	secretsgarbagecollectionworker "github.com/grafana/grafana/pkg/registry/apis/secret/garbagecollectionworker"
 	secretinline "github.com/grafana/grafana/pkg/registry/apis/secret/inline"
 	secretmutator "github.com/grafana/grafana/pkg/registry/apis/secret/mutator"
 	secretsecurevalueservice "github.com/grafana/grafana/pkg/registry/apis/secret/service"
@@ -312,6 +314,7 @@ var wireBasicSet = wire.NewSet(
 	wire.Bind(new(secrets.Service), new(*secretsManager.SecretsService)),
 	secretsDatabase.ProvideSecretsStore,
 	wire.Bind(new(secrets.Store), new(*secretsDatabase.SecretsStoreImpl)),
+	secretsgarbagecollectionworker.ProvideWorker,
 	grafanads.ProvideService,
 	wire.Bind(new(dashboardsnapshots.Store), new(*dashsnapstore.DashboardSnapshotStore)),
 	dashsnapstore.ProvideStore,
@@ -442,7 +445,9 @@ var wireBasicSet = wire.NewSet(
 	secretmutator.ProvideSecureValueMutator,
 	secretmigrator.NewWithEngine,
 	secretdatabase.ProvideDatabase,
+	secretclock.ProvideClock,
 	wire.Bind(new(secretcontracts.Database), new(*secretdatabase.Database)),
+	wire.Bind(new(secretcontracts.Clock), new(*secretclock.Clock)),
 	encryptionManager.ProvideEncryptionManager,
 	cipher.ProvideAESGCMCipherService,
 	// Unified storage

pkg/setting/setting_secrets_manager.go

@@ -2,6 +2,7 @@ package setting
 
 import (
 	"strings"
+	"time"
 )
 
 const (
@@ -22,6 +23,17 @@ type SecretsManagerSettings struct {
 	GrpcServerTLSServerName string // Server name to use for TLS verification
 	GrpcServerAddress       string // Address for gRPC secrets server
 	GrpcGrafanaServiceName  string // Service name to use for background grafana decryption/inline
+
+	// Used for testing. Set to false to disable the control loop.
+	GCWorkerEnabled bool
+	// Max number of inactive secure values to fetch from the database.
+	GCWorkerMaxBatchSize uint16
+	// Max number of tasks to delete secure values that can be inflight at a time.
+	GCWorkerMaxConcurrentCleanups uint16
+	// How long to wait for between fetching inactive secure values for cleanup.
+	GCWorkerPollInterval time.Duration
+	// How long to wait for the process to clean up a secure value to complete.
+	GCWorkerPerSecureValueCleanupTimeout time.Duration
 }
 
 func (cfg *Cfg) readSecretsManagerSettings() {
@@ -35,6 +47,12 @@ func (cfg *Cfg) readSecretsManagerSettings() {
 	cfg.SecretsManagement.GrpcServerAddress = valueAsString(secretsMgmt, "grpc_server_address", "")
 	cfg.SecretsManagement.GrpcGrafanaServiceName = valueAsString(secretsMgmt, "grpc_grafana_service_name", "")
 
+	cfg.SecretsManagement.GCWorkerEnabled = secretsMgmt.Key("gc_worker_enabled").MustBool(true)
+	cfg.SecretsManagement.GCWorkerMaxBatchSize = uint16(secretsMgmt.Key("gc_worker_batch_size").MustUint(16))
+	cfg.SecretsManagement.GCWorkerMaxConcurrentCleanups = uint16(secretsMgmt.Key("gc_worker_max_concurrency").MustUint(16))
+	cfg.SecretsManagement.GCWorkerPollInterval = secretsMgmt.Key("gc_worker_poll_interval").MustDuration(1 * time.Minute)
+	cfg.SecretsManagement.GCWorkerPerSecureValueCleanupTimeout = secretsMgmt.Key("gc_worker_per_request_timeout").MustDuration(5 * time.Second)
+
 	// Extract available KMS providers from configuration sections
 	providers := make(map[string]map[string]string)
 	for _, section := range cfg.Raw.Sections() {

pkg/storage/secret/metadata/data/secure_value_delete.sql

@@ -0,0 +1,7 @@
+DELETE FROM 
+  {{ .Ident "secret_secure_value" }}
+WHERE 
+  {{ .Ident "namespace" }} = {{ .Arg .Namespace }} AND
+  {{ .Ident "name" }} = {{ .Arg .Name }} AND
+  {{ .Ident "version" }} = {{ .Arg .Version }}
+;

pkg/storage/secret/metadata/data/secure_value_lease_inactive.sql

@@ -0,0 +1,14 @@
+UPDATE
+  {{ .Ident "secret_secure_value" }}
+SET
+  {{ .Ident "lease_token" }} = {{ .Arg .LeaseToken }},
+  {{ .Ident "lease_created" }} = {{ .Arg .Now }}
+WHERE 
+  {{ .Ident "guid" }} IN (SELECT {{ .Ident "guid"}} 
+                          FROM {{ .Ident "secret_secure_value" }} 
+                          WHERE 
+                            {{ .Ident "active" }} = FALSE AND
+                            {{ .Arg .Now }} - {{ .Ident "created" }} > {{ .Arg .MinAge }} AND
+                            {{ .Arg .Now }} - {{ .Ident "lease_created" }} > {{ .Arg .LeaseTTL }}
+                          LIMIT {{ .Arg .MaxBatchSize }})
+;

pkg/storage/secret/metadata/data/secure_value_list_by_lease_token.sql

@@ -0,0 +1,26 @@
+SELECT
+  {{ .Ident "guid" }},
+  {{ .Ident "name" }},
+  {{ .Ident "namespace" }},
+  {{ .Ident "annotations" }},
+  {{ .Ident "labels" }},
+  {{ .Ident "created" }},
+  {{ .Ident "created_by" }},
+  {{ .Ident "updated" }},
+  {{ .Ident "updated_by" }},
+  {{ .Ident "description" }},
+  {{ .Ident "keeper" }},
+  {{ .Ident "decrypters" }},
+  {{ .Ident "ref" }},
+  {{ .Ident "external_id" }},
+  {{ .Ident "version" }},
+  {{ .Ident "active" }},
+  {{ .Ident "owner_reference_api_group" }},
+  {{ .Ident "owner_reference_api_version" }},
+  {{ .Ident "owner_reference_kind" }},
+  {{ .Ident "owner_reference_name" }}
+FROM
+  {{ .Ident "secret_secure_value" }}
+WHERE 
+  {{ .Ident "lease_token" }} = {{ .Arg .LeaseToken }}
+;

pkg/storage/secret/metadata/metrics/metrics.go

@@ -30,6 +30,7 @@ type StorageMetrics struct {
 	SecureValueMetadataListDuration   *prometheus.HistogramVec
 	SecureValueSetExternalIDDuration  *prometheus.HistogramVec
 	SecureValueSetStatusDuration      *prometheus.HistogramVec
+	SecureValueDeleteDuration         *prometheus.HistogramVec
 
 	DecryptDuration *prometheus.HistogramVec
 }
@@ -116,6 +117,13 @@ func newStorageMetrics() *StorageMetrics {
 			Help:      "Duration of secure value set status operations",
 			Buckets:   prometheus.DefBuckets,
 		}, []string{successLabel}),
+		SecureValueDeleteDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "secure_value_delete_duration_seconds",
+			Help:      "Duration of secure value delete operations",
+			Buckets:   prometheus.DefBuckets,
+		}, []string{successLabel}),
 
 		// Decrypt metrics
 		DecryptDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
@@ -151,6 +159,7 @@ func NewStorageMetrics(reg prometheus.Registerer) *StorageMetrics {
 				m.SecureValueMetadataListDuration,
 				m.SecureValueSetExternalIDDuration,
 				m.SecureValueSetStatusDuration,
+				m.SecureValueDeleteDuration,
 				m.DecryptDuration,
 			)
 		}

pkg/storage/secret/metadata/query.go

@@ -28,6 +28,9 @@ var (
 	sqlSecureValueList             = mustTemplate("secure_value_list.sql")
 	sqlSecureValueCreate           = mustTemplate("secure_value_create.sql")
 	sqlSecureValueUpdateExternalId = mustTemplate("secure_value_updateExternalId.sql")
+	sqlSecureValueDelete           = mustTemplate("secure_value_delete.sql")
+	sqlSecureValueLeaseInactive    = mustTemplate("secure_value_lease_inactive.sql")
+	sqlSecureValueListByLeaseToken = mustTemplate("secure_value_list_by_lease_token.sql")
 
 	sqlGetLatestSecureValueVersion     = mustTemplate("secure_value_get_latest_version.sql")
 	sqlSecureValueSetVersionToActive   = mustTemplate("secure_value_set_version_to_active.sql")
@@ -208,3 +211,39 @@ type updateExternalIdSecureValue struct {
 func (r updateExternalIdSecureValue) Validate() error {
 	return nil // TODO
 }
+
+type deleteSecureValue struct {
+	sqltemplate.SQLTemplate
+	Namespace string
+	Name      string
+	Version   int64
+}
+
+// Validate is only used if we use `dbutil` from `unifiedstorage`
+func (r deleteSecureValue) Validate() error {
+	return nil // TODO
+}
+
+type leaseInactiveSecureValues struct {
+	sqltemplate.SQLTemplate
+	LeaseToken   string
+	MaxBatchSize uint16
+	MinAge       int64
+	LeaseTTL     int64
+	Now          int64
+}
+
+// Validate is only used if we use `dbutil` from `unifiedstorage`
+func (r leaseInactiveSecureValues) Validate() error {
+	return nil // TODO
+}
+
+type listSecureValuesByLeaseToken struct {
+	sqltemplate.SQLTemplate
+	LeaseToken string
+}
+
+// Validate is only used if we use `dbutil` from `unifiedstorage`
+func (r listSecureValuesByLeaseToken) Validate() error {
+	return nil // TODO
+}

pkg/storage/secret/metadata/query_test.go

@@ -3,6 +3,7 @@ package metadata
 import (
 	"testing"
 	"text/template"
+	"time"
 
 	"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
 	"k8s.io/utils/ptr"
@@ -238,6 +239,39 @@ func TestSecureValueQueries(t *testing.T) {
 					},
 				},
 			},
+			sqlSecureValueDelete: {
+				{
+					Name: "deleteSecureValue",
+					Data: &deleteSecureValue{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						Namespace:   "ns",
+						Name:        "name",
+						Version:     1,
+					},
+				},
+			},
+			sqlSecureValueLeaseInactive: {
+				{
+					Name: "lease inactive",
+					Data: &leaseInactiveSecureValues{
+						SQLTemplate:  mocks.NewTestingSQLTemplate(),
+						Now:          10,
+						LeaseToken:   "token",
+						LeaseTTL:     int64((30 * time.Second).Seconds()),
+						MaxBatchSize: 10,
+						MinAge:       int64((300 * time.Second).Seconds()),
+					},
+				},
+			},
+			sqlSecureValueListByLeaseToken: {
+				{
+					Name: "list by lease token",
+					Data: &listSecureValuesByLeaseToken{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						LeaseToken:  "token",
+					},
+				},
+			},
 		},
 	})
 }

pkg/storage/secret/metadata/secure_value_model.go

@@ -122,18 +122,18 @@ func (sv *secureValueDB) toKubernetes() (*secretv1beta1.SecureValue, error) {
 }
 
 // toCreateRow maps a Kubernetes resource into a DB row for new resources being created/inserted.
-func toCreateRow(sv *secretv1beta1.SecureValue, actorUID string) (*secureValueDB, error) {
+func toCreateRow(now time.Time, sv *secretv1beta1.SecureValue, actorUID string) (*secureValueDB, error) {
 	row, err := toRow(sv, "")
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert SecureValue to secureValueDB: %w", err)
 	}
 
-	now := time.Now().UTC().Unix()
+	timestamp := now.UTC().Unix()
 
 	row.GUID = uuid.New().String()
-	row.Created = now
+	row.Created = timestamp
 	row.CreatedBy = actorUID
-	row.Updated = now
+	row.Updated = timestamp
 	row.UpdatedBy = actorUID
 
 	return row, nil

pkg/storage/secret/metadata/secure_value_store.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -23,11 +24,13 @@ import (
 var _ contracts.SecureValueMetadataStorage = (*secureValueMetadataStorage)(nil)
 
 func ProvideSecureValueMetadataStorage(
+	clock contracts.Clock,
 	db contracts.Database,
 	tracer trace.Tracer,
 	reg prometheus.Registerer,
 ) (contracts.SecureValueMetadataStorage, error) {
 	return &secureValueMetadataStorage{
+		clock:   clock,
 		db:      db,
 		dialect: sqltemplate.DialectForDriver(db.DriverName()),
 		metrics: metrics.NewStorageMetrics(reg),
@@ -37,6 +40,7 @@ func ProvideSecureValueMetadataStorage(
 
 // secureValueMetadataStorage is the actual implementation of the secure value (metadata) storage.
 type secureValueMetadataStorage struct {
+	clock   contracts.Clock
 	db      contracts.Database
 	dialect sqltemplate.Dialect
 	metrics *metrics.StorageMetrics
@@ -44,7 +48,7 @@ type secureValueMetadataStorage struct {
 }
 
 func (s *secureValueMetadataStorage) Create(ctx context.Context, sv *secretv1beta1.SecureValue, actorUID string) (_ *secretv1beta1.SecureValue, svmCreateErr error) {
-	start := time.Now()
+	start := s.clock.Now()
 	name := sv.GetName()
 	namespace := sv.GetNamespace()
 	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Create", trace.WithAttributes(
@@ -123,7 +127,7 @@ func (s *secureValueMetadataStorage) Create(ctx context.Context, sv *secretv1bet
 		for {
 			sv.Status.Version = version
 
-			row, err = toCreateRow(sv, actorUID)
+			row, err = toCreateRow(s.clock.Now(), sv, actorUID)
 			if err != nil {
 				return fmt.Errorf("to create row: %w", err)
 			}
@@ -256,7 +260,7 @@ func (s *secureValueMetadataStorage) readActiveVersion(ctx context.Context, name
 }
 
 func (s *secureValueMetadataStorage) Read(ctx context.Context, namespace xkube.Namespace, name string, opts contracts.ReadOpts) (_ *secretv1beta1.SecureValue, readErr error) {
-	start := time.Now()
+	start := s.clock.Now()
 	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Read", trace.WithAttributes(
 		attribute.String("name", name),
 		attribute.String("namespace", namespace.String()),
@@ -297,7 +301,7 @@ func (s *secureValueMetadataStorage) Read(ctx context.Context, namespace xkube.N
 }
 
 func (s *secureValueMetadataStorage) List(ctx context.Context, namespace xkube.Namespace) (svList []secretv1beta1.SecureValue, listErr error) {
-	start := time.Now()
+	start := s.clock.Now()
 	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.List", trace.WithAttributes(
 		attribute.String("namespace", namespace.String()),
 	))
@@ -400,7 +404,7 @@ func (s *secureValueMetadataStorage) SetVersionToActive(ctx context.Context, nam
 		return fmt.Errorf("setting secure value version to active: namespace=%+v name=%+v version=%+v %w", namespace, name, version, err)
 	}
 
-	// validate modified cound
+	// validate modified count
 	modifiedCount, err := res.RowsAffected()
 	if err != nil {
 		return fmt.Errorf("fetching number of modified rows: %w", err)
@@ -449,7 +453,7 @@ func (s *secureValueMetadataStorage) SetVersionToInactive(ctx context.Context, n
 }
 
 func (s *secureValueMetadataStorage) SetExternalID(ctx context.Context, namespace xkube.Namespace, name string, version int64, externalID contracts.ExternalID) (setExtIDErr error) {
-	start := time.Now()
+	start := s.clock.Now()
 	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.SetExternalID", trace.WithAttributes(
 		attribute.String("name", name),
 		attribute.String("namespace", namespace.String()),
@@ -508,3 +512,164 @@ func (s *secureValueMetadataStorage) SetExternalID(ctx context.Context, namespac
 
 	return nil
 }
+
+func (s *secureValueMetadataStorage) Delete(ctx context.Context, namespace xkube.Namespace, name string, version int64) (err error) {
+	start := s.clock.Now()
+	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Delete", trace.WithAttributes(
+		attribute.String("name", name),
+		attribute.String("namespace", namespace.String()),
+		attribute.Int64("version", version),
+	))
+
+	defer span.End()
+
+	defer func() {
+		success := err == nil
+		args := []any{
+			"namespace", namespace.String(),
+			"name", name,
+			"version", strconv.FormatInt(version, 10),
+			"success", success,
+		}
+
+		if !success {
+			span.SetStatus(codes.Error, "SecureValueMetadataStorage.Delete failed")
+			span.RecordError(err)
+			args = append(args, "error", err)
+		}
+
+		logging.FromContext(ctx).Info("SecureValueMetadataStorage.Delete", args...)
+		s.metrics.SecureValueDeleteDuration.WithLabelValues(strconv.FormatBool(success)).Observe(time.Since(start).Seconds())
+	}()
+
+	req := deleteSecureValue{
+		SQLTemplate: sqltemplate.New(s.dialect),
+		Namespace:   namespace.String(),
+		Name:        name,
+		Version:     version,
+	}
+
+	q, err := sqltemplate.Execute(sqlSecureValueDelete, req)
+	if err != nil {
+		return fmt.Errorf("execute template %q: %w", sqlSecureValueDelete.Name(), err)
+	}
+
+	res, err := s.db.ExecContext(ctx, q, req.GetArgs()...)
+	if err != nil {
+		return fmt.Errorf("deleting secure value: namespace=%+v name=%+v version=%+v %w", namespace, name, version, err)
+	}
+
+	modifiedCount, err := res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("getting rows affected: %w", err)
+	}
+	// Deleting is idempotent so modifiedCunt must be in {0, 1}
+	if modifiedCount > 1 {
+		return fmt.Errorf("secureValueMetadataStorage.Delete: delete more than one secret, this is a bug, check the where condition: modifiedCount=%d", modifiedCount)
+	}
+
+	return nil
+}
+
+func (s *secureValueMetadataStorage) LeaseInactiveSecureValues(ctx context.Context, maxBatchSize uint16) (out []secretv1beta1.SecureValue, err error) {
+	start := s.clock.Now()
+	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.LeaseInactiveSecureValues", trace.WithAttributes(
+		attribute.Int("maxBatchSize", int(maxBatchSize)),
+	))
+
+	defer span.End()
+
+	defer func() {
+		success := err == nil
+
+		if !success {
+			span.SetStatus(codes.Error, "SecureValueMetadataStorage.LeaseInactiveSecureValues failed")
+			span.RecordError(err)
+		}
+
+		s.metrics.SecureValueDeleteDuration.WithLabelValues(strconv.FormatBool(success)).Observe(time.Since(start).Seconds())
+	}()
+
+	leaseToken := uuid.NewString()
+	if err := s.acquireLeases(ctx, leaseToken, maxBatchSize); err != nil {
+		return nil, fmt.Errorf("acquiring leases for inactive secure values: %w", err)
+	}
+
+	secureValues, err := s.listByLeaseToken(ctx, leaseToken)
+	if err != nil {
+		return nil, fmt.Errorf("fetching secure values by lease token: %w", err)
+	}
+
+	return secureValues, nil
+}
+
+func (s *secureValueMetadataStorage) acquireLeases(ctx context.Context, leaseToken string, maxBatchSize uint16) error {
+	req := leaseInactiveSecureValues{
+		SQLTemplate:  sqltemplate.New(s.dialect),
+		LeaseToken:   leaseToken,
+		MaxBatchSize: maxBatchSize,
+		MinAge:       int64((300 * time.Second).Seconds()),
+		LeaseTTL:     int64((30 * time.Second).Seconds()),
+		Now:          s.clock.Now().UTC().Unix(),
+	}
+
+	q, err := sqltemplate.Execute(sqlSecureValueLeaseInactive, req)
+	if err != nil {
+		return fmt.Errorf("execute template %q: %w", sqlSecureValueLeaseInactive.Name(), err)
+	}
+
+	if _, err := s.db.ExecContext(ctx, q, req.GetArgs()...); err != nil {
+		return fmt.Errorf("leasing inactive secure values: %w", err)
+	}
+
+	return nil
+}
+
+func (s *secureValueMetadataStorage) listByLeaseToken(ctx context.Context, leaseToken string) ([]secretv1beta1.SecureValue, error) {
+	req := listSecureValuesByLeaseToken{
+		SQLTemplate: sqltemplate.New(s.dialect),
+		LeaseToken:  leaseToken,
+	}
+
+	q, err := sqltemplate.Execute(sqlSecureValueListByLeaseToken, req)
+	if err != nil {
+		return nil, fmt.Errorf("execute template %q: %w", sqlSecureValueListByLeaseToken.Name(), err)
+	}
+
+	rows, err := s.db.QueryContext(ctx, q, req.GetArgs()...)
+	if err != nil {
+		return nil, fmt.Errorf("listing secure values: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	secureValues := make([]secretv1beta1.SecureValue, 0)
+	for rows.Next() {
+		row := secureValueDB{}
+
+		err = rows.Scan(&row.GUID,
+			&row.Name, &row.Namespace, &row.Annotations,
+			&row.Labels,
+			&row.Created, &row.CreatedBy,
+			&row.Updated, &row.UpdatedBy,
+			&row.Description, &row.Keeper, &row.Decrypters,
+			&row.Ref, &row.ExternalID, &row.Version, &row.Active,
+			&row.OwnerReferenceAPIGroup, &row.OwnerReferenceAPIVersion, &row.OwnerReferenceKind, &row.OwnerReferenceName,
+		)
+
+		if err != nil {
+			return nil, fmt.Errorf("error reading secure value row: %w", err)
+		}
+
+		secureValue, err := row.toKubernetes()
+		if err != nil {
+			return nil, fmt.Errorf("convert to kubernetes object: %w", err)
+		}
+
+		secureValues = append(secureValues, *secureValue)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("read rows error: %w", err)
+	}
+
+	return secureValues, nil
+}

pkg/storage/secret/metadata/secure_value_store_test.go

@@ -3,13 +3,17 @@ package metadata_test
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel/trace/noop"
 	"k8s.io/utils/ptr"
+	"pgregory.net/rapid"
 
 	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/clock"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/testutils"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/storage/secret/database"
@@ -43,7 +47,7 @@ func Test_SecureValueMetadataStorage_CreateAndRead(t *testing.T) {
 	db := database.ProvideDatabase(testDB, tracer)
 
 	// Initialize the secure value storage
-	secureValueStorage, err := metadata.ProvideSecureValueMetadataStorage(db, tracer, nil)
+	secureValueStorage, err := metadata.ProvideSecureValueMetadataStorage(clock.ProvideClock(), db, tracer, nil)
 	require.NoError(t, err)
 
 	// Initialize the keeper storage
@@ -142,3 +146,96 @@ func Test_SecureValueMetadataStorage_CreateAndRead(t *testing.T) {
 		require.Equal(t, contracts.ErrSecureValueNotFound, err)
 	})
 }
+
+func TestLeaseInactiveSecureValues(t *testing.T) {
+	t.Parallel()
+
+	t.Run("no secure value exists", func(t *testing.T) {
+		t.Parallel()
+
+		sut := testutils.Setup(t)
+		svs, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		require.Empty(t, svs)
+	})
+
+	t.Run("secure values are not visible to other requests during lease duration", func(t *testing.T) {
+		sut := testutils.Setup(t)
+		sv, err := sut.CreateSv(t.Context())
+		require.NoError(t, err)
+		_, err = sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
+		require.NoError(t, err)
+		// Advance clock to handle grace period
+		sut.Clock.AdvanceBy(10 * time.Minute)
+		// Acquire a lease on inactive secure values
+		values1, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		require.Equal(t, 1, len(values1))
+		require.Equal(t, sv.UID, values1[0].UID)
+		// Try to acquire a lease again
+		values2, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		// There's only one inactive secure value and it is already leased
+		require.Empty(t, values2)
+		// Advance clock to expire lease
+		sut.Clock.AdvanceBy(10 * time.Minute)
+		values3, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		// Should acquire a new lease since the previous one expired
+		require.Equal(t, 1, len(values3))
+		require.Equal(t, sv.UID, values3[0].UID)
+	})
+}
+
+func TestPropertySecureValueMetadataStorage(t *testing.T) {
+	t.Parallel()
+
+	tt := t
+
+	rapid.Check(t, func(t *rapid.T) {
+		sut := testutils.Setup(tt)
+		model := newModel()
+
+		t.Repeat(map[string]func(*rapid.T){
+			"create": func(t *rapid.T) {
+				sv := anySecureValueGen.Draw(t, "sv")
+				modelCreatedSv, modelErr := model.create(sut.Clock.Now(), deepCopy(sv))
+				createdSv, err := sut.CreateSv(t.Context(), testutils.CreateSvWithSv(deepCopy(sv)))
+				if err != nil || modelErr != nil {
+					require.ErrorIs(t, err, modelErr)
+					return
+				}
+				require.Equal(t, modelCreatedSv.Namespace, createdSv.Namespace)
+				require.Equal(t, modelCreatedSv.Name, createdSv.Name)
+				require.Equal(t, modelCreatedSv.Status.Version, createdSv.Status.Version)
+			},
+			"delete": func(t *rapid.T) {
+				ns := namespaceGen.Draw(t, "ns")
+				name := nameGen.Draw(t, "name")
+				modelSv, modelErr := model.delete(ns, name)
+				sv, err := sut.DeleteSv(t.Context(), ns, name)
+				if err != nil || modelErr != nil {
+					require.ErrorIs(t, err, modelErr)
+					return
+				}
+				require.Equal(t, modelSv.Namespace, sv.Namespace)
+				require.Equal(t, modelSv.Name, sv.Name)
+				require.Equal(t, modelSv.Status.Version, sv.Status.Version)
+			},
+			"lease": func(t *rapid.T) {
+				// Taken from secureValueMetadataStorage.acquireLeases
+				minAge := 300 * time.Second
+				leaseTTL := 30 * time.Second
+				maxBatchSize := rapid.Uint16Range(1, 10).Draw(t, "maxBatchSize")
+				modelSvs, modelErr := model.leaseInactiveSecureValues(sut.Clock.Now(), minAge, leaseTTL, maxBatchSize)
+				svs, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), maxBatchSize)
+				require.ErrorIs(t, err, modelErr)
+				require.Equal(t, len(modelSvs), len(svs))
+			},
+			"advanceTime": func(t *rapid.T) {
+				duration := time.Duration(rapid.IntRange(1, 10).Draw(t, "minutes")) * time.Minute
+				sut.Clock.AdvanceBy(duration)
+			},
+		})
+	})
+}

pkg/storage/secret/metadata/secure_value_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"slices"
 	"testing"
+	"time"
 
 	"github.com/mitchellh/copystructure"
 	"github.com/stretchr/testify/require"
@@ -20,7 +21,9 @@ import (
 
 type modelSecureValue struct {
 	*secretv1beta1.SecureValue
-	active bool
+	active       bool
+	created      time.Time
+	leaseCreated time.Time
 }
 
 // A simplified model of the grafana secrets manager
@@ -69,8 +72,8 @@ func (m *model) readActiveVersion(namespace, name string) *modelSecureValue {
 	return nil
 }
 
-func (m *model) create(sv *secretv1beta1.SecureValue, actorUID string) (*secretv1beta1.SecureValue, error) {
+func (m *model) create(now time.Time, sv *secretv1beta1.SecureValue) (*secretv1beta1.SecureValue, error) {
-	modelSv := &modelSecureValue{sv, false}
+	modelSv := &modelSecureValue{SecureValue: sv, active: false, created: now}
 	modelSv.Status.Version = m.getNewVersionNumber(modelSv.Namespace, modelSv.Name)
 	modelSv.Status.ExternalID = fmt.Sprintf("%d", modelSv.Status.Version)
 	m.secureValues = append(m.secureValues, modelSv)
@@ -78,7 +81,7 @@ func (m *model) create(sv *secretv1beta1.SecureValue, actorUID string) (*secretv
 	return modelSv.SecureValue, nil
 }
 
-func (m *model) update(newSecureValue *secretv1beta1.SecureValue, actorUID string) (*secretv1beta1.SecureValue, bool, error) {
+func (m *model) update(now time.Time, newSecureValue *secretv1beta1.SecureValue) (*secretv1beta1.SecureValue, bool, error) {
 	// If the payload doesn't contain a value, get the value from current version
 	if newSecureValue.Spec.Value == nil {
 		sv := m.readActiveVersion(newSecureValue.Namespace, newSecureValue.Name)
@@ -87,7 +90,7 @@ func (m *model) update(newSecureValue *secretv1beta1.SecureValue, actorUID strin
 		}
 		newSecureValue.Spec.Value = sv.Spec.Value
 	}
-	createdSv, err := m.create(newSecureValue, actorUID)
+	createdSv, err := m.create(now, newSecureValue)
 	return createdSv, true, err
 }
 
@@ -141,6 +144,22 @@ func (m *model) read(namespace, name string) (*secretv1beta1.SecureValue, error)
 	return modelSv.SecureValue, nil
 }
 
+func (m *model) leaseInactiveSecureValues(now time.Time, minAge, leaseTTL time.Duration, maxBatchSize uint16) ([]*modelSecureValue, error) {
+	out := make([]*modelSecureValue, 0)
+
+	for _, sv := range m.secureValues {
+		if len(out) >= int(maxBatchSize) {
+			break
+		}
+		if !sv.active && now.Sub(sv.created) > minAge && now.Sub(sv.leaseCreated) > leaseTTL {
+			sv.leaseCreated = now
+			out = append(out, sv)
+		}
+	}
+
+	return out, nil
+}
+
 var (
 	decryptersGen     = rapid.SampledFrom([]string{"svc1", "svc2", "svc3", "svc4", "svc5"})
 	nameGen           = rapid.SampledFrom([]string{"n1", "n2", "n3", "n4", "n5"})
@@ -204,16 +223,17 @@ func TestModel(t *testing.T) {
 		t.Parallel()
 
 		m := newModel()
+		now := time.Now()
 
 		// Create a secure value
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 		require.Equal(t, sv.Namespace, sv1.Namespace)
 		require.Equal(t, sv.Name, sv1.Name)
 		require.EqualValues(t, 1, sv1.Status.Version)
 
 		// Create a new version of a secure value
-		sv2, err := m.create(deepCopy(sv), "actor-uid")
+		sv2, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 		require.Equal(t, sv.Namespace, sv2.Namespace)
 		require.Equal(t, sv.Name, sv2.Name)
@@ -225,11 +245,13 @@ func TestModel(t *testing.T) {
 
 		m := newModel()
 
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		now := time.Now()
+
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 
 		// Create a new version of a secure value by updating it
-		sv2, _, err := m.update(deepCopy(sv1), "actor-uid")
+		sv2, _, err := m.update(now, deepCopy(sv1))
 		require.NoError(t, err)
 		require.Equal(t, sv.Namespace, sv2.Namespace)
 		require.Equal(t, sv.Name, sv2.Name)
@@ -239,14 +261,14 @@ func TestModel(t *testing.T) {
 		sv3 := deepCopy(sv2)
 		sv3.Name = "i_dont_exist"
 		sv3.Spec.Value = nil
-		_, _, err = m.update(sv3, "actor-uid")
+		_, _, err = m.update(now, sv3)
 		require.ErrorIs(t, err, contracts.ErrSecureValueNotFound)
 
 		// Updating a value that doesn't exist creates a new version
 		sv4 := deepCopy(sv3)
 		sv4.Name = "i_dont_exist"
 		sv4.Spec.Value = ptr.To(secretv1beta1.NewExposedSecureValue("sv4"))
-		sv4, _, err = m.update(sv4, "actor-uid")
+		sv4, _, err = m.update(now, sv4)
 		require.NoError(t, err)
 		require.EqualValues(t, 1, sv4.Status.Version)
 	})
@@ -255,8 +277,9 @@ func TestModel(t *testing.T) {
 		t.Parallel()
 
 		m := newModel()
+		now := time.Now()
 
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 
 		// Deleting a secure value
@@ -275,6 +298,7 @@ func TestModel(t *testing.T) {
 		t.Parallel()
 
 		m := newModel()
+		now := time.Now()
 
 		// No secure values exist yet
 		list, err := m.list(sv.Namespace)
@@ -282,7 +306,7 @@ func TestModel(t *testing.T) {
 		require.Equal(t, 0, len(list.Items))
 
 		// Create a secure value
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 
 		// 1 secure value exists and it should be returned
@@ -298,6 +322,7 @@ func TestModel(t *testing.T) {
 		t.Parallel()
 
 		m := newModel()
+		now := time.Now()
 
 		// Decrypting a secure value that does not exist
 		result, err := m.decrypt("decrypter", "namespace", "name")
@@ -308,7 +333,7 @@ func TestModel(t *testing.T) {
 
 		// Create a secure value
 		secret := "v1"
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 
 		// Decrypt the just created secure value
@@ -333,7 +358,7 @@ func TestStateMachine(t *testing.T) {
 			"create": func(t *rapid.T) {
 				sv := anySecureValueGen.Draw(t, "sv")
 
-				modelCreatedSv, modelErr := model.create(deepCopy(sv), "actor-uid")
+				modelCreatedSv, modelErr := model.create(sut.Clock.Now(), deepCopy(sv))
 
 				createdSv, err := sut.CreateSv(t.Context(), testutils.CreateSvWithSv(deepCopy(sv)))
 				if err != nil || modelErr != nil {
@@ -346,7 +371,7 @@ func TestStateMachine(t *testing.T) {
 			},
 			"update": func(t *rapid.T) {
 				sv := updateSecureValueGen.Draw(t, "sv")
-				modelCreatedSv, _, modelErr := model.update(deepCopy(sv), "actor-uid")
+				modelCreatedSv, _, modelErr := model.update(sut.Clock.Now(), deepCopy(sv))
 				createdSv, err := sut.UpdateSv(t.Context(), deepCopy(sv))
 				if err != nil || modelErr != nil {
 					require.ErrorIs(t, err, modelErr)

pkg/storage/secret/metadata/testdata/mysql--secure_value_delete-deleteSecureValue.sql

@@ -0,0 +1,7 @@
+DELETE FROM 
+  `secret_secure_value`
+WHERE 
+  `namespace` = 'ns' AND
+  `name` = 'name' AND
+  `version` = 1
+;

pkg/storage/secret/metadata/testdata/mysql--secure_value_lease_inactive-lease inactive.sql

@@ -0,0 +1,14 @@
+UPDATE
+  `secret_secure_value`
+SET
+  `lease_token` = 'token',
+  `lease_created` = 10
+WHERE 
+  `guid` IN (SELECT `guid` 
+                          FROM `secret_secure_value` 
+                          WHERE 
+                            `active` = FALSE AND
+                            10 - `created` > 300 AND
+                            10 - `lease_created` > 30
+                          LIMIT 10)
+;

pkg/storage/secret/metadata/testdata/mysql--secure_value_list_by_lease_token-list by lease token.sql

@@ -0,0 +1,26 @@
+SELECT
+  `guid`,
+  `name`,
+  `namespace`,
+  `annotations`,
+  `labels`,
+  `created`,
+  `created_by`,
+  `updated`,
+  `updated_by`,
+  `description`,
+  `keeper`,
+  `decrypters`,
+  `ref`,
+  `external_id`,
+  `version`,
+  `active`,
+  `owner_reference_api_group`,
+  `owner_reference_api_version`,
+  `owner_reference_kind`,
+  `owner_reference_name`
+FROM
+  `secret_secure_value`
+WHERE 
+  `lease_token` = 'token'
+;

pkg/storage/secret/metadata/testdata/postgres--secure_value_delete-deleteSecureValue.sql

@@ -0,0 +1,7 @@
+DELETE FROM 
+  "secret_secure_value"
+WHERE 
+  "namespace" = 'ns' AND
+  "name" = 'name' AND
+  "version" = 1
+;

pkg/storage/secret/metadata/testdata/postgres--secure_value_lease_inactive-lease inactive.sql

@@ -0,0 +1,14 @@
+UPDATE
+  "secret_secure_value"
+SET
+  "lease_token" = 'token',
+  "lease_created" = 10
+WHERE 
+  "guid" IN (SELECT "guid" 
+                          FROM "secret_secure_value" 
+                          WHERE 
+                            "active" = FALSE AND
+                            10 - "created" > 300 AND
+                            10 - "lease_created" > 30
+                          LIMIT 10)
+;

pkg/storage/secret/metadata/testdata/postgres--secure_value_list_by_lease_token-list by lease token.sql

@@ -0,0 +1,26 @@
+SELECT
+  "guid",
+  "name",
+  "namespace",
+  "annotations",
+  "labels",
+  "created",
+  "created_by",
+  "updated",
+  "updated_by",
+  "description",
+  "keeper",
+  "decrypters",
+  "ref",
+  "external_id",
+  "version",
+  "active",
+  "owner_reference_api_group",
+  "owner_reference_api_version",
+  "owner_reference_kind",
+  "owner_reference_name"
+FROM
+  "secret_secure_value"
+WHERE 
+  "lease_token" = 'token'
+;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_delete-deleteSecureValue.sql

@@ -0,0 +1,7 @@
+DELETE FROM 
+  "secret_secure_value"
+WHERE 
+  "namespace" = 'ns' AND
+  "name" = 'name' AND
+  "version" = 1
+;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_lease_inactive-lease inactive.sql

@@ -0,0 +1,14 @@
+UPDATE
+  "secret_secure_value"
+SET
+  "lease_token" = 'token',
+  "lease_created" = 10
+WHERE 
+  "guid" IN (SELECT "guid" 
+                          FROM "secret_secure_value" 
+                          WHERE 
+                            "active" = FALSE AND
+                            10 - "created" > 300 AND
+                            10 - "lease_created" > 30
+                          LIMIT 10)
+;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_list_by_lease_token-list by lease token.sql

@@ -0,0 +1,26 @@
+SELECT
+  "guid",
+  "name",
+  "namespace",
+  "annotations",
+  "labels",
+  "created",
+  "created_by",
+  "updated",
+  "updated_by",
+  "description",
+  "keeper",
+  "decrypters",
+  "ref",
+  "external_id",
+  "version",
+  "active",
+  "owner_reference_api_group",
+  "owner_reference_api_version",
+  "owner_reference_kind",
+  "owner_reference_name"
+FROM
+  "secret_secure_value"
+WHERE 
+  "lease_token" = 'token'
+;

pkg/storage/secret/migrator/migrator.go

@@ -181,4 +181,23 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {
 		Length:   253, // Limit enforced by K8s.
 		Nullable: true,
 	}))
+
+	mg.AddMigration("add lease_token column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "lease_token",
+		Type:     migrator.DB_NVarchar,
+		Length:   36,
+		Nullable: true,
+	}))
+	mg.AddMigration("add lease_token index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
+		Cols: []string{"lease_token"},
+	}))
+	mg.AddMigration("add lease_created column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "lease_created",
+		Type:     migrator.DB_BigInt,
+		Nullable: false,
+		Default:  "0",
+	}))
+	mg.AddMigration("add lease_created index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
+		Cols: []string{"lease_created"},
+	}))
 }