From b76711205b67c89ea3b61be27638b358401d0f36 Mon Sep 17 00:00:00 2001 From: Matheus Macabu Date: Mon, 28 Apr 2025 21:45:46 +0200 Subject: [PATCH] Outbox: Migrate storage to sql templates --- .../data/secure_value_outbox_append.sql | 31 +++ .../data/secure_value_outbox_delete.sql | 5 + .../data/secure_value_outbox_receiveN.sql | 17 ++ pkg/storage/secret/metadata/outbox_store.go | 248 ++++++++++++------ .../secret/metadata/outbox_store_test.go | 5 +- pkg/storage/secret/metadata/query.go | 28 ++ pkg/storage/secret/metadata/query_test.go | 92 +++++++ ...value_outbox_append-all-fields-present.sql | 19 ++ ...alue_outbox_append-no-encrypted-secret.sql | 17 ++ ...ure_value_outbox_append-no-external-id.sql | 17 ++ ...ure_value_outbox_append-no-keeper-name.sql | 17 ++ ...ysql--secure_value_outbox_delete-basic.sql | 5 + ...ql--secure_value_outbox_receiveN-basic.sql | 17 ++ ...value_outbox_append-all-fields-present.sql | 19 ++ ...alue_outbox_append-no-encrypted-secret.sql | 17 ++ ...ure_value_outbox_append-no-external-id.sql | 17 ++ ...ure_value_outbox_append-no-keeper-name.sql | 17 ++ ...gres--secure_value_outbox_delete-basic.sql | 5 + ...es--secure_value_outbox_receiveN-basic.sql | 17 ++ ...value_outbox_append-all-fields-present.sql | 19 ++ ...alue_outbox_append-no-encrypted-secret.sql | 17 ++ ...ure_value_outbox_append-no-external-id.sql | 17 ++ ...ure_value_outbox_append-no-keeper-name.sql | 17 ++ ...lite--secure_value_outbox_delete-basic.sql | 5 + ...te--secure_value_outbox_receiveN-basic.sql | 16 ++ pkg/storage/secret/migrator/migrator.go | 2 +- 26 files changed, 620 insertions(+), 83 deletions(-) create mode 100644 pkg/storage/secret/metadata/data/secure_value_outbox_append.sql create mode 100644 pkg/storage/secret/metadata/data/secure_value_outbox_delete.sql create mode 100644 pkg/storage/secret/metadata/data/secure_value_outbox_receiveN.sql create mode 100755 pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-all-fields-present.sql create mode 100755 pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-encrypted-secret.sql create mode 100755 pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-external-id.sql create mode 100755 pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-keeper-name.sql create mode 100755 pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_delete-basic.sql create mode 100755 pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_receiveN-basic.sql create mode 100755 pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-all-fields-present.sql create mode 100755 pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-encrypted-secret.sql create mode 100755 pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-external-id.sql create mode 100755 pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-keeper-name.sql create mode 100755 pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_delete-basic.sql create mode 100755 pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_receiveN-basic.sql create mode 100755 pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-all-fields-present.sql create mode 100755 pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-encrypted-secret.sql create mode 100755 pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-external-id.sql create mode 100755 pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-keeper-name.sql create mode 100755 pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_delete-basic.sql create mode 100755 pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_receiveN-basic.sql diff --git a/pkg/storage/secret/metadata/data/secure_value_outbox_append.sql b/pkg/storage/secret/metadata/data/secure_value_outbox_append.sql new file mode 100644 index 00000000000..dd92180912a --- /dev/null +++ b/pkg/storage/secret/metadata/data/secure_value_outbox_append.sql @@ -0,0 +1,31 @@ +INSERT INTO {{ .Ident "secret_secure_value_outbox" }} ( + {{ .Ident "uid" }}, + {{ .Ident "message_type" }}, + {{ .Ident "name" }}, + {{ .Ident "namespace" }}, +{{ if .Row.EncryptedSecret.Valid }} + {{ .Ident "encrypted_secret" }}, +{{ end }} +{{ if .Row.KeeperName.Valid }} + {{ .Ident "keeper_name" }}, +{{ end }} +{{ if .Row.ExternalID.Valid }} + {{ .Ident "external_id" }}, +{{ end }} + {{ .Ident "created" }} +) VALUES ( + {{ .Arg .Row.MessageID }}, + {{ .Arg .Row.MessageType }}, + {{ .Arg .Row.Name }}, + {{ .Arg .Row.Namespace }}, +{{ if .Row.EncryptedSecret.Valid }} + {{ .Arg .Row.EncryptedSecret.String }}, +{{ end }} +{{ if .Row.KeeperName.Valid }} + {{ .Arg .Row.KeeperName.String }}, +{{ end }} +{{ if .Row.ExternalID.Valid }} + {{ .Arg .Row.ExternalID.String }}, +{{ end }} + {{ .Arg .Row.Created }} +); diff --git a/pkg/storage/secret/metadata/data/secure_value_outbox_delete.sql b/pkg/storage/secret/metadata/data/secure_value_outbox_delete.sql new file mode 100644 index 00000000000..fa3a0c2580a --- /dev/null +++ b/pkg/storage/secret/metadata/data/secure_value_outbox_delete.sql @@ -0,0 +1,5 @@ +DELETE FROM + {{ .Ident "secret_secure_value_outbox" }} +WHERE + {{ .Ident "uid" }} = {{ .Arg .MessageID }} +; diff --git a/pkg/storage/secret/metadata/data/secure_value_outbox_receiveN.sql b/pkg/storage/secret/metadata/data/secure_value_outbox_receiveN.sql new file mode 100644 index 00000000000..851e830baab --- /dev/null +++ b/pkg/storage/secret/metadata/data/secure_value_outbox_receiveN.sql @@ -0,0 +1,17 @@ +SELECT + {{ .Ident "uid" }}, + {{ .Ident "message_type" }}, + {{ .Ident "name" }}, + {{ .Ident "namespace" }}, + {{ .Ident "encrypted_secret" }}, + {{ .Ident "keeper_name" }}, + {{ .Ident "external_id" }}, + {{ .Ident "created" }} +FROM + {{ .Ident "secret_secure_value_outbox" }} +ORDER BY + {{ .Ident "created" }} ASC +LIMIT + {{ .Arg .ReceiveLimit }} +{{ .SelectFor "UPDATE SKIP LOCKED" }} +; diff --git a/pkg/storage/secret/metadata/outbox_store.go b/pkg/storage/secret/metadata/outbox_store.go index ab7e92a2b80..988bd309490 100644 --- a/pkg/storage/secret/metadata/outbox_store.go +++ b/pkg/storage/secret/metadata/outbox_store.go @@ -2,112 +2,200 @@ package metadata import ( "context" + "database/sql" "fmt" "time" "github.com/google/uuid" "github.com/grafana/grafana/pkg/apis/secret/v0alpha1" - "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/registry/apis/secret/assert" "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" - "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/storage/secret/migrator" + "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate" ) -// Implements contracts.OutboxQueue type outboxStore struct { - db db.DB + db contracts.Database + dialect sqltemplate.Dialect } -func ProvideOutboxQueue(db db.DB) contracts.OutboxQueue { - return &outboxStore{db: db} +func ProvideOutboxQueue(db contracts.Database) contracts.OutboxQueue { + return &outboxStore{ + db: db, + dialect: sqltemplate.DialectForDriver(db.DriverName()), + } } type outboxMessageDB struct { - MessageID string `xorm:"pk 'uid'"` - MessageType contracts.OutboxMessageType `xorm:"message_type"` - Name string `xorm:"name"` - Namespace string `xorm:"namespace"` - EncryptedSecret string `xorm:"encrypted_secret"` - KeeperName *string `xorm:"keeper_name"` - ExternalID *string `xorm:"external_id"` - Created int64 `xorm:"created"` -} - -func (*outboxMessageDB) TableName() string { - return migrator.TableNameSecureValueOutbox + MessageID string + MessageType contracts.OutboxMessageType + Name string + Namespace string + EncryptedSecret sql.NullString + KeeperName sql.NullString + ExternalID sql.NullString + Created int64 } func (s *outboxStore) Append(ctx context.Context, input contracts.AppendOutboxMessage) (string, error) { assert.True(input.Type != "", "outboxStore.Append: outbox message type is required") - var messageID string - err := s.db.InTransaction(ctx, func(ctx context.Context) error { - return s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { - outboxDB := outboxMessageDB{ - MessageID: uuid.New().String(), - MessageType: input.Type, - Name: input.Name, - Namespace: input.Namespace, - KeeperName: input.KeeperName, - ExternalID: input.ExternalID, - Created: time.Now().UTC().UnixMilli(), - } - if input.Type == contracts.CreateSecretOutboxMessage || input.Type == contracts.UpdateSecretOutboxMessage { - outboxDB.EncryptedSecret = input.EncryptedSecret.DangerouslyExposeAndConsumeValue() - } - _, err := sess.Table(migrator.TableNameSecureValueOutbox).Insert(outboxDB) - if err != nil { - return fmt.Errorf("inserting message into secure value outbox table: %+w", err) - } + keeperName := sql.NullString{} + if input.KeeperName != nil { + keeperName = sql.NullString{ + Valid: true, + String: *input.KeeperName, + } + } - messageID = outboxDB.MessageID - return nil - }) - }) - return messageID, err + externalID := sql.NullString{} + if input.ExternalID != nil { + externalID = sql.NullString{ + Valid: true, + String: *input.ExternalID, + } + } + + encryptedSecret := sql.NullString{} + if input.Type == contracts.CreateSecretOutboxMessage || input.Type == contracts.UpdateSecretOutboxMessage { + encryptedSecret = sql.NullString{ + Valid: true, + // TODO: this type does not need to be exposed when encrypted (maybe []byte or string) + String: input.EncryptedSecret.DangerouslyExposeAndConsumeValue(), + } + } + + messageID := uuid.New().String() + + req := appendSecureValueOutbox{ + SQLTemplate: sqltemplate.New(s.dialect), + Row: &outboxMessageDB{ + MessageID: messageID, + MessageType: input.Type, + Name: input.Name, + Namespace: input.Namespace, + EncryptedSecret: encryptedSecret, + KeeperName: keeperName, + ExternalID: externalID, + Created: time.Now().UTC().UnixMilli(), + }, + } + + query, err := sqltemplate.Execute(sqlSecureValueOutboxAppend, req) + if err != nil { + return "", fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxAppend.Name(), err) + } + + result, err := s.db.ExecContext(ctx, query, req.GetArgs()...) + if err != nil { + return "", fmt.Errorf("inserting message into secure value outbox table: %w", err) + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return "", fmt.Errorf("get rows affected: %w", err) + } + + if rowsAffected != 1 { + return "", fmt.Errorf("expected to affect 1 row, but affected %d", rowsAffected) + } + + return messageID, nil } func (s *outboxStore) ReceiveN(ctx context.Context, n uint) ([]contracts.OutboxMessage, error) { + req := receiveNSecureValueOutbox{ + SQLTemplate: sqltemplate.New(s.dialect), + ReceiveLimit: n, + } + + query, err := sqltemplate.Execute(sqlSecureValueOutboxReceiveN, req) + if err != nil { + return nil, fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxReceiveN.Name(), err) + } + + rows, err := s.db.QueryContext(ctx, query, req.GetArgs()...) + if err != nil { + return nil, fmt.Errorf("fetching rows from secure value outbox table: %w", err) + } + defer func() { _ = rows.Close() }() + messages := make([]contracts.OutboxMessage, 0) - err := s.db.InTransaction(ctx, func(ctx context.Context) error { - return s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { - rows := make([]outboxMessageDB, 0) - // TODO: skip locked rows - if err := sess.Table(migrator.TableNameSecureValueOutbox).ForUpdate().OrderBy("secret_secure_value_outbox.created ASC").Limit(int(n)).Find(&rows); err != nil { - return fmt.Errorf("fetching rows from secure value outbox table: %w", err) - } - for _, row := range rows { - msg := contracts.OutboxMessage{ - Type: row.MessageType, - MessageID: row.MessageID, - Name: row.Name, - Namespace: row.Namespace, - KeeperName: row.KeeperName, - ExternalID: row.ExternalID, - } - if row.MessageType != contracts.DeleteSecretOutboxMessage { - msg.EncryptedSecret = v0alpha1.ExposedSecureValue(row.EncryptedSecret) - } - messages = append(messages, msg) - } - return nil - }) - }) - return messages, err + + for rows.Next() { + var row outboxMessageDB + if err := rows.Scan( + &row.MessageID, + &row.MessageType, + &row.Name, + &row.Namespace, + &row.EncryptedSecret, + &row.KeeperName, + &row.ExternalID, + &row.Created, + ); err != nil { + return nil, fmt.Errorf("scanning row from secure value outbox table: %w", err) + } + + var keeperName *string + if row.KeeperName.Valid { + keeperName = &row.KeeperName.String + } + + var externalID *string + if row.ExternalID.Valid { + externalID = &row.ExternalID.String + } + + msg := contracts.OutboxMessage{ + Type: row.MessageType, + MessageID: row.MessageID, + Name: row.Name, + Namespace: row.Namespace, + KeeperName: keeperName, + ExternalID: externalID, + } + + if row.MessageType != contracts.DeleteSecretOutboxMessage && row.EncryptedSecret.Valid { + // TODO: dont do this because it is encrypted! + msg.EncryptedSecret = v0alpha1.ExposedSecureValue(row.EncryptedSecret.String) + } + + messages = append(messages, msg) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating over rows: %w", err) + } + + return messages, nil } func (s *outboxStore) Delete(ctx context.Context, messageID string) error { - return s.db.InTransaction(ctx, func(ctx context.Context) error { - return s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { - deleted, err := sess.Delete(&outboxMessageDB{MessageID: messageID}) - if err != nil { - return fmt.Errorf("deleting message from outbox table: messageID=%+v %w", messageID, err) - } - if deleted > 1 { - return fmt.Errorf("bug: deleted more than one row from the outbox table, should delete only one at a time: deleted=%+v", deleted) - } - return nil - }) - }) + assert.True(messageID != "", "outboxStore.Delete: messageID is required") + + req := deleteSecureValueOutbox{ + SQLTemplate: sqltemplate.New(s.dialect), + MessageID: messageID, + } + + query, err := sqltemplate.Execute(sqlSecureValueOutboxDelete, req) + if err != nil { + return fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxDelete.Name(), err) + } + + result, err := s.db.ExecContext(ctx, query, req.GetArgs()...) + if err != nil { + return fmt.Errorf("deleting message id=%v from secure value outbox table: %w", messageID, err) + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("get rows affected: %w", err) + } + + if rowsAffected != 1 { + return fmt.Errorf("bug: deleted more than one row from the outbox table, should delete only one at a time: deleted=%v", rowsAffected) + } + + return nil } diff --git a/pkg/storage/secret/metadata/outbox_store_test.go b/pkg/storage/secret/metadata/outbox_store_test.go index b62f6220b81..a2b39bbde2a 100644 --- a/pkg/storage/secret/metadata/outbox_store_test.go +++ b/pkg/storage/secret/metadata/outbox_store_test.go @@ -11,6 +11,7 @@ import ( secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/storage/secret/database" "github.com/grafana/grafana/pkg/storage/secret/migrator" "github.com/stretchr/testify/require" ) @@ -109,7 +110,7 @@ func TestOutboxStore(t *testing.T) { ctx := context.Background() - outbox := ProvideOutboxQueue(testDB) + outbox := ProvideOutboxQueue(database.ProvideDatabase(testDB)) m1 := contracts.AppendOutboxMessage{ Type: contracts.CreateSecretOutboxMessage, @@ -178,7 +179,7 @@ func TestOutboxStoreProperty(t *testing.T) { testDB := sqlstore.NewTestStore(t) require.NoError(t, migrator.MigrateSecretSQL(testDB.GetEngine(), nil)) - outbox := ProvideOutboxQueue(testDB) + outbox := ProvideOutboxQueue(database.ProvideDatabase(testDB)) model := newOutboxStoreModel() diff --git a/pkg/storage/secret/metadata/query.go b/pkg/storage/secret/metadata/query.go index f5e2a277e41..3aa5e0b2dfa 100644 --- a/pkg/storage/secret/metadata/query.go +++ b/pkg/storage/secret/metadata/query.go @@ -39,6 +39,10 @@ var ( // sqlSecureValueUpdate = mustTemplate("secure_value_update.sql") sqlSecureValueUpdateExternalId = mustTemplate("secure_value_updateExternalId.sql") sqlSecureValueUpdateStatus = mustTemplate("secure_value_updateStatus.sql") + + sqlSecureValueOutboxAppend = mustTemplate("secure_value_outbox_append.sql") + sqlSecureValueOutboxReceiveN = mustTemplate("secure_value_outbox_receiveN.sql") + sqlSecureValueOutboxDelete = mustTemplate("secure_value_outbox_delete.sql") ) func mustTemplate(filename string) *template.Template { @@ -222,3 +226,27 @@ type updateStatusSecureValue struct { func (r updateStatusSecureValue) Validate() error { return nil // TODO } + +/*************************************/ +/**-- Secure Value Outbox Queries --**/ +/*************************************/ +type appendSecureValueOutbox struct { + sqltemplate.SQLTemplate + Row *outboxMessageDB +} + +func (appendSecureValueOutbox) Validate() error { return nil } + +type receiveNSecureValueOutbox struct { + sqltemplate.SQLTemplate + ReceiveLimit uint +} + +func (receiveNSecureValueOutbox) Validate() error { return nil } + +type deleteSecureValueOutbox struct { + sqltemplate.SQLTemplate + MessageID string +} + +func (deleteSecureValueOutbox) Validate() error { return nil } diff --git a/pkg/storage/secret/metadata/query_test.go b/pkg/storage/secret/metadata/query_test.go index 0d9a3821adc..89bde26a1bc 100644 --- a/pkg/storage/secret/metadata/query_test.go +++ b/pkg/storage/secret/metadata/query_test.go @@ -1,6 +1,7 @@ package metadata import ( + "database/sql" "testing" "text/template" @@ -233,3 +234,94 @@ func TestSecureValueQueries(t *testing.T) { }, }) } + +func TestSecureValueOutboxQueries(t *testing.T) { + mocks.CheckQuerySnapshots(t, mocks.TemplateTestSetup{ + RootDir: "testdata", + Templates: map[*template.Template][]mocks.TemplateTestCase{ + sqlSecureValueOutboxAppend: { + { + Name: "no-encrypted-secret", + Data: &appendSecureValueOutbox{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Row: &outboxMessageDB{ + MessageID: "my-uuid", + MessageType: "some-type", + Name: "name", + Namespace: "namespace", + ExternalID: sql.NullString{Valid: true, String: "external-id"}, + KeeperName: sql.NullString{Valid: true, String: "keeper"}, + Created: 1234, + }, + }, + }, + { + Name: "no-external-id", + Data: &appendSecureValueOutbox{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Row: &outboxMessageDB{ + MessageID: "my-uuid", + MessageType: "some-type", + Name: "name", + Namespace: "namespace", + EncryptedSecret: sql.NullString{Valid: true, String: "encrypted"}, + KeeperName: sql.NullString{Valid: true, String: "keeper"}, + Created: 1234, + }, + }, + }, + { + Name: "no-keeper-name", + Data: &appendSecureValueOutbox{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Row: &outboxMessageDB{ + MessageID: "my-uuid", + MessageType: "some-type", + Name: "name", + Namespace: "namespace", + EncryptedSecret: sql.NullString{Valid: true, String: "encrypted"}, + ExternalID: sql.NullString{Valid: true, String: "external-id"}, + Created: 1234, + }, + }, + }, + { + Name: "all-fields-present", + Data: &appendSecureValueOutbox{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Row: &outboxMessageDB{ + MessageID: "my-uuid", + MessageType: "some-type", + Name: "name", + Namespace: "namespace", + EncryptedSecret: sql.NullString{Valid: true, String: "encrypted"}, + ExternalID: sql.NullString{Valid: true, String: ""}, // can be empty string + KeeperName: sql.NullString{Valid: true, String: "keeper"}, + Created: 1234, + }, + }, + }, + }, + + sqlSecureValueOutboxReceiveN: { + { + Name: "basic", + Data: &receiveNSecureValueOutbox{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + ReceiveLimit: 10, + }, + }, + }, + + sqlSecureValueOutboxDelete: { + { + Name: "basic", + Data: &deleteSecureValueOutbox{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + MessageID: "my-uuid", + }, + }, + }, + }, + }) +} diff --git a/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-all-fields-present.sql b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-all-fields-present.sql new file mode 100755 index 00000000000..82cf437e05a --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-all-fields-present.sql @@ -0,0 +1,19 @@ +INSERT INTO `secret_secure_value_outbox` ( + `uid`, + `message_type`, + `name`, + `namespace`, + `encrypted_secret`, + `keeper_name`, + `external_id`, + `created` +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'keeper', + '', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-encrypted-secret.sql b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-encrypted-secret.sql new file mode 100755 index 00000000000..9b60a534508 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-encrypted-secret.sql @@ -0,0 +1,17 @@ +INSERT INTO `secret_secure_value_outbox` ( + `uid`, + `message_type`, + `name`, + `namespace`, + `keeper_name`, + `external_id`, + `created` +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'keeper', + 'external-id', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-external-id.sql b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-external-id.sql new file mode 100755 index 00000000000..063ef99472b --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-external-id.sql @@ -0,0 +1,17 @@ +INSERT INTO `secret_secure_value_outbox` ( + `uid`, + `message_type`, + `name`, + `namespace`, + `encrypted_secret`, + `keeper_name`, + `created` +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'keeper', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-keeper-name.sql b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-keeper-name.sql new file mode 100755 index 00000000000..8541f190bce --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-keeper-name.sql @@ -0,0 +1,17 @@ +INSERT INTO `secret_secure_value_outbox` ( + `uid`, + `message_type`, + `name`, + `namespace`, + `encrypted_secret`, + `external_id`, + `created` +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'external-id', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_delete-basic.sql b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_delete-basic.sql new file mode 100755 index 00000000000..f4f1acaa6f7 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_delete-basic.sql @@ -0,0 +1,5 @@ +DELETE FROM + `secret_secure_value_outbox` +WHERE + `uid` = 'my-uuid' +; diff --git a/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_receiveN-basic.sql b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_receiveN-basic.sql new file mode 100755 index 00000000000..09b1458a11f --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_receiveN-basic.sql @@ -0,0 +1,17 @@ +SELECT + `uid`, + `message_type`, + `name`, + `namespace`, + `encrypted_secret`, + `keeper_name`, + `external_id`, + `created` +FROM + `secret_secure_value_outbox` +ORDER BY + `created` ASC +LIMIT + 10 +FOR UPDATE SKIP LOCKED +; diff --git a/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-all-fields-present.sql b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-all-fields-present.sql new file mode 100755 index 00000000000..50e8ca41f6f --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-all-fields-present.sql @@ -0,0 +1,19 @@ +INSERT INTO "secret_secure_value_outbox" ( + "uid", + "message_type", + "name", + "namespace", + "encrypted_secret", + "keeper_name", + "external_id", + "created" +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'keeper', + '', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-encrypted-secret.sql b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-encrypted-secret.sql new file mode 100755 index 00000000000..73574027ca8 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-encrypted-secret.sql @@ -0,0 +1,17 @@ +INSERT INTO "secret_secure_value_outbox" ( + "uid", + "message_type", + "name", + "namespace", + "keeper_name", + "external_id", + "created" +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'keeper', + 'external-id', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-external-id.sql b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-external-id.sql new file mode 100755 index 00000000000..050a85b5740 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-external-id.sql @@ -0,0 +1,17 @@ +INSERT INTO "secret_secure_value_outbox" ( + "uid", + "message_type", + "name", + "namespace", + "encrypted_secret", + "keeper_name", + "created" +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'keeper', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-keeper-name.sql b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-keeper-name.sql new file mode 100755 index 00000000000..7bd8bccb624 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-keeper-name.sql @@ -0,0 +1,17 @@ +INSERT INTO "secret_secure_value_outbox" ( + "uid", + "message_type", + "name", + "namespace", + "encrypted_secret", + "external_id", + "created" +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'external-id', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_delete-basic.sql b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_delete-basic.sql new file mode 100755 index 00000000000..8b9b9f3cc52 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_delete-basic.sql @@ -0,0 +1,5 @@ +DELETE FROM + "secret_secure_value_outbox" +WHERE + "uid" = 'my-uuid' +; diff --git a/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_receiveN-basic.sql b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_receiveN-basic.sql new file mode 100755 index 00000000000..437b506652d --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_receiveN-basic.sql @@ -0,0 +1,17 @@ +SELECT + "uid", + "message_type", + "name", + "namespace", + "encrypted_secret", + "keeper_name", + "external_id", + "created" +FROM + "secret_secure_value_outbox" +ORDER BY + "created" ASC +LIMIT + 10 +FOR UPDATE SKIP LOCKED +; diff --git a/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-all-fields-present.sql b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-all-fields-present.sql new file mode 100755 index 00000000000..50e8ca41f6f --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-all-fields-present.sql @@ -0,0 +1,19 @@ +INSERT INTO "secret_secure_value_outbox" ( + "uid", + "message_type", + "name", + "namespace", + "encrypted_secret", + "keeper_name", + "external_id", + "created" +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'keeper', + '', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-encrypted-secret.sql b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-encrypted-secret.sql new file mode 100755 index 00000000000..73574027ca8 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-encrypted-secret.sql @@ -0,0 +1,17 @@ +INSERT INTO "secret_secure_value_outbox" ( + "uid", + "message_type", + "name", + "namespace", + "keeper_name", + "external_id", + "created" +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'keeper', + 'external-id', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-external-id.sql b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-external-id.sql new file mode 100755 index 00000000000..050a85b5740 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-external-id.sql @@ -0,0 +1,17 @@ +INSERT INTO "secret_secure_value_outbox" ( + "uid", + "message_type", + "name", + "namespace", + "encrypted_secret", + "keeper_name", + "created" +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'keeper', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-keeper-name.sql b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-keeper-name.sql new file mode 100755 index 00000000000..7bd8bccb624 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-keeper-name.sql @@ -0,0 +1,17 @@ +INSERT INTO "secret_secure_value_outbox" ( + "uid", + "message_type", + "name", + "namespace", + "encrypted_secret", + "external_id", + "created" +) VALUES ( + 'my-uuid', + 'some-type', + 'name', + 'namespace', + 'encrypted', + 'external-id', + 1234 +); diff --git a/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_delete-basic.sql b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_delete-basic.sql new file mode 100755 index 00000000000..8b9b9f3cc52 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_delete-basic.sql @@ -0,0 +1,5 @@ +DELETE FROM + "secret_secure_value_outbox" +WHERE + "uid" = 'my-uuid' +; diff --git a/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_receiveN-basic.sql b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_receiveN-basic.sql new file mode 100755 index 00000000000..f11cf559024 --- /dev/null +++ b/pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_receiveN-basic.sql @@ -0,0 +1,16 @@ +SELECT + "uid", + "message_type", + "name", + "namespace", + "encrypted_secret", + "keeper_name", + "external_id", + "created" +FROM + "secret_secure_value_outbox" +ORDER BY + "created" ASC +LIMIT + 10 +; diff --git a/pkg/storage/secret/migrator/migrator.go b/pkg/storage/secret/migrator/migrator.go index 5a07dbe192b..00c617456d2 100644 --- a/pkg/storage/secret/migrator/migrator.go +++ b/pkg/storage/secret/migrator/migrator.go @@ -124,7 +124,7 @@ func initSecretStore(mg *migrator.Migrator) string { {Name: "message_type", Type: migrator.DB_NVarchar, Length: 16, Nullable: false}, {Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. - {Name: "encrypted_secret", Type: migrator.DB_Blob, Nullable: false}, + {Name: "encrypted_secret", Type: migrator.DB_Blob, Nullable: true}, {Name: "keeper_name", Type: migrator.DB_NVarchar, Length: 253, Nullable: true}, // Keeper name, if not set, use default keeper. {Name: "external_id", Type: migrator.DB_NVarchar, Length: 36, Nullable: true}, // Fixed size of a UUID. {Name: "created", Type: migrator.DB_BigInt, Nullable: false},