Storage: Show history+trash using the list command (#99009)

Co-authored-by: Stephanie Hingtgen <stephanie.hingtgen@grafana.com>

pkg/apimachinery/utils/meta.go

@@ -17,6 +17,18 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 )
 
+// LabelKeyGetHistory is used to select object history for an given resource
+const LabelKeyGetHistory = "grafana.app/get-history"
+
+// LabelKeyGetTrash is used to list objects that have been (soft) deleted
+const LabelKeyGetTrash = "grafana.app/get-trash"
+
+// AnnoKeyKubectlLastAppliedConfig is the annotation kubectl writes with the entire previous config
+const AnnoKeyKubectlLastAppliedConfig = "kubectl.kubernetes.io/last-applied-configuration"
+
+// DeletedGeneration is set on Resources that have been (soft) deleted
+const DeletedGeneration = int64(-999)
+
 // Annotation keys
 
 const AnnoKeyCreatedBy = "grafana.app/createdBy"

pkg/registry/apis/dashboard/legacy/query_dashboards.sql

@@ -24,6 +24,9 @@ SELECT
     WHERE dashboard.is_folder = false
       AND dashboard.org_id = {{ .Arg .Query.OrgID }}
     {{ if .Query.UseHistoryTable }}
+      {{ if .Query.UID }}
+      AND dashboard.uid = {{ .Arg .Query.UID }}
+      {{ end }}
       {{ if .Query.Version }}
       AND dashboard_version.version = {{ .Arg .Query.Version }}
       {{ else if .Query.LastID }}

pkg/registry/apis/dashboard/legacy/sql_dashboards.go

@@ -98,8 +98,10 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, sql *legacysql.LegacyD
 		return nil, fmt.Errorf("execute template %q: %w", tmpl.Name(), err)
 	}
 	q := rawQuery
-	// q = sqltemplate.RemoveEmptyLines(rawQuery)
-	// fmt.Printf(">>%s [%+v]", q, req.GetArgs())
+	// if true {
+	// 	pretty := sqltemplate.RemoveEmptyLines(rawQuery)
+	// 	fmt.Printf("DASHBOARD QUERY: %s [%+v] // %+v\n", pretty, req.GetArgs(), query)
+	// }
 
 	rows, err := sql.DB.GetSqlxSession().Query(ctx, q, req.GetArgs()...)
 	if err != nil {
@@ -267,6 +269,7 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
 
 		if deleted.Valid {
 			meta.SetDeletionTimestamp(ptr.To(metav1.NewTime(deleted.Time)))
+			meta.SetGeneration(utils.DeletedGeneration)
 		}
 
 		if message.String != "" {

pkg/registry/apis/dashboard/legacy/storage.go

@@ -192,6 +192,16 @@ func (a *dashboardSqlAccess) ListIterator(ctx context.Context, req *resource.Lis
 		return 0, err
 	}
 
+	switch req.Source {
+	case resource.ListRequest_HISTORY:
+		query.GetHistory = true
+		query.UID = req.Options.Key.Name
+	case resource.ListRequest_TRASH:
+		query.GetTrash = true
+	case resource.ListRequest_STORE:
+		// normal
+	}
+
 	listRV, err := sql.GetResourceVersion(ctx, "dashboard", "updated")
 	if err != nil {
 		return 0, err
@@ -250,87 +260,6 @@ func (a *dashboardSqlAccess) Search(ctx context.Context, req *resource.ResourceS
 	return nil, fmt.Errorf("not yet (filter)")
 }
 
-/**
-func (a *dashboardSqlAccess) History(ctx context.Context, req *resource.HistoryRequest) (*resource.HistoryResponse, error) {
-	info, err := claims.ParseNamespace(req.Key.Namespace)
-	if err == nil {
-		err = isDashboardKey(req.Key, false)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	token, err := readContinueToken(req.NextPageToken)
-	if err != nil {
-		return nil, err
-	}
-	if token.orgId > 0 && token.orgId != info.OrgID {
-		return nil, fmt.Errorf("token and orgID mismatch")
-	}
-	limit := int(req.Limit)
-	if limit < 1 {
-		limit = 15
-	}
-	query := &DashboardQuery{
-		OrgID:  info.OrgID,
-		Limit:  limit + 1,
-		LastID: token.id,
-		UID:    req.Key.Name,
-	}
-	if req.ShowDeleted {
-		query.GetTrash = true
-	} else {
-		query.GetHistory = true
-	}
-
-	sql, err := a.sql(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	rows, err := a.getRows(ctx, sql, query)
-	if err != nil {
-		return nil, err
-	}
-	defer func() { _ = rows.Close() }()
-
-	list := &resource.HistoryResponse{}
-	for rows.Next() {
-		if rows.err != nil || rows.row == nil {
-			return list, err
-		}
-		row := rows.row
-
-		partial := &metav1.PartialObjectMetadata{
-			ObjectMeta: row.Dash.ObjectMeta,
-		}
-		partial.UID = "" // it is not useful/helpful/accurate and just confusing now
-
-		val, err := json.Marshal(partial)
-		if err != nil {
-			return list, err
-		}
-
-		if len(list.Items) >= limit {
-			// if query.Requirements.Folder != nil {
-			// 	row.token.folder = *query.Requirements.Folder
-			// }
-			row.token.id = getVersionFromRV(row.RV) // Use the version as the increment
-			list.NextPageToken = row.token.String() // will skip this one but start here next time
-			return list, err
-		}
-
-		list.Items = append(list.Items, &resource.ResourceMeta{
-			ResourceVersion:   row.RV,
-			PartialObjectMeta: val,
-			Size:              int32(len(rows.Value())),
-			Hash:              "??", // hash the full?
-		})
-	}
-	return list, err
-}
-	**/
-
 func (a *dashboardSqlAccess) ListRepositoryObjects(ctx context.Context, req *resource.ListRepositoryObjectsRequest) (*resource.ListRepositoryObjectsResponse, error) {
 	return nil, fmt.Errorf("not implemented")
 }

pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_at_version.sql

@@ -16,5 +16,6 @@ SELECT
     LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard.updated_by = updated_user.id
     WHERE dashboard.is_folder = false
       AND dashboard.org_id = 2
+      AND dashboard.uid = 'UUU'
       AND dashboard_version.version = 3
     ORDER BY dashboard_version.version DESC

pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_at_version.sql

@@ -16,5 +16,6 @@ SELECT
     LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
     WHERE dashboard.is_folder = false
       AND dashboard.org_id = 2
+      AND dashboard.uid = 'UUU'
       AND dashboard_version.version = 3
     ORDER BY dashboard_version.version DESC

pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_at_version.sql

@@ -16,5 +16,6 @@ SELECT
     LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
     WHERE dashboard.is_folder = false
       AND dashboard.org_id = 2
+      AND dashboard.uid = 'UUU'
       AND dashboard_version.version = 3
     ORDER BY dashboard_version.version DESC

pkg/storage/unified/apistore/util.go

@@ -14,8 +14,11 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apiserver/pkg/storage"
 
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
+
 	grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
 )
@@ -39,6 +42,38 @@ func toListRequest(k *resource.ResourceKey, opts storage.ListOptions) (*resource
 		for _, r := range requirements {
 			v := r.Key()
 
+			// Parse the history request from labels
+			if v == utils.LabelKeyGetHistory || v == utils.LabelKeyGetTrash {
+				if len(requirements) != 1 {
+					return nil, predicate, apierrors.NewBadRequest("single label supported with: " + v)
+				}
+				if !opts.Predicate.Field.Empty() {
+					return nil, predicate, apierrors.NewBadRequest("field selector not supported with: " + v)
+				}
+				if r.Operator() != selection.Equals {
+					return nil, predicate, apierrors.NewBadRequest("only = operator supported with: " + v)
+				}
+
+				vals := r.Values().List()
+				if len(vals) != 1 {
+					return nil, predicate, apierrors.NewBadRequest("expecting single value for: " + v)
+				}
+
+				if v == utils.LabelKeyGetTrash {
+					req.Source = resource.ListRequest_TRASH
+					if vals[0] != "true" {
+						return nil, predicate, apierrors.NewBadRequest("expecting true for: " + v)
+					}
+				} else {
+					req.Source = resource.ListRequest_HISTORY
+					req.Options.Key.Name = vals[0]
+				}
+
+				req.Options.Labels = nil
+				req.Options.Fields = nil
+				return req, storage.Everything, nil
+			}
+
 			req.Options.Labels = append(req.Options.Labels, &resource.Requirement{
 				Key:      v,
 				Operator: string(r.Operator()),

pkg/storage/unified/resource/cdk_backend.go

@@ -21,6 +21,8 @@ import (
 	_ "gocloud.dev/blob/memblob"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
 )
 
 type CDKBackendOptions struct {
@@ -192,7 +194,7 @@ func (s *cdkBackend) ReadResource(ctx context.Context, req *ReadRequest) *Backen
 			err = nil
 		}
 	}
-	if err == nil && isDeletedMarker(raw) {
+	if err == nil && isDeletedValue(raw) {
 		raw = nil
 	}
 	if raw == nil {
@@ -206,11 +208,11 @@ func (s *cdkBackend) ReadResource(ctx context.Context, req *ReadRequest) *Backen
 	}
 }
 
-func isDeletedMarker(raw []byte) bool {
-	if bytes.Contains(raw, []byte(`"DeletedMarker"`)) {
+func isDeletedValue(raw []byte) bool {
+	if bytes.Contains(raw, []byte(`"generation":-999`)) {
 		tmp := &unstructured.Unstructured{}
 		err := tmp.UnmarshalJSON(raw)
-		if err == nil && tmp.GetKind() == "DeletedMarker" {
+		if err == nil && tmp.GetGeneration() == utils.DeletedGeneration {
 			return true
 		}
 	}
@@ -218,6 +220,10 @@ func isDeletedMarker(raw []byte) bool {
 }
 
 func (s *cdkBackend) ListIterator(ctx context.Context, req *ListRequest, cb func(ListIterator) error) (int64, error) {
+	if req.Source != ListRequest_STORE {
+		return 0, fmt.Errorf("listing from history not supported in CDK backend")
+	}
+
 	resources, err := buildTree(ctx, s, req.Options.Key)
 	if err != nil {
 		return 0, err
@@ -286,7 +292,7 @@ func (c *cdkListIterator) Next() bool {
 			c.err = err
 			return false
 		}
-		if !isDeletedMarker(raw) {
+		if !isDeletedValue(raw) {
 			c.currentRV = latest.rv
 			c.currentKey = latest.key
 			c.currentVal = raw

pkg/storage/unified/resource/deleted.go

@@ -1,37 +0,0 @@
-package resource
-
-import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-)
-
-// This object is written when an object is deleted
-type DeletedMarker struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DeletedMarker) DeepCopyInto(out *DeletedMarker) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeletedMarker.
-func (in *DeletedMarker) DeepCopy() *DeletedMarker {
-	if in == nil {
-		return nil
-	}
-	out := new(DeletedMarker)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *DeletedMarker) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}

pkg/storage/unified/resource/resource.proto

@@ -225,6 +225,12 @@ enum ResourceVersionMatch {
 }
 
 message ListRequest {
+  enum Source { 
+    STORE = 0; // the standard place
+    HISTORY = 1; 
+    TRASH = 2;
+  }
+
   // Starting from the requested page (other query parameters must match!)
   string next_page_token = 1;
 
@@ -240,6 +246,9 @@ message ListRequest {
 
   // Filtering
   ListOptions options = 5;
+
+  // Select values from history or trash
+  Source source = 6; 
 }
 
 message ListResponse {

pkg/storage/unified/resource/server.go

@@ -371,6 +371,13 @@ func (s *server) newEvent(ctx context.Context, user claims.AuthInfo, key *Resour
 		s.log.Error("object must not include a resource version", "key", key)
 	}
 
+	// Make sure the command labels are not saved
+	for k := range obj.GetLabels() {
+		if k == utils.LabelKeyGetHistory || k == utils.LabelKeyGetTrash {
+			return nil, NewBadRequestError("can not save label: " + k)
+		}
+	}
+
 	check := authz.CheckRequest{
 		Verb:      utils.VerbCreate,
 		Group:     key.Group,
@@ -612,7 +619,7 @@ func (s *server) Delete(ctx context.Context, req *DeleteRequest) (*DeleteRespons
 	if !ok {
 		return nil, apierrors.NewBadRequest("unable to get user")
 	}
-	marker := &DeletedMarker{}
+	marker := &unstructured.Unstructured{}
 	err = json.Unmarshal(latest.Value, marker)
 	if err != nil {
 		return nil, apierrors.NewBadRequest(
@@ -627,12 +634,9 @@ func (s *server) Delete(ctx context.Context, req *DeleteRequest) (*DeleteRespons
 	obj.SetManagedFields(nil)
 	obj.SetFinalizers(nil)
 	obj.SetUpdatedBy(requester.GetUID())
-	marker.TypeMeta = metav1.TypeMeta{
-		Kind:       "DeletedMarker",
-		APIVersion: "common.grafana.app/v0alpha1", // ?? or can we stick this in common?
-	}
-	marker.Annotations["RestoreResourceVersion"] = fmt.Sprintf("%d", event.PreviousRV)
-	event.Value, err = json.Marshal(marker)
+	obj.SetGeneration(utils.DeletedGeneration)
+	obj.SetAnnotation(utils.AnnoKeyKubectlLastAppliedConfig, "") // clears it
+	event.Value, err = marker.MarshalJSON()
 	if err != nil {
 		return nil, apierrors.NewBadRequest(
 			fmt.Sprintf("unable creating deletion marker, %v", err))
@@ -693,6 +697,15 @@ func (s *server) List(ctx context.Context, req *ListRequest) (*ListResponse, err
 	ctx, span := s.tracer.Start(ctx, "storage_server.List")
 	defer span.End()
 
+	// The history + trash queries do not yet support additional filters
+	if req.Source != ListRequest_STORE {
+		if len(req.Options.Fields) > 0 || len(req.Options.Labels) > 0 {
+			return &ListResponse{
+				Error: NewBadRequestError("unexpected field/label selector for history query"),
+			}, nil
+		}
+	}
+
 	user, ok := claims.From(ctx)
 	if !ok || user == nil {
 		return &ListResponse{
@@ -702,6 +715,13 @@ func (s *server) List(ctx context.Context, req *ListRequest) (*ListResponse, err
 			}}, nil
 	}
 
+	// Do not allow label query for trash/history
+	for _, v := range req.Options.Labels {
+		if v.Key == utils.LabelKeyGetHistory || v.Key == utils.LabelKeyGetTrash {
+			return &ListResponse{Error: NewBadRequestError("history and trash must be requested as source")}, nil
+		}
+	}
+
 	if req.Limit < 1 {
 		req.Limit = 50 // default max 50 items in a page
 	}

pkg/storage/unified/resource/server_test.go

@@ -119,14 +119,27 @@ func TestSimpleServer(t *testing.T) {
 		obj.SetAnnotation("test", "hello")
 		obj.SetUpdatedTimestampMillis(now)
 		obj.SetUpdatedBy(testUserA.GetUID())
+		obj.SetLabels(map[string]string{
+			utils.LabelKeyGetTrash: "", // should not be allowed to save this!
+		})
 		raw, err = json.Marshal(tmp)
 		require.NoError(t, err)
-
 		updated, err := server.Update(ctx, &UpdateRequest{
 			Key:             key,
 			Value:           raw,
 			ResourceVersion: created.ResourceVersion})
 		require.NoError(t, err)
+		require.Equal(t, int32(400), updated.Error.Code) // bad request
+
+		// remove the invalid labels
+		obj.SetLabels(nil)
+		raw, err = json.Marshal(tmp)
+		require.NoError(t, err)
+		updated, err = server.Update(ctx, &UpdateRequest{
+			Key:             key,
+			Value:           raw,
+			ResourceVersion: created.ResourceVersion})
+		require.NoError(t, err)
 		require.Nil(t, updated.Error)
 		require.True(t, updated.ResourceVersion > created.ResourceVersion)
 

pkg/storage/unified/sql/backend.go

@@ -37,6 +37,7 @@ type BackendOptions struct {
 	DBProvider        db.DBProvider
 	Tracer            trace.Tracer
 	PollingInterval   time.Duration
+	SkipDataMigration bool
 }
 
 func NewBackend(opts BackendOptions) (Backend, error) {
@@ -59,6 +60,7 @@ func NewBackend(opts BackendOptions) (Backend, error) {
 		tracer:            opts.Tracer,
 		dbProvider:        opts.DBProvider,
 		pollingInterval:   pollingInterval,
+		skipDataMigration: opts.SkipDataMigration,
 	}, nil
 }
 
@@ -77,6 +79,7 @@ type backend struct {
 	dbProvider        db.DBProvider
 	db                db.DB
 	dialect           sqltemplate.Dialect
+	skipDataMigration bool
 
 	// watch streaming
 	//stream chan *resource.WatchEvent
@@ -103,6 +106,12 @@ func (b *backend) initLocked(ctx context.Context) error {
 		return fmt.Errorf("no dialect for driver %q", driverName)
 	}
 
+	// Process any data manipulation migrations
+	err = b.runStartupDataMigrations(ctx)
+	if err != nil {
+		return err
+	}
+
 	return b.db.PingContext(ctx)
 }
 
@@ -477,13 +486,17 @@ func (b *backend) ReadResource(ctx context.Context, req *resource.ReadRequest) *
 }
 
 func (b *backend) ListIterator(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
-	_, span := b.tracer.Start(ctx, tracePrefix+"List")
+	ctx, span := b.tracer.Start(ctx, tracePrefix+"List")
 	defer span.End()
 
 	if req.Options == nil || req.Options.Key.Group == "" || req.Options.Key.Resource == "" {
 		return 0, fmt.Errorf("missing group or resource")
 	}
 
+	if req.Source != resource.ListRequest_STORE {
+		return b.getHistory(ctx, req, cb)
+	}
+
 	// TODO: think about how to handler VersionMatch. We should be able to use latest for the first page (only).
 
 	// TODO: add support for RemainingItemCount
@@ -647,6 +660,48 @@ func (b *backend) listAtRevision(ctx context.Context, req *resource.ListRequest,
 	return iter.listRV, err
 }
 
+// listLatest fetches the resources from the resource table.
+func (b *backend) getHistory(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
+	listReq := sqlGetHistoryRequest{
+		SQLTemplate: sqltemplate.New(b.dialect),
+		Key:         req.Options.Key,
+		Trash:       req.Source == resource.ListRequest_TRASH,
+	}
+
+	iter := &listIter{}
+	if req.NextPageToken != "" {
+		continueToken, err := GetContinueToken(req.NextPageToken)
+		if err != nil {
+			return 0, fmt.Errorf("get continue token: %w", err)
+		}
+		listReq.StartRV = continueToken.ResourceVersion
+	}
+
+	err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error {
+		var err error
+		iter.listRV, err = fetchLatestRV(ctx, tx, b.dialect, req.Options.Key.Group, req.Options.Key.Resource)
+		if err != nil {
+			return err
+		}
+
+		rows, err := dbutil.QueryRows(ctx, tx, sqlResourceHistoryGet, listReq)
+		if rows != nil {
+			defer func() {
+				if err := rows.Close(); err != nil {
+					b.log.Warn("listLatest error closing rows", "error", err)
+				}
+			}()
+		}
+		if err != nil {
+			return err
+		}
+
+		iter.rows = rows
+		return cb(iter)
+	})
+	return iter.listRV, err
+}
+
 func (b *backend) WatchWriteEvents(ctx context.Context) (<-chan *resource.WrittenEvent, error) {
 	// Get the latest RV
 	since, err := b.listLatestRVs(ctx)

pkg/storage/unified/sql/backend_test.go

@@ -62,7 +62,10 @@ func setupBackendTest(t *testing.T) (testBackend, context.Context) {
 
 	ctx := testutil.NewDefaultTestContext(t)
 	dbp := test.NewDBProviderMatchWords(t)
-	b, err := NewBackend(BackendOptions{DBProvider: dbp})
+	b, err := NewBackend(BackendOptions{
+		DBProvider:        dbp,
+		SkipDataMigration: true, // Calling migrations makes startup SQL calls (avoid the mock)
+	})
 	require.NoError(t, err)
 	require.NotNil(t, b)
 
@@ -109,7 +112,7 @@ func TestBackend_Init(t *testing.T) {
 
 		ctx := testutil.NewDefaultTestContext(t)
 		dbp := test.NewDBProviderWithPing(t)
-		b, err := NewBackend(BackendOptions{DBProvider: dbp})
+		b, err := NewBackend(BackendOptions{DBProvider: dbp, SkipDataMigration: true})
 		require.NoError(t, err)
 		require.NotNil(t, b)
 
@@ -166,7 +169,7 @@ func TestBackend_Init(t *testing.T) {
 
 		ctx := testutil.NewDefaultTestContext(t)
 		dbp := test.NewDBProviderWithPing(t)
-		b, err := NewBackend(BackendOptions{DBProvider: dbp})
+		b, err := NewBackend(BackendOptions{DBProvider: dbp, SkipDataMigration: true})
 		require.NoError(t, err)
 		require.NotNil(t, dbp.DB)
 
@@ -182,7 +185,7 @@ func TestBackend_IsHealthy(t *testing.T) {
 
 	ctx := testutil.NewDefaultTestContext(t)
 	dbp := test.NewDBProviderWithPing(t)
-	b, err := NewBackend(BackendOptions{DBProvider: dbp})
+	b, err := NewBackend(BackendOptions{DBProvider: dbp, SkipDataMigration: true})
 	require.NoError(t, err)
 	require.NotNil(t, dbp.DB)
 

pkg/storage/unified/sql/data/migrator_get_deletion_markers.sql

@@ -0,0 +1,9 @@
+SELECT
+    {{ .Ident "guid"  }},
+    {{ .Ident "value" }},
+    {{ .Ident "group" }},
+    {{ .Ident "resource" }},
+    {{ .Ident "previous_resource_version" }}
+ FROM {{ .Ident "resource_history" }}
+WHERE {{ .Ident "action" }} = 3
+  AND {{ .Ident "value" }} LIKE {{ .Arg .MarkerQuery }};

pkg/storage/unified/sql/data/migrator_get_value_from_rv.sql

@@ -0,0 +1,5 @@
+SELECT {{ .Ident "value" }}
+  FROM {{ .Ident "resource_history" }}
+ WHERE {{ .Ident "group"    }} = {{ .Arg .Group }}
+   AND {{ .Ident "resource" }} = {{ .Arg .Resource }}
+   AND {{ .Ident "resource_version" }} = {{ .Arg .RV }};

pkg/storage/unified/sql/data/migrator_update_value_with_guid.sql

@@ -0,0 +1,4 @@
+UPDATE {{ .Ident "resource_history" }}
+   SET {{ .Ident "value" }} = {{ .Arg .Value }}
+ WHERE {{ .Ident "guid"  }} = {{ .Arg .GUID }}
+;

pkg/storage/unified/sql/data/resource_history_delete.sql

@@ -0,0 +1,4 @@
+DELETE FROM {{ .Ident "resource_history" }}
+ WHERE 1 = 1
+   AND {{ .Ident "guid" }} = {{ .Arg .GUID }}
+   

pkg/storage/unified/sql/data/resource_history_get.sql

@@ -0,0 +1,21 @@
+SELECT
+  {{ .Ident "resource_version" }},
+  {{ .Ident "namespace" }},
+  {{ .Ident "name" }},
+  {{ .Ident "folder" }},
+  {{ .Ident "value" }}
+FROM {{ .Ident "resource_history" }}
+WHERE 1 = 1
+  AND {{ .Ident "namespace" }} = {{ .Arg .Key.Namespace }}
+  AND {{ .Ident "group" }}     = {{ .Arg .Key.Group }}
+  AND {{ .Ident "resource" }}  = {{ .Arg .Key.Resource }}
+  {{ if .Key.Name }}
+  AND {{ .Ident "name" }}      = {{ .Arg .Key.Name }}
+  {{ end }}
+  {{ if .Trash }}
+  AND {{ .Ident "action" }} = 3
+  {{ end }}
+  {{ if (gt .StartRV 0) }}
+  AND {{ .Ident "resource_version" }} > {{ .Arg .StartRV }}
+  {{ end }}
+ORDER BY resource_version DESC

pkg/storage/unified/sql/migrations.go

@@ -0,0 +1,133 @@
+package sql
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
+	"github.com/grafana/grafana/pkg/storage/unified/sql/db"
+	"github.com/grafana/grafana/pkg/storage/unified/sql/dbutil"
+	"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
+)
+
+// This runs functions before the server is returned as healthy
+func (b *backend) runStartupDataMigrations(ctx context.Context) error {
+	if b.skipDataMigration {
+		return nil
+	}
+
+	type migrateRow struct {
+		GUID       string
+		Marker     *unstructured.Unstructured
+		Group      string
+		Resource   string
+		PreviousRV int64
+	}
+
+	// Migrate DeletedMarker to regular resource
+	err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
+		req := &sqlMigrationQueryRequest{
+			SQLTemplate: sqltemplate.New(b.dialect),
+			MarkerQuery: `{"kind":"DeletedMarker"%`,
+		}
+
+		// 1. Find rows with the existing deletion marker
+		rows, err := dbutil.QueryRows(ctx, tx, sqlMigratorGetDeletionMarkers, req)
+		if err != nil {
+			return err
+		}
+		migrateRows := make([]migrateRow, 0)
+		for rows.Next() {
+			item := migrateRow{Marker: &unstructured.Unstructured{}}
+			err = rows.Scan(&item.GUID, &req.Value, &item.Group, &item.Resource, &item.PreviousRV)
+			if err != nil {
+				return err
+			}
+
+			err = item.Marker.UnmarshalJSON([]byte(req.Value))
+			if err != nil {
+				return err
+			}
+
+			migrateRows = append(migrateRows, item)
+		}
+		err = rows.Close()
+		if err != nil {
+			return err
+		}
+
+		for _, item := range migrateRows {
+			// 2. Load the previous value referenced by that marker
+			req := &sqlMigrationQueryRequest{
+				SQLTemplate: sqltemplate.New(b.dialect),
+				Group:       item.Group,
+				Resource:    item.Resource,
+				RV:          item.PreviousRV,
+				GUID:        item.GUID,
+			}
+			rows, err = dbutil.QueryRows(ctx, tx, sqlMigratorGetValueFromRV, req)
+			if err != nil {
+				return err
+			}
+			if rows.Next() {
+				err = rows.Scan(&req.Value)
+				if err != nil {
+					return err
+				}
+			}
+			err = rows.Close()
+			if err != nil {
+				return err
+			}
+			req.Reset()
+
+			if len(req.Value) > 0 {
+				previous := &unstructured.Unstructured{}
+				err = previous.UnmarshalJSON([]byte(req.Value))
+				if err != nil {
+					return err
+				}
+
+				// 3. Prepare a new payload
+				metaMarker, _ := utils.MetaAccessor(item.Marker)
+				metaPrev, _ := utils.MetaAccessor(previous)
+				metaPrev.SetDeletionTimestamp(metaMarker.GetDeletionTimestamp())
+				metaPrev.SetFinalizers(nil)
+				metaPrev.SetManagedFields(nil)
+				metaPrev.SetGeneration(utils.DeletedGeneration)
+				metaPrev.SetAnnotation(utils.AnnoKeyKubectlLastAppliedConfig, "") // clears it
+				ts, _ := metaMarker.GetUpdatedTimestamp()
+				if ts != nil {
+					metaPrev.SetUpdatedTimestamp(ts)
+				}
+				buff, err := previous.MarshalJSON()
+				if err != nil {
+					return err
+				}
+				req.Value = string(buff)
+
+				// 4. Update the SQL row with this new value
+				b.log.Info("Migrating DeletedMarker", "guid", req.GUID, "group", req.Group, "resource", req.Resource)
+				_, err = dbutil.Exec(ctx, tx, sqlMigratorUpdateValueWithGUID, req)
+				if err != nil {
+					return err
+				}
+			} else {
+				// 5. If the previous version is missing, we delete it -- there is nothing to help us restore anyway
+				b.log.Warn("Removing orphan deletion marker", "guid", req.GUID, "group", req.Group, "resource", req.Resource)
+				_, err = dbutil.Exec(ctx, tx, sqlResourceHistoryDelete, &sqlResourceHistoryDeleteRequest{
+					SQLTemplate: sqltemplate.New(b.dialect),
+					GUID:        req.GUID,
+				})
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return err
+}

pkg/storage/unified/sql/queries.go

@@ -42,6 +42,8 @@ var (
 	sqlResoureceHistoryUpdateUid = mustTemplate("resource_history_update_uid.sql")
 	sqlResourceHistoryInsert     = mustTemplate("resource_history_insert.sql")
 	sqlResourceHistoryPoll       = mustTemplate("resource_history_poll.sql")
+	sqlResourceHistoryGet        = mustTemplate("resource_history_get.sql")
+	sqlResourceHistoryDelete     = mustTemplate("resource_history_delete.sql")
 
 	// sqlResourceLabelsInsert = mustTemplate("resource_labels_insert.sql")
 	sqlResourceVersionGet    = mustTemplate("resource_version_get.sql")
@@ -51,6 +53,10 @@ var (
 
 	sqlResourceBlobInsert = mustTemplate("resource_blob_insert.sql")
 	sqlResourceBlobQuery  = mustTemplate("resource_blob_query.sql")
+
+	sqlMigratorGetDeletionMarkers  = mustTemplate("migrator_get_deletion_markers.sql")
+	sqlMigratorGetValueFromRV      = mustTemplate("migrator_get_value_from_rv.sql")
+	sqlMigratorUpdateValueWithGUID = mustTemplate("migrator_update_value_with_guid.sql")
 )
 
 // TxOptions.
@@ -197,6 +203,27 @@ func (r sqlResourceHistoryListRequest) Results() (*resource.ResourceWrapper, err
 	}, nil
 }
 
+type sqlResourceHistoryDeleteRequest struct {
+	sqltemplate.SQLTemplate
+	GUID string
+	// TODO, add other constraints
+}
+
+func (r *sqlResourceHistoryDeleteRequest) Validate() error {
+	return nil // TODO
+}
+
+type sqlGetHistoryRequest struct {
+	sqltemplate.SQLTemplate
+	Key     *resource.ResourceKey
+	Trash   bool  // only deleted items
+	StartRV int64 // from NextPageToken
+}
+
+func (r sqlGetHistoryRequest) Validate() error {
+	return nil // TODO
+}
+
 // update resource history
 
 type sqlResourceHistoryUpdateRequest struct {
@@ -303,3 +330,19 @@ func (r *sqlResourceVersionListRequest) Results() (*groupResourceVersion, error)
 	x := *r.groupResourceVersion
 	return &x, nil
 }
+
+// This holds all the variables used in migration queries
+
+type sqlMigrationQueryRequest struct {
+	sqltemplate.SQLTemplate
+	MarkerQuery string //
+	Group       string
+	Resource    string
+	RV          int64
+	GUID        string
+	Value       string
+}
+
+func (r sqlMigrationQueryRequest) Validate() error {
+	return nil // TODO
+}

pkg/storage/unified/sql/queries_test.go

@@ -207,6 +207,46 @@ func TestUnifiedStorageQueries(t *testing.T) {
 				},
 			},
 
+			sqlResourceHistoryGet: {
+				{
+					Name: "read object history",
+					Data: &sqlGetHistoryRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						Key: &resource.ResourceKey{
+							Namespace: "nn",
+							Group:     "gg",
+							Resource:  "rr",
+							Name:      "name",
+						},
+					},
+				},
+				{
+					Name: "read trash",
+					Data: &sqlGetHistoryRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						Key: &resource.ResourceKey{
+							Namespace: "nn",
+							Group:     "gg",
+							Resource:  "rr",
+						},
+						Trash: true,
+					},
+				},
+				{
+					Name: "read trash second page",
+					Data: &sqlGetHistoryRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						Key: &resource.ResourceKey{
+							Namespace: "nn",
+							Group:     "gg",
+							Resource:  "rr",
+						},
+						Trash:   true,
+						StartRV: 123456,
+					},
+				},
+			},
+
 			sqlResourceVersionGet: {
 				{
 					Name: "single path",
@@ -317,5 +357,44 @@ func TestUnifiedStorageQueries(t *testing.T) {
 					},
 				},
 			},
+			sqlResourceHistoryDelete: {
+				{
+					Name: "guid",
+					Data: &sqlResourceHistoryDeleteRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						GUID:        `xxxx`,
+					},
+				},
+			},
+			sqlMigratorGetDeletionMarkers: {
+				{
+					Name: "list",
+					Data: &sqlMigrationQueryRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						MarkerQuery: `{"kind":"DeletedMarker"%`,
+					},
+				},
+			},
+			sqlMigratorGetValueFromRV: {
+				{
+					Name: "get",
+					Data: &sqlMigrationQueryRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						Group:       "ggg",
+						Resource:    "rrr",
+						RV:          1234,
+					},
+				},
+			},
+			sqlMigratorUpdateValueWithGUID: {
+				{
+					Name: "update",
+					Data: &sqlMigrationQueryRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						GUID:        "ggggg",
+						Value:       "{new value}",
+					},
+				},
+			},
 		}})
 }

pkg/storage/unified/sql/testdata/mysql--migrator_get_deletion_markers-list.sql

@@ -0,0 +1,9 @@
+SELECT
+    `guid`,
+    `value`,
+    `group`,
+    `resource`,
+    `previous_resource_version`
+ FROM `resource_history`
+WHERE `action` = 3
+  AND `value` LIKE '{"kind":"DeletedMarker"%';

pkg/storage/unified/sql/testdata/mysql--migrator_get_value_from_rv-get.sql

@@ -0,0 +1,5 @@
+SELECT `value`
+  FROM `resource_history`
+ WHERE `group` = 'ggg'
+   AND `resource` = 'rrr'
+   AND `resource_version` = 1234;

pkg/storage/unified/sql/testdata/mysql--migrator_update_value_with_guid-update.sql

@@ -0,0 +1,4 @@
+UPDATE `resource_history`
+   SET `value` = '{new value}'
+ WHERE `guid` = 'ggggg'
+;

pkg/storage/unified/sql/testdata/mysql--resource_history_delete-guid.sql

@@ -0,0 +1,3 @@
+DELETE FROM `resource_history`
+ WHERE 1 = 1
+   AND `guid` = 'xxxx'

pkg/storage/unified/sql/testdata/mysql--resource_history_get-read object history.sql

@@ -0,0 +1,13 @@
+SELECT
+  `resource_version`,
+  `namespace`,
+  `name`,
+  `folder`,
+  `value`
+FROM `resource_history`
+WHERE 1 = 1
+  AND `namespace` = 'nn'
+  AND `group`     = 'gg'
+  AND `resource`  = 'rr'
+  AND `name`      = 'name'
+ORDER BY resource_version DESC

pkg/storage/unified/sql/testdata/mysql--resource_history_get-read trash second page.sql

@@ -0,0 +1,14 @@
+SELECT
+  `resource_version`,
+  `namespace`,
+  `name`,
+  `folder`,
+  `value`
+FROM `resource_history`
+WHERE 1 = 1
+  AND `namespace` = 'nn'
+  AND `group`     = 'gg'
+  AND `resource`  = 'rr'
+  AND `action` = 3
+  AND `resource_version` > 123456
+ORDER BY resource_version DESC

pkg/storage/unified/sql/testdata/mysql--resource_history_get-read trash.sql

@@ -0,0 +1,13 @@
+SELECT
+  `resource_version`,
+  `namespace`,
+  `name`,
+  `folder`,
+  `value`
+FROM `resource_history`
+WHERE 1 = 1
+  AND `namespace` = 'nn'
+  AND `group`     = 'gg'
+  AND `resource`  = 'rr'
+  AND `action` = 3
+ORDER BY resource_version DESC

pkg/storage/unified/sql/testdata/postgres--migrator_get_deletion_markers-list.sql

@@ -0,0 +1,9 @@
+SELECT
+    "guid",
+    "value",
+    "group",
+    "resource",
+    "previous_resource_version"
+ FROM "resource_history"
+WHERE "action" = 3
+  AND "value" LIKE '{"kind":"DeletedMarker"%';

pkg/storage/unified/sql/testdata/postgres--migrator_get_value_from_rv-get.sql

@@ -0,0 +1,5 @@
+SELECT "value"
+  FROM "resource_history"
+ WHERE "group" = 'ggg'
+   AND "resource" = 'rrr'
+   AND "resource_version" = 1234;

pkg/storage/unified/sql/testdata/postgres--migrator_update_value_with_guid-update.sql

@@ -0,0 +1,4 @@
+UPDATE "resource_history"
+   SET "value" = '{new value}'
+ WHERE "guid" = 'ggggg'
+;

pkg/storage/unified/sql/testdata/postgres--resource_history_delete-guid.sql

@@ -0,0 +1,3 @@
+DELETE FROM "resource_history"
+ WHERE 1 = 1
+   AND "guid" = 'xxxx'

pkg/storage/unified/sql/testdata/postgres--resource_history_get-read object history.sql

@@ -0,0 +1,13 @@
+SELECT
+  "resource_version",
+  "namespace",
+  "name",
+  "folder",
+  "value"
+FROM "resource_history"
+WHERE 1 = 1
+  AND "namespace" = 'nn'
+  AND "group"     = 'gg'
+  AND "resource"  = 'rr'
+  AND "name"      = 'name'
+ORDER BY resource_version DESC

pkg/storage/unified/sql/testdata/postgres--resource_history_get-read trash second page.sql

@@ -0,0 +1,14 @@
+SELECT
+  "resource_version",
+  "namespace",
+  "name",
+  "folder",
+  "value"
+FROM "resource_history"
+WHERE 1 = 1
+  AND "namespace" = 'nn'
+  AND "group"     = 'gg'
+  AND "resource"  = 'rr'
+  AND "action" = 3
+  AND "resource_version" > 123456
+ORDER BY resource_version DESC

pkg/storage/unified/sql/testdata/postgres--resource_history_get-read trash.sql

@@ -0,0 +1,13 @@
+SELECT
+  "resource_version",
+  "namespace",
+  "name",
+  "folder",
+  "value"
+FROM "resource_history"
+WHERE 1 = 1
+  AND "namespace" = 'nn'
+  AND "group"     = 'gg'
+  AND "resource"  = 'rr'
+  AND "action" = 3
+ORDER BY resource_version DESC

pkg/storage/unified/sql/testdata/sqlite--migrator_get_deletion_markers-list.sql

@@ -0,0 +1,9 @@
+SELECT
+    "guid",
+    "value",
+    "group",
+    "resource",
+    "previous_resource_version"
+ FROM "resource_history"
+WHERE "action" = 3
+  AND "value" LIKE '{"kind":"DeletedMarker"%';

pkg/storage/unified/sql/testdata/sqlite--migrator_get_value_from_rv-get.sql

@@ -0,0 +1,5 @@
+SELECT "value"
+  FROM "resource_history"
+ WHERE "group" = 'ggg'
+   AND "resource" = 'rrr'
+   AND "resource_version" = 1234;

pkg/storage/unified/sql/testdata/sqlite--migrator_update_value_with_guid-update.sql

@@ -0,0 +1,4 @@
+UPDATE "resource_history"
+   SET "value" = '{new value}'
+ WHERE "guid" = 'ggggg'
+;

pkg/storage/unified/sql/testdata/sqlite--resource_history_delete-guid.sql

@@ -0,0 +1,3 @@
+DELETE FROM "resource_history"
+ WHERE 1 = 1
+   AND "guid" = 'xxxx'

pkg/storage/unified/sql/testdata/sqlite--resource_history_get-read object history.sql

@@ -0,0 +1,13 @@
+SELECT
+  "resource_version",
+  "namespace",
+  "name",
+  "folder",
+  "value"
+FROM "resource_history"
+WHERE 1 = 1
+  AND "namespace" = 'nn'
+  AND "group"     = 'gg'
+  AND "resource"  = 'rr'
+  AND "name"      = 'name'
+ORDER BY resource_version DESC

pkg/storage/unified/sql/testdata/sqlite--resource_history_get-read trash second page.sql

@@ -0,0 +1,14 @@
+SELECT
+  "resource_version",
+  "namespace",
+  "name",
+  "folder",
+  "value"
+FROM "resource_history"
+WHERE 1 = 1
+  AND "namespace" = 'nn'
+  AND "group"     = 'gg'
+  AND "resource"  = 'rr'
+  AND "action" = 3
+  AND "resource_version" > 123456
+ORDER BY resource_version DESC

pkg/storage/unified/sql/testdata/sqlite--resource_history_get-read trash.sql

@@ -0,0 +1,13 @@
+SELECT
+  "resource_version",
+  "namespace",
+  "name",
+  "folder",
+  "value"
+FROM "resource_history"
+WHERE 1 = 1
+  AND "namespace" = 'nn'
+  AND "group"     = 'gg'
+  AND "resource"  = 'rr'
+  AND "action" = 3
+ORDER BY resource_version DESC