root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 20 Packages Projects Releases Wiki Activity

K8s/Annotations: Use manager/source annotations rather than repo (#101313) 

Co-authored-by: Stephanie Hingtgen <stephanie.hingtgen@grafana.com>
This commit is contained in:
Ryan McKinley 2025-03-05 09:54:20 +03:00 committed by GitHub
parent e7baf9804e
commit dc2defd84f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
37 changed files with 589 additions and 573 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/api/dtos/folder.go
View File

@ -3,6 +3,7 @@ package dtos
import (
"time"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/services/accesscontrol"
)
@ -31,7 +32,7 @@ type Folder struct {
// When the folder belongs to a repository
// NOTE: this is only populated when folders are managed by unified storage
Repository string `json:"repository,omitempty"`
ManagedBy utils.ManagerKind `json:"managedBy,omitempty"`
}
type FolderSearchHit struct {
@ -42,5 +43,5 @@ type FolderSearchHit struct {
// When the folder belongs to a repository
// NOTE: this is only populated when folders are managed by unified storage
Repository string `json:"repository,omitempty"`
ManagedBy utils.ManagerKind `json:"managedBy,omitempty"`
}

12
pkg/api/folder.go
View File

@ -94,11 +94,11 @@ func (hs *HTTPServer) GetFolders(c *contextmodel.ReqContext) response.Response {
hits := make([]dtos.FolderSearchHit, 0)
for _, f := range folders {
hits = append(hits, dtos.FolderSearchHit{
ID: f.ID, // nolint:staticcheck
UID: f.UID,
Title: f.Title,
ParentUID: f.ParentUID,
Repository: f.Repository,
ID: f.ID, // nolint:staticcheck
UID: f.UID,
Title: f.Title,
ParentUID: f.ParentUID,
ManagedBy: f.ManagedBy,
})
metrics.MFolderIDsAPICount.WithLabelValues(metrics.GetFolders).Inc()
}
@ -427,7 +427,7 @@ func (hs *HTTPServer) newToFolderDto(c *contextmodel.ReqContext, f *folder.Folde
Version: f.Version,
AccessControl: acMetadata,
ParentUID: f.ParentUID,
Repository: f.Repository,
ManagedBy: f.ManagedBy,
}, nil
}

28
pkg/apimachinery/utils/manager.go
View File

@ -1,28 +1,27 @@
package utils
import "time"
// ManagerProperties is used to identify the manager of the resource.
type ManagerProperties struct {
// The kind of manager, which is responsible for managing the resource.
// Examples include "git", "terraform", "kubectl", etc.
Kind ManagerKind
Kind ManagerKind `json:"kind,omitempty"`
// The identity of the manager, which refers to a specific instance of the manager.
// The format & the value depends on the manager kind.
Identity string
Identity string `json:"id,omitempty"`
// AllowsEdits indicates whether the manager allows edits to the resource.
// If set to true, it means that other requesters can edit the resource.
AllowsEdits bool
AllowsEdits bool `json:"allowEdits,omitempty"`
// Suspended indicates whether the manager is suspended.
// If set to true, then the manager skip updates to the resource.
Suspended bool
Suspended bool `json:"suspended,omitempty"`
}
// ManagerKind is the type of manager, which is responsible for managing the resource.
// It can be a user or a tool or a generic API client.
// +enum
type ManagerKind string
// Known values for ManagerKind.
@ -31,6 +30,11 @@ const (
ManagerKindRepo ManagerKind = "repo"
ManagerKindTerraform ManagerKind = "terraform"
ManagerKindKubectl ManagerKind = "kubectl"
ManagerKindPlugin ManagerKind = "plugin"
// Deprecated: this is used as a shim/migration path for legacy file provisioning
// Previously this was a "file:" prefix
ManagerKindClassicFP ManagerKind = "classic-file-provisioning"
)
// ParseManagerKindString parses a string into a ManagerKind.
@ -44,6 +48,10 @@ func ParseManagerKindString(v string) ManagerKind {
return ManagerKindTerraform
case string(ManagerKindKubectl):
return ManagerKindKubectl
case string(ManagerKindPlugin):
return ManagerKindPlugin
case string(ManagerKindClassicFP): // nolint:staticcheck
return ManagerKindClassicFP // nolint:staticcheck
default:
return ManagerKindUnknown
}
@ -55,13 +63,13 @@ func ParseManagerKindString(v string) ManagerKind {
type SourceProperties struct {
// The path to the source of the resource.
// Can be a file path, a URL, etc.
Path string
Path string `json:"path,omitempty"`
// The checksum of the source of the resource.
// An example could be a git commit hash.
Checksum string
Checksum string `json:"checksum,omitempty"`
// The timestamp of the source of the resource.
// The unix millis timestamp of the source of the resource.
// An example could be the file modification time.
Timestamp time.Time
TimestampMillis int64 `json:"timestampMillis,omitempty"`
}

200
pkg/apimachinery/utils/meta.go
View File

@ -41,10 +41,10 @@ const AnnoKeyMessage = "grafana.app/message"
// Identify where values came from
const AnnoKeyRepoName = "grafana.app/repoName"
const AnnoKeyRepoPath = "grafana.app/repoPath"
const AnnoKeyRepoHash = "grafana.app/repoHash"
const AnnoKeyRepoTimestamp = "grafana.app/repoTimestamp"
const oldAnnoKeyRepoName = "grafana.app/repoName"
const oldAnnoKeyRepoPath = "grafana.app/repoPath"
const oldAnnoKeyRepoHash = "grafana.app/repoHash"
const oldAnnoKeyRepoTimestamp = "grafana.app/repoTimestamp"
// Annotations used to store manager properties
@ -56,41 +56,13 @@ const AnnoKeyManagerSuspended = "grafana.app/managerSuspended"
// Annotations used to store source properties
const AnnoKeySourcePath = "grafana.app/sourcePath"
const AnnoKeySourceHash = "grafana.app/sourceHash"
const AnnoKeySourceChecksum = "grafana.app/sourceChecksum"
const AnnoKeySourceTimestamp = "grafana.app/sourceTimestamp"
// LabelKeyDeprecatedInternalID gives the deprecated internal ID of a resource
// Deprecated: will be removed in grafana 13
const LabelKeyDeprecatedInternalID = "grafana.app/deprecatedInternalID"
// These can be removed once we verify that non of the dual-write sources
// (for dashboards/playlists/etc) depend on the saved internal ID in SQL
const oldAnnoKeyOriginName = "grafana.app/originName"
const oldAnnoKeyOriginPath = "grafana.app/originPath"
const oldAnnoKeyOriginHash = "grafana.app/originHash"
const oldAnnoKeyOriginTimestamp = "grafana.app/originTimestamp"
// ResourceRepositoryInfo is encoded into kubernetes metadata annotations.
// This value identifies indicates the state of the resource in its provisioning source when
// the spec was last saved. Currently this is derived from the dashboards provisioning table.
type ResourceRepositoryInfo struct {
// Name of the repository/provisioning source
Name string `json:"name,omitempty"`
// The path within the named repository above (external_id in the existing dashboard provisioning)
Path string `json:"path,omitempty"`
// Verification/identification hash (check_sum in existing dashboard provisioning)
Hash string `json:"hash,omitempty"`
// Origin modification timestamp when the resource was saved
// This will be before the resource updated time
Timestamp *time.Time `json:"time,omitempty"`
// Avoid extending
_ any `json:"-"`
}
// Accessor functions for k8s objects
type GrafanaMetaAccessor interface {
metav1.Object
@ -128,13 +100,6 @@ type GrafanaMetaAccessor interface {
// Deprecated: This will be removed in Grafana 13
SetDeprecatedInternalID(id int64)
GetRepositoryInfo() (*ResourceRepositoryInfo, error)
SetRepositoryInfo(info *ResourceRepositoryInfo)
GetRepositoryName() string
GetRepositoryPath() string
GetRepositoryHash() string
GetRepositoryTimestamp() (*time.Time, error)
GetSpec() (any, error)
SetSpec(any) error
@ -351,90 +316,6 @@ func (m *grafanaMetaAccessor) SetDeprecatedInternalID(id int64) {
m.obj.SetLabels(labels)
}
// This allows looking up a primary and secondary key -- if either exist the value will be returned
func (m *grafanaMetaAccessor) getAnnoValue(primary, secondary string) (string, bool) {
v, ok := m.obj.GetAnnotations()[primary]
if !ok {
v, ok = m.obj.GetAnnotations()[secondary]
}
return v, ok
}
func (m *grafanaMetaAccessor) SetRepositoryInfo(info *ResourceRepositoryInfo) {
anno := m.obj.GetAnnotations()
if anno == nil {
if info == nil {
return
}
anno = make(map[string]string, 0)
}
// remove legacy values
delete(anno, oldAnnoKeyOriginHash)
delete(anno, oldAnnoKeyOriginPath)
delete(anno, oldAnnoKeyOriginHash)
delete(anno, oldAnnoKeyOriginTimestamp)
delete(anno, AnnoKeyRepoName)
delete(anno, AnnoKeyRepoPath)
delete(anno, AnnoKeyRepoHash)
delete(anno, AnnoKeyRepoTimestamp)
if info != nil && info.Name != "" {
anno[AnnoKeyRepoName] = info.Name
if info.Path != "" {
anno[AnnoKeyRepoPath] = info.Path
}
if info.Hash != "" {
anno[AnnoKeyRepoHash] = info.Hash
}
if info.Timestamp != nil {
anno[AnnoKeyRepoTimestamp] = info.Timestamp.UTC().Format(time.RFC3339)
}
}
m.obj.SetAnnotations(anno)
}
func (m *grafanaMetaAccessor) GetRepositoryInfo() (*ResourceRepositoryInfo, error) {
v, ok := m.getAnnoValue(AnnoKeyRepoName, oldAnnoKeyOriginName)
if !ok {
return nil, nil
}
t, err := m.GetRepositoryTimestamp()
return &ResourceRepositoryInfo{
Name: v,
Path: m.GetRepositoryPath(),
Hash: m.GetRepositoryHash(),
Timestamp: t,
}, err
}
func (m *grafanaMetaAccessor) GetRepositoryName() string {
v, _ := m.getAnnoValue(AnnoKeyRepoName, oldAnnoKeyOriginName)
return v // will be empty string
}
func (m *grafanaMetaAccessor) GetRepositoryPath() string {
v, _ := m.getAnnoValue(AnnoKeyRepoPath, oldAnnoKeyOriginPath)
return v // will be empty string
}
func (m *grafanaMetaAccessor) GetRepositoryHash() string {
v, _ := m.getAnnoValue(AnnoKeyRepoHash, oldAnnoKeyOriginHash)
return v // will be empty string
}
func (m *grafanaMetaAccessor) GetRepositoryTimestamp() (*time.Time, error) {
v, ok := m.getAnnoValue(AnnoKeyRepoTimestamp, oldAnnoKeyOriginTimestamp)
if !ok || v == "" {
return nil, nil
}
t, err := time.Parse(time.RFC3339, v)
if err != nil {
return nil, fmt.Errorf("invalid origin timestamp: %s", err.Error())
}
return &t, nil
}
// GetAnnotations implements GrafanaMetaAccessor.
func (m *grafanaMetaAccessor) GetAnnotations() map[string]string {
return m.obj.GetAnnotations()
@ -751,7 +632,7 @@ func (m *grafanaMetaAccessor) GetManagerProperties() (ManagerProperties, bool) {
res := ManagerProperties{
Identity: "",
Kind: ManagerKindUnknown,
AllowsEdits: true,
AllowsEdits: false,
Suspended: false,
}
@ -759,6 +640,15 @@ func (m *grafanaMetaAccessor) GetManagerProperties() (ManagerProperties, bool) {
id, ok := annot[AnnoKeyManagerIdentity]
if !ok || id == "" {
// Temporarily support the repo name annotation
repo := annot[oldAnnoKeyRepoName]
if repo != "" {
return ManagerProperties{
Kind: ManagerKindRepo,
Identity: repo,
}, true
}
// If the identity is not set, we should ignore the other annotations and return the default values.
//
// This is to prevent inadvertently marking resources as managed,
@ -788,10 +678,31 @@ func (m *grafanaMetaAccessor) SetManagerProperties(v ManagerProperties) {
annot = make(map[string]string, 4)
}
annot[AnnoKeyManagerIdentity] = v.Identity
annot[AnnoKeyManagerKind] = string(v.Kind)
annot[AnnoKeyManagerAllowsEdits] = strconv.FormatBool(v.AllowsEdits)
annot[AnnoKeyManagerSuspended] = strconv.FormatBool(v.Suspended)
if v.Identity != "" {
annot[AnnoKeyManagerIdentity] = v.Identity
} else {
delete(annot, AnnoKeyManagerIdentity)
}
if string(v.Kind) != "" {
annot[AnnoKeyManagerKind] = string(v.Kind)
} else {
delete(annot, AnnoKeyManagerKind)
}
if v.AllowsEdits {
annot[AnnoKeyManagerAllowsEdits] = strconv.FormatBool(v.AllowsEdits)
} else {
delete(annot, AnnoKeyManagerAllowsEdits)
}
if v.Suspended {
annot[AnnoKeyManagerSuspended] = strconv.FormatBool(v.Suspended)
} else {
delete(annot, AnnoKeyManagerSuspended)
}
// Clean up old annotation access
delete(annot, oldAnnoKeyRepoName)
m.obj.SetAnnotations(annot)
}
@ -810,16 +721,27 @@ func (m *grafanaMetaAccessor) GetSourceProperties() (SourceProperties, bool) {
if path, ok := annot[AnnoKeySourcePath]; ok && path != "" {
res.Path = path
found = true
} else if path, ok := annot[oldAnnoKeyRepoPath]; ok && path != "" {
res.Path = path
found = true
}
if hash, ok := annot[AnnoKeySourceHash]; ok && hash != "" {
if hash, ok := annot[AnnoKeySourceChecksum]; ok && hash != "" {
res.Checksum = hash
found = true
} else if hash, ok := annot[oldAnnoKeyRepoHash]; ok && hash != "" {
res.Checksum = hash
found = true
}
if timestamp, ok := annot[AnnoKeySourceTimestamp]; ok && timestamp != "" {
if t, err := time.Parse(time.RFC3339, timestamp); err == nil {
res.Timestamp = t
t, ok := annot[AnnoKeySourceTimestamp]
if !ok {
t, ok = annot[oldAnnoKeyRepoTimestamp]
}
if ok && t != "" {
var err error
res.TimestampMillis, err = strconv.ParseInt(t, 10, 64)
if err != nil {
found = true
}
}
@ -835,14 +757,20 @@ func (m *grafanaMetaAccessor) SetSourceProperties(v SourceProperties) {
if v.Path != "" {
annot[AnnoKeySourcePath] = v.Path
} else {
delete(annot, AnnoKeySourcePath)
}
if v.Checksum != "" {
annot[AnnoKeySourceHash] = v.Checksum
annot[AnnoKeySourceChecksum] = v.Checksum
} else {
delete(annot, AnnoKeySourceChecksum)
}
if !v.Timestamp.IsZero() {
annot[AnnoKeySourceTimestamp] = v.Timestamp.Format(time.RFC3339)
if v.TimestampMillis > 0 {
annot[AnnoKeySourceTimestamp] = strconv.FormatInt(v.TimestampMillis, 10)
} else {
delete(annot, AnnoKeySourceTimestamp)
}
m.obj.SetAnnotations(annot)

94
pkg/apimachinery/utils/meta_test.go
View File

@ -131,10 +131,13 @@ func (in *Spec2) DeepCopy() *Spec2 {
}
func TestMetaAccessor(t *testing.T) {
repoInfo := &utils.ResourceRepositoryInfo{
Name: "test",
Path: "a/b/c",
Hash: "kkk",
repoInfo := utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "test",
}
sourceInfo := utils.SourceProperties{
Path: "a/b/c",
Checksum: "kkk",
}
t.Run("fails for non resource objects", func(t *testing.T) {
@ -202,14 +205,13 @@ func TestMetaAccessor(t *testing.T) {
},
}
meta.SetRepositoryInfo(repoInfo)
meta.SetManagerProperties(repoInfo)
meta.SetFolder("folderUID")
require.Equal(t, map[string]string{
"grafana.app/repoName": "test",
"grafana.app/repoPath": "a/b/c",
"grafana.app/repoHash": "kkk",
"grafana.app/folder": "folderUID",
"grafana.app/managedBy": "repo",
"grafana.app/managerId": "test",
"grafana.app/folder": "folderUID",
}, res.GetAnnotations())
meta.SetNamespace("aaa")
@ -255,14 +257,16 @@ func TestMetaAccessor(t *testing.T) {
meta, err := utils.MetaAccessor(res)
require.NoError(t, err)
meta.SetRepositoryInfo(repoInfo)
meta.SetManagerProperties(repoInfo)
meta.SetSourceProperties(sourceInfo)
meta.SetFolder("folderUID")
require.Equal(t, map[string]string{
"grafana.app/repoName": "test",
"grafana.app/repoPath": "a/b/c",
"grafana.app/repoHash": "kkk",
"grafana.app/folder": "folderUID",
"grafana.app/managedBy": "repo",
"grafana.app/managerId": "test",
"grafana.app/sourcePath": "a/b/c",
"grafana.app/sourceChecksum": "kkk",
"grafana.app/folder": "folderUID",
}, res.GetAnnotations())
meta.SetNamespace("aaa")
@ -306,14 +310,13 @@ func TestMetaAccessor(t *testing.T) {
meta, err := utils.MetaAccessor(res)
require.NoError(t, err)
meta.SetRepositoryInfo(repoInfo)
meta.SetManagerProperties(repoInfo)
meta.SetFolder("folderUID")
require.Equal(t, map[string]string{
"grafana.app/repoName": "test",
"grafana.app/repoPath": "a/b/c",
"grafana.app/repoHash": "kkk",
"grafana.app/folder": "folderUID",
"grafana.app/managedBy": "repo",
"grafana.app/managerId": "test",
"grafana.app/folder": "folderUID",
}, res.GetAnnotations())
meta.SetNamespace("aaa")
@ -347,7 +350,7 @@ func TestMetaAccessor(t *testing.T) {
require.Equal(t, "ZZ", res.Status.Title)
})
t.Run("test reading old originInfo (now repository)", func(t *testing.T) {
t.Run("test reading old repo fields (now manager+source)", func(t *testing.T) {
res := &TestResource2{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
@ -362,11 +365,15 @@ func TestMetaAccessor(t *testing.T) {
meta, err := utils.MetaAccessor(res)
require.NoError(t, err)
info, err := meta.GetRepositoryInfo()
require.NoError(t, err)
require.Equal(t, "test", info.Name)
require.Equal(t, "a/b/c", info.Path)
require.Equal(t, "zzz", info.Hash)
manager, ok := meta.GetManagerProperties()
require.True(t, ok)
require.Equal(t, utils.ManagerKindRepo, manager.Kind)
require.Equal(t, "test", manager.Identity)
source, ok := meta.GetSourceProperties()
require.True(t, ok)
require.Equal(t, "a/b/c", source.Path)
require.Equal(t, "zzz", source.Checksum)
})
t.Run("blob info", func(t *testing.T) {
@ -391,14 +398,16 @@ func TestMetaAccessor(t *testing.T) {
meta, err := utils.MetaAccessor(obj)
require.NoError(t, err)
meta.SetRepositoryInfo(repoInfo)
meta.SetManagerProperties(repoInfo)
meta.SetSourceProperties(sourceInfo)
meta.SetFolder("folderUID")
require.Equal(t, map[string]string{
"grafana.app/repoName": "test",
"grafana.app/repoPath": "a/b/c",
"grafana.app/repoHash": "kkk",
"grafana.app/folder": "folderUID",
"grafana.app/managedBy": "repo",
"grafana.app/managerId": "test",
"grafana.app/sourcePath": "a/b/c",
"grafana.app/sourceChecksum": "kkk",
"grafana.app/folder": "folderUID",
}, obj.GetAnnotations())
require.Equal(t, "HELLO", obj.Spec.Title)
@ -414,14 +423,13 @@ func TestMetaAccessor(t *testing.T) {
meta, err = utils.MetaAccessor(obj2)
require.NoError(t, err)
meta.SetRepositoryInfo(repoInfo)
meta.SetManagerProperties(repoInfo)
meta.SetFolder("folderUID")
require.Equal(t, map[string]string{
"grafana.app/repoName": "test",
"grafana.app/repoPath": "a/b/c",
"grafana.app/repoHash": "kkk",
"grafana.app/folder": "folderUID",
"grafana.app/managedBy": "repo",
"grafana.app/managerId": "test",
"grafana.app/folder": "folderUID",
}, obj2.GetAnnotations())
require.Equal(t, "xxx", meta.FindTitle("xxx"))
@ -447,7 +455,7 @@ func TestMetaAccessor(t *testing.T) {
wantProperties: utils.ManagerProperties{
Identity: "",
Kind: utils.ManagerKindUnknown,
AllowsEdits: true,
AllowsEdits: false,
Suspended: false,
},
wantOK: false,
@ -479,7 +487,7 @@ func TestMetaAccessor(t *testing.T) {
wantProperties: utils.ManagerProperties{
Identity: "",
Kind: utils.ManagerKindUnknown,
AllowsEdits: true,
AllowsEdits: false,
Suspended: false,
},
wantOK: false,
@ -534,14 +542,14 @@ func TestMetaAccessor(t *testing.T) {
{
name: "set and get valid values",
setProperties: &utils.SourceProperties{
Path: "path",
Checksum: "hash",
Timestamp: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC),
Path: "path",
Checksum: "hash",
TimestampMillis: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC).UnixMilli(),
},
wantProperties: utils.SourceProperties{
Path: "path",
Checksum: "hash",
Timestamp: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC),
Path: "path",
Checksum: "hash",
TimestampMillis: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC).UnixMilli(),
},
wantOK: true,
},

42
pkg/apis/dashboard/utils.go
View File

@ -1,51 +1,31 @@
package dashboard
import (
"strings"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
var PluginIDRepoName = "plugin"
var fileProvisionedRepoPrefix = "file:"
// ProvisionedFileNameWithPrefix adds the `file:` prefix to the
// provisioner name, to be used as the annotation for dashboards
// provisioned from files
func ProvisionedFileNameWithPrefix(name string) string {
if name == "" {
return ""
}
return fileProvisionedRepoPrefix + name
}
// GetProvisionedFileNameFromMeta returns the provisioner name
// from a given annotation string, which is in the form file:<name>
func GetProvisionedFileNameFromMeta(annotation string) (string, bool) {
return strings.CutPrefix(annotation, fileProvisionedRepoPrefix)
}
// SetPluginIDMeta sets the repo name to "plugin" and the path to the plugin ID
func SetPluginIDMeta(obj unstructured.Unstructured, pluginID string) {
func SetPluginIDMeta(obj *unstructured.Unstructured, pluginID string) {
if pluginID == "" {
return
}
annotations := obj.GetAnnotations()
if annotations == nil {
annotations = map[string]string{}
meta, err := utils.MetaAccessor(obj)
if err == nil {
meta.SetManagerProperties(utils.ManagerProperties{
Kind: utils.ManagerKindPlugin,
Identity: pluginID,
})
}
annotations[utils.AnnoKeyRepoName] = PluginIDRepoName
annotations[utils.AnnoKeyRepoPath] = pluginID
obj.SetAnnotations(annotations)
}
// GetPluginIDFromMeta returns the plugin ID from the meta if the repo name is "plugin"
func GetPluginIDFromMeta(obj utils.GrafanaMetaAccessor) string {
if obj.GetRepositoryName() == PluginIDRepoName {
return obj.GetRepositoryPath()
p, ok := obj.GetManagerProperties()
if ok && p.Kind == utils.ManagerKindPlugin {
return p.Identity
}
return ""
}

24
pkg/registry/apis/dashboard/legacy/sql_dashboards.go
View File

@ -317,13 +317,6 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRo
}
if origin_name.String != "" {
ts := time.Unix(origin_ts.Int64, 0)
repo := &utils.ResourceRepositoryInfo{
Name: dashboardOG.ProvisionedFileNameWithPrefix(origin_name.String),
Hash: origin_hash.String,
Timestamp: &ts,
}
// if the reader cannot be found, it may be an orphaned provisioned dashboard
resolvedPath := a.provisioning.GetDashboardProvisionerResolvedPath(origin_name.String)
if resolvedPath != "" {
@ -334,13 +327,20 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRo
if err != nil {
return nil, err
}
repo.Path = originPath
meta.SetSourceProperties(utils.SourceProperties{
Path: originPath, // relative path within source
Checksum: origin_hash.String,
TimestampMillis: origin_ts.Int64,
})
meta.SetManagerProperties(utils.ManagerProperties{
Kind: utils.ManagerKindClassicFP, // nolint:staticcheck
Identity: origin_name.String,
})
}
meta.SetRepositoryInfo(repo)
} else if plugin_id.String != "" {
meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{
Name: dashboardOG.PluginIDRepoName,
Path: plugin_id.String,
meta.SetManagerProperties(utils.ManagerProperties{
Kind: utils.ManagerKindPlugin,
Identity: plugin_id.String,
})
}

25
pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go
View File

@ -83,12 +83,18 @@ func TestScanRow(t *testing.T) {
meta, err := utils.MetaAccessor(row.Dash)
require.NoError(t, err)
require.Equal(t, "file:provisioner", meta.GetRepositoryName()) // should be prefixed by file:
require.Equal(t, "../"+pathToFile, meta.GetRepositoryPath()) // relative to provisioner
require.Equal(t, "hashing", meta.GetRepositoryHash())
ts, err := meta.GetRepositoryTimestamp()
m, ok := meta.GetManagerProperties()
require.True(t, ok)
s, ok := meta.GetSourceProperties()
require.True(t, ok)
require.Equal(t, utils.ManagerKindClassicFP, m.Kind) // nolint:staticcheck
require.Equal(t, "provisioner", m.Identity)
require.Equal(t, "../"+pathToFile, s.Path) // relative to provisioner
require.Equal(t, "hashing", s.Checksum)
require.NoError(t, err)
require.Equal(t, int64(100000), ts.Unix())
require.Equal(t, int64(100000), s.TimestampMillis)
})
t.Run("Plugin provisioned dashboard should have annotations", func(t *testing.T) {
@ -105,8 +111,11 @@ func TestScanRow(t *testing.T) {
meta, err := utils.MetaAccessor(row.Dash)
require.NoError(t, err)
require.Equal(t, "plugin", meta.GetRepositoryName())
require.Equal(t, "slo", meta.GetRepositoryPath()) // the ID of the plugin
require.Equal(t, "", meta.GetRepositoryHash()) // hash is not used on plugins
manager, ok := meta.GetManagerProperties()
require.True(t, ok)
require.Equal(t, utils.ManagerKindPlugin, manager.Kind)
require.Equal(t, "slo", manager.Identity) // the ID of the plugin
require.Equal(t, "", meta.GetAnnotations()[utils.AnnoKeySourceChecksum]) // hash is not used on plugins
})
}

40
pkg/registry/apis/dashboard/legacysearcher/search_client.go
View File

@ -13,7 +13,6 @@ import (
claims "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
dashboardOG "github.com/grafana/grafana/pkg/apis/dashboard"
dashboard "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
folderv0alpha1 "github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
"github.com/grafana/grafana/pkg/services/dashboards"
@ -181,18 +180,22 @@ func (c *DashboardSearchClient) Search(ctx context.Context, req *resource.Resour
}
query.FolderUIDs = folders
case resource.SEARCH_FIELD_REPOSITORY_PATH:
case resource.SEARCH_FIELD_SOURCE_PATH:
// only one value is supported in legacy search
if len(vals) != 1 {
return nil, fmt.Errorf("only one repo path query is supported")
}
query.ProvisionedPath = vals[0]
case resource.SEARCH_FIELD_REPOSITORY_NAME:
query.SourcePath = vals[0]
case resource.SEARCH_FIELD_MANAGER_KIND:
if len(vals) != 1 {
return nil, fmt.Errorf("only one manager kind supported")
}
query.ManagedBy = utils.ManagerKind(vals[0])
case resource.SEARCH_FIELD_MANAGER_ID:
if field.Operator == string(selection.NotIn) {
for _, val := range vals {
name, _ := dashboardOG.GetProvisionedFileNameFromMeta(val)
query.ProvisionedReposNotIn = append(query.ProvisionedReposNotIn, name)
}
query.ManagerIdentityNotIn = vals
continue
}
@ -200,8 +203,7 @@ func (c *DashboardSearchClient) Search(ctx context.Context, req *resource.Resour
if len(vals) != 1 {
return nil, fmt.Errorf("only one repo name is supported")
}
query.ProvisionedRepo, _ = dashboardOG.GetProvisionedFileNameFromMeta(vals[0])
query.ManagerIdentity = vals[0]
}
}
searchFields := resource.StandardSearchFields()
@ -221,17 +223,21 @@ func (c *DashboardSearchClient) Search(ctx context.Context, req *resource.Resour
// if we are querying for provisioning information, we need to use a different
// legacy sql query, since legacy search does not support this
if query.ProvisionedRepo != "" || len(query.ProvisionedReposNotIn) > 0 {
if query.ManagerIdentity != "" || len(query.ManagerIdentityNotIn) > 0 {
if query.ManagedBy == utils.ManagerKindUnknown {
return nil, fmt.Errorf("query by manager identity also requires manager.kind parameter")
}
var dashes []*dashboards.Dashboard
if query.ProvisionedRepo == dashboardOG.PluginIDRepoName {
if query.ManagedBy == utils.ManagerKindPlugin {
dashes, err = c.dashboardStore.GetDashboardsByPluginID(ctx, &dashboards.GetDashboardsByPluginIDQuery{
PluginID: query.ProvisionedPath,
PluginID: query.ManagerIdentity,
OrgID: user.GetOrgID(),
})
} else if query.ProvisionedRepo != "" {
dashes, err = c.dashboardStore.GetProvisionedDashboardsByName(ctx, query.ProvisionedRepo)
} else if len(query.ProvisionedReposNotIn) > 0 {
dashes, err = c.dashboardStore.GetOrphanedProvisionedDashboards(ctx, query.ProvisionedReposNotIn)
} else if query.ManagerIdentity != "" {
dashes, err = c.dashboardStore.GetProvisionedDashboardsByName(ctx, query.ManagerIdentity)
} else if len(query.ManagerIdentityNotIn) > 0 {
dashes, err = c.dashboardStore.GetOrphanedProvisionedDashboards(ctx, query.ManagerIdentityNotIn)
}
if err != nil {
return nil, err

22
pkg/registry/apis/dashboard/legacysearcher/search_client_test.go
View File

@ -373,12 +373,12 @@ func TestDashboardSearchClient_Search(t *testing.T) {
Key: dashboardKey,
Fields: []*resource.Requirement{
{
Key: resource.SEARCH_FIELD_REPOSITORY_PATH,
Key: resource.SEARCH_FIELD_MANAGER_ID,
Operator: "in",
Values: []string{"slo"},
},
{
Key: resource.SEARCH_FIELD_REPOSITORY_NAME,
Key: resource.SEARCH_FIELD_MANAGER_KIND,
Operator: "in",
Values: []string{"plugin"},
},
@ -402,9 +402,14 @@ func TestDashboardSearchClient_Search(t *testing.T) {
Key: dashboardKey,
Fields: []*resource.Requirement{
{
Key: resource.SEARCH_FIELD_REPOSITORY_NAME,
Key: resource.SEARCH_FIELD_MANAGER_KIND,
Operator: "=",
Values: []string{string(utils.ManagerKindClassicFP)}, // nolint:staticcheck
},
{
Key: resource.SEARCH_FIELD_MANAGER_ID,
Operator: "in",
Values: []string{"file:test"}, // file prefix should be removed before going to legacy
Values: []string{"test"},
},
},
},
@ -426,9 +431,14 @@ func TestDashboardSearchClient_Search(t *testing.T) {
Key: dashboardKey,
Fields: []*resource.Requirement{
{
Key: resource.SEARCH_FIELD_REPOSITORY_NAME,
Key: resource.SEARCH_FIELD_MANAGER_KIND,
Operator: "=",
Values: []string{string(utils.ManagerKindClassicFP)}, // nolint:staticcheck
},
{
Key: resource.SEARCH_FIELD_MANAGER_ID,
Operator: string(selection.NotIn),
Values: []string{"file:test", "file:test2"}, // file prefix should be removed before going to legacy
Values: []string{"test", "test2"},
},
},
},

10
pkg/registry/apis/dashboard/sub_dto.go
View File

@ -134,13 +134,9 @@ func (r *DTOConnector) Connect(ctx context.Context, name string, opts runtime.Ob
OrgID: info.OrgID,
ID: obj.GetDeprecatedInternalID(), // nolint:staticcheck
}
repo, err := obj.GetRepositoryInfo()
if err != nil {
responder.Error(err)
return
}
if repo != nil && repo.Name == dashboard.PluginIDRepoName {
dto.PluginID = repo.Path
manager, ok := obj.GetManagerProperties()
if ok && manager.Kind == utils.ManagerKindPlugin {
dto.PluginID = manager.Identity
}
guardian, err := guardian.NewByDashboard(ctx, dto, info.OrgID, user)

8
pkg/services/dashboards/models.go
View File

@ -5,6 +5,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/infra/slugify"
@ -438,9 +439,10 @@ type FindPersistedDashboardsQuery struct {
Sort model.SortOption
IsDeleted bool
ProvisionedRepo string
ProvisionedPath string
ProvisionedReposNotIn []string
ManagedBy utils.ManagerKind
ManagerIdentity string
SourcePath string
ManagerIdentityNotIn []string
Filters []any

139
pkg/services/dashboards/service/dashboard_service.go
View File

@ -237,7 +237,8 @@ func (dr *DashboardServiceImpl) GetProvisionedDashboardData(ctx context.Context,
func(orgID int64) {
g.Go(func() error {
res, err := dr.searchProvisionedDashboardsThroughK8s(ctx, &dashboards.FindPersistedDashboardsQuery{
ProvisionedRepo: name,
ManagedBy: utils.ManagerKindClassicFP, // nolint:staticcheck
ManagerIdentity: name,
OrgId: orgID,
})
if err != nil {
@ -568,8 +569,8 @@ func (dr *DashboardServiceImpl) DeleteOrphanedProvisionedDashboards(ctx context.
ctx, _ := identity.WithServiceIdentity(ctx, org.ID)
// find all dashboards in the org that have a file repo set that is not in the given readers list
foundDashs, err := dr.searchProvisionedDashboardsThroughK8s(ctx, &dashboards.FindPersistedDashboardsQuery{
ProvisionedReposNotIn: cmd.ReaderNames,
OrgId: org.ID,
ManagerIdentityNotIn: cmd.ReaderNames,
OrgId: org.ID,
})
if err != nil {
return err
@ -962,8 +963,8 @@ func (dr *DashboardServiceImpl) GetDashboardsByPluginID(ctx context.Context, que
if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesClientDashboardsFolders) {
dashs, err := dr.searchDashboardsThroughK8s(ctx, &dashboards.FindPersistedDashboardsQuery{
OrgId: query.OrgID,
ProvisionedRepo: dashboard.PluginIDRepoName,
ProvisionedPath: query.PluginID,
ManagedBy: utils.ManagerKindPlugin,
ManagerIdentity: query.PluginID,
})
if err != nil {
return nil, err
@ -1553,22 +1554,22 @@ func (dr *DashboardServiceImpl) saveProvisionedDashboardThroughK8s(ctx context.C
return nil, err
}
annotations := obj.GetAnnotations()
if annotations == nil {
annotations = map[string]string{}
meta, err := utils.MetaAccessor(obj)
if err != nil {
return nil, err
}
if unprovision {
delete(annotations, utils.AnnoKeyRepoName)
delete(annotations, utils.AnnoKeyRepoPath)
delete(annotations, utils.AnnoKeyRepoHash)
delete(annotations, utils.AnnoKeyRepoTimestamp)
} else {
annotations[utils.AnnoKeyRepoName] = dashboard.ProvisionedFileNameWithPrefix(provisioning.Name)
annotations[utils.AnnoKeyRepoPath] = provisioning.ExternalID
annotations[utils.AnnoKeyRepoHash] = provisioning.CheckSum
annotations[utils.AnnoKeyRepoTimestamp] = time.Unix(provisioning.Updated, 0).UTC().Format(time.RFC3339)
m := utils.ManagerProperties{}
s := utils.SourceProperties{}
if !unprovision {
m.Kind = utils.ManagerKindClassicFP // nolint:staticcheck
m.Identity = provisioning.Name
s.Path = provisioning.ExternalID
s.Checksum = provisioning.CheckSum
s.TimestampMillis = time.Unix(provisioning.Updated, 0).UnixMilli()
}
obj.SetAnnotations(annotations)
meta.SetManagerProperties(m)
meta.SetSourceProperties(s)
out, err := dr.createOrUpdateDash(ctx, obj, cmd.OrgID)
if err != nil {
@ -1594,16 +1595,16 @@ func (dr *DashboardServiceImpl) saveDashboardThroughK8s(ctx context.Context, cmd
return out, nil
}
func (dr *DashboardServiceImpl) createOrUpdateDash(ctx context.Context, obj unstructured.Unstructured, orgID int64) (*dashboards.Dashboard, error) {
func (dr *DashboardServiceImpl) createOrUpdateDash(ctx context.Context, obj *unstructured.Unstructured, orgID int64) (*dashboards.Dashboard, error) {
var out *unstructured.Unstructured
current, err := dr.k8sclient.Get(ctx, obj.GetName(), orgID, v1.GetOptions{})
if current == nil || err != nil {
out, err = dr.k8sclient.Create(ctx, &obj, orgID)
out, err = dr.k8sclient.Create(ctx, obj, orgID)
if err != nil {
return nil, err
}
} else {
out, err = dr.k8sclient.Update(ctx, &obj, orgID)
out, err = dr.k8sclient.Update(ctx, obj, orgID)
if err != nil {
return nil, err
}
@ -1723,30 +1724,35 @@ func (dr *DashboardServiceImpl) searchDashboardsThroughK8sRaw(ctx context.Contex
})
}
if query.ProvisionedRepo != "" {
req := []*resource.Requirement{{
Key: resource.SEARCH_FIELD_REPOSITORY_NAME,
Operator: string(selection.In),
Values: []string{query.ProvisionedRepo},
}}
request.Options.Fields = append(request.Options.Fields, req...)
if query.ManagedBy != "" {
request.Options.Fields = append(request.Options.Fields, &resource.Requirement{
Key: resource.SEARCH_FIELD_MANAGER_KIND,
Operator: string(selection.Equals),
Values: []string{string(query.ManagedBy)},
})
}
if len(query.ProvisionedReposNotIn) > 0 {
req := []*resource.Requirement{{
Key: resource.SEARCH_FIELD_REPOSITORY_NAME,
Operator: string(selection.NotIn),
Values: query.ProvisionedReposNotIn,
}}
request.Options.Fields = append(request.Options.Fields, req...)
}
if query.ProvisionedPath != "" {
req := []*resource.Requirement{{
Key: resource.SEARCH_FIELD_REPOSITORY_PATH,
if query.ManagerIdentity != "" {
request.Options.Fields = append(request.Options.Fields, &resource.Requirement{
Key: resource.SEARCH_FIELD_MANAGER_ID,
Operator: string(selection.In),
Values: []string{query.ProvisionedPath},
}}
request.Options.Fields = append(request.Options.Fields, req...)
Values: []string{query.ManagerIdentity},
})
}
if len(query.ManagerIdentityNotIn) > 0 {
request.Options.Fields = append(request.Options.Fields, &resource.Requirement{
Key: resource.SEARCH_FIELD_MANAGER_ID,
Operator: string(selection.NotIn),
Values: query.ManagerIdentityNotIn,
})
}
if query.SourcePath != "" {
request.Options.Fields = append(request.Options.Fields, &resource.Requirement{
Key: resource.SEARCH_FIELD_SOURCE_PATH,
Operator: string(selection.In),
Values: []string{query.SourcePath},
})
}
if query.Title != "" {
@ -1840,18 +1846,6 @@ func (dr *DashboardServiceImpl) searchProvisionedDashboardsThroughK8s(ctx contex
ctx, _ = identity.WithServiceIdentity(ctx, query.OrgId)
if query.ProvisionedRepo != "" {
query.ProvisionedRepo = dashboard.ProvisionedFileNameWithPrefix(query.ProvisionedRepo)
}
if len(query.ProvisionedReposNotIn) > 0 {
repos := make([]string, len(query.ProvisionedReposNotIn))
for i, v := range query.ProvisionedReposNotIn {
repos[i] = dashboard.ProvisionedFileNameWithPrefix(v)
}
query.ProvisionedReposNotIn = repos
}
query.Type = searchstore.TypeDashboard
searchResults, err := dr.searchDashboardsThroughK8sRaw(ctx, query)
@ -1878,26 +1872,27 @@ func (dr *DashboardServiceImpl) searchProvisionedDashboardsThroughK8s(ctx contex
return err
}
// ensure the repo is set due to file provisioning, otherwise skip it
fileRepo, found := dashboard.GetProvisionedFileNameFromMeta(meta.GetRepositoryName())
if !found {
m, ok := meta.GetManagerProperties()
if !ok || m.Kind != utils.ManagerKindClassicFP { // nolint:staticcheck
return nil
}
source, ok := meta.GetSourceProperties()
if !ok {
return nil
}
provisioning := &dashboardProvisioningWithUID{
DashboardProvisioning: dashboards.DashboardProvisioning{
Name: m.Identity,
ExternalID: source.Path,
CheckSum: source.Checksum,
DashboardID: meta.GetDeprecatedInternalID(), // nolint:staticcheck
},
DashboardUID: hit.Name,
}
provisioning.Name = fileRepo
provisioning.ExternalID = meta.GetRepositoryPath()
provisioning.CheckSum = meta.GetRepositoryHash()
provisioning.DashboardID = meta.GetDeprecatedInternalID() // nolint:staticcheck
updated, err := meta.GetRepositoryTimestamp()
if err != nil {
return err
}
if updated != nil {
provisioning.Updated = updated.Unix()
if source.TimestampMillis > 0 {
provisioning.Updated = time.UnixMilli(source.TimestampMillis).Unix()
}
mu.Lock()
@ -2027,13 +2022,13 @@ func (dr *DashboardServiceImpl) UnstructuredToLegacyDashboard(ctx context.Contex
return &out, nil
}
func LegacySaveCommandToUnstructured(cmd *dashboards.SaveDashboardCommand, namespace string) (unstructured.Unstructured, error) {
func LegacySaveCommandToUnstructured(cmd *dashboards.SaveDashboardCommand, namespace string) (*unstructured.Unstructured, error) {
uid := cmd.GetDashboardModel().UID
if uid == "" {
uid = uuid.NewString()
}
finalObj := unstructured.Unstructured{
finalObj := &unstructured.Unstructured{
Object: map[string]interface{}{},
}
@ -2061,7 +2056,7 @@ func LegacySaveCommandToUnstructured(cmd *dashboards.SaveDashboardCommand, names
finalObj.SetNamespace(namespace)
finalObj.SetGroupVersionKind(dashboardv0alpha1.DashboardResourceInfo.GroupVersionKind())
meta, err := utils.MetaAccessor(&finalObj)
meta, err := utils.MetaAccessor(finalObj)
if err != nil {
return finalObj, err
}

156
pkg/services/dashboards/service/dashboard_service_test.go
View File

@ -2,6 +2,7 @@ package service
import (
"context"
"fmt"
"reflect"
"testing"
"time"
@ -9,13 +10,12 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/apis/dashboard"
dashboardv0alpha1 "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/apiserver/client"
@ -540,31 +540,38 @@ func TestGetProvisionedDashboardData(t *testing.T) {
t.Run("Should use Kubernetes client if feature flags are enabled and get from relevant org", func(t *testing.T) {
ctx, k8sCliMock := setupK8sDashboardTests(service)
provisioningTimestamp := int64(1234567)
k8sCliMock.On("GetNamespace", mock.Anything, mock.Anything).Return("default")
k8sCliMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
"metadata": map[string]any{
"name": "uid",
"labels": map[string]any{
utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck
k8sCliMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": dashboardv0alpha1.DashboardResourceInfo.GroupVersion().String(),
"kind": dashboardv0alpha1.DashboardResourceInfo.GroupVersionKind().Kind,
"metadata": map[string]interface{}{
"name": "uid",
"labels": map[string]interface{}{
utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck
},
"annotations": map[string]interface{}{
utils.AnnoKeyManagerKind: string(utils.ManagerKindClassicFP), // nolint:staticcheck
utils.AnnoKeyManagerIdentity: "test",
utils.AnnoKeySourceChecksum: "hash",
utils.AnnoKeySourcePath: "path/to/file",
utils.AnnoKeySourceTimestamp: fmt.Sprintf("%d", time.Unix(provisioningTimestamp, 0).UnixMilli()),
},
},
"annotations": map[string]any{
utils.AnnoKeyRepoName: dashboard.ProvisionedFileNameWithPrefix("test"),
utils.AnnoKeyRepoHash: "hash",
utils.AnnoKeyRepoPath: "path/to/file",
utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
"spec": map[string]interface{}{
"test": "test",
"version": int64(1),
"title": "testing slugify",
},
},
"spec": map[string]any{
"test": "test",
"version": int64(1),
"title": "testing slugify",
},
}}, nil).Once()
}, nil).Once()
repo := "test"
k8sCliMock.On("Search", mock.Anything, int64(1),
mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
// ensure the prefix is added to the query
return req.Options.Fields[0].Values[0] == dashboard.ProvisionedFileNameWithPrefix(repo)
// make sure the kind is added to the query
return req.Options.Fields[0].Values[0] == string(utils.ManagerKindClassicFP) && // nolint:staticcheck
req.Options.Fields[1].Values[0] == repo
})).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{},
@ -573,8 +580,9 @@ func TestGetProvisionedDashboardData(t *testing.T) {
TotalHits: 0,
}, nil).Once()
k8sCliMock.On("Search", mock.Anything, int64(2), mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
// ensure the prefix is added to the query
return req.Options.Fields[0].Values[0] == dashboard.ProvisionedFileNameWithPrefix(repo)
// make sure the kind is added to the query
return req.Options.Fields[0].Values[0] == string(utils.ManagerKindClassicFP) && // nolint:staticcheck
req.Options.Fields[1].Values[0] == repo
})).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
@ -611,7 +619,7 @@ func TestGetProvisionedDashboardData(t *testing.T) {
Name: "test",
ExternalID: "path/to/file",
CheckSum: "hash",
Updated: 1735689600,
Updated: provisioningTimestamp,
})
k8sCliMock.AssertExpectations(t)
})
@ -639,21 +647,25 @@ func TestGetProvisionedDashboardDataByDashboardID(t *testing.T) {
t.Run("Should use Kubernetes client if feature flags are enabled and get from whatever org it is in", func(t *testing.T) {
ctx, k8sCliMock := setupK8sDashboardTests(service)
provisioningTimestamp := int64(1234567)
k8sCliMock.On("GetNamespace", mock.Anything, mock.Anything).Return("default")
k8sCliMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
"metadata": map[string]any{
k8sCliMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]interface{}{
"apiVersion": dashboardv0alpha1.DashboardResourceInfo.GroupVersion().String(),
"kind": dashboardv0alpha1.DashboardResourceInfo.GroupVersionKind().Kind,
"metadata": map[string]interface{}{
"name": "uid",
"labels": map[string]any{
"labels": map[string]interface{}{
utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck
},
"annotations": map[string]any{
utils.AnnoKeyRepoName: dashboard.ProvisionedFileNameWithPrefix("test"),
utils.AnnoKeyRepoHash: "hash",
utils.AnnoKeyRepoPath: "path/to/file",
utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
"annotations": map[string]interface{}{
utils.AnnoKeyManagerKind: string(utils.ManagerKindClassicFP), // nolint:staticcheck
utils.AnnoKeyManagerIdentity: "test",
utils.AnnoKeySourceChecksum: "hash",
utils.AnnoKeySourcePath: "path/to/file",
utils.AnnoKeySourceTimestamp: fmt.Sprintf("%d", time.Unix(provisioningTimestamp, 0).UnixMilli()),
},
},
"spec": map[string]any{
"spec": map[string]interface{}{
"test": "test",
"version": int64(1),
"title": "testing slugify",
@ -701,7 +713,7 @@ func TestGetProvisionedDashboardDataByDashboardID(t *testing.T) {
Name: "test",
ExternalID: "path/to/file",
CheckSum: "hash",
Updated: 1735689600,
Updated: provisioningTimestamp,
})
k8sCliMock.AssertExpectations(t)
})
@ -729,21 +741,25 @@ func TestGetProvisionedDashboardDataByDashboardUID(t *testing.T) {
t.Run("Should use Kubernetes client if feature flags are enabled", func(t *testing.T) {
ctx, k8sCliMock := setupK8sDashboardTests(service)
provisioningTimestamp := int64(1234567)
k8sCliMock.On("GetNamespace", mock.Anything, mock.Anything).Return("default")
k8sCliMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
"metadata": map[string]any{
k8sCliMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]interface{}{
"apiVersion": dashboardv0alpha1.DashboardResourceInfo.GroupVersion().String(),
"kind": dashboardv0alpha1.DashboardResourceInfo.GroupVersionKind().Kind,
"metadata": map[string]interface{}{
"name": "uid",
"labels": map[string]any{
"labels": map[string]interface{}{
utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck
},
"annotations": map[string]any{
utils.AnnoKeyRepoName: dashboard.ProvisionedFileNameWithPrefix("test"),
utils.AnnoKeyRepoHash: "hash",
utils.AnnoKeyRepoPath: "path/to/file",
utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
"annotations": map[string]interface{}{
utils.AnnoKeyManagerKind: string(utils.ManagerKindClassicFP), // nolint:staticcheck
utils.AnnoKeyManagerIdentity: "test",
utils.AnnoKeySourceChecksum: "hash",
utils.AnnoKeySourcePath: "path/to/file",
utils.AnnoKeySourceTimestamp: fmt.Sprintf("%d", time.Unix(provisioningTimestamp, 0).UnixMilli()),
},
},
"spec": map[string]any{
"spec": map[string]interface{}{
"test": "test",
"version": int64(1),
"title": "testing slugify",
@ -784,7 +800,7 @@ func TestGetProvisionedDashboardDataByDashboardUID(t *testing.T) {
Name: "test",
ExternalID: "path/to/file",
CheckSum: "hash",
Updated: 1735689600,
Updated: provisioningTimestamp,
})
k8sCliMock.AssertExpectations(t)
})
@ -826,10 +842,11 @@ func TestDeleteOrphanedProvisionedDashboards(t *testing.T) {
"metadata": map[string]any{
"name": "uid",
"annotations": map[string]any{
utils.AnnoKeyRepoName: dashboard.ProvisionedFileNameWithPrefix("orphaned"),
utils.AnnoKeyRepoHash: "hash",
utils.AnnoKeyRepoPath: "path/to/file",
utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
utils.AnnoKeyManagerKind: string(utils.ManagerKindClassicFP), // nolint:staticcheck
utils.AnnoKeyManagerIdentity: "orphaned",
utils.AnnoKeySourceChecksum: "hash",
utils.AnnoKeySourcePath: "path/to/file",
utils.AnnoKeySourceTimestamp: "2025-01-01T00:00:00Z",
},
},
"spec": map[string]any{},
@ -839,8 +856,8 @@ func TestDeleteOrphanedProvisionedDashboards(t *testing.T) {
"metadata": map[string]any{
"name": "uid2",
"annotations": map[string]any{
utils.AnnoKeyRepoName: dashboard.PluginIDRepoName,
utils.AnnoKeyRepoHash: "app",
utils.AnnoKeyManagerKind: string(utils.ManagerKindPlugin),
utils.AnnoKeyManagerIdentity: "app",
},
},
"spec": map[string]any{},
@ -850,16 +867,17 @@ func TestDeleteOrphanedProvisionedDashboards(t *testing.T) {
"metadata": map[string]any{
"name": "uid3",
"annotations": map[string]any{
utils.AnnoKeyRepoName: dashboard.ProvisionedFileNameWithPrefix("orphaned"),
utils.AnnoKeyRepoHash: "hash",
utils.AnnoKeyRepoPath: "path/to/file",
utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
utils.AnnoKeyManagerKind: string(utils.ManagerKindClassicFP), // nolint:staticcheck
utils.AnnoKeyManagerIdentity: "orphaned",
utils.AnnoKeySourceChecksum: "hash",
utils.AnnoKeySourcePath: "path/to/file",
utils.AnnoKeySourceTimestamp: "2025-01-01T00:00:00Z",
},
},
"spec": map[string]any{},
}}, nil).Once()
k8sCliMock.On("Search", mock.Anything, int64(1), mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
return req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == dashboard.ProvisionedFileNameWithPrefix("test") && req.Options.Fields[0].Operator == "notin"
return req.Options.Fields[0].Key == "manager.id" && req.Options.Fields[0].Values[0] == "test" && req.Options.Fields[0].Operator == "notin"
})).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
@ -889,7 +907,7 @@ func TestDeleteOrphanedProvisionedDashboards(t *testing.T) {
}, nil).Once()
k8sCliMock.On("Search", mock.Anything, int64(2), mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
return req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == dashboard.ProvisionedFileNameWithPrefix("test") && req.Options.Fields[0].Operator == "notin"
return req.Options.Fields[0].Key == "manager.id" && req.Options.Fields[0].Values[0] == "test" && req.Options.Fields[0].Operator == "notin"
})).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
@ -960,10 +978,11 @@ func TestUnprovisionDashboard(t *testing.T) {
"metadata": map[string]any{
"name": "uid",
"annotations": map[string]any{
utils.AnnoKeyRepoName: dashboard.ProvisionedFileNameWithPrefix("test"),
utils.AnnoKeyRepoHash: "hash",
utils.AnnoKeyRepoPath: "path/to/file",
utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
utils.AnnoKeyManagerKind: utils.ManagerKindClassicFP, // nolint:staticcheck
utils.AnnoKeyManagerIdentity: "test",
utils.AnnoKeySourceChecksum: "hash",
utils.AnnoKeySourcePath: "path/to/file",
utils.AnnoKeySourceTimestamp: "2025-01-01T00:00:00Z",
},
},
"spec": map[string]any{},
@ -983,7 +1002,7 @@ func TestUnprovisionDashboard(t *testing.T) {
},
}}
// should update it to be without annotations
k8sCliMock.On("Update", mock.Anything, dashWithoutAnnotations, mock.Anything, mock.Anything).Return(dashWithoutAnnotations, nil)
k8sCliMock.On("Update", mock.Anything, dashWithoutAnnotations, mock.Anything).Return(dashWithoutAnnotations, nil)
k8sCliMock.On("GetNamespace", mock.Anything).Return("default")
k8sCliMock.On("GetUserFromMeta", mock.Anything, mock.Anything).Return(&user.User{}, nil)
k8sCliMock.On("Search", mock.Anything, mock.Anything, mock.Anything).Return(&resource.ResourceSearchResponse{
@ -1054,8 +1073,9 @@ func TestGetDashboardsByPluginID(t *testing.T) {
k8sCliMock.On("Get", mock.Anything, "uid", mock.Anything, mock.Anything, mock.Anything).Return(uidUnstructured, nil)
k8sCliMock.On("GetUserFromMeta", mock.Anything, mock.Anything).Return(&user.User{}, nil)
k8sCliMock.On("Search", mock.Anything, mock.Anything, mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
return req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == dashboard.PluginIDRepoName &&
req.Options.Fields[1].Key == "repo.path" && req.Options.Fields[1].Values[0] == "testing"
return ( // gofmt comment helper
req.Options.Fields[0].Key == "manager.kind" && req.Options.Fields[0].Values[0] == string(utils.ManagerKindPlugin) &&
req.Options.Fields[1].Key == "manager.id" && req.Options.Fields[1].Values[0] == "testing")
})).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
@ -1989,14 +2009,16 @@ func TestSearchProvisionedDashboardsThroughK8sRaw(t *testing.T) {
query := &dashboards.FindPersistedDashboardsQuery{
OrgId: 1,
}
provisioningTimestamp := int64(1234567)
dashboardUnstructuredProvisioned := unstructured.Unstructured{Object: map[string]any{
"metadata": map[string]any{
"name": "uid",
"annotations": map[string]any{
utils.AnnoKeyRepoName: dashboard.ProvisionedFileNameWithPrefix("test"),
utils.AnnoKeyRepoHash: "hash",
utils.AnnoKeyRepoPath: "path/to/file",
utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
utils.AnnoKeyManagerKind: string(utils.ManagerKindClassicFP), // nolint:staticcheck
utils.AnnoKeyManagerIdentity: "test",
utils.AnnoKeySourceChecksum: "hash",
utils.AnnoKeySourcePath: "path/to/file",
utils.AnnoKeySourceTimestamp: fmt.Sprintf("%d", time.Unix(provisioningTimestamp, 0).UnixMilli()),
},
},
"spec": map[string]any{},
@ -2056,7 +2078,7 @@ func TestSearchProvisionedDashboardsThroughK8sRaw(t *testing.T) {
Name: "test",
ExternalID: "path/to/file",
CheckSum: "hash",
Updated: 1735689600,
Updated: provisioningTimestamp,
},
},
}, res) // only should return the one provisioned dashboard

9
pkg/services/dashboards/service/search/search.go
View File

@ -29,10 +29,11 @@ var (
resource.SEARCH_FIELD_CREATED_BY,
resource.SEARCH_FIELD_UPDATED,
resource.SEARCH_FIELD_UPDATED_BY,
resource.SEARCH_FIELD_REPOSITORY_NAME,
resource.SEARCH_FIELD_REPOSITORY_PATH,
resource.SEARCH_FIELD_REPOSITORY_HASH,
resource.SEARCH_FIELD_REPOSITORY_TIME,
resource.SEARCH_FIELD_MANAGER_KIND,
resource.SEARCH_FIELD_MANAGER_ID,
resource.SEARCH_FIELD_SOURCE_PATH,
resource.SEARCH_FIELD_SOURCE_CHECKSUM,
resource.SEARCH_FIELD_SOURCE_TIME,
}
)

7
pkg/services/folder/folderimpl/conversions.go
View File

@ -6,13 +6,14 @@ import (
"strconv"
"strings"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
authlib "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/slugify"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/user"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func (ss *FolderUnifiedStoreImpl) UnstructuredToLegacyFolder(ctx context.Context, item *unstructured.Unstructured) (*folder.Folder, error) {
@ -57,7 +58,7 @@ func (ss *FolderUnifiedStoreImpl) UnstructuredToLegacyFolder(ctx context.Context
if updater.UID == "" {
updater = creator
}
manager, _ := meta.GetManagerProperties()
return &folder.Folder{
UID: uid,
Title: title,
@ -65,7 +66,7 @@ func (ss *FolderUnifiedStoreImpl) UnstructuredToLegacyFolder(ctx context.Context
ID: meta.GetDeprecatedInternalID(), // nolint:staticcheck
ParentUID: meta.GetFolder(),
Version: int(meta.GetGeneration()),
Repository: meta.GetRepositoryName(),
ManagedBy: manager.Kind,
URL: url,
Created: created,

3
pkg/services/folder/folderimpl/conversions_test.go
View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/services/user/usertest"
@ -63,7 +64,7 @@ func TestFolderConversions(t *testing.T) {
Title: "test folder",
Description: "Something set in the file",
URL: "/dashboards/f/be79sztagf20wd/test-folder",
Repository: "example-repo",
ManagedBy: utils.ManagerKindRepo,
Created: created,
Updated: created.Add(time.Hour * 5),
CreatedBy: 10,

5
pkg/services/folder/model.go
View File

@ -7,6 +7,7 @@ import (
"github.com/grafana/grafana/pkg/apimachinery/errutil"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/infra/slugify"
"github.com/grafana/grafana/pkg/services/dashboards/dashboardaccess"
@ -56,10 +57,10 @@ type Folder struct {
Fullpath string `xorm:"fullpath"`
FullpathUIDs string `xorm:"fullpath_uids"`
// When the folder belongs to a repository
// The folder is managed by an external process
// NOTE: this is only populated when folders are managed by unified storage
// This is not ever used by xorm, but the translation functions flow through this type
Repository string `json:"repository,omitempty"`
ManagedBy utils.ManagerKind `json:"managedBy,omitempty"`
}
var GeneralFolder = Folder{ID: 0, Title: "General"}

15
pkg/storage/unified/apistore/prepare.go
View File

@ -9,12 +9,13 @@ import (
"time"
"github.com/google/uuid"
authtypes "github.com/grafana/authlib/types"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/storage"
"k8s.io/klog/v2"
authtypes "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
@ -73,12 +74,6 @@ func (s *Storage) prepareObjectForStorage(ctx context.Context, newObject runtime
obj.SetResourceVersion("")
obj.SetSelfLink("")
// Read+write will verify that repository format is accurate
repo, err := obj.GetRepositoryInfo()
if err != nil {
return nil, err
}
obj.SetRepositoryInfo(repo)
obj.SetUpdatedBy("")
obj.SetUpdatedTimestamp(nil)
obj.SetCreatedBy(info.GetUID())
@ -136,12 +131,6 @@ func (s *Storage) prepareObjectForUpdate(ctx context.Context, updateObject runti
obj.SetDeprecatedInternalID(previousInternalID) // nolint:staticcheck
}
// Read+write will verify that origin format is accurate
repo, err := obj.GetRepositoryInfo()
if err != nil {
return nil, err
}
obj.SetRepositoryInfo(repo)
obj.SetUpdatedBy(info.GetUID())
obj.SetUpdatedTimestampMillis(time.Now().UnixMilli())

40
pkg/storage/unified/apistore/prepare_test.go
View File

@ -6,16 +6,17 @@ import (
"time"
"github.com/bwmarrin/snowflake"
authtypes "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
"github.com/stretchr/testify/require"
"golang.org/x/exp/rand"
"k8s.io/apimachinery/pkg/api/apitesting"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apiserver/pkg/storage"
authtypes "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
)
var scheme = runtime.NewScheme()
@ -87,11 +88,14 @@ func TestPrepareObjectForStorage(t *testing.T) {
meta, err := utils.MetaAccessor(obj)
require.NoError(t, err)
now := time.Now()
meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{
Name: "test-repo",
Path: "test/path",
Hash: "hash",
Timestamp: &now,
meta.SetManagerProperties(utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "test-repo",
})
meta.SetSourceProperties(utils.SourceProperties{
Path: "test/path",
Checksum: "hash",
TimestampMillis: now.UnixMilli(),
})
encodedData, err := s.prepareObjectForStorage(ctx, obj)
@ -101,14 +105,16 @@ func TestPrepareObjectForStorage(t *testing.T) {
require.NoError(t, err)
meta, err = utils.MetaAccessor(newObject)
require.NoError(t, err)
require.Equal(t, meta.GetRepositoryHash(), "hash")
require.Equal(t, meta.GetRepositoryName(), "test-repo")
require.Equal(t, meta.GetRepositoryPath(), "test/path")
ts, err := meta.GetRepositoryTimestamp()
require.NoError(t, err)
parsed, err := time.Parse(time.RFC3339, now.UTC().Format(time.RFC3339))
require.NoError(t, err)
require.Equal(t, ts, &parsed)
m, ok := meta.GetManagerProperties()
require.True(t, ok)
s, ok := meta.GetSourceProperties()
require.True(t, ok)
require.Equal(t, m.Identity, "test-repo")
require.Equal(t, s.Checksum, "hash")
require.Equal(t, s.Path, "test/path")
require.Equal(t, s.TimestampMillis, now.UnixMilli())
})
s.opts.RequireDeprecatedInternalID = true

23
pkg/storage/unified/resource/document.go
View File

@ -102,7 +102,10 @@ type IndexableDocument struct {
References ResourceReferences `json:"reference,omitempty"`
// When the resource is managed by an upstream repository
RepoInfo *utils.ResourceRepositoryInfo `json:"repo,omitempty"`
Manager *utils.ManagerProperties `json:"manager,omitempty"`
// When the manager knows about file paths
Source *utils.SourceProperties `json:"source,omitempty"`
}
func (m *IndexableDocument) Type() string {
@ -173,7 +176,14 @@ func NewIndexableDocument(key *ResourceKey, rv int64, obj utils.GrafanaMetaAcces
CreatedBy: obj.GetCreatedBy(),
UpdatedBy: obj.GetUpdatedBy(),
}
doc.RepoInfo, _ = obj.GetRepositoryInfo()
m, ok := obj.GetManagerProperties()
if ok {
doc.Manager = &m
}
s, ok := obj.GetSourceProperties()
if ok {
doc.Source = &s
}
ts := obj.GetCreationTimestamp()
if !ts.Time.IsZero() {
doc.Created = ts.Time.UnixMilli()
@ -265,10 +275,11 @@ const SEARCH_FIELD_CREATED_BY = "createdBy"
const SEARCH_FIELD_UPDATED = "updated"
const SEARCH_FIELD_UPDATED_BY = "updatedBy"
const SEARCH_FIELD_REPOSITORY_NAME = "repo.name"
const SEARCH_FIELD_REPOSITORY_PATH = "repo.path"
const SEARCH_FIELD_REPOSITORY_HASH = "repo.hash"
const SEARCH_FIELD_REPOSITORY_TIME = "repo.time"
const SEARCH_FIELD_MANAGER_KIND = "manager.kind"
const SEARCH_FIELD_MANAGER_ID = "manager.id"
const SEARCH_FIELD_SOURCE_PATH = "source.path"
const SEARCH_FIELD_SOURCE_CHECKSUM = "source.checksum"
const SEARCH_FIELD_SOURCE_TIME = "source.timestampMillis"
const SEARCH_FIELD_SCORE = "_score" // the match score
const SEARCH_FIELD_EXPLAIN = "_explain" // score explanation as JSON object

13
pkg/storage/unified/resource/document_test.go
View File

@ -33,17 +33,20 @@ func TestStandardDocumentBuilder(t *testing.T) {
"resource": "playlists",
"name": "test1"
},
"name": "test1",
"rv": 10,
"title": "test playlist unified storage",
"title_phrase": "test playlist unified storage",
"created": 1717236672000,
"createdBy": "user:ABC",
"updatedBy": "user:XYZ",
"name": "test1",
"repo": {
"name": "something",
"manager": {
"kind": "repo",
"id": "something"
},
"source": {
"path": "path/in/system.json",
"hash": "xyz"
"checksum": "xyz"
}
}`, string(jj))
}`, string(jj))
}

9
pkg/storage/unified/resource/server.go
View File

@ -456,12 +456,9 @@ func (s *server) newEvent(ctx context.Context, user claims.AuthInfo, key *Resour
}
}
repo, err := obj.GetRepositoryInfo()
if err != nil {
return nil, NewBadRequestError("invalid repository info")
}
if repo != nil {
err = s.writeHooks.CanWriteValueFromRepository(ctx, user, repo.Name)
m, ok := obj.GetManagerProperties()
if ok && m.Kind == utils.ManagerKindRepo {
err = s.writeHooks.CanWriteValueFromRepository(ctx, user, m.Identity)
if err != nil {
return nil, AsErrorResult(err)
}

30
pkg/storage/unified/search/bleve.go
View File

@ -18,11 +18,12 @@ import (
"github.com/blevesearch/bleve/v2/search/query"
bleveSearch "github.com/blevesearch/bleve/v2/search/searcher"
index "github.com/blevesearch/bleve_index_api"
"github.com/grafana/grafana/pkg/services/dashboards/dashboardaccess"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"go.opentelemetry.io/otel/trace"
"k8s.io/apimachinery/pkg/selection"
"github.com/grafana/grafana/pkg/services/dashboards/dashboardaccess"
"github.com/grafana/grafana/pkg/services/featuremgmt"
authlib "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/log"
@ -304,19 +305,20 @@ func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.Li
found, err := b.index.SearchInContext(ctx, &bleve.SearchRequest{
Query: &query.TermQuery{
Term: req.Name,
FieldVal: resource.SEARCH_FIELD_REPOSITORY_NAME,
FieldVal: resource.SEARCH_FIELD_MANAGER_ID,
},
Fields: []string{
resource.SEARCH_FIELD_TITLE,
resource.SEARCH_FIELD_FOLDER,
resource.SEARCH_FIELD_REPOSITORY_NAME,
resource.SEARCH_FIELD_REPOSITORY_PATH,
resource.SEARCH_FIELD_REPOSITORY_HASH,
resource.SEARCH_FIELD_REPOSITORY_TIME,
resource.SEARCH_FIELD_MANAGER_KIND,
resource.SEARCH_FIELD_MANAGER_ID,
resource.SEARCH_FIELD_SOURCE_PATH,
resource.SEARCH_FIELD_SOURCE_CHECKSUM,
resource.SEARCH_FIELD_SOURCE_TIME,
},
Sort: search.SortOrder{
&search.SortField{
Field: resource.SEARCH_FIELD_REPOSITORY_PATH,
Field: resource.SEARCH_FIELD_SOURCE_PATH,
Type: search.SortFieldAsString,
Desc: false,
},
@ -347,6 +349,10 @@ func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.Li
if ok {
return intV
}
floatV, ok := v.(float64)
if ok {
return int64(floatV)
}
str, ok := v.(string)
if ok {
t, _ := time.Parse(time.RFC3339, str)
@ -359,9 +365,9 @@ func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.Li
for _, hit := range found.Hits {
item := &resource.ListRepositoryObjectsResponse_Item{
Object: &resource.ResourceKey{},
Hash: asString(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_HASH]),
Path: asString(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_PATH]),
Time: asTime(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_TIME]),
Hash: asString(hit.Fields[resource.SEARCH_FIELD_SOURCE_CHECKSUM]),
Path: asString(hit.Fields[resource.SEARCH_FIELD_SOURCE_PATH]),
Time: asTime(hit.Fields[resource.SEARCH_FIELD_SOURCE_TIME]),
Title: asString(hit.Fields[resource.SEARCH_FIELD_TITLE]),
Folder: asString(hit.Fields[resource.SEARCH_FIELD_FOLDER]),
}
@ -379,7 +385,7 @@ func (b *bleveIndex) CountRepositoryObjects(ctx context.Context) ([]*resource.Co
Query: bleve.NewMatchAllQuery(),
Size: 0,
Facets: bleve.FacetsRequest{
"count": bleve.NewFacetRequest(resource.SEARCH_FIELD_REPOSITORY_NAME, 1000), // typically less then 5
"count": bleve.NewFacetRequest(resource.SEARCH_FIELD_MANAGER_ID, 1000), // typically less then 5
},
})
if err != nil {

28
pkg/storage/unified/search/bleve_mappings.go
View File

@ -69,9 +69,9 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_FOLDER, folderMapping)
// Repositories
repo := bleve.NewDocumentStaticMapping()
repo.AddFieldMappingsAt("name", &mapping.FieldMapping{
Name: "name",
manager := bleve.NewDocumentStaticMapping()
manager.AddFieldMappingsAt("kind", &mapping.FieldMapping{
Name: "kind",
Type: "text",
Analyzer: keyword.Name,
Store: true,
@ -79,7 +79,18 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
})
repo.AddFieldMappingsAt("path", &mapping.FieldMapping{
manager.AddFieldMappingsAt("id", &mapping.FieldMapping{
Name: "id",
Type: "text",
Analyzer: keyword.Name,
Store: true,
Index: true,
IncludeTermVectors: false,
IncludeInAll: true,
})
source := bleve.NewDocumentStaticMapping()
source.AddFieldMappingsAt("path", &mapping.FieldMapping{
Name: "path",
Type: "text",
Analyzer: keyword.Name,
@ -88,8 +99,8 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
})
repo.AddFieldMappingsAt("hash", &mapping.FieldMapping{
Name: "hash",
source.AddFieldMappingsAt("checksum", &mapping.FieldMapping{
Name: "checksum",
Type: "text",
Analyzer: keyword.Name,
Store: true,
@ -97,9 +108,10 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
})
repo.AddFieldMappingsAt("time", mapping.NewDateTimeFieldMapping())
source.AddFieldMappingsAt("timestampMillis", mapping.NewNumericFieldMapping())
mapper.AddSubDocumentMapping("repo", repo)
mapper.AddSubDocumentMapping("manager", manager)
mapper.AddSubDocumentMapping("source", source)
labelMapper := bleve.NewDocumentMapping()
mapper.AddSubDocumentMapping(resource.SEARCH_FIELD_LABELS, labelMapper)

15
pkg/storage/unified/search/bleve_mappings_test.go
View File

@ -25,11 +25,14 @@ func TestDocumentMapping(t *testing.T) {
"x": "y",
},
RV: 1234,
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "nnn",
Path: "ppp",
Hash: "hhh",
Timestamp: asTimePointer(1234),
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "rrr",
},
Source: &utils.SourceProperties{
Path: "ppp",
Checksum: "ooo",
TimestampMillis: 1234,
},
}
@ -43,5 +46,5 @@ func TestDocumentMapping(t *testing.T) {
fmt.Printf("DOC: fields %d\n", len(doc.Fields))
fmt.Printf("DOC: size %d\n", doc.Size())
require.Equal(t, 13, len(doc.Fields))
require.Equal(t, 14, len(doc.Fields))
}

63
pkg/storage/unified/search/bleve_test.go
View File

@ -7,20 +7,18 @@ import (
"os"
"path/filepath"
"testing"
"time"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
authlib "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
authzextv1 "github.com/grafana/grafana/pkg/services/authz/proto/v1"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/tracing"
authzextv1 "github.com/grafana/grafana/pkg/services/authz/proto/v1"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
@ -90,11 +88,14 @@ func TestBleveBackend(t *testing.T) {
utils.LabelKeyDeprecatedInternalID: "10", // nolint:staticcheck
},
Tags: []string{"aa", "bb"},
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/aaa.json",
Hash: "xyz",
Timestamp: asTimePointer(1609462800000), // 2021
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo-1",
},
Source: &utils.SourceProperties{
Path: "path/to/aaa.json",
Checksum: "xyz",
TimestampMillis: 1609462800000, // 2021
},
})
_ = index.Write(&resource.IndexableDocument{
@ -119,11 +120,14 @@ func TestBleveBackend(t *testing.T) {
"region": "east",
utils.LabelKeyDeprecatedInternalID: "11", // nolint:staticcheck
},
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/bbb.json",
Hash: "hijk",
Timestamp: asTimePointer(1640998800000), // 2022
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo-1",
},
Source: &utils.SourceProperties{
Path: "path/to/bbb.json",
Checksum: "hijk",
TimestampMillis: 1640998800000, // 2022
},
})
_ = index.Write(&resource.IndexableDocument{
@ -138,8 +142,11 @@ func TestBleveBackend(t *testing.T) {
Title: "ccc (dash)",
TitlePhrase: "ccc (dash)",
Folder: "zzz",
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo2",
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo2",
},
Source: &utils.SourceProperties{
Path: "path/in/repo2.yaml",
},
Fields: map[string]any{},
@ -263,6 +270,7 @@ func TestBleveBackend(t *testing.T) {
jj, err := json.MarshalIndent(found, "", " ")
require.NoError(t, err)
fmt.Printf("%s\n", string(jj))
// NOTE "hash" -> "checksum" requires changing the protobuf
require.JSONEq(t, `{
"items": [
{
@ -334,11 +342,14 @@ func TestBleveBackend(t *testing.T) {
},
Title: "zzz (folder)",
TitlePhrase: "zzz (folder)",
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/folder.json",
Hash: "xxxx",
Timestamp: asTimePointer(300),
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo-1",
},
Source: &utils.SourceProperties{
Path: "path/to/folder.json",
Checksum: "xxxx",
TimestampMillis: 300,
},
})
_ = index.Write(&resource.IndexableDocument{
@ -559,14 +570,6 @@ func TestGetSortFields(t *testing.T) {
})
}
func asTimePointer(milli int64) *time.Time {
if milli > 0 {
t := time.UnixMilli(milli)
return &t
}
return nil
}
var _ authlib.AccessClient = (*StubAccessClient)(nil)
func NewStubAccessClient(permissions map[string]bool) *StubAccessClient {

18
pkg/storage/unified/search/document_test.go
View File

@ -81,14 +81,26 @@ func TestDashboardDocumentBuilder(t *testing.T) {
// Standard
builder = resource.StandardDocumentBuilder()
doSnapshotTests(t, builder, "folder", key, []string{
doSnapshotTests(t, builder, "folder", &resource.ResourceKey{
Namespace: "default",
Group: "folder.grafana.app",
Resource: "folders",
}, []string{
"aaa",
"bbb",
})
doSnapshotTests(t, builder, "playlist", key, []string{
doSnapshotTests(t, builder, "playlist", &resource.ResourceKey{
Namespace: "default",
Group: "playlist.grafana.app",
Resource: "playlists",
}, []string{
"aaa",
})
doSnapshotTests(t, builder, "report", key, []string{
doSnapshotTests(t, builder, "report", &resource.ResourceKey{
Namespace: "default",
Group: "reporting.grafana.app",
Resource: "reports",
}, []string{
"aaa",
})
}

9
pkg/storage/unified/search/testdata/doc/folder-aaa-out.json vendored
View File

@ -1,8 +1,8 @@
{
"key": {
"namespace": "default",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"group": "folder.grafana.app",
"resource": "folders",
"name": "aaa"
},
"name": "aaa",
@ -11,7 +11,8 @@
"title_phrase": "test-aaa",
"created": 1730490142000,
"createdBy": "user:1",
"repo": {
"name": "SQL"
"manager": {
"kind": "repo",
"id": "MyGIT"
}
}

2
pkg/storage/unified/search/testdata/doc/folder-aaa.json vendored
View File

@ -8,7 +8,7 @@
"creationTimestamp": "2024-11-01T19:42:22Z",
"annotations": {
"grafana.app/createdBy": "user:1",
"grafana.app/originName": "SQL"
"grafana.app/repoName": "MyGIT"
}
},
"spec": {

9
pkg/storage/unified/search/testdata/doc/folder-bbb-out.json vendored
View File

@ -1,8 +1,8 @@
{
"key": {
"namespace": "default",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"group": "folder.grafana.app",
"resource": "folders",
"name": "bbb"
},
"name": "bbb",
@ -11,7 +11,8 @@
"title_phrase": "test-bbb",
"created": 1730490142000,
"createdBy": "user:1",
"repo": {
"name": "SQL"
"manager": {
"kind": "repo",
"id": "MyGIT"
}
}

2
pkg/storage/unified/search/testdata/doc/folder-bbb.json vendored
View File

@ -8,7 +8,7 @@
"creationTimestamp": "2024-11-01T19:42:22Z",
"annotations": {
"grafana.app/createdBy": "user:1",
"grafana.app/originName": "SQL"
"grafana.app/repoName": "MyGIT"
}
},
"spec": {

11
pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json vendored
View File

@ -1,8 +1,8 @@
{
"key": {
"namespace": "default",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"group": "playlist.grafana.app",
"resource": "playlists",
"name": "aaa"
},
"name": "aaa",
@ -10,10 +10,5 @@
"title": "Test AAA",
"title_phrase": "test aaa",
"created": 1731336353000,
"createdBy": "user:t000000001",
"repo": {
"name": "UI",
"path": "/playlists/new",
"hash": "Grafana v11.4.0-pre (c0de407fee)"
}
"createdBy": "user:t000000001"
}

4
pkg/storage/unified/search/testdata/doc/report-aaa-out.json vendored
View File

@ -1,8 +1,8 @@
{
"key": {
"namespace": "default",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"group": "reporting.grafana.app",
"resource": "reports",
"name": "aaa"
},
"name": "aaa",

4
pkg/tests/apis/dashboard/dashboards_test.go
View File

@ -68,7 +68,9 @@ func runDashboardTest(t *testing.T, helper *apis.K8sTestHelper, gvr schema.Group
wrap, err := utils.MetaAccessor(obj)
require.NoError(t, err)
require.Empty(t, wrap.GetRepositoryName()) // no SQL repo stub
m, _ := wrap.GetManagerProperties()
require.Empty(t, m.Identity) // no SQL repo stub
require.Equal(t, helper.Org1.Admin.Identity.GetUID(), wrap.GetCreatedBy())
// Commented out because the dynamic client does not like lists as sub-resource

19
public/api-merged.json
View File

@ -15514,6 +15514,9 @@
"type": "integer",
"format": "int64"
},
"managedBy": {
"$ref": "#/definitions/ManagerKind"
},
"orgId": {
"type": "integer",
"format": "int64"
@ -15529,10 +15532,6 @@
"$ref": "#/definitions/Folder"
}
},
"repository": {
"description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
"type": "string"
},
"title": {
"type": "string"
},
@ -15562,11 +15561,10 @@
"type": "integer",
"format": "int64"
},
"parentUid": {
"type": "string"
"managedBy": {
"$ref": "#/definitions/ManagerKind"
},
"repository": {
"description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
"parentUid": {
"type": "string"
},
"title": {
@ -17047,6 +17045,11 @@
}
}
},
"ManagerKind": {
"description": "It can be a user or a tool or a generic API client.\n+enum",
"type": "string",
"title": "ManagerKind is the type of manager, which is responsible for managing the resource."
},
"MassDeleteAnnotationsCmd": {
"type": "object",
"properties": {

19
public/openapi3.json
View File

@ -5569,6 +5569,9 @@
"format": "int64",
"type": "integer"
},
"managedBy": {
"$ref": "#/components/schemas/ManagerKind"
},
"orgId": {
"format": "int64",
"type": "integer"
@ -5584,10 +5587,6 @@
},
"type": "array"
},
"repository": {
"description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
"type": "string"
},
"title": {
"type": "string"
},
@ -5617,11 +5616,10 @@
"format": "int64",
"type": "integer"
},
"parentUid": {
"type": "string"
"managedBy": {
"$ref": "#/components/schemas/ManagerKind"
},
"repository": {
"description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
"parentUid": {
"type": "string"
},
"title": {
@ -7103,6 +7101,11 @@
},
"type": "object"
},
"ManagerKind": {
"description": "It can be a user or a tool or a generic API client.\n+enum",
"title": "ManagerKind is the type of manager, which is responsible for managing the resource.",
"type": "string"
},
"MassDeleteAnnotationsCmd": {
"properties": {
"annotationId": {