From 0cb6c3d7bf894d7074a33ff281c7c839c149e25c Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 21 Nov 2024 08:53:25 +0300 Subject: [PATCH] UnifiedSearch: Introduce a DocumentBuilder interface (#96738) --- pkg/server/wire.go | 2 + pkg/storage/unified/client.go | 3 +- pkg/storage/unified/resource/document.go | 320 ++++++++++++++++++ pkg/storage/unified/resource/document_test.go | 47 +++ pkg/storage/unified/resource/keys.go | 41 +++ pkg/storage/unified/resource/keys_test.go | 38 +++ pkg/storage/unified/resource/search.go | 208 ++++++++++++ pkg/storage/unified/resource/server.go | 29 ++ .../resource/testdata/playlist-resource.json | 11 +- pkg/storage/unified/search/dashboard.go | 220 ++++++++++++ pkg/storage/unified/search/document.go | 32 ++ pkg/storage/unified/search/document_test.go | 94 +++++ .../testdata/doc/dashboard-aaa-out.json | 65 ++++ .../search/testdata/doc/dashboard-aaa.json | 127 +++++++ .../search/testdata/doc/folder-aaa-out.json | 15 + .../search/testdata/doc/folder-aaa.json | 17 + .../search/testdata/doc/folder-bbb-out.json | 15 + .../search/testdata/doc/folder-bbb.json | 17 + .../search/testdata/doc/playlist-aaa-out.json | 17 + .../search/testdata/doc/playlist-aaa.json | 30 ++ .../search/testdata/doc/report-aaa-out.json | 15 + .../search/testdata/doc/report-aaa.json | 56 +++ pkg/storage/unified/sql/server.go | 7 +- pkg/storage/unified/sql/service.go | 10 +- 24 files changed, 1428 insertions(+), 8 deletions(-) create mode 100644 pkg/storage/unified/resource/document.go create mode 100644 pkg/storage/unified/resource/document_test.go create mode 100644 pkg/storage/unified/resource/search.go create mode 100644 pkg/storage/unified/search/dashboard.go create mode 100644 pkg/storage/unified/search/document.go create mode 100644 pkg/storage/unified/search/document_test.go create mode 100644 pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json create mode 100644 pkg/storage/unified/search/testdata/doc/dashboard-aaa.json create mode 100644 pkg/storage/unified/search/testdata/doc/folder-aaa-out.json create mode 100644 pkg/storage/unified/search/testdata/doc/folder-aaa.json create mode 100644 pkg/storage/unified/search/testdata/doc/folder-bbb-out.json create mode 100644 pkg/storage/unified/search/testdata/doc/folder-bbb.json create mode 100644 pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json create mode 100644 pkg/storage/unified/search/testdata/doc/playlist-aaa.json create mode 100644 pkg/storage/unified/search/testdata/doc/report-aaa-out.json create mode 100644 pkg/storage/unified/search/testdata/doc/report-aaa.json diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 05e00a995e7..31c78ceb27b 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -156,6 +156,7 @@ import ( "github.com/grafana/grafana/pkg/services/user/userimpl" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified" + unifiedsearch "github.com/grafana/grafana/pkg/storage/unified/search" "github.com/grafana/grafana/pkg/tsdb/azuremonitor" cloudmonitoring "github.com/grafana/grafana/pkg/tsdb/cloud-monitoring" "github.com/grafana/grafana/pkg/tsdb/cloudwatch" @@ -229,6 +230,7 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(login.AuthInfoService), new(*authinfoimpl.Service)), authinfoimpl.ProvideStore, datasourceproxy.ProvideService, + unifiedsearch.ProvideDocumentBuilders, search.ProvideService, searchV2.ProvideService, searchV2.ProvideSearchHTTPService, diff --git a/pkg/storage/unified/client.go b/pkg/storage/unified/client.go index 9cdd69d6a19..e3c8a13ddbd 100644 --- a/pkg/storage/unified/client.go +++ b/pkg/storage/unified/client.go @@ -34,6 +34,7 @@ func ProvideUnifiedStorageClient( tracer tracing.Tracer, reg prometheus.Registerer, authzc authz.Client, + docs resource.DocumentBuilderSupplier, ) (resource.ResourceClient, error) { // See: apiserver.ApplyGrafanaConfig(cfg, features, o) apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver") @@ -97,7 +98,7 @@ func ProvideUnifiedStorageClient( // Use the local SQL default: - server, err := sql.NewResourceServer(ctx, db, cfg, features, tracer, reg, authzc) + server, err := sql.NewResourceServer(ctx, db, cfg, features, docs, tracer, reg, authzc) if err != nil { return nil, err } diff --git a/pkg/storage/unified/resource/document.go b/pkg/storage/unified/resource/document.go new file mode 100644 index 00000000000..e787bb64b98 --- /dev/null +++ b/pkg/storage/unified/resource/document.go @@ -0,0 +1,320 @@ +package resource + +import ( + "context" + "fmt" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/grafana/grafana/pkg/apimachinery/utils" +) + +// Convert raw resource bytes into an IndexableDocument +type DocumentBuilder interface { + // Convert raw bytes into an document that can be written + BuildDocument(ctx context.Context, key *ResourceKey, rv int64, value []byte) (*IndexableDocument, error) +} + +// Registry of the searchable document fields +type SearchableDocumentFields interface { + Fields() []string + Field(name string) *ResourceTableColumnDefinition +} + +// Some kinds will require special processing for their namespace +type NamespacedDocumentSupplier = func(ctx context.Context, namespace string, blob BlobSupport) (DocumentBuilder, error) + +// Register how documents can be built for a resource +type DocumentBuilderInfo struct { + // The target resource (empty will be used to match anything) + GroupResource schema.GroupResource + + // Defines the searchable fields + // NOTE: this does not include the root/common fields, only values specific to the the builder + Fields SearchableDocumentFields + + // simple/static builders that do not depend on the environment can be declared once + Builder DocumentBuilder + + // Complicated builders (eg dashboards!) will be declared dynamically and managed by the ResourceServer + Namespaced NamespacedDocumentSupplier +} + +type DocumentBuilderSupplier interface { + GetDocumentBuilders() ([]DocumentBuilderInfo, error) +} + +// IndexableDocument can be written to a ResourceIndex +// Although public, this is *NOT* an end user interface +type IndexableDocument struct { + // The resource key + Key *ResourceKey `json:"key"` + + // Resource version for the resource (if known) + RV int64 `json:"rv,omitempty"` + + // The generic display name + Title string `json:"title,omitempty"` + + // A generic description -- helpful in global search + Description string `json:"description,omitempty"` + + // Like dashboard tags + Tags []string `json:"tags,omitempty"` + + // Generic metadata labels + Labels map[string]string `json:"labels,omitempty"` + + // The folder (K8s name) + Folder string `json:"folder,omitempty"` + + // The first time this resource was saved + Created int64 `json:"created,omitempty"` + + // Who created the resource (will be in the form `user:uid`) + CreatedBy string `json:"createdBy,omitempty"` + + // The last time a user updated the spec + Updated int64 `json:"updated,omitempty"` + + // Who updated the resource (will be in the form `user:uid`) + UpdatedBy string `json:"updatedBy,omitempty"` + + // Searchable nested keys + // The key should exist from the fields defined in DocumentBuilderInfo + // This should not contain duplicate information from the results above + // The meaning of these fields changes depending on the field type + // These values typically come from the Spec, but may also come from status + // metadata, annotations, or external data linked at index time + Fields map[string]any `json:"fields,omitempty"` + + // Maintain a list of resource references. + // Someday this will likely be part of https://github.com/grafana/gamma + References ResourceReferences `json:"reference,omitempty"` + + // When the resource is managed by an upstream repository + RepoInfo *utils.ResourceRepositoryInfo `json:"repository,omitempty"` +} + +type ResourceReference struct { + Relation string `json:"relation"` // eg: depends-on + Group string `json:"group,omitempty"` // the api group + Version string `json:"version,omitempty"` // the api version + Kind string `json:"kind,omitempty"` // panel, data source (for now) + Name string `json:"name"` // the UID / panel name +} + +func (m ResourceReference) String() string { + var sb strings.Builder + sb.WriteString(m.Relation) + sb.WriteString(">>") + sb.WriteString(m.Group) + if m.Version != "" { + sb.WriteString("/") + sb.WriteString(m.Version) + } + if m.Kind != "" { + sb.WriteString("/") + sb.WriteString(m.Kind) + } + sb.WriteString("/") + sb.WriteString(m.Name) + return sb.String() +} + +// Sortable list of references +type ResourceReferences []ResourceReference + +func (m ResourceReferences) Len() int { return len(m) } +func (m ResourceReferences) Swap(i, j int) { m[i], m[j] = m[j], m[i] } +func (m ResourceReferences) Less(i, j int) bool { + a := m[i].String() + b := m[j].String() + return strings.Compare(a, b) > 0 +} + +// Create a new indexable document based on a generic k8s resource +func NewIndexableDocument(key *ResourceKey, rv int64, obj utils.GrafanaMetaAccessor) *IndexableDocument { + doc := &IndexableDocument{ + Key: key, + RV: rv, + Title: obj.FindTitle(key.Name), // We always want *something* to display + Labels: obj.GetLabels(), + Folder: obj.GetFolder(), + CreatedBy: obj.GetCreatedBy(), + UpdatedBy: obj.GetUpdatedBy(), + } + doc.RepoInfo, _ = obj.GetRepositoryInfo() + ts := obj.GetCreationTimestamp() + if !ts.Time.IsZero() { + doc.Created = ts.Time.UnixMilli() + } + tt, err := obj.GetUpdatedTimestamp() + if err != nil && tt != nil { + doc.Updated = tt.UnixMilli() + } + return doc +} + +func StandardDocumentBuilder() DocumentBuilderInfo { + return DocumentBuilderInfo{ + Builder: &standardDocumentBuilder{}, + Fields: StandardSearchFields(), + } +} + +type standardDocumentBuilder struct{} + +func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *ResourceKey, rv int64, value []byte) (*IndexableDocument, error) { + tmp := &unstructured.Unstructured{} + err := tmp.UnmarshalJSON(value) + if err != nil { + return nil, err + } + + obj, err := utils.MetaAccessor(tmp) + if err != nil { + return nil, err + } + + doc := NewIndexableDocument(key, rv, obj) + doc.Title = obj.FindTitle(key.Name) + return doc, nil +} + +type searchableDocumentFields struct { + names []string + fields map[string]*resourceTableColumn +} + +// This requires unique names +func NewSearchableDocumentFields(columns []*ResourceTableColumnDefinition) (SearchableDocumentFields, error) { + f := &searchableDocumentFields{ + names: make([]string, len(columns)), + fields: make(map[string]*resourceTableColumn), + } + for i, c := range columns { + if f.fields[c.Name] != nil { + return nil, fmt.Errorf("duplicate name") + } + col, err := newResourceTableColumn(c, i) + if err != nil { + return nil, err + } + f.names[i] = c.Name + f.fields[c.Name] = col + } + return f, nil +} + +func (x *searchableDocumentFields) Fields() []string { + return x.names +} + +func (x *searchableDocumentFields) Field(name string) *ResourceTableColumnDefinition { + f, ok := x.fields[name] + if ok { + return f.def + } + return nil +} + +const SEARCH_FIELD_ID = "_id" // {namespace}/{group}/{resource}/{name} +const SEARCH_FIELD_GROUP_RESOURCE = "gr" // group/resource +const SEARCH_FIELD_NAMESPACE = "namespace" +const SEARCH_FIELD_NAME = "name" +const SEARCH_FIELD_RV = "rv" +const SEARCH_FIELD_TITLE = "title" +const SEARCH_FIELD_DESCRIPTION = "description" +const SEARCH_FIELD_TAGS = "tags" +const SEARCH_FIELD_LABELS = "labels" // All labels, not a specific one + +const SEARCH_FIELD_FOLDER = "folder" +const SEARCH_FIELD_CREATED = "created" +const SEARCH_FIELD_CREATED_BY = "createdBy" +const SEARCH_FIELD_UPDATED = "updated" +const SEARCH_FIELD_UPDATED_BY = "updatedBy" +const SEARCH_FIELD_REPOSITORY = "repository" +const SEARCH_FIELD_REPOSITORY_HASH = "repository_hash" + +const SEARCH_FIELD_SCORE = "_score" // the match score +const SEARCH_FIELD_EXPLAIN = "_explain" // score explanation as JSON object + +var standardSearchFieldsInit sync.Once +var standardSearchFields SearchableDocumentFields + +func StandardSearchFields() SearchableDocumentFields { + standardSearchFieldsInit.Do(func() { + var err error + standardSearchFields, err = NewSearchableDocumentFields([]*ResourceTableColumnDefinition{ + { + Name: SEARCH_FIELD_ID, + Type: ResourceTableColumnDefinition_STRING, + Description: "Unique Identifier. {namespace}/{group}/{resource}/{name}", + Properties: &ResourceTableColumnDefinition_Properties{ + NotNull: true, + }, + }, + { + Name: SEARCH_FIELD_GROUP_RESOURCE, + Type: ResourceTableColumnDefinition_STRING, + Description: "The resource kind: {group}/{resource}", + Properties: &ResourceTableColumnDefinition_Properties{ + NotNull: true, + }, + }, + { + Name: SEARCH_FIELD_NAMESPACE, + Type: ResourceTableColumnDefinition_STRING, + Description: "Tenant isolation", + Properties: &ResourceTableColumnDefinition_Properties{ + NotNull: true, + }, + }, + { + Name: SEARCH_FIELD_NAME, + Type: ResourceTableColumnDefinition_STRING, + Description: "Kubernetes name. Unique identifier within a namespace+group+resource", + Properties: &ResourceTableColumnDefinition_Properties{ + NotNull: true, + }, + }, + { + Name: SEARCH_FIELD_TITLE, + Type: ResourceTableColumnDefinition_STRING, + Description: "Display name for the resource", + }, + { + Name: SEARCH_FIELD_DESCRIPTION, + Type: ResourceTableColumnDefinition_STRING, + Description: "An account of the resource.", + Properties: &ResourceTableColumnDefinition_Properties{ + FreeText: true, + }, + }, + }) + if err != nil { + panic("failed to initialize standard search fields") + } + }) + return standardSearchFields +} + +// // Helper function to convert everything except the "Fields" property to values +// // NOTE: this is really to help testing things absent real backend index +// func IndexableDocumentStandardFields(doc *IndexableDocument) map[string]any { +// fields := make(map[string]any) + +// // These should always exist +// fields[SEARCH_FIELD_ID] = doc.Key.SearchID() +// fields[SEARCH_FIELD_NAMESPACE] = doc.Key.Namespace +// fields[SEARCH_FIELD_NAME] = doc.Key.Name +// fields[SEARCH_FIELD_GROUP_RESOURCE] = fmt.Sprintf("%s/%s", doc.Key.Group, doc.Key.Resource) + +// fields[SEARCH_FIELD_TITLE] = doc.Title + +// return fields +// } diff --git a/pkg/storage/unified/resource/document_test.go b/pkg/storage/unified/resource/document_test.go new file mode 100644 index 00000000000..c195b7c1479 --- /dev/null +++ b/pkg/storage/unified/resource/document_test.go @@ -0,0 +1,47 @@ +package resource + +import ( + "context" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStandardDocumentBuilder(t *testing.T) { + ctx := context.Background() + builder := StandardDocumentBuilder().Builder + + body, err := os.ReadFile("testdata/playlist-resource.json") + require.NoError(t, err) + doc, err := builder.BuildDocument(ctx, &ResourceKey{ + Namespace: "default", + Group: "playlists.grafana.app", + Resource: "playlists", + Name: "test1", + }, 10, body) + require.NoError(t, err) + + jj, _ := json.MarshalIndent(doc, "", " ") + fmt.Printf("%s\n", string(jj)) + require.JSONEq(t, `{ + "key": { + "namespace": "default", + "group": "playlists.grafana.app", + "resource": "playlists", + "name": "test1" + }, + "rv": 10, + "title": "test1", + "created": 1717236672000, + "createdBy": "user:ABC", + "updatedBy": "user:XYZ", + "repository": { + "name": "SQL", + "path": "15", + "hash": "xyz" + } + }`, string(jj)) +} diff --git a/pkg/storage/unified/resource/keys.go b/pkg/storage/unified/resource/keys.go index a4b554476a6..7e0c7e049ab 100644 --- a/pkg/storage/unified/resource/keys.go +++ b/pkg/storage/unified/resource/keys.go @@ -1,5 +1,10 @@ package resource +import ( + "fmt" + "strings" +) + func verifyRequestKey(key *ResourceKey) *ErrorResult { if key == nil { return NewBadRequestError("missing resource key") @@ -28,3 +33,39 @@ func matchesQueryKey(query *ResourceKey, key *ResourceKey) bool { } return true } + +const clusterNamespace = "**cluster**" + +// Convert the key to a search ID string +func (x *ResourceKey) SearchID() string { + var sb strings.Builder + if x.Namespace == "" { + sb.WriteString(clusterNamespace) + } else { + sb.WriteString(x.Namespace) + } + sb.WriteString("/") + sb.WriteString(x.Group) + sb.WriteString("/") + sb.WriteString(x.Resource) + sb.WriteString("/") + sb.WriteString(x.Name) + return sb.String() +} + +func (x *ResourceKey) ReadSearchID(v string) error { + parts := strings.Split(v, "/") + if len(parts) != 4 { + return fmt.Errorf("invalid search id (expecting 3 slashes)") + } + + x.Namespace = parts[0] + x.Group = parts[1] + x.Resource = parts[2] + x.Name = parts[3] + + if x.Namespace == clusterNamespace { + x.Namespace = "" + } + return nil +} diff --git a/pkg/storage/unified/resource/keys_test.go b/pkg/storage/unified/resource/keys_test.go index deaaff5bedb..0e9de7325f2 100644 --- a/pkg/storage/unified/resource/keys_test.go +++ b/pkg/storage/unified/resource/keys_test.go @@ -19,3 +19,41 @@ func TestKeyMatching(t *testing.T) { })) }) } + +func TestSearchIDKeys(t *testing.T) { + tests := []struct { + input string + expected *ResourceKey // nil error + }{ + {input: "a"}, // error + {input: "default/group/resource/name", + expected: &ResourceKey{ + Namespace: "default", + Group: "group", + Resource: "resource", + Name: "name", + }}, + {input: "/group/resource/", // missing name + expected: &ResourceKey{ + Namespace: "", + Group: "group", + Resource: "resource", + Name: "", + }}, + {input: "**cluster**/group/resource/aaa", // cluster namespace + expected: &ResourceKey{ + Namespace: "", + Group: "group", + Resource: "resource", + Name: "aaa", + }}, + } + + for _, test := range tests { + tmp := &ResourceKey{} + err := tmp.ReadSearchID(test.input) + if err == nil { + require.Equal(t, test.expected, tmp, test.input) + } + } +} diff --git a/pkg/storage/unified/resource/search.go b/pkg/storage/unified/resource/search.go new file mode 100644 index 00000000000..ee71c7408d6 --- /dev/null +++ b/pkg/storage/unified/resource/search.go @@ -0,0 +1,208 @@ +package resource + +import ( + "context" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/hashicorp/golang-lru/v2/expirable" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type NamespacedResource struct { + Namespace string + Group string + Resource string +} + +type SearchBackend interface { + // TODO +} + +const tracingPrexfixSearch = "unified_search." + +// This supports indexing+search regardless of implementation +type searchSupport struct { + tracer trace.Tracer + log *slog.Logger + storage StorageBackend + search SearchBackend + builders *builderCache + initWorkers int +} + +func newSearchSupport(opts SearchOptions, storage StorageBackend, blob BlobSupport, tracer trace.Tracer) (support *searchSupport, err error) { + // No backend search support + if opts.Backend == nil { + return nil, nil + } + + if opts.WorkerThreads < 1 { + opts.WorkerThreads = 1 + } + + support = &searchSupport{ + tracer: tracer, + storage: storage, + search: opts.Backend, + log: slog.Default().With("logger", "resource-search"), + initWorkers: opts.WorkerThreads, + } + + info, err := opts.Resources.GetDocumentBuilders() + if err != nil { + return nil, err + } + + support.builders, err = newBuilderCache(info, 100, time.Minute*2) // TODO? opts + if support.builders != nil { + support.builders.blob = blob + } + + return support, err +} + +// init is called during startup. any failure will block startup and continued execution +func (s *searchSupport) init(ctx context.Context) error { + _, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Init") + defer span.End() + + // TODO, replace namespaces with a query that gets top values + namespaces, err := s.storage.Namespaces(ctx) + if err != nil { + return err + } + + // Hardcoded for now... should come from the query + kinds := []schema.GroupResource{ + {Group: "dashboard.grafana.app", Resource: "dashboards"}, + {Group: "playlist.grafana.app", Resource: "playlists"}, + } + + totalBatchesIndexed := 0 + group := errgroup.Group{} + group.SetLimit(s.initWorkers) + + // Prepare all the (large) indexes + // TODO, threading and query real information: + // SELECT namespace,"group",resource,COUNT(*),resource_version FROM resource + // GROUP BY "group", "resource", "namespace" + // ORDER BY resource_version desc; + for _, ns := range namespaces { + for _, gr := range kinds { + group.Go(func() error { + s.log.Debug("initializing search index", "namespace", ns, "gr", gr) + totalBatchesIndexed++ + _, _, err = s.build(ctx, NamespacedResource{ + Group: gr.Group, + Resource: gr.Resource, + Namespace: ns, + }, 10, 0) // TODO, approximate size + return err + }) + } + } + + err = group.Wait() + if err != nil { + return err + } + span.AddEvent("namespaces indexed", trace.WithAttributes(attribute.Int("namespaced_indexed", totalBatchesIndexed))) + + s.log.Debug("TODO, listen to all events") + + return nil +} + +func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size int64, rv int64) (any, int64, error) { + _, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Build") + defer span.End() + + builder, err := s.builders.get(ctx, nsr) + if err != nil { + return nil, 0, err + } + + s.log.Debug(fmt.Sprintf("TODO, build %+v (size:%d, rv:%d) // builder:%+v\n", nsr, size, rv, builder)) + + return nil, 0, nil +} + +type builderCache struct { + // The default builder + defaultBuilder DocumentBuilder + + // Possible blob support + blob BlobSupport + + // lookup by group, then resource (namespace) + // This is only modified at startup, so we do not need mutex for access + lookup map[string]map[string]DocumentBuilderInfo + + // For namespaced based resources that require a cache + ns *expirable.LRU[NamespacedResource, DocumentBuilder] + mu sync.Mutex // only locked for a cache miss +} + +func newBuilderCache(cfg []DocumentBuilderInfo, nsCacheSize int, ttl time.Duration) (*builderCache, error) { + cache := &builderCache{ + lookup: make(map[string]map[string]DocumentBuilderInfo), + ns: expirable.NewLRU[NamespacedResource, DocumentBuilder](nsCacheSize, nil, ttl), + } + if len(cfg) == 0 { + return cache, fmt.Errorf("no builders configured") + } + + for _, b := range cfg { + // the default + if b.GroupResource.Group == "" && b.GroupResource.Resource == "" { + if b.Builder == nil { + return cache, fmt.Errorf("default document builder is missing") + } + cache.defaultBuilder = b.Builder + continue + } + g, ok := cache.lookup[b.GroupResource.Group] + if !ok { + g = make(map[string]DocumentBuilderInfo) + cache.lookup[b.GroupResource.Group] = g + } + g[b.GroupResource.Resource] = b + } + return cache, nil +} + +// context is typically background. Holds an LRU cache for a +func (s *builderCache) get(ctx context.Context, key NamespacedResource) (DocumentBuilder, error) { + g, ok := s.lookup[key.Group] + if ok { + r, ok := g[key.Resource] + if ok { + if r.Builder != nil { + return r.Builder, nil + } + + // The builder needs context + builder, ok := s.ns.Get(key) + if ok { + return builder, nil + } + { + s.mu.Lock() + defer s.mu.Unlock() + + b, err := r.Namespaced(ctx, key.Namespace, s.blob) + if err == nil { + _ = s.ns.Add(key, b) + } + return b, err + } + } + } + return s.defaultBuilder, nil +} diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go index 24fcee7260a..db2909914af 100644 --- a/pkg/storage/unified/resource/server.go +++ b/pkg/storage/unified/resource/server.go @@ -123,6 +123,18 @@ type BlobConfig struct { Backend BlobSupport } +// Passed as input to the constructor +type SearchOptions struct { + // The raw index backend (eg, bleve, frames, parquet, etc) + Backend SearchBackend + + // The supported resource types + Resources DocumentBuilderSupplier + + // How many threads should build indexes + WorkerThreads int +} + type ResourceServerOptions struct { // OTel tracer Tracer trace.Tracer @@ -136,6 +148,9 @@ type ResourceServerOptions struct { // Requests based on a search index Index ResourceIndexServer + // Search options + Search SearchOptions + // Diagnostics Diagnostics DiagnosticsServer @@ -225,6 +240,14 @@ func NewResourceServer(opts ResourceServerOptions) (ResourceServer, error) { cancel: cancel, } + if opts.Search.Resources != nil { + var err error + s.search, err = newSearchSupport(opts.Search, s.backend, s.blob, opts.Tracer) + if err != nil { + return nil, err + } + } + return s, nil } @@ -235,6 +258,7 @@ type server struct { log *slog.Logger backend StorageBackend blob BlobSupport + search *searchSupport index ResourceIndexServer diagnostics DiagnosticsServer access authz.AccessClient @@ -269,6 +293,11 @@ func (s *server) Init(ctx context.Context) error { s.initErr = s.initWatcher() } + // initialize the search index + if s.initErr == nil && s.search != nil { + s.initErr = s.search.init(ctx) + } + if s.initErr != nil { s.log.Error("error initializing resource server", "error", s.initErr) } diff --git a/pkg/storage/unified/resource/testdata/playlist-resource.json b/pkg/storage/unified/resource/testdata/playlist-resource.json index 89b70a65eb3..bab6a01c29f 100644 --- a/pkg/storage/unified/resource/testdata/playlist-resource.json +++ b/pkg/storage/unified/resource/testdata/playlist-resource.json @@ -5,15 +5,18 @@ "name": "ae2ntrqxefvnke", "namespace": "default", "uid": "playlist-1", - "creationTimestamp": "2024-11-01T19:42:22Z", + "creationTimestamp": "2024-06-01T10:11:12Z", "annotations": { - "grafana.app/createdBy": "user:1", + "grafana.app/createdBy": "user:ABC", + "grafana.app/updatedBy": "user:XYZ", "grafana.app/repoName": "SQL", "grafana.app/repoPath": "15", - "grafana.app/repoTimestamp": "2024-11-01T19:42:22Z" + "grafana.app/repoHash": "xyz", + "grafana.app/updatedTimestamp": "2024-07-01T10:11:12Z" } }, "spec": { - "title": "test-us-playlist" + "title": "test playlist unified storage", + "description": "description for the test playlist" } } \ No newline at end of file diff --git a/pkg/storage/unified/search/dashboard.go b/pkg/storage/unified/search/dashboard.go new file mode 100644 index 00000000000..edf13abc1f6 --- /dev/null +++ b/pkg/storage/unified/search/dashboard.go @@ -0,0 +1,220 @@ +package search + +import ( + "bytes" + "context" + "fmt" + "sort" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/grafana/grafana/pkg/apimachinery/utils" + "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" + "github.com/grafana/grafana/pkg/services/store/kind/dashboard" + "github.com/grafana/grafana/pkg/storage/unified/resource" +) + +//------------------------------------------------------------ +// Standard dashboard fields +//------------------------------------------------------------ + +const DASHBOARD_LEGACY_ID = "legacy_id" +const DASHBOARD_SCHEMA_VERSION = "schema_version" +const DASHBOARD_LINK_COUNT = "link_count" +const DASHBOARD_PANEL_TYPES = "panel_types" +const DASHBOARD_DS_TYPES = "ds_types" +const DASHBOARD_TRANSFORMATIONS = "transformation" + +//------------------------------------------------------------ +// The following fields are added in enterprise +//------------------------------------------------------------ + +const DASHBOARD_VIEWS_LAST_1_DAYS = "views_last_1_days" +const DASHBOARD_VIEWS_LAST_7_DAYS = "views_last_7_days" +const DASHBOARD_VIEWS_LAST_30_DAYS = "views_last_30_days" +const DASHBOARD_VIEWS_TOTAL = "views_total" +const DASHBOARD_VIEWS_TODAY = "views_today" +const DASHBOARD_QUERIES_LAST_1_DAYS = "queries_last_1_days" +const DASHBOARD_QUERIES_LAST_7_DAYS = "queries_last_7_days" +const DASHBOARD_QUERIES_LAST_30_DAYS = "queries_last_30_days" +const DASHBOARD_QUERIES_TOTAL = "queries_total" +const DASHBOARD_QUERIES_TODAY = "queries_today" +const DASHBOARD_ERRORS_LAST_1_DAYS = "errors_last_1_days" +const DASHBOARD_ERRORS_LAST_7_DAYS = "errors_last_7_days" +const DASHBOARD_ERRORS_LAST_30_DAYS = "errors_last_30_days" +const DASHBOARD_ERRORS_TOTAL = "errors_total" +const DASHBOARD_ERRORS_TODAY = "errors_today" + +func DashboardBuilder(namespaced resource.NamespacedDocumentSupplier) (resource.DocumentBuilderInfo, error) { + fields, err := resource.NewSearchableDocumentFields([]*resource.ResourceTableColumnDefinition{ + { + Name: DASHBOARD_SCHEMA_VERSION, + Type: resource.ResourceTableColumnDefinition_INT32, + Description: "Numeric version saying when the schema was saved", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + NotNull: true, + }, + }, + { + Name: DASHBOARD_LINK_COUNT, + Type: resource.ResourceTableColumnDefinition_INT32, + Description: "How many links appear on the page", + }, + { + Name: DASHBOARD_PANEL_TYPES, + Type: resource.ResourceTableColumnDefinition_STRING, + IsArray: true, + Description: "How many links appear on the page", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + }) + if namespaced == nil { + namespaced = func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { + return &DashboardDocumentBuilder{ + Namespace: namespace, + Blob: blob, + Stats: NewDashboardStatsLookup(nil), + DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{ + // empty values (does not resolve anything) + }), + }, nil + } + } + return resource.DocumentBuilderInfo{ + GroupResource: v0alpha1.DashboardResourceInfo.GroupResource(), + Fields: fields, + Namespaced: namespaced, + }, err +} + +type DashboardDocumentBuilder struct { + // Scoped to a single tenant + Namespace string + + // Cached stats for this namespace + // TODO, load this from apiserver request + Stats DashboardStatsLookup + + // data source lookup + DatasourceLookup dashboard.DatasourceLookup + + // For large dashboards we will need to load them from blob store + Blob resource.BlobSupport +} + +type DashboardStatsLookup = func(ctx context.Context, uid string) map[string]int64 + +func NewDashboardStatsLookup(stats map[string]map[string]int64) DashboardStatsLookup { + return func(ctx context.Context, uid string) map[string]int64 { + if stats == nil { + return nil + } + return stats[uid] + } +} + +var _ resource.DocumentBuilder = &DashboardDocumentBuilder{} + +func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resource.ResourceKey, rv int64, value []byte) (*resource.IndexableDocument, error) { + if s.Namespace != "" && s.Namespace != key.Namespace { + return nil, fmt.Errorf("invalid namespace") + } + + tmp := &unstructured.Unstructured{} + err := tmp.UnmarshalJSON(value) + if err != nil { + return nil, err + } + + obj, err := utils.MetaAccessor(tmp) + if err != nil { + return nil, err + } + + blob := obj.GetBlob() + if blob != nil { + rsp, err := s.Blob.GetResourceBlob(ctx, key, blob, true) + if err != nil { + return nil, err + } + if rsp.Error != nil { + return nil, fmt.Errorf("error reading blob: %+v", rsp.Error) + } + value = rsp.Value + } + + summary, err := dashboard.ReadDashboard(bytes.NewReader(value), s.DatasourceLookup) + if err != nil { + return nil, err + } + + doc := resource.NewIndexableDocument(key, rv, obj) + doc.Title = summary.Title + doc.Description = summary.Description + doc.Tags = summary.Tags + + panelTypes := []string{} + transformations := []string{} + dsTypes := []string{} + + for _, p := range summary.Panels { + if p.Type != "" { + panelTypes = append(panelTypes, p.Type) + } + if len(p.Transformer) > 0 { + transformations = append(transformations, p.Transformer...) + } + if p.LibraryPanel != "" { + doc.References = append(doc.References, resource.ResourceReference{ + Group: "dashboards.grafana.app", + Kind: "LibraryPanel", + Name: p.LibraryPanel, + Relation: "depends-on", + }) + } + } + + for _, ds := range summary.Datasource { + dsTypes = append(dsTypes, ds.Type) + doc.References = append(doc.References, resource.ResourceReference{ + Group: ds.Type, + Kind: "DataSource", + Name: ds.UID, + Relation: "depends-on", + }) + } + if doc.References != nil { + sort.Sort(doc.References) + } + + doc.Fields = map[string]any{ + DASHBOARD_SCHEMA_VERSION: summary.SchemaVersion, + DASHBOARD_LINK_COUNT: summary.LinkCount, + } + + if summary.ID > 0 { + doc.Fields[DASHBOARD_LEGACY_ID] = summary.ID + } + if len(panelTypes) > 0 { + sort.Strings(panelTypes) + doc.Fields[DASHBOARD_PANEL_TYPES] = panelTypes + } + if len(dsTypes) > 0 { + sort.Strings(dsTypes) + doc.Fields[DASHBOARD_DS_TYPES] = dsTypes + } + if len(transformations) > 0 { + sort.Strings(transformations) + doc.Fields[DASHBOARD_TRANSFORMATIONS] = transformations + } + + // Add the stats fields + stats := s.Stats(ctx, key.Name) // summary.UID + for k, v := range stats { + doc.Fields[k] = v + } + + return doc, nil +} diff --git a/pkg/storage/unified/search/document.go b/pkg/storage/unified/search/document.go new file mode 100644 index 00000000000..b5bc8debd32 --- /dev/null +++ b/pkg/storage/unified/search/document.go @@ -0,0 +1,32 @@ +package search + +import ( + "context" + + "github.com/grafana/grafana/pkg/services/store/kind/dashboard" + "github.com/grafana/grafana/pkg/storage/unified/resource" +) + +// The default list of open source document builders +type StandardDocumentBuilders struct{} + +// Hooked up so wire can fill in different sprinkles +func ProvideDocumentBuilders() resource.DocumentBuilderSupplier { + return &StandardDocumentBuilders{} +} + +func (s *StandardDocumentBuilders) GetDocumentBuilders() ([]resource.DocumentBuilderInfo, error) { + dashboards, err := DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { + return &DashboardDocumentBuilder{ + Namespace: namespace, + Blob: blob, + Stats: NewDashboardStatsLookup(nil), // empty stats + DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{{}}), + }, nil + }) + + return []resource.DocumentBuilderInfo{ + resource.StandardDocumentBuilder(), + dashboards, + }, err +} diff --git a/pkg/storage/unified/search/document_test.go b/pkg/storage/unified/search/document_test.go new file mode 100644 index 00000000000..1ea0feeffed --- /dev/null +++ b/pkg/storage/unified/search/document_test.go @@ -0,0 +1,94 @@ +package search + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/services/store/kind/dashboard" + "github.com/grafana/grafana/pkg/storage/unified/resource" +) + +func doSnapshotTests(t *testing.T, builder resource.DocumentBuilder, kind string, key *resource.ResourceKey, names []string) { + t.Helper() + + for _, name := range names { + key.Name = name + prefix := fmt.Sprintf("%s-%s", kind, key.Name) + t.Run(prefix, func(t *testing.T) { + // nolint:gosec + in, err := os.ReadFile(filepath.Join("testdata", "doc", prefix+".json")) + require.NoError(t, err) + + doc, err := builder.BuildDocument(context.Background(), key, int64(1234), in) + require.NoError(t, err) + + out, err := json.MarshalIndent(doc, "", " ") + require.NoError(t, err) + + outpath := filepath.Join("testdata", "doc", prefix+"-out.json") + + // test path + // nolint:gosec + expect, _ := os.ReadFile(outpath) + if !assert.JSONEq(t, string(expect), string(out)) { + err = os.WriteFile(outpath, out, 0600) + require.NoError(t, err) + } + }) + } +} + +func TestDashboardDocumentBuilder(t *testing.T) { + key := &resource.ResourceKey{ + Namespace: "default", + Group: "dashboard.grafana.app", + Resource: "dashboards", + } + + info, err := DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { + return &DashboardDocumentBuilder{ + Namespace: namespace, + Blob: blob, + Stats: NewDashboardStatsLookup(map[string]map[string]int64{ + "aaa": { + DASHBOARD_ERRORS_LAST_1_DAYS: 1, + DASHBOARD_ERRORS_LAST_7_DAYS: 1, + }, + }), + DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{{ + Name: "TheDisplayName", // used to be the unique ID! + Type: "my-custom-plugin", + UID: "DSUID", + }}), + }, nil + }) + require.NoError(t, err) + + builder, err := info.Namespaced(context.Background(), key.Namespace, nil) + require.NoError(t, err) + + // Dashboards (custom) + doSnapshotTests(t, builder, "dashboard", key, []string{ + "aaa", + }) + + // Standard + builder = resource.StandardDocumentBuilder().Builder + doSnapshotTests(t, builder, "folder", key, []string{ + "aaa", + "bbb", + }) + doSnapshotTests(t, builder, "playlist", key, []string{ + "aaa", + }) + doSnapshotTests(t, builder, "report", key, []string{ + "aaa", + }) +} diff --git a/pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json b/pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json new file mode 100644 index 00000000000..10782ea0123 --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json @@ -0,0 +1,65 @@ +{ + "key": { + "namespace": "default", + "group": "dashboard.grafana.app", + "resource": "dashboards", + "name": "aaa" + }, + "rv": 1234, + "title": "Test title", + "description": "test description", + "tags": [ + "a", + "b", + "c" + ], + "labels": { + "host": "abc", + "region": "xyz" + }, + "folder": "the-folder-uid", + "created": 1730313054000, + "createdBy": "user:be2g71ke8yoe8b", + "fields": { + "ds_types": [ + "datasource", + "my-custom-plugin" + ], + "errors_last_1_days": 1, + "errors_last_7_days": 1, + "legacy_id": 141, + "link_count": 0, + "panel_types": [ + "barchart", + "graph", + "row" + ], + "schema_version": 38 + }, + "reference": [ + { + "relation": "depends-on", + "group": "my-custom-plugin", + "kind": "DataSource", + "name": "DSUID" + }, + { + "relation": "depends-on", + "group": "datasource", + "kind": "DataSource", + "name": "grafana" + }, + { + "relation": "depends-on", + "group": "dashboards.grafana.app", + "kind": "LibraryPanel", + "name": "e1d5f519-dabd-47c6-9ad7-83d181ce1cee" + }, + { + "relation": "depends-on", + "group": "dashboards.grafana.app", + "kind": "LibraryPanel", + "name": "a7975b7a-fb53-4ab7-951d-15810953b54f" + } + ] +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/dashboard-aaa.json b/pkg/storage/unified/search/testdata/doc/dashboard-aaa.json new file mode 100644 index 00000000000..d7e09d40570 --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/dashboard-aaa.json @@ -0,0 +1,127 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v0alpha1", + "metadata": { + "name": "aaa", + "namespace": "default", + "uid": "b396894e-56bf-4a01-837b-64157912ca00", + "creationTimestamp": "2024-10-30T18:30:54Z", + "annotations": { + "grafana.app/folder": "the-folder-uid", + "grafana.app/createdBy": "user:be2g71ke8yoe8b", + "grafana.app/repositoryName": "MyGit" + }, + "labels": { + "host": "abc", + "region": "xyz" + } + }, + "spec": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 141, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "libraryPanel": { + "name": "green pie", + "uid": "a7975b7a-fb53-4ab7-951d-15810953b54f" + }, + "title": "green pie" + }, + { + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "libraryPanel": { + "name": "red pie", + "uid": "e1d5f519-dabd-47c6-9ad7-83d181ce1cee" + }, + "title": "green pie" + }, + { + "id": 7, + "type": "barchart", + "datasource": "TheDisplayName" + }, + { + "id": 8, + "type": "graph" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 3, + "panels": [ + { + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 42, + "libraryPanel": { + "name": "blue pie", + "uid": "l3d2s634-fdgf-75u4-3fg3-67j966ii7jur" + }, + "title": "blue pie" + } + ], + "title": "collapsed row", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": ["a", "b", "c"], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Test title", + "description": "test description", + "uid": "adfbg6f", + "version": 3, + "weekStart": "" + } +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/folder-aaa-out.json b/pkg/storage/unified/search/testdata/doc/folder-aaa-out.json new file mode 100644 index 00000000000..b5789739d2d --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/folder-aaa-out.json @@ -0,0 +1,15 @@ +{ + "key": { + "namespace": "default", + "group": "dashboard.grafana.app", + "resource": "dashboards", + "name": "aaa" + }, + "rv": 1234, + "title": "aaa", + "created": 1730490142000, + "createdBy": "user:1", + "repository": { + "name": "SQL" + } +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/folder-aaa.json b/pkg/storage/unified/search/testdata/doc/folder-aaa.json new file mode 100644 index 00000000000..97262d5926e --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/folder-aaa.json @@ -0,0 +1,17 @@ +{ + "kind": "Folder", + "apiVersion": "folder.grafana.app/v0alpha1", + "metadata": { + "name": "aaa", + "namespace": "default", + "uid": "b396894e-56bf-4a01-837b-64157912ca00", + "creationTimestamp": "2024-11-01T19:42:22Z", + "annotations": { + "grafana.app/createdBy": "user:1", + "grafana.app/originName": "SQL" + } + }, + "spec": { + "title": "test-aaa" + } +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/folder-bbb-out.json b/pkg/storage/unified/search/testdata/doc/folder-bbb-out.json new file mode 100644 index 00000000000..4fee88712f9 --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/folder-bbb-out.json @@ -0,0 +1,15 @@ +{ + "key": { + "namespace": "default", + "group": "dashboard.grafana.app", + "resource": "dashboards", + "name": "bbb" + }, + "rv": 1234, + "title": "bbb", + "created": 1730490142000, + "createdBy": "user:1", + "repository": { + "name": "SQL" + } +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/folder-bbb.json b/pkg/storage/unified/search/testdata/doc/folder-bbb.json new file mode 100644 index 00000000000..6d5f66d648d --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/folder-bbb.json @@ -0,0 +1,17 @@ +{ + "kind": "Folder", + "apiVersion": "folder.grafana.app/v0alpha1", + "metadata": { + "name": "bbb", + "namespace": "default", + "uid": "b396894e-56bf-4a01-837b-64157912ca00", + "creationTimestamp": "2024-11-01T19:42:22Z", + "annotations": { + "grafana.app/createdBy": "user:1", + "grafana.app/originName": "SQL" + } + }, + "spec": { + "title": "test-bbb" + } +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json b/pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json new file mode 100644 index 00000000000..3dd069e0ed6 --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json @@ -0,0 +1,17 @@ +{ + "key": { + "namespace": "default", + "group": "dashboard.grafana.app", + "resource": "dashboards", + "name": "aaa" + }, + "rv": 1234, + "title": "aaa", + "created": 1731336353000, + "createdBy": "user:t000000001", + "repository": { + "name": "UI", + "path": "/playlists/new", + "hash": "Grafana v11.4.0-pre (c0de407fee)" + } +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/playlist-aaa.json b/pkg/storage/unified/search/testdata/doc/playlist-aaa.json new file mode 100644 index 00000000000..d795e8e19f8 --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/playlist-aaa.json @@ -0,0 +1,30 @@ +{ + "kind": "Playlist", + "apiVersion": "playlist.grafana.app/v0alpha1", + "metadata": { + "name": "aaa", + "namespace": "default", + "uid": "8a2b984d-4663-4182-861b-edec2f987dff", + "creationTimestamp": "2024-11-11T14:45:53Z", + "annotations": { + "grafana.app/createdBy": "user:t000000001", + "grafana.app/originHash": "Grafana v11.4.0-pre (c0de407fee)", + "grafana.app/originName": "UI", + "grafana.app/originPath": "/playlists/new" + } + }, + "spec": { + "title": "Test AAA", + "interval": "5m", + "items": [ + { + "type": "dashboard_by_uid", + "value": "xCmMwXdVz" + }, + { + "type": "dashboard_by_tag", + "value": "panel-tests" + } + ] + } +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/report-aaa-out.json b/pkg/storage/unified/search/testdata/doc/report-aaa-out.json new file mode 100644 index 00000000000..7ee68646d0a --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/report-aaa-out.json @@ -0,0 +1,15 @@ +{ + "key": { + "namespace": "default", + "group": "dashboard.grafana.app", + "resource": "dashboards", + "name": "aaa" + }, + "rv": 1234, + "title": "aaa", + "created": 1706690655000, + "createdBy": "user:abc", + "repository": { + "name": "SQL" + } +} \ No newline at end of file diff --git a/pkg/storage/unified/search/testdata/doc/report-aaa.json b/pkg/storage/unified/search/testdata/doc/report-aaa.json new file mode 100644 index 00000000000..e6cb9bca024 --- /dev/null +++ b/pkg/storage/unified/search/testdata/doc/report-aaa.json @@ -0,0 +1,56 @@ +{ + "kind": "Report", + "apiVersion": "reports.grafana.app/v0alpha1", + "metadata": { + "name": "aaa", + "namespace": "default", + "uid": "GPShlB3AMM0KIVRtn3H2DbGEckFHGXltAkO1o0XD79cX", + "resourceVersion": "1706690655000", + "creationTimestamp": "2024-01-31T08:44:15Z", + "annotations": { + "grafana.app/createdBy": "user:abc", + "grafana.app/originName": "SQL" + } + }, + "spec": { + "title": "Test AAA", + "recipients": "", + "replyTo": "", + "message": "Hi, \nPlease find attached a PDF status report. If you have any questions, feel free to contact me!\nBest,", + "schedule": { + "startDate": "2024-01-31T08:43:15Z", + "endDate": null, + "frequency": "weekly", + "workdaysOnly": false, + "timeZone": "America/Los_Angeles" + }, + "options": { + "orientation": "landscape", + "layout": "simple", + "pdfShowTemplateVariables": false, + "pdfCombineOneFile": false + }, + "enableDashboardUrl": true, + "state": "draft", + "dashboards": [ + { + "uid": "vmie2cmWz", + "timeRange": { + "from": "now-15m", + "to": "now" + } + }, + { + "uid": "xMsQdBfWz", + "timeRange": { + "from": "now-1h", + "to": "now" + } + } + ], + "formats": [ + "pdf" + ], + "scaleFactor": 3 + } +} \ No newline at end of file diff --git a/pkg/storage/unified/sql/server.go b/pkg/storage/unified/sql/server.go index ddbbd99ec7d..796793f2a79 100644 --- a/pkg/storage/unified/sql/server.go +++ b/pkg/storage/unified/sql/server.go @@ -19,7 +19,9 @@ import ( ) // Creates a new ResourceServer -func NewResourceServer(ctx context.Context, db infraDB.DB, cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracer tracing.Tracer, reg prometheus.Registerer, ac authz.Client) (resource.ResourceServer, error) { +func NewResourceServer(ctx context.Context, db infraDB.DB, cfg *setting.Cfg, + features featuremgmt.FeatureToggles, docs resource.DocumentBuilderSupplier, + tracer tracing.Tracer, reg prometheus.Registerer, ac authz.Client) (resource.ResourceServer, error) { apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver") opts := resource.ResourceServerOptions{ Tracer: tracer, @@ -52,6 +54,9 @@ func NewResourceServer(ctx context.Context, db infraDB.DB, cfg *setting.Cfg, fea opts.Backend = store opts.Diagnostics = store opts.Lifecycle = store + opts.Search = resource.SearchOptions{ + Resources: docs, + } if features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageSearch) { opts.Index = resource.NewResourceIndexServer(cfg, tracer) diff --git a/pkg/storage/unified/sql/service.go b/pkg/storage/unified/sql/service.go index 62cf7612f40..50c0608475e 100644 --- a/pkg/storage/unified/sql/service.go +++ b/pkg/storage/unified/sql/service.go @@ -3,10 +3,11 @@ package sql import ( "context" - "github.com/grafana/dskit/services" "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc/health/grpc_health_v1" + "github.com/grafana/dskit/services" + infraDB "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" @@ -19,6 +20,7 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resource/grpc" + "github.com/grafana/grafana/pkg/storage/unified/search" ) var ( @@ -99,7 +101,11 @@ func (s *service) start(ctx context.Context) error { return err } - server, err := NewResourceServer(ctx, s.db, s.cfg, s.features, s.tracing, s.reg, authzClient) + // TODO, for standalone this will need to be started from enterprise + // Connecting to the correct remote services + docs := search.ProvideDocumentBuilders() + + server, err := NewResourceServer(ctx, s.db, s.cfg, s.features, docs, s.tracing, s.reg, authzClient) if err != nil { return err }