Make it safe to run new config CRUD APIs in DS API server

Using `configCrudUseNewApis` passed through for DS API servers from the
`grafana-enterprise` codebase.

go.mod

@@ -651,6 +651,8 @@ require (
 	sigs.k8s.io/yaml v1.6.0 // indirect
 )
 
+require github.com/go-jose/go-jose/v3 v3.0.4
+
 require github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 
 // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream

go.sum

@@ -1224,6 +1224,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=
+github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI=
 github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=

pkg/extensions/enterprise_imports.go

@@ -15,7 +15,7 @@ import (
 	_ "github.com/blugelabs/bluge"
 	_ "github.com/blugelabs/bluge_segment_api"
 	_ "github.com/crewjam/saml"
-	_ "github.com/go-jose/go-jose/v4"
+	_ "github.com/go-jose/go-jose/v3"
 	_ "github.com/gobwas/glob"
 	_ "github.com/googleapis/gax-go/v2"
 	_ "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
@@ -54,7 +54,6 @@ import (
 	_ "github.com/grafana/e2e"
 	_ "github.com/grafana/gofpdf"
 	_ "github.com/grafana/gomemcache/memcache"
-	_ "github.com/grafana/tempo/pkg/traceql"
-
 	_ "github.com/grafana/grafana/apps/alerting/alertenrichment/pkg/apis/alertenrichment/v1beta1"
+	_ "github.com/grafana/tempo/pkg/traceql"
 )

pkg/registry/apis/datasource/connections.go

@@ -0,0 +1,59 @@
+package datasource
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
+	"k8s.io/apimachinery/pkg/apis/meta/internalversion"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/registry/rest"
+)
+
+var (
+	_ rest.Scoper               = (*connectionAccess)(nil)
+	_ rest.SingularNameProvider = (*connectionAccess)(nil)
+	_ rest.Getter               = (*connectionAccess)(nil)
+	_ rest.Lister               = (*connectionAccess)(nil)
+	_ rest.Storage              = (*connectionAccess)(nil)
+)
+
+type connectionAccess struct {
+	resourceInfo   utils.ResourceInfo
+	tableConverter rest.TableConvertor
+	datasources    PluginDatasourceProvider
+}
+
+func (s *connectionAccess) New() runtime.Object {
+	return s.resourceInfo.NewFunc()
+}
+
+func (s *connectionAccess) Destroy() {}
+
+func (s *connectionAccess) NamespaceScoped() bool {
+	return true
+}
+
+func (s *connectionAccess) GetSingularName() string {
+	return s.resourceInfo.GetSingularName()
+}
+
+func (s *connectionAccess) ShortNames() []string {
+	return s.resourceInfo.GetShortNames()
+}
+
+func (s *connectionAccess) NewList() runtime.Object {
+	return s.resourceInfo.NewListFunc()
+}
+
+func (s *connectionAccess) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
+	return s.tableConverter.ConvertToTable(ctx, object, tableOptions)
+}
+
+func (s *connectionAccess) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
+	return s.datasources.GetDataSource(ctx, name)
+}
+
+func (s *connectionAccess) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) {
+	return s.datasources.ListDataSources(ctx)
+}

pkg/registry/apis/datasource/register.go

@@ -3,8 +3,10 @@ package datasource
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"maps"
+	"path/filepath"
 
 	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,11 +26,13 @@ import (
 	"github.com/grafana/grafana/pkg/configprovider"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/manager/sources"
 	"github.com/grafana/grafana/pkg/promlib/models"
 	"github.com/grafana/grafana/pkg/registry/apis/query/queryschema"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/apiserver/builder"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource/kinds"
 )
 
@@ -40,13 +44,14 @@ var (
 type DataSourceAPIBuilder struct {
 	datasourceResourceInfo utils.ResourceInfo
 
-	pluginJSON      plugins.JSONData
-	client          PluginClient // will only ever be called with the same plugin id!
-	datasources     PluginDatasourceProvider
-	contextProvider PluginContextWrapper
-	accessControl   accesscontrol.AccessControl
-	queryTypes      *queryV0.QueryTypeDefinitionList
-	log             log.Logger
+	pluginJSON           plugins.JSONData
+	client               PluginClient // will only ever be called with the same plugin id!
+	datasources          PluginDatasourceProvider
+	contextProvider      PluginContextWrapper
+	accessControl        accesscontrol.AccessControl
+	queryTypes           *queryV0.QueryTypeDefinitionList
+	log                  log.Logger
+	configCrudUseNewApis bool
 }
 
 func RegisterAPIService(
@@ -109,16 +114,12 @@ func RegisterAPIService(
 			contextProvider,
 			accessControl,
 			features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes),
+			false,
 		)
 		if err != nil {
 			return nil, err
 		}
 
-		// TODO: load the schema provider from a static manifest
-		// if ds.ID == "grafana-testdata-datasource" {
-		// 	builder.schemaProvider = hardcoded.TestdataOpenAPIExtension
-		// }
-
 		apiRegistrar.RegisterAPI(builder)
 	}
 	return builder, nil // only used for wire
@@ -140,6 +141,7 @@ func NewDataSourceAPIBuilder(
 	contextProvider PluginContextWrapper,
 	accessControl accesscontrol.AccessControl,
 	loadQueryTypes bool,
+	configCrudUseNewApis bool,
 ) (*DataSourceAPIBuilder, error) {
 	group, err := plugins.GetDatasourceGroupNameFromPluginID(plugin.ID)
 	if err != nil {
@@ -154,6 +156,7 @@ func NewDataSourceAPIBuilder(
 		contextProvider:        contextProvider,
 		accessControl:          accessControl,
 		log:                    log.New("grafana-apiserver.datasource"),
+		configCrudUseNewApis:   configCrudUseNewApis,
 	}
 	if loadQueryTypes {
 		// In the future, this will somehow come from the plugin
@@ -232,19 +235,6 @@ func (b *DataSourceAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver
 
 	// Register the raw datasource connection
 	ds := b.datasourceResourceInfo
-	legacyStore := &legacyStorage{
-		datasources:  b.datasources,
-		resourceInfo: &ds,
-	}
-	unified, err := grafanaregistry.NewRegistryStore(opts.Scheme, ds, opts.OptsGetter)
-	if err != nil {
-		return err
-	}
-	storage[ds.StoragePath()], err = opts.DualWriteBuilder(ds.GroupResource(), legacyStore, unified)
-	if err != nil {
-		return err
-	}
-
 	storage[ds.StoragePath("query")] = &subQueryREST{builder: b}
 	storage[ds.StoragePath("health")] = &subHealthREST{builder: b}
 	storage[ds.StoragePath("resource")] = &subResourceREST{builder: b}
@@ -255,13 +245,34 @@ func (b *DataSourceAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver
 	storage["connections"] = &noopREST{}                            // hidden from openapi
 	storage["connections/query"] = storage[ds.StoragePath("query")] // deprecated in openapi
 
+	if b.configCrudUseNewApis {
+		legacyStore := &legacyStorage{
+			datasources:  b.datasources,
+			resourceInfo: &ds,
+		}
+		unified, err := grafanaregistry.NewRegistryStore(opts.Scheme, ds, opts.OptsGetter)
+		if err != nil {
+			return err
+		}
+		storage[ds.StoragePath()], err = opts.DualWriteBuilder(ds.GroupResource(), legacyStore, unified)
+		if err != nil {
+			return err
+		}
+	} else {
+		storage[ds.StoragePath()] = &connectionAccess{
+			datasources:    b.datasources,
+			resourceInfo:   ds,
+			tableConverter: ds.TableConverter(),
+		}
+	}
+
 	// Frontend proxy
 	if len(b.pluginJSON.Routes) > 0 {
 		storage[ds.StoragePath("proxy")] = &subProxyREST{pluginJSON: b.pluginJSON}
 	}
 
 	// Register hardcoded query schemas
-	err = queryschema.RegisterQueryTypes(b.queryTypes, storage)
+	err := queryschema.RegisterQueryTypes(b.queryTypes, storage)
 	if err != nil {
 		return err
 	}
@@ -287,3 +298,22 @@ func (b *DataSourceAPIBuilder) GetOpenAPIDefinitions() openapi.GetOpenAPIDefinit
 		return defs
 	}
 }
+
+func getCorePlugins(cfg *setting.Cfg) ([]plugins.JSONData, error) {
+	coreDataSourcesPath := filepath.Join(cfg.StaticRootPath, "app", "plugins", "datasource")
+	coreDataSourcesSrc := sources.NewLocalSource(
+		plugins.ClassCore,
+		[]string{coreDataSourcesPath},
+	)
+
+	res, err := coreDataSourcesSrc.Discover(context.Background())
+	if err != nil {
+		return nil, errors.New("failed to load core data source plugins")
+	}
+
+	pluginJSONs := make([]plugins.JSONData, 0, len(res))
+	for _, p := range res {
+		pluginJSONs = append(pluginJSONs, p.Primary.JSONData)
+	}
+	return pluginJSONs, nil
+}

pkg/registry/apis/datasource/testdata/convert-resource-empty-to-cmd-update-roundtrip.json

@@ -2,8 +2,7 @@
   "metadata": {
     "name": "cejobd88i85j4d",
     "namespace": "org-0",
-    "uid": "boDNh7zU3nXj46rOXIJI7r44qaxjs8yy9I9dOj1MyBoX",
-    "creationTimestamp": null
+    "uid": "boDNh7zU3nXj46rOXIJI7r44qaxjs8yy9I9dOj1MyBoX"
   },
   "spec": {
     "jsonData": null,

pkg/registry/apis/datasource/testdata/convert-resource-full-to-cmd-update-roundtrip.json

@@ -3,8 +3,7 @@
     "name": "cejobd88i85j4d",
     "namespace": "org-0",
     "uid": "boDNh7zU3nXj46rOXIJI7r44qaxjs8yy9I9dOj1MyBoX",
-    "generation": 2,
-    "creationTimestamp": null
+    "generation": 2
   },
   "spec": {
     "access": "proxy",