root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 169 Packages Projects Releases Wiki Activity

EntityStore: Move slug+folder to summary metadata (#59620)

This commit is contained in:
Ryan McKinley 2022-12-01 11:45:08 -08:00 committed by GitHub
parent 6dbe3b555f
commit fb98a97efa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 468 additions and 1074 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/models/object.go → pkg/models/entity.go
View File

@ -88,6 +88,12 @@ type EntitySummary struct {
// Key value pairs. Tags are are represented as keys with empty values // Key value pairs. Tags are are represented as keys with empty values
Labels map[string]string `json:"labels,omitempty"` Labels map[string]string `json:"labels,omitempty"`
// Parent folder UID
Folder string `json:"folder,omitempty"`
// URL safe version of the name. It will be unique within the folder
Slug string `json:"slug,omitempty"`
// URL should only be set if the value is not derived directly from kind+uid // URL should only be set if the value is not derived directly from kind+uid
// NOTE: this may go away with a more robust GRN solution /!\ // NOTE: this may go away with a more robust GRN solution /!\
URL string `json:"URL,omitempty"` URL string `json:"URL,omitempty"`

4
pkg/services/playlist/playlistimpl/entity_store.go
View File

@ -152,13 +152,13 @@ func (s *entityStoreImpl) Get(ctx context.Context, q *playlist.GetPlaylistByUidQ
if err != nil { if err != nil {
return nil, err return nil, err
} }
if rsp.Entity == nil || rsp.Entity.Body == nil { if rsp == nil || rsp.Body == nil {
return nil, fmt.Errorf("missing object") return nil, fmt.Errorf("missing object")
} }
// Get the object from payload // Get the object from payload
found := &playlist.PlaylistDTO{} found := &playlist.PlaylistDTO{}
err = json.Unmarshal(rsp.Entity.Body, found) err = json.Unmarshal(rsp.Body, found)
return found, err return found, err
} }

398
pkg/services/store/entity/dummy/dummy_server.go
View File

@ -1,398 +0,0 @@
package dummy
import (
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strconv"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/x/persistentcollection"
"github.com/grafana/grafana/pkg/services/grpcserver"
"github.com/grafana/grafana/pkg/services/store"
"github.com/grafana/grafana/pkg/services/store/entity"
"github.com/grafana/grafana/pkg/services/store/kind"
"github.com/grafana/grafana/pkg/setting"
)
type EntityVersionWithBody struct {
*entity.EntityVersionInfo `json:"info,omitempty"`
Body []byte `json:"body,omitempty"`
}
type EntityWithHistory struct {
Entity *entity.Entity `json:"entity,omitempty"`
History []*EntityVersionWithBody `json:"history,omitempty"`
}
var (
// increment when Entity changes
rawEntityVersion = 10
)
// Make sure we implement both store + admin
var _ entity.EntityStoreServer = &dummyEntityServer{}
var _ entity.EntityStoreAdminServer = &dummyEntityServer{}
func ProvideDummyEntityServer(cfg *setting.Cfg, grpcServerProvider grpcserver.Provider, kinds kind.KindRegistry) entity.EntityStoreServer {
objectServer := &dummyEntityServer{
collection: persistentcollection.NewLocalFSPersistentCollection[*EntityWithHistory]("raw-object", cfg.DataPath, rawEntityVersion),
log: log.New("in-memory-object-server"),
kinds: kinds,
}
entity.RegisterEntityStoreServer(grpcServerProvider.GetServer(), objectServer)
return objectServer
}
type dummyEntityServer struct {
log log.Logger
collection persistentcollection.PersistentCollection[*EntityWithHistory]
kinds kind.KindRegistry
}
func namespaceFromUID(grn *entity.GRN) string {
// TODO
return "orgId-1"
}
func (i *dummyEntityServer) findEntity(ctx context.Context, grn *entity.GRN, version string) (*EntityWithHistory, *entity.Entity, error) {
if grn == nil {
return nil, nil, errors.New("GRN must not be nil")
}
obj, err := i.collection.FindFirst(ctx, namespaceFromUID(grn), func(i *EntityWithHistory) (bool, error) {
return grn.Equals(i.Entity.GRN), nil
})
if err != nil {
return nil, nil, err
}
if obj == nil {
return nil, nil, nil
}
getLatestVersion := version == ""
if getLatestVersion {
return obj, obj.Entity, nil
}
for _, objVersion := range obj.History {
if objVersion.Version == version {
copy := &entity.Entity{
GRN: obj.Entity.GRN,
CreatedAt: obj.Entity.CreatedAt,
CreatedBy: obj.Entity.CreatedBy,
UpdatedAt: objVersion.UpdatedAt,
UpdatedBy: objVersion.UpdatedBy,
ETag: objVersion.ETag,
Version: objVersion.Version,
// Body is added from the dummy server cache (it does not exist in EntityVersionInfo)
Body: objVersion.Body,
}
return obj, copy, nil
}
}
return obj, nil, nil
}
func (i *dummyEntityServer) Read(ctx context.Context, r *entity.ReadEntityRequest) (*entity.ReadEntityResponse, error) {
grn := getFullGRN(ctx, r.GRN)
_, objVersion, err := i.findEntity(ctx, grn, r.Version)
if err != nil {
return nil, err
}
if objVersion == nil {
return &entity.ReadEntityResponse{
Entity: nil,
SummaryJson: nil,
}, nil
}
rsp := &entity.ReadEntityResponse{
Entity: objVersion,
}
if r.WithSummary {
// Since we do not store the summary, we can just recreate on demand
builder := i.kinds.GetSummaryBuilder(r.GRN.Kind)
if builder != nil {
summary, _, e2 := builder(ctx, r.GRN.UID, objVersion.Body)
if e2 != nil {
return nil, e2
}
rsp.SummaryJson, err = json.Marshal(summary)
}
}
return rsp, err
}
func (i *dummyEntityServer) BatchRead(ctx context.Context, batchR *entity.BatchReadEntityRequest) (*entity.BatchReadEntityResponse, error) {
results := make([]*entity.ReadEntityResponse, 0)
for _, r := range batchR.Batch {
resp, err := i.Read(ctx, r)
if err != nil {
return nil, err
}
results = append(results, resp)
}
return &entity.BatchReadEntityResponse{Results: results}, nil
}
func createContentsHash(contents []byte) string {
hash := md5.Sum(contents)
return hex.EncodeToString(hash[:])
}
func (i *dummyEntityServer) update(ctx context.Context, r *entity.AdminWriteEntityRequest, namespace string) (*entity.WriteEntityResponse, error) {
builder := i.kinds.GetSummaryBuilder(r.GRN.Kind)
if builder == nil {
return nil, fmt.Errorf("unsupported kind: " + r.GRN.Kind)
}
rsp := &entity.WriteEntityResponse{}
updatedCount, err := i.collection.Update(ctx, namespace, func(i *EntityWithHistory) (bool, *EntityWithHistory, error) {
if !r.GRN.Equals(i.Entity.GRN) {
return false, nil, nil
}
if r.PreviousVersion != "" && i.Entity.Version != r.PreviousVersion {
return false, nil, fmt.Errorf("expected the previous version to be %s, but was %s", r.PreviousVersion, i.Entity.Version)
}
prevVersion, err := strconv.Atoi(i.Entity.Version)
if err != nil {
return false, nil, err
}
modifier := store.UserFromContext(ctx)
updated := &entity.Entity{
GRN: r.GRN,
CreatedAt: i.Entity.CreatedAt,
CreatedBy: i.Entity.CreatedBy,
UpdatedAt: time.Now().UnixMilli(),
UpdatedBy: store.GetUserIDString(modifier),
Size: int64(len(r.Body)),
ETag: createContentsHash(r.Body),
Body: r.Body,
Version: fmt.Sprintf("%d", prevVersion+1),
}
versionInfo := &EntityVersionWithBody{
Body: r.Body,
EntityVersionInfo: &entity.EntityVersionInfo{
Version: updated.Version,
UpdatedAt: updated.UpdatedAt,
UpdatedBy: updated.UpdatedBy,
Size: updated.Size,
ETag: updated.ETag,
Comment: r.Comment,
},
}
rsp.Entity = versionInfo.EntityVersionInfo
rsp.Status = entity.WriteEntityResponse_UPDATED
// When saving, it must be different than the head version
if i.Entity.ETag == updated.ETag {
versionInfo.EntityVersionInfo.Version = i.Entity.Version
rsp.Status = entity.WriteEntityResponse_UNCHANGED
return false, nil, nil
}
return true, &EntityWithHistory{
Entity: updated,
History: append(i.History, versionInfo),
}, nil
})
if err != nil {
return nil, err
}
if updatedCount == 0 && rsp.Entity == nil {
return nil, fmt.Errorf("could not find object: %v", r.GRN)
}
return rsp, nil
}
func (i *dummyEntityServer) insert(ctx context.Context, r *entity.AdminWriteEntityRequest, namespace string) (*entity.WriteEntityResponse, error) {
modifier := store.GetUserIDString(store.UserFromContext(ctx))
rawObj := &entity.Entity{
GRN: r.GRN,
UpdatedAt: time.Now().UnixMilli(),
CreatedAt: time.Now().UnixMilli(),
CreatedBy: modifier,
UpdatedBy: modifier,
Size: int64(len(r.Body)),
ETag: createContentsHash(r.Body),
Body: r.Body,
Version: fmt.Sprintf("%d", 1),
}
info := &entity.EntityVersionInfo{
Version: rawObj.Version,
UpdatedAt: rawObj.UpdatedAt,
UpdatedBy: rawObj.UpdatedBy,
Size: rawObj.Size,
ETag: rawObj.ETag,
Comment: r.Comment,
}
newObj := &EntityWithHistory{
Entity: rawObj,
History: []*EntityVersionWithBody{{
EntityVersionInfo: info,
Body: r.Body,
}},
}
err := i.collection.Insert(ctx, namespace, newObj)
if err != nil {
return nil, err
}
return &entity.WriteEntityResponse{
Error: nil,
Entity: info,
Status: entity.WriteEntityResponse_CREATED,
}, nil
}
func (i *dummyEntityServer) Write(ctx context.Context, r *entity.WriteEntityRequest) (*entity.WriteEntityResponse, error) {
return i.doWrite(ctx, entity.ToAdminWriteEntityRequest(r))
}
func (i *dummyEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEntityRequest) (*entity.WriteEntityResponse, error) {
// Check permissions?
return i.doWrite(ctx, r)
}
func (i *dummyEntityServer) doWrite(ctx context.Context, r *entity.AdminWriteEntityRequest) (*entity.WriteEntityResponse, error) {
grn := getFullGRN(ctx, r.GRN)
namespace := namespaceFromUID(grn)
obj, err := i.collection.FindFirst(ctx, namespace, func(i *EntityWithHistory) (bool, error) {
if i == nil || r == nil {
return false, nil
}
return grn.Equals(i.Entity.GRN), nil
})
if err != nil {
return nil, err
}
if obj == nil {
return i.insert(ctx, r, namespace)
}
return i.update(ctx, r, namespace)
}
func (i *dummyEntityServer) Delete(ctx context.Context, r *entity.DeleteEntityRequest) (*entity.DeleteEntityResponse, error) {
grn := getFullGRN(ctx, r.GRN)
_, err := i.collection.Delete(ctx, namespaceFromUID(grn), func(i *EntityWithHistory) (bool, error) {
if grn.Equals(i.Entity.GRN) {
if r.PreviousVersion != "" && i.Entity.Version != r.PreviousVersion {
return false, fmt.Errorf("expected the previous version to be %s, but was %s", r.PreviousVersion, i.Entity.Version)
}
return true, nil
}
return false, nil
})
if err != nil {
return nil, err
}
return &entity.DeleteEntityResponse{
OK: true,
}, nil
}
func (i *dummyEntityServer) History(ctx context.Context, r *entity.EntityHistoryRequest) (*entity.EntityHistoryResponse, error) {
grn := getFullGRN(ctx, r.GRN)
obj, _, err := i.findEntity(ctx, grn, "")
if err != nil {
return nil, err
}
rsp := &entity.EntityHistoryResponse{}
if obj != nil {
// Return the most recent versions first
// Better? save them in this order?
for i := len(obj.History) - 1; i >= 0; i-- {
rsp.Versions = append(rsp.Versions, obj.History[i].EntityVersionInfo)
}
}
return rsp, nil
}
func (i *dummyEntityServer) Search(ctx context.Context, r *entity.EntitySearchRequest) (*entity.EntitySearchResponse, error) {
var kindMap map[string]bool
if len(r.Kind) != 0 {
kindMap = make(map[string]bool)
for _, k := range r.Kind {
kindMap[k] = true
}
}
// TODO more filters
objects, err := i.collection.Find(ctx, namespaceFromUID(&entity.GRN{}), func(i *EntityWithHistory) (bool, error) {
if len(r.Kind) != 0 {
if _, ok := kindMap[i.Entity.GRN.Kind]; !ok {
return false, nil
}
}
return true, nil
})
if err != nil {
return nil, err
}
searchResults := make([]*entity.EntitySearchResult, 0)
for _, o := range objects {
builder := i.kinds.GetSummaryBuilder(o.Entity.GRN.Kind)
if builder == nil {
continue
}
summary, clean, e2 := builder(ctx, o.Entity.GRN.UID, o.Entity.Body)
if e2 != nil {
continue
}
searchResults = append(searchResults, &entity.EntitySearchResult{
GRN: o.Entity.GRN,
Version: o.Entity.Version,
UpdatedAt: o.Entity.UpdatedAt,
UpdatedBy: o.Entity.UpdatedBy,
Name: summary.Name,
Description: summary.Description,
Body: clean,
})
}
return &entity.EntitySearchResponse{
Results: searchResults,
}, nil
}
// This sets the TenantId on the request GRN
func getFullGRN(ctx context.Context, grn *entity.GRN) *entity.GRN {
if grn.TenantId == 0 {
modifier := store.UserFromContext(ctx)
grn.TenantId = modifier.OrgID
}
return grn
}

89
pkg/services/store/entity/dummy/dummy_server_test.go
View File

@ -1,89 +0,0 @@
package dummy
import (
"encoding/json"
"fmt"
"testing"
"github.com/grafana/grafana/pkg/services/store/entity"
"github.com/stretchr/testify/require"
)
func TestRawEncoders(t *testing.T) {
body, err := json.Marshal(map[string]interface{}{
"hello": "world",
"field": 1.23,
})
require.NoError(t, err)
raw := &EntityVersionWithBody{
&entity.EntityVersionInfo{
Version: "A",
},
body,
}
b, err := json.Marshal(raw)
require.NoError(t, err)
str := string(b)
fmt.Printf("expect: %s", str)
require.JSONEq(t, `{"info":{"version":"A"},"body":"eyJmaWVsZCI6MS4yMywiaGVsbG8iOiJ3b3JsZCJ9"}`, str)
copy := &EntityVersionWithBody{}
err = json.Unmarshal(b, copy)
require.NoError(t, err)
}
func TestRawEntityWithHistory(t *testing.T) {
body, err := json.Marshal(map[string]interface{}{
"hello": "world",
"field": 1.23,
})
require.NoError(t, err)
raw := &EntityWithHistory{
Entity: &entity.Entity{
GRN: &entity.GRN{UID: "x"},
Version: "A",
Body: body,
},
History: make([]*EntityVersionWithBody, 0),
}
raw.History = append(raw.History, &EntityVersionWithBody{
&entity.EntityVersionInfo{
Version: "B",
},
body,
})
b, err := json.MarshalIndent(raw, "", " ")
require.NoError(t, err)
str := string(b)
//fmt.Printf("expect: %s", str)
require.JSONEq(t, `{
"entity": {
"GRN": {
"UID": "x"
},
"version": "A",
"body": {
"field": 1.23,
"hello": "world"
}
},
"history": [
{
"info": {
"version": "B"
},
"body": "eyJmaWVsZCI6MS4yMywiaGVsbG8iOiJ3b3JsZCJ9"
}
]
}`, str)
copy := &EntityVersionWithBody{}
err = json.Unmarshal(b, copy)
require.NoError(t, err)
}

2
pkg/services/store/entity/dummy/fake_store.go
View File

@ -25,7 +25,7 @@ func (i fakeEntityStore) Write(ctx context.Context, r *entity.WriteEntityRequest
return nil, fmt.Errorf("unimplemented") return nil, fmt.Errorf("unimplemented")
} }
func (i fakeEntityStore) Read(ctx context.Context, r *entity.ReadEntityRequest) (*entity.ReadEntityResponse, error) { func (i fakeEntityStore) Read(ctx context.Context, r *entity.ReadEntityRequest) (*entity.Entity, error) {
return nil, fmt.Errorf("unimplemented") return nil, fmt.Errorf("unimplemented")
} }

821
pkg/services/store/entity/entity.pb.go
View File

File diff suppressed because it is too large Load Diff

48
pkg/services/store/entity/entity.proto
View File

@ -21,47 +21,43 @@ message Entity {
// Entity identifier // Entity identifier
GRN GRN = 1; GRN GRN = 1;
// The version will change when the entity is saved. It is not necessarily sortable
string version = 2;
// Time in epoch milliseconds that the entity was created // Time in epoch milliseconds that the entity was created
int64 created_at = 2; int64 created_at = 3;
// Time in epoch milliseconds that the entity was updated // Time in epoch milliseconds that the entity was updated
int64 updated_at = 3; int64 updated_at = 4;
// Who created the entity // Who created the entity
string created_by = 4; string created_by = 5;
// Who updated the entity // Who updated the entity
string updated_by = 5; string updated_by = 6;
// Content Length // Content Length
int64 size = 6; int64 size = 7;
// MD5 digest of the body // MD5 digest of the body
string ETag = 7; string ETag = 8;
// Raw bytes of the storage entity. The kind will determine what is a valid payload // Raw bytes of the storage entity. The kind will determine what is a valid payload
bytes body = 8; bytes body = 9;
// Folder UID // Entity summary as JSON
string folder = 9; bytes summary_json = 10;
// Unique slug within folder (may be UID)
string slug = 10;
// The version will change when the entity is saved. It is not necessarily sortable
//
// NOTE: currently managed by the dashboard+dashboard_version tables
string version = 11;
// External location info // External location info
EntityOriginInfo origin = 12; EntityOriginInfo origin = 11;
} }
// This stores additional metadata for items entities that were synced from external systmes
message EntityOriginInfo { message EntityOriginInfo {
// NOTE: currently managed by the dashboard_provisioning table // identify the external source (plugin, git instance, etc)
string source = 1; string source = 1;
// Key in the upstream system // Key in the upstream system (git hash, file path, etc)
string key = 2; string key = 2;
// Time in epoch milliseconds that the entity was last synced with an external system (provisioning/git) // Time in epoch milliseconds that the entity was last synced with an external system (provisioning/git)
@ -122,14 +118,6 @@ message ReadEntityRequest {
bool with_summary = 4; bool with_summary = 4;
} }
message ReadEntityResponse {
// Entity details with the body removed
Entity entity = 1;
// Entity summary as JSON
bytes summary_json = 2;
}
//------------------------------------------------------ //------------------------------------------------------
// Make many read requests at once (by Kind+ID+version) // Make many read requests at once (by Kind+ID+version)
//------------------------------------------------------ //------------------------------------------------------
@ -139,7 +127,7 @@ message BatchReadEntityRequest {
} }
message BatchReadEntityResponse { message BatchReadEntityResponse {
repeated ReadEntityResponse results = 1; repeated Entity results = 1;
} }
//----------------------------------------------- //-----------------------------------------------
@ -375,7 +363,7 @@ message EntitySearchResponse {
// The entity store provides a basic CRUD (+watch eventually) interface for generic entitys // The entity store provides a basic CRUD (+watch eventually) interface for generic entitys
service EntityStore { service EntityStore {
rpc Read(ReadEntityRequest) returns (ReadEntityResponse); rpc Read(ReadEntityRequest) returns (Entity);
rpc BatchRead(BatchReadEntityRequest) returns (BatchReadEntityResponse); rpc BatchRead(BatchReadEntityRequest) returns (BatchReadEntityResponse);
rpc Write(WriteEntityRequest) returns (WriteEntityResponse); rpc Write(WriteEntityRequest) returns (WriteEntityResponse);
rpc Delete(DeleteEntityRequest) returns (DeleteEntityResponse); rpc Delete(DeleteEntityRequest) returns (DeleteEntityResponse);

10
pkg/services/store/entity/entity_grpc.pb.go
View File

@ -22,7 +22,7 @@ const _ = grpc.SupportPackageIsVersion7
// //
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type EntityStoreClient interface { type EntityStoreClient interface {
Read(ctx context.Context, in *ReadEntityRequest, opts ...grpc.CallOption) (*ReadEntityResponse, error) Read(ctx context.Context, in *ReadEntityRequest, opts ...grpc.CallOption) (*Entity, error)
BatchRead(ctx context.Context, in *BatchReadEntityRequest, opts ...grpc.CallOption) (*BatchReadEntityResponse, error) BatchRead(ctx context.Context, in *BatchReadEntityRequest, opts ...grpc.CallOption) (*BatchReadEntityResponse, error)
Write(ctx context.Context, in *WriteEntityRequest, opts ...grpc.CallOption) (*WriteEntityResponse, error) Write(ctx context.Context, in *WriteEntityRequest, opts ...grpc.CallOption) (*WriteEntityResponse, error)
Delete(ctx context.Context, in *DeleteEntityRequest, opts ...grpc.CallOption) (*DeleteEntityResponse, error) Delete(ctx context.Context, in *DeleteEntityRequest, opts ...grpc.CallOption) (*DeleteEntityResponse, error)
@ -40,8 +40,8 @@ func NewEntityStoreClient(cc grpc.ClientConnInterface) EntityStoreClient {
return &entityStoreClient{cc} return &entityStoreClient{cc}
} }
func (c *entityStoreClient) Read(ctx context.Context, in *ReadEntityRequest, opts ...grpc.CallOption) (*ReadEntityResponse, error) { func (c *entityStoreClient) Read(ctx context.Context, in *ReadEntityRequest, opts ...grpc.CallOption) (*Entity, error) {
out := new(ReadEntityResponse) out := new(Entity)
err := c.cc.Invoke(ctx, "/entity.EntityStore/Read", in, out, opts...) err := c.cc.Invoke(ctx, "/entity.EntityStore/Read", in, out, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
@ -107,7 +107,7 @@ func (c *entityStoreClient) AdminWrite(ctx context.Context, in *AdminWriteEntity
// All implementations should embed UnimplementedEntityStoreServer // All implementations should embed UnimplementedEntityStoreServer
// for forward compatibility // for forward compatibility
type EntityStoreServer interface { type EntityStoreServer interface {
Read(context.Context, *ReadEntityRequest) (*ReadEntityResponse, error) Read(context.Context, *ReadEntityRequest) (*Entity, error)
BatchRead(context.Context, *BatchReadEntityRequest) (*BatchReadEntityResponse, error) BatchRead(context.Context, *BatchReadEntityRequest) (*BatchReadEntityResponse, error)
Write(context.Context, *WriteEntityRequest) (*WriteEntityResponse, error) Write(context.Context, *WriteEntityRequest) (*WriteEntityResponse, error)
Delete(context.Context, *DeleteEntityRequest) (*DeleteEntityResponse, error) Delete(context.Context, *DeleteEntityRequest) (*DeleteEntityResponse, error)
@ -121,7 +121,7 @@ type EntityStoreServer interface {
type UnimplementedEntityStoreServer struct { type UnimplementedEntityStoreServer struct {
} }
func (UnimplementedEntityStoreServer) Read(context.Context, *ReadEntityRequest) (*ReadEntityResponse, error) { func (UnimplementedEntityStoreServer) Read(context.Context, *ReadEntityRequest) (*Entity, error) {
return nil, status.Errorf(codes.Unimplemented, "method Read not implemented") return nil, status.Errorf(codes.Unimplemented, "method Read not implemented")
} }
func (UnimplementedEntityStoreServer) BatchRead(context.Context, *BatchReadEntityRequest) (*BatchReadEntityResponse, error) { func (UnimplementedEntityStoreServer) BatchRead(context.Context, *BatchReadEntityRequest) (*BatchReadEntityResponse, error) {

16
pkg/services/store/entity/httpentitystore/service.go
View File

@ -90,17 +90,17 @@ func (s *httpEntityStore) doGetEntity(c *models.ReqContext) response.Response {
if err != nil { if err != nil {
return response.Error(500, "error fetching entity", err) return response.Error(500, "error fetching entity", err)
} }
if rsp.Entity == nil { if rsp == nil {
return response.Error(404, "not found", nil) return response.Error(404, "not found", nil)
} }
// Configure etag support // Configure etag support
currentEtag := rsp.Entity.ETag currentEtag := rsp.ETag
previousEtag := c.Req.Header.Get("If-None-Match") previousEtag := c.Req.Header.Get("If-None-Match")
if previousEtag == currentEtag { if previousEtag == currentEtag {
return response.CreateNormalResponse( return response.CreateNormalResponse(
http.Header{ http.Header{
"ETag": []string{rsp.Entity.ETag}, "ETag": []string{rsp.ETag},
}, },
[]byte{}, // nothing []byte{}, // nothing
http.StatusNotModified, // 304 http.StatusNotModified, // 304
@ -130,14 +130,14 @@ func (s *httpEntityStore) doGetRawEntity(c *models.ReqContext) response.Response
return response.Error(400, "Unsupported kind", err) return response.Error(400, "Unsupported kind", err)
} }
if rsp.Entity != nil && rsp.Entity.Body != nil { if rsp != nil && rsp.Body != nil {
// Configure etag support // Configure etag support
currentEtag := rsp.Entity.ETag currentEtag := rsp.ETag
previousEtag := c.Req.Header.Get("If-None-Match") previousEtag := c.Req.Header.Get("If-None-Match")
if previousEtag == currentEtag { if previousEtag == currentEtag {
return response.CreateNormalResponse( return response.CreateNormalResponse(
http.Header{ http.Header{
"ETag": []string{rsp.Entity.ETag}, "ETag": []string{rsp.ETag},
}, },
[]byte{}, // nothing []byte{}, // nothing
http.StatusNotModified, // 304 http.StatusNotModified, // 304
@ -152,7 +152,7 @@ func (s *httpEntityStore) doGetRawEntity(c *models.ReqContext) response.Response
"Content-Type": []string{mime}, "Content-Type": []string{mime},
"ETag": []string{currentEtag}, "ETag": []string{currentEtag},
}, },
rsp.Entity.Body, rsp.Body,
200, 200,
) )
} }
@ -279,7 +279,7 @@ func (s *httpEntityStore) doUpload(c *models.ReqContext) response.Response {
if err != nil { if err != nil {
return response.Error(500, "Internal Server Error", err) return response.Error(500, "Internal Server Error", err)
} }
if result.Entity != nil { if result.GRN != nil {
return response.Error(400, "File name already in use", err) return response.Error(400, "File name already in use", err)
} }
} }

59
pkg/services/store/entity/json.go
View File

@ -12,7 +12,6 @@ import (
func init() { //nolint:gochecknoinits func init() { //nolint:gochecknoinits
jsoniter.RegisterTypeEncoder("entity.EntitySearchResult", &searchResultCodec{}) jsoniter.RegisterTypeEncoder("entity.EntitySearchResult", &searchResultCodec{})
jsoniter.RegisterTypeEncoder("entity.WriteEntityResponse", &writeResponseCodec{}) jsoniter.RegisterTypeEncoder("entity.WriteEntityResponse", &writeResponseCodec{})
jsoniter.RegisterTypeEncoder("entity.ReadEntityResponse", &readResponseCodec{})
jsoniter.RegisterTypeEncoder("entity.Entity", &rawEntityCodec{}) jsoniter.RegisterTypeEncoder("entity.Entity", &rawEntityCodec{})
jsoniter.RegisterTypeDecoder("entity.Entity", &rawEntityCodec{}) jsoniter.RegisterTypeDecoder("entity.Entity", &rawEntityCodec{})
@ -91,33 +90,26 @@ func (codec *rawEntityCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream)
stream.WriteString(sEnc) // works for strings stream.WriteString(sEnc) // works for strings
} }
} }
if len(obj.SummaryJson) > 0 {
stream.WriteMore()
stream.WriteObjectField("summary")
writeRawJson(stream, obj.SummaryJson)
}
if obj.ETag != "" { if obj.ETag != "" {
stream.WriteMore() stream.WriteMore()
stream.WriteObjectField("etag") stream.WriteObjectField("etag")
stream.WriteString(obj.ETag) stream.WriteString(obj.ETag)
} }
if obj.Folder != "" {
stream.WriteMore()
stream.WriteObjectField("folder")
stream.WriteString(obj.Folder)
}
if obj.Slug != "" {
stream.WriteMore()
stream.WriteObjectField("slug")
stream.WriteString(obj.Slug)
}
if obj.Size > 0 { if obj.Size > 0 {
stream.WriteMore() stream.WriteMore()
stream.WriteObjectField("size") stream.WriteObjectField("size")
stream.WriteInt64(obj.Size) stream.WriteInt64(obj.Size)
} }
if obj.Origin != nil { if obj.Origin != nil {
stream.WriteMore() stream.WriteMore()
stream.WriteObjectField("origin") stream.WriteObjectField("origin")
stream.WriteVal(obj.Origin) stream.WriteVal(obj.Origin)
} }
stream.WriteObjectEnd() stream.WriteObjectEnd()
} }
@ -145,15 +137,20 @@ func readEntity(iter *jsoniter.Iterator, raw *Entity) {
raw.Size = iter.ReadInt64() raw.Size = iter.ReadInt64()
case "etag": case "etag":
raw.ETag = iter.ReadString() raw.ETag = iter.ReadString()
case "folder":
raw.Folder = iter.ReadString()
case "slug":
raw.Slug = iter.ReadString()
case "version": case "version":
raw.Version = iter.ReadString() raw.Version = iter.ReadString()
case "origin": case "origin":
raw.Origin = &EntityOriginInfo{} raw.Origin = &EntityOriginInfo{}
iter.ReadVal(raw.Origin) iter.ReadVal(raw.Origin)
case "summary":
var val interface{}
iter.ReadVal(&val) // ??? is there a smarter way to just keep the underlying bytes without read+marshal
body, err := json.Marshal(val)
if err != nil {
iter.ReportError("raw entity", "error reading summary body")
return
}
raw.SummaryJson = body
case "body": case "body":
var val interface{} var val interface{}
@ -181,34 +178,6 @@ func readEntity(iter *jsoniter.Iterator, raw *Entity) {
} }
} }
// Unlike the standard JSON marshal, this will write bytes as JSON when it can
type readResponseCodec struct{}
func (obj *ReadEntityResponse) MarshalJSON() ([]byte, error) {
var json = jsoniter.ConfigCompatibleWithStandardLibrary
return json.Marshal(obj)
}
func (codec *readResponseCodec) IsEmpty(ptr unsafe.Pointer) bool {
f := (*ReadEntityResponse)(ptr)
return f == nil
}
func (codec *readResponseCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) {
obj := (*ReadEntityResponse)(ptr)
stream.WriteObjectStart()
stream.WriteObjectField("entity")
stream.WriteVal(obj.Entity)
if len(obj.SummaryJson) > 0 {
stream.WriteMore()
stream.WriteObjectField("summary")
writeRawJson(stream, obj.SummaryJson)
}
stream.WriteObjectEnd()
}
// Unlike the standard JSON marshal, this will write bytes as JSON when it can // Unlike the standard JSON marshal, this will write bytes as JSON when it can
type searchResultCodec struct{} type searchResultCodec struct{}

58
pkg/services/store/entity/sqlstash/sql_storage_server.go
View File

@ -46,8 +46,7 @@ type sqlEntityServer struct {
func getReadSelect(r *entity.ReadEntityRequest) string { func getReadSelect(r *entity.ReadEntityRequest) string {
fields := []string{ fields := []string{
"tenant_id", "kind", "uid", // The PK "tenant_id", "kind", "uid", // The PK
"version", "slug", "folder", "version", "size", "etag", "errors", // errors are always returned
"size", "etag", "errors", // errors are always returned
"created_at", "created_by", "created_at", "created_by",
"updated_at", "updated_by", "updated_at", "updated_by",
"origin", "origin_key", "origin_ts"} "origin", "origin_key", "origin_ts"}
@ -56,23 +55,21 @@ func getReadSelect(r *entity.ReadEntityRequest) string {
fields = append(fields, `body`) fields = append(fields, `body`)
} }
if r.WithSummary { if r.WithSummary {
fields = append(fields, `name`, `description`, `labels`, `fields`) fields = append(fields, "name", "slug", "folder", "description", "labels", "fields")
} }
return "SELECT " + strings.Join(fields, ",") + " FROM entity WHERE " return "SELECT " + strings.Join(fields, ",") + " FROM entity WHERE "
} }
func (s *sqlEntityServer) rowToReadEntityResponse(ctx context.Context, rows *sql.Rows, r *entity.ReadEntityRequest) (*entity.ReadEntityResponse, error) { func (s *sqlEntityServer) rowToReadEntityResponse(ctx context.Context, rows *sql.Rows, r *entity.ReadEntityRequest) (*entity.Entity, error) {
raw := &entity.Entity{ raw := &entity.Entity{
GRN: &entity.GRN{}, GRN: &entity.GRN{},
Origin: &entity.EntityOriginInfo{}, Origin: &entity.EntityOriginInfo{},
} }
slug := ""
summaryjson := &summarySupport{} summaryjson := &summarySupport{}
args := []interface{}{ args := []interface{}{
&raw.GRN.TenantId, &raw.GRN.Kind, &raw.GRN.UID, &raw.GRN.TenantId, &raw.GRN.Kind, &raw.GRN.UID,
&raw.Version, &slug, &raw.Folder, &raw.Version, &raw.Size, &raw.ETag, &summaryjson.errors,
&raw.Size, &raw.ETag, &summaryjson.errors,
&raw.CreatedAt, &raw.CreatedBy, &raw.CreatedAt, &raw.CreatedBy,
&raw.UpdatedAt, &raw.UpdatedBy, &raw.UpdatedAt, &raw.UpdatedBy,
&raw.Origin.Source, &raw.Origin.Key, &raw.Origin.Time, &raw.Origin.Source, &raw.Origin.Key, &raw.Origin.Time,
@ -81,7 +78,7 @@ func (s *sqlEntityServer) rowToReadEntityResponse(ctx context.Context, rows *sql
args = append(args, &raw.Body) args = append(args, &raw.Body)
} }
if r.WithSummary { if r.WithSummary {
args = append(args, &summaryjson.name, &summaryjson.description, &summaryjson.labels, &summaryjson.fields) args = append(args, &summaryjson.name, &summaryjson.slug, &summaryjson.folder, &summaryjson.description, &summaryjson.labels, &summaryjson.fields)
} }
err := rows.Scan(args...) err := rows.Scan(args...)
@ -93,10 +90,6 @@ func (s *sqlEntityServer) rowToReadEntityResponse(ctx context.Context, rows *sql
raw.Origin = nil raw.Origin = nil
} }
rsp := &entity.ReadEntityResponse{
Entity: raw,
}
if r.WithSummary || summaryjson.errors != nil { if r.WithSummary || summaryjson.errors != nil {
summary, err := summaryjson.toEntitySummary() summary, err := summaryjson.toEntitySummary()
if err != nil { if err != nil {
@ -107,9 +100,9 @@ func (s *sqlEntityServer) rowToReadEntityResponse(ctx context.Context, rows *sql
if err != nil { if err != nil {
return nil, err return nil, err
} }
rsp.SummaryJson = js raw.SummaryJson = js
} }
return rsp, nil return raw, nil
} }
func (s *sqlEntityServer) validateGRN(ctx context.Context, grn *entity.GRN) (*entity.GRN, error) { func (s *sqlEntityServer) validateGRN(ctx context.Context, grn *entity.GRN) (*entity.GRN, error) {
@ -138,7 +131,7 @@ func (s *sqlEntityServer) validateGRN(ctx context.Context, grn *entity.GRN) (*en
return grn, nil return grn, nil
} }
func (s *sqlEntityServer) Read(ctx context.Context, r *entity.ReadEntityRequest) (*entity.ReadEntityResponse, error) { func (s *sqlEntityServer) Read(ctx context.Context, r *entity.ReadEntityRequest) (*entity.Entity, error) {
if r.Version != "" { if r.Version != "" {
return s.readFromHistory(ctx, r) return s.readFromHistory(ctx, r)
} }
@ -157,13 +150,13 @@ func (s *sqlEntityServer) Read(ctx context.Context, r *entity.ReadEntityRequest)
defer func() { _ = rows.Close() }() defer func() { _ = rows.Close() }()
if !rows.Next() { if !rows.Next() {
return &entity.ReadEntityResponse{}, nil return &entity.Entity{}, nil
} }
return s.rowToReadEntityResponse(ctx, rows, r) return s.rowToReadEntityResponse(ctx, rows, r)
} }
func (s *sqlEntityServer) readFromHistory(ctx context.Context, r *entity.ReadEntityRequest) (*entity.ReadEntityResponse, error) { func (s *sqlEntityServer) readFromHistory(ctx context.Context, r *entity.ReadEntityRequest) (*entity.Entity, error) {
grn, err := s.validateGRN(ctx, r.GRN) grn, err := s.validateGRN(ctx, r.GRN)
if err != nil { if err != nil {
return nil, err return nil, err
@ -185,15 +178,12 @@ func (s *sqlEntityServer) readFromHistory(ctx context.Context, r *entity.ReadEnt
// Version or key not found // Version or key not found
if !rows.Next() { if !rows.Next() {
return &entity.ReadEntityResponse{}, nil return &entity.Entity{}, nil
} }
raw := &entity.Entity{ raw := &entity.Entity{
GRN: r.GRN, GRN: r.GRN,
} }
rsp := &entity.ReadEntityResponse{
Entity: raw,
}
err = rows.Scan(&raw.Body, &raw.Size, &raw.ETag, &raw.UpdatedAt, &raw.UpdatedBy) err = rows.Scan(&raw.Body, &raw.Size, &raw.ETag, &raw.UpdatedAt, &raw.UpdatedBy)
if err != nil { if err != nil {
return nil, err return nil, err
@ -210,7 +200,7 @@ func (s *sqlEntityServer) readFromHistory(ctx context.Context, r *entity.ReadEnt
val, out, err := builder(ctx, r.GRN.UID, raw.Body) val, out, err := builder(ctx, r.GRN.UID, raw.Body)
if err == nil { if err == nil {
raw.Body = out // cleaned up raw.Body = out // cleaned up
rsp.SummaryJson, err = json.Marshal(val) raw.SummaryJson, err = json.Marshal(val)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -220,10 +210,10 @@ func (s *sqlEntityServer) readFromHistory(ctx context.Context, r *entity.ReadEnt
// Clear the body if not requested // Clear the body if not requested
if !r.WithBody { if !r.WithBody {
rsp.Entity.Body = nil raw.Body = nil
} }
return rsp, err return raw, err
} }
func (s *sqlEntityServer) BatchRead(ctx context.Context, b *entity.BatchReadEntityRequest) (*entity.BatchReadEntityResponse, error) { func (s *sqlEntityServer) BatchRead(ctx context.Context, b *entity.BatchReadEntityRequest) (*entity.BatchReadEntityResponse, error) {
@ -306,12 +296,6 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
return nil, err return nil, err
} }
t := summary.name
if t == "" {
t = r.GRN.UID
}
slug := slugify.Slugify(t)
etag := createContentsHash(body) etag := createContentsHash(body)
rsp := &entity.WriteEntityResponse{ rsp := &entity.WriteEntityResponse{
GRN: grn, GRN: grn,
@ -479,8 +463,8 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
" ?, ?, ?)", " ?, ?, ?)",
oid, grn.TenantId, grn.Kind, grn.UID, r.Folder, oid, grn.TenantId, grn.Kind, grn.UID, r.Folder,
versionInfo.Size, body, etag, versionInfo.Version, versionInfo.Size, body, etag, versionInfo.Version,
updatedAt, createdBy, createdAt, createdBy, // created + updated are the same updatedAt, createdBy, createdAt, createdBy,
summary.model.Name, summary.model.Description, slug, summary.model.Name, summary.model.Description, summary.model.Slug,
summary.labels, summary.fields, summary.errors, summary.labels, summary.fields, summary.errors,
origin.Source, origin.Key, origin.Time, origin.Source, origin.Key, origin.Time,
) )
@ -550,6 +534,16 @@ func (s *sqlEntityServer) prepare(ctx context.Context, r *entity.AdminWriteEntit
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
// Update a summary based on the name (unless the root suggested one)
if summary.Slug == "" {
t := summary.Name
if t == "" {
t = r.GRN.UID
}
summary.Slug = slugify.Slugify(t)
}
return summaryjson, body, nil return summaryjson, body, nil
} }

15
pkg/services/store/entity/sqlstash/summary_handler.go
View File

@ -10,6 +10,8 @@ type summarySupport struct {
model *models.EntitySummary model *models.EntitySummary
name string name string
description *string // null or empty description *string // null or empty
slug *string // null or empty
folder *string // null or empty
labels *string labels *string
fields *string fields *string
errors *string // should not allow saving with this! errors *string // should not allow saving with this!
@ -32,7 +34,12 @@ func newSummarySupport(summary *models.EntitySummary) (*summarySupport, error) {
if summary.Description != "" { if summary.Description != "" {
s.description = &summary.Description s.description = &summary.Description
} }
if summary.Slug != "" {
s.slug = &summary.Slug
}
if summary.Folder != "" {
s.folder = &summary.Folder
}
if len(summary.Labels) > 0 { if len(summary.Labels) > 0 {
js, err = json.Marshal(summary.Labels) js, err = json.Marshal(summary.Labels)
if err != nil { if err != nil {
@ -71,6 +78,12 @@ func (s summarySupport) toEntitySummary() (*models.EntitySummary, error) {
if s.description != nil { if s.description != nil {
summary.Description = *s.description summary.Description = *s.description
} }
if s.slug != nil {
summary.Slug = *s.slug
}
if s.folder != nil {
summary.Folder = *s.folder
}
if s.labels != nil { if s.labels != nil {
b := []byte(*s.labels) b := []byte(*s.labels)
err = json.Unmarshal(b, &summary.Labels) err = json.Unmarshal(b, &summary.Labels)

16
pkg/services/store/entity/tests/server_integration_test.go
View File

@ -134,7 +134,7 @@ func TestIntegrationEntityServer(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, resp) require.NotNil(t, resp)
require.Nil(t, resp.Entity) require.Nil(t, resp.GRN)
}) })
t.Run("should be able to read persisted objects", func(t *testing.T) { t.Run("should be able to read persisted objects", func(t *testing.T) {
@ -162,9 +162,9 @@ func TestIntegrationEntityServer(t *testing.T) {
}) })
require.NoError(t, err) require.NoError(t, err)
require.Nil(t, readResp.SummaryJson) require.Nil(t, readResp.SummaryJson)
require.NotNil(t, readResp.Entity) require.NotNil(t, readResp)
foundGRN := readResp.Entity.GRN foundGRN := readResp.GRN
require.NotNil(t, foundGRN) require.NotNil(t, foundGRN)
require.Equal(t, testCtx.user.OrgID, foundGRN.TenantId) // orgId becomes the tenant id when not set require.Equal(t, testCtx.user.OrgID, foundGRN.TenantId) // orgId becomes the tenant id when not set
require.Equal(t, grn.Kind, foundGRN.Kind) require.Equal(t, grn.Kind, foundGRN.Kind)
@ -179,7 +179,7 @@ func TestIntegrationEntityServer(t *testing.T) {
body: body, body: body,
version: &firstVersion, version: &firstVersion,
} }
requireEntityMatch(t, readResp.Entity, objectMatcher) requireEntityMatch(t, readResp, objectMatcher)
deleteResp, err := testCtx.client.Delete(ctx, &entity.DeleteEntityRequest{ deleteResp, err := testCtx.client.Delete(ctx, &entity.DeleteEntityRequest{
GRN: grn, GRN: grn,
@ -194,7 +194,7 @@ func TestIntegrationEntityServer(t *testing.T) {
WithBody: true, WithBody: true,
}) })
require.NoError(t, err) require.NoError(t, err)
require.Nil(t, readRespAfterDelete.Entity) require.Nil(t, readRespAfterDelete.GRN)
}) })
t.Run("should be able to update an object", func(t *testing.T) { t.Run("should be able to update an object", func(t *testing.T) {
@ -258,7 +258,7 @@ func TestIntegrationEntityServer(t *testing.T) {
}) })
require.NoError(t, err) require.NoError(t, err)
require.Nil(t, readRespLatest.SummaryJson) require.Nil(t, readRespLatest.SummaryJson)
requireEntityMatch(t, readRespLatest.Entity, latestMatcher) requireEntityMatch(t, readRespLatest, latestMatcher)
readRespFirstVer, err := testCtx.client.Read(ctx, &entity.ReadEntityRequest{ readRespFirstVer, err := testCtx.client.Read(ctx, &entity.ReadEntityRequest{
GRN: grn, GRN: grn,
@ -268,8 +268,8 @@ func TestIntegrationEntityServer(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.Nil(t, readRespFirstVer.SummaryJson) require.Nil(t, readRespFirstVer.SummaryJson)
require.NotNil(t, readRespFirstVer.Entity) require.NotNil(t, readRespFirstVer)
requireEntityMatch(t, readRespFirstVer.Entity, rawEntityMatcher{ requireEntityMatch(t, readRespFirstVer, rawEntityMatcher{
grn: grn, grn: grn,
createdRange: []time.Time{before, time.Now()}, createdRange: []time.Time{before, time.Now()},
updatedRange: []time.Time{before, time.Now()}, updatedRange: []time.Time{before, time.Now()},