Fix: velaql fail to parse query result to json format (#4011)

Signed-off-by: yangsoon <songyang.song@alibaba-inc.com>

Co-authored-by: yangsoon <songyang.song@alibaba-inc.com>

pkg/cue/utils.go

@@ -16,6 +16,16 @@ limitations under the License.
 
 package cue
 
+import (
+	"bytes"
+
+	"cuelang.org/go/pkg/encoding/json"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	"github.com/oam-dev/kubevela/pkg/cue/model/value"
+)
+
 // int data can evaluate with number in CUE, so it's OK if we convert the original float type data to int
 func isIntegral(val float64) bool {
 	return val == float64(int(val))
@@ -55,3 +65,16 @@ func intifyMap(m map[string]interface{}) interface{} {
 	}
 	return m2
 }
+
+// FillUnstructuredObject fill runtime.Unstructured to *value.Value
+func FillUnstructuredObject(v *value.Value, obj runtime.Unstructured, paths ...string) error {
+	var buf bytes.Buffer
+	if err := unstructured.UnstructuredJSONScheme.Encode(obj, &buf); err != nil {
+		return v.FillObject(err.Error(), "err")
+	}
+	expr, err := json.Unmarshal(buf.Bytes())
+	if err != nil {
+		return v.FillObject(err.Error(), "err")
+	}
+	return v.FillObject(expr, paths...)
+}

pkg/cue/utils_test.go

@@ -0,0 +1,74 @@
+/*
+ Copyright 2022. The KubeVela Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package cue
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/oam-dev/kubevela/pkg/cue/model/value"
+)
+
+func TestFillUnstructuredObject(t *testing.T) {
+	testcases := map[string]struct {
+		obj  *unstructured.Unstructured
+		json string
+	}{
+		"test unstructured object with nil value": {
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"spec": map[string]interface{}{
+						"template": map[string]interface{}{
+							"metadata": map[string]interface{}{
+								"creationTimestamp": nil,
+							},
+						},
+					},
+				},
+			},
+			json: `{"object":{"apiVersion":"apps/v1","kind":"Deployment","spec":{"template":{"metadata":{"creationTimestamp":null}}}}}`,
+		},
+		"test unstructured object without nil value": {
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"creationTimestamp": "2022-05-25T12:07:02Z",
+					},
+				},
+			},
+			json: `{"object":{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"creationTimestamp":"2022-05-25T12:07:02Z"}}}`,
+		},
+	}
+
+	for name, testcase := range testcases {
+		t.Run(name, func(t *testing.T) {
+			value, err := value.NewValue("", nil, "")
+			assert.NoError(t, err)
+			err = FillUnstructuredObject(value, testcase.obj, "object")
+			assert.NoError(t, err)
+			json, err := value.CueValue().MarshalJSON()
+			assert.NoError(t, err)
+			assert.Equal(t, testcase.json, string(json))
+		})
+	}
+}

pkg/velaql/providers/query/handler.go

@@ -110,7 +110,7 @@ func (h *provider) ListResourcesInApp(ctx wfContext.Context, v *value.Value, act
 	if err != nil {
 		return v.FillObject(err.Error(), "err")
 	}
-	return v.FillObject(appResList, "list")
+	return fillQueryResult(v, appResList, "list")
 }
 
 // ListAppliedResources list applied resource from tracker, this provider only queries the metadata.
@@ -133,10 +133,10 @@ func (h *provider) ListAppliedResources(ctx wfContext.Context, v *value.Value, a
 	if err != nil {
 		return v.FillObject(err.Error(), "err")
 	}
-	return v.FillObject(appResList, "list")
+	return fillQueryResult(v, appResList, "list")
 }
 
-// ListAppliedResources list applied resource from tracker
+// GetApplicationResourceTree get resource tree of application
 func (h *provider) GetApplicationResourceTree(ctx wfContext.Context, v *value.Value, act types.Action) error {
 	val, err := v.LookupValue("app")
 	if err != nil {
@@ -198,7 +198,7 @@ func (h *provider) GetApplicationResourceTree(ctx wfContext.Context, v *value.Va
 		}
 		resource.ResourceTree = &root
 	}
-	return v.FillObject(appResList, "list")
+	return fillQueryResult(v, appResList, "list")
 }
 
 func (h *provider) CollectPods(ctx wfContext.Context, v *value.Value, act types.Action) error {
@@ -229,7 +229,7 @@ func (h *provider) CollectPods(ctx wfContext.Context, v *value.Value, act types.
 	if err != nil {
 		return v.FillObject(err.Error(), "err")
 	}
-	return v.FillObject(pods, "list")
+	return fillQueryResult(v, pods, "list")
 }
 
 func (h *provider) SearchEvents(ctx wfContext.Context, v *value.Value, act types.Action) error {
@@ -258,7 +258,7 @@ func (h *provider) SearchEvents(ctx wfContext.Context, v *value.Value, act types
 	if err := h.cli.List(listCtx, &eventList, listOpts...); err != nil {
 		return v.FillObject(err.Error(), "err")
 	}
-	return v.FillObject(eventList.Items, "list")
+	return fillQueryResult(v, eventList.Items, "list")
 }
 
 // GeneratorServiceEndpoints generator service endpoints is available for common component type,
@@ -388,7 +388,7 @@ func (h *provider) GeneratorServiceEndpoints(wfctx wfContext.Context, v *value.V
 			serviceEndpoints = append(serviceEndpoints, generatorFromService(service, selectorNodeIP, cluster, resource.Component, fmt.Sprintf("/seldon/%s/%s", resource.Namespace, resource.Name))...)
 		}
 	}
-	return v.FillObject(serviceEndpoints, "list")
+	return fillQueryResult(v, serviceEndpoints, "list")
 }
 
 var (

pkg/velaql/providers/query/utils.go

@@ -0,0 +1,38 @@
+/*
+ Copyright 2022. The KubeVela Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package query
+
+import (
+	"encoding/json"
+
+	cuejson "cuelang.org/go/pkg/encoding/json"
+
+	"github.com/oam-dev/kubevela/pkg/cue/model/value"
+)
+
+// fillQueryResult help fill query result which contains k8s object to *value.Value
+func fillQueryResult(v *value.Value, res interface{}, paths ...string) error {
+	b, err := json.Marshal(res)
+	if err != nil {
+		return v.FillObject(err, "err")
+	}
+	expr, err := cuejson.Unmarshal(b)
+	if err != nil {
+		return v.FillObject(err, "err")
+	}
+	return v.FillObject(expr, paths...)
+}

pkg/velaql/providers/query/utils_test.go

@@ -0,0 +1,83 @@
+/*
+ Copyright 2022. The KubeVela Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package query
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/oam-dev/kubevela/pkg/cue/model/value"
+)
+
+func TestFillQueryResult(t *testing.T) {
+	testcases := map[string]struct {
+		queryRes interface{}
+		json     string
+	}{
+		"test fill query result which contains *unstructured.Unstructured": {
+			queryRes: []Resource{
+				{
+					Cluster:   "local",
+					Component: "web",
+					Revision:  "v1",
+					Object: &unstructured.Unstructured{
+						Object: map[string]interface{}{
+							"apiVersion": "apps/v1",
+							"kind":       "Deployment",
+							"spec": map[string]interface{}{
+								"template": map[string]interface{}{
+									"metadata": map[string]interface{}{
+										"creationTimestamp": nil,
+									},
+								},
+							},
+						},
+					},
+				},
+				{
+					Cluster:   "ap-southeast-1",
+					Component: "web",
+					Revision:  "v2",
+					Object: &unstructured.Unstructured{
+						Object: map[string]interface{}{
+							"apiVersion": "apps/v1",
+							"kind":       "Deployment",
+							"metadata": map[string]interface{}{
+								"creationTimestamp": "2022-05-25T12:07:02Z",
+							},
+						},
+					},
+				},
+			},
+			json: `{"list":[{"cluster":"local","component":"web","revision":"v1","object":{"apiVersion":"apps/v1","kind":"Deployment","spec":{"template":{"metadata":{"creationTimestamp":null}}}}},{"cluster":"ap-southeast-1","component":"web","revision":"v2","object":{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"creationTimestamp":"2022-05-25T12:07:02Z"}}}]}`,
+		},
+	}
+
+	for name, testcase := range testcases {
+		t.Run(name, func(t *testing.T) {
+			value, err := value.NewValue("", nil, "")
+			assert.NoError(t, err)
+			err = fillQueryResult(value, testcase.queryRes, "list")
+			assert.NoError(t, err)
+			json, err := value.CueValue().MarshalJSON()
+			assert.NoError(t, err)
+			assert.Equal(t, testcase.json, string(json))
+		})
+	}
+}

pkg/workflow/providers/kube/handle.go

@@ -26,6 +26,7 @@ import (
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	"github.com/oam-dev/kubevela/pkg/auth"
+	"github.com/oam-dev/kubevela/pkg/cue"
 	"github.com/oam-dev/kubevela/pkg/cue/model"
 	"github.com/oam-dev/kubevela/pkg/cue/model/value"
 	"github.com/oam-dev/kubevela/pkg/multicluster"
@@ -92,7 +93,7 @@ func (h *provider) Apply(ctx wfContext.Context, v *value.Value, act types.Action
 	if err := h.apply(deployCtx, cluster, common.WorkflowResourceCreator, workload); err != nil {
 		return err
 	}
-	return v.FillObject(workload.Object, "value")
+	return cue.FillUnstructuredObject(v, workload, "value")
 }
 
 // ApplyInParallel create or update CRs in parallel.
@@ -136,7 +137,6 @@ func (h *provider) Read(ctx wfContext.Context, v *value.Value, act types.Action)
 	if err != nil {
 		return err
 	}
-
 	obj := new(unstructured.Unstructured)
 	if err := val.UnmarshalTo(obj); err != nil {
 		return err
@@ -154,7 +154,7 @@ func (h *provider) Read(ctx wfContext.Context, v *value.Value, act types.Action)
 	if err := h.cli.Get(readCtx, key, obj); err != nil {
 		return v.FillObject(err.Error(), "err")
 	}
-	return v.FillObject(obj.Object, "value")
+	return cue.FillUnstructuredObject(v, obj, "value")
 }
 
 // List lists CRs from cluster.
@@ -197,7 +197,7 @@ func (h *provider) List(ctx wfContext.Context, v *value.Value, act types.Action)
 	if err := h.cli.List(readCtx, list, listOpts...); err != nil {
 		return v.FillObject(err.Error(), "err")
 	}
-	return v.FillObject(list, "list")
+	return cue.FillUnstructuredObject(v, list, "list")
 }
 
 // Delete deletes CR from cluster.

references/cli/velaql_test.go

@@ -61,68 +61,76 @@ var _ = Describe("Test velaQL", func() {
 					},
 				},
 			},
-			Status: common.AppStatus{
-				AppliedResources: []common.ClusterObjectReference{
-					{
-						Cluster: "",
-						ObjectReference: corev1.ObjectReference{
-							Kind:       "Ingress",
-							Namespace:  "default",
-							Name:       "ingress-http",
-							APIVersion: "networking.k8s.io/v1beta1",
-						},
+		}
+
+		err := k8sClient.Create(context.TODO(), testApp)
+		Expect(err).Should(BeNil())
+
+		testApp.Status = common.AppStatus{
+			AppliedResources: []common.ClusterObjectReference{
+				{
+					Cluster: "",
+					ObjectReference: corev1.ObjectReference{
+						Kind:       "Ingress",
+						Namespace:  "default",
+						Name:       "ingress-http",
+						APIVersion: "networking.k8s.io/v1beta1",
 					},
-					{
-						Cluster: "",
-						ObjectReference: corev1.ObjectReference{
-							Kind:       "Ingress",
-							Namespace:  "default",
-							Name:       "ingress-https",
-							APIVersion: "networking.k8s.io/v1",
-						},
+				},
+				{
+					Cluster: "",
+					ObjectReference: corev1.ObjectReference{
+						Kind:       "Ingress",
+						Namespace:  "default",
+						Name:       "ingress-https",
+						APIVersion: "networking.k8s.io/v1",
 					},
-					{
-						Cluster: "",
-						ObjectReference: corev1.ObjectReference{
-							Kind:       "Ingress",
-							Namespace:  "default",
-							Name:       "ingress-paths",
-							APIVersion: "networking.k8s.io/v1",
-						},
+				},
+				{
+					Cluster: "",
+					ObjectReference: corev1.ObjectReference{
+						Kind:       "Ingress",
+						Namespace:  "default",
+						Name:       "ingress-paths",
+						APIVersion: "networking.k8s.io/v1",
 					},
-					{
-						Cluster: "",
-						ObjectReference: corev1.ObjectReference{
-							Kind:      "Service",
-							Namespace: "default",
-							Name:      "nodeport",
-						},
+				},
+				{
+					Cluster: "",
+					ObjectReference: corev1.ObjectReference{
+						Kind:      "Service",
+						Namespace: "default",
+						Name:      "nodeport",
 					},
-					{
-						Cluster: "",
-						ObjectReference: corev1.ObjectReference{
-							Kind:      "Service",
-							Namespace: "default",
-							Name:      "loadbalancer",
-						},
+				},
+				{
+					Cluster: "",
+					ObjectReference: corev1.ObjectReference{
+						Kind:      "Service",
+						Namespace: "default",
+						Name:      "loadbalancer",
 					},
-					{
-						Cluster: "",
-						ObjectReference: corev1.ObjectReference{
-							Kind:      helmapi.HelmReleaseGVK.Kind,
-							Namespace: "default",
-							Name:      "helmRelease",
-						},
+				},
+				{
+					Cluster: "",
+					ObjectReference: corev1.ObjectReference{
+						Kind:      helmapi.HelmReleaseGVK.Kind,
+						Namespace: "default",
+						Name:      "helmRelease",
 					},
 				},
 			},
 		}
-		err := k8sClient.Create(context.TODO(), testApp)
+
+		err = k8sClient.Status().Update(context.TODO(), testApp)
 		Expect(err).Should(BeNil())
 
 		var mr []v1beta1.ManagedResource
 		for i := range testApp.Status.AppliedResources {
 			mr = append(mr, v1beta1.ManagedResource{
+				OAMObjectReference: common.OAMObjectReference{
+					Component: "endpoints-test",
+				},
 				ClusterObjectReference: testApp.Status.AppliedResources[i],
 			})
 		}

test/e2e-apiserver-test/testdata/component-pod-view.yaml

@@ -52,7 +52,7 @@ data:
         }
       }
 
-      podsWithCluster: [ for i, pods in podsMap for podObj in pods.list {
+      podsWithCluster: [ for i, pods in podsMap if pods.list != null for podObj in pods.list {
         cluster: pods.cluster
         obj:     podObj
       }]