mirror of https://github.com/kubevela/kubevela.git
Feat: Enable CueX compiler in component & trait templating (#6720)
* Feat: Enable CueX compiler in component & trait templating * Feat: Enable CueX compiler in component & trait templating Signed-off-by: Brian Kane <briankane1@gmail.com> --------- Signed-off-by: Brian Kane <briankane1@gmail.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Brian Kane
GitHub
parent
0751c15ee5
commit
8ee02c6506
|
|
@ -20,6 +20,8 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/kubevela/pkg/cue/cuex"
|
||||
|
||||
pkgclient "github.com/kubevela/pkg/controller/client"
|
||||
ctrlrec "github.com/kubevela/pkg/controller/reconciler"
|
||||
"github.com/kubevela/pkg/controller/sharding"
|
||||
|
|
@ -35,7 +37,6 @@ import (
|
|||
oamcontroller "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev"
|
||||
"github.com/oam-dev/kubevela/pkg/oam"
|
||||
"github.com/oam-dev/kubevela/pkg/resourcekeeper"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/providers"
|
||||
)
|
||||
|
||||
// CoreOptions contains everything necessary to create and run vela-core
|
||||
|
|
@ -129,8 +130,8 @@ func (s *CoreOptions) Flags() cliflag.NamedFlagSets {
|
|||
gfs.BoolVar(&s.EnableClusterGateway, "enable-cluster-gateway", s.EnableClusterGateway, "Enable cluster-gateway to use multicluster, disabled by default.")
|
||||
gfs.BoolVar(&s.EnableClusterMetrics, "enable-cluster-metrics", s.EnableClusterMetrics, "Enable cluster-metrics-management to collect metrics from clusters with cluster-gateway, disabled by default. When this param is enabled, enable-cluster-gateway should be enabled")
|
||||
gfs.DurationVar(&s.ClusterMetricsInterval, "cluster-metrics-interval", s.ClusterMetricsInterval, "The interval that ClusterMetricsMgr will collect metrics from clusters, default value is 15 seconds.")
|
||||
gfs.BoolVar(&providers.EnableExternalPackageForDefaultCompiler, "enable-external-package-for-default-compiler", providers.EnableExternalPackageForDefaultCompiler, "Enable external package for default compiler")
|
||||
gfs.BoolVar(&providers.EnableExternalPackageWatchForDefaultCompiler, "enable-external-package-watch-for-default-compiler", providers.EnableExternalPackageWatchForDefaultCompiler, "Enable external package watch for default compiler")
|
||||
gfs.BoolVar(&cuex.EnableExternalPackageForDefaultCompiler, "enable-external-package-for-default-compiler", cuex.EnableExternalPackageForDefaultCompiler, "Enable external package for default compiler")
|
||||
gfs.BoolVar(&cuex.EnableExternalPackageWatchForDefaultCompiler, "enable-external-package-watch-for-default-compiler", cuex.EnableExternalPackageWatchForDefaultCompiler, "Enable external package watch for default compiler")
|
||||
|
||||
s.ControllerArgs.AddFlags(fss.FlagSet("controllerArgs"), s.ControllerArgs)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,9 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/kubevela/pkg/cue/cuex"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
oamcontroller "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev"
|
||||
)
|
||||
|
|
@ -96,3 +98,26 @@ func TestCoreOptions_Flags(t *testing.T) {
|
|||
t.Errorf("Flags() diff: %v", cmp.Diff(opt, expected, cmp.AllowUnexported(CoreOptions{})))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCuexOptions_Flags(t *testing.T) {
|
||||
pflag.NewFlagSet("test", pflag.ContinueOnError)
|
||||
cuex.EnableExternalPackageForDefaultCompiler = false
|
||||
cuex.EnableExternalPackageWatchForDefaultCompiler = false
|
||||
|
||||
opts := &CoreOptions{
|
||||
ControllerArgs: &oamcontroller.Args{},
|
||||
}
|
||||
fss := opts.Flags()
|
||||
|
||||
args := []string{
|
||||
"--enable-external-package-for-default-compiler=true",
|
||||
"--enable-external-package-watch-for-default-compiler=true",
|
||||
}
|
||||
err := fss.FlagSet("generic").Parse(args)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
assert.True(t, cuex.EnableExternalPackageForDefaultCompiler, "The --enable-external-package-for-default-compiler flag should be enabled")
|
||||
assert.True(t, cuex.EnableExternalPackageWatchForDefaultCompiler, "The --enable-external-package-watch-for-default-compiler flag should be enabled")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,351 @@
|
|||
/*
|
||||
Copyright 2025 The KubeVela Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cuex_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kubevela/pkg/cue/cuex"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
||||
"github.com/kubevela/pkg/util/singleton"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/rest"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
|
||||
"github.com/oam-dev/kubevela/apis/types"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/definition"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/process"
|
||||
)
|
||||
|
||||
var testCtx = struct {
|
||||
K8sClient client.Client
|
||||
ReturnVal string
|
||||
CueXTestPackage string
|
||||
Namespace string
|
||||
CueXPath string
|
||||
ExternalFnName string
|
||||
InputParamName string
|
||||
OutputParamName string
|
||||
}{
|
||||
ReturnVal: "external",
|
||||
CueXTestPackage: "cuex-test-package",
|
||||
Namespace: "default",
|
||||
CueXPath: "cuex/ext",
|
||||
ExternalFnName: "external",
|
||||
InputParamName: "input",
|
||||
OutputParamName: "output",
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
testEnv := &envtest.Environment{
|
||||
CRDDirectoryPaths: []string{
|
||||
filepath.Join("..", "..", "..", "charts", "vela-core", "crds"),
|
||||
},
|
||||
}
|
||||
|
||||
var err error
|
||||
cfg, err := testEnv.Start()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to start envtest: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if cfg == nil {
|
||||
fmt.Fprintf(os.Stderr, "envtest config is nil")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
testCtx.K8sClient, err = createK8sClient(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create k8s Client: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
mockServer := createMockServer()
|
||||
defer mockServer.Close()
|
||||
|
||||
singleton.KubeConfig.Set(cfg)
|
||||
|
||||
if err = createTestPackage(mockServer.URL); err != nil {
|
||||
_, _ = fmt.Fprintf(os.Stderr, "Setup failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer func() {
|
||||
if err = deleteTestPackage(); err != nil {
|
||||
_, _ = fmt.Fprintf(os.Stderr, "Teardown failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}()
|
||||
|
||||
code := m.Run()
|
||||
|
||||
singleton.KubeConfig.Reload()
|
||||
|
||||
if err := testEnv.Stop(); err != nil {
|
||||
_, _ = fmt.Fprintf(os.Stderr, "Failed to stop envtest: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestWorkloadCompiler(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
cuexEnabled bool
|
||||
workloadTemplate string
|
||||
params map[string]interface{}
|
||||
expectedObj runtime.Object
|
||||
expectedAdditionalObjs map[string]runtime.Object
|
||||
hasCompileErr bool
|
||||
errorString string
|
||||
}{
|
||||
"cuex disabled with no external packages": {
|
||||
cuexEnabled: false,
|
||||
workloadTemplate: getWorkloadTemplate(false),
|
||||
params: make(map[string]interface{}),
|
||||
expectedObj: getExpectedObj(false),
|
||||
expectedAdditionalObjs: make(map[string]runtime.Object),
|
||||
hasCompileErr: false,
|
||||
errorString: "",
|
||||
},
|
||||
"cuex enabled with no external packages": {
|
||||
cuexEnabled: true,
|
||||
workloadTemplate: getWorkloadTemplate(false),
|
||||
params: make(map[string]interface{}),
|
||||
expectedObj: getExpectedObj(false),
|
||||
expectedAdditionalObjs: make(map[string]runtime.Object),
|
||||
hasCompileErr: false,
|
||||
errorString: "",
|
||||
},
|
||||
"cuex disabled with external packages": {
|
||||
cuexEnabled: false,
|
||||
workloadTemplate: getWorkloadTemplate(true),
|
||||
params: make(map[string]interface{}),
|
||||
expectedObj: getExpectedObj(true),
|
||||
expectedAdditionalObjs: make(map[string]runtime.Object),
|
||||
hasCompileErr: true,
|
||||
errorString: "builtin package \"cuex/ext\" undefined",
|
||||
},
|
||||
"cuex enabled with external packages": {
|
||||
cuexEnabled: true,
|
||||
workloadTemplate: getWorkloadTemplate(true),
|
||||
params: make(map[string]interface{}),
|
||||
expectedObj: getExpectedObj(true),
|
||||
expectedAdditionalObjs: make(map[string]runtime.Object),
|
||||
hasCompileErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
cuex.EnableExternalPackageForDefaultCompiler = tc.cuexEnabled
|
||||
cuex.DefaultCompiler.Reload()
|
||||
|
||||
ctx := process.NewContext(process.ContextData{
|
||||
AppName: "test-app",
|
||||
CompName: "test-component",
|
||||
Namespace: testCtx.Namespace,
|
||||
AppRevisionName: "test-app-v1",
|
||||
ClusterVersion: types.ClusterVersion{Minor: "19+"},
|
||||
})
|
||||
|
||||
wt := definition.NewWorkloadAbstractEngine("test-workload")
|
||||
err := wt.Complete(ctx, tc.workloadTemplate, tc.params)
|
||||
assert.Equal(t, tc.hasCompileErr, err != nil)
|
||||
if tc.hasCompileErr {
|
||||
assert.NotNil(t, err)
|
||||
assert.Contains(t, err.Error(), tc.errorString)
|
||||
} else {
|
||||
output, _ := ctx.Output()
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, output)
|
||||
outputObj, _ := output.Unstructured()
|
||||
assert.Equal(t, tc.expectedObj, outputObj)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createMockServer() *httptest.Server {
|
||||
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/"+testCtx.ExternalFnName {
|
||||
http.Error(w, fmt.Sprintf("unexpected path: %s, expected: /%s", r.URL.Path, testCtx.ExternalFnName), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, err := w.Write([]byte(fmt.Sprintf("{\"%s\": \"%s\"}", testCtx.OutputParamName, testCtx.ReturnVal)))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}))
|
||||
return mockServer
|
||||
}
|
||||
|
||||
func createTestPackage(url string) error {
|
||||
ctx := context.Background()
|
||||
|
||||
packageObj := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"apiVersion": "cue.oam.dev/v1alpha1",
|
||||
"kind": "Package",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": testCtx.CueXTestPackage,
|
||||
"namespace": testCtx.Namespace,
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"path": testCtx.CueXPath,
|
||||
"provider": map[string]interface{}{
|
||||
"endpoint": url,
|
||||
"protocol": "http",
|
||||
},
|
||||
"templates": map[string]interface{}{
|
||||
"ext/cue": strings.TrimSpace(fmt.Sprintf(`
|
||||
package ext
|
||||
|
||||
#ExternalFunction: {
|
||||
#do: "%s",
|
||||
#provider: "%s",
|
||||
$params: {
|
||||
%s: string
|
||||
},
|
||||
$returns: {
|
||||
%s: string
|
||||
}
|
||||
}
|
||||
`, testCtx.ExternalFnName, testCtx.CueXTestPackage, testCtx.InputParamName, testCtx.OutputParamName)),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err := testCtx.K8sClient.Create(ctx, packageObj)
|
||||
|
||||
err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
|
||||
err = testCtx.K8sClient.Get(ctx, client.ObjectKey{
|
||||
Name: testCtx.CueXTestPackage,
|
||||
Namespace: testCtx.Namespace,
|
||||
}, packageObj)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create test package: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteTestPackage() error {
|
||||
ctx := context.Background()
|
||||
|
||||
testPkg := &unstructured.Unstructured{}
|
||||
testPkg.SetGroupVersionKind(schema.GroupVersionKind{
|
||||
Group: "cue.oam.dev",
|
||||
Version: "v1alpha1",
|
||||
Kind: "Package",
|
||||
})
|
||||
testPkg.SetName(testCtx.CueXTestPackage)
|
||||
testPkg.SetNamespace(testCtx.Namespace)
|
||||
|
||||
err := testCtx.K8sClient.Delete(ctx, testPkg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete test package: %w", err)
|
||||
}
|
||||
err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
|
||||
err := testCtx.K8sClient.Get(ctx, client.ObjectKey{
|
||||
Name: testCtx.CueXTestPackage,
|
||||
Namespace: testCtx.Namespace,
|
||||
}, testPkg)
|
||||
if err != nil {
|
||||
if k8serrors.IsNotFound(err) {
|
||||
return true, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete test package: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getWorkloadTemplate(includeExt bool) string {
|
||||
tmpl := ""
|
||||
name := "test-deployment"
|
||||
if includeExt {
|
||||
name = "test-deployment-\\(external.$returns.output)"
|
||||
tmpl = tmpl + strings.TrimSpace(fmt.Sprintf(`
|
||||
import (
|
||||
"%s"
|
||||
)
|
||||
|
||||
external: ext.#ExternalFunction & {
|
||||
$params: {
|
||||
%s: "external"
|
||||
}
|
||||
}
|
||||
`, testCtx.CueXPath, testCtx.InputParamName)) + "\n"
|
||||
|
||||
}
|
||||
|
||||
tmpl = tmpl + strings.TrimSpace(fmt.Sprintf(`
|
||||
output: {
|
||||
apiVersion: "apps/v1"
|
||||
kind: "Deployment"
|
||||
metadata: name: "%s"
|
||||
spec: replicas: 1
|
||||
}
|
||||
`, name))
|
||||
return tmpl
|
||||
}
|
||||
|
||||
func getExpectedObj(includeExt bool) *unstructured.Unstructured {
|
||||
name := "test-deployment"
|
||||
if includeExt {
|
||||
name = fmt.Sprintf("test-deployment-%s", testCtx.ReturnVal)
|
||||
}
|
||||
return &unstructured.Unstructured{Object: map[string]interface{}{
|
||||
"apiVersion": "apps/v1",
|
||||
"kind": "Deployment",
|
||||
"metadata": map[string]interface{}{"name": name},
|
||||
"spec": map[string]interface{}{"replicas": int64(1)},
|
||||
}}
|
||||
}
|
||||
|
||||
func createK8sClient(config *rest.Config) (client.Client, error) {
|
||||
scheme := runtime.NewScheme()
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
return nil, fmt.Errorf("failed to add corev1 to scheme: %w", err)
|
||||
}
|
||||
return client.New(config, client.Options{Scheme: scheme})
|
||||
}
|
||||
|
|
@ -22,6 +22,8 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/kubevela/pkg/cue/cuex"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
"github.com/kubevela/pkg/multicluster"
|
||||
|
|
@ -111,10 +113,14 @@ func (wd *workloadDef) Complete(ctx process.Context, abstractTemplate string, pa
|
|||
return err
|
||||
}
|
||||
|
||||
val := cuecontext.New().CompileString(strings.Join([]string{
|
||||
val, err := cuex.DefaultCompiler.Get().CompileString(ctx.GetCtx(), strings.Join([]string{
|
||||
renderTemplate(abstractTemplate), paramFile, c,
|
||||
}, "\n"))
|
||||
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed to compile workload %s after merge parameter and context", wd.name)
|
||||
}
|
||||
|
||||
if err := val.Validate(); err != nil {
|
||||
return errors.WithMessagef(err, "invalid cue template of workload %s after merge parameter and context", wd.name)
|
||||
}
|
||||
|
|
@ -318,10 +324,16 @@ func (td *traitDef) Complete(ctx process.Context, abstractTemplate string, param
|
|||
}
|
||||
buff += c
|
||||
|
||||
val := cuecontext.New().CompileString(buff)
|
||||
val, err := cuex.DefaultCompiler.Get().CompileString(ctx.GetCtx(), buff)
|
||||
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed to compile trait %s after merge parameter and context", td.name)
|
||||
}
|
||||
|
||||
if err := val.Validate(); err != nil {
|
||||
return errors.WithMessagef(err, "invalid template of trait %s after merge with parameter and context", td.name)
|
||||
}
|
||||
|
||||
processing := val.LookupPath(value.FieldPath("processing"))
|
||||
if processing.Exists() {
|
||||
if val, err = task.Process(val); err != nil {
|
||||
|
|
|
|||
|
|
@ -49,13 +49,6 @@ const (
|
|||
QLProviderName = "ql"
|
||||
)
|
||||
|
||||
var (
|
||||
// EnableExternalPackageForDefaultCompiler .
|
||||
EnableExternalPackageForDefaultCompiler = true
|
||||
// EnableExternalPackageWatchForDefaultCompiler .
|
||||
EnableExternalPackageWatchForDefaultCompiler = false
|
||||
)
|
||||
|
||||
// compiler is the workflow default compiler
|
||||
var compiler = singleton.NewSingletonE[*cuex.Compiler](func() (*cuex.Compiler, error) {
|
||||
return cuex.NewCompilerWithInternalPackages(
|
||||
|
|
@ -84,12 +77,12 @@ var compiler = singleton.NewSingletonE[*cuex.Compiler](func() (*cuex.Compiler, e
|
|||
// DefaultCompiler compiler for cuex to compile
|
||||
var DefaultCompiler = singleton.NewSingleton[*cuex.Compiler](func() *cuex.Compiler {
|
||||
c := compiler.Get()
|
||||
if EnableExternalPackageForDefaultCompiler {
|
||||
if cuex.EnableExternalPackageForDefaultCompiler {
|
||||
if err := c.LoadExternalPackages(context.Background()); err != nil {
|
||||
klog.Errorf("failed to load external packages for cuex default compiler: %v", err.Error())
|
||||
}
|
||||
}
|
||||
if EnableExternalPackageWatchForDefaultCompiler {
|
||||
if cuex.EnableExternalPackageWatchForDefaultCompiler {
|
||||
go c.ListenExternalPackages(nil)
|
||||
}
|
||||
return c
|
||||
|
|
|
|||
Loading…
Reference in New Issue