Feat: request token when vela auth gen-kubeconfig (#4687)

* Feat: request token when vela auth gen-kubeconfig

Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

* 1.24 test

Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

* fix test

Signed-off-by: qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

* fix test

Signed-off-by: qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

* format

Signed-off-by: qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

* more test on 1.24

Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

* rollback some logic and fix test

Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

* fix

Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
Signed-off-by: qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

.github/workflows/apiserver-test.yaml

@@ -19,8 +19,8 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.24\"]'
-  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.24\"]'
 
 jobs:
 
@@ -138,7 +138,7 @@ jobs:
       - name: Calculate K3d args
         run: |
           EGRESS_ARG=""
-          if [[ "${{ matrix.k8s-version }}" == v1.23 ]]; then
+          if [[ "${{ matrix.k8s-version }}" == v1.24 ]]; then
             EGRESS_ARG="--k3s-arg --egress-selector-mode=disabled@server:0"
           fi
           echo "EGRESS_ARG=${EGRESS_ARG}" >> $GITHUB_ENV 

.github/workflows/e2e-multicluster-test.yml

@@ -17,8 +17,8 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.24\"]'
-  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.24\"]'
 
 jobs:
 
@@ -84,7 +84,7 @@ jobs:
       - name: Calculate K3d args
         run: |
           EGRESS_ARG=""
-          if [[ "${{ matrix.k8s-version }}" == v1.23 ]]; then
+          if [[ "${{ matrix.k8s-version }}" == v1.24 ]]; then
             EGRESS_ARG="--k3s-arg --egress-selector-mode=disabled@server:0"
           fi
           echo "EGRESS_ARG=${EGRESS_ARG}" >> $GITHUB_ENV 

.github/workflows/e2e-rollout-test.yml

@@ -17,8 +17,8 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.24\"]'
-  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.24\"]'
 
 jobs:
 
@@ -83,7 +83,7 @@ jobs:
       - name: Calculate K3d args
         run: |
           EGRESS_ARG=""
-          if [[ "${{ matrix.k8s-version }}" == v1.23 ]]; then
+          if [[ "${{ matrix.k8s-version }}" == v1.24 ]]; then
             EGRESS_ARG="--k3s-arg --egress-selector-mode=disabled@server:0"
           fi
           echo "EGRESS_ARG=${EGRESS_ARG}" >> $GITHUB_ENV 

.github/workflows/e2e-test.yml

@@ -17,8 +17,8 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.24\"]'
-  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.24\"]'
 
 jobs:
 
@@ -83,7 +83,7 @@ jobs:
       - name: Calculate K3d args
         run: |
           EGRESS_ARG=""
-          if [[ "${{ matrix.k8s-version }}" == v1.23 ]]; then
+          if [[ "${{ matrix.k8s-version }}" == v1.24 ]]; then
             EGRESS_ARG="--k3s-arg --egress-selector-mode=disabled@server:0"
           fi
           echo "EGRESS_ARG=${EGRESS_ARG}" >> $GITHUB_ENV 

pkg/auth/kubeconfig.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	authenticationv1 "k8s.io/api/authentication/v1"
 	certificatesv1 "k8s.io/api/certificates/v1"
 	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
 	corev1 "k8s.io/api/core/v1"
@@ -46,6 +47,9 @@ import (
 	"github.com/oam-dev/kubevela/pkg/utils"
 )
 
+// DefaultExpireTime is default expire time for both X.509 and SA token apply
+const DefaultExpireTime = time.Hour * 24 * 365
+
 // KubeConfigGenerateOptions options for create KubeConfig
 type KubeConfigGenerateOptions struct {
 	X509           *KubeConfigGenerateX509Options
@@ -64,6 +68,7 @@ type KubeConfigGenerateX509Options struct {
 type KubeConfigGenerateServiceAccountOptions struct {
 	ServiceAccountName      string
 	ServiceAccountNamespace string
+	ExpireTime              time.Duration
 }
 
 // KubeConfigWithUserGenerateOption option for setting user in KubeConfig
@@ -96,6 +101,7 @@ func (opt KubeConfigWithServiceAccountGenerateOption) ApplyToOptions(options *Ku
 	options.ServiceAccount = &KubeConfigGenerateServiceAccountOptions{
 		ServiceAccountName:      opt.Name,
 		ServiceAccountNamespace: opt.Namespace,
+		ExpireTime:              DefaultExpireTime,
 	}
 }
 
@@ -128,7 +134,7 @@ func newKubeConfigGenerateOptions(options ...KubeConfigGenerateOption) *KubeConf
 		X509: &KubeConfigGenerateX509Options{
 			User:           user.Anonymous,
 			Groups:         []string{KubeVelaClientGroup},
-			ExpireTime:     time.Hour * 24 * 365,
+			ExpireTime:     DefaultExpireTime,
 			PrivateKeyBits: 2048,
 		},
 		ServiceAccount: nil,
@@ -329,30 +335,53 @@ func generateX509KubeConfigV1Beta(ctx context.Context, cli kubernetes.Interface,
 }
 
 func generateServiceAccountKubeConfig(ctx context.Context, cli kubernetes.Interface, cfg *clientcmdapi.Config, writer io.Writer, opts *KubeConfigGenerateServiceAccountOptions) (*clientcmdapi.Config, error) {
+	var (
+		token string
+		CA    []byte
+	)
 	sa, err := cli.CoreV1().ServiceAccounts(opts.ServiceAccountNamespace).Get(ctx, opts.ServiceAccountName, metav1.GetOptions{})
 	if err != nil {
 		return nil, err
 	}
 	_, _ = fmt.Fprintf(writer, "ServiceAccount %s/%s found.\n", opts.ServiceAccountNamespace, opts.ServiceAccountName)
 	if len(sa.Secrets) == 0 {
-		return nil, errors.Errorf("no secret found in serviceaccount %s/%s", opts.ServiceAccountNamespace, opts.ServiceAccountName)
+		_, _ = fmt.Fprintf(writer, "ServiceAccount %s/%s has no secret. Requesting token", opts.ServiceAccountNamespace, opts.ServiceAccountName)
+		request := authenticationv1.TokenRequest{
+			Spec: authenticationv1.TokenRequestSpec{
+				Audiences:         []string{},
+				ExpirationSeconds: pointer.Int64(int64(opts.ExpireTime.Seconds())),
+			},
+		}
+		tokenRequest, err := cli.CoreV1().ServiceAccounts(opts.ServiceAccountNamespace).CreateToken(ctx, opts.ServiceAccountName, &request, metav1.CreateOptions{})
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to request token")
+		}
+		token = tokenRequest.Status.Token
+		CAConfigMap, err := cli.CoreV1().ConfigMaps(sa.Namespace).Get(ctx, "kube-root-ca.crt", metav1.GetOptions{})
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get root CA secret")
+		}
+		CA = []byte(CAConfigMap.Data["ca.crt"])
+	} else {
+		secretKey := sa.Secrets[0]
+		if secretKey.Namespace == "" {
+			secretKey.Namespace = sa.Namespace
+		}
+		secret, err := cli.CoreV1().Secrets(secretKey.Namespace).Get(ctx, secretKey.Name, metav1.GetOptions{})
+		if err != nil {
+			return nil, err
+		}
+		_, _ = fmt.Fprintf(writer, "ServiceAccount secret %s/%s found.\n", secretKey.Namespace, secret.Name)
+		if len(secret.Data["token"]) == 0 {
+			return nil, errors.Errorf("no token found in secret %s/%s", secret.Namespace, secret.Name)
+		}
+		_, _ = fmt.Fprintf(writer, "ServiceAccount token found.\n")
+		token = string(secret.Data["token"])
+		CA = secret.Data["ca.crt"]
 	}
-	secretKey := sa.Secrets[0]
-	if secretKey.Namespace == "" {
-		secretKey.Namespace = sa.Namespace
-	}
-	secret, err := cli.CoreV1().Secrets(secretKey.Namespace).Get(ctx, secretKey.Name, metav1.GetOptions{})
-	if err != nil {
-		return nil, err
-	}
-	_, _ = fmt.Fprintf(writer, "ServiceAccount secret %s/%s found.\n", secretKey.Namespace, secret.Name)
-	if len(secret.Data["token"]) == 0 {
-		return nil, errors.Errorf("no token found in secret %s/%s", secret.Namespace, secret.Name)
-	}
-	_, _ = fmt.Fprintf(writer, "ServiceAccount token found.\n")
 	return genKubeConfig(cfg, &clientcmdapi.AuthInfo{
-		Token: string(secret.Data["token"]),
+		Token: token,
-	}, secret.Data["ca.crt"])
+	}, CA)
 }
 
 // ReadIdentityFromKubeConfig extract identity from kubeconfig

test/e2e-multicluster-test/multicluster_test.go

@@ -25,27 +25,26 @@ import (
 	"strings"
 	"time"
 
-	"k8s.io/apimachinery/pkg/runtime"
-
-	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
-	"github.com/oam-dev/kubevela/pkg/oam"
-	"github.com/oam-dev/kubevela/pkg/utils"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/yaml"
 
+	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha1"
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	"github.com/oam-dev/kubevela/pkg/multicluster"
+	"github.com/oam-dev/kubevela/pkg/oam"
+	"github.com/oam-dev/kubevela/pkg/utils"
 )
 
 func initializeContext() (hubCtx context.Context, workerCtx context.Context) {
@@ -151,11 +150,12 @@ var _ = Describe("Test multicluster scenario", func() {
 
 		It("Test generate service account kubeconfig", func() {
 			_, workerCtx := initializeContext()
-			// create service account kubeconfig in worker cluster
+			By("create service account kubeconfig in worker cluster")
 			key := time.Now().UnixNano()
 			serviceAccountName := fmt.Sprintf("test-service-account-%d", key)
+			serviceAccountNamespace := "kube-system"
 			serviceAccount := &corev1.ServiceAccount{
-				ObjectMeta: metav1.ObjectMeta{Namespace: "kube-system", Name: serviceAccountName},
+				ObjectMeta: metav1.ObjectMeta{Namespace: serviceAccountNamespace, Name: serviceAccountName},
 			}
 			Expect(k8sClient.Create(workerCtx, serviceAccount)).Should(Succeed())
 			defer func() {
@@ -165,30 +165,26 @@ var _ = Describe("Test multicluster scenario", func() {
 			clusterRoleBindingName := fmt.Sprintf("test-cluster-role-binding-%d", key)
 			clusterRoleBinding := &rbacv1.ClusterRoleBinding{
 				ObjectMeta: metav1.ObjectMeta{Name: clusterRoleBindingName},
-				Subjects:   []rbacv1.Subject{{Kind: "ServiceAccount", Name: serviceAccountName, Namespace: "kube-system"}},
+				Subjects:   []rbacv1.Subject{{Kind: "ServiceAccount", Name: serviceAccountName, Namespace: serviceAccountNamespace}},
 				RoleRef:    rbacv1.RoleRef{Name: "cluster-admin", APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole"},
 			}
 			Expect(k8sClient.Create(workerCtx, clusterRoleBinding)).Should(Succeed())
 			defer func() {
-				Expect(k8sClient.Get(workerCtx, types.NamespacedName{Namespace: "kube-system", Name: clusterRoleBindingName}, clusterRoleBinding)).Should(Succeed())
+				Expect(k8sClient.Get(workerCtx, types.NamespacedName{Namespace: serviceAccountNamespace, Name: clusterRoleBindingName}, clusterRoleBinding)).Should(Succeed())
 				Expect(k8sClient.Delete(workerCtx, clusterRoleBinding)).Should(Succeed())
 			}()
 			serviceAccount = &corev1.ServiceAccount{}
-			Eventually(func(g Gomega) {
+			By("Generating a token for SA")
-				g.Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccountName, Namespace: "kube-system"}, serviceAccount)).Should(Succeed())
+			tr := &v1.TokenRequest{}
-				g.Expect(len(serviceAccount.Secrets)).Should(Equal(1))
+			token, err := k8sCli.CoreV1().ServiceAccounts(serviceAccountNamespace).CreateToken(workerCtx, serviceAccountName, tr, metav1.CreateOptions{})
-			}, time.Second*30).Should(Succeed())
+			Expect(err).Should(BeNil())
-			secret := &corev1.Secret{}
-			Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccount.Secrets[0].Name, Namespace: "kube-system"}, secret)).Should(Succeed())
-			token, ok := secret.Data["token"]
-			Expect(ok).Should(BeTrue())
 			config, err := clientcmd.LoadFromFile(WorkerClusterKubeConfigPath)
 			Expect(err).Should(Succeed())
 			currentContext, ok := config.Contexts[config.CurrentContext]
 			Expect(ok).Should(BeTrue())
 			authInfo, ok := config.AuthInfos[currentContext.AuthInfo]
 			Expect(ok).Should(BeTrue())
-			authInfo.Token = string(token)
+			authInfo.Token = token.Status.Token
 			authInfo.ClientKeyData = nil
 			authInfo.ClientCertificateData = nil
 			kubeconfigFilePath := fmt.Sprintf("/tmp/worker.sa-%d.kubeconfig", key)

test/e2e-multicluster-test/suite_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/strings/slices"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -44,6 +45,7 @@ const (
 
 var (
 	k8sClient client.Client
+	k8sCli    kubernetes.Interface
 )
 
 func execCommand(args ...string) (string, error) {
@@ -70,6 +72,8 @@ var _ = BeforeSuite(func() {
 	config.Wrap(multicluster.NewSecretModeMultiClusterRoundTripper)
 	k8sClient, err = client.New(config, options)
 	Expect(err).Should(Succeed())
+	k8sCli, err = kubernetes.NewForConfig(config)
+	Expect(err).Should(Succeed())
 
 	// join worker cluster
 	_, err = execCommand("cluster", "join", WorkerClusterKubeConfigPath, "--name", WorkerClusterName)