Fix: webhook TLS caBundle breakage during failed Helm upgrades (#6919)

* Fix: Add post-rollback hook to admission-related resources for better upgrade handling

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Add webhook upgrade validation workflow for handling failed Helm upgrades

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Update webhook upgrade validation template for improved resource handling

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Refactor webhook upgrade validation script for improved readability and resource handling

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* webhook-upgrade-validation.yml

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Update KubeVela CLI installation method in webhook upgrade validation workflow

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Update KubeVela CLI installation method in webhook upgrade validation workflow

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Add verification step for active webhook validation in upgrade workflow

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Update test command in webhook upgrade validation workflow to use Ginkgo

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Update cluster deletion command in webhook upgrade validation workflow

Signed-off-by: vishal210893 <vishal210893@gmail.com>

* Fix: Refactor environment setup in webhook upgrade validation workflow

Signed-off-by: Vaibhav Agrawal <vaibhav.agrawal0096@gmail.com>

* Fix: Update KinD setup in webhook upgrade validation workflow

Signed-off-by: Vaibhav Agrawal <vaibhav.agrawal0096@gmail.com>

---------

Signed-off-by: vishal210893 <vishal210893@gmail.com>
Signed-off-by: Vaibhav Agrawal <vaibhav.agrawal0096@gmail.com>

.github/workflows/webhook-upgrade-validation.yml

@@ -0,0 +1,165 @@
+name: Webhook Upgrade Validation
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+    tags:
+      - v*
+  workflow_dispatch: {}
+  pull_request:
+    branches:
+      - master
+      - release-*
+
+permissions:
+  contents: read
+
+env:
+  GO_VERSION: '1.23.8'
+
+jobs:
+  webhook-upgrade-check:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+
+      - name: Setup Env
+        uses: ./.github/actions/env-setup
+
+      - name: Setup KinD
+        run: |
+          go install sigs.k8s.io/kind@v0.29.0
+          kind delete cluster || true
+          kind create cluster --image=kindest/node:v1.31.9
+
+      - name: Install KubeVela CLI
+        run: curl -fsSL https://kubevela.io/script/install.sh | bash
+
+      - name: Install KubeVela baseline
+        run: |
+          vela install --set featureGates.enableCueValidation=true
+          kubectl wait --namespace vela-system --for=condition=Available deployment/kubevela-vela-core --timeout=300s
+
+      - name: Prepare failing chart changes
+        run: |
+          cat <<'CHART' > charts/vela-core/templates/defwithtemplate/resource.yaml
+          # Code generated by KubeVela templates. DO NOT EDIT. Please edit the original cue file.
+          # Definition source cue file: vela-templates/definitions/internal/resource.cue
+          apiVersion: core.oam.dev/v1beta1
+          kind: TraitDefinition
+          metadata:
+            annotations:
+              definition.oam.dev/description: Add resource requests and limits on K8s pod for your workload which follows the pod spec in path 'spec.template.'
+            name: resource
+            namespace: {{ include "systemDefinitionNamespace" . }}
+          spec:
+            appliesToWorkloads:
+              - deployments.apps
+              - statefulsets.apps
+              - daemonsets.apps
+              - jobs.batch
+              - cronjobs.batch
+            podDisruptive: true
+            schematic:
+              cue:
+                template: |2
+                  let resourceContent = {
+                    resources: {
+                      if parameter.cpu != _|_ if parameter.memory != _|_ if parameter.requests == _|_ if parameter.limits == _|_ {
+                        // +patchStrategy=retainKeys
+                        requests: {
+                          cpu:    parameter.cpu
+                          memory: parameter.memory
+                        }
+                        // +patchStrategy=retainKeys
+                        limits: {
+                          cpu:    parameter.cpu
+                          memory: parameter.memory
+                        }
+                      }
+                      if parameter.requests != _|_ {
+                        // +patchStrategy=retainKeys
+                        requests: {
+                          cpu:    parameter.requests.cpu
+                          memory: parameter.requests.memory
+                        }
+                      }
+                      if parameter.limits != _|_ {
+                        // +patchStrategy=retainKeys
+                        limits: {
+                          cpu:    parameter.limits.cpu
+                          memory: parameter.limits.memory
+                        }
+                      }
+                    }
+                  }
+                  if context.output.spec != _|_ if context.output.spec.template != _|_ {
+                    patch: spec: template: spec: {
+                      // +patchKey=name
+                      containers: [resourceContent]
+                    }
+                  }
+                  if context.output.spec != _|_ if context.output.spec.jobTemplate != _|_ {
+                    patch: spec: jobTemplate: spec: template: spec: {
+                      // +patchKey=name
+                      containers: [resourceContent]
+                    }
+                  }
+                  parameter: {
+                    // +usage=Specify the amount of cpu for requests and limits
+                    cpu?: *1 | number | string
+                    // +usage=Specify the amount of memory for requests and limits
+                    memory?: *"2048Mi" | =~"^([1-9][0-9]{0,63})(E|P|T|G|M|K|Ei|Pi|Ti|Gi|Mi|Ki)$"
+                    // +usage=Specify the resources in requests
+                    requests?: {
+                      // +usage=Specify the amount of cpu for requests
+                      cpu: *1 | number | string
+                      // +usage=Specify the amount of memory for requests
+                      memory: *"2048Mi" | =~"^([1-9][0-9]{0,63})(E|P|T|G|M|K|Ei|Pi|Ti|Gi|Mi|Ki)$"
+                    }
+                    // +usage=Specify the resources in limits
+                    limits?: {
+                      // +usage=Specify the amount of cpu for limits
+                      cpu: *1 | number | string
+                      // +usage=Specify the amount of memory for limits
+                      memory: *"2048Mi" | =~"^([1-9][0-9]{0,63})(E|P|T|G|M|K|Ei|Pi|Ti|Gi|Mi|Ki)$"
+                    }
+                  }
+
+      - name: Load image
+        run: |
+          mkdir -p $HOME/tmp/
+          TMPDIR=$HOME/tmp/ make image-load
+
+      - name: Run Helm upgrade (expected to fail)
+        run: |
+          set +e
+          helm upgrade \
+            --set image.repository=vela-core-test \
+            --set image.tag=$(git rev-parse --short HEAD) \
+            --set featureGates.enableCueValidation=true \
+            --wait kubevela ./charts/vela-core --debug -n vela-system
+          status=$?
+          echo "Helm upgrade exit code: ${status}"
+          if [ $status -eq 0 ]; then
+            echo "Expected helm upgrade to fail" >&2
+            exit 1
+          fi
+          echo "Helm upgrade failed as expected"
+
+      - name: Dump webhook configurations
+        if: ${{ always() }}
+        run: |
+          kubectl get mutatingwebhookconfiguration kubevela-vela-core-admission -o yaml
+          kubectl get validatingwebhookconfiguration kubevela-vela-core-admission -o yaml
+
+      - name: Verify webhook validation remains active
+        run: ginkgo -v --focus-file requiredparam_validation_test.go ./test/e2e-test
+
+      - name: Cleanup kind cluster
+        if: ${{ always() }}
+        run: kind delete cluster --name kind

charts/vela-core/templates/admission-webhooks/job-patch/clusterrole.yaml

@@ -4,7 +4,7 @@ kind: ClusterRole
 metadata:
   name:  {{ template "kubevela.fullname" . }}-admission
   annotations:
-    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade,post-rollback
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
   labels:
     app: {{ template "kubevela.name" . }}-admission

charts/vela-core/templates/admission-webhooks/job-patch/clusterrolebinding.yaml

@@ -4,7 +4,7 @@ kind: ClusterRoleBinding
 metadata:
   name:  {{ template "kubevela.fullname" . }}-admission
   annotations:
-    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade,post-rollback
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
   labels:
     app: {{ template "kubevela.name" . }}-admission

charts/vela-core/templates/admission-webhooks/job-patch/job-patchWebhook.yaml

@@ -5,7 +5,7 @@ metadata:
   name: {{ template "kubevela.fullname" . }}-admission-patch
   namespace: {{ .Release.Namespace }}
   annotations:
-    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook": post-install,post-upgrade,post-rollback
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
   labels:
     app: {{ template "kubevela.name" . }}-admission-patch

charts/vela-core/templates/admission-webhooks/job-patch/role.yaml

@@ -5,7 +5,7 @@ metadata:
   name:  {{ template "kubevela.fullname" . }}-admission
   namespace: {{ .Release.Namespace }}
   annotations:
-    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade,post-rollback
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
   labels:
     app: {{ template "kubevela.name" . }}-admission

charts/vela-core/templates/admission-webhooks/job-patch/rolebinding.yaml

@@ -5,7 +5,7 @@ metadata:
   name:  {{ template "kubevela.fullname" . }}-admission
   namespace: {{ .Release.Namespace }}
   annotations:
-    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade,post-rollback
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
   labels:
     app: {{ template "kubevela.name" . }}-admission

charts/vela-core/templates/admission-webhooks/job-patch/serviceaccount.yaml

@@ -5,7 +5,7 @@ metadata:
   name:  {{ template "kubevela.fullname" . }}-admission
   namespace: {{ .Release.Namespace }}
   annotations:
-    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade,post-rollback
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
   labels:
     app: {{ template "kubevela.name" . }}-admission

charts/vela-core/templates/admission-webhooks/mutatingWebhookConfiguration.yaml

@@ -1,4 +1,14 @@
 {{- if .Values.admissionWebhooks.enabled -}}
+{{- /* Preserve existing caBundle on upgrade to avoid breaking admission if hooks fail. */}}
+{{- $mName := printf "%s-admission" (include "kubevela.fullname" .) -}}
+{{- $existing := (lookup "admissionregistration.k8s.io/v1" "MutatingWebhookConfiguration" "" $mName) -}}
+{{- $vals := dict "apps" "" "comps" "" -}}
+{{- if $existing -}}
+{{- range $existing.webhooks -}}
+{{- if eq .name "mutating.core.oam.dev.v1beta1.applications" -}}{{- $_ := set $vals "apps" .clientConfig.caBundle -}}{{- end -}}
+{{- if eq .name "mutating.core.oam-dev.v1beta1.componentdefinitions" -}}{{- $_ := set $vals "comps" .clientConfig.caBundle -}}{{- end -}}
+{{- end -}}
+{{- end -}}
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
@@ -10,7 +20,7 @@ metadata:
   {{- end }}
 webhooks:
   - clientConfig:
-      caBundle: Cg==
+      caBundle: {{ default "Cg==" (get $vals "apps") }}
       service:
         name: {{ template "kubevela.name" . }}-webhook
         namespace: {{ .Release.Namespace }}
@@ -36,7 +46,7 @@ webhooks:
         resources:
           - applications
   - clientConfig:
-      caBundle: Cg==
+      caBundle: {{ default "Cg==" (get $vals "comps") }}
       service:
         name: {{ template "kubevela.name" . }}-webhook
         namespace: {{ .Release.Namespace }}

charts/vela-core/templates/admission-webhooks/validatingWebhookConfiguration.yaml

@@ -1,4 +1,16 @@
 {{- if .Values.admissionWebhooks.enabled -}}
+{{- /* Preserve existing caBundle on upgrade to avoid breaking admission if hooks fail. */}}
+{{- $vName := printf "%s-admission" (include "kubevela.fullname" .) -}}
+{{- $existing := (lookup "admissionregistration.k8s.io/v1" "ValidatingWebhookConfiguration" "" $vName) -}}
+{{- $vals := dict "traits" "" "apps" "" "comps" "" "policies" "" -}}
+{{- if $existing -}}
+{{- range $existing.webhooks -}}
+{{- if eq .name "validating.core.oam.dev.v1beta1.traitdefinitions" -}}{{- $_ := set $vals "traits" .clientConfig.caBundle -}}{{- end -}}
+{{- if eq .name "validating.core.oam.dev.v1beta1.applications" -}}{{- $_ := set $vals "apps" .clientConfig.caBundle -}}{{- end -}}
+{{- if eq .name "validating.core.oam-dev.v1beta1.componentdefinitions" -}}{{- $_ := set $vals "comps" .clientConfig.caBundle -}}{{- end -}}
+{{- if eq .name "validating.core.oam-dev.v1beta1.policydefinitions" -}}{{- $_ := set $vals "policies" .clientConfig.caBundle -}}{{- end -}}
+{{- end -}}
+{{- end -}}
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
@@ -10,7 +22,7 @@ metadata:
   {{- end }}
 webhooks:
   - clientConfig:
-      caBundle: Cg==
+      caBundle: {{ default "Cg==" (get $vals "traits") }}
       service:
         name: {{ template "kubevela.name" . }}-webhook
         namespace: {{ .Release.Namespace }}
@@ -37,7 +49,7 @@ webhooks:
           - traitdefinitions
     timeoutSeconds: 5
   - clientConfig:
-      caBundle: Cg==
+      caBundle: {{ default "Cg==" (get $vals "apps") }}
       service:
         name: {{ template "kubevela.name" . }}-webhook
         namespace: {{ .Release.Namespace }}
@@ -63,7 +75,7 @@ webhooks:
         resources:
           - applications
   - clientConfig:
-      caBundle: Cg==
+      caBundle: {{ default "Cg==" (get $vals "comps") }}
       service:
         name: {{ template "kubevela.name" . }}-webhook
         namespace: {{ .Release.Namespace }}
@@ -89,7 +101,7 @@ webhooks:
         resources:
           - componentdefinitions
   - clientConfig:
-      caBundle: Cg==
+      caBundle: {{ default "Cg==" (get $vals "policies") }}
       service:
         name: {{ template "kubevela.name" . }}-webhook
         namespace: {{ .Release.Namespace }}