[cherry-pick] udpate storage to s3 (#21999) (#22002)

udpate storage to s3 (#21999) move the build storage from google storage to the CNCF S3 storage Currently, we use the internal GCR to store all dev builds for nightly testing, development, and as candidates for RC and GA releases. However, this internal Google storage will no longer be available, this pull request it to move to the CNCF-hosted S3 storage. Signed-off-by: wang yan <wangyan@vmware.com>
2025-05-19 10:56:10 +08:00 · 2025-05-19 10:56:10 +08:00 · 647842f419
parent b8ee76373f
commit 647842f419
7 changed files with 39 additions and 58 deletions
--- a/.github/workflows/build-package.yml
+++ b/.github/workflows/build-package.yml
@ -15,14 +15,12 @@ jobs:
    runs-on:
      - ubuntu-22.04
    steps:
-      - uses: actions/checkout@v3
-      - uses: 'google-github-actions/auth@v2'
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4.1.0
        with:
-          credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
-      - uses: google-github-actions/setup-gcloud@v2
-        with:
-          version: '430.0.0'
-      - run: gcloud info
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
      - name: Set up Go 1.22
        uses: actions/setup-go@v5
        with:
--- a/.github/workflows/conformance_test.yml
+++ b/.github/workflows/conformance_test.yml
@ -17,14 +17,12 @@ jobs:
      #- self-hosted
      - ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
-      - id: 'auth'
-        name: 'Authenticate to Google Cloud'
-        uses: google-github-actions/auth@v2
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4.1.0
        with:
-          credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
-      - uses: google-github-actions/setup-gcloud@v2
-      - run: gcloud info
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
      - name: Set up Go 1.21
        uses: actions/setup-go@v5
        with:
@ -65,6 +63,5 @@ jobs:
      - name: upload test result to gs
        run: |
          cd src/github.com/goharbor/harbor
-          gsutil cp ./distribution-spec/conformance/report.html gs://harbor-conformance-test/report.html
-          gsutil acl ch -u AllUsers:R gs://harbor-conformance-test/report.html
+          aws s3 cp ./distribution-spec/conformance/report.html s3://harbor-conformance-test/report.html
        if: always()
--- a/.github/workflows/publish_release.yml
+++ b/.github/workflows/publish_release.yml
@ -19,12 +19,12 @@ jobs:
          echo "PRE_TAG=$(echo $release | jq -r '.body' | jq -r '.preTag')" >> $GITHUB_ENV
          echo "BRANCH=$(echo $release | jq -r '.target_commitish')" >> $GITHUB_ENV
          echo "PRERELEASE=$(echo $release | jq -r '.prerelease')" >> $GITHUB_ENV
-      - uses: 'google-github-actions/auth@v2'
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4.1.0
        with:
-          credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
-      - uses: google-github-actions/setup-gcloud@v2
-        with:
-          version: '430.0.0'
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
      - name: Prepare Assets
        run: |
          if [ ! ${{ env.BUILD_NO }} -o ${{ env.BUILD_NO }} = "null" ]
@ -39,8 +39,8 @@ jobs:
          src_online_package=harbor-online-installer-${{ env.BASE_TAG }}-${{ env.BUILD_NO }}.tgz
          dst_offline_package=harbor-offline-installer-${{ env.CUR_TAG }}.tgz
          dst_online_package=harbor-online-installer-${{ env.CUR_TAG }}.tgz
-          gsutil cp gs://${{ secrets.HARBOR_RELEASE_BUILD }}/${{ env.BRANCH }}/${src_offline_package} gs://${{ secrets.HARBOR_RELEASE_BUILD }}/${{ env.BRANCH }}/${dst_offline_package}
-          gsutil cp gs://${{ secrets.HARBOR_RELEASE_BUILD }}/${{ env.BRANCH }}/${src_online_package} gs://${{ secrets.HARBOR_RELEASE_BUILD }}/${{ env.BRANCH }}/${dst_online_package}
+          aws s3 cp s3://${{ secrets.HARBOR_RELEASE_BUILD }}/${{ env.BRANCH }}/${src_offline_package} s3://${{ secrets.HARBOR_RELEASE_BUILD }}/${{ env.BRANCH }}/${dst_offline_package}
+          aws s3 cp s3://${{ secrets.HARBOR_RELEASE_BUILD }}/${{ env.BRANCH }}/${src_online_package} s3://${{ secrets.HARBOR_RELEASE_BUILD }}/${{ env.BRANCH }}/${dst_online_package}

          assets_path=$(pwd)/assets
          source tools/release/release_utils.sh && getAssets ${{ secrets.HARBOR_RELEASE_BUILD }} ${{ env.BRANCH }} $dst_offline_package $dst_online_package ${{ env.PRERELEASE }} $assets_path
--- a/tests/ci/api_run.sh
+++ b/tests/ci/api_run.sh
@ -3,7 +3,7 @@ set -x

 #source gskey.sh

-sudo gsutil version -l
+sudo aws --version

 harbor_logs_bucket="harbor-ci-logs"

@ -12,8 +12,7 @@ E2E_IMAGE="goharbor/harbor-e2e-engine:latest-api"

 # GS util
 function uploader {
-   sudo gsutil cp $1 gs://$2/$1
-   sudo gsutil acl ch -u AllUsers:R gs://$2/$1
+   sudo aws s3 cp $1 s3://$2/$1
 }

 set +e
@ -38,18 +37,18 @@ else
 fi
 rc=$?
 ## --------------------------------------------- Upload Harbor CI Logs -------------------------------------------
-timestamp=$(date +%s)
-GIT_COMMIT=$(git rev-parse --short "$GITHUB_SHA")
-outfile="integration_logs_$timestamp$GIT_COMMIT.tar.gz"
-sudo tar -zcvf $outfile output.xml log.html /var/log/harbor/*
-if [ -f "$outfile" ]; then
-   uploader $outfile $harbor_logs_bucket
-   echo "----------------------------------------------"
-   echo "Download test logs:"
-   echo "https://storage.googleapis.com/harbor-ci-logs/$outfile"
-   echo "----------------------------------------------"
-else
-   echo "No log output file to upload"
-fi
+#timestamp=$(date +%s)
+#GIT_COMMIT=$(git rev-parse --short "$GITHUB_SHA")
+#outfile="integration_logs_$timestamp$GIT_COMMIT.tar.gz"
+#sudo tar -zcvf $outfile output.xml log.html /var/log/harbor/*
+#if [ -f "$outfile" ]; then
+#   uploader $outfile $harbor_logs_bucket
+#   echo "----------------------------------------------"
+#   echo "Download test logs:"
+#   echo "https://storage.googleapis.com/harbor-ci-logs/$outfile"
+#   echo "----------------------------------------------"
+#else
+#   echo "No log output file to upload"
+#fi

 exit $rc
--- a/tests/ci/build_util.sh
+++ b/tests/ci/build_util.sh
@ -4,8 +4,7 @@ set -x
 set -e

 function uploader {
-    gsutil cp $1 gs://$2/$1
-    gsutil -D setacl public-read gs://$2/$1 &> /dev/null
+    aws s3 cp $1 s3://$2/$1
 }

 function publishImage {
--- a/tests/integration.sh
+++ b/tests/integration.sh
@ -14,7 +14,7 @@
 # limitations under the License.
 #
 set -x
-gsutil version -l
+aws --version
 set +x

 docker-compose version
@ -58,17 +58,6 @@ else
  harbor_target_bucket=$harbor_releases_bucket/$DRONE_BRANCH
 fi

-# GC credentials
-keyfile="/root/harbor-ci-logs.key"
-botofile="/root/.boto"
-echo -n $GS_PRIVATE_KEY > $keyfile
-chmod 400 $keyfile
-echo "[Credentials]" >> $botofile
-echo "gs_service_key_file = $keyfile" >> $botofile
-echo "gs_service_client_id = $GS_CLIENT_EMAIL" >> $botofile
-echo "[GSUtil]" >> $botofile
-echo "content_language = en" >> $botofile
-echo "default_project_id = $GS_PROJECT_ID" >> $botofile
 container_ip=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
 echo $container_ip

@ -106,10 +95,9 @@ echo "Harbor Assets version: $Harbor_Assets_Version"
 echo "Harbor Build Base tag: $Harbor_Build_Base_Tag"
 echo "--------------------------------------------------"

-# GS util
+# AWS S3 util
 function uploader {
-    gsutil cp $1 gs://$2/$1
-    gsutil -D setacl public-read gs://$2/$1 &> /dev/null
+    aws s3 cp $1 s3://$2/$1
 }

 function package_installer {
--- a/tools/release/release_utils.sh
+++ b/tools/release/release_utils.sh
@ -9,12 +9,12 @@ function getAssets {
    local prerelease=$5
    local assetsPath=$6
    mkdir $assetsPath && pushd $assetsPath
-    gsutil cp gs://$bucket/$branch/$offlinePackage .
+    aws s3 cp s3://$bucket/$branch/$offlinePackage .
    md5sum $offlinePackage > md5sum
    # Pre-release does not handle online installer packages
    if [ $prerelease = "false" ]
    then
-        gsutil cp gs://$bucket/$branch/$onlinePackage .
+        aws s3 cp s3://$bucket/$branch/$onlinePackage .
        md5sum $onlinePackage >> md5sum
    fi
    popd