From fa936e7e088061c206420d2808f193a46845d823 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Tue, 8 Oct 2024 15:40:27 -0700 Subject: [PATCH] KAFKA-17735: release.py must not use home.apache.org (#17421) Previously, Apache Kafka was uploading release candidate (RC) artifacts to users' home directories on home.apache.org. However, since this resource has been decommissioned, we need to follow the standard approach of putting release candidate artifacts into the appropriate subversion directory, at https://dist.apache.org/repos/dist/dev/kafka/. Reviewers: Justine Olshan --- release/README.md | 1 - release/release.py | 12 ++++++--- release/runtime.py | 4 +-- release/sftp.py | 59 ------------------------------------------- release/svn.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ release/templates.py | 24 +++++++++--------- 6 files changed, 82 insertions(+), 78 deletions(-) delete mode 100644 release/sftp.py create mode 100644 release/svn.py diff --git a/release/README.md b/release/README.md index 0222164cd3d..1f56f679011 100644 --- a/release/README.md +++ b/release/README.md @@ -8,7 +8,6 @@ This directory contains the tools used to publish a release. * python 3.12 * git * gpg 2.4 -* sftp The full instructions for producing a release are available in https://cwiki.apache.org/confluence/display/KAFKA/Release+Process. diff --git a/release/release.py b/release/release.py index 67fc46c831d..ac324726277 100644 --- a/release/release.py +++ b/release/release.py @@ -70,10 +70,12 @@ import git import gpg import notes import preferences -import sftp +import svn import templates import textfiles +from svn import SVN_DEV_URL + def get_jdk(version): """ @@ -243,7 +245,6 @@ def verify_prerequeisites(): except Exception as e: fail(f"Pre-requisite not met: {name}. Error: {e}") prereq('Apache Maven CLI (mvn) in PATH', lambda: "Apache Maven" in execute("mvn -v")) - prereq('Apache sftp connection', lambda: sftp.test(apache_id)) prereq("svn CLI in PATH", lambda: "svn" in execute("svn --version")) prereq("Verifying that you have no unstaged git changes", lambda: git.has_unstaged_changes()) prereq("Verifying that you have no staged git changes", lambda: git.has_staged_changes()) @@ -298,7 +299,7 @@ git.commit(f"Bump version to {release_version}") git.create_tag(rc_tag) git.switch_branch(starting_branch) -# Note that we don't use tempfile here because mkdtemp causes problems with sftp and being able to determine the absolute path to a file. +# Note that we don't use tempfile here because mkdtemp causes problems with being able to determine the absolute path to a file. # Instead we rely on a fixed path work_dir = os.path.join(repo_dir, ".release_work_dir") clean_up_work_dir = lambda: cmd("Cleaning up work directory", f"rm -rf {work_dir}") @@ -315,6 +316,7 @@ git.clone(repo_dir, 'kafka', cwd=work_dir) git.create_branch(release_version, rc_tag, cwd=kafka_dir) current_year = datetime.datetime.now().year cmd("Verifying the correct year in NOTICE", f"grep {current_year} NOTICE", cwd=kafka_dir) +svn.checkout_svn_dev(work_dir) print("Generating release notes") try: @@ -348,7 +350,9 @@ for filename in os.listdir(artifacts_dir): cmd("Listing artifacts to be uploaded:", f"ls -R {artifacts_dir}") cmd("Zipping artifacts", f"tar -czf {artifact_name}.tar.gz {artifact_name}", cwd=work_dir) -sftp.upload_artifacts(apache_id, artifacts_dir) + +confirm_or_fail(f"Going to check in artifacts to svn under {SVN_DEV_URL}/{rc_tag}. OK?") +svn.commit_artifacts(rc_tag, artifacts_dir, work_dir) confirm_or_fail("Going to build and upload mvn artifacts based on these settings:\n" + textfiles.read(global_gradle_props) + '\nOK?') cmd("Building and uploading archives", "./gradlew publish -PscalaVersion=2.13", cwd=kafka_dir, env=jdk8_env, shell=True) diff --git a/release/runtime.py b/release/runtime.py index a4cfd11e66b..c0d5d67eafb 100644 --- a/release/runtime.py +++ b/release/runtime.py @@ -133,12 +133,12 @@ def cmd(action, cmd_arg, *args, **kwargs): try: output = execute(cmd_arg, *args, stderr=subprocess.STDOUT, **kwargs) print(_prefix("> ", output.strip())) - return + return True except subprocess.CalledProcessError as e: print(e.output.decode("utf-8")) if allow_failure: - return + return False retry = confirm("Retry?") diff --git a/release/sftp.py b/release/sftp.py deleted file mode 100644 index f9ac042cf1e..00000000000 --- a/release/sftp.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -Auxiliary functions to interact with sftp(1). -""" - -import subprocess - -from runtime import ( - cmd, - confirm_or_fail, - execute, - fail, -) - -REMOTE_DIR = "public_html" - - -def mkdirp(apache_id, dir): - cmd_desc = f"Creating '{dir}' in your Apache home directory" - cmd_str = f"sftp -b - {apache_id}@home.apache.org" - stdin_str = f"mkdir {dir}\n" - cmd(cmd_desc, cmd_str, stdin=stdin_str, allow_failure=True) - - -def upload(apache_id, destination, dir): - cmd_desc = f"Uploading '{dir}' under {REMOTE_DIR} in your Apache home directory, this may take a while." - cmd_str = f"sftp -b - {apache_id}@home.apache.org" - stdin_str = f"cd {destination}\nput -r {dir}\n" - cmd(cmd_desc, cmd_str, stdin=stdin_str) - - -def upload_artifacts(apache_id, dir): - mkdirp(apache_id, REMOTE_DIR) - upload(apache_id, REMOTE_DIR, dir) - confirm_or_fail(f"Are the artifacts present in your Apache home: https://home.apache.org/~{apache_id}/ ?") - - -def test(apache_id): - """ - Test the ability to estalish an sftp session. - """ - execute(f"sftp {apache_id}@home.apache.org", input="bye") - diff --git a/release/svn.py b/release/svn.py new file mode 100644 index 00000000000..4869f79ddf1 --- /dev/null +++ b/release/svn.py @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Auxiliary functions to interact with svn(1). +""" + +import os +import shutil +import subprocess + +from runtime import cmd + +SVN_DEV_URL="https://dist.apache.org/repos/dist/dev/kafka" + +def delete_old_rc_directory_if_needed(rc_tag, src, work_dir): + svn_dev = os.path.join(work_dir, "svn_dev") + cmd_desc = f"Check if {rc_tag} exists in the subversion repository." + cmd_str = f"svn info --show-item revision {SVN_DEV_URL}/{rc_tag}" + if not cmd(cmd_desc, cmd_str, cwd = svn_dev, allow_failure = True): + print(f"Nothing under {SVN_DEV_URL}/{rc_tag}. Continuing.") + return + cmd_desc = f"Committing the deletion of {SVN_DEV_URL}/{rc_tag} from the svn repo." + cmd_str = f"svn delete -m Remove_{rc_tag} {SVN_DEV_URL}/{rc_tag}" + cmd(cmd_desc, cmd_str, cwd = svn_dev) + +def commit_artifacts(rc_tag, src, work_dir): + delete_old_rc_directory_if_needed(rc_tag, src, work_dir) + svn_dev = os.path.join(work_dir, "svn_dev") + dst = os.path.join(svn_dev, rc_tag) + print(f"Copying {src} to {dst}") + shutil.copytree(src, dst) + cmd_desc = f"Adding {SVN_DEV_URL}/{rc_tag} to the svn repo." + cmd_str = f"svn add ./{rc_tag}" + cmd(cmd_desc, cmd_str, cwd = svn_dev) + cmd_desc = f"Committing the addition of {SVN_DEV_URL}/{rc_tag} to the svn repo. Please wait, this may take a while." + cmd_str = f"svn commit -m Add_{rc_tag}" + cmd(cmd_desc, cmd_str, cwd = svn_dev) + +def checkout_svn_dev(work_dir): + svn_dev = os.path.join(work_dir, "svn_dev") + if os.path.exists(svn_dev): + shutil.rmtree(svn_dev) + cmd_desc = f"Checking out {SVN_DEV_URL} at {svn_dev}" + cmd_str = f"svn checkout --depth empty {SVN_DEV_URL}/ {svn_dev}" + cmd(cmd_desc, cmd_str) diff --git a/release/templates.py b/release/templates.py index 5eb3fc74f93..89826e1097c 100644 --- a/release/templates.py +++ b/release/templates.py @@ -167,23 +167,23 @@ Now you should sanity check it before proceeding. All subsequent steps start mak Some suggested steps: - * Grab the source archive and make sure it compiles: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz - * Grab one of the binary distros and run the quickstarts against them: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka_2.13-{release_version}.tgz - * Extract and verify one of the site docs jars: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka_2.13-{release_version}-site-docs.tgz + * Grab the source archive and make sure it compiles: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz + * Grab one of the binary distros and run the quickstarts against them: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz + * Extract and verify one of the site docs jars: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}-site-docs.tgz * Build a sample against jars in the staging repo: (TODO: Can we get a temporary URL before "closing" the staged artifacts?) * Validate GPG signatures on at least one file: - wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz && - wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.asc && - wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.md5 && - wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.sha1 && - wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.sha512 && + wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz && + wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.asc && + wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.md5 && + wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.sha1 && + wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.sha512 && gpg --verify kafka-{release_version}-src.tgz.asc kafka-{release_version}-src.tgz && gpg --print-md md5 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.md5 && gpg --print-md sha1 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.sha1 && gpg --print-md sha512 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.sha512 && rm kafka-{release_version}-src.tgz* && echo "OK" || echo "Failed" - * Validate the javadocs look ok. They are at https://home.apache.org/~{apache_id}/kafka-{rc_tag}/javadoc/ + * Validate the javadocs look ok. They are at https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/javadoc/ ******************************************************************************************************************************************************* """ @@ -201,7 +201,7 @@ This is the first candidate for release of Apache Kafka {release_version}. Release notes for the {release_version} release: -https://home.apache.org/~{apache_id}/kafka-{rc_tag}/RELEASE_NOTES.html +https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/RELEASE_NOTES.html *** Please download, test and vote by @@ -210,7 +210,7 @@ Kafka's KEYS file containing PGP keys we use to sign the release: https://kafka.apache.org/KEYS * Release artifacts to be voted upon (source and binary): -https://home.apache.org/~{apache_id}/kafka-{rc_tag}/ +https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/ * Docker release artifacts to be voted upon: @@ -221,7 +221,7 @@ apache/kafka-native:{rc_tag} https://repository.apache.org/content/groups/staging/org/apache/kafka/ * Javadoc: -https://home.apache.org/~{apache_id}/kafka-{rc_tag}/javadoc/ +https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/javadoc/ * Tag to be voted upon (off {dev_branch} branch) is the {release_version} tag: https://github.com/apache/kafka/releases/tag/{rc_tag}