KAFKA-17735: release.py must not use home.apache.org (#17421)

Previously, Apache Kafka was uploading release candidate (RC) artifacts
to users' home directories on home.apache.org. However, since this
resource has been decommissioned, we need to follow the standard
approach of putting release candidate artifacts into the appropriate
subversion directory, at https://dist.apache.org/repos/dist/dev/kafka/.

Reviewers: Justine Olshan <jolshan@confluent.io>

release/README.md

@@ -8,7 +8,6 @@ This directory contains the tools used to publish a release.
 * python 3.12
 * git
 * gpg 2.4
-* sftp
 
 The full instructions for producing a release are available in
 https://cwiki.apache.org/confluence/display/KAFKA/Release+Process.

release/release.py

@@ -70,10 +70,12 @@ import git
 import gpg
 import notes
 import preferences
-import sftp
+import svn
 import templates
 import textfiles
 
+from svn import SVN_DEV_URL
+
 
 def get_jdk(version):
     """
@@ -243,7 +245,6 @@ def verify_prerequeisites():
         except Exception as e:
             fail(f"Pre-requisite not met: {name}. Error: {e}")
     prereq('Apache Maven CLI (mvn) in PATH', lambda: "Apache Maven" in execute("mvn -v"))
-    prereq('Apache sftp connection', lambda: sftp.test(apache_id))
     prereq("svn CLI in PATH", lambda: "svn" in execute("svn --version"))
     prereq("Verifying that you have no unstaged git changes", lambda: git.has_unstaged_changes())
     prereq("Verifying that you have no staged git changes", lambda: git.has_staged_changes())
@@ -298,7 +299,7 @@ git.commit(f"Bump version to {release_version}")
 git.create_tag(rc_tag)
 git.switch_branch(starting_branch)
 
-# Note that we don't use tempfile here because mkdtemp causes problems with sftp and being able to determine the absolute path to a file.
+# Note that we don't use tempfile here because mkdtemp causes problems with being able to determine the absolute path to a file.
 # Instead we rely on a fixed path
 work_dir = os.path.join(repo_dir, ".release_work_dir")
 clean_up_work_dir = lambda: cmd("Cleaning up work directory", f"rm -rf {work_dir}")
@@ -315,6 +316,7 @@ git.clone(repo_dir, 'kafka', cwd=work_dir)
 git.create_branch(release_version, rc_tag, cwd=kafka_dir)
 current_year = datetime.datetime.now().year
 cmd("Verifying the correct year in NOTICE", f"grep {current_year} NOTICE", cwd=kafka_dir)
+svn.checkout_svn_dev(work_dir)
 
 print("Generating release notes")
 try:
@@ -348,7 +350,9 @@ for filename in os.listdir(artifacts_dir):
 
 cmd("Listing artifacts to be uploaded:", f"ls -R {artifacts_dir}")
 cmd("Zipping artifacts", f"tar -czf {artifact_name}.tar.gz {artifact_name}", cwd=work_dir)
-sftp.upload_artifacts(apache_id, artifacts_dir)
+
+confirm_or_fail(f"Going to check in artifacts to svn under {SVN_DEV_URL}/{rc_tag}. OK?")
+svn.commit_artifacts(rc_tag, artifacts_dir, work_dir)
 
 confirm_or_fail("Going to build and upload mvn artifacts based on these settings:\n" + textfiles.read(global_gradle_props) + '\nOK?')
 cmd("Building and uploading archives", "./gradlew publish -PscalaVersion=2.13", cwd=kafka_dir, env=jdk8_env, shell=True)

release/runtime.py

@@ -133,12 +133,12 @@ def cmd(action, cmd_arg, *args, **kwargs):
         try:
             output = execute(cmd_arg, *args, stderr=subprocess.STDOUT, **kwargs)
             print(_prefix("> ", output.strip()))
-            return
+            return True
         except subprocess.CalledProcessError as e:
             print(e.output.decode("utf-8"))
 
             if allow_failure:
-                return
+                return False
 
             retry = confirm("Retry?")
 

release/sftp.py

@@ -1,59 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Auxiliary functions to interact with sftp(1).
-"""
-
-import subprocess
-
-from runtime import (
-    cmd,
-    confirm_or_fail,
-    execute,
-    fail,
-)
-
-REMOTE_DIR = "public_html"
-
-
-def mkdirp(apache_id, dir):
-    cmd_desc = f"Creating '{dir}' in your Apache home directory"
-    cmd_str = f"sftp -b - {apache_id}@home.apache.org"
-    stdin_str  = f"mkdir {dir}\n"
-    cmd(cmd_desc, cmd_str, stdin=stdin_str, allow_failure=True)
-
-
-def upload(apache_id, destination, dir):
-    cmd_desc = f"Uploading '{dir}' under {REMOTE_DIR} in your Apache home directory, this may take a while."
-    cmd_str = f"sftp -b - {apache_id}@home.apache.org"
-    stdin_str  = f"cd {destination}\nput -r {dir}\n"
-    cmd(cmd_desc, cmd_str, stdin=stdin_str)
-
-
-def upload_artifacts(apache_id, dir):
-    mkdirp(apache_id, REMOTE_DIR)
-    upload(apache_id, REMOTE_DIR, dir)
-    confirm_or_fail(f"Are the artifacts present in your Apache home: https://home.apache.org/~{apache_id}/ ?")
-
-
-def test(apache_id):
-    """
-    Test the ability to estalish an sftp session.
-    """
-    execute(f"sftp {apache_id}@home.apache.org", input="bye")
-

release/svn.py

@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Auxiliary functions to interact with svn(1).
+"""
+
+import os
+import shutil
+import subprocess
+
+from runtime import cmd
+
+SVN_DEV_URL="https://dist.apache.org/repos/dist/dev/kafka"
+
+def delete_old_rc_directory_if_needed(rc_tag, src, work_dir):
+    svn_dev = os.path.join(work_dir, "svn_dev")
+    cmd_desc = f"Check if {rc_tag} exists in the subversion repository."
+    cmd_str = f"svn info --show-item revision {SVN_DEV_URL}/{rc_tag}"
+    if not cmd(cmd_desc, cmd_str, cwd = svn_dev, allow_failure = True):
+        print(f"Nothing under {SVN_DEV_URL}/{rc_tag}. Continuing.")
+        return
+    cmd_desc = f"Committing the deletion of {SVN_DEV_URL}/{rc_tag} from the svn repo."
+    cmd_str = f"svn delete -m Remove_{rc_tag} {SVN_DEV_URL}/{rc_tag}"
+    cmd(cmd_desc, cmd_str, cwd = svn_dev)
+
+def commit_artifacts(rc_tag, src, work_dir):
+    delete_old_rc_directory_if_needed(rc_tag, src, work_dir)
+    svn_dev = os.path.join(work_dir, "svn_dev")
+    dst = os.path.join(svn_dev, rc_tag)
+    print(f"Copying {src} to {dst}")
+    shutil.copytree(src, dst)
+    cmd_desc = f"Adding {SVN_DEV_URL}/{rc_tag} to the svn repo."
+    cmd_str = f"svn add ./{rc_tag}"
+    cmd(cmd_desc, cmd_str, cwd = svn_dev)
+    cmd_desc = f"Committing the addition of {SVN_DEV_URL}/{rc_tag} to the svn repo. Please wait, this may take a while."
+    cmd_str = f"svn commit -m Add_{rc_tag}"
+    cmd(cmd_desc, cmd_str, cwd = svn_dev)
+
+def checkout_svn_dev(work_dir):
+    svn_dev = os.path.join(work_dir, "svn_dev")
+    if os.path.exists(svn_dev):
+        shutil.rmtree(svn_dev)
+    cmd_desc = f"Checking out {SVN_DEV_URL} at {svn_dev}"
+    cmd_str = f"svn checkout --depth empty {SVN_DEV_URL}/ {svn_dev}"
+    cmd(cmd_desc, cmd_str)

release/templates.py

@@ -167,23 +167,23 @@ Now you should sanity check it before proceeding. All subsequent steps start mak
 
 Some suggested steps:
 
- * Grab the source archive and make sure it compiles: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz
+ * Grab the source archive and make sure it compiles: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz
- * Grab one of the binary distros and run the quickstarts against them: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka_2.13-{release_version}.tgz
+ * Grab one of the binary distros and run the quickstarts against them: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz
- * Extract and verify one of the site docs jars: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka_2.13-{release_version}-site-docs.tgz
+ * Extract and verify one of the site docs jars: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}-site-docs.tgz
  * Build a sample against jars in the staging repo: (TODO: Can we get a temporary URL before "closing" the staged artifacts?)
  * Validate GPG signatures on at least one file:
-      wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz &&
+      wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz &&
-      wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.asc &&
+      wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.asc &&
-      wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.md5 &&
+      wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.md5 &&
-      wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.sha1 &&
+      wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.sha1 &&
-      wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.sha512 &&
+      wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.sha512 &&
       gpg --verify kafka-{release_version}-src.tgz.asc kafka-{release_version}-src.tgz &&
       gpg --print-md md5 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.md5 &&
       gpg --print-md sha1 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.sha1 &&
       gpg --print-md sha512 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.sha512 &&
       rm kafka-{release_version}-src.tgz* &&
       echo "OK" || echo "Failed"
- * Validate the javadocs look ok. They are at https://home.apache.org/~{apache_id}/kafka-{rc_tag}/javadoc/
+ * Validate the javadocs look ok. They are at https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/javadoc/
 
 *******************************************************************************************************************************************************
 """
@@ -201,7 +201,7 @@ This is the first candidate for release of Apache Kafka {release_version}.
 <DESCRIPTION OF MAJOR CHANGES, INCLUDE INDICATION OF MAJOR/MINOR RELEASE>
 
 Release notes for the {release_version} release:
-https://home.apache.org/~{apache_id}/kafka-{rc_tag}/RELEASE_NOTES.html
+https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/RELEASE_NOTES.html
 
 *** Please download, test and vote by <VOTING DEADLINE, e.g. Monday, March 28, 9am PT>
 <THE RELEASE POLICY (https://www.apache.org/legal/release-policy.html#release-approval) REQUIRES VOTES TO BE OPEN FOR MINIMUM OF 3 DAYS THEREFORE VOTING DEADLINE SHOULD BE AT LEAST 72 HOURS FROM THE TIME THIS EMAIL IS SENT.>
@@ -210,7 +210,7 @@ Kafka's KEYS file containing PGP keys we use to sign the release:
 https://kafka.apache.org/KEYS
 
 * Release artifacts to be voted upon (source and binary):
-https://home.apache.org/~{apache_id}/kafka-{rc_tag}/
+https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/
 
 <USE docker/README.md FOR STEPS TO CREATE RELEASE CANDIDATE DOCKER IMAGE>
 * Docker release artifacts to be voted upon:
@@ -221,7 +221,7 @@ apache/kafka-native:{rc_tag}
 https://repository.apache.org/content/groups/staging/org/apache/kafka/
 
 * Javadoc:
-https://home.apache.org/~{apache_id}/kafka-{rc_tag}/javadoc/
+https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/javadoc/
 
 * Tag to be voted upon (off {dev_branch} branch) is the {release_version} tag:
 https://github.com/apache/kafka/releases/tag/{rc_tag}