From 675552a724fc8da5c3b09083f00bf2500930b50d Mon Sep 17 00:00:00 2001 From: PoAn Yang Date: Wed, 10 Sep 2025 13:02:03 +0800 Subject: [PATCH] KAFKA-19490: Remove usages of distutils in docker scripts (#20178) The [distutils](https://docs.python.org/3.13/whatsnew/3.12.html#distutils) package is removed from Python 3.12. Change `distutils` usage to `shutil`. Reviewers: Mickael Maison --------- Signed-off-by: PoAn Yang --- docker/common.py | 11 ++++------- docker/docker_build_test.py | 3 +-- docker/docker_official_image_build_test.py | 10 +++++----- docker/prepare_docker_official_image_source.py | 11 ++++------- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/docker/common.py b/docker/common.py index f04a484a187..5099a789da1 100644 --- a/docker/common.py +++ b/docker/common.py @@ -18,9 +18,6 @@ import subprocess import tempfile import os -from distutils.file_util import copy_file - -from distutils.dir_util import copy_tree import shutil def execute(command): @@ -36,11 +33,11 @@ def get_input(message): def build_docker_image_runner(command, image_type, kafka_archive=None): temp_dir_path = tempfile.mkdtemp() current_dir = os.path.dirname(os.path.realpath(__file__)) - copy_tree(f"{current_dir}/{image_type}", f"{temp_dir_path}/{image_type}") - copy_tree(f"{current_dir}/resources", f"{temp_dir_path}/{image_type}/resources") - copy_file(f"{current_dir}/server.properties", f"{temp_dir_path}/{image_type}") + shutil.copytree(f"{current_dir}/{image_type}", f"{temp_dir_path}/{image_type}", dirs_exist_ok=True) + shutil.copytree(f"{current_dir}/resources", f"{temp_dir_path}/{image_type}/resources", dirs_exist_ok=True) + shutil.copy(f"{current_dir}/server.properties", f"{temp_dir_path}/{image_type}") if kafka_archive: - copy_file(kafka_archive, f"{temp_dir_path}/{image_type}/kafka.tgz") + shutil.copy(kafka_archive, f"{temp_dir_path}/{image_type}/kafka.tgz") command = command.replace("$DOCKER_FILE", f"{temp_dir_path}/{image_type}/Dockerfile") command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/{image_type}") try: diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index fab6e65263d..9a986875fe3 100755 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -36,7 +36,6 @@ Usage: from datetime import date import argparse -from distutils.dir_util import copy_tree import shutil from test.docker_sanity_test import run_tests from common import execute, build_docker_image_runner @@ -47,7 +46,7 @@ def run_docker_tests(image, tag, kafka_url, image_type): temp_dir_path = tempfile.mkdtemp() try: current_dir = os.path.dirname(os.path.realpath(__file__)) - copy_tree(f"{current_dir}/test/fixtures", f"{temp_dir_path}/fixtures") + shutil.copytree(f"{current_dir}/test/fixtures", f"{temp_dir_path}/fixtures", dirs_exist_ok=True) execute(["wget", "-nv", "-O", f"{temp_dir_path}/kafka.tgz", kafka_url]) execute(["mkdir", f"{temp_dir_path}/fixtures/kafka"]) execute(["tar", "xfz", f"{temp_dir_path}/kafka.tgz", "-C", f"{temp_dir_path}/fixtures/kafka", "--strip-components", "1"]) diff --git a/docker/docker_official_image_build_test.py b/docker/docker_official_image_build_test.py index 3da68854c23..32869a1f4b2 100644 --- a/docker/docker_official_image_build_test.py +++ b/docker/docker_official_image_build_test.py @@ -34,7 +34,6 @@ Usage: """ import argparse -from distutils.dir_util import copy_tree import shutil from common import execute from docker_build_test import run_docker_tests @@ -46,10 +45,11 @@ def build_docker_official_image(image, tag, kafka_version, image_type): image = f'{image}:{tag}' current_dir = os.path.dirname(os.path.realpath(__file__)) temp_dir_path = tempfile.mkdtemp() - copy_tree(f"{current_dir}/docker_official_images/{kafka_version}/{image_type}", - f"{temp_dir_path}/{image_type}") - copy_tree(f"{current_dir}/docker_official_images/{kafka_version}/jvm/resources", - f"{temp_dir_path}/{image_type}/resources") + shutil.copytree(f"{current_dir}/docker_official_images/{kafka_version}/{image_type}", + f"{temp_dir_path}/{image_type}", dirs_exist_ok=True) + shutil.copytree(f"{current_dir}/docker_official_images/{kafka_version}/jvm/resources", + f"{temp_dir_path}/{image_type}/resources", dirs_exist_ok=True) + shutil.copy(f"{current_dir}/server.properties", f"{temp_dir_path}/{image_type}") command = f"docker build -f $DOCKER_FILE -t {image} $DOCKER_DIR" command = command.replace("$DOCKER_FILE", f"{temp_dir_path}/{image_type}/Dockerfile") command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/{image_type}") diff --git a/docker/prepare_docker_official_image_source.py b/docker/prepare_docker_official_image_source.py index 25d57c53e0f..bbc539b5c4c 100644 --- a/docker/prepare_docker_official_image_source.py +++ b/docker/prepare_docker_official_image_source.py @@ -33,7 +33,6 @@ Usage: from datetime import date import argparse -from distutils.dir_util import copy_tree import os import shutil import re @@ -61,12 +60,10 @@ if __name__ == '__main__': args = parser.parse_args() kafka_url = f"https://archive.apache.org/dist/kafka/{args.kafka_version}/kafka_2.13-{args.kafka_version}.tgz" current_dir = os.path.dirname(os.path.realpath(__file__)) - new_dir = os.path.join( - current_dir, f'docker_official_images', args.kafka_version) + new_dir = os.path.join(current_dir, 'docker_official_images', args.kafka_version) if os.path.exists(new_dir): shutil.rmtree(new_dir) os.makedirs(new_dir) - copy_tree(os.path.join(current_dir, args.image_type), os.path.join(new_dir, args.kafka_version, args.image_type)) - copy_tree(os.path.join(current_dir, 'resources'), os.path.join(new_dir, args.kafka_version, args.image_type, 'resources')) - remove_args_and_hardcode_values( - os.path.join(new_dir, args.kafka_version, args.image_type, 'Dockerfile'), args.kafka_version, kafka_url) + shutil.copytree(os.path.join(current_dir, args.image_type), os.path.join(new_dir, args.image_type), dirs_exist_ok=True) + shutil.copytree(os.path.join(current_dir, 'resources'), os.path.join(new_dir, args.image_type, 'resources'), dirs_exist_ok=True) + remove_args_and_hardcode_values(os.path.join(new_dir, args.image_type, 'Dockerfile'), args.kafka_version, kafka_url)