KAFKA-5484: Refactor kafkatest docker support

Author: Colin P. Mccabe <cmccabe@confluent.io>

Reviewers: Ewen Cheslack-Postava <ewen@confluent.io>

Closes #3389 from cmccabe/KAFKA-5484

tests/cluster_file_generator.sh

@@ -1,63 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cat<<EOF
-{
-  "_comment": [
-    "Licensed to the Apache Software Foundation (ASF) under one or more",
-    "contributor license agreements.  See the NOTICE file distributed with",
-    "this work for additional information regarding copyright ownership.",
-    "The ASF licenses this file to You under the Apache License, Version 2.0",
-    "(the \"License\"); you may not use this file except in compliance with",
-    "the License.  You may obtain a copy of the License at",
-    "",
-    "http://www.apache.org/licenses/LICENSE-2.0",
-    "",
-    "Unless required by applicable law or agreed to in writing, software",
-    "distributed under the License is distributed on an \"AS IS\" BASIS,",
-    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.",
-    "See the License for the specific language governing permissions and",
-    "limitations under the License."
-  ],
-  "nodes": [
-EOF
-
-for K in $(seq -w 02 ${KAFKA_NUM_CONTAINERS}); do
-  KNODE="knode${K}"
-  if [ $K -eq ${KAFKA_NUM_CONTAINERS} ]; then
-    SUFFIX=""
-  else
-    SUFFIX=","
-  fi
-  cat<<EOF
-    {
-      "externally_routable_ip": "$KNODE",
-      "ssh_config": {
-        "host": "$KNODE",
-        "hostname": "$KNODE",
-        "identityfile": "/root/.ssh/id_rsa",
-        "password": "",
-        "port": 22,
-        "user": "root"
-      }
-    }$SUFFIX
-EOF
-done
-
-cat<<EOF
-  ]
-}
-EOF

tests/docker/Dockerfile

@@ -19,15 +19,35 @@ MAINTAINER Apache Kafka dev@kafka.apache.org
 VOLUME ["/opt/kafka-dev"]
 ENV MIRROR="http://apache.cs.utah.edu/"
 
-ADD ssh /root/.ssh
+# Set the timezone.
-RUN chmod 600 /root/.ssh/id_rsa
+ENV TZ="/usr/share/zoneinfo/America/Los_Angeles"
-RUN apt update && apt install -y unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev
+
-RUN pip install -U pip && pip install --upgrade cffi ducktape==0.6.0
+# Do not ask for confirmations when running apt-get, etc.
-# Test with multiple Scala version
+ENV DEBIAN_FRONTEND noninteractive
+
+# Set the ducker.creator label so that we know that this is a ducker image.  This will make it
+# visible to 'ducker purge'.  The ducker.creator label also lets us know what UNIX user built this
+# image.
+ARG ducker_creator=default
+LABEL ducker.creator=$ducker_creator
+
+# Update Linux and install necessary utilities.
+RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev && apt-get -y clean
+RUN pip install -U pip && pip install --upgrade cffi virtualenv pyasn1 && pip install --upgrade ducktape==0.6.0
+
+# Set up ssh
+COPY ./ssh-config /root/.ssh/config
+RUN ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+# Install binary test dependencies.
 RUN mkdir -p "/opt/kafka-0.8.2.2" && curl -s "${MIRROR}kafka/0.8.2.2/kafka_2.10-0.8.2.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.8.2.2"
 RUN mkdir -p "/opt/kafka-0.9.0.1" && curl -s "${MIRROR}kafka/0.9.0.1/kafka_2.11-0.9.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.9.0.1"
 RUN mkdir -p "/opt/kafka-0.10.0.1" && curl -s "${MIRROR}kafka/0.10.0.1/kafka_2.11-0.10.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.0.1"
 RUN mkdir -p "/opt/kafka-0.10.1.1" && curl -s "${MIRROR}kafka/0.10.1.1/kafka_2.11-0.10.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.1.1"
 RUN mkdir -p "/opt/kafka-0.10.2.1" && curl -s "${MIRROR}kafka/0.10.2.1/kafka_2.11-0.10.2.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.2.1"
 
-CMD service ssh start && tail -f /dev/null
+# Set up the ducker user.
+RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq /root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ && echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+USER ducker
+
+CMD sudo service ssh start && tail -f /dev/null

tests/docker/ducker-ak

@@ -0,0 +1,511 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Ducker-AK: a tool for running Apache Kafka system tests inside Docker images.
+#
+# Note: this should be compatible with the version of bash that ships on most
+# Macs, bash 3.2.57.
+#
+
+script_path="${0}"
+
+# The absolute path to the directory which this script is in.  This will also be the directory
+# which we run docker build from.
+ducker_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# The absolute path to the root Kafka directory
+kafka_dir="$( cd "${ducker_dir}/../.." && pwd )"
+
+# The memory consumption to allow during the docker build.
+# This does not include swap.
+docker_build_memory_limit="3200m"
+
+# The maximum mmemory consumption to allow in containers.
+docker_run_memory_limit="2000m"
+
+# The default number of cluster nodes to bring up if a number is not specified.
+default_num_nodes=14
+
+# The default ducker-ak image name.
+default_image_name="ducker-ak"
+
+# Display a usage message on the terminal and exit.
+#
+# $1: The exit status to use
+usage() {
+    local exit_status="${1}"
+    cat <<EOF
+ducker-ak: a tool for running Apache Kafka tests inside Docker images.
+
+Usage: ${script_path} [command] [options]
+
+help|-h|--help
+    Display this help message
+
+up [-n|--num-nodes NUM_NODES] [-f|--force] [docker-image]
+    Bring up a cluster with the specified amount of nodes (defaults to ${default_num_nodes}).
+    The docker image name defaults to ${default_image_name}.  If --force is specified, we will
+    attempt to bring up an image even some parameters are not valid.
+
+test [test-name(s)]
+    Run a test or set of tests inside the currently active Ducker nodes.
+    For example, to run the Muckrake test simple_consumer_shell_test, you would run:
+        ./tests/docker/ducker-ak test ./tests/kafkatest/test/core/simple_consumer_shell_test.py
+
+ssh [node-name|user-name@node-name] [command]
+    Log in to a running ducker container.  If node-name is not given, it prints
+    the names of all running nodes.  If node-name is 'all', we will run the
+    command on every node.  If user-name is given, we will try to log in as
+    that user.  Otherwise, we will log in as the 'ducker' user.  If a command
+    is specified, we will run that command.  Otherwise, we will provide a login
+    shell.
+
+down [-q|--quiet]
+    Tear down all the currently active ducker-ak nodes.  If --quiet is specified,
+    only error messages are printed.
+
+purge [--f|--force]
+    Purge Docker images created by ducker-ak.  This will free disk space.
+    If --force is set, we run 'docker rmi -f'.
+EOF
+    exit "${exit_status}"
+}
+
+# Exit with an error message.
+die() {
+    echo $@
+    exit 1
+}
+
+# Check for the presence of certain commands.
+#
+# $@: The commands to check for.  This function will die if any of these commands are not found by
+#       the 'which' command.
+require_commands() {
+    local cmds="${@}"
+    for cmd in ${cmds}; do
+        which -- "${cmd}" &> /dev/null || die "You must install ${cmd} to run this script."
+    done
+}
+
+# Set a global variable to a value.
+#
+# $1: The variable name to set.  This function will die if the variable already has a value.  The
+#     variable will be made readonly to prevent any future modifications.
+# $2: The value to set the variable to.  This function will die if the value is empty or starts
+#     with a dash.
+# $3: A human-readable description of the variable.
+set_once() {
+    local key="${1}"
+    local value="${2}"
+    local what="${3}"
+    [[ -n "${!key}" ]] && die "Error: more than one value specified for ${what}."
+    verify_command_line_argument "${value}" "${what}"
+    # It would be better to use declare -g, but older bash versions don't support it.
+    export ${key}="${value}"
+}
+
+# Verify that a command-line argument is present and does not start with a slash.
+#
+# $1: The command-line argument to verify.
+# $2: A human-readable description of the variable.
+verify_command_line_argument() {
+    local value="${1}"
+    local what="${2}"
+    [[ -n "${value}" ]] || die "Error: no value specified for ${what}"
+    [[ ${value} == -* ]] && die "Error: invalid value ${value} specified for ${what}"
+}
+
+# Echo a message if a flag is set.
+#
+# $1: If this is 1, the message will be echoed.
+# $@: The message
+maybe_echo() {
+    local verbose="${1}"
+    shift
+    [[ "${verbose}" -eq 1 ]] && echo "${@}"
+}
+
+# Counts the number of elements passed to this subroutine.
+count() {
+    echo $#
+}
+
+# Push a new directory on to the bash directory stack, or exit with a failure message.
+#
+# $1: The directory push on to the directory stack.
+must_pushd() {
+    local target_dir="${1}"
+    pushd -- "${target_dir}" &> /dev/null || die "failed to change directory to ${target_dir}"
+}
+
+# Pop a directory from the bash directory stack, or exit with a failure message.
+must_popd() {
+    popd &> /dev/null || die "failed to popd"
+}
+
+# Run a command and die if it fails.
+#
+# $1: If this equals 1, we will print the command before executing it.
+# $2: If this equals 1, we will show the output of the command.
+# $@: The command to run.
+must_do() {
+    local verbose="${1}"
+    local output="/dev/null"
+    [[ "${2}" -eq 1 ]] && output="/dev/stdout"
+    shift 2
+    local cmd="${@}"
+    [[ "${verbose}" -eq 1 ]] && echo "${cmd}"
+    ${cmd} >${output} || die "${1} failed"
+}
+
+# Ask the user a yes/no question.
+#
+# $1: The prompt to use
+# $_return: 0 if the user answered no; 1 if the user answered yes.
+ask_yes_no() {
+    local prompt="${1}"
+    while true; do
+        read -r -p "${prompt} " response
+        case "${response}" in
+            [yY]|[yY][eE][sS]) _return=1; return;;
+            [nN]|[nN][oO]) _return=0; return;;
+            *);;
+        esac
+        echo "Please respond 'yes' or 'no'."
+        echo
+    done
+}
+
+# Build a docker image.
+#
+# $1: The name of the image to build.
+ducker_build() {
+    local image_name="${1}"
+
+    # Use SECONDS, a builtin bash variable that gets incremented each second, to measure the docker
+    # build duration.
+    SECONDS=0
+
+    must_pushd "${ducker_dir}"
+    must_do 1 1 docker build --memory="${docker_build_memory_limit}" \
+        --build-arg "ducker_creator=${user_name}" -t "${image_name}" \
+        -f "${ducker_dir}/Dockerfile" ${docker_args} -- .
+    docker_status=$?
+    must_popd
+    duration="${SECONDS}"
+    if [[ ${docker_status} -ne 0 ]]; then
+        die "** ERROR: Failed to build ${what} image after $((${duration} / 60))m \
+$((${duration} % 60))s.  See ${build_log} for details."
+    fi
+    echo "** Successfully built ${what} image in $((${duration} / 60))m \
+$((${duration} % 60))s.  See ${build_log} for details."
+}
+
+ducker_up() {
+    require_commands docker
+    while [[ $# -ge 1 ]]; do
+        case "${1}" in
+            -f|--force) force=1; shift;;
+            -n|--num-nodes) set_once num_nodes "${2}" "number of nodes"; shift 2;;
+            *) set_once image_name "${1}" "docker image name"; shift;;
+        esac
+    done
+    [[ -n "${num_nodes}" ]] || num_nodes="${default_num_nodes}"
+    [[ -n "${image_name}" ]] || image_name=ducker-ak
+    [[ "${num_nodes}" =~ ^-?[0-9]+$ ]] || \
+        die "ducker_up: the number of nodes must be an integer."
+    [[ "${num_nodes}" -gt 0 ]] || die "ducker_up: the number of nodes must be greater than 0."
+    if [[ "${num_nodes}" -lt 2 ]]; then
+        if [[ "${force}" -ne 1 ]]; then
+            echo "ducker_up: It is recommended to run at least 2 nodes, since ducker01 is only \
+used to run ducktape itself.  If you want to do it anyway, you can use --force to attempt to \
+use only ${num_nodes}."
+            exit 1
+        fi
+    fi
+    ducker_build "${image_name}"
+
+    docker ps >/dev/null || die "ducker_up: failed to run docker.  Please check that the daemon is started."
+    docker inspect --format='{{.Config.Labels}}' --type=image "${image_name}" | grep -q 'ducker.type'
+    local docker_status=${PIPESTATUS[0]}
+    local grep_status=${PIPESTATUS[1]}
+    [[ "${docker_status}" -eq 0 ]] || die "ducker_up: failed to inspect image ${image_name}.  \
+Please check that it exists."
+    if [[ "${grep_status}" -ne 0 ]]; then
+        if [[ "${force}" -ne 1 ]]; then
+            echo "ducker_up: ${image_name} does not appear to be a ducker image.  It lacks the \
+ducker.type label.  If you think this is a mistake, you can use --force to attempt to bring \
+it up anyway."
+            exit 1
+        fi
+    fi
+    local running_containers="$(docker ps -f=network=ducknet -q)"
+    local num_running_containers=$(count ${running_containers})
+    if [[ ${num_running_containers} -gt 0 ]]; then
+        die "ducker_up: there are ${num_running_containers} ducker containers \
+running already.  Use ducker down to bring down these containers before \
+attempting to start new ones."
+    fi
+
+    echo "ducker_up: Bringing up ${image_name} with ${num_nodes} nodes..."
+    if docker network inspect ducknet &>/dev/null; then
+        must_do 1 0 docker network rm ducknet
+    fi
+    must_do 1 0 docker network create ducknet
+    for n in $(seq -f %02g 1 ${num_nodes}); do
+        local node="ducker${n}"
+        # Invoke docker-run. Enable the NET_ADMIN and NET_RAW capabilities so that we can use
+        # iptables inside the container.
+        must_do 1 0 docker run --cap-add=NET_ADMIN --cap-add=NET_RAW \
+            -d -t --net=host -h "${node}" --network ducknet \
+            --memory=${docker_run_memory_limit} --memory-swappiness=1 \
+            -v "${kafka_dir}:/opt/kafka-dev" --name "${node}" -- "${image_name}"
+    done
+    mkdir -p "${ducker_dir}/build"
+    exec 3<> "${ducker_dir}/build/node_hosts"
+    for n in $(seq -f %02g 1 ${num_nodes}); do
+        local node="ducker${n}"
+        docker exec --user=root "${node}" grep "${node}" /etc/hosts >&3
+        [[ $? -ne 0 ]] && die "failed to find the /etc/hosts entry for ${node}"
+    done
+    exec 3>&-
+    for n in $(seq -f %02g 1 ${num_nodes}); do
+        local node="ducker${n}"
+        docker exec --user=root "${node}" \
+            bash -c "grep -v ${node} /opt/kafka-dev/tests/docker/build/node_hosts >> /etc/hosts"
+        [[ $? -ne 0 ]] && die "failed to append to the /etc/hosts file on ${node}"
+    done
+
+    echo "ducker_up: added the latest entries to /etc/hosts on each node."
+    generate_cluster_json_file "${num_nodes}" "${ducker_dir}/build/cluster.json"
+    echo "ducker_up: successfully wrote ${ducker_dir}/build/cluster.json"
+    echo "** ducker_up: successfully brought up ${num_nodes} nodes."
+}
+
+# Generate the cluster.json file used by ducktape to identify cluster nodes.
+#
+# $1: The number of cluster nodes.
+# $2: The path to write the cluster.json file to.
+generate_cluster_json_file() {
+    local num_nodes="${1}"
+    local path="${2}"
+    exec 3<> "${path}"
+cat<<EOF >&3
+{
+  "_comment": [
+    "Licensed to the Apache Software Foundation (ASF) under one or more",
+    "contributor license agreements.  See the NOTICE file distributed with",
+    "this work for additional information regarding copyright ownership.",
+    "The ASF licenses this file to You under the Apache License, Version 2.0",
+    "(the \"License\"); you may not use this file except in compliance with",
+    "the License.  You may obtain a copy of the License at",
+    "",
+    "http://www.apache.org/licenses/LICENSE-2.0",
+    "",
+    "Unless required by applicable law or agreed to in writing, software",
+    "distributed under the License is distributed on an \"AS IS\" BASIS,",
+    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.",
+    "See the License for the specific language governing permissions and",
+    "limitations under the License."
+  ],
+  "nodes": [
+EOF
+    for n in $(seq 2 ${num_nodes}); do
+      if [[ ${n} -eq ${num_nodes} ]]; then
+        suffix=""
+      else
+        suffix=","
+      fi
+      local node=$(printf ducker%02d ${n})
+cat<<EOF >&3
+    {
+      "externally_routable_ip": "${node}",
+      "ssh_config": {
+        "host": "${node}",
+        "hostname": "${node}",
+        "identityfile": "/home/ducker/.ssh/id_rsa",
+        "password": "",
+        "port": 22,
+        "user": "ducker"
+      }
+    }${suffix}
+EOF
+    done
+cat<<EOF >&3
+  ]
+}
+EOF
+    exec 3>&-
+}
+
+ducker_test() {
+    require_commands docker
+    docker inspect ducker01 &>/dev/null || \
+        die "ducker_test: the ducker01 instance appears to be down. Did you run 'ducker up'?"
+    [[ $# -lt 1 ]] && \
+        die "ducker_test: you must supply at least one system test to run. Type --help for help."
+    local args=""
+    local kafka_test=0
+    for arg in "${@}"; do
+        local regex=".*\/kafkatest\/(.*)"
+        if [[ $arg =~ $regex ]]; then
+            local kpath=${BASH_REMATCH[1]}
+            args="${args} ./tests/kafkatest/${kpath}"
+        else
+            args="${args} ${arg}"
+        fi
+    done
+    must_pushd "${kafka_dir}"
+    ls ./core/build/distributions/kafka_*.tgz &> /dev/null
+    [[ $? -eq 0 ]] || die "Failed to find core/build/distributions/kafka_*.tgz.  Did you run ./gradlew releaseTarGz?"
+    must_popd
+    cmd="cd /opt/kafka-dev && ducktape --cluster-file /opt/kafka-dev/tests/docker/build/cluster.json $args"
+    echo "docker exec -it ducker01 bash -c \"${cmd}\""
+    exec docker exec --user=ducker -it ducker01 bash -c "${cmd}"
+}
+
+ducker_ssh() {
+    require_commands docker
+    [[ $# -eq 0 ]] && die "ducker_ssh: Please specify a container name to log into. \
+Currently active containers: $(echo_running_container_names)"
+    local node_info="${1}"
+    shift
+    local guest_command="$*"
+    local user_name="ducker"
+    if [[ "${node_info}" =~ @ ]]; then
+        user_name="${node_info%%@*}"
+        local node_name="${node_info##*@}"
+    else
+        local node_name="${node_info}"
+    fi
+    local docker_flags=""
+    if [[ -z "${guest_command}" ]]; then
+        local docker_flags="${docker_flags} -t"
+        local guest_command_prefix=""
+        guest_command=bash
+    else
+        local guest_command_prefix="bash -c"
+    fi
+    if [[ "${node_name}" == "all" ]]; then
+        local nodes=$(echo_running_container_names)
+        [[ "${nodes}" == "(none)" ]] && die "ducker_ssh: can't locate any running ducker nodes."
+        for node in ${nodes}; do
+            docker exec --user=${user_name} -i ${docker_flags} "${node}" \
+                ${guest_command_prefix} "${guest_command}" || die "docker exec ${node} failed"
+        done
+    else
+        docker inspect --type=container -- "${node_name}" &>/dev/null || \
+            die "ducker_ssh: can't locate node ${node_name}. Currently running nodes: \
+$(echo_running_container_names)"
+        exec docker exec --user=${user_name} -i ${docker_flags} "${node_name}" \
+            ${guest_command_prefix} "${guest_command}"
+    fi
+}
+
+# Echo all the running Ducker container names, or (none) if there are no running Ducker containers.
+echo_running_container_names() {
+    node_names="$(docker ps -f=network=ducknet -q --format '{{.Names}}' | sort)"
+    if [[ -z "${node_names}" ]]; then
+        echo "(none)"
+    else
+        echo ${node_names//$'\n'/ }
+    fi
+}
+
+ducker_down() {
+    require_commands docker
+    local verbose=1
+    while [[ $# -ge 1 ]]; do
+        case "${1}" in
+            -q|--quiet) verbose=0; shift;;
+            *) die "ducker_down: unexpected command-line argument ${1}";;
+        esac
+    done
+    local running_containers
+    running_containers="$(docker ps -f=network=ducknet -q)"
+    [[ $? -eq 0 ]]  || die "ducker_down: docker command failed.  Is the docker daemon running?"
+    running_containers=${running_containers//$'\n'/ }
+    local all_containers="$(docker ps -a -f=network=ducknet -q)"
+    all_containers=${all_containers//$'\n'/ }
+    if [[ -z "${all_containers}" ]]; then
+        maybe_echo "${verbose}" "No ducker containers found."
+        return
+    fi
+    if [[ -n "${running_containers}" ]]; then
+        must_do ${verbose} 0 docker kill "${running_containers}"
+    fi
+    must_do ${verbose} 0 docker rm "${all_containers}"
+    must_do ${verbose} 1 rm -f -- "${ducker_dir}/build/node_hosts" "${ducker_dir}/build/cluster.json"
+    if docker network inspect ducknet &>/dev/null; then
+        must_do 1 0 docker network rm ducknet
+    fi
+    maybe_echo "${verbose}" "ducker_down: removed $(count ${all_containers}) containers."
+}
+
+ducker_purge() {
+    require_commands docker
+    local force_str=""
+    while [[ $# -ge 1 ]]; do
+        case "${1}" in
+            -f|--force) force_str="-f"; shift;;
+            *) die "ducker_purge: unknown argument ${1}";;
+        esac
+    done
+    echo "** ducker_purge: attempting to locate ducker images to purge"
+    local images
+    images=$(docker images -q -a -f label=ducker.creator)
+    [[ $? -ne 0 ]] && die "docker images command failed"
+    images=${images//$'\n'/ }
+    declare -a purge_images=()
+    if [[ -z "${images}" ]]; then
+        echo "** ducker_purge: no images found to purge."
+        exit 0
+    fi
+    echo "** ducker_purge: images to delete:"
+    for image in ${images}; do
+        echo -n "${image} "
+        docker inspect --format='{{.Config.Labels}} {{.Created}}' --type=image "${image}"
+        [[ $? -ne 0 ]] && die "docker inspect ${image} failed"
+    done
+    ask_yes_no "Delete these docker images? [y/n]"
+    [[ "${_return}" -eq 0 ]] && exit 0
+    must_do 1 1 docker rmi ${force_str} ${images}
+}
+
+# Parse command-line arguments
+[[ $# -lt 1 ]] && usage 0
+# Display the help text if -h or --help appears in the command line
+for arg in ${@}; do
+    case "${arg}" in
+        -h|--help) usage 0;;
+        --) break;;
+        *);;
+    esac
+done
+action="${1}"
+shift
+case "${action}" in
+    help) usage 0;;
+
+    up|test|ssh|down|purge)
+        ducker_${action} "${@}"; exit 0;;
+
+    *)  echo "Unknown command '${action}'.  Type '${script_path} --help' for usage information."
+        exit 1;;
+esac

tests/docker/run_tests.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,55 +15,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# To run tests use a command like:
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-#   TC_PATHS="tests/kafkatest/tests/streams tests/kafkatest/tests/tools" bash tests/docker/run_tests.sh
+KAFKA_NUM_CONTAINERS=${KAFKA_NUM_CONTAINERS:-14}
-set -x
+TC_PATHS=${TC_PATHS:-./kafkatest/}
 
 die() {
     echo $@
     exit 1
 }
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+if ${SCRIPT_DIR}/ducker-ak ssh | grep -q '(none)'; then
-TESTS_DIR=`dirname ${SCRIPT_DIR}`
+    ${SCRIPT_DIR}/ducker-ak up -n "${KAFKA_NUM_CONTAINERS}" || die "ducker-ak up failed"
-KAFKA_SRC=`dirname ${TESTS_DIR}`
-KAFKA_VERSION=$(grep "version=.*" ${KAFKA_SRC}/gradle.properties | cut -f 2 -d =)
-JDK_INFO="openjdk8"
-KAFKA_IMAGE=${KAFKA_IMAGE:-kafkadev/kafka-image:${KAFKA_VERSION}}_${JDK_INFO}
-export KAFKA_NUM_CONTAINERS=13
-
-chmod 600 ${SCRIPT_DIR}/ssh/id_rsa
-cd ${KAFKA_SRC}
-(( $(ls -1 core/build/distributions/kafka_*SNAPSHOT.tgz | wc -l) != 1 )) && echo 'Expecting a single file like core/build/distributions/kafka_*SNAPSHOT.tgz, found:' $(ls -1 core/build/distributions/kafka_*SNAPSHOT.tgz) && echo "Did you run ./gradlew clean releaseTarGz ?" && exit 1
-
-docker network rm knw
-docker network create knw
-
-docker kill $(docker ps -f=network=knw -q)
-docker rm $(docker ps -a -f=network=knw -q)
-
-docker run --rm -it ${KAFKA_IMAGE} "true"
-if [[ $? != 0 || ${KAFKA_IMAGE_REBUILD} != "" ]]; then
-    echo "kafka image ${KAFKA_IMAGE} does not exist. Building it from scratch."
-    COMMIT_INFO=$(git describe HEAD)
-    docker build -t ${KAFKA_IMAGE} --label=commit_info=${COMMIT_INFO} ${SCRIPT_DIR} \
-        || die "docker build failed"
 fi
-
+${SCRIPT_DIR}/ducker-ak test ${TC_PATHS} || die "ducker-ak test failed"
-echo "Using kafka image: ${KAFKA_IMAGE}"
-docker inspect ${KAFKA_IMAGE}
-for i in $(seq -w 1 ${KAFKA_NUM_CONTAINERS}); do
-  docker run -d -t --name knode${i} --network knw -v ${KAFKA_SRC}:/opt/kafka-dev ${KAFKA_IMAGE}
-done
-
-docker info
-docker ps
-docker network inspect knw
-
-for i in $(seq -w 1 ${KAFKA_NUM_CONTAINERS}); do
-  echo knode${i}
-  docker exec knode01 bash -c "ssh knode$i hostname"
-done
-
-bash tests/cluster_file_generator.sh > tests/cluster_file.json
-docker exec knode01 bash -c "cd /opt/kafka-dev; ducktape ${_DUCKTAPE_OPTIONS} --cluster-file tests/cluster_file.json ${TC_PATHS:-tests/kafkatest/tests}"

tests/docker/ssh-config

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Host *
+  ControlMaster auto
+  ControlPath ~/.ssh/master-%r@%h:%p
+  StrictHostKeyChecking no
+  ConnectTimeout=10
+  IdentityFile ~/.ssh/id_rsa

tests/setup.py

@@ -42,6 +42,7 @@ class PyTest(TestCommand):
         errno = pytest.main(self.pytest_args)
         sys.exit(errno)
 
+# Note: when changing the version of ducktape, also revise tests/docker/Dockerfile
 setup(name="kafkatest",
       version=version,
       description="Apache Kafka System Tests",