KAFKA-13041: Enable connecting VS Code remote debugger (#10915)

The changes in this PR enable connecting VS Code's remote debugger to a system test running locally with ducker-ak.
Changes include:
- added zip_safe=False to setup.py - this enables installing kafkatest module together with source code when running `python setup.py  develop/install`.
- install [debugpy](https://github.com/microsoft/debugpy) on ducker nodes
- expose 5678 (default debugpy port) on ducker01 node - ducker01 is the one that actually executes tests, so that's where you'd connect to.
- added `-d|--debug` option to `ducker-ak test` command - if used, tests will run via `python3.7 -m debugpy` command, which would listen on 5678 and pause until debugger is connected.
- changed the logic of the `ducker-ak test` command so that ducktape args are collected separately after `--` - otherwise any argument we add to the `test` command in the future might potentially
shadow a similar ducktape argument. 
	- we don't really check that `ducktape_args` are args while `test_name_args` are actual test names, so the difference between the two is minimal actually - most importantly we do check that `test_name_args` is not empty, but we are ok if `ducktape_args` is.

Reviewers: Ewen Cheslack-Postava <ewen@confluent.io>, Manikumar Reddy <manikumar.reddy@gmail.com>

tests/README.md

@@ -51,6 +51,40 @@ bash tests/docker/ducker-ak up -j 'openjdk:11'; tests/docker/run_tests.sh
 ```
 REBUILD="t" bash tests/docker/run_tests.sh
 ```
+* Debug tests in VS Code:
+  - Run test with `--debug` flag (can be before or after file name):
+    ```
+    tests/docker/ducker-ak up; tests/docker/ducker-ak test tests/kafkatest/tests/core/security_test.py --debug
+    ```
+  - Test will run in debug mode and wait for a debugger to attach. 
+  - Launch VS Code debugger with `"attach"` request - here's an example:
+    ```json
+    {
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Attach to Ducker",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 5678
+            },
+            "justMyCode": false,
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "."
+                }
+            ]
+        }
+      ]
+    }
+    ```
+  - To pass `--debug` flag to ducktape itself, use `--`:
+    ```
+    tests/docker/ducker-ak test tests/kafkatest/tests/core/security_test.py --debug -- --debug
+    ```
 
 * Notes
   - The scripts to run tests creates and destroys docker network named *knw*.

tests/docker/Dockerfile

@@ -35,7 +35,7 @@ LABEL ducker.creator=$ducker_creator
 # we have to install git since it is included in openjdk:8 but not openjdk:11
 RUN apt update && apt install -y sudo git netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python3-pip python3-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute && apt-get -y clean
 RUN python3 -m pip install -U pip==21.1.1;
-RUN pip3 install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip3 install --upgrade ducktape==0.8.1
+RUN pip3 install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 debugpy && pip3 install --upgrade ducktape==0.8.1
 
 # Set up ssh
 COPY ./ssh-config /root/.ssh/config

tests/docker/ducker-ak

@@ -47,6 +47,9 @@ default_jdk="openjdk:8"
 # The default ducker-ak image name.
 default_image_name="ducker-ak"
 
+# Port to listen on when debugging
+debugpy_port=5678
+
 # Display a usage message on the terminal and exit.
 #
 # $1: The exit status to use
@@ -75,11 +78,20 @@ up [-n|--num-nodes NUM_NODES] [-f|--force] [docker-image]
     or a combination of port/port-range separated by comma (like 2181,9092 or 2181,5005-5008).
     By default no port is exposed. See README.md for more detail on this option.
     
-test [test-name(s)]
+    Note that port 5678 will be automatically exposed for ducker01 node and will be mapped to 5678 
+    on your local machine to enable debugging in VS Code.
+
+test [-d|--debug] [test-name(s)] [-- [ducktape args]]
     Run a test or set of tests inside the currently active Ducker nodes.
     For example, to run the system test produce_bench_test, you would run:
         ./tests/docker/ducker-ak test ./tests/kafkatest/tests/core/produce_bench_test.py
     
+    If --debug is passed, the tests will wait for remote VS Code debugger to connect on port 5678:
+        ./tests/docker/ducker-ak test --debug ./tests/kafkatest/tests/core/produce_bench_test.py
+    
+    To pass arguments to underlying ducktape invocation, pass them after `--`, e.g.:
+        ./tests/docker/ducker-ak test ./tests/kafkatest/tests/core/produce_bench_test.py -- --test-runner-timeout 1800000
+
 ssh [node-name|user-name@node-name] [command]
     Log in to a running ducker container.  If node-name is not given, it prints
     the names of all running nodes.  If node-name is 'all', we will run the
@@ -246,6 +258,7 @@ docker_run() {
     local node=${1}
     local image_name=${2}
     local ports_option=${3}
+    local port_mapping=${4}
 
     local expose_ports=""
     if [[ -n ${ports_option} ]]; then
@@ -254,6 +267,9 @@ docker_run() {
             expose_ports="${expose_ports} --expose ${expose_port}"
         done
     fi
+    if [[ -n ${port_mapping} ]]; then
+        expose_ports="${expose_ports} -p ${port_mapping}:${port_mapping}"
+    fi
 
     # Invoke docker-run. We need privileged mode to be able to run iptables
     # and mount FUSE filesystems inside the container.  We also need it to
@@ -340,7 +356,8 @@ attempting to start new ones."
     if [[ -n "${custom_ducktape}" ]]; then
         setup_custom_ducktape "${custom_ducktape}" "${image_name}"
     fi
-    for n in $(seq -f %02g 1 ${num_nodes}); do
+    docker_run ducker01 "${image_name}" "${expose_ports}" "${debugpy_port}"
+    for n in $(seq -f %02g 2 ${num_nodes}); do
         local node="ducker${n}"
         docker_run "${node}" "${image_name}" "${expose_ports}"
     done
@@ -425,23 +442,41 @@ ducker_test() {
     require_commands docker
     docker inspect ducker01 &>/dev/null || \
         die "ducker_test: the ducker01 instance appears to be down. Did you run 'ducker up'?"
-    [[ $# -lt 1 ]] && \
+    declare -a test_name_args=()
+    local debug=0
+    while [[ $# -ge 1 ]]; do
+        case "${1}" in
+            -d|--debug) debug=1; shift;;
+            --) shift; break;;
+            *) test_name_args+=("${1}"); shift;;
+        esac
+    done
+    local ducktape_args=${*}
+    
+    [[ ${#test_name_args} -lt 1 ]] && \
         die "ducker_test: you must supply at least one system test to run. Type --help for help."
-    local args=""
-    local kafka_test=0
-    for arg in "${@}"; do
+    local test_names=""
+
+    for test_name in ${test_name_args[*]}; do
         local regex=".*\/kafkatest\/(.*)"
-        if [[ $arg =~ $regex ]]; then
+        if [[ $test_name =~ $regex ]]; then
             local kpath=${BASH_REMATCH[1]}
-            args="${args} ./tests/kafkatest/${kpath}"
+            test_names="${test_names} ./tests/kafkatest/${kpath}"
         else
-            args="${args} ${arg}"
+            test_names="${test_names} ${test_name}"
         fi
     done
+    
     must_pushd "${kafka_dir}"
     (test -f ./gradlew || gradle) && ./gradlew systemTestLibs
     must_popd
-    cmd="cd /opt/kafka-dev && ducktape --cluster-file /opt/kafka-dev/tests/docker/build/cluster.json $args"
+    if [[ "${debug}" -eq 1 ]]; then
+        local ducktape_cmd="python3.7 -m debugpy --listen 0.0.0.0:${debugpy_port} --wait-for-client /usr/local/bin/ducktape"
+    else
+        local ducktape_cmd="ducktape"
+    fi
+
+    cmd="cd /opt/kafka-dev && ${ducktape_cmd} --cluster-file /opt/kafka-dev/tests/docker/build/cluster.json $test_names $ducktape_args"
     echo "docker exec ducker01 bash -c \"${cmd}\""
     exec docker exec --user=ducker ducker01 bash -c "${cmd}"
 }

tests/docker/run_tests.sh

@@ -32,4 +32,7 @@ fi
 if ${SCRIPT_DIR}/ducker-ak ssh | grep -q '(none)'; then
     ${SCRIPT_DIR}/ducker-ak up -n "${KAFKA_NUM_CONTAINERS}" || die "ducker-ak up failed"
 fi
+
+[[ -n ${_DUCKTAPE_OPTIONS} ]] && _DUCKTAPE_OPTIONS="-- ${_DUCKTAPE_OPTIONS}"
+
 ${SCRIPT_DIR}/ducker-ak test ${TC_PATHS} ${_DUCKTAPE_OPTIONS} || die "ducker-ak test failed"

tests/setup.py

@@ -51,7 +51,8 @@ setup(name="kafkatest",
       license="apache2.0",
       packages=find_packages(),
       include_package_data=True,
-      install_requires=["ducktape==0.8.1", "requests==2.24.0"],
+      install_requires=["ducktape==0.8.8", "requests==2.24.0"],
       tests_require=["pytest", "mock"],
       cmdclass={'test': PyTest},
+      zip_safe=False
       )