2017-01-09 08:14:16 +08:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
|
|
# this work for additional information regarding copyright ownership.
|
|
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
# (the "License"); you may not use this file except in compliance with
|
|
|
|
# the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2019-01-12 07:11:55 +08:00
|
|
|
ARG jdk_version=openjdk:8
|
|
|
|
FROM $jdk_version
|
2017-01-09 08:14:16 +08:00
|
|
|
|
|
|
|
MAINTAINER Apache Kafka dev@kafka.apache.org
|
2017-03-01 04:21:46 +08:00
|
|
|
VOLUME ["/opt/kafka-dev"]
|
2017-01-09 08:14:16 +08:00
|
|
|
|
2017-06-30 04:28:35 +08:00
|
|
|
# Set the timezone.
|
|
|
|
ENV TZ="/usr/share/zoneinfo/America/Los_Angeles"
|
|
|
|
|
|
|
|
# Do not ask for confirmations when running apt-get, etc.
|
|
|
|
ENV DEBIAN_FRONTEND noninteractive
|
|
|
|
|
|
|
|
# Set the ducker.creator label so that we know that this is a ducker image. This will make it
|
|
|
|
# visible to 'ducker purge'. The ducker.creator label also lets us know what UNIX user built this
|
|
|
|
# image.
|
|
|
|
ARG ducker_creator=default
|
|
|
|
LABEL ducker.creator=$ducker_creator
|
|
|
|
|
|
|
|
# Update Linux and install necessary utilities.
|
2020-09-15 06:22:00 +08:00
|
|
|
# we have to install git since it is included in openjdk:8 but not openjdk:11
|
2023-04-13 09:30:56 +08:00
|
|
|
RUN apt update && apt install -y sudo git netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python3-pip python3-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute iproute2 iputils-ping && apt-get -y clean
|
MINOR: upgrade pip from 20.2.2 to 21.1.1 (#10661)
The following error happens on my mac m1 when building docker image for system tests.
Collecting pynacl
Using cached PyNaCl-1.4.0.tar.gz (3.4 MB)
Installing build dependencies ... error
ERROR: Command errored out with exit status 1:
command: /usr/bin/python3 /usr/local/lib/python3.8/dist-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-k867aac0/overlay --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- 'setuptools>=40.8.0' wheel 'cffi>=1.4.1; python_implementation != '"'"'PyPy'"'"''
cwd: None
Complete output (14 lines):
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.8/dist-packages/pip/__main__.py", line 23, in <module>
from pip._internal.cli.main import main as _main # isort:skip # noqa
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/cli/main.py", line 5, in <module>
import locale
File "/usr/lib/python3.8/locale.py", line 16, in <module>
import re
File "/usr/lib/python3.8/re.py", line 145, in <module>
class RegexFlag(enum.IntFlag):
AttributeError: module 'enum' has no attribute 'IntFlag'
----------------------------------------
ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/local/lib/python3.8/dist-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-k867aac0/overlay --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- 'setuptools>=40.8.0' wheel 'cffi>=1.4.1; python_implementation != '"'"'PyPy'"'"'' Check the logs for full command output.
There was a related issue: pypa/pip#9689 and it is already fixed by pypa/pip#9689 (included by pip 21.1.1). I test the pip 21.1.1 and it works well on mac m1.
Reviewers: Ismael Juma <ismael@juma.me.uk>
2021-05-29 14:49:25 +08:00
|
|
|
RUN python3 -m pip install -U pip==21.1.1;
|
2022-03-11 20:18:19 +08:00
|
|
|
RUN pip3 install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 debugpy && pip3 install --upgrade "ducktape>0.8"
|
2017-06-30 04:28:35 +08:00
|
|
|
|
|
|
|
# Set up ssh
|
|
|
|
COPY ./ssh-config /root/.ssh/config
|
2019-01-12 07:11:55 +08:00
|
|
|
# NOTE: The paramiko library supports the PEM-format private key, but does not support the RFC4716 format.
|
|
|
|
RUN ssh-keygen -m PEM -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
|
2019-06-08 05:23:49 +08:00
|
|
|
RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config
|
2017-06-30 04:28:35 +08:00
|
|
|
|
|
|
|
# Install binary test dependencies.
|
2018-04-07 08:00:52 +08:00
|
|
|
# we use the same versions as in vagrant/base.sh
|
2017-11-21 22:39:11 +08:00
|
|
|
ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
|
2018-04-07 08:00:52 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-0.8.2.2" && chmod a+rw /opt/kafka-0.8.2.2 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.8.2.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.8.2.2"
|
2018-03-16 05:42:43 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-0.9.0.1" && chmod a+rw /opt/kafka-0.9.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.9.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.9.0.1"
|
|
|
|
RUN mkdir -p "/opt/kafka-0.10.0.1" && chmod a+rw /opt/kafka-0.10.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.0.1"
|
|
|
|
RUN mkdir -p "/opt/kafka-0.10.1.1" && chmod a+rw /opt/kafka-0.10.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.1.1"
|
MINOR: Enable ignored upgrade system tests - trunk (#5605)
Removed ignore annotations from the upgrade tests. This PR includes the following changes for updating the upgrade tests:
* Uploaded new versions 0.10.2.2, 0.11.0.3, 1.0.2, 1.1.1, and 2.0.0 (in the associated scala versions) to kafka-packages
* Update versions in version.py, Dockerfile, base.sh
* Added new versions to StreamsUpgradeTest.test_upgrade_downgrade_brokers including version 2.0.0
* Added new versions StreamsUpgradeTest.test_simple_upgrade_downgrade test excluding version 2.0.0
* Version 2.0.0 is excluded from the streams upgrade/downgrade test as StreamsConfig needs an update for the new version, requiring a KIP. Once the community votes the KIP in, a minor follow-up PR can be pushed to add the 2.0.0 version to the upgrade test.
* Fixed minor bug in kafka-run-class.sh for classpath in upgrade/downgrade tests across versions.
* Follow on PRs for 0.10.2x, 0.11.0x, 1.0.x, 1.1.x, and 2.0.x will be pushed soon with the same updates required for the specific version.
Reviewers: Eno Thereska <eno.thereska@gmail.com>, John Roesler <vvcephei@users.noreply.github.com>, Guozhang Wang <wangguoz@gmail.com>, Matthias J. Sax <matthias@confluent.io>
2018-09-14 04:46:47 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-0.10.2.2" && chmod a+rw /opt/kafka-0.10.2.2 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.2.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.2.2"
|
|
|
|
RUN mkdir -p "/opt/kafka-0.11.0.3" && chmod a+rw /opt/kafka-0.11.0.3 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.11.0.3.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.11.0.3"
|
|
|
|
RUN mkdir -p "/opt/kafka-1.0.2" && chmod a+rw /opt/kafka-1.0.2 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.0.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.0.2"
|
|
|
|
RUN mkdir -p "/opt/kafka-1.1.1" && chmod a+rw /opt/kafka-1.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.1.1"
|
2019-01-07 15:03:54 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-2.0.1" && chmod a+rw /opt/kafka-2.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.0.1"
|
2019-05-24 04:34:00 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-2.1.1" && chmod a+rw /opt/kafka-2.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.1.1"
|
2019-12-07 05:44:56 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-2.2.2" && chmod a+rw /opt/kafka-2.2.2 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.2.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.2.2"
|
|
|
|
RUN mkdir -p "/opt/kafka-2.3.1" && chmod a+rw /opt/kafka-2.3.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.3.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.3.1"
|
2020-03-31 00:55:35 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-2.4.1" && chmod a+rw /opt/kafka-2.4.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.4.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.4.1"
|
2020-08-12 04:18:33 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-2.5.1" && chmod a+rw /opt/kafka-2.5.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.5.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.5.1"
|
2023-10-06 01:23:23 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-2.6.3" && chmod a+rw /opt/kafka-2.6.3 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.6.3.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.6.3"
|
|
|
|
RUN mkdir -p "/opt/kafka-2.7.2" && chmod a+rw /opt/kafka-2.7.2 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.7.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.7.2"
|
2022-09-11 23:20:07 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-2.8.2" && chmod a+rw /opt/kafka-2.8.2 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.8.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.8.2"
|
2022-09-19 19:43:40 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-3.0.2" && chmod a+rw /opt/kafka-3.0.2 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.0.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.0.2"
|
|
|
|
RUN mkdir -p "/opt/kafka-3.1.2" && chmod a+rw /opt/kafka-3.1.2 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.1.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.1.2"
|
|
|
|
RUN mkdir -p "/opt/kafka-3.2.3" && chmod a+rw /opt/kafka-3.2.3 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.2.3.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.2.3"
|
2023-10-06 01:23:23 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-3.3.2" && chmod a+rw /opt/kafka-3.3.2 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.3.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.3.2"
|
2023-07-12 16:11:44 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-3.4.1" && chmod a+rw /opt/kafka-3.4.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.4.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.4.1"
|
2023-07-27 18:33:34 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-3.5.1" && chmod a+rw /opt/kafka-3.5.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.5.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.5.1"
|
2023-10-08 07:59:06 +08:00
|
|
|
RUN mkdir -p "/opt/kafka-3.6.0" && chmod a+rw /opt/kafka-3.6.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.6.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.6.0"
|
2023-07-12 16:11:44 +08:00
|
|
|
|
2018-03-16 05:42:43 +08:00
|
|
|
|
|
|
|
# Streams test dependencies
|
2018-04-07 08:00:52 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.0.1-test.jar" -o /opt/kafka-0.10.0.1/libs/kafka-streams-0.10.0.1-test.jar
|
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.1.1-test.jar" -o /opt/kafka-0.10.1.1/libs/kafka-streams-0.10.1.1-test.jar
|
MINOR: Enable ignored upgrade system tests - trunk (#5605)
Removed ignore annotations from the upgrade tests. This PR includes the following changes for updating the upgrade tests:
* Uploaded new versions 0.10.2.2, 0.11.0.3, 1.0.2, 1.1.1, and 2.0.0 (in the associated scala versions) to kafka-packages
* Update versions in version.py, Dockerfile, base.sh
* Added new versions to StreamsUpgradeTest.test_upgrade_downgrade_brokers including version 2.0.0
* Added new versions StreamsUpgradeTest.test_simple_upgrade_downgrade test excluding version 2.0.0
* Version 2.0.0 is excluded from the streams upgrade/downgrade test as StreamsConfig needs an update for the new version, requiring a KIP. Once the community votes the KIP in, a minor follow-up PR can be pushed to add the 2.0.0 version to the upgrade test.
* Fixed minor bug in kafka-run-class.sh for classpath in upgrade/downgrade tests across versions.
* Follow on PRs for 0.10.2x, 0.11.0x, 1.0.x, 1.1.x, and 2.0.x will be pushed soon with the same updates required for the specific version.
Reviewers: Eno Thereska <eno.thereska@gmail.com>, John Roesler <vvcephei@users.noreply.github.com>, Guozhang Wang <wangguoz@gmail.com>, Matthias J. Sax <matthias@confluent.io>
2018-09-14 04:46:47 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.2.2-test.jar" -o /opt/kafka-0.10.2.2/libs/kafka-streams-0.10.2.2-test.jar
|
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.11.0.3-test.jar" -o /opt/kafka-0.11.0.3/libs/kafka-streams-0.11.0.3-test.jar
|
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.0.2-test.jar" -o /opt/kafka-1.0.2/libs/kafka-streams-1.0.2-test.jar
|
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.1.1-test.jar" -o /opt/kafka-1.1.1/libs/kafka-streams-1.1.1-test.jar
|
2019-01-07 15:03:54 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.0.1-test.jar" -o /opt/kafka-2.0.1/libs/kafka-streams-2.0.1-test.jar
|
2019-05-31 03:50:30 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.1.1-test.jar" -o /opt/kafka-2.1.1/libs/kafka-streams-2.1.1-test.jar
|
2019-12-07 05:44:56 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.2.2-test.jar" -o /opt/kafka-2.2.2/libs/kafka-streams-2.2.2-test.jar
|
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.3.1-test.jar" -o /opt/kafka-2.3.1/libs/kafka-streams-2.3.1-test.jar
|
2020-04-02 08:56:47 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.4.1-test.jar" -o /opt/kafka-2.4.1/libs/kafka-streams-2.4.1-test.jar
|
2020-08-12 04:18:33 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.5.1-test.jar" -o /opt/kafka-2.5.1/libs/kafka-streams-2.5.1-test.jar
|
2023-10-06 01:23:23 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.6.3-test.jar" -o /opt/kafka-2.6.3/libs/kafka-streams-2.6.3-test.jar
|
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.7.2-test.jar" -o /opt/kafka-2.7.2/libs/kafka-streams-2.7.2-test.jar
|
2022-09-11 23:20:07 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.8.2-test.jar" -o /opt/kafka-2.8.2/libs/kafka-streams-2.8.2-test.jar
|
2022-09-19 19:43:40 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.0.2-test.jar" -o /opt/kafka-3.0.2/libs/kafka-streams-3.0.2-test.jar
|
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.1.2-test.jar" -o /opt/kafka-3.1.2/libs/kafka-streams-3.1.2-test.jar
|
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.2.3-test.jar" -o /opt/kafka-3.2.3/libs/kafka-streams-3.2.3-test.jar
|
2023-10-06 01:23:23 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.3.2-test.jar" -o /opt/kafka-3.3.2/libs/kafka-streams-3.3.2-test.jar
|
2023-07-12 16:11:44 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.4.1-test.jar" -o /opt/kafka-3.4.1/libs/kafka-streams-3.4.1-test.jar
|
2023-07-27 18:33:34 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.5.1-test.jar" -o /opt/kafka-3.5.1/libs/kafka-streams-3.5.1-test.jar
|
2023-10-08 07:59:06 +08:00
|
|
|
RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.6.0-test.jar" -o /opt/kafka-3.6.0/libs/kafka-streams-3.6.0-test.jar
|
2017-11-21 22:39:11 +08:00
|
|
|
|
|
|
|
# The version of Kibosh to use for testing.
|
2019-11-16 07:13:32 +08:00
|
|
|
# If you update this, also update vagrant/base.sh
|
|
|
|
ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a"
|
2017-01-09 08:14:16 +08:00
|
|
|
|
2020-12-03 10:39:58 +08:00
|
|
|
# Aligning uid inside/outside docker enables containers to modify files of kafka source (mounted in /opt/kafka-dev")
|
|
|
|
# By default, the outside user id is 1000 (UID_MIN). The known exception in QA is travis which gives non-1000 id.
|
|
|
|
ARG UID="1000"
|
|
|
|
|
2017-11-17 01:59:24 +08:00
|
|
|
# Install Kibosh
|
|
|
|
RUN apt-get install fuse
|
2017-11-21 22:39:11 +08:00
|
|
|
RUN cd /opt && git clone -q https://github.com/confluentinc/kibosh.git && cd "/opt/kibosh" && git reset --hard $KIBOSH_VERSION && mkdir "/opt/kibosh/build" && cd "/opt/kibosh/build" && ../configure && make -j 2
|
2017-11-17 01:59:24 +08:00
|
|
|
|
2017-06-30 04:28:35 +08:00
|
|
|
# Set up the ducker user.
|
2020-12-03 10:39:58 +08:00
|
|
|
RUN useradd -u $UID -ms /bin/bash ducker \
|
|
|
|
&& mkdir -p /home/ducker/ \
|
|
|
|
&& rsync -aiq /root/.ssh/ /home/ducker/.ssh \
|
|
|
|
&& chown -R ducker /home/ducker/ /mnt/ /var/log/ \
|
|
|
|
&& echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >> /home/ducker/.ssh/environment \
|
|
|
|
&& echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile \
|
|
|
|
&& echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
|
|
|
|
|
2017-06-30 04:28:35 +08:00
|
|
|
USER ducker
|
|
|
|
|
|
|
|
CMD sudo service ssh start && tail -f /dev/null
|