kafka/vagrant/base.sh

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -ex

# The version of Kibosh to use for testing.
# If you update this, also update tests/docker/Dockerfile
export KIBOSH_VERSION=8841dd392e6fbf02986e2fb1f1ebf04df344b65a

path_to_jdk_cache() {
  jdk_version=$1
  echo "/tmp/jdk-${jdk_version}.tar.gz"
}

fetch_jdk_tgz() {
  jdk_version=$1

  path=$(path_to_jdk_cache $jdk_version)

  if [ ! -e $path ]; then
    mkdir -p $(dirname $path)
    curl --retry 5 -s -L "https://s3-us-west-2.amazonaws.com/kafka-packages/jdk-${jdk_version}.tar.gz" -o $path
  fi
}

JDK_MAJOR="${JDK_MAJOR:-17}"
JDK_FULL="${JDK_FULL:-17-linux-x64}"

if [ -z `which javac` ]; then
    apt-get -y update
    apt-get install -y software-properties-common python-software-properties binutils java-common

    echo "===> Installing JDK..." 

    mkdir -p /opt/jdk
    cd /opt/jdk
    rm -rf $JDK_MAJOR
    mkdir -p $JDK_MAJOR
    cd $JDK_MAJOR
    fetch_jdk_tgz $JDK_FULL
    tar x --strip-components=1 -zf $(path_to_jdk_cache $JDK_FULL)
    for bin in /opt/jdk/$JDK_MAJOR/bin/* ; do 
      name=$(basename $bin)
      update-alternatives --install /usr/bin/$name $name $bin 1081 && update-alternatives --set $name $bin
    done
    echo -e "export JAVA_HOME=/opt/jdk/$JDK_MAJOR\nexport PATH=\$PATH:\$JAVA_HOME/bin" > /etc/profile.d/jdk.sh
    echo "JDK installed: $(javac -version 2>&1)"

fi

chmod a+rw /opt
if [ -h /opt/kafka-dev ]; then
    # reset symlink
    rm /opt/kafka-dev
fi
ln -s /vagrant /opt/kafka-dev


get_kafka() {
    version=$1
    scala_version=$2

    kafka_dir=/opt/kafka-$version
    url=https://s3-us-west-2.amazonaws.com/kafka-packages/kafka_$scala_version-$version.tgz
    # the .tgz above does not include the streams test jar hence we need to get it separately
    url_streams_test=https://s3-us-west-2.amazonaws.com/kafka-packages/kafka-streams-$version-test.jar
    if [ ! -d /opt/kafka-$version ]; then
        pushd /tmp
        curl --retry 5 -O $url
        curl --retry 5 -O $url_streams_test || true
        file_tgz=`basename $url`
        file_streams_jar=`basename $url_streams_test` || true
        tar -xzf $file_tgz
        rm -rf $file_tgz

        file=`basename $file_tgz .tgz`
        mv $file $kafka_dir
        mv $file_streams_jar $kafka_dir/libs || true
        popd
    fi
}

# Install Kibosh
apt-get update -y && apt-get install -y git cmake pkg-config libfuse-dev
pushd /opt
rm -rf /opt/kibosh
git clone -q  https://github.com/confluentinc/kibosh.git
pushd "/opt/kibosh"
git reset --hard $KIBOSH_VERSION
mkdir "/opt/kibosh/build"
pushd "/opt/kibosh/build"
../configure && make -j 2
popd
popd
popd

# Install iperf
apt-get install -y iperf traceroute

# Test multiple Kafka versions
# We want to use the latest Scala version per Kafka version
# Previously we could not pull in Scala 2.12 builds, because Scala 2.12 requires Java 8 and we were running the system
# tests with Java 7. We have since switched to Java 8, so 2.0.0 and later use Scala 2.12.
get_kafka 2.1.1 2.12
chmod a+rw /opt/kafka-2.1.1
get_kafka 2.2.2 2.12
chmod a+rw /opt/kafka-2.2.2
get_kafka 2.3.1 2.12
chmod a+rw /opt/kafka-2.3.1
get_kafka 2.4.1 2.12
chmod a+rw /opt/kafka-2.4.1
get_kafka 2.5.1 2.12
chmod a+rw /opt/kafka-2.5.1
get_kafka 2.6.3 2.12
chmod a+rw /opt/kafka-2.6.3
get_kafka 2.7.2 2.12
chmod a+rw /opt/kafka-2.7.2
get_kafka 2.8.2 2.12
chmod a+rw /opt/kafka-2.8.2
get_kafka 3.0.2 2.12
chmod a+rw /opt/kafka-3.0.2
get_kafka 3.1.2 2.12
chmod a+rw /opt/kafka-3.1.2
get_kafka 3.2.3 2.12
chmod a+rw /opt/kafka-3.2.3
get_kafka 3.3.2 2.12
chmod a+rw /opt/kafka-3.3.2
get_kafka 3.4.1 2.12
chmod a+rw /opt/kafka-3.4.1
get_kafka 3.5.2 2.12
chmod a+rw /opt/kafka-3.5.2
get_kafka 3.6.2 2.12
chmod a+rw /opt/kafka-3.6.2
get_kafka 3.7.2 2.12
chmod a+rw /opt/kafka-3.7.2
get_kafka 3.8.1 2.12
chmod a+rw /opt/kafka-3.8.1
get_kafka 3.9.1 2.12
chmod a+rw /opt/kafka-3.9.1
get_kafka 4.0.0 2.13
chmod a+rw /opt/kafka-4.0.0

# To ensure the Kafka cluster starts successfully under JDK 17, we need to update the Zookeeper
# client from version 3.4.x to 3.5.7 in Kafka versions 2.1.1, 2.2.2, and 2.3.1, as the older Zookeeper
# client is incompatible with JDK 17. See KAFKA-17888 for more details.
curl -s "https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar" -o /opt/zookeeper-3.5.7.jar
curl -s "https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper-jute/3.5.7/zookeeper-jute-3.5.7.jar" -o /opt/zookeeper-jute-3.5.7.jar
rm -f /opt/kafka-2.1.1/libs/zookeeper-*
rm -f /opt/kafka-2.2.2/libs/zookeeper-*
rm -f /opt/kafka-2.3.1/libs/zookeeper-*

cp /opt/zookeeper-3.5.7.jar /opt/kafka-2.1.1/libs/zookeeper-3.5.7.jar
chmod a+rw /opt/kafka-2.1.1/libs/zookeeper-3.5.7.jar
cp /opt/zookeeper-3.5.7.jar /opt/kafka-2.2.2/libs/zookeeper-3.5.7.jar
chmod a+rw /opt/kafka-2.2.2/libs/zookeeper-3.5.7.jar
cp /opt/zookeeper-3.5.7.jar /opt/kafka-2.3.1/libs/zookeeper-3.5.7.jar
chmod a+rw /opt/kafka-2.3.1/libs/zookeeper-3.5.7.jar
cp /opt/zookeeper-jute-3.5.7.jar /opt/kafka-2.1.1/libs/zookeeper-jute-3.5.7.jar
chmod a+rw /opt/kafka-2.1.1/libs/zookeeper-jute-3.5.7.jar
cp /opt/zookeeper-jute-3.5.7.jar /opt/kafka-2.2.2/libs/zookeeper-jute-3.5.7.jar
chmod a+rw /opt/kafka-2.2.2/libs/zookeeper-jute-3.5.7.jar
cp /opt/zookeeper-jute-3.5.7.jar /opt/kafka-2.3.1/libs/zookeeper-jute-3.5.7.jar
chmod a+rw /opt/kafka-2.3.1/libs/zookeeper-jute-3.5.7.jar

# For EC2 nodes, we want to use /mnt, which should have the local disk. On local
# VMs, we can just create it if it doesn't exist and use it like we'd use
# /tmp. Eventually, we'd like to also support more directories, e.g. when EC2
# instances have multiple local disks.
if [ ! -e /mnt ]; then
    mkdir /mnt
fi
chmod a+rwx /mnt

# Run ntpdate once to sync to ntp servers
# use -u option to avoid port collision in case ntp daemon is already running
ntpdate -u pool.ntp.org
# Install ntp daemon - it will automatically start on boot
apt-get -y install ntp

# Increase the ulimit
mkdir -p /etc/security/limits.d
echo "* soft nofile 128000" >> /etc/security/limits.d/nofile.conf
echo "* hard nofile 128000" >> /etc/security/limits.d/nofile.conf

ulimit -Hn 128000
ulimit -Sn 128000