mirror of https://github.com/apache/kafka.git
MINOR: Added scripts to automate Vagrant setup for system tests
Updated testing README accordingly. Author: Geoff Anderson <geoff@confluent.io> Reviewers: Ewen Cheslack-Postava, Gwen Shapira Closes #201 from granders/minor-vagrant-package-script
This commit is contained in:
Geoff Anderson
Gwen Shapira
parent
d88b5bdfde
commit
7199c87c32
|
|
@ -14,51 +14,29 @@ https://cwiki.apache.org/confluence/display/KAFKA/tutorial+-+set+up+and+run+Kafk
|
|||
|
||||
* Install Virtual Box from [https://www.virtualbox.org/](https://www.virtualbox.org/) (run `$ vboxmanage --version` to check if it's installed).
|
||||
* Install Vagrant >= 1.6.4 from [http://www.vagrantup.com/](http://www.vagrantup.com/) (run `vagrant --version` to check if it's installed).
|
||||
* Install Vagrant Plugins:
|
||||
* Install system test dependiences, including ducktape, a command-line tool and library for testing distributed systems.
|
||||
|
||||
# Required
|
||||
# Note that vagrant-hostmanager v1.6.0 and up breaks our Vagrant scripts
|
||||
$ vagrant plugin install vagrant-hostmanager --plugin-version 1.5.0
|
||||
$ vagrant plugin install vagrant-cachier
|
||||
$ python tests/setup.py develop
|
||||
|
||||
* Build a specific branch of Kafka
|
||||
|
||||
$ cd kafka
|
||||
$ git checkout $BRANCH
|
||||
$ gradle
|
||||
$ ./gradlew jar
|
||||
|
||||
* Setup a testing cluster with Vagrant. Configure your Vagrant setup by creating the file
|
||||
`Vagrantfile.local` in the directory of your Kafka checkout. For testing purposes,
|
||||
`num_brokers` and `num_kafka` should be 0, and `num_workers` should be set high enough
|
||||
to run all of you tests. An example resides in kafka/vagrant/system-test-Vagrantfile.local
|
||||
* Run the bootstrap script to set up Vagrant for testing
|
||||
|
||||
# Example Vagrantfile.local for use on local machine
|
||||
# Vagrantfile.local should reside in the base Kafka directory
|
||||
num_zookeepers = 0
|
||||
num_kafka = 0
|
||||
num_workers = 9
|
||||
$ tests/bootstrap-test-env.sh
|
||||
|
||||
* Bring up the cluster (note that the initial provisioning process can be slow since it involves
|
||||
installing dependencies and updates on every vm.):
|
||||
* Bring up the test cluster
|
||||
|
||||
$ vagrant up
|
||||
|
||||
* Install ducktape:
|
||||
* Build the desired branch of Kafka
|
||||
|
||||
$ pip install ducktape
|
||||
|
||||
$ git checkout $BRANCH
|
||||
$ gradle # (only if necessary)
|
||||
$ ./gradlew jar
|
||||
|
||||
* Run the system tests using ducktape:
|
||||
|
||||
$ cd tests
|
||||
$ ducktape kafkatest/tests
|
||||
|
||||
* If you make changes to your Kafka checkout, you'll need to rebuild and resync to your Vagrant cluster:
|
||||
|
||||
$ cd kafka
|
||||
$ ./gradlew jar
|
||||
$ vagrant rsync # Re-syncs build output to cluster
|
||||
|
||||
EC2 Quickstart
|
||||
--------------
|
||||
This quickstart will help you run the Kafka system tests on EC2. In this setup, all logic is run
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script automates the process of setting up a local machine for running Kafka system tests
|
||||
|
||||
# Helper function which prints version numbers so they can be compared lexically or numerically
|
||||
function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d%03d\n", $1,$2,$3,$4); }'; }
|
||||
|
||||
base_dir=`dirname $0`/..
|
||||
cd $base_dir
|
||||
|
||||
echo "Checking Virtual Box installation..."
|
||||
bad_vb=false
|
||||
if [ -z `vboxmanage --version` ]; then
|
||||
echo "It appears that Virtual Box is not installed. Please install and try again (see https://www.virtualbox.org/ for details)"
|
||||
bad_vb=true
|
||||
else
|
||||
echo "Virtual Box looks good."
|
||||
fi
|
||||
|
||||
echo "Checking Vagrant installation..."
|
||||
vagrant_version=`vagrant --version | egrep -o "\d+\.\d+\.\d+"`
|
||||
bad_vagrant=false
|
||||
if [ "$(version $vagrant_version)" -lt "$(version 1.6.4)" ]; then
|
||||
echo "Found Vagrant version $vagrant_version. Please upgrade to 1.6.4 or higher (see http://www.vagrantup.com for details)"
|
||||
bad_vagrant=true
|
||||
else
|
||||
echo "Vagrant installation looks good."
|
||||
fi
|
||||
|
||||
if [ "x$bad_vagrant" == "xtrue" -o "x$bad_vb" == "xtrue" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Checking for necessary Vagrant plugins..."
|
||||
install_hostmanager=false
|
||||
hostmanager_version=`vagrant plugin list | grep vagrant-hostmanager | egrep -o "\d+\.\d+\.\d+"`
|
||||
if [ -z "$hostmanager_version" ]; then
|
||||
install_hostmanager=true
|
||||
elif [ "$hostmanager_version" != "1.5.0" ]; then
|
||||
echo "You have the wrong version of vagrant plugin vagrant-hostmanager. Uninstalling..."
|
||||
vagrant plugin uninstall vagrant-hostmanager
|
||||
install_hostmanager=true
|
||||
fi
|
||||
if [ "x$install_hostmanager" == "xtrue" ]; then
|
||||
vagrant plugin install vagrant-hostmanager --plugin-version 1.5.0
|
||||
fi
|
||||
|
||||
echo "Creating and packaging a reusable base box for Vagrant..."
|
||||
vagrant/package-base-box.sh
|
||||
|
||||
# Set up Vagrantfile.local if necessary
|
||||
if [ ! -e Vagrantfile.local ]; then
|
||||
echo "Creating Vagrantfile.local..."
|
||||
cp vagrant/system-test-Vagrantfile.local Vagrantfile.local
|
||||
else
|
||||
echo "Found an existing Vagrantfile.local. Keeping without overwriting..."
|
||||
fi
|
||||
|
||||
# Sanity check contents of Vagrantfile.local
|
||||
echo "Checking Vagrantfile.local..."
|
||||
vagrantfile_ok=true
|
||||
num_brokers=`egrep -o "num_brokers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs`
|
||||
num_zookeepers=`egrep -o "num_zookeepers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs`
|
||||
num_workers=`egrep -o "num_workers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs`
|
||||
if [ "x$num_brokers" == "x" -o "$num_brokers" != 0 ]; then
|
||||
echo "Vagrantfile.local: bad num_brokers. Update to: num_brokers = 0"
|
||||
vagrantfile_ok=false
|
||||
fi
|
||||
if [ "x$num_zookeepers" == "x" -o "$num_zookeepers" != 0 ]; then
|
||||
echo "Vagrantfile.local: bad num_zookeepers. Update to: num_zookeepers = 0"
|
||||
vagrantfile_ok=false
|
||||
fi
|
||||
if [ "x$num_workers" == "x" -o "$num_workers" == 0 ]; then
|
||||
echo "Vagrantfile.local: bad num_workers (size of test cluster). Set num_workers high enough to run your tests."
|
||||
vagrantfile_ok=false
|
||||
fi
|
||||
|
||||
if [ "$vagrantfile_ok" == "true" ]; then
|
||||
echo "Vagrantfile.local looks good."
|
||||
fi
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script automates the process of creating and packaging
|
||||
# a new vagrant base_box. For use locally (not aws).
|
||||
|
||||
base_dir=`dirname $0`/..
|
||||
cd $base_dir
|
||||
|
||||
backup_vagrantfile=backup_Vagrantfile.local
|
||||
local_vagrantfile=Vagrantfile.local
|
||||
|
||||
# Restore original Vagrantfile.local, if it exists
|
||||
function revert_vagrantfile {
|
||||
rm -f $local_vagrantfile
|
||||
if [ -e $backup_vagrantfile ]; then
|
||||
mv $backup_vagrantfile $local_vagrantfile
|
||||
fi
|
||||
}
|
||||
|
||||
function clean_up {
|
||||
echo "Cleaning up..."
|
||||
vagrant destroy -f
|
||||
rm -f package.box
|
||||
revert_vagrantfile
|
||||
}
|
||||
|
||||
# Name of the new base box
|
||||
base_box="kafkatest-worker"
|
||||
|
||||
# vagrant VM name
|
||||
worker_name="worker1"
|
||||
|
||||
echo "Destroying vagrant machines..."
|
||||
vagrant destroy -f
|
||||
|
||||
echo "Removing $base_box from vagrant..."
|
||||
vagrant box remove $base_box
|
||||
|
||||
echo "Bringing up a single vagrant machine from scratch..."
|
||||
if [ -e $local_vagrantfile ]; then
|
||||
mv $local_vagrantfile $backup_vagrantfile
|
||||
fi
|
||||
echo "num_workers = 1" > $local_vagrantfile
|
||||
echo "num_brokers = 0" >> $local_vagrantfile
|
||||
echo "num_zookeepers = 0" >> $local_vagrantfile
|
||||
vagrant up
|
||||
up_status=$?
|
||||
if [ $up_status != 0 ]; then
|
||||
echo "Failed to bring up a template vm, please try running again."
|
||||
clean_up
|
||||
exit $up_status
|
||||
fi
|
||||
|
||||
echo "Packaging $worker_name..."
|
||||
vagrant package $worker_name
|
||||
|
||||
echo "Adding new base box $base_box to vagrant..."
|
||||
vagrant box add $base_box package.box
|
||||
|
||||
clean_up
|
||||
|
||||
|
|
@ -20,3 +20,4 @@
|
|||
num_zookeepers = 0
|
||||
num_brokers = 0
|
||||
num_workers = 9
|
||||
base_box = "kafkatest-worker"
|
||||
|
|
|
|||
Loading…
Reference in New Issue