rabbit_ct_vm_helpers, tools/terraform: Support Erlang built from sources

... instead of installing Debian packages. To use this feature, the
caller has to specify an Erlang Git reference (a branch, a tag or a
plain commit hash).

This is useful now to test with Erlang 21.0 which is not released yet at
the time of this commit. In the case of Erlang 21.0, if no Git reference
is specified, it defaults to `OTP-21.0-rc2`.

The time we wait for VMs to be ready is bumped from 7 to 20 minutes
because compiling Erlang and Elixir takes time.

rabbit_ct_vm_helpers takes a look at the return value of
`erlang:system_info(system_version)` because it may contain the commit
hash of the running Erlang system (if it was built from sources). It
allows to use the same commit on the remote VMs.

[#157964874]

deps/rabbitmq_ct_helpers/src/rabbit_ct_vm_helpers.erl

@@ -31,6 +31,7 @@
 
          ensure_terraform_cmd/1,
          determine_erlang_version/1,
+         determine_erlang_git_ref/1,
          compute_code_path/1,
          find_terraform_ssh_key/1,
          set_terraform_files_suffix/1,
@@ -69,6 +70,7 @@ setup_steps() ->
     [
      fun ensure_terraform_cmd/1,
      fun determine_erlang_version/1,
+     fun determine_erlang_git_ref/1,
      fun compute_code_path/1,
      fun find_terraform_ssh_key/1,
      fun set_terraform_files_suffix/1,
@@ -133,6 +135,30 @@ determine_erlang_version(Config) ->
     rabbit_ct_helpers:set_config(
       Config, {erlang_version, ErlangVersion}).
 
+determine_erlang_git_ref(Config) ->
+    GitRef = case rabbit_ct_helpers:get_config(Config, erlang_git_ref) of
+                  undefined ->
+                      case os:getenv("ERLANG_GIT_REF") of
+                          false ->
+                              Version = erlang:system_info(system_version),
+                              ReOpts = [{capture, all_but_first, list}],
+                              Match = re:run(Version,
+                                             "source-([0-9a-fA-F]+)",
+                                             ReOpts),
+                              case Match of
+                                  {match, [V]} -> V;
+                                  _            -> ""
+                              end;
+                          V ->
+                              V
+                      end;
+                  V ->
+                      V
+              end,
+    ct:pal(?LOW_IMPORTANCE, "Erlang Git reference: ~s", [GitRef]),
+    rabbit_ct_helpers:set_config(
+      Config, {erlang_git_ref, GitRef}).
+
 compute_code_path(Config) ->
     EntireCodePath = code:get_path(),
     CodePath = rabbit_ct_broker_helpers:filter_out_erlang_code_path(
@@ -453,6 +479,7 @@ destroy_terraform_vms(Config) ->
 
 terraform_var_flags(Config) ->
     ErlangVersion = ?config(erlang_version, Config),
+    GitRef = ?config(erlang_git_ref, Config),
     SshKey = ?config(terraform_ssh_key, Config),
     Suffix = ?config(terraform_files_suffix, Config),
     EC2Region = ?config(terraform_aws_ec2_region, Config),
@@ -474,6 +501,7 @@ terraform_var_flags(Config) ->
     Archive = ?config(upload_dirs_archive, Config),
     [
      {"-var=erlang_version=~s", [ErlangVersion]},
+     {"-var=erlang_git_ref=~s", [GitRef]},
      {"-var=erlang_cookie=~s", [erlang:get_cookie()]},
      {"-var=erlang_nodename=~s", [?ERLANG_REMOTE_NODENAME]},
      {"-var=ssh_key=~s", [SshKey]},
@@ -724,7 +752,7 @@ write_inetrc(Config) ->
 
 wait_for_ct_peers(Config) ->
     CTPeers = get_ct_peers(Config),
-    Timeout = 7 * 60 * 1000,
+    Timeout = 20 * 60 * 1000,
     {ok, TRef} = timer:send_after(Timeout, ct_peers_timeout),
     wait_for_ct_peers(Config, CTPeers, TRef).
 

deps/rabbitmq_ct_helpers/tools/terraform/direct-vms/main.tf

@@ -171,6 +171,7 @@ data "template_file" "user_data" {
     erlang_cookie    = "${var.erlang_cookie}"
     erlang_nodename  = "${var.erlang_nodename}"
     erlang_version   = "${var.erlang_version}"
+    erlang_git_ref   = "${var.erlang_git_ref}"
   }
 }
 

deps/rabbitmq_ct_helpers/tools/terraform/direct-vms/setup-vms.sh

@@ -118,6 +118,7 @@ start_vms() {
   terraform apply \
     -auto-approve=true \
     -var="erlang_version=$erlang_branch" \
+    -var="erlang_git_ref=$erlang_git_ref" \
     -var="erlang_cookie=$erlang_cookie" \
     -var="erlang_nodename=$erlang_nodename" \
     -var="ssh_key=$ssh_key" \
@@ -131,6 +132,7 @@ destroy_vms() {
   terraform destroy \
     -force \
     -var="erlang_version=$erlang_branch" \
+    -var="erlang_git_ref=$erlang_git_ref" \
     -var="erlang_cookie=$erlang_cookie" \
     -var="erlang_nodename=$erlang_nodename" \
     -var="ssh_key=$ssh_key" \
@@ -140,6 +142,13 @@ destroy_vms() {
     "$terraform_dir"
 }
 
+case "$erlang_version" in
+  *@*)
+    erlang_git_ref=${erlang_version#*@}
+    erlang_version=${erlang_version%@*}
+    ;;
+esac
+
 erlang_branch=$(canonicalize_erlang_version "$erlang_version")
 if test -z "$erlang_branch"; then
   echo "Erlang version '$erlang_version' malformed or unrecognized" 1>&2

deps/rabbitmq_ct_helpers/tools/terraform/direct-vms/templates/setup-erlang.sh

@@ -17,6 +17,8 @@ export DEBIAN_FRONTEND
 # shellcheck disable=SC2016
 readonly erlang_version='${erlang_version}'
 # shellcheck disable=SC2016
+erlang_git_ref='${erlang_git_ref}'
+# shellcheck disable=SC2016
 readonly erlang_nodename='${erlang_nodename}'
 # shellcheck disable=SC2016
 readonly default_user='${default_user}'
@@ -30,6 +32,11 @@ readonly erlang_cookie='${erlang_cookie}'
 readonly debian_codename="$${distribution#debian-*}"
 
 case "$erlang_version" in
+  21.0)
+    if test -z "$erlang_git_ref"; then
+      erlang_git_ref='OTP-21.0-rc2'
+    fi
+    ;;
   20.[1-3])
     readonly erlang_package_version="1:$erlang_version-1"
     ;;
@@ -45,40 +52,61 @@ case "$erlang_version" in
     ;;
 esac
 
-# Enable backports.
-cat >/etc/apt/sources.list.d/backports.list << EOF
+setup_backports() {
+  # Enable backports.
+  cat >/etc/apt/sources.list.d/backports.list << EOF
 deb http://httpredir.debian.org/debian $debian_codename-backports main
 EOF
+  apt-get -qq update
+}
 
-# Setup repository to get Erlang.
-readonly esl_package=/tmp/erlang-solutions_1.0_all.deb
-wget -q -O"$esl_package" https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
-dpkg -i "$esl_package"
-apt-get -qq update
+# --------------------------------------------------------------------
+# Functions to take Erlang and Elixir from Debian packages.
+# --------------------------------------------------------------------
 
-# Configure Erlang version pinning.
-cat >/etc/apt/preferences.d/erlang <<EOF
+setup_erlang_deb_repository() {
+  # Setup repository to get Erlang.
+  readonly esl_package=/tmp/erlang-solutions_1.0_all.deb
+  wget -q -O"$esl_package" \
+    https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
+  dpkg -i "$esl_package"
+  apt-get -qq update
+
+  # Configure Erlang version pinning.
+  cat >/etc/apt/preferences.d/erlang <<EOF
 Package: erlang*
 Pin: version $erlang_package_version
 Pin-Priority: 1000
 EOF
+}
 
 apt_install_erlang() {
   apt-get -qq install -y --no-install-recommends \
     erlang-base-hipe erlang-nox erlang-dev erlang-src erlang-common-test
 }
 
-apt_install_extra() {
+apt_install_elixir() {
   case "$debian_codename" in
     wheezy)
       # We don't install Elixir because we only use Debian Wheezy to
       # run Erlang R16B03. This version of Erlang is only useful for
       # RabbitMQ 3.6.x which doesn't depend on Elixir.
+      ;;
+    *)
+      apt-get -qq install -y --no-install-recommends \
+        elixir
+      ;;
+  esac
+}
+
+apt_install_extra() {
+  case "$debian_codename" in
+    wheezy)
       readonly extra_pkgs='make rsync vim-nox zip'
       readonly extra_backports='git'
       ;;
     *)
-      readonly extra_pkgs='elixir git make rsync vim-nox xz-utils zip'
+      readonly extra_pkgs='git make rsync vim-nox xz-utils zip'
       readonly extra_backports=''
       ;;
   esac
@@ -95,8 +123,79 @@ apt_install_extra() {
     $extra_backports
 }
 
+# --------------------------------------------------------------------
+# Functions to build Erlang and Elixir from sources.
+# --------------------------------------------------------------------
+
+install_kerl() {
+  apt-get -qq install -y --no-install-recommends \
+    curl
+
+  mkdir -p /usr/local/bin
+  cd /usr/local/bin
+  curl -O https://raw.githubusercontent.com/kerl/kerl/master/kerl
+  chmod a+x kerl
+}
+
+kerl_install_erlang() {
+  apt-get -qq install -y --no-install-recommends \
+    git \
+    build-essential \
+    autoconf automake libtool \
+    libssl-dev \
+    libncurses5-dev \
+    libsctp1 libsctp-dev
+
+  kerl build git https://github.com/erlang/otp.git "$erlang_git_ref" "$erlang_version"
+  kerl install "$erlang_version" /usr/local/erlang
+
+  . /usr/local/erlang/activate
+  echo '. /usr/local/erlang/activate' > /etc/profile.d/erlang.sh
+}
+
+install_kiex() {
+  curl -sSL https://raw.githubusercontent.com/taylor/kiex/master/install | bash -s
+
+  mv "$HOME/.kiex" /usr/local/kiex
+  sed -E \
+    -e 's,\\\$HOME/\.kiex,/usr/local/kiex,' \
+    -e 's,\$HOME/\.kiex,/usr/local/kiex,' \
+    < /usr/local/kiex/bin/kiex \
+    > /usr/local/kiex/bin/kiex.patched
+  mv /usr/local/kiex/bin/kiex.patched /usr/local/kiex/bin/kiex
+  chmod a+x /usr/local/kiex/bin/kiex
+}
+
+kiex_install_elixir() {
+  export PATH=/usr/local/kiex/bin:$PATH
+  elixir_version=$(kiex list releases | tail -n 1 | awk '{print $1}')
+  kiex install $elixir_version
+
+  . /usr/local/kiex/elixirs/elixir-$elixir_version.env
+  cat >> /etc/profile.d/erlang.sh <<EOF
+
+. /usr/local/kiex/elixirs/elixir-$elixir_version.env
+EOF
+}
+
+# --------------------------------------------------------------------
+# Main.
+# --------------------------------------------------------------------
+
+setup_backports
+
 # Install Erlang + various tools.
-apt_install_erlang
+if test "$erlang_package_version"; then
+  setup_erlang_deb_repository
+  apt_install_erlang
+  apt_install_elixir
+elif test "$erlang_git_ref"; then
+  install_kerl
+  kerl_install_erlang
+  install_kiex
+  kiex_install_elixir
+fi
+
 apt_install_extra
 
 # Store Erlang cookie file for both root and the default user.

deps/rabbitmq_ct_helpers/tools/terraform/direct-vms/variables.tf

@@ -7,6 +7,14 @@ the underlying OS.
 EOF
 }
 
+variable "erlang_git_ref" {
+  default     = ""
+  description = <<EOF
+Git reference if building Erlang from Git. Specifying the Erlang
+version is still required.
+EOF
+}
+
 variable "erlang_cookie" {
   description = <<EOF
 Erlang cookie to deploy on VMs.
@@ -72,6 +80,7 @@ variable "erlang_version_to_system" {
     "20.1"   = "debian-stretch"
     "20.2"   = "debian-stretch"
     "20.3"   = "debian-stretch"
+    "21.0"   = "debian-stretch"
   }
 }