Add mixed version testing for the deps/rabbit suites

Mix clusters with v3.7 and v3.8, currently on both erlang versions. As
this also increases the actions duration rabbit ct-* suites have been
de-grouped such that multiple ct suites are no longer run as part of a
GitHub Actions job.

.github/workflows/base-images.yaml

@@ -1,4 +1,4 @@
-name: Base Images
+name: Workflow Base Images
 on:
 - workflow_dispatch
 jobs:
@@ -8,14 +8,24 @@ jobs:
     steps:
     - name: CHECKOUT REPOSITORY
       uses: actions/checkout@v2
-    - name: CREATE BASE CI IMAGE
+    - name: CREATE ERLANG+ELIXIR IMAGE (22.3)
+      uses: docker/build-push-action@v1
+      with:
+        username: _json_key
+        password: ${{ secrets.GCR_JSON_KEY }}
+        registry: eu.gcr.io
+        repository: cf-rabbitmq-core/erlang_elixir
+        dockerfile: ci/dockerfiles/22.3/erlang_elixir
+        tags: "22.3"
+    - name: CREATE BASE CI IMAGE (22.3)
       uses: docker/build-push-action@v1
       with:
         username: _json_key
         password: ${{ secrets.GCR_JSON_KEY }}
         registry: eu.gcr.io
         repository: cf-rabbitmq-core/ci-base
-        dockerfile: ci/dockerfiles/22.3/base
+        dockerfile: ci/dockerfiles/ci-base
+        build_args: ERLANG_VERSION=22.3,SECONDARY_UMBRELLA_GITREFS=v3.7.28 v3.8.9
         tags: "22.3"
   ci-base-23_1:
     name: ci-base-23_1
@@ -23,12 +33,22 @@ jobs:
     steps:
     - name: CHECKOUT REPOSITORY
       uses: actions/checkout@v2
-    - name: CREATE BASE CI IMAGE
+    - name: CREATE ERLANG+ELIXIR IMAGE (23.1)
+      uses: docker/build-push-action@v1
+      with:
+        username: _json_key
+        password: ${{ secrets.GCR_JSON_KEY }}
+        registry: eu.gcr.io
+        repository: cf-rabbitmq-core/erlang_elixir
+        dockerfile: ci/dockerfiles/23.1/erlang_elixir
+        tags: "23.1"
+    - name: CREATE BASE CI IMAGE (23.1)
       uses: docker/build-push-action@v1
       with:
         username: _json_key
         password: ${{ secrets.GCR_JSON_KEY }}
         registry: eu.gcr.io
         repository: cf-rabbitmq-core/ci-base
-        dockerfile: ci/dockerfiles/23.1/base
+        dockerfile: ci/dockerfiles/ci-base
+        build_args: ERLANG_VERSION=23.1,SECONDARY_UMBRELLA_GITREFS=v3.7.28 v3.8.9
         tags: "23.1"

ci/dockerfiles/22.3/erlang_elixir

@@ -16,9 +16,3 @@ RUN set -xe \
 	&& rm elixir-src.tar.gz \
 	&& cd /usr/local/src/elixir \
 	&& make install clean
-
-RUN apt-get update && apt-get install -y rsync
-
-RUN curl -L -o buildevents https://github.com/honeycombio/buildevents/releases/latest/download/buildevents-linux-amd64
-RUN chmod 755 buildevents
-RUN mv buildevents /usr/bin/

ci/dockerfiles/23.1/erlang_elixir

@@ -16,9 +16,3 @@ RUN set -xe \
 	&& rm elixir-src.tar.gz \
 	&& cd /usr/local/src/elixir \
 	&& make install clean
-
-RUN apt-get update && apt-get install -y rsync
-
-RUN curl -L -o buildevents https://github.com/honeycombio/buildevents/releases/latest/download/buildevents-linux-amd64
-RUN chmod 755 buildevents
-RUN mv buildevents /usr/bin/

ci/dockerfiles/ci-base

@@ -0,0 +1,20 @@
+ARG ERLANG_VERSION
+
+FROM eu.gcr.io/cf-rabbitmq-core/erlang_elixir:${ERLANG_VERSION}
+
+RUN apt-get update && apt-get install -y rsync zip
+
+RUN curl -L -o buildevents https://github.com/honeycombio/buildevents/releases/latest/download/buildevents-linux-amd64
+RUN chmod 755 buildevents
+RUN mv buildevents /usr/bin/
+
+WORKDIR /workspace
+
+COPY ci/scripts/fetch_secondary_umbrellas.sh .
+
+# If we clone the monorepo at a ref when the monorepo was still rabbitmq-server,
+# then we just get rabbitmq-server (not the monorepo as it would have looked, had
+# it existed at that time). So for the time being, secondary umbrellas will derive
+# from rabbitmq-public-umbrella (as they always have)
+ARG SECONDARY_UMBRELLA_GITREFS
+RUN bash fetch_secondary_umbrellas.sh ${SECONDARY_UMBRELLA_GITREFS}

ci/scripts/ct-suite.sh

@@ -14,8 +14,15 @@ catch() {
     fi
 }
 
-buildevents cmd ${GITHUB_RUN_ID} ${GITHUB_RUN_ID}-${project} ct-${CT_SUITE} -- \
+CMD=ct-${CT_SUITE}
+SECONDARY_UMBRELLA_ARGS=""
+if [[ "${SECONDARY_UMBRELLA_VERSION:-}" != "" ]]; then
+    CMD=ct-${CT_SUITE}-mixed-${SECONDARY_UMBRELLA_VERSION}
+    SECONDARY_UMBRELLA_ARGS="SECONDARY_UMBRELLA=/workspace/rabbitmq-${SECONDARY_UMBRELLA_VERSION} RABBITMQ_FEATURE_FLAGS="
+fi
+
+buildevents cmd ${GITHUB_RUN_ID} ${GITHUB_RUN_ID}-${project} ${CMD} -- \
             make ct-${CT_SUITE} \
                  FULL= \
                  FAIL_FAST=1 \
-                 SKIP_AS_ERROR=1
+                 SKIP_AS_ERROR=1 ${SECONDARY_UMBRELLA_ARGS}

ci/scripts/fetch_secondary_umbrellas.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -euo pipefail
+
+refs="$@"
+
+for version in ${refs}; do
+  umbrella="umbrellas/$version"
+  if ! test -d "$umbrella"  ||
+     ! make -C "$umbrella/deps/rabbit" test-dist; then
+    rm -rf "$umbrella"
+
+    # Fetch the master Umbrella; the final umbrellas are created from
+    # the master copy.
+    if ! test -d umbrellas/master; then
+      git config --global advice.detachedHead false
+      git clone \
+        https://github.com/rabbitmq/rabbitmq-public-umbrella.git \
+        umbrellas/master
+      make -C umbrellas/master co # To get RabbitMQ components.
+    fi
+
+    # We copy the master Umbrella and checkout the appropriate tag.
+    cp -a umbrellas/master "$umbrella"
+    git -C "$umbrella" checkout "master"
+    make -C "$umbrella" up BRANCH="$version"
+    # To remove third-party deps which were checked out when the
+    # projects were on the `master` branch. Thus, possibly not the
+    # version pinning we expect. We update the Umbrella one last time
+    # to fetch the correct third-party deps.
+    make -C "$umbrella" clean-3rd-party-repos
+    make -C "$umbrella" up
+    make -C "$umbrella/deps/rabbit" test-dist
+    rm -rf "$umbrella"/deps/rabbitmq_website
+    rm -rf "$umbrella"/deps/rabbitmq_prometheus/docker
+    rm -rf "$umbrella"/deps/*/{.git,test} "$umbrella"/.git
+  fi
+done
+
+for version in ${refs}; do
+  umbrella="umbrellas/$version"
+  mv ${umbrella} rabbitmq-${version}
+done
+
+rm -fr umbrellas

mk/github-actions.mk

@@ -155,14 +155,3 @@ docker: local-ci-image
 		--oom-score-adj -500 \
 		$(LOCAL_IMAGE) \
 		/bin/bash
-
-# A literal space.
-space :=
-space +=
-
-comma := ,
-
-# Joins elements of the list in arg 2 with the given separator.
-#   1. Element separator.
-#   2. The list.
-join-with = $(subst $(space),$1,$(strip $(comma)))

workflow_sources/base_image/workflow.yml

@@ -9,24 +9,41 @@
 #@   return jobs
 #@ end
 
+#@ def to_build_args(d):
+#@   return ",".join(['{0}={1}'.format(k,d[k]) for k in d.keys()])
+#@ end
+
 #@ def base_image_job(erlang_version):
 name: #@ ("ci-base-" + erlang_version).replace(".", "_")
 runs-on: ubuntu-18.04
 steps:
   - name: CHECKOUT REPOSITORY
     uses: actions/checkout@v2
-  - name: CREATE BASE CI IMAGE
+  - name: #@ 'CREATE ERLANG+ELIXIR IMAGE ({})'.format(erlang_version)
+    uses: docker/build-push-action@v1
+    with:
+      username: _json_key
+      password: ${{ secrets.GCR_JSON_KEY }}
+      registry: eu.gcr.io
+      repository: cf-rabbitmq-core/erlang_elixir
+      dockerfile: #@ 'ci/dockerfiles/{}/erlang_elixir'.format(erlang_version)
+      tags: #@ erlang_version
+  - name: #@ 'CREATE BASE CI IMAGE ({})'.format(erlang_version)
     uses: docker/build-push-action@v1
     with:
       username: _json_key
       password: ${{ secrets.GCR_JSON_KEY }}
       registry: eu.gcr.io
       repository: cf-rabbitmq-core/ci-base
-      dockerfile: #@ "ci/dockerfiles/" + erlang_version + "/base"
+      dockerfile: ci/dockerfiles/ci-base
+      #@ build_args = {}
+      #@ build_args["ERLANG_VERSION"] = erlang_version
+      #@ build_args["SECONDARY_UMBRELLA_GITREFS"] = ' '.join(data.values.secondary_umbrella_gitrefs)
+      build_args: #@ to_build_args(build_args)
       tags: #@ erlang_version
 #@ end
 ---
-name: Base Images
+name: Workflow Base Images
 #! https://github.com/k14s/ytt/issues/189
 a_magic_string_that_we_will_sed_to_on: [workflow_dispatch]
 jobs: #@ jobs_map_from_list([base_image_job(v) for v in data.values.erlang_versions])

workflow_sources/base_values.yml

@@ -1,6 +1,9 @@
 #@data/values
 ---
 base_rmq_ref: master
+secondary_umbrella_gitrefs:
+- v3.7.28
+- v3.8.9
 #! these values are injected at template time from the cli,
 #! but must be declared first here
 erlang_versions: []

workflow_sources/test/ct.lib.yml

@@ -1,6 +1,6 @@
 #@ load("@ytt:data", "data")
 #@ load("@ytt:assert", "assert")
-#@ load("util.star", "is_unique", "group_by_time", "to_build_args")
+#@ load("util.star", "is_unique", "group_by_one", "to_build_args")
 #@ load("helpers.star", "ci_image_tag", "ci_dep_image")
 
 #@ def checks_job(dep):
@@ -77,12 +77,36 @@ steps:
     with:
       name: (@= dep.name @)-ct-(@= suite @)-logs
       path: "ct-(@= suite @)-logs/*-ct-logs-*.tar.xz"
+  #@ for version in data.values.secondary_umbrella_gitrefs:
+  #@   logs_dir = 'ct-{}-logs-{}'.format(suite, version)
+  - name: #@ "RUN ct-{} [mixed {}]".format(suite, version)
+    run: |
+      mkdir (@= logs_dir @) && chmod 777 (@= logs_dir @)
+      docker run \
+        --env project=(@= dep.name @) \
+        --env CT_SUITE=(@= suite @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env SECONDARY_UMBRELLA_VERSION=(@= version @) \
+        --volume ${PWD}/(@= logs_dir @):/workspace/ct-logs \
+        (@= ci_dep_image(dep.name) @) \
+        /workspace/rabbitmq/ci/scripts/ct-suite.sh
+  - name: #@ 'ON FAILURE UPLOAD TESTS LOGS ARTIFACT [mixed {}]'.format(version)
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    #! For whatever reason, "if: steps.(@= suite @).outcome == 'failure'" never executes,
+    #! so just run always
+    if: always()
+    with:
+      name: (@= dep.name @)-ct-(@= suite @)-logs-mixed-(@= version @)
+      path: "(@= logs_dir @)/*-ct-logs-*.tar.xz"
+  #@ end
   #@ end
 #@ end
 
 #@ def collect_job(dep):
 name: #@ dep.name
-needs: #@ [dep.name + "-checks"] + [dep.name + "-ct-" + group["name"] for group in group_by_time(dep.suites)]
+needs: #@ [dep.name + "-checks"] + [dep.name + "-ct-" + group["name"] for group in group_by_one(dep.suites)]
 runs-on: ubuntu-18.04
 if: always()
 #@yaml/text-templated-strings

workflow_sources/test/dep.star

@@ -1,4 +1,4 @@
-load("util.star", "group_by_time")
+load("util.star", "group_by_one")
 load("rabbitmq_cli.lib.yml", "rabbitmq_cli_job")
 load("ct.lib.yml", "checks_job", "ct_suites_job", "collect_job")
 load("tests.lib.yml", "tests_job")
@@ -10,7 +10,7 @@ def dep_jobs(dep):
       jobs[dep.name] = rabbitmq_cli_job(dep)
     elif len(dep.suites) > 20:
       jobs[dep.name + "-checks"] = checks_job(dep)
-      for group in group_by_time(dep.suites):
+      for group in group_by_one(dep.suites):
         jobs[dep.name + "-ct-" + group["name"]] = ct_suites_job(dep, group)
       end
       jobs[dep.name] = collect_job(dep)

workflow_sources/test/util.star

@@ -48,6 +48,11 @@ def group_by_time(suites):
   return [{"name": name(suites), "suites": [suite.name for suite in suites]} for suites in groups]
 end
 
+# Used when we don't actually want multiple ct-suites per job
+def group_by_one(suites):
+  return [{"name": suite.name, "suites": [suite.name]} for suite in suites]
+end
+
 def to_build_args(d):
   return ",".join(['{0}={1}'.format(k,d[k]) for k in d.keys()])
 end