Fetch all cli deps with bazel (#7875)

* Fetch all prod cli deps with bazel

This avoids issues with hex and OTP 26, and is needed for offline
bazel builds anyway

* Fetch test cli deps with bazel

* mix format

MODULE.bazel

@@ -31,7 +31,7 @@ bazel_dep(
 
 bazel_dep(
     name = "rules_erlang",
-    version = "3.9.9",
+    version = "3.9.11",
 )
 
 erlang_config = use_extension(
@@ -250,6 +250,13 @@ erlang_package.git_package(
     tag = "v1.5.0",
 )
 
+erlang_package.hex_package(
+    name = "parallel_stream",
+    build_file = "@rabbitmq-server//bazel:BUILD.parallel_stream",
+    sha256 = "639b2e8749e11b87b9eb42f2ad325d161c170b39b288ac8d04c4f31f8f0823eb",
+    version = "1.0.6",
+)
+
 erlang_package.hex_package(
     name = "prometheus",
     build_file = "@rabbitmq-server//bazel:BUILD.prometheus",
@@ -343,6 +350,7 @@ use_repo(
     "json",
     "observer_cli",
     "osiris",
+    "parallel_stream",
     "prometheus",
     "ra",
     "ranch",
@@ -361,12 +369,26 @@ erlang_dev_package = use_extension(
     "erlang_package",
 )
 
+erlang_dev_package.hex_package(
+    name = "amqp",
+    build_file = "@rabbitmq-server//bazel:BUILD.amqp",
+    sha256 = "b6d926770e4508e30e3e9e476c57b6c8aeda44f7715663bdc38935620ce5be6f",
+    version = "2.1.1",
+)
+
 erlang_dev_package.git_package(
     branch = "master",
     build_file = "@//:bazel/BUILD.ct_helper",
     repository = "ninenines/ct_helper",
 )
 
+erlang_dev_package.hex_package(
+    name = "dialyxir",
+    build_file = "@rabbitmq-server//bazel:BUILD.dialyxir",
+    sha256 = "6c32a70ed5d452c6650916555b1f96c79af5fc4bf286997f8b15f213de786f73",
+    version = "0.5.1",
+)
+
 erlang_dev_package.git_package(
     name = "emqtt",
     build_file = "@rabbitmq-server//bazel:BUILD.emqtt",
@@ -393,13 +415,31 @@ erlang_dev_package.git_package(
     repository = "manopapad/proper",
 )
 
+erlang_dev_package.hex_package(
+    name = "temp",
+    build_file = "@rabbitmq-server//bazel:BUILD.temp",
+    sha256 = "6af19e7d6a85a427478be1021574d1ae2a1e1b90882586f06bde76c63cd03e0d",
+    version = "0.4.7",
+)
+
+erlang_dev_package.hex_package(
+    name = "x509",
+    build_file = "@rabbitmq-server//bazel:BUILD.x509",
+    sha256 = "5ff9c79e77d64a62ccffd90aaeb23e8f5b6e47844ef7bc8fed931ecf238662e0",
+    version = "0.7.0",
+)
+
 use_repo(
     erlang_dev_package,
+    "amqp",
     "ct_helper",
+    "dialyxir",
     "emqtt",
     "inet_tcp_proxy_dist",
     "meck",
     "proper",
+    "temp",
+    "x509",
 )
 
 rbe = use_extension(

WORKSPACE

@@ -6,7 +6,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_r
 git_repository(
     name = "rules_erlang",
     remote = "https://github.com/rabbitmq/rules_erlang.git",
-    tag = "3.9.9",
+    tag = "3.9.11",
 )
 
 load("@rules_erlang//:internal_deps.bzl", "rules_erlang_internal_deps")
@@ -206,3 +206,10 @@ git_repository(
     branch = "linux-rbe",
     remote = "https://github.com/rabbitmq/rbe-erlang-platform.git",
 )
+
+http_archive(
+    name = "hex",
+    strip_prefix = "hex-2.0.5",
+    urls = ["https://github.com/hexpm/hex/archive/refs/tags/v2.0.5.zip"],
+    build_file = "@rabbitmq-server//bazel:BUILD.hex",
+)

bazel/BUILD.amqp

@@ -0,0 +1,10 @@
+filegroup(
+    name = "sources",
+    srcs = [
+        "mix.exs",
+    ] + glob([
+        "LICENSE*",
+        "lib/**/*",
+    ]),
+    visibility = ["//visibility:public"],
+)

bazel/BUILD.dialyxir

@@ -0,0 +1,10 @@
+filegroup(
+    name = "sources",
+    srcs = [
+        "mix.exs",
+    ] + glob([
+        "LICENSE*",
+        "lib/**/*",
+    ]),
+    visibility = ["//visibility:public"],
+)

bazel/BUILD.ex_doc

@@ -0,0 +1,10 @@
+filegroup(
+    name = "sources",
+    srcs = [
+        "mix.exs",
+    ] + glob([
+        "LICENSE*",
+        "lib/**/*",
+    ]),
+    visibility = ["//visibility:public"],
+)

bazel/BUILD.hex

@@ -0,0 +1,15 @@
+load(
+    "@rabbitmq-server//bazel/elixir:mix_archive_build.bzl",
+    "mix_archive_build",
+)
+
+mix_archive_build(
+    name = "archive",
+    srcs = [
+        "mix.exs",
+    ] + glob([
+        "lib/**/*",
+    ]),
+    out = "hex.ez",
+    visibility = ["//visibility:public"],
+)

bazel/BUILD.parallel_stream

@@ -0,0 +1,10 @@
+filegroup(
+    name = "sources",
+    srcs = [
+        "mix.exs",
+    ] + glob([
+        "LICENSE*",
+        "lib/**/*",
+    ]),
+    visibility = ["//visibility:public"],
+)

bazel/BUILD.temp

@@ -0,0 +1,10 @@
+filegroup(
+    name = "sources",
+    srcs = [
+        "mix.exs",
+    ] + glob([
+        "LICENSE*",
+        "lib/**/*",
+    ]),
+    visibility = ["//visibility:public"],
+)

bazel/BUILD.x509

@@ -0,0 +1,10 @@
+filegroup(
+    name = "sources",
+    srcs = [
+        "mix.exs",
+    ] + glob([
+        "LICENSE*",
+        "lib/**/*",
+    ]),
+    visibility = ["//visibility:public"],
+)

bazel/elixir/mix_archive_build.bzl

@@ -0,0 +1,118 @@
+load(
+    "@rules_erlang//:util.bzl",
+    "path_join",
+)
+load(
+    "@rules_erlang//private:util.bzl",
+    "additional_file_dest_relative_path",
+)
+load(
+    "//bazel/elixir:elixir_toolchain.bzl",
+    "elixir_dirs",
+    "erlang_dirs",
+    "maybe_install_erlang",
+)
+
+def _impl(ctx):
+    (erlang_home, _, erlang_runfiles) = erlang_dirs(ctx)
+    (elixir_home, elixir_runfiles) = elixir_dirs(ctx)
+
+    out = ctx.actions.declare_file(ctx.attr.out.name)
+    mix_invocation_dir = ctx.actions.declare_directory("{}_mix".format(ctx.label.name))
+
+    package_dir = path_join(
+        ctx.label.workspace_root,
+        ctx.label.package,
+    )
+
+    copy_srcs_commands = []
+    for src in ctx.files.srcs:
+        dest = additional_file_dest_relative_path(ctx.label, src)
+        copy_srcs_commands.extend([
+            "mkdir -p $(dirname ${{MIX_INVOCATION_DIR}}/{dest})".format(
+                dest = dest,
+            ),
+            "cp {flags}{src} ${{MIX_INVOCATION_DIR}}/{dest}".format(
+                flags = "-r " if src.is_directory else "",
+                src = src.path,
+                dest = dest,
+            ),
+        ])
+
+    script = """set -euo pipefail
+
+{maybe_install_erlang}
+
+if [[ "{elixir_home}" == /* ]]; then
+    ABS_ELIXIR_HOME="{elixir_home}"
+else
+    ABS_ELIXIR_HOME=$PWD/{elixir_home}
+fi
+
+ABS_OUT_PATH=$PWD/{out}
+
+export PATH="$ABS_ELIXIR_HOME"/bin:"{erlang_home}"/bin:${{PATH}}
+
+export LANG="en_US.UTF-8"
+export LC_ALL="en_US.UTF-8"
+
+MIX_INVOCATION_DIR="{mix_invocation_dir}"
+
+{copy_srcs_commands}
+
+cd ${{MIX_INVOCATION_DIR}}
+export HOME=${{PWD}}
+export MIX_ENV=prod
+export ERL_COMPILER_OPTIONS=deterministic
+"${{ABS_ELIXIR_HOME}}"/bin/mix archive.build -o ${{ABS_OUT_PATH}}
+
+# remove symlinks from the _build directory since it
+# is an unused output, and bazel does not allow them
+find . -type l -delete
+""".format(
+        maybe_install_erlang = maybe_install_erlang(ctx),
+        erlang_home = erlang_home,
+        elixir_home = elixir_home,
+        mix_invocation_dir = mix_invocation_dir.path,
+        copy_srcs_commands = "\n".join(copy_srcs_commands),
+        package_dir = package_dir,
+        out = out.path,
+    )
+
+    inputs = depset(
+        direct = ctx.files.srcs,
+        transitive = [
+            erlang_runfiles.files,
+            elixir_runfiles.files,
+        ],
+    )
+
+    ctx.actions.run_shell(
+        inputs = inputs,
+        outputs = [
+            out,
+            mix_invocation_dir,
+        ],
+        command = script,
+        mnemonic = "MIX",
+    )
+
+    return [
+        DefaultInfo(
+            files = depset([out]),
+        ),
+    ]
+
+mix_archive_build = rule(
+    implementation = _impl,
+    attrs = {
+        "srcs": attr.label_list(
+            mandatory = True,
+            allow_files = True,
+        ),
+        "out": attr.output(),
+    },
+    toolchains = [
+        ":toolchain_type",
+    ],
+)

deps/rabbitmq_cli/BUILD.bazel

@@ -23,9 +23,13 @@ rabbitmqctl(
         "@observer_cli//:erlang_app",
         "@stdout_formatter//:erlang_app",
     ],
+    archives = [
+        "@hex//:archive",
+    ],
     source_deps = {
         "@csv//:sources": "csv",
         "@json//:sources": "json",
+        "@parallel_stream//:sources": "parallel_stream", # transitive dep of csv 2.x
     },
 )
 
@@ -128,9 +132,17 @@ rabbitmqctl_test(
         "@observer_cli//:erlang_app",
         "@stdout_formatter//:erlang_app",
     ],
+    archives = [
+        "@hex//:archive",
+    ],
     source_deps = {
+        "@amqp//:sources": "amqp",
         "@csv//:sources": "csv",
+        "@dialyxir//:sources": "dialyxir",
         "@json//:sources": "json",
+        "@parallel_stream//:sources": "parallel_stream", # transitive dep of csv 2.x
+        "@temp//:sources": "temp",
+        "@x509//:sources": "x509",
     },
 )
 

deps/rabbitmq_cli/Makefile

@@ -1,11 +1,16 @@
 PROJECT = rabbitmq_cli
 
 BUILD_DEPS = rabbit_common
-DEPS = csv json observer_cli stdout_formatter
+DEPS = csv json observer_cli parallel_stream stdout_formatter
-TEST_DEPS = amqp_client rabbit
+TEST_DEPS = amqp amqp_client dialyxir temp x509 rabbit
 
+dep_amqp = hex 2.1.1
 dep_csv = hex 2.4.1
+dep_dialyxir = hex 0.5.1
+dep_parallel_stream = hex 1.0.6
 dep_json = hex 1.4.1
+dep_temp = hex 0.4.7
+dep_x509 = hex 0.7.0
 
 DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk
 DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk

deps/rabbitmq_cli/mix.exs

@@ -15,7 +15,7 @@ defmodule RabbitMQCtl.MixfileBase do
       build_embedded: Mix.env() == :prod,
       start_permanent: Mix.env() == :prod,
       escript: [main_module: RabbitMQCtl, emu_args: "-hidden", path: "escript/rabbitmqctl"],
-      deps: deps(),
+      deps: deps(Mix.env()),
       aliases: aliases(),
       xref: [
         exclude: [
@@ -122,7 +122,7 @@ defmodule RabbitMQCtl.MixfileBase do
   # from Hex.pm in RabbitMQ source archive (the source archive must be
   # self-contained and RabbitMQ must be buildable offline). However, we
   # don't have the equivalent for other methods.
-  defp deps() do
+  defp deps(env) do
     deps_dir = System.get_env("DEPS_DIR", "deps")
 
     # Mix is confused by any `rebar.{config,lock}` we might have left in
@@ -146,6 +146,10 @@ defmodule RabbitMQCtl.MixfileBase do
         :csv,
         path: Path.join(deps_dir, "csv")
       },
+      {
+        :parallel_stream,
+        path: Path.join(deps_dir, "parallel_stream"), override: true
+      },
       {
         :stdout_formatter,
         path: Path.join(deps_dir, "stdout_formatter"),
@@ -156,24 +160,43 @@ defmodule RabbitMQCtl.MixfileBase do
         path: Path.join(deps_dir, "observer_cli"),
         compile: if(is_bazel, do: false, else: make_cmd)
       },
-      {:amqp, "~> 2.1.0", only: :test},
-      {:dialyxir, "~> 0.5", only: :test, runtime: false},
-      {:temp, "~> 0.4", only: :test},
-      {:x509, "~> 0.7", only: :test},
       {
         :rabbit_common,
         path: Path.join(deps_dir, "rabbit_common"),
         compile: if(is_bazel, do: false, else: make_cmd),
         override: true
-      },
-      {
-        :amqp_client,
-        path: Path.join(deps_dir, "amqp_client"),
-        compile: if(is_bazel, do: false, else: make_cmd),
-        override: true,
-        only: :test
       }
-    ]
+    ] ++
+      case env do
+        :test ->
+          [
+            {
+              :amqp,
+              path: Path.join(deps_dir, "amqp")
+            },
+            {
+              :dialyxir,
+              path: Path.join(deps_dir, "dialyxir"), runtime: false
+            },
+            {
+              :temp,
+              path: Path.join(deps_dir, "temp")
+            },
+            {
+              :x509,
+              path: Path.join(deps_dir, "x509")
+            },
+            {
+              :amqp_client,
+              path: Path.join(deps_dir, "amqp_client"),
+              compile: if(is_bazel, do: false, else: make_cmd),
+              override: true
+            }
+          ]
+
+        _ ->
+          []
+      end
   end
 
   defp aliases do

deps/rabbitmq_cli/rabbitmqctl.bzl

@@ -49,7 +49,7 @@ def deps_dir_contents(ctx, deps, dir):
                     output = f,
                     target_file = src,
                 )
-                files.append(f)
+                files.extend([src, f])
     return files
 
 def _impl(ctx):
@@ -60,7 +60,6 @@ def _impl(ctx):
     ebin = ctx.actions.declare_directory("ebin")
     consolidated = ctx.actions.declare_directory("consolidated")
     mix_invocation_dir = ctx.actions.declare_directory("{}_mix".format(ctx.label.name))
-    fetched_srcs = ctx.actions.declare_file("deps.tar")
 
     deps = flat_deps(ctx.attr.deps)
 
@@ -100,7 +99,6 @@ fi
 ABS_EBIN_DIR=$PWD/{ebin_dir}
 ABS_CONSOLIDATED_DIR=$PWD/{consolidated_dir}
 ABS_ESCRIPT_PATH=$PWD/{escript_path}
-ABS_FETCHED_SRCS=$PWD/{fetched_srcs}
 
 export PATH="$ABS_ELIXIR_HOME"/bin:"{erlang_home}"/bin:${{PATH}}
 
@@ -113,17 +111,18 @@ cp -r {package_dir}/config ${{MIX_INVOCATION_DIR}}/config
 cp -r {package_dir}/lib ${{MIX_INVOCATION_DIR}}/lib
 cp    {package_dir}/mix.exs ${{MIX_INVOCATION_DIR}}/mix.exs
 
+ORIGINAL_DIR=$PWD
 cd ${{MIX_INVOCATION_DIR}}
 export IS_BAZEL=true
 export HOME=${{PWD}}
 export DEPS_DIR=$(dirname $ABS_EBIN_DIR)/{deps_dir}
 export MIX_ENV=prod
 export ERL_COMPILER_OPTIONS=deterministic
-"${{ABS_ELIXIR_HOME}}"/bin/mix local.hex --force
+for archive in {archives}; do
-"${{ABS_ELIXIR_HOME}}"/bin/mix local.rebar --force
+    "${{ABS_ELIXIR_HOME}}"/bin/mix archive.install --force $ORIGINAL_DIR/$archive
-"${{ABS_ELIXIR_HOME}}"/bin/mix deps.get --only prod
+done
 for d in {precompiled_deps}; do
-    mkdir _build/${{MIX_ENV}}/lib/$d
+    mkdir -p _build/${{MIX_ENV}}/lib/$d
     ln -s ${{DEPS_DIR}}/$d/ebin _build/${{MIX_ENV}}/lib/$d
     ln -s ${{DEPS_DIR}}/$d/include _build/${{MIX_ENV}}/lib/$d
 done
@@ -136,9 +135,6 @@ cp escript/rabbitmqctl ${{ABS_ESCRIPT_PATH}}
 cp _build/${{MIX_ENV}}/lib/rabbitmqctl/ebin/* ${{ABS_EBIN_DIR}}
 cp _build/${{MIX_ENV}}/lib/rabbitmqctl/consolidated/* ${{ABS_CONSOLIDATED_DIR}}
 
-tar --file ${{ABS_FETCHED_SRCS}} \\
-    --create deps
-
 # remove symlinks from the _build directory since it
 # is not used, and bazel does not allow them
 find . -type l -delete
@@ -152,7 +148,7 @@ find . -type l -delete
         escript_path = escript.path,
         ebin_dir = ebin.path,
         consolidated_dir = consolidated.path,
-        fetched_srcs = fetched_srcs.path,
+        archives = "".join([a.path for a in ctx.files.archives]),
         precompiled_deps = " ".join([
             dep[ErlangAppInfo].app_name
             for dep in ctx.attr.deps
@@ -164,6 +160,7 @@ find . -type l -delete
         transitive = [
             erlang_runfiles.files,
             elixir_runfiles.files,
+            depset(ctx.files.archives),
             depset(deps_dir_files),
         ],
     )
@@ -175,7 +172,6 @@ find . -type l -delete
             ebin,
             consolidated,
             mix_invocation_dir,
-            fetched_srcs,
         ],
         command = script,
         mnemonic = "MIX",
@@ -192,7 +188,7 @@ find . -type l -delete
     return [
         DefaultInfo(
             executable = escript,
-            files = depset([ebin, consolidated, fetched_srcs]),
+            files = depset([ebin, consolidated]),
             runfiles = runfiles,
         ),
         ElixirAppInfo(
@@ -224,6 +220,9 @@ rabbitmqctl_private = rule(
         "deps": attr.label_list(
             providers = [ErlangAppInfo],
         ),
+        "archives": attr.label_list(
+            allow_files = [".ez"],
+        ),
         "source_deps": attr.label_keyed_string_dict(),
     },
     toolchains = [

deps/rabbitmq_cli/rabbitmqctl_test.bzl

@@ -96,11 +96,11 @@ export HOME=${{PWD}}
 export DEPS_DIR=$TEST_SRCDIR/$TEST_WORKSPACE/{package_dir}/{deps_dir}
 export MIX_ENV=test
 export ERL_COMPILER_OPTIONS=deterministic
-"${{ABS_ELIXIR_HOME}}"/bin/mix local.hex --force
+for archive in {archives}; do
-"${{ABS_ELIXIR_HOME}}"/bin/mix local.rebar --force
+    "${{ABS_ELIXIR_HOME}}"/bin/mix archive.install --force $INITIAL_DIR/$archive
-"${{ABS_ELIXIR_HOME}}"/bin/mix deps.get
+done
 for d in {precompiled_deps}; do
-    mkdir _build/${{MIX_ENV}}/lib/$d
+    mkdir -p _build/${{MIX_ENV}}/lib/$d
     ln -s ${{DEPS_DIR}}/$d/ebin _build/${{MIX_ENV}}/lib/$d
     ln -s ${{DEPS_DIR}}/$d/include _build/${{MIX_ENV}}/lib/$d
 done
@@ -138,6 +138,7 @@ set -x
             elixir_home = elixir_home,
             package_dir = package_dir,
             deps_dir = deps_dir,
+            archives = "".join([a.short_path for a in ctx.files.archives]),
             precompiled_deps = precompiled_deps,
             rabbitmq_run_cmd = ctx.attr.rabbitmq_run[DefaultInfo].files_to_run.executable.short_path,
         )
@@ -167,9 +168,11 @@ set DEPS_DIR=%TEST_SRCDIR%/%TEST_WORKSPACE%/{package_dir}/{deps_dir}
 set DEPS_DIR=%DEPS_DIR:/=\\%
 set ERL_COMPILER_OPTIONS=deterministic
 set MIX_ENV=test
-echo y | "{elixir_home}\\bin\\mix" local.hex --force || goto :error
+for %%a in ({archives}) do (
-echo y | "{elixir_home}\\bin\\mix" local.rebar --force || goto :error
+    set ARCH=%TEST_SRCDIR%/%TEST_WORKSPACE%/%%a
-"{elixir_home}\\bin\\mix" deps.get || goto :error
+    set ARCH=%ARCH:/=\\%
+    "{elixir_home}\\bin\\mix" archive.install --force %ARCH% || goto :error
+)
 for %%d in ({precompiled_deps}) do (
     mkdir _build\\%MIX_ENV%\\lib\\%%d
     robocopy %DEPS_DIR%\\%%d\\ebin _build\\%MIX_ENV%\\lib\\%%d /E /NFL /NDL /NJH /NJS /nc /ns /np
@@ -192,6 +195,7 @@ exit /b 1
             elixir_home = windows_path(elixir_home),
             package_dir = windows_path(ctx.label.package),
             deps_dir = deps_dir,
+            archives = "".join([a.short_path for a in ctx.files.archives]),
             precompiled_deps = precompiled_deps,
             rabbitmq_run_cmd = ctx.attr.rabbitmq_run[DefaultInfo].files_to_run.executable.short_path,
         )
@@ -202,7 +206,7 @@ exit /b 1
     )
 
     runfiles = ctx.runfiles(
-        files = ctx.files.srcs + ctx.files.data,
+        files = ctx.files.srcs + ctx.files.data + ctx.files.archives,
         transitive_files = depset(deps_dir_files),
     ).merge_all([
         erlang_runfiles,
@@ -222,6 +226,9 @@ rabbitmqctl_private_test = rule(
         "srcs": attr.label_list(allow_files = [".ex", ".exs"]),
         "data": attr.label_list(allow_files = True),
         "deps": attr.label_list(providers = [ErlangAppInfo]),
+        "archives": attr.label_list(
+            allow_files = [".ez"],
+        ),
         "source_deps": attr.label_keyed_string_dict(),
         "rabbitmq_run": attr.label(
             executable = True,