Fetch all cli deps with bazel (#7875)

* Fetch all prod cli deps with bazel

This avoids issues with hex and OTP 26, and is needed for offline
bazel builds anyway

* Fetch test cli deps with bazel

* mix format
This commit is contained in:
Rin Kuryloski 2023-04-12 14:27:32 +02:00 committed by GitHub
parent 3c19348c58
commit f09b31e8c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 325 additions and 39 deletions

View File

@ -31,7 +31,7 @@ bazel_dep(
bazel_dep(
name = "rules_erlang",
version = "3.9.9",
version = "3.9.11",
)
erlang_config = use_extension(
@ -250,6 +250,13 @@ erlang_package.git_package(
tag = "v1.5.0",
)
erlang_package.hex_package(
name = "parallel_stream",
build_file = "@rabbitmq-server//bazel:BUILD.parallel_stream",
sha256 = "639b2e8749e11b87b9eb42f2ad325d161c170b39b288ac8d04c4f31f8f0823eb",
version = "1.0.6",
)
erlang_package.hex_package(
name = "prometheus",
build_file = "@rabbitmq-server//bazel:BUILD.prometheus",
@ -343,6 +350,7 @@ use_repo(
"json",
"observer_cli",
"osiris",
"parallel_stream",
"prometheus",
"ra",
"ranch",
@ -361,12 +369,26 @@ erlang_dev_package = use_extension(
"erlang_package",
)
erlang_dev_package.hex_package(
name = "amqp",
build_file = "@rabbitmq-server//bazel:BUILD.amqp",
sha256 = "b6d926770e4508e30e3e9e476c57b6c8aeda44f7715663bdc38935620ce5be6f",
version = "2.1.1",
)
erlang_dev_package.git_package(
branch = "master",
build_file = "@//:bazel/BUILD.ct_helper",
repository = "ninenines/ct_helper",
)
erlang_dev_package.hex_package(
name = "dialyxir",
build_file = "@rabbitmq-server//bazel:BUILD.dialyxir",
sha256 = "6c32a70ed5d452c6650916555b1f96c79af5fc4bf286997f8b15f213de786f73",
version = "0.5.1",
)
erlang_dev_package.git_package(
name = "emqtt",
build_file = "@rabbitmq-server//bazel:BUILD.emqtt",
@ -393,13 +415,31 @@ erlang_dev_package.git_package(
repository = "manopapad/proper",
)
erlang_dev_package.hex_package(
name = "temp",
build_file = "@rabbitmq-server//bazel:BUILD.temp",
sha256 = "6af19e7d6a85a427478be1021574d1ae2a1e1b90882586f06bde76c63cd03e0d",
version = "0.4.7",
)
erlang_dev_package.hex_package(
name = "x509",
build_file = "@rabbitmq-server//bazel:BUILD.x509",
sha256 = "5ff9c79e77d64a62ccffd90aaeb23e8f5b6e47844ef7bc8fed931ecf238662e0",
version = "0.7.0",
)
use_repo(
erlang_dev_package,
"amqp",
"ct_helper",
"dialyxir",
"emqtt",
"inet_tcp_proxy_dist",
"meck",
"proper",
"temp",
"x509",
)
rbe = use_extension(

View File

@ -6,7 +6,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_r
git_repository(
name = "rules_erlang",
remote = "https://github.com/rabbitmq/rules_erlang.git",
tag = "3.9.9",
tag = "3.9.11",
)
load("@rules_erlang//:internal_deps.bzl", "rules_erlang_internal_deps")
@ -206,3 +206,10 @@ git_repository(
branch = "linux-rbe",
remote = "https://github.com/rabbitmq/rbe-erlang-platform.git",
)
http_archive(
name = "hex",
strip_prefix = "hex-2.0.5",
urls = ["https://github.com/hexpm/hex/archive/refs/tags/v2.0.5.zip"],
build_file = "@rabbitmq-server//bazel:BUILD.hex",
)

10
bazel/BUILD.amqp Normal file
View File

@ -0,0 +1,10 @@
filegroup(
name = "sources",
srcs = [
"mix.exs",
] + glob([
"LICENSE*",
"lib/**/*",
]),
visibility = ["//visibility:public"],
)

10
bazel/BUILD.dialyxir Normal file
View File

@ -0,0 +1,10 @@
filegroup(
name = "sources",
srcs = [
"mix.exs",
] + glob([
"LICENSE*",
"lib/**/*",
]),
visibility = ["//visibility:public"],
)

10
bazel/BUILD.ex_doc Normal file
View File

@ -0,0 +1,10 @@
filegroup(
name = "sources",
srcs = [
"mix.exs",
] + glob([
"LICENSE*",
"lib/**/*",
]),
visibility = ["//visibility:public"],
)

15
bazel/BUILD.hex Normal file
View File

@ -0,0 +1,15 @@
load(
"@rabbitmq-server//bazel/elixir:mix_archive_build.bzl",
"mix_archive_build",
)
mix_archive_build(
name = "archive",
srcs = [
"mix.exs",
] + glob([
"lib/**/*",
]),
out = "hex.ez",
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,10 @@
filegroup(
name = "sources",
srcs = [
"mix.exs",
] + glob([
"LICENSE*",
"lib/**/*",
]),
visibility = ["//visibility:public"],
)

10
bazel/BUILD.temp Normal file
View File

@ -0,0 +1,10 @@
filegroup(
name = "sources",
srcs = [
"mix.exs",
] + glob([
"LICENSE*",
"lib/**/*",
]),
visibility = ["//visibility:public"],
)

10
bazel/BUILD.x509 Normal file
View File

@ -0,0 +1,10 @@
filegroup(
name = "sources",
srcs = [
"mix.exs",
] + glob([
"LICENSE*",
"lib/**/*",
]),
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,118 @@
load(
"@rules_erlang//:util.bzl",
"path_join",
)
load(
"@rules_erlang//private:util.bzl",
"additional_file_dest_relative_path",
)
load(
"//bazel/elixir:elixir_toolchain.bzl",
"elixir_dirs",
"erlang_dirs",
"maybe_install_erlang",
)
def _impl(ctx):
(erlang_home, _, erlang_runfiles) = erlang_dirs(ctx)
(elixir_home, elixir_runfiles) = elixir_dirs(ctx)
out = ctx.actions.declare_file(ctx.attr.out.name)
mix_invocation_dir = ctx.actions.declare_directory("{}_mix".format(ctx.label.name))
package_dir = path_join(
ctx.label.workspace_root,
ctx.label.package,
)
copy_srcs_commands = []
for src in ctx.files.srcs:
dest = additional_file_dest_relative_path(ctx.label, src)
copy_srcs_commands.extend([
"mkdir -p $(dirname ${{MIX_INVOCATION_DIR}}/{dest})".format(
dest = dest,
),
"cp {flags}{src} ${{MIX_INVOCATION_DIR}}/{dest}".format(
flags = "-r " if src.is_directory else "",
src = src.path,
dest = dest,
),
])
script = """set -euo pipefail
{maybe_install_erlang}
if [[ "{elixir_home}" == /* ]]; then
ABS_ELIXIR_HOME="{elixir_home}"
else
ABS_ELIXIR_HOME=$PWD/{elixir_home}
fi
ABS_OUT_PATH=$PWD/{out}
export PATH="$ABS_ELIXIR_HOME"/bin:"{erlang_home}"/bin:${{PATH}}
export LANG="en_US.UTF-8"
export LC_ALL="en_US.UTF-8"
MIX_INVOCATION_DIR="{mix_invocation_dir}"
{copy_srcs_commands}
cd ${{MIX_INVOCATION_DIR}}
export HOME=${{PWD}}
export MIX_ENV=prod
export ERL_COMPILER_OPTIONS=deterministic
"${{ABS_ELIXIR_HOME}}"/bin/mix archive.build -o ${{ABS_OUT_PATH}}
# remove symlinks from the _build directory since it
# is an unused output, and bazel does not allow them
find . -type l -delete
""".format(
maybe_install_erlang = maybe_install_erlang(ctx),
erlang_home = erlang_home,
elixir_home = elixir_home,
mix_invocation_dir = mix_invocation_dir.path,
copy_srcs_commands = "\n".join(copy_srcs_commands),
package_dir = package_dir,
out = out.path,
)
inputs = depset(
direct = ctx.files.srcs,
transitive = [
erlang_runfiles.files,
elixir_runfiles.files,
],
)
ctx.actions.run_shell(
inputs = inputs,
outputs = [
out,
mix_invocation_dir,
],
command = script,
mnemonic = "MIX",
)
return [
DefaultInfo(
files = depset([out]),
),
]
mix_archive_build = rule(
implementation = _impl,
attrs = {
"srcs": attr.label_list(
mandatory = True,
allow_files = True,
),
"out": attr.output(),
},
toolchains = [
":toolchain_type",
],
)

View File

@ -23,9 +23,13 @@ rabbitmqctl(
"@observer_cli//:erlang_app",
"@stdout_formatter//:erlang_app",
],
archives = [
"@hex//:archive",
],
source_deps = {
"@csv//:sources": "csv",
"@json//:sources": "json",
"@parallel_stream//:sources": "parallel_stream", # transitive dep of csv 2.x
},
)
@ -128,9 +132,17 @@ rabbitmqctl_test(
"@observer_cli//:erlang_app",
"@stdout_formatter//:erlang_app",
],
archives = [
"@hex//:archive",
],
source_deps = {
"@amqp//:sources": "amqp",
"@csv//:sources": "csv",
"@dialyxir//:sources": "dialyxir",
"@json//:sources": "json",
"@parallel_stream//:sources": "parallel_stream", # transitive dep of csv 2.x
"@temp//:sources": "temp",
"@x509//:sources": "x509",
},
)

View File

@ -1,11 +1,16 @@
PROJECT = rabbitmq_cli
BUILD_DEPS = rabbit_common
DEPS = csv json observer_cli stdout_formatter
TEST_DEPS = amqp_client rabbit
DEPS = csv json observer_cli parallel_stream stdout_formatter
TEST_DEPS = amqp amqp_client dialyxir temp x509 rabbit
dep_amqp = hex 2.1.1
dep_csv = hex 2.4.1
dep_dialyxir = hex 0.5.1
dep_parallel_stream = hex 1.0.6
dep_json = hex 1.4.1
dep_temp = hex 0.4.7
dep_x509 = hex 0.7.0
DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk
DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk

View File

@ -15,7 +15,7 @@ defmodule RabbitMQCtl.MixfileBase do
build_embedded: Mix.env() == :prod,
start_permanent: Mix.env() == :prod,
escript: [main_module: RabbitMQCtl, emu_args: "-hidden", path: "escript/rabbitmqctl"],
deps: deps(),
deps: deps(Mix.env()),
aliases: aliases(),
xref: [
exclude: [
@ -122,7 +122,7 @@ defmodule RabbitMQCtl.MixfileBase do
# from Hex.pm in RabbitMQ source archive (the source archive must be
# self-contained and RabbitMQ must be buildable offline). However, we
# don't have the equivalent for other methods.
defp deps() do
defp deps(env) do
deps_dir = System.get_env("DEPS_DIR", "deps")
# Mix is confused by any `rebar.{config,lock}` we might have left in
@ -146,6 +146,10 @@ defmodule RabbitMQCtl.MixfileBase do
:csv,
path: Path.join(deps_dir, "csv")
},
{
:parallel_stream,
path: Path.join(deps_dir, "parallel_stream"), override: true
},
{
:stdout_formatter,
path: Path.join(deps_dir, "stdout_formatter"),
@ -156,24 +160,43 @@ defmodule RabbitMQCtl.MixfileBase do
path: Path.join(deps_dir, "observer_cli"),
compile: if(is_bazel, do: false, else: make_cmd)
},
{:amqp, "~> 2.1.0", only: :test},
{:dialyxir, "~> 0.5", only: :test, runtime: false},
{:temp, "~> 0.4", only: :test},
{:x509, "~> 0.7", only: :test},
{
:rabbit_common,
path: Path.join(deps_dir, "rabbit_common"),
compile: if(is_bazel, do: false, else: make_cmd),
override: true
},
{
:amqp_client,
path: Path.join(deps_dir, "amqp_client"),
compile: if(is_bazel, do: false, else: make_cmd),
override: true,
only: :test
}
]
] ++
case env do
:test ->
[
{
:amqp,
path: Path.join(deps_dir, "amqp")
},
{
:dialyxir,
path: Path.join(deps_dir, "dialyxir"), runtime: false
},
{
:temp,
path: Path.join(deps_dir, "temp")
},
{
:x509,
path: Path.join(deps_dir, "x509")
},
{
:amqp_client,
path: Path.join(deps_dir, "amqp_client"),
compile: if(is_bazel, do: false, else: make_cmd),
override: true
}
]
_ ->
[]
end
end
defp aliases do

View File

@ -49,7 +49,7 @@ def deps_dir_contents(ctx, deps, dir):
output = f,
target_file = src,
)
files.append(f)
files.extend([src, f])
return files
def _impl(ctx):
@ -60,7 +60,6 @@ def _impl(ctx):
ebin = ctx.actions.declare_directory("ebin")
consolidated = ctx.actions.declare_directory("consolidated")
mix_invocation_dir = ctx.actions.declare_directory("{}_mix".format(ctx.label.name))
fetched_srcs = ctx.actions.declare_file("deps.tar")
deps = flat_deps(ctx.attr.deps)
@ -100,7 +99,6 @@ fi
ABS_EBIN_DIR=$PWD/{ebin_dir}
ABS_CONSOLIDATED_DIR=$PWD/{consolidated_dir}
ABS_ESCRIPT_PATH=$PWD/{escript_path}
ABS_FETCHED_SRCS=$PWD/{fetched_srcs}
export PATH="$ABS_ELIXIR_HOME"/bin:"{erlang_home}"/bin:${{PATH}}
@ -113,17 +111,18 @@ cp -r {package_dir}/config ${{MIX_INVOCATION_DIR}}/config
cp -r {package_dir}/lib ${{MIX_INVOCATION_DIR}}/lib
cp {package_dir}/mix.exs ${{MIX_INVOCATION_DIR}}/mix.exs
ORIGINAL_DIR=$PWD
cd ${{MIX_INVOCATION_DIR}}
export IS_BAZEL=true
export HOME=${{PWD}}
export DEPS_DIR=$(dirname $ABS_EBIN_DIR)/{deps_dir}
export MIX_ENV=prod
export ERL_COMPILER_OPTIONS=deterministic
"${{ABS_ELIXIR_HOME}}"/bin/mix local.hex --force
"${{ABS_ELIXIR_HOME}}"/bin/mix local.rebar --force
"${{ABS_ELIXIR_HOME}}"/bin/mix deps.get --only prod
for archive in {archives}; do
"${{ABS_ELIXIR_HOME}}"/bin/mix archive.install --force $ORIGINAL_DIR/$archive
done
for d in {precompiled_deps}; do
mkdir _build/${{MIX_ENV}}/lib/$d
mkdir -p _build/${{MIX_ENV}}/lib/$d
ln -s ${{DEPS_DIR}}/$d/ebin _build/${{MIX_ENV}}/lib/$d
ln -s ${{DEPS_DIR}}/$d/include _build/${{MIX_ENV}}/lib/$d
done
@ -136,9 +135,6 @@ cp escript/rabbitmqctl ${{ABS_ESCRIPT_PATH}}
cp _build/${{MIX_ENV}}/lib/rabbitmqctl/ebin/* ${{ABS_EBIN_DIR}}
cp _build/${{MIX_ENV}}/lib/rabbitmqctl/consolidated/* ${{ABS_CONSOLIDATED_DIR}}
tar --file ${{ABS_FETCHED_SRCS}} \\
--create deps
# remove symlinks from the _build directory since it
# is not used, and bazel does not allow them
find . -type l -delete
@ -152,7 +148,7 @@ find . -type l -delete
escript_path = escript.path,
ebin_dir = ebin.path,
consolidated_dir = consolidated.path,
fetched_srcs = fetched_srcs.path,
archives = "".join([a.path for a in ctx.files.archives]),
precompiled_deps = " ".join([
dep[ErlangAppInfo].app_name
for dep in ctx.attr.deps
@ -164,6 +160,7 @@ find . -type l -delete
transitive = [
erlang_runfiles.files,
elixir_runfiles.files,
depset(ctx.files.archives),
depset(deps_dir_files),
],
)
@ -175,7 +172,6 @@ find . -type l -delete
ebin,
consolidated,
mix_invocation_dir,
fetched_srcs,
],
command = script,
mnemonic = "MIX",
@ -192,7 +188,7 @@ find . -type l -delete
return [
DefaultInfo(
executable = escript,
files = depset([ebin, consolidated, fetched_srcs]),
files = depset([ebin, consolidated]),
runfiles = runfiles,
),
ElixirAppInfo(
@ -224,6 +220,9 @@ rabbitmqctl_private = rule(
"deps": attr.label_list(
providers = [ErlangAppInfo],
),
"archives": attr.label_list(
allow_files = [".ez"],
),
"source_deps": attr.label_keyed_string_dict(),
},
toolchains = [

View File

@ -96,11 +96,11 @@ export HOME=${{PWD}}
export DEPS_DIR=$TEST_SRCDIR/$TEST_WORKSPACE/{package_dir}/{deps_dir}
export MIX_ENV=test
export ERL_COMPILER_OPTIONS=deterministic
"${{ABS_ELIXIR_HOME}}"/bin/mix local.hex --force
"${{ABS_ELIXIR_HOME}}"/bin/mix local.rebar --force
"${{ABS_ELIXIR_HOME}}"/bin/mix deps.get
for archive in {archives}; do
"${{ABS_ELIXIR_HOME}}"/bin/mix archive.install --force $INITIAL_DIR/$archive
done
for d in {precompiled_deps}; do
mkdir _build/${{MIX_ENV}}/lib/$d
mkdir -p _build/${{MIX_ENV}}/lib/$d
ln -s ${{DEPS_DIR}}/$d/ebin _build/${{MIX_ENV}}/lib/$d
ln -s ${{DEPS_DIR}}/$d/include _build/${{MIX_ENV}}/lib/$d
done
@ -138,6 +138,7 @@ set -x
elixir_home = elixir_home,
package_dir = package_dir,
deps_dir = deps_dir,
archives = "".join([a.short_path for a in ctx.files.archives]),
precompiled_deps = precompiled_deps,
rabbitmq_run_cmd = ctx.attr.rabbitmq_run[DefaultInfo].files_to_run.executable.short_path,
)
@ -167,9 +168,11 @@ set DEPS_DIR=%TEST_SRCDIR%/%TEST_WORKSPACE%/{package_dir}/{deps_dir}
set DEPS_DIR=%DEPS_DIR:/=\\%
set ERL_COMPILER_OPTIONS=deterministic
set MIX_ENV=test
echo y | "{elixir_home}\\bin\\mix" local.hex --force || goto :error
echo y | "{elixir_home}\\bin\\mix" local.rebar --force || goto :error
"{elixir_home}\\bin\\mix" deps.get || goto :error
for %%a in ({archives}) do (
set ARCH=%TEST_SRCDIR%/%TEST_WORKSPACE%/%%a
set ARCH=%ARCH:/=\\%
"{elixir_home}\\bin\\mix" archive.install --force %ARCH% || goto :error
)
for %%d in ({precompiled_deps}) do (
mkdir _build\\%MIX_ENV%\\lib\\%%d
robocopy %DEPS_DIR%\\%%d\\ebin _build\\%MIX_ENV%\\lib\\%%d /E /NFL /NDL /NJH /NJS /nc /ns /np
@ -192,6 +195,7 @@ exit /b 1
elixir_home = windows_path(elixir_home),
package_dir = windows_path(ctx.label.package),
deps_dir = deps_dir,
archives = "".join([a.short_path for a in ctx.files.archives]),
precompiled_deps = precompiled_deps,
rabbitmq_run_cmd = ctx.attr.rabbitmq_run[DefaultInfo].files_to_run.executable.short_path,
)
@ -202,7 +206,7 @@ exit /b 1
)
runfiles = ctx.runfiles(
files = ctx.files.srcs + ctx.files.data,
files = ctx.files.srcs + ctx.files.data + ctx.files.archives,
transitive_files = depset(deps_dir_files),
).merge_all([
erlang_runfiles,
@ -222,6 +226,9 @@ rabbitmqctl_private_test = rule(
"srcs": attr.label_list(allow_files = [".ex", ".exs"]),
"data": attr.label_list(allow_files = True),
"deps": attr.label_list(providers = [ErlangAppInfo]),
"archives": attr.label_list(
allow_files = [".ez"],
),
"source_deps": attr.label_keyed_string_dict(),
"rabbitmq_run": attr.label(
executable = True,