Check additional applications when comparing bazel and make results (#8209)

* Check additional applications when comparing bazel and make results

* Sync bazel/make for amqp_client

* Do not fail-fast in build system comparison

* promethus -> prometheus

* Regenerate BUILD.redbug

* When comparing build systems & .app files ignore empty 'registered'

It's listed as a required key in
https://www.erlang.org/doc/man/app.html, but the same docs state the
default is "[]". It seems to ignore it if it's empty.

* Copy bazel/BUILD.osiris from BUILD.bazel in the osiris repo

Normally it would be generated with `bazel run gazelle-update-repos --
-args osiris@1.5.1=github.com/rabbitmq/osiris@v1.5.1`, but in this
case we just want to match it's compilation with erlang.mk with some
manual tweaks.

* Use elixir 1.15, otherwise mix format fails

* Sync bazel/make for rabbitmq_web_dispatch, rabbitmq_management_agent

.github/workflows/check-build-system-equivalence.yaml

@@ -4,10 +4,11 @@ on:
   - cron: '0 2 * * *'
   workflow_dispatch:
 env:
-  erlang_version: 25.3
+  erlang_version: 26.0
-  elixir_version: 1.14
+  elixir_version: 1.15
   VERSION: 3.13.0
   PLUGINS: amqp10_common amqp10_client rabbitmq_amqp1_0 rabbitmq_auth_backend_cache rabbitmq_auth_backend_http rabbitmq_auth_backend_ldap rabbitmq_auth_backend_oauth2 rabbitmq_auth_mechanism_ssl rabbitmq_consistent_hash_exchange rabbitmq_event_exchange rabbitmq_federation rabbitmq_jms_topic_exchange rabbitmq_mqtt rabbitmq_random_exchange rabbitmq_recent_history_exchange rabbitmq_sharding rabbitmq_shovel rabbitmq_stomp rabbitmq_stream rabbitmq_trust_store rabbitmq_web_dispatch rabbitmq_management_agent rabbitmq_management rabbitmq_prometheus rabbitmq_federation_management rabbitmq_shovel_management rabbitmq_stream_management rabbitmq_top rabbitmq_tracing rabbitmq_web_mqtt rabbitmq_web_mqtt_examples rabbitmq_web_stomp rabbitmq_web_stomp_examples rabbitmq_aws rabbitmq_peer_discovery_common rabbitmq_peer_discovery_aws rabbitmq_peer_discovery_k8s rabbitmq_peer_discovery_consul rabbitmq_peer_discovery_etcd
+  EXTRA_PLUGINS: accept amqp_client aten base64url cowboy cowlib credentials_obfuscation cuttlefish eetcd enough gen_batch_server getopt gun jose observer_cli osiris prometheus quantile_estimator ra ranch recon redbug seshat stdout_formatter syslog sysmon_handler systemd thoas
 jobs:
   build-with-bazel:
     name: bazel build package-generic-unix.tar.xz

MODULE.bazel

@@ -268,6 +268,7 @@ erlang_package.hex_package(
 )
 
 erlang_package.git_package(
+    build_file = "@rabbitmq-server//bazel:BUILD.osiris",
     repository = "rabbitmq/osiris",
     tag = "v1.6.0",
 )

bazel/BUILD.osiris

@@ -0,0 +1,144 @@
+load("@rules_erlang//:erlang_app.bzl", "erlang_app", "test_erlang_app")
+load("@rules_erlang//:xref2.bzl", "xref")
+load("@rules_erlang//:dialyze.bzl", "DEFAULT_PLT_APPS", "dialyze", "plt")
+load("@rules_erlang//:eunit2.bzl", "eunit")
+load("@rules_erlang//:ct.bzl", "assert_suites2", "ct_suite")
+
+NAME = "osiris"
+
+VERSION = "1.5.1"
+
+DESCRIPTION = "New project"
+
+APP_ENV = """[
+	{data_dir, "/tmp/osiris"},
+	{port_range, {6000, 6500}},
+	{max_segment_size_chunks, 256000},
+	{replication_transport, tcp},
+	{replica_forced_gc_default_interval, 4999}
+]"""
+
+EXTRA_APPS = [
+    "sasl",
+    "crypto",
+]
+
+DEPS = [
+    "@gen_batch_server//:erlang_app",
+]
+
+RUNTIME_DEPS = [
+    "@seshat//:erlang_app",
+]
+
+erlang_app(
+    app_description = DESCRIPTION,
+    app_env = APP_ENV,
+    app_name = NAME,
+    app_version = VERSION,
+    extra_apps = EXTRA_APPS,
+    runtime_deps = RUNTIME_DEPS,
+    deps = DEPS,
+)
+
+test_erlang_app(
+    app_description = DESCRIPTION,
+    app_env = APP_ENV,
+    app_name = NAME,
+    app_version = VERSION,
+    extra_apps = EXTRA_APPS,
+    runtime_deps = RUNTIME_DEPS,
+    deps = DEPS,
+)
+
+xref()
+
+PLT_APPS = DEFAULT_PLT_APPS + [
+    "compiler",
+    "crypto",
+    "tools",
+    "runtime_tools",
+    "mnesia",
+    "public_key",
+    "asn1",
+    "ssl",
+    "inets",
+]
+
+plt(
+    name = "deps_plt",
+    apps = PLT_APPS,
+    for_target = ":erlang_app",
+)
+
+dialyze(
+    name = "dialyze",
+    size = "small",
+    dialyzer_opts = [
+        "-Werror_handling",
+        "-Wunmatched_returns",
+    ],
+    plt = ":deps_plt",
+    target = ":erlang_app",
+)
+
+plt(
+    name = "test_deps_plt",
+    apps = PLT_APPS + [
+        "eunit",
+        "common_test",
+        "debugger",
+        "xmerl",
+        "ftp",
+        "ssh",
+        "snmp",
+        "wx",
+        "syntax_tools",
+    ],
+    for_target = ":test_erlang_app",
+)
+
+dialyze(
+    name = "dialyze_tests",
+    size = "small",
+    beam = [
+        f.replace("test/", "").replace(".erl", "_beam_files")
+        for f in glob(["test/*_SUITE.erl"])
+    ],
+    dialyzer_opts = [
+        "-Werror_handling",
+        "-Wunmatched_returns",
+    ],
+    plt = ":test_deps_plt",
+)
+
+eunit(
+    name = "eunit",
+    eunit_opts = [
+        "no_tty",
+        "{report, {eunit_progress, [colored, profile]}}",
+    ],
+    target = ":test_erlang_app",
+    deps = [
+        "@eunit_formatters//:erlang_app",
+    ],
+)
+
+[ct_suite(
+    name = f.replace("test/", "").replace(".erl", ""),
+) for f in glob(
+    ["test/*_SUITE.erl"],
+    exclude = ["test/osiris_SUITE.erl"],
+)]
+
+ct_suite(
+    name = "osiris_SUITE",
+    data = [
+        "@tls_gen//:basic",
+    ],
+    test_env = {
+        "DEPS_DIR": "$TEST_SRCDIR/$TEST_WORKSPACE/external",
+    },
+)
+
+assert_suites2()

bazel/BUILD.redbug

@@ -25,17 +25,9 @@ erlang_bytecode(
         "src/redbug_parser.erl",
         "src/redbug_targ.erl",
     ],
-    outs = [
+    hdrs = [":public_and_private_hdrs"],
-        "ebin/redbug.beam",
-        "ebin/redbug_compiler.beam",
-        "ebin/redbug_dtop.beam",
-        "ebin/redbug_lexer.beam",
-        "ebin/redbug_parser.beam",
-        "ebin/redbug_targ.beam",
-    ],
-    hdrs = [],
     app_name = "redbug",
-    beam = [],
+    dest = "ebin",
     erlc_opts = "//:erlc_opts",
 )
 
@@ -57,20 +49,11 @@ filegroup(
     ],
 )
 
-filegroup(
+filegroup(name = "private_hdrs")
-    name = "private_hdrs",
-    srcs = [],
-)
 
-filegroup(
+filegroup(name = "public_hdrs")
-    name = "public_hdrs",
-    srcs = [],
-)
 
-filegroup(
+filegroup(name = "priv")
-    name = "priv",
-    srcs = [],
-)
 
 filegroup(
     name = "licenses",
@@ -96,9 +79,12 @@ filegroup(
 erlang_app(
     name = "erlang_app",
     srcs = [":all_srcs"],
+    hdrs = [":public_hdrs"],
     app_name = "redbug",
     beam_files = [":beam_files"],
     extra_apps = ["runtime_tools"],
+    license_files = [":license_files"],
+    priv = [":priv"],
 )
 
 alias(
@@ -106,3 +92,10 @@ alias(
     actual = ":erlang_app",
     visibility = ["//visibility:public"],
 )
+
+filegroup(
+    name = "license_files",
+    srcs = [
+        "LICENSE",
+    ],
+)

deps/amqp_client/BUILD.bazel

@@ -29,7 +29,7 @@ APP_EXTRA_KEYS = """%% Hex.pm package informations.
 	{licenses, ["MPL-2.0"]},
 	{links, [
 	    {"Website", "https://www.rabbitmq.com/"},
-	    {"GitHub", "https://github.com/rabbitmq/rabbitmq-server/deps/amqp_client"},
+	    {"GitHub", "https://github.com/rabbitmq/rabbitmq-server/tree/main/deps/amqp_client"},
 	    {"User guide", "https://www.rabbitmq.com/erlang-client-user-guide.html"}
 	  ]},
 	{build_tools, ["make", "rebar3"]},
@@ -41,7 +41,8 @@ APP_EXTRA_KEYS = """%% Hex.pm package informations.
 	    "Makefile",
 	    "rabbitmq-components.mk",
 	    "README",
-	    "README.md"
+	    "README.md",
+	    "src"
 	  ]}
 """
 

deps/amqp_client/Makefile

@@ -39,7 +39,7 @@ endef
 # Release artifacts are put in $(PACKAGES_DIR).
 PACKAGES_DIR ?= $(abspath PACKAGES)
 
-LOCAL_DEPS = xmerl
+LOCAL_DEPS = xmerl ssl public_key
 DEPS = rabbit_common credentials_obfuscation
 TEST_DEPS = rabbitmq_ct_helpers rabbit meck
 

deps/rabbitmq_management_agent/Makefile

@@ -17,7 +17,7 @@ define PROJECT_APP_EXTRA_KEYS
 	{broker_version_requirements, []}
 endef
 
-DEPS = rabbit_common rabbit
+DEPS = rabbit_common rabbit rabbitmq_web_dispatch
 TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers
 LOCAL_DEPS += xmerl mnesia ranch ssl crypto public_key
 DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk

deps/rabbitmq_web_dispatch/Makefile

@@ -7,7 +7,7 @@ define PROJECT_APP_EXTRA_KEYS
 endef
 
 LOCAL_DEPS = inets
-DEPS = rabbit_common rabbit cowboy
+DEPS = amqp_client rabbit_common rabbit cowboy
 TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers
 
 DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk

tools/compare_dist.sh

@@ -1,44 +1,62 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -uo pipefail
 
 GOLDEN=$1
 SECOND=$2
 
+failure_count=0
+
 echo "Check both have INSTALL"
-test -f $GOLDEN/rabbitmq_server-${VERSION}/INSTALL
+test -f $GOLDEN/rabbitmq_server-${VERSION}/INSTALL || ((failure_count++))
-test -f $SECOND/rabbitmq_server-${VERSION}/INSTALL
+test -f $SECOND/rabbitmq_server-${VERSION}/INSTALL || ((failure_count++))
 
 echo "Check LICENSEs"
 diff \
     <(grep LICENSE make.manifest) \
-    <(grep LICENSE bazel.manifest | grep -v ".md" | grep -v ".txt")
+    <(grep LICENSE bazel.manifest | grep -v ".md" | grep -v ".txt") \
+         || ((failure_count++))
 
 echo "Check plugins"
 plugins_rel=rabbitmq_server-${VERSION}/plugins
-diff <(grep $plugins_rel make.manifest | grep -v ".ez") <(grep $plugins_rel bazel.manifest | grep -v ".ez")
+diff \
+    <(grep $plugins_rel make.manifest | grep -v ".ez") \
+    <(grep $plugins_rel bazel.manifest | grep -v ".ez") \
+         || ((failure_count++))
 
 echo "Plugins exist with same version and deps"
-for p in ${PLUGINS}; do
+for p in ${PLUGINS} ${EXTRA_PLUGINS}; do
     echo "$p"
     f="$(cd $GOLDEN && ls -d $plugins_rel/$p-*)"
-    test -f $GOLDEN/$f/ebin/$p.app || (echo "$GOLDEN/$f/ebin/$p.app does not exist"; exit 1)
+    test -f $GOLDEN/$f/ebin/$p.app || (echo "$GOLDEN/$f/ebin/$p.app does not exist"; ((failure_count++)))
-    test -d $SECOND/$f || (echo "$SECOND/$f does not exist"; exit 1)
+    test -d $SECOND/$f || (echo "$SECOND/$f does not exist"; ((failure_count++)))
-    test -f $SECOND/$f/ebin/$p.app || (echo "$SECOND/$f/ebin/$p.app does not exist"; exit 1)
+    test -f $SECOND/$f/ebin/$p.app || (echo "$SECOND/$f/ebin/$p.app does not exist"; ((failure_count++)))
     ./rabbitmq-server/tools/erlang_app_equal \
         $GOLDEN/$f/ebin/$p.app \
-    $SECOND/$f/ebin/$p.app
+        $SECOND/$f/ebin/$p.app \
+             || ((failure_count++))
 done
 
 echo "Both have escript"
 escript_rel=rabbitmq_server-${VERSION}/escript
-diff <(grep $escript_rel make.manifest) <(grep $escript_rel bazel.manifest)
+diff \
+    <(grep $escript_rel make.manifest) \
+    <(grep $escript_rel bazel.manifest) \
+         || ((failure_count++))
 
 echo "Both have sbin"
 sbin_rel=rabbitmq_server-${VERSION}/sbin
-diff <(grep $sbin_rel make.manifest) <(grep $sbin_rel bazel.manifest)
+diff \
+    <(grep $sbin_rel make.manifest) \
+    <(grep $sbin_rel bazel.manifest) \
+         || ((failure_count++))
 
 echo "Both have manpages"
 manpages_rel=rabbitmq_server-${VERSION}/share/man
-diff <(grep $manpages_rel make.manifest) <(grep $manpages_rel bazel.manifest)
+diff \
+    <(grep $manpages_rel make.manifest) \
+    <(grep $manpages_rel bazel.manifest) \
+         || ((failure_count++))
 
-echo "PASS"
+echo "There were $failure_count failures."
+
+exit $failure_count

tools/erlang_app_equal

@@ -28,8 +28,15 @@ compare(LeftMetadata, RightMetadata) ->
                       "Warning:\t 'id' does not match (~p != ~p)~n", [LeftId, RightId])
     end,
 
-    LeftPropsMap = proplists:to_map(proplists:delete(id, LeftProps)),
+    FilterEmptyRegistered = fun
-    RightPropsMap = proplists:to_map(proplists:delete(id, RightProps)),
+        (registered, []) -> false;
+        (_, _) -> true
+    end,
+
+    LeftPropsMap = maps:filter(FilterEmptyRegistered,
+                               proplists:to_map(proplists:delete(id, LeftProps))),
+    RightPropsMap = maps:filter(FilterEmptyRegistered,
+                                proplists:to_map(proplists:delete(id, RightProps))),
     assert_equal(
         lists:sort(maps:keys(LeftPropsMap)),
         lists:sort(maps:keys(RightPropsMap)),