root
/
rabbitmq-server
mirror of https://github.com/rabbitmq/rabbitmq-server.git
1
0
Fork 0
Code Issues Actions 7 Packages Projects Releases Wiki Activity

make: Additional cleanup of RabbitMQ components 

Because of the monorepo most components do not need to be
listed. Only the community plugins and third party dependencies.

Community plugins can now be fetched and acted on from the top
level Makefile by adding COMMUNITY_PLUGINS=1 to the command line
or the environment. This will fetch and build community plugins:

  make COMMUNITY_PLUGINS=1

Once fetched they can be targeted directly as usual:

  make -C deps/rabbitmq_metronome

This cleanup has a net positive effect on build performance,
especially the performance of the top-level Makefile:

  make nope  0,04s user 0,02s system 106% cpu 0,061 total
  make nope  0,02s user 0,01s system 104% cpu 0,033 total

But also a minor improvement for application Makefiles:

  make -C deps/rabbit nope  0,02s user 0,00s system 98% cpu 0,022 total
  make -C deps/rabbit nope  0,01s user 0,00s system 98% cpu 0,020 total

And that improvement adds up when going through dependencies:

  make -C deps/rabbitmq_management  0,59s user 0,23s system 100% cpu 0,808 total
  make -C deps/rabbitmq_management  0,60s user 0,19s system 101% cpu 0,780 total
This commit is contained in:
Loïc Hoguin 2024-07-24 15:56:29 +02:00
parent d4222f8216
commit 5aefd919d3
No known key found for this signature in database
GPG Key ID: C69E26E3A9DF618F
2 changed files with 36 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Makefile
View File

@ -60,6 +60,20 @@ include rabbitmq-components.mk
# multiple times (including for release file names and whatnot).
PROJECT_VERSION := $(PROJECT_VERSION)
# Fetch/build community plugins.
#
# To include community plugins in commands, use
# `make COMMUNITY_PLUGINS=1` or export the variable.
# They are not included otherwise. Note that only
# the top-level Makefile can do this.
#
# Note that the community plugins will be fetched using
# SSH and therefore may be subject to GH authentication.
ifdef COMMUNITY_PLUGINS
DEPS += $(RABBITMQ_COMMUNITY)
endif
include erlang.mk
include mk/github-actions.mk
include mk/bazel.mk

234
rabbitmq-components.mk
View File

@ -34,84 +34,6 @@ endif
# RabbitMQ components.
# --------------------------------------------------------------------
# For RabbitMQ repositories, we want to checkout branches which match
# the parent project. For instance, if the parent project is on a
# release tag, dependencies must be on the same release tag. If the
# parent project is on a topic branch, dependencies must be on the same
# topic branch or fallback to `stable` or `main` whichever was the
# base of the topic branch.
dep_amqp_client = git_rmq-subfolder rabbitmq-erlang-client $(current_rmq_ref) $(base_rmq_ref) main
dep_amqp10_client = git_rmq-subfolder rabbitmq-amqp1.0-client $(current_rmq_ref) $(base_rmq_ref) main
dep_oauth2_client = git_rmq-subfolder oauth2-client $(current_rmq_ref) $(base_rmq_ref) main
dep_amqp10_common = git_rmq-subfolder rabbitmq-amqp1.0-common $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbit = git_rmq-subfolder rabbitmq-server $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbit_common = git_rmq-subfolder rabbitmq-common $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_amqp1_0 = git_rmq-subfolder rabbitmq-amqp1.0 $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_auth_backend_amqp = git_rmq rabbitmq-auth-backend-amqp $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_auth_backend_cache = git_rmq-subfolder rabbitmq-auth-backend-cache $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_auth_backend_http = git_rmq-subfolder rabbitmq-auth-backend-http $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_auth_backend_ldap = git_rmq-subfolder rabbitmq-auth-backend-ldap $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_auth_backend_oauth2 = git_rmq-subfolder rabbitmq-auth-backend-oauth2 $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_auth_mechanism_ssl = git_rmq-subfolder rabbitmq-auth-mechanism-ssl $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_aws = git_rmq-subfolder rabbitmq-aws $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_boot_steps_visualiser = git_rmq rabbitmq-boot-steps-visualiser $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_cli = git_rmq-subfolder rabbitmq-cli $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_codegen = git_rmq-subfolder rabbitmq-codegen $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_consistent_hash_exchange = git_rmq-subfolder rabbitmq-consistent-hash-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_ct_client_helpers = git_rmq-subfolder rabbitmq-ct-client-helpers $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_ct_helpers = git_rmq-subfolder rabbitmq-ct-helpers $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_delayed_message_exchange = git_rmq rabbitmq-delayed-message-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_dotnet_client = git_rmq rabbitmq-dotnet-client $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_event_exchange = git_rmq-subfolder rabbitmq-event-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_federation = git_rmq-subfolder rabbitmq-federation $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_federation_management = git_rmq-subfolder rabbitmq-federation-management $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_federation_prometheus = git_rmq-subfolder rabbitmq-federation-prometheus $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_java_client = git_rmq rabbitmq-java-client $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_jms_client = git_rmq rabbitmq-jms-client $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_jms_cts = git_rmq rabbitmq-jms-cts $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_jms_topic_exchange = git_rmq-subfolder rabbitmq-jms-topic-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_lvc_exchange = git_rmq rabbitmq-lvc-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_management = git_rmq-subfolder rabbitmq-management $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_management_agent = git_rmq-subfolder rabbitmq-management-agent $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_management_exchange = git_rmq rabbitmq-management-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_management_themes = git_rmq rabbitmq-management-themes $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_message_timestamp = git_rmq rabbitmq-message-timestamp $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_metronome = git_rmq rabbitmq-metronome $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_mqtt = git_rmq-subfolder rabbitmq-mqtt $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_objc_client = git_rmq rabbitmq-objc-client $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_peer_discovery_aws = git_rmq-subfolder rabbitmq-peer-discovery-aws $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_peer_discovery_common = git_rmq-subfolder rabbitmq-peer-discovery-common $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_peer_discovery_consul = git_rmq-subfolder rabbitmq-peer-discovery-consul $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_peer_discovery_etcd = git_rmq-subfolder rabbitmq-peer-discovery-etcd $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_peer_discovery_k8s = git_rmq-subfolder rabbitmq-peer-discovery-k8s $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_prelaunch = git_rmq-subfolder rabbitmq-prelaunch $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_prometheus = git_rmq-subfolder rabbitmq-prometheus $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_random_exchange = git_rmq-subfolder rabbitmq-random-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_recent_history_exchange = git_rmq-subfolder rabbitmq-recent-history-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_routing_node_stamp = git_rmq rabbitmq-routing-node-stamp $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_rtopic_exchange = git_rmq rabbitmq-rtopic-exchange $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_sharding = git_rmq-subfolder rabbitmq-sharding $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_shovel = git_rmq-subfolder rabbitmq-shovel $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_shovel_management = git_rmq-subfolder rabbitmq-shovel-management $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_shovel_prometheus = git_rmq-subfolder rabbitmq-shovel-prometheus $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_stomp = git_rmq-subfolder rabbitmq-stomp $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_stream = git_rmq-subfolder rabbitmq-stream $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_stream_common = git_rmq-subfolder rabbitmq-stream-common $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_stream_management = git_rmq-subfolder rabbitmq-stream-management $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_toke = git_rmq rabbitmq-toke $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_top = git_rmq-subfolder rabbitmq-top $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_tracing = git_rmq-subfolder rabbitmq-tracing $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_trust_store = git_rmq-subfolder rabbitmq-trust-store $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_test = git_rmq rabbitmq-test $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_web_dispatch = git_rmq-subfolder rabbitmq-web-dispatch $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_web_stomp = git_rmq-subfolder rabbitmq-web-stomp $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_web_stomp_examples = git_rmq-subfolder rabbitmq-web-stomp-examples $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_web_mqtt = git_rmq-subfolder rabbitmq-web-mqtt $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_web_mqtt_examples = git_rmq-subfolder rabbitmq-web-mqtt-examples $(current_rmq_ref) $(base_rmq_ref) main
dep_rabbitmq_website = git_rmq rabbitmq-website $(current_rmq_ref) $(base_rmq_ref) live main
dep_toke = git_rmq toke $(current_rmq_ref) $(base_rmq_ref) master
# Third-party dependencies version pinning.
#
# We do that in this file, which is included by all projects, to ensure
@ -198,16 +120,33 @@ RABBITMQ_BUILTIN = \
trust_store_http
# Applications outside of the monorepo maintained by Team RabbitMQ.
RABBITMQ_COMMUNITY = \
rabbitmq_auth_backend_amqp \
rabbitmq_boot_steps_visualiser \
rabbitmq_delayed_message_exchange \
rabbitmq_lvc_exchange \
rabbitmq_management_exchange \
rabbitmq_management_themes \
rabbitmq_message_timestamp \
rabbitmq_metronome \
rabbitmq_routing_node_stamp \
rabbitmq_rtopic_exchange
community_dep = git git@github.com:rabbitmq/$1.git $(if $2,$2,main)
dep_rabbitmq_auth_backend_amqp = $(call community_dep,rabbitmq-auth-backend-amqp)
dep_rabbitmq_boot_steps_visualiser = $(call community_dep,rabbitmq-boot-steps-visualiser,master)
dep_rabbitmq_delayed_message_exchange = $(call community_dep,rabbitmq-delayed-message-exchange)
dep_rabbitmq_lvc_exchange = $(call community_dep,rabbitmq-lvc-exchange)
dep_rabbitmq_management_exchange = $(call community_dep,rabbitmq-management-exchange)
dep_rabbitmq_management_themes = $(call community_dep,rabbitmq-management-themes,master)
dep_rabbitmq_message_timestamp = $(call community_dep,rabbitmq-message-timestamp)
dep_rabbitmq_metronome = $(call community_dep,rabbitmq-metronome,master)
dep_rabbitmq_routing_node_stamp = $(call community_dep,rabbitmq-routing-node-stamp)
dep_rabbitmq_rtopic_exchange = $(call community_dep,rabbitmq-rtopic-exchange)
# All RabbitMQ applications.
RABBITMQ_COMPONENTS = $(RABBITMQ_BUILTIN) $(RABBITMQ_COMMUNITY)
# Erlang.mk does not rebuild dependencies by default, once they were
@ -219,139 +158,10 @@ RABBITMQ_COMPONENTS = $(RABBITMQ_BUILTIN) $(RABBITMQ_COMMUNITY)
FORCE_REBUILD = $(RABBITMQ_COMPONENTS)
# Several components have a custom erlang.mk/build.config, mainly
# to disable eunit. Therefore, we can't use the top-level project's
# erlang.mk copy. Note that this is not needed for components that
# sit in the monorepo.
NO_AUTOPATCH += $(RABBITMQ_COMPONENTS)
ifeq ($(origin current_rmq_ref),undefined)
ifneq ($(wildcard .git),)
current_rmq_ref := $(shell (\
ref=$$(LANG=C git branch --list | awk '/^\* \(.*detached / {ref=$$0; sub(/.*detached [^ ]+ /, "", ref); sub(/\)$$/, "", ref); print ref; exit;} /^\* / {ref=$$0; sub(/^\* /, "", ref); print ref; exit}');\
if test "$$(git rev-parse --short HEAD)" != "$$ref"; then echo "$$ref"; fi))
else
current_rmq_ref := main
endif
endif
export current_rmq_ref
ifeq ($(origin base_rmq_ref),undefined)
ifneq ($(wildcard .git),)
possible_base_rmq_ref := main
ifeq ($(possible_base_rmq_ref),$(current_rmq_ref))
base_rmq_ref := $(current_rmq_ref)
else
base_rmq_ref := $(shell \
(git rev-parse --verify -q main >/dev/null && \
git rev-parse --verify -q $(possible_base_rmq_ref) >/dev/null && \
git merge-base --is-ancestor $$(git merge-base main HEAD) $(possible_base_rmq_ref) && \
echo $(possible_base_rmq_ref)) || \
echo main)
endif
else
base_rmq_ref := main
endif
endif
export base_rmq_ref
# Repository URL selection.
#
# First, we infer other components' location from the current project
# repository URL, if it's a Git repository:
# - We take the "origin" remote URL as the base
# - The current project name and repository name is replaced by the
# target's properties:
# eg. rabbitmq-common is replaced by rabbitmq-codegen
# eg. rabbit_common is replaced by rabbitmq_codegen
#
# If cloning from this computed location fails, we fallback to RabbitMQ
# upstream which is GitHub.
# Macro to transform eg. "rabbit_common" to "rabbitmq-common".
rmq_cmp_repo_name = $(word 2,$(dep_$(1)))
# Upstream URL for the current project.
RABBITMQ_COMPONENT_REPO_NAME := $(call rmq_cmp_repo_name,$(PROJECT))
# Current URL for the current project. If this is not a Git clone,
# default to the upstream Git repository.
ifneq ($(wildcard .git),)
git_origin_fetch_url := $(shell git config remote.origin.url)
git_origin_push_url := $(shell git config remote.origin.pushurl || git config remote.origin.url)
RABBITMQ_CURRENT_FETCH_URL ?= $(git_origin_fetch_url)
RABBITMQ_CURRENT_PUSH_URL ?= $(git_origin_push_url)
else
RABBITMQ_CURRENT_FETCH_URL ?= $(RABBITMQ_UPSTREAM_FETCH_URL)
RABBITMQ_CURRENT_PUSH_URL ?= $(RABBITMQ_UPSTREAM_PUSH_URL)
endif
# Macro to replace the following pattern:
# 1. /foo.git -> /bar.git
# 2. /foo -> /bar
# 3. /foo/ -> /bar/
subst_repo_name = $(patsubst %/$(1)/%,%/$(2)/%,$(patsubst %/$(1),%/$(2),$(patsubst %/$(1).git,%/$(2).git,$(3))))
# We need to provide a repo mapping for deps resolved via git_rmq fetch method
query_repo_git_rmq = https://github.com/rabbitmq/$(call rmq_cmp_repo_name,$(1))
# Macro to replace both the project's name (eg. "rabbit_common") and
# repository name (eg. "rabbitmq-common") by the target's equivalent.
#
# This macro is kept on one line because we don't want whitespaces in
# the returned value, as it's used in $(dep_fetch_git_rmq) in a shell
# single-quoted string.
dep_rmq_repo = $(if $(dep_$(2)),$(call subst_repo_name,$(PROJECT),$(2),$(call subst_repo_name,$(RABBITMQ_COMPONENT_REPO_NAME),$(call rmq_cmp_repo_name,$(2)),$(1))),$(pkg_$(1)_repo))
dep_rmq_commits = $(if $(dep_$(1)), \
$(wordlist 3,$(words $(dep_$(1))),$(dep_$(1))), \
$(pkg_$(1)_commit))
define dep_fetch_git_rmq
fetch_url1='$(call dep_rmq_repo,$(RABBITMQ_CURRENT_FETCH_URL),$(1))'; \
fetch_url2='$(call dep_rmq_repo,$(RABBITMQ_UPSTREAM_FETCH_URL),$(1))'; \
if test "$$$$fetch_url1" != '$(RABBITMQ_CURRENT_FETCH_URL)' && \
git clone -q -n -- "$$$$fetch_url1" $(DEPS_DIR)/$(call dep_name,$(1)); then \
fetch_url="$$$$fetch_url1"; \
push_url='$(call dep_rmq_repo,$(RABBITMQ_CURRENT_PUSH_URL),$(1))'; \
elif git clone -q -n -- "$$$$fetch_url2" $(DEPS_DIR)/$(call dep_name,$(1)); then \
fetch_url="$$$$fetch_url2"; \
push_url='$(call dep_rmq_repo,$(RABBITMQ_UPSTREAM_PUSH_URL),$(1))'; \
fi; \
cd $(DEPS_DIR)/$(call dep_name,$(1)) && ( \
$(foreach ref,$(call dep_rmq_commits,$(1)), \
git checkout -q $(ref) >/dev/null 2>&1 || \
) \
(echo "error: no valid pathspec among: $(call dep_rmq_commits,$(1))" \
1>&2 && false) ) && \
(test "$$$$fetch_url" = "$$$$push_url" || \
git remote set-url --push origin "$$$$push_url")
endef
define dep_fetch_git_rmq-subfolder
fetch_url1='https://github.com/rabbitmq/rabbitmq-server.git'; \
fetch_url2='git@github.com:rabbitmq/rabbitmq-server.git'; \
if [ ! -d $(ERLANG_MK_TMP)/rabbitmq-server ]; then \
if test "$$$$fetch_url1" != '$(RABBITMQ_CURRENT_FETCH_URL)' && \
git clone -q -n -- "$$$$fetch_url1" $(ERLANG_MK_TMP)/rabbitmq-server; then \
fetch_url="$$$$fetch_url1"; \
push_url='$(call dep_rmq_repo,$(RABBITMQ_CURRENT_PUSH_URL),rabbitmq-server)'; \
elif git clone -q -n -- "$$$$fetch_url2" $(ERLANG_MK_TMP)/rabbitmq-server; then \
fetch_url="$$$$fetch_url2"; \
push_url='$(call dep_rmq_repo,$(RABBITMQ_UPSTREAM_PUSH_URL),rabbitmq-server)'; \
fi; \
fi; \
cd $(ERLANG_MK_TMP)/rabbitmq-server && ( \
$(foreach ref,$(call dep_rmq_commits,$(1)), \
git checkout -q $(ref) >/dev/null 2>&1 || \
) \
(echo "error: no valid pathspec among: $(call dep_rmq_commits,$(1))" \
1>&2 && false) ) && \
(test "$$$$fetch_url" = "$$$$push_url" || \
git remote set-url --push origin "$$$$push_url")
ln -s $(ERLANG_MK_TMP)/rabbitmq-server/deps/$(call dep_name,$(1)) \
$(DEPS_DIR)/$(call dep_name,$(1));
endef
# We disable autopatching for community plugins as they sit in
# their own repository and we want to avoid polluting the git
# status with changes that should not be committed.
NO_AUTOPATCH += $(RABBITMQ_COMMUNITY)
# --------------------------------------------------------------------
# Component distribution.
@ -364,7 +174,7 @@ prepare-dist::
@:
# --------------------------------------------------------------------
# Monorepo-specific settings.
# RabbitMQ-specific settings.
# --------------------------------------------------------------------
# If the top-level project is a RabbitMQ component, we override