Add the option to build Redis with modules (#13524)

A new BUILD_WITH_MODULES flag was added to the Makefile to control
building the module directory.

The new module directory includes a general Makefile that iterates
over each module, fetch a specific version, and build it.

Co-authored-by: YaacovHazan <yaacov.hazan@redislabs.com>

Makefile

@@ -1,11 +1,16 @@
-# Top level makefile, the real stuff is at src/Makefile
+# Top level makefile, the real stuff is at ./src/Makefile and in ./modules/Makefile
+
+SUBDIRS = src
+ifeq ($(BUILD_WITH_MODULES), yes)
+	SUBDIRS += modules
+endif
 
 default: all
 
 .DEFAULT:
-	cd src && $(MAKE) $@
+	for dir in $(SUBDIRS); do $(MAKE) -C $$dir $@; done
 
 install:
-	cd src && $(MAKE) $@
+	for dir in $(SUBDIRS); do $(MAKE) -C $$dir $@; done
 
 .PHONY: install

modules/Makefile

@@ -0,0 +1,72 @@
+
+SUBDIRS = redisjson redistimeseries redisbloom redisearch
+
+define submake
+	for dir in $(SUBDIRS); do $(MAKE) -C $$dir $(1); done
+endef
+
+all: prepare_source
+	$(call submake,$@)
+
+get_source:
+	$(call submake,$@)
+
+prepare_source: get_source handle-werrors setup_environment
+
+clean:
+	$(call submake,$@)
+
+distclean: clean_environment
+	$(call submake,$@)
+
+pristine:
+	$(call submake,$@)
+
+install:
+	$(call submake,$@)
+
+setup_environment: install-rust handle-werrors
+
+clean_environment: uninstall-rust
+
+# Keep all of the Rust stuff in one place
+install-rust:
+ifeq ($(INSTALL_RUST_TOOLCHAIN),yes)
+	@RUST_VERSION=1.80.1; \
+	case "$$(uname -m)" in \
+		'x86_64') RUST_INSTALLER="rust-$${RUST_VERSION}-x86_64-unknown-linux-gnu"; RUST_SHA256="85e936d5d36970afb80756fa122edcc99bd72a88155f6bdd514f5d27e778e00a" ;; \
+		'aarch64') RUST_INSTALLER="rust-$${RUST_VERSION}-aarch64-unknown-linux-gnu"; RUST_SHA256="2e89bad7857711a1c11d017ea28fbfeec54076317763901194f8f5decbac1850" ;; \
+		*) echo >&2 "Unsupported architecture: '$$(uname -m)'"; exit 1 ;; \
+	esac; \
+	wget --quiet -O $${RUST_INSTALLER}.tar.xz https://static.rust-lang.org/dist/$${RUST_INSTALLER}.tar.xz; \
+	echo "$${RUST_SHA256} $${RUST_INSTALLER}.tar.xz" | sha256sum -c --quiet || { echo "Rust standalone installer checksum failed!"; exit 1; }; \
+	tar -xf $${RUST_INSTALLER}.tar.xz; \
+	(cd $${RUST_INSTALLER} && ./install.sh); \
+	rm -rf $${RUST_INSTALLER}
+endif
+
+uninstall-rust:
+ifeq ($(INSTALL_RUST_TOOLCHAIN),yes)
+	@if [ -x "/usr/local/lib/rustlib/uninstall.sh" ]; then \
+		echo "Uninstalling Rust using uninstall.sh script"; \
+		rm -rf ~/.cargo; \
+		/usr/local/lib/rustlib/uninstall.sh; \
+	else \
+		echo "WARNING: Rust toolchain not found or uninstall script is missing."; \
+	fi
+endif
+
+handle-werrors: get_source
+ifeq ($(DISABLE_WERRORS),yes)
+	@echo "Disabling -Werror for all modules"
+	@for dir in $(SUBDIRS); do \
+		echo "Processing $$dir"; \
+		find $$dir/src -type f \
+			\( -name "Makefile" \
+			-o -name "*.mk" \
+			-o -name "CMakeLists.txt" \) \
+			-exec sed -i 's/-Werror//g' {} +; \
+	done
+endif
+
+.PHONY: all clean distclean install $(SUBDIRS) setup_environment clean_environment install-rust uninstall-rust handle-werrors

modules/common.mk

@@ -0,0 +1,49 @@
+PREFIX ?= /usr/local
+INSTALL_DIR ?= $(DESTDIR)$(PREFIX)/lib/redis/modules
+INSTALL ?= install
+
+# This logic *partially* follows the current module build system. It is a bit awkward and
+# should be changed if/when the modules' build process is refactored.
+
+ARCH_MAP_x86_64 := x64
+ARCH_MAP_i386 := x86
+ARCH_MAP_i686 := x86
+ARCH_MAP_aarch64 := arm64v8
+ARCH_MAP_arm64 := arm64v8
+
+OS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+ARCH := $(ARCH_MAP_$(shell uname -m))
+ifeq ($(ARCH),)
+	$(error Unrecognized CPU architecture $(shell uname -m))
+endif
+
+FULL_VARIANT := $(OS)-$(ARCH)-release
+
+# Common rules for all modules, based on per-module configuration
+
+all: $(TARGET_MODULE)
+
+$(TARGET_MODULE): get_source
+	$(MAKE) -C $(SRC_DIR)
+
+get_source: $(SRC_DIR)/.prepared
+
+$(SRC_DIR)/.prepared:
+	mkdir -p $(SRC_DIR)
+	git clone --recursive --depth 1 --branch $(MODULE_VERSION) $(MODULE_REPO) $(SRC_DIR)
+	touch $@
+
+clean:
+	-$(MAKE) -C $(SRC_DIR) clean
+
+distclean:
+	-$(MAKE) -C $(SRC_DIR) distclean
+
+pristine:
+	-rm -rf $(SRC_DIR)
+
+install: $(TARGET_MODULE)
+	mkdir -p $(INSTALL_DIR)
+	$(INSTALL) -m 0755 -D $(TARGET_MODULE) $(INSTALL_DIR)
+
+.PHONY: all clean distclean pristine install

modules/redisbloom/Makefile

@@ -0,0 +1,6 @@
+SRC_DIR = src
+MODULE_VERSION = v9.99.99
+MODULE_REPO = https://github.com/redisbloom/redisbloom
+TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/redisbloom.so
+
+include ../common.mk

modules/redisearch/Makefile

@@ -0,0 +1,7 @@
+SRC_DIR = src
+MODULE_VERSION = v9.99.99
+MODULE_REPO = https://github.com/redisearch/redisearch
+TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/coord-oss/redisearch.so
+
+include ../common.mk
+

modules/redisjson/Makefile

@@ -0,0 +1,11 @@
+SRC_DIR = src
+MODULE_VERSION = v9.99.99
+MODULE_REPO = https://github.com/redisjson/redisjson
+TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/rejson.so
+
+include ../common.mk
+
+$(SRC_DIR)/.cargo_fetched:
+	cd $(SRC_DIR) && cargo fetch
+
+get_source: $(SRC_DIR)/.cargo_fetched

modules/redistimeseries/Makefile

@@ -0,0 +1,6 @@
+SRC_DIR = src
+MODULE_VERSION = v9.99.99
+MODULE_REPO = https://github.com/redistimeseries/redistimeseries
+TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/redistimeseries.so
+
+include ../common.mk