Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/rails.gitlab-ci.yml

@@ -115,6 +115,18 @@ rspec background_migration pg14 praefect:
     - .praefect-with-db
     - .rails:rules:praefect-with-db
 
+rspec migration pg14 clusterwide-db:
+  extends:
+    - rspec migration pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
+rspec background_migration pg14 clusterwide-db:
+  extends:
+    - rspec background_migration pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec unit pg14:
   extends:
     - .rspec-base-pg14
@@ -139,6 +151,12 @@ rspec unit pg14 single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection
 
+rspec unit pg14 clusterwide-db:
+  extends:
+    - rspec unit pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec unit pg14 praefect:
   extends:
     - rspec unit pg14
@@ -169,6 +187,12 @@ rspec integration pg14 single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection
 
+rspec integration pg14 clusterwide-db:
+  extends:
+    - rspec integration pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec integration pg14 praefect:
   extends:
     - rspec integration pg14
@@ -201,6 +225,12 @@ rspec system pg14 single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection
 
+rspec system pg14 clusterwide-db:
+  extends:
+    - rspec system pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec system pg14 praefect:
   extends:
     - rspec system pg14
@@ -604,6 +634,18 @@ rspec background_migration pg14-as-if-foss single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection-as-if-foss
 
+rspec migration pg14-as-if-foss clusterwide-db:
+  extends:
+    - rspec migration pg14-as-if-foss
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
+rspec background_migration pg14-as-if-foss clusterwide-db:
+  extends:
+    - rspec background_migration pg14-as-if-foss
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec unit pg14-as-if-foss:
   extends:
     - .rspec-base-pg14-as-if-foss
@@ -622,6 +664,12 @@ rspec unit pg14-as-if-foss single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection-as-if-foss
 
+rspec unit pg14-as-if-foss clusterwide-db:
+  extends:
+    - rspec unit pg14-as-if-foss
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec integration pg14-as-if-foss:
   extends:
     - .rspec-base-pg14-as-if-foss
@@ -640,6 +688,12 @@ rspec integration pg14-as-if-foss single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection-as-if-foss
 
+rspec integration pg14-as-if-foss clusterwide-db:
+  extends:
+    - rspec integration pg14-as-if-foss
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec system pg14-as-if-foss:
   extends:
     - .rspec-base-pg14-as-if-foss
@@ -658,6 +712,12 @@ rspec system pg14-as-if-foss single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection-as-if-foss
 
+rspec system pg14-as-if-foss clusterwide-db:
+  extends:
+    - rspec system pg14-as-if-foss
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec-ee migration pg14:
   extends:
     - .rspec-ee-base-pg14
@@ -696,6 +756,18 @@ rspec-ee background_migration pg14 single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection
 
+rspec-ee migration pg14 clusterwide-db:
+  extends:
+    - rspec-ee migration pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
+rspec-ee background_migration pg14 clusterwide-db:
+  extends:
+    - rspec-ee background_migration pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec-ee migration pg14 praefect:
   extends:
     - rspec migration pg14
@@ -737,6 +809,12 @@ rspec-ee unit pg14 single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection
 
+rspec-ee unit pg14 clusterwide-db:
+  extends:
+    - rspec-ee unit pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec-ee integration pg14:
   extends:
     - .rspec-ee-base-pg14
@@ -766,6 +844,12 @@ rspec-ee integration pg14 single-db-ci-connection:
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection
 
+rspec-ee integration pg14 clusterwide-db:
+  extends:
+    - rspec-ee integration pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
+
 rspec-ee system pg14:
   extends:
     - .rspec-ee-base-pg14
@@ -794,6 +878,12 @@ rspec-ee system pg14 single-db-ci-connection:
     - rspec-ee system pg14
     - .single-db-ci-connection-rspec
     - .rails:rules:single-db-ci-connection
+
+rspec-ee system pg14 clusterwide-db:
+  extends:
+    - rspec-ee system pg14
+    - .clusterwide-db
+    - .rails:rules:clusterwide-db
 # EE: default refs (MRs, default branch, schedules) jobs #
 ##################################################
 

.gitlab/ci/rails/shared.gitlab-ci.yml

@@ -30,6 +30,10 @@ include:
   variables:
     USE_REDIS_CLUSTER: "false"
 
+.clusterwide-db:
+  variables:
+    CLUSTERWIDE_DB: "true"
+
 .single-db:
   variables:
     DECOMPOSED_DB: "false"

.gitlab/ci/rules.gitlab-ci.yml

@@ -77,6 +77,9 @@
 .if-merge-request-labels-run-single-db: &if-merge-request-labels-run-single-db
   if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-single-db/'
 
+.if-merge-request-labels-run-clusterwide-db: &if-merge-request-labels-run-clusterwide-db
+  if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-clusterwide-db/'
+
 .if-merge-request-labels-run-review-app: &if-merge-request-labels-run-review-app
   if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-review-app/'
 
@@ -1601,6 +1604,10 @@
       changes: *setup-test-env-patterns
     - <<: *if-merge-request-labels-run-all-rspec
 
+.rails:rules:clusterwide-db:
+  rules:
+    - <<: *if-merge-request-labels-run-clusterwide-db
+
 .rails:rules:single-redis:
   rules:
     - <<: *if-merge-request-labels-run-single-db

config/database.yml.decomposed-clusterwide-postgresql

@@ -0,0 +1,86 @@
+# Defaults
+
+defaults: &defaults
+  adapter: postgresql
+  encoding: unicode
+  host: localhost
+  username: git
+  password: "secure password"
+
+#
+# PRODUCTION
+#
+production:
+  main: &main
+    <<: *defaults
+    database: gitlabhq_production
+  ci:
+    <<: *defaults
+    database: gitlabhq_production_ci
+  main_clusterwide:
+    <: *main
+    database_tasks: false
+  geo:
+    <<: *defaults
+    database: gitlabhq_geo_production
+
+#
+# Development specific
+#
+development:
+  main: &main
+    <<: *defaults
+    database: gitlabhq_development
+    username: postgres
+    variables:
+      statement_timeout: 15s
+  ci:
+    <<: *defaults
+    database: gitlabhq_development_ci
+    username: postgres
+    variables:
+      statement_timeout: 15s
+  main_clusterwide:
+    <<: *main
+    database_tasks: false
+    variables:
+      statement_timeout: 15s
+  geo:
+    <<: *defaults
+    database: gitlabhq_geo_development
+    username: postgres
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  main: &main
+    <<: *defaults
+    database: gitlabhq_test
+    username: postgres
+    password:
+    prepared_statements: false
+    variables:
+      statement_timeout: 15s
+  ci:
+    <<: *defaults
+    database: gitlabhq_test_ci
+    username: postgres
+    password:
+    prepared_statements: false
+    variables:
+      statement_timeout: 15s
+  main_clusterwide:
+    <<: *main
+    database_tasks: false
+  geo:
+    <<: *defaults
+    database: gitlabhq_geo_test
+    username: postgres
+    password:
+  embedding:
+    <<: *defaults
+    database: gitlabhq_embedding_test
+    username: postgres
+    password:
+

doc/development/database/multiple_databases.md

@@ -604,6 +604,13 @@ to limit the modes where tests can run, and skip them on any other modes.
 | `skip_if_multiple_databases_are_setup(:ci)` | Only on **single-db**   |
 | `skip_if_multiple_databases_not_setup(:ci)` | On **single-db-ci-connection** and **multiple databases** |
 
+## Testing for multiple databases, including main_clusterwide
+
+By default, we do not setup the `main_clusterwide` connection in CI pipelines. However, if you add the label `~"pipeline:run-clusterwide-db"`, the pipelines will run with 3 connections, `main`, `ci` and `main_clusterwide`.
+
+NOTE:
+This setup is not completely ready yet, and running pipelines in the setup may fail some jobs. As of July 2023, this is only used by  **group::tenant scale**  to test out changes while building [Cells](../../architecture/blueprints/cells/index.md).
+
 ## Locking writes on the tables that don't belong to the database schemas
 
 When the CI database is promoted and the two databases are fully split,

scripts/utils.sh

@@ -129,10 +129,15 @@ function assets_compile_script() {
 
 function setup_database_yml() {
   if [ "$DECOMPOSED_DB" == "true" ]; then
-    echo "Using decomposed database config (config/database.yml.decomposed-postgresql)"
-    cp config/database.yml.decomposed-postgresql config/database.yml
+    if [ "$CLUSTERWIDE_DB" == "true" ]; then
+      echo "Using decomposed database config, containing clusterwide connection (config/database.yml.decomposed-clusterwide-postgresql)"
+      cp config/database.yml.decomposed-clusterwide-postgresql config/database.yml
+    else
+      echo "Using decomposed database config (config/database.yml.decomposed-postgresql)"
+      cp config/database.yml.decomposed-postgresql config/database.yml
+    fi
   else
-    echo "Using decomposed database config (config/database.yml.postgresql)"
+    echo "Using two connections, single database config (config/database.yml.postgresql)"
     cp config/database.yml.postgresql config/database.yml
 
     if [ "$CI_CONNECTION_DB" != "true" ]; then