From fd767b7d65e0e9cb6f313b1803b859b8a7c0da57 Mon Sep 17 00:00:00 2001
From: GitLab Bot <gitlab-bot@gitlab.com>
Date: Fri, 23 Sep 2022 09:13:24 +0000
Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master

---
 .gitlab/ci/rules.gitlab-ci.yml                |  20 +-
 .rubocop.yml                                  |   4 +-
 GITALY_SERVER_VERSION                         |   2 +-
 .../components/profile_preferences.vue        |   2 +-
 ...220905120848_backfill_epic_cache_counts.rb |  29 +++
 ...pipeline_variables_raw_with_new_default.rb |  10 +
 ...ci_group_variables_raw_with_new_default.rb |  10 +
 ...instance_variables_raw_with_new_default.rb |  10 +
 ...d_ci_job_variables_raw_with_new_default.rb |  10 +
 ...schedule_variables_raw_with_new_default.rb |  10 +
 ...d_add_ci_variables_raw_with_new_default.rb |  10 +
 db/schema_migrations/20220905120848           |   1 +
 db/schema_migrations/20220921111823           |   1 +
 db/schema_migrations/20220921111831           |   1 +
 db/schema_migrations/20220921111840           |   1 +
 db/schema_migrations/20220921111849           |   1 +
 db/schema_migrations/20220921111857           |   1 +
 db/schema_migrations/20220921111907           |   1 +
 db/structure.sql                              |  14 +-
 .../blueprints/runner_scaling/index.md        | 172 ++++++++++++++----
 doc/integration/jira/development_panel.md     |   2 +-
 doc/topics/autodevops/index.md                | 128 +++++--------
 .../backfill_epic_cache_counts.rb             |  15 ++
 rubocop/rubocop-ruby30.yml                    |  16 ++
 .../backfill_epic_cache_counts_spec.rb        |  32 ++++
 .../web_hooks/log_execution_service_spec.rb   |   2 +
 26 files changed, 372 insertions(+), 133 deletions(-)
 create mode 100644 db/post_migrate/20220905120848_backfill_epic_cache_counts.rb
 create mode 100644 db/post_migrate/20220921111823_remove_and_add_ci_pipeline_variables_raw_with_new_default.rb
 create mode 100644 db/post_migrate/20220921111831_remove_and_add_ci_group_variables_raw_with_new_default.rb
 create mode 100644 db/post_migrate/20220921111840_remove_and_add_ci_instance_variables_raw_with_new_default.rb
 create mode 100644 db/post_migrate/20220921111849_remove_and_add_ci_job_variables_raw_with_new_default.rb
 create mode 100644 db/post_migrate/20220921111857_remove_and_add_ci_pipeline_schedule_variables_raw_with_new_default.rb
 create mode 100644 db/post_migrate/20220921111907_remove_and_add_ci_variables_raw_with_new_default.rb
 create mode 100644 db/schema_migrations/20220905120848
 create mode 100644 db/schema_migrations/20220921111823
 create mode 100644 db/schema_migrations/20220921111831
 create mode 100644 db/schema_migrations/20220921111840
 create mode 100644 db/schema_migrations/20220921111849
 create mode 100644 db/schema_migrations/20220921111857
 create mode 100644 db/schema_migrations/20220921111907
 create mode 100644 lib/gitlab/background_migration/backfill_epic_cache_counts.rb
 create mode 100644 rubocop/rubocop-ruby30.yml
 create mode 100644 spec/migrations/backfill_epic_cache_counts_spec.rb

diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index bc8f7596ff0..203aa779e5b 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -31,11 +31,14 @@
 .if-merge-request: &if-merge-request
   if: '$CI_MERGE_REQUEST_IID'
 
+# Once https://gitlab.com/gitlab-org/gitlab/-/issues/373904 is implemented, we should be able to change this back to
+# if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVALS_COUNT > 0'
+# or any similar condition to check that the MR has *any* approval (not just required approval).
 .if-merge-request-approved: &if-merge-request-approved
-  if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVED'
+  if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-full-rspec/'
 
 .if-merge-request-not-approved: &if-merge-request-not-approved
-  if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVED != "true"'
+  if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_LABELS !~ /pipeline:run-full-rspec/'
 
 .if-automated-merge-request: &if-automated-merge-request
   if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "release-tools/update-gitaly" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /stable-ee$/'
@@ -532,9 +535,6 @@
   rules:
     - <<: *if-merge-request-approved
       when: never
-    # Temporarily disabled minimal rspec jobs before and after approval because of https://gitlab.com/gitlab-org/gitlab/-/issues/373064.
-    - <<: *if-merge-request-not-approved
-      when: never
     - <<: *if-automated-merge-request
       when: never
     - <<: *if-security-merge-request
@@ -554,6 +554,8 @@
       changes: *backend-patterns
     - <<: *if-security-merge-request
       changes: *backend-patterns
+    - <<: *if-merge-request-not-approved
+      when: never
 
 .rails:rules:as-if-foss-migration-unit-integration:minimal-default-rules:
   rules:
@@ -581,6 +583,8 @@
       changes: *code-backstage-patterns
     - <<: *if-security-merge-request
       changes: *code-backstage-patterns
+    - <<: *if-merge-request-not-approved
+      when: never
 
 .rails:rules:system:minimal-default-rules:
   rules:
@@ -1007,6 +1011,8 @@
       changes: *db-patterns
     - <<: *if-security-merge-request
       changes: *db-patterns
+    - <<: *if-merge-request-not-approved
+      when: never
     - changes: *db-patterns
 
 .rails:rules:ee-and-foss-migration:minimal:
@@ -1108,6 +1114,8 @@
       changes: *db-patterns
     - <<: *if-security-merge-request
       changes: *db-patterns
+    - <<: *if-merge-request-not-approved
+      when: never
     - changes: *db-patterns
 
 .rails:rules:ee-only-migration:minimal:
@@ -1195,6 +1203,8 @@
       changes: *db-patterns
     - <<: *if-security-merge-request
       changes: *db-patterns
+    - <<: *if-merge-request-not-approved
+      when: never
 
 .rails:rules:as-if-foss-migration:minimal:
   rules:
diff --git a/.rubocop.yml b/.rubocop.yml
index ee54d03d6af..21e2f8f2827 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -13,6 +13,9 @@ inherit_from:
     <% end %>
   - '.rubocop_todo.yml'
   <% end %>
+  <% if RUBY_VERSION[/^\d+\.\d+/, 0] == '3.0' %>
+  - ./rubocop/rubocop-ruby30.yml
+  <% end %>
   - ./rubocop/rubocop-migrations.yml
   - ./rubocop/rubocop-usage-data.yml
   - ./rubocop/rubocop-code_reuse.yml
@@ -84,7 +87,6 @@ Lint/EmptyFile:
 # This cop checks whether some constant value isn't a
 # mutable literal (e.g. array or hash).
 Style/MutableConstant:
-  Enabled: true
   Exclude:
     - 'db/migrate/**/*'
     - 'db/post_migrate/**/*'
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION
index 80c8a866e1f..beb66fcc2b4 100644
--- a/GITALY_SERVER_VERSION
+++ b/GITALY_SERVER_VERSION
@@ -1 +1 @@
-9498ab9459048cc595d8e2e411b027d080c0ab0f
+e4d8f69ffa2efd3f2cb0adff5fa66f367f66f6fb
diff --git a/app/assets/javascripts/profile/preferences/components/profile_preferences.vue b/app/assets/javascripts/profile/preferences/components/profile_preferences.vue
index aa6d001c00e..b30b6a2faa9 100644
--- a/app/assets/javascripts/profile/preferences/components/profile_preferences.vue
+++ b/app/assets/javascripts/profile/preferences/components/profile_preferences.vue
@@ -110,7 +110,7 @@ export default {
 </script>
 
 <template>
-  <div class="row gl-mt-3 js-preferences-form">
+  <div class="row gl-mt-3 js-preferences-form js-search-settings-section">
     <div v-if="integrationViews.length" class="col-sm-12">
       <hr data-testid="profile-preferences-integrations-rule" />
     </div>
diff --git a/db/post_migrate/20220905120848_backfill_epic_cache_counts.rb b/db/post_migrate/20220905120848_backfill_epic_cache_counts.rb
new file mode 100644
index 00000000000..32fe04eb99c
--- /dev/null
+++ b/db/post_migrate/20220905120848_backfill_epic_cache_counts.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class BackfillEpicCacheCounts < Gitlab::Database::Migration[2.0]
+  MIGRATION = 'BackfillEpicCacheCounts'
+  DELAY_INTERVAL = 2.minutes.to_i
+  BATCH_SIZE = 200
+  MAX_BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 20
+
+  disable_ddl_transaction!
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :epics,
+      :id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      max_batch_size: MAX_BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE,
+      gitlab_schema: :gitlab_main
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :epics, :id, [])
+  end
+end
diff --git a/db/post_migrate/20220921111823_remove_and_add_ci_pipeline_variables_raw_with_new_default.rb b/db/post_migrate/20220921111823_remove_and_add_ci_pipeline_variables_raw_with_new_default.rb
new file mode 100644
index 00000000000..fd56a8f320a
--- /dev/null
+++ b/db/post_migrate/20220921111823_remove_and_add_ci_pipeline_variables_raw_with_new_default.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class RemoveAndAddCiPipelineVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
+  enable_lock_retries!
+
+  def change
+    remove_column :ci_pipeline_variables, :raw, :boolean, null: false, default: true
+    add_column :ci_pipeline_variables, :raw, :boolean, null: false, default: false
+  end
+end
diff --git a/db/post_migrate/20220921111831_remove_and_add_ci_group_variables_raw_with_new_default.rb b/db/post_migrate/20220921111831_remove_and_add_ci_group_variables_raw_with_new_default.rb
new file mode 100644
index 00000000000..f4b7de3faf4
--- /dev/null
+++ b/db/post_migrate/20220921111831_remove_and_add_ci_group_variables_raw_with_new_default.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class RemoveAndAddCiGroupVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
+  enable_lock_retries!
+
+  def change
+    remove_column :ci_group_variables, :raw, :boolean, null: false, default: true
+    add_column :ci_group_variables, :raw, :boolean, null: false, default: false
+  end
+end
diff --git a/db/post_migrate/20220921111840_remove_and_add_ci_instance_variables_raw_with_new_default.rb b/db/post_migrate/20220921111840_remove_and_add_ci_instance_variables_raw_with_new_default.rb
new file mode 100644
index 00000000000..019db1bbaa3
--- /dev/null
+++ b/db/post_migrate/20220921111840_remove_and_add_ci_instance_variables_raw_with_new_default.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class RemoveAndAddCiInstanceVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
+  enable_lock_retries!
+
+  def change
+    remove_column :ci_instance_variables, :raw, :boolean, null: false, default: true
+    add_column :ci_instance_variables, :raw, :boolean, null: false, default: false
+  end
+end
diff --git a/db/post_migrate/20220921111849_remove_and_add_ci_job_variables_raw_with_new_default.rb b/db/post_migrate/20220921111849_remove_and_add_ci_job_variables_raw_with_new_default.rb
new file mode 100644
index 00000000000..736cd46382d
--- /dev/null
+++ b/db/post_migrate/20220921111849_remove_and_add_ci_job_variables_raw_with_new_default.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class RemoveAndAddCiJobVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
+  enable_lock_retries!
+
+  def change
+    remove_column :ci_job_variables, :raw, :boolean, null: false, default: true
+    add_column :ci_job_variables, :raw, :boolean, null: false, default: false
+  end
+end
diff --git a/db/post_migrate/20220921111857_remove_and_add_ci_pipeline_schedule_variables_raw_with_new_default.rb b/db/post_migrate/20220921111857_remove_and_add_ci_pipeline_schedule_variables_raw_with_new_default.rb
new file mode 100644
index 00000000000..159b0303d71
--- /dev/null
+++ b/db/post_migrate/20220921111857_remove_and_add_ci_pipeline_schedule_variables_raw_with_new_default.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class RemoveAndAddCiPipelineScheduleVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
+  enable_lock_retries!
+
+  def change
+    remove_column :ci_pipeline_schedule_variables, :raw, :boolean, null: false, default: true
+    add_column :ci_pipeline_schedule_variables, :raw, :boolean, null: false, default: false
+  end
+end
diff --git a/db/post_migrate/20220921111907_remove_and_add_ci_variables_raw_with_new_default.rb b/db/post_migrate/20220921111907_remove_and_add_ci_variables_raw_with_new_default.rb
new file mode 100644
index 00000000000..129e5b1f020
--- /dev/null
+++ b/db/post_migrate/20220921111907_remove_and_add_ci_variables_raw_with_new_default.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class RemoveAndAddCiVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
+  enable_lock_retries!
+
+  def change
+    remove_column :ci_variables, :raw, :boolean, null: false, default: true
+    add_column :ci_variables, :raw, :boolean, null: false, default: false
+  end
+end
diff --git a/db/schema_migrations/20220905120848 b/db/schema_migrations/20220905120848
new file mode 100644
index 00000000000..ebfd3a0b59c
--- /dev/null
+++ b/db/schema_migrations/20220905120848
@@ -0,0 +1 @@
+f8196de8a4c8f6e8c6790c0d741b0deb455c533a35f665fffeb70c833d0ecd29
\ No newline at end of file
diff --git a/db/schema_migrations/20220921111823 b/db/schema_migrations/20220921111823
new file mode 100644
index 00000000000..eeca2c10dad
--- /dev/null
+++ b/db/schema_migrations/20220921111823
@@ -0,0 +1 @@
+f06d7555d3541abbb9fd671df3718645203aef111674b91ad386dac434c4ede2
\ No newline at end of file
diff --git a/db/schema_migrations/20220921111831 b/db/schema_migrations/20220921111831
new file mode 100644
index 00000000000..a03ad3f67dd
--- /dev/null
+++ b/db/schema_migrations/20220921111831
@@ -0,0 +1 @@
+a515d3e34f4b467c5e6fbd9b55135a676277ff6388eb1e3fc14df4b11d8eb3c0
\ No newline at end of file
diff --git a/db/schema_migrations/20220921111840 b/db/schema_migrations/20220921111840
new file mode 100644
index 00000000000..7e84d96f124
--- /dev/null
+++ b/db/schema_migrations/20220921111840
@@ -0,0 +1 @@
+0fdbb888bdbd4f68619466a8f4384e44062b0cf854790c648a6a060ab1e71806
\ No newline at end of file
diff --git a/db/schema_migrations/20220921111849 b/db/schema_migrations/20220921111849
new file mode 100644
index 00000000000..3989dd78aeb
--- /dev/null
+++ b/db/schema_migrations/20220921111849
@@ -0,0 +1 @@
+2041cb81c534be0cc45b1cc77fa7fd2e31615129e1ba60a146cca88d58b77605
\ No newline at end of file
diff --git a/db/schema_migrations/20220921111857 b/db/schema_migrations/20220921111857
new file mode 100644
index 00000000000..578b2c853f4
--- /dev/null
+++ b/db/schema_migrations/20220921111857
@@ -0,0 +1 @@
+cbef1d036abb0035d710cf912e554e32fa88df3abaed17cb938e0b18032c7448
\ No newline at end of file
diff --git a/db/schema_migrations/20220921111907 b/db/schema_migrations/20220921111907
new file mode 100644
index 00000000000..013f79024c9
--- /dev/null
+++ b/db/schema_migrations/20220921111907
@@ -0,0 +1 @@
+fbc4c12aedd9d0f183e9444f2cb42c11a2b894c11684e80a5dbe847c7bccb21f
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 7cbe648f4f0..0f6ff043d6e 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -12772,7 +12772,7 @@ CREATE TABLE ci_group_variables (
     masked boolean DEFAULT false NOT NULL,
     variable_type smallint DEFAULT 1 NOT NULL,
     environment_scope text DEFAULT '*'::text NOT NULL,
-    raw boolean DEFAULT true NOT NULL,
+    raw boolean DEFAULT false NOT NULL,
     CONSTRAINT check_dfe009485a CHECK ((char_length(environment_scope) <= 255))
 );
 
@@ -12793,7 +12793,7 @@ CREATE TABLE ci_instance_variables (
     key text NOT NULL,
     encrypted_value text,
     encrypted_value_iv text,
-    raw boolean DEFAULT true NOT NULL,
+    raw boolean DEFAULT false NOT NULL,
     CONSTRAINT check_07a45a5bcb CHECK ((char_length(encrypted_value_iv) <= 255)),
     CONSTRAINT check_5aede12208 CHECK ((char_length(key) <= 255)),
     CONSTRAINT check_956afd70f1 CHECK ((char_length(encrypted_value) <= 13579))
@@ -12875,7 +12875,7 @@ CREATE TABLE ci_job_variables (
     job_id bigint NOT NULL,
     variable_type smallint DEFAULT 1 NOT NULL,
     source smallint DEFAULT 0 NOT NULL,
-    raw boolean DEFAULT true NOT NULL
+    raw boolean DEFAULT false NOT NULL
 );
 
 CREATE SEQUENCE ci_job_variables_id_seq
@@ -13057,7 +13057,7 @@ CREATE TABLE ci_pipeline_schedule_variables (
     created_at timestamp with time zone,
     updated_at timestamp with time zone,
     variable_type smallint DEFAULT 1 NOT NULL,
-    raw boolean DEFAULT true NOT NULL
+    raw boolean DEFAULT false NOT NULL
 );
 
 CREATE SEQUENCE ci_pipeline_schedule_variables_id_seq
@@ -13101,8 +13101,8 @@ CREATE TABLE ci_pipeline_variables (
     encrypted_value_iv character varying,
     pipeline_id integer NOT NULL,
     variable_type smallint DEFAULT 1 NOT NULL,
-    raw boolean DEFAULT true NOT NULL,
-    partition_id bigint DEFAULT 100 NOT NULL
+    partition_id bigint DEFAULT 100 NOT NULL,
+    raw boolean DEFAULT false NOT NULL
 );
 
 CREATE SEQUENCE ci_pipeline_variables_id_seq
@@ -13564,7 +13564,7 @@ CREATE TABLE ci_variables (
     environment_scope character varying DEFAULT '*'::character varying NOT NULL,
     masked boolean DEFAULT false NOT NULL,
     variable_type smallint DEFAULT 1 NOT NULL,
-    raw boolean DEFAULT true NOT NULL
+    raw boolean DEFAULT false NOT NULL
 );
 
 CREATE SEQUENCE ci_variables_id_seq
diff --git a/doc/architecture/blueprints/runner_scaling/index.md b/doc/architecture/blueprints/runner_scaling/index.md
index 8f7062a1148..700fd6367cd 100644
--- a/doc/architecture/blueprints/runner_scaling/index.md
+++ b/doc/architecture/blueprints/runner_scaling/index.md
@@ -94,7 +94,7 @@ data that can be shared between job runs.
 Because there is no viable replacement and we might be unable to support all
 cloud providers that Docker Machine used to support, the key design requirement
 is to make it really simple and easy for the wider community to write a custom
-GitLab auto-scaling plugin, whatever cloud provider they might be using. We
+GitLab plugin for whatever cloud provider they might be using. We
 want to design a simple abstraction that users will be able to build on top, as
 will we to support existing workflows on GitLab.com.
 
@@ -129,12 +129,11 @@ the need of rebuilding GitLab Runner whenever it happens.
 
 ### 💡 Write a solid documentation about how to build your own plugin
 
-It is important to show users how to build an auto-scaling plugin, so that they
+It is important to show users how to build a plugin, so that they
 can implement support for their own cloud infrastructure.
 
-Building new plugins should be simple, and with the support of great
-documentation it should not require advanced skills, like understanding how
-gRPC works. We want to design the plugin system in a way that the entry barrier
+Building new plugins should be simple and supported with great
+documentation. We want to design the plugin system in a way that the entry barrier
 for contributing new plugins is very low.
 
 ### 💡 Build a PoC to run multiple builds on a single machine
@@ -171,38 +170,65 @@ configures the Docker daemon there to allow external authenticated requests. It
 stores credentials to such ephemeral Docker environments on disk. Once a
 machine has been provisioned and made available for GitLab Runner Manager to
 run builds, it is using one of the existing executors to run a user-provided
-script. In auto-scaling, this is typically done using Docker executor.
+script. In auto-scaling, this is typically done using the Docker executor.
 
-### Custom provider
+### Separation of concerns
 
-In order to reduce the scope of work, we only want to introduce the new
-abstraction layer in one place.
+There are several concerns represented in the current architecture. They are
+coupled in the current implementation so we will break them out here to consider
+them each separately.
 
-A few years ago we introduced the [Custom Executor](https://docs.gitlab.com/runner/executors/custom.html)
-feature in GitLab Runner. It allows users to design custom build execution
-methods. The custom executor driver can be implemented in any way - from a
-simple shell script to a dedicated binary - that is then used by a Runner
-through os/exec system calls.
+1. **Virtual Machine (VM) shape**. The underlying provider of a VM requires configuration to
+   know what kind of machine to create. E.g. Cores, memory, failure domain,
+   etc... This information is very provider specific.
+1. **VM lifecycle management**. Multiple machines will be created and a
+   system must keep track of which machines belong to this executor. Typically
+   a cloud provider will have a way to manage a set of homogenous machines.
+   E.g. GCE Instance Group. The basic operations are increase, decrease and
+   usually delete a specific machine.
+1. **VM autoscaling**. In addition to low-level lifecycle management,
+   job-aware capacity decisions must be made to the set of machines to provide
+   capacity when it is needed but not maintain excess capacity for cost reasons.
+1. **Job to VM mapping (routing)**. Currently the system assigns only one job to a
+   given a machine. A machine may be reused based on the specific executor
+   configuration.
+1. **In-VM job execution**. Within each VM a job must be driven through
+   various pre-defined stages and results and trace information returned
+   to the Runner system. These details are highly dependent on the VM
+   architecture and operating system as well as Executor type.
 
-Thanks to the custom executor abstraction there is no more need to implement
-new executors internally in Runner. Users who have specific needs can implement
-their own drivers and don't need to wait for us to make their work part of the
-"official" GitLab Runner. As each driver is a separate project, it also makes
-it easier to create communities around them, where interested people can
-collaborate together on improvements and bug fixes.
+The current architecture has several points of coupling between concerns.
+Coupling reduces opportunities for abstraction (e.g. community supported
+plugins) and increases complexity, making the code harder to understand,
+test, maintain and extend.
 
-We want to design the new Custom Provider to replicate the success of the
-Custom Executor. It will make it easier for users to build their own ways to
-provide a context and an environment in which a build will be executed by one
-of the Custom Executors.
+A primary design decision will be which concerns to externalize to the plugin
+and which should remain with the runner system. The current implementation
+has several abstractions internally which could be used as cut points for a
+new abstraction. 
 
-There are multiple solutions to implementing a custom provider abstraction. We
-can use raw Go plugins, Hashcorp's Go Plugin, HTTP interface or gRPC based
-facade service. There are many solutions, and we want to choose the most
-optimal one. In order to do that, we will describe the solutions in a separate
-document, define requirements and score the solution accordingly. This will
-allow us to choose a solution that will work best for us and the wider
-community.
+For example the [`Build`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L125)
+type uses the [`GetExecutorProvider`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/executor.go#L171)
+function to get an executor provider based on a dispatching executor string.
+Various executor types register with the system by being imported and calling
+[`RegisterExecutorProvider`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/executor.go#L154)
+during initialization. Here the abstractions are the [`ExecutorProvider`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/executor.go#L80)
+and [`Executor`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/executor.go#L59)
+interfaces.
+
+Within the `docker+autoscaling` executor the [`machineExecutor`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/machine.go#L19)
+type has a [`Machine`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/helpers/docker/machine.go#L7)
+interface which it uses to aquire a VM during the common [`Prepare`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/machine.go#L71)
+phase. This abstraction primarily creates, accesses and deletes VMs.
+
+There is no current abstraction for the VM autoscaling logic. It is tightly
+coupled with the VM lifecycle and job routing logic. Creating idle capacity
+happens as a side-effect of calling [`Acquire`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/provider.go#L449) on the `machineProvider` while binding a job to a VM.
+
+There is also no current abstraction for in-VM job execution. VM-specific
+commands are generated by the Runner Manager using the [`GenerateShellScript`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L336)
+function and [injected](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L373)
+into the VM as the manager drives the job execution stages.
 
 ### Design principles
 
@@ -235,12 +261,84 @@ abstraction.
 1. Build an abstraction that serves our community well but allows us to ship it quickly.
 1. Invest in a flexible solution, avoid one-way-door decisions, foster iteration.
 1. When in doubts err on the side of making things more simple for the wider community.
+1. Limit coupling between concerns in order to make the system more simple and extensible.
+1. Concerns should live on one side of the plug or the other--not both, which
+   duplicates effort and increases coupling.
 
 #### The most important technical details
 
 1. Favor gRPC communication between a plugin and GitLab Runner.
 1. Make it possible to version communication interface and support many versions.
 1. Make Go a primary language for writing plugins but accept other languages too.
+1. Prefer a GitLab job-aware autoscaler to provider specific autoscalers. Cloud provider
+   autoscalers don't know which VM to delete when scaling down so they make sub-optimal
+   decisions. Rather than teaching all autoscalers about GitLab jobs, we prefer to
+   have one, GitLab-owned autoscaler (not in the plugin).
+
+## Plugin boundary proposals
+
+The following are proposals for where to draw the plugin boundary. We will evaluate
+these proposals and others by the design principles and technical constraints
+listed above.
+
+### Custom provider
+
+In order to reduce the scope of work, we only want to introduce the new
+abstraction layer in one place.
+
+A few years ago we introduced the [Custom Executor](https://docs.gitlab.com/runner/executors/custom.html)
+feature in GitLab Runner. It allows users to design custom build execution
+methods. The custom executor driver can be implemented in any way - from a
+simple shell script to a dedicated binary - that is then used by a Runner
+through os/exec system calls.
+
+Thanks to the custom executor abstraction there is no more need to implement
+new executors internally in Runner. Users who have specific needs can implement
+their own drivers and don't need to wait for us to make their work part of the
+"official" GitLab Runner. As each driver is a separate project, it also makes
+it easier to create communities around them, where interested people can
+collaborate together on improvements and bug fixes.
+
+We want to design the new Custom Provider to replicate the success of the
+Custom Executor. It will make it easier for users to build their own ways to
+provide a context and an environment in which a build will be executed by one
+of the Custom Executors.
+
+There are multiple solutions to implementing a custom provider abstraction. We
+can use raw Go plugins, Hashcorp's Go Plugin, HTTP interface or gRPC based
+facade service. There are many solutions, and we want to choose the most
+optimal one. In order to do that, we will describe the solutions in a separate
+document, define requirements and score the solution accordingly. This will
+allow us to choose a solution that will work best for us and the wider
+community.
+
+This proposal places VM lifecycle and autoscaling concerns as well as job to
+VM mapping (routing) into the plugin. The build need only ask for a VM and
+it will get one with all aspects of lifecycle and routing already accounted
+for by the plugin.
+
+Rationale: [Description of the Custom Executor Provider proposal](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28848#note_823321515)
+
+### Fleeting VM provider
+
+We can introduce a more simple version of the `Machine` abstraction in the
+form of a "Fleeting" interface. Fleeting provides a low-level interface to
+a homogenous VM group which allows increasing and decreasing the set size
+as well as consuming a VM from within the set.
+
+Plugins for cloud providers and other VM sources are implemented via the
+Hashicorp go-plugin library. This is in practice gRPC over STDIN/STDOUT
+but other wire protocols can be used also.
+
+In order to make use of the new interface, the autoscaling logic is pulled
+out of the Docker Executor and placed into a new Taskscaler library.
+
+This places the concerns of VM lifecycle, VM shape and job routing within
+the plugin. It also places the conern of VM autoscaling into a separate
+component so it can be used by multiple Runner Executors (not just `docker+autoscaling`).
+
+Rationale: [Description of the InstanceGroup / Fleeting proposal](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28848#note_823430883)
+POC: [Merge request](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3315)
 
 ## Status
 
@@ -253,12 +351,12 @@ Proposal:
 <!-- vale gitlab.Spelling = NO -->
 
 | Role                         | Who
-|------------------------------|------------------------------------------|
-| Authors                      | Grzegorz Bizon, Tomasz Maczukin          |
-| Architecture Evolution Coach | Kamil Trzciński                          |
-| Engineering Leader           | Elliot Rushton, Cheryl Li                |
-| Product Manager              | Darren Eastman, Jackie Porter            |
-| Domain Expert / Runner       | Arran Walker                             |
+|------------------------------|-------------------------------------------------|
+| Authors                      | Grzegorz Bizon, Tomasz Maczukin, Joseph Burnett |
+| Architecture Evolution Coach | Kamil Trzciński                                 |
+| Engineering Leader           | Elliot Rushton, Cheryl Li                       |
+| Product Manager              | Darren Eastman, Jackie Porter                   |
+| Domain Expert / Runner       | Arran Walker                                    |
 
 DRIs:
 
diff --git a/doc/integration/jira/development_panel.md b/doc/integration/jira/development_panel.md
index 66619c338c4..654608387d2 100644
--- a/doc/integration/jira/development_panel.md
+++ b/doc/integration/jira/development_panel.md
@@ -70,7 +70,7 @@ To simplify administration, we recommend that a GitLab group maintainer or group
 | Jira usage | GitLab.com customers need | GitLab self-managed customers need |
 |------------|---------------------------|------------------------------------|
 | [Atlassian cloud](https://www.atlassian.com/migration/assess/why-cloud) | The [GitLab.com for Jira Cloud app](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud?hosting=cloud&tab=overview) installed from the [Atlassian Marketplace](https://marketplace.atlassian.com). This offers real-time sync between GitLab.com and Jira. For more information, see the documentation for the [GitLab.com for Jira Cloud app](connect-app.md). | The [GitLab.com for Jira Cloud app](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud?hosting=cloud&tab=overview), using a workaround process. See the documentation for [installing the GitLab.com for Jira Cloud app for self-managed instances](connect-app.md#install-the-gitlabcom-for-jira-cloud-app-for-self-managed-instances) for more information. |
-| Your own server | The [Jira DVCS (distributed version control system) connector](dvcs.md). This syncs data hourly. | The [Jira DVCS Connector](dvcs.md). |
+| Your own server | The [Jira DVCS (distributed version control system) connector](dvcs.md). This syncs data hourly. | The [Jira DVCS (distributed version control system) connector](dvcs.md). This syncs data hourly. |
 
 Each GitLab project can be configured to connect to an entire Jira instance. That means after
 configuration, one GitLab project can interact with all Jira projects in that instance. For:
diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md
index 09d656a5e4f..73e87aa190f 100644
--- a/doc/topics/autodevops/index.md
+++ b/doc/topics/autodevops/index.md
@@ -26,36 +26,24 @@ For an introduction to Auto DevOps, watch [Auto DevOps in GitLab 11.0](https://y
 
 ## Auto DevOps features
 
-Based on the DevOps [stages](stages.md), use Auto DevOps to:
+Auto DevOps supports development during each of the [DevOps stages](stages.md).
 
-**Build your app:**
-
-- [Auto Build](stages.md#auto-build)
-- [Auto Dependency Scanning](stages.md#auto-dependency-scanning)
-
-**Test your app:**
-
-- [Auto Test](stages.md#auto-test)
-- [Auto Browser Performance Testing](stages.md#auto-browser-performance-testing)
-- [Auto Code Intelligence](stages.md#auto-code-intelligence)
-- [Auto Code Quality](stages.md#auto-code-quality)
-- [Auto Container Scanning](stages.md#auto-container-scanning)
-- [Auto License Compliance](stages.md#auto-license-compliance)
-
-**Deploy your app:**
-
-- [Auto Review Apps](stages.md#auto-review-apps)
-- [Auto Deploy](stages.md#auto-deploy)
-
-**Monitor your app:**
-
-- [Auto Monitoring](stages.md#auto-monitoring)
-
-**Secure your app:**
-
-- [Auto Dynamic Application Security Testing (DAST)](stages.md#auto-dast)
-- [Auto Static Application Security Testing (SAST)](stages.md#auto-sast)
-- [Auto Secret Detection](stages.md#auto-secret-detection)
+| Stage | Auto DevOps feature |
+|---------|-------------|
+| Build | [Auto Build](stages.md#auto-build) |
+| Build | [Auto Dependency Scanning](stages.md#auto-dependency-scanning) |
+| Test | [Auto Test](stages.md#auto-test) |
+| Test | [Auto Browser Performance Testing](stages.md#auto-browser-performance-testing) |
+| Test | [Auto Code Intelligence](stages.md#auto-code-intelligence) |
+| Test | [Auto Code Quality](stages.md#auto-code-quality) |
+| Test | [Auto Container Scanning](stages.md#auto-container-scanning) |
+| Test | [Auto License Compliance](stages.md#auto-license-compliance) |
+| Deploy | [Auto Review Apps](stages.md#auto-review-apps) |
+| Deploy | [Auto Deploy](stages.md#auto-deploy) |
+| Monitor | [Auto Monitoring](stages.md#auto-monitoring) |
+| Secure | [Auto Dynamic Application Security Testing (DAST)](stages.md#auto-dast) |
+| Secure | [Auto Static Application Security Testing (SAST)](stages.md#auto-sast) |
+| Secure | [Auto Secret Detection](stages.md#auto-secret-detection) |
 
 ### Comparison to application platforms and PaaS
 
@@ -80,24 +68,12 @@ If you want to build, test, and deploy your app:
 
 1. View the [requirements for deployment](requirements.md).
 1. [Enable Auto DevOps](#enable-or-disable-auto-devops).
-1. Follow the [quick start guide](#quick-start).
-
-As Auto DevOps relies on many components, be familiar with:
-
-- [Continuous methodologies](../../ci/introduction/index.md)
-- [Docker](https://docs.docker.com)
-- [GitLab Runner](https://docs.gitlab.com/runner/)
-
-When deploying to a Kubernetes cluster make sure you're also familiar with:
-
-- [Kubernetes](https://kubernetes.io/docs/home/)
-- [Helm](https://helm.sh/docs/)
-- [Prometheus](https://prometheus.io/docs/introduction/overview/)
+1. [Deploy your app to a cloud provider](#deploy-your-app-to-a-cloud-provider).
 
 ### Enable or disable Auto DevOps
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41729) in GitLab 11.3, Auto DevOps is enabled by default.
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26655) GitLab 12.7, Auto DevOps runs pipelines automatically only if a [`Dockerfile` or matching buildpack](stages.md#auto-build) exists.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26655) in GitLab 12.7, Auto DevOps runs pipelines automatically only if a [`Dockerfile` or matching buildpack](stages.md#auto-build) exists.
 
 Depending on your instance type, you can enable or disable Auto DevOps at the
 following levels:
@@ -116,7 +92,7 @@ To use Auto DevOps for individual projects, you can enable it in a
 project-by-project basis. If you intend to use it for more projects,
 you can enable it for a [group](#at-the-group-level) or an
 [instance](#at-the-instance-level). This can save you the time of
-enabling it one by one.
+enabling it in each project.
 
 Prerequisites:
 
@@ -143,9 +119,10 @@ To disable it, follow the same process and clear the
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52447) in GitLab 11.10.
 
-When you enable Auto DevOps at group level, the subgroups and projects in that
-group inherit the configuration. This saves you some time by batch-enabling it
-rather than enabling individually for each subgroup or project.
+When you enable Auto DevOps at the group level, the subgroups and
+projects in that group inherit the configuration. You can save time by
+enabling Auto DevOps for a group instead of enabling it for each
+subgroup or project.
 
 When enabled for a group, you can still disable Auto DevOps
 for the subgroups and projects where you don't want to use it.
@@ -162,7 +139,7 @@ To enable Auto DevOps for a group:
 1. Select the **Default to Auto DevOps pipeline** checkbox.
 1. Select **Save changes**.
 
-To disable Auto DevOps on the group level, follow the same process and
+To disable Auto DevOps at the group level, follow the same process and
 clear the **Default to Auto DevOps pipeline** checkbox.
 
 After enabling Auto DevOps at the group level, you can trigger the
@@ -175,10 +152,9 @@ Auto DevOps pipeline for any project that belongs to that group:
 
 #### At the instance level **(FREE SELF)**
 
-By enabling Auto DevOps in the instance level, all projects created in that
-instance become enabled. This is convenient when you want to run Auto DevOps by
-default for all projects. You can still disable Auto DevOps individually for
-the groups and projects where you don't want to run it.
+To enable Auto DevOps by default for all projects, you can enable it at the instance level.
+You can still disable Auto DevOps for each group and project
+where you don't want to run it.
 
 Even when disabled for an instance, group Owners and project Maintainers
 can still enable Auto DevOps at the group and project levels.
@@ -196,24 +172,17 @@ To enable Auto DevOps for your instance:
 1. Optional. Add the Auto DevOps [base domain](requirements.md#auto-devops-base-domain).
 1. Select **Save changes**.
 
-When enabled, it attempts to run Auto DevOps pipelines in every project. If the
+When enabled, Auto DevOps attempts to run pipelines in every project. If the
 pipeline fails in a particular project, it disables itself.
 GitLab administrators can change this in the [Auto DevOps settings](../../user/admin_area/settings/continuous_integration.md#auto-devops).
 
 If a [CI/CD configuration file](../../ci/yaml/index.md) is present,
-it remains unchanged and Auto DevOps doesn't affect it.
+it remains unchanged and Auto DevOps does not affect it.
 
 To disable Auto DevOps in the instance level, follow the same process
 and clear the **Default to Auto DevOps pipeline** checkbox.
 
-### Private registry support
-
-There is no guarantee that you can use a private container registry with Auto DevOps.
-
-Instead, use the [GitLab Container Registry](../../user/packages/container_registry/index.md) with Auto DevOps to
-simplify configuration and prevent any unforeseen issues.
-
-### Quick start
+### Deploy your app to a cloud provider
 
 - [Use Auto DevOps to deploy to a Kubernetes cluster on Google Kubernetes Engine (GKE)](cloud_deployments/auto_devops_with_gke.md)
 - [Use Auto DevOps to deploy to EC2](cloud_deployments/auto_devops_with_ec2.md)
@@ -221,7 +190,7 @@ simplify configuration and prevent any unforeseen issues.
 
 ## Upgrade Auto DevOps dependencies when updating GitLab
 
-When updating GitLab, you may need to upgrade Auto DevOps dependencies to
+When updating GitLab, you might need to upgrade Auto DevOps dependencies to
 match your new GitLab version:
 
 - [Upgrading Auto DevOps resources](upgrading_auto_deploy_dependencies.md):
@@ -233,30 +202,29 @@ match your new GitLab version:
   - Environment variables.
 - [Upgrading PostgreSQL](upgrading_postgresql.md).
 
+## Private registry support
+
+There is no guarantee that you can use a private container registry with Auto DevOps.
+
+Instead, use the [GitLab Container Registry](../../user/packages/container_registry/index.md) with Auto DevOps to
+simplify configuration and prevent any unforeseen issues.
+
 ## Install applications behind a proxy
 
 The GitLab integration with Helm does not support installing applications when
 behind a proxy.
 
-To do so, inject proxy settings into the installation pods at runtime.
-For example, you can use a `PodPreset`:
+If you want to do so, you must inject proxy settings into the
+installation pods at runtime.
 
-NOTE:
-[PodPreset was removed in Kubernetes v1.20](https://github.com/kubernetes/kubernetes/pull/94090).
+## Related topics
 
-```yaml
-apiVersion: settings.k8s.io/v1alpha1
-kind: PodPreset
-metadata:
-  name: gitlab-managed-apps-default-proxy
-  namespace: gitlab-managed-apps
-spec:
-  env:
-    - name: http_proxy
-      value: "PUT_YOUR_HTTP_PROXY_HERE"
-    - name: https_proxy
-      value: "PUT_YOUR_HTTPS_PROXY_HERE"
-```
+- [Continuous methodologies](../../ci/introduction/index.md)
+- [Docker](https://docs.docker.com)
+- [GitLab Runner](https://docs.gitlab.com/runner/)
+- [Helm](https://helm.sh/docs/)
+- [Kubernetes](https://kubernetes.io/docs/home/)
+- [Prometheus](https://prometheus.io/docs/introduction/overview/)
 
 ## Troubleshooting
 
diff --git a/lib/gitlab/background_migration/backfill_epic_cache_counts.rb b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb
new file mode 100644
index 00000000000..bd61d1a0f07
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class BackfillEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob
+      def perform; end
+    end
+    # rubocop: enable Style/Documentation
+  end
+end
+
+# rubocop: disable Layout/LineLength
+Gitlab::BackgroundMigration::BackfillEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts')
+# rubocop: enable Layout/LineLength
diff --git a/rubocop/rubocop-ruby30.yml b/rubocop/rubocop-ruby30.yml
new file mode 100644
index 00000000000..6cd3f66ce55
--- /dev/null
+++ b/rubocop/rubocop-ruby30.yml
@@ -0,0 +1,16 @@
+# RuboCop configuration adjustments during the transition time from Ruby 2.7 to Ruby 3.0.
+# This configuration should be removed after the transition has been completed.
+
+# Disable cops for now since their behavior changed in Ruby 3.0.
+# See https://gitlab.com/gitlab-org/gitlab/-/jobs/3068345492
+#
+# Migration plan:
+# * Generate TODOs for these cops (with Ruby 3.0) right before the switch to Ruby 3.0
+# * Put these cops back in "grace period" to ensure `master` stability
+# * Remove "grace period" after the switch
+# * Incrementally fix TODOs
+#
+Style/MutableConstant:
+  Enabled: false
+Style/RedundantFreeze:
+  Enabled: false
diff --git a/spec/migrations/backfill_epic_cache_counts_spec.rb b/spec/migrations/backfill_epic_cache_counts_spec.rb
new file mode 100644
index 00000000000..6084fdad0a6
--- /dev/null
+++ b/spec/migrations/backfill_epic_cache_counts_spec.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe BackfillEpicCacheCounts, :migration do
+  let(:migration) { described_class::MIGRATION }
+
+  describe '#up' do
+    it 'schedules a batched background migration' do
+      migrate!
+
+      expect(migration).to have_scheduled_batched_migration(
+        table_name: :epics,
+        column_name: :id,
+        interval: described_class::DELAY_INTERVAL,
+        batch_size: described_class::BATCH_SIZE,
+        max_batch_size: described_class::MAX_BATCH_SIZE,
+        sub_batch_size: described_class::SUB_BATCH_SIZE
+      )
+    end
+  end
+
+  describe '#down' do
+    it 'deletes all batched migration records' do
+      migrate!
+      schema_migrate_down!
+
+      expect(migration).not_to have_scheduled_batched_migration
+    end
+  end
+end
diff --git a/spec/services/web_hooks/log_execution_service_spec.rb b/spec/services/web_hooks/log_execution_service_spec.rb
index 1967a8368fb..7e3f33c5fbb 100644
--- a/spec/services/web_hooks/log_execution_service_spec.rb
+++ b/spec/services/web_hooks/log_execution_service_spec.rb
@@ -69,6 +69,8 @@ RSpec.describe WebHooks::LogExecutionService do
           subject(:service) { described_class.new(hook: project_hook, log_data: data, response_category: response_category) }
 
           before do
+            # stub LOCK_RETRY to be 0 in order for tests to run quicker
+            stub_const("#{described_class.name}::LOCK_RETRY", 0)
             stub_exclusive_lease_taken(lease_key, timeout: described_class::LOCK_TTL)
             allow(project_hook).to receive(:executable?).and_return(executable)
           end