Add latest changes from gitlab-org/gitlab@master

GITLAB_KAS_VERSION

@@ -1 +1 @@
-v16.9.1
+v16.9.2

app/models/project_export_job.rb

@@ -37,12 +37,4 @@ class ProjectExportJob < ApplicationRecord
     state :finished, value: STATUS[:finished]
     state :failed, value: STATUS[:failed]
   end
-
-  class << self
-    def prune_expired_jobs
-      prunable.each_batch do |relation|
-        relation.delete_all
-      end
-    end
-  end
 end

app/models/projects/import_export/relation_export_upload.rb

@@ -12,6 +12,12 @@ module Projects
         foreign_key: :project_relation_export_id,
         inverse_of: :upload
 
+      scope :for_project_export_jobs, ->(export_job_ids) do
+        joins(:relation_export).where(
+          relation_export: { project_export_job_id: export_job_ids }
+        )
+      end
+
       mount_uploader :export_file, ImportExportUploader
     end
   end

app/models/upload.rb

@@ -16,6 +16,7 @@ class Upload < ApplicationRecord
 
   scope :with_files_stored_locally, -> { where(store: ObjectStorage::Store::LOCAL) }
   scope :with_files_stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) }
+  scope :for_model_type_and_id, ->(type, id) { where(model_type: type, model_id: id) }
 
   before_save :calculate_checksum!, if: :foreground_checksummable?
   # as the FileUploader is not mounted, the default CarrierWave ActiveRecord

app/services/click_house/rebuild_materialized_view_service.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  class RebuildMaterializedViewService
+    INSERT_BATCH_SIZE = 10_000_000
+
+    VIEW_DEFINITION_QUERY = <<~SQL
+      SELECT view_definition FROM information_schema.views
+      WHERE table_name = {view_name:String} AND
+      table_schema = {database_name:String}
+    SQL
+
+    def initialize(connection:, state: {})
+      @connection = connection
+
+      @view_name = state.fetch(:view_name)
+      @tmp_view_name = state.fetch(:tmp_view_name)
+      @view_table_name = state.fetch(:view_table_name)
+      @tmp_view_table_name = state.fetch(:tmp_view_table_name)
+      @source_table_name = state.fetch(:source_table_name)
+    end
+
+    def execute
+      create_tmp_materialized_view_table
+      create_tmp_materialized_view
+
+      backfill_data
+
+      rename_table
+      drop_tmp_tables
+    end
+
+    private
+
+    attr_reader :connection, :view_name, :tmp_view_name, :view_table_name, :tmp_view_table_name, :source_table_name
+
+    def create_tmp_materialized_view_table
+      # Create a tmp table from the existing table, use IF NOT EXISTS to avoid failure when the table exists.
+      create_statement = show_create_table(view_table_name)
+        .gsub("#{connection.database_name}.#{view_table_name}",
+          "#{connection.database_name}.#{quote(tmp_view_table_name)}")
+        .gsub('CREATE TABLE', 'CREATE TABLE IF NOT EXISTS')
+
+      connection.execute(create_statement)
+    end
+
+    def create_tmp_materialized_view
+      # Create a tmp materialized view from the existing view, use IF NOT EXISTS to avoid failure when the view exists.
+      create_statement = show_create_table(view_name)
+        .gsub("#{connection.database_name}.#{view_name}",
+          "#{connection.database_name}.#{quote(tmp_view_name)}")
+        .gsub("#{connection.database_name}.#{view_table_name}",
+          "#{connection.database_name}.#{quote(tmp_view_table_name)}")
+        .gsub('CREATE MATERIALIZED VIEW', 'CREATE MATERIALIZED VIEW IF NOT EXISTS')
+
+      connection.execute(create_statement)
+    end
+
+    def backfill_data
+      # Take the query from the materialized view definition.
+      query = ClickHouse::Client::Query.new(raw_query: VIEW_DEFINITION_QUERY, placeholders: {
+        view_name: view_name,
+        database_name: connection.database_name
+      })
+      view_query = connection.select(query).first['view_definition']
+
+      iterator.each_batch(column: :id, of: INSERT_BATCH_SIZE) do |scope|
+        # Use the materialized view query to backfill the new temporary table.
+        # The materialized view query selects from the source table, example: FROM events.
+        # Replace the FROM part and select data from a batched subquery.
+        # Old: FROM events
+        # New: FROM (SELECT .. FROM events WHERE id > x and id < y) events
+        inner_query = "(#{scope.to_sql}) #{quote(source_table_name)}"
+
+        query = view_query.gsub("FROM #{connection.database_name}.#{source_table_name}", "FROM #{inner_query}")
+
+        # Insert the batch
+        connection.execute("INSERT INTO #{quote(tmp_view_table_name)} #{query}")
+      end
+    end
+
+    def rename_table
+      # Swap the tables
+      connection.execute("EXCHANGE TABLES #{quote(view_table_name)} AND #{quote(tmp_view_table_name)}")
+    end
+
+    def drop_tmp_tables
+      connection.execute("DROP TABLE IF EXISTS #{quote(tmp_view_table_name)}")
+      connection.execute("DROP TABLE IF EXISTS #{quote(tmp_view_name)}")
+    end
+
+    def show_create_table(table)
+      result = connection.select("SHOW CREATE TABLE #{quote(table)}")
+
+      raise "Table or view not found: #{table}" if result.empty?
+
+      result.first['statement']
+    end
+
+    def quote(table)
+      ApplicationRecord.connection.quote_table_name(table)
+    end
+
+    def iterator
+      builder = ClickHouse::QueryBuilder.new(source_table_name)
+      ClickHouse::Iterator.new(query_builder: builder, connection: connection)
+    end
+  end
+end

app/services/projects/import_export/prune_expired_export_jobs_service.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Projects
+  module ImportExport
+    class PruneExpiredExportJobsService
+      class << self
+        def execute
+          prunable_jobs = ProjectExportJob.prunable
+
+          delete_uploads_for_expired_jobs(prunable_jobs)
+          delete_expired_jobs(prunable_jobs)
+        end
+
+        private
+
+        def delete_expired_jobs(prunable_jobs)
+          prunable_jobs.each_batch do |relation|
+            relation.delete_all
+          end
+        end
+
+        def delete_uploads_for_expired_jobs(prunable_jobs)
+          prunable_uploads = get_uploads_for_prunable_jobs(prunable_jobs)
+          prunable_upload_keys = prunable_uploads.begin_fast_destroy
+
+          prunable_uploads.each_batch do |relation|
+            relation.delete_all
+          end
+
+          Upload.finalize_fast_destroy(prunable_upload_keys)
+        end
+
+        def get_uploads_for_prunable_jobs(prunable_jobs)
+          prunable_export_uploads = Projects::ImportExport::RelationExportUpload
+            .for_project_export_jobs(prunable_jobs.select(:id))
+
+          Upload.for_model_type_and_id(
+            Projects::ImportExport::RelationExportUpload,
+            prunable_export_uploads.select(:id)
+          )
+        end
+      end
+    end
+  end
+end

app/workers/all_queues.yml

@@ -390,6 +390,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: cronjob:click_house_rebuild_materialized_view_cron
+  :worker_name: ClickHouse::RebuildMaterializedViewCronWorker
+  :feature_category: :value_stream_management
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:concurrency_limit_resume
   :worker_name: ConcurrencyLimit::ResumeWorker
   :feature_category: :global_search

app/workers/click_house/rebuild_materialized_view_cron_worker.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  class RebuildMaterializedViewCronWorker
+    include ApplicationWorker
+    include ClickHouseWorker
+    include Gitlab::ExclusiveLeaseHelpers
+
+    idempotent!
+    queue_namespace :cronjob
+    data_consistency :delayed
+    worker_has_external_dependencies! # the worker interacts with a ClickHouse database
+    feature_category :value_stream_management
+
+    MATERIALIZED_VIEWS = [
+      {
+        view_name: 'contributions_mv',
+        view_table_name: 'contributions',
+        tmp_view_name: 'tmp_contributions_mv',
+        tmp_view_table_name: 'tmp_contributions',
+        source_table_name: 'events'
+      }.freeze
+    ].freeze
+
+    def perform
+      connection = ClickHouse::Connection.new(:main)
+      ClickHouse::RebuildMaterializedViewService
+        .new(connection: connection, state: MATERIALIZED_VIEWS.first)
+        .execute
+    end
+  end
+end

app/workers/gitlab/export/prune_project_export_jobs_worker.rb

@@ -16,7 +16,7 @@ module Gitlab
       idempotent!
 
       def perform
-        ProjectExportJob.prune_expired_jobs
+        Projects::ImportExport::PruneExpiredExportJobsService.execute
       end
     end
   end

db/docs/merge_requests_compliance_violations.yml

@@ -7,4 +7,19 @@ feature_categories:
 description: TODO
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74290
 milestone: '14.6'
-gitlab_schema: gitlab_main
+gitlab_schema: gitlab_main_cell
+allow_cross_joins:
+- gitlab_main_clusterwide
+allow_cross_transactions:
+- gitlab_main_clusterwide
+allow_cross_foreign_keys:
+- gitlab_main_clusterwide
+desired_sharding_key:
+  project_id:
+    references: projects
+    backfill_via:
+      parent:
+        foreign_key: merge_request_id
+        table: merge_requests
+        sharding_key: target_project_id
+        belongs_to: merge_request

db/docs/status_check_responses.yml

@@ -7,4 +7,19 @@ feature_categories:
 description: TODO
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61135
 milestone: '13.12'
-gitlab_schema: gitlab_main
+gitlab_schema: gitlab_main_cell
+allow_cross_joins:
+- gitlab_main_clusterwide
+allow_cross_transactions:
+- gitlab_main_clusterwide
+allow_cross_foreign_keys:
+- gitlab_main_clusterwide
+desired_sharding_key:
+  project_id:
+    references: projects
+    backfill_via:
+      parent:
+        foreign_key: merge_request_id
+        table: merge_requests
+        sharding_key: target_project_id
+        belongs_to: merge_request

db/post_migrate/20240202120723_finalize_bigint_conversion_of_geo_event_id.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class FinalizeBigintConversionOfGeoEventId < Gitlab::Database::Migration[2.2]
+  include Gitlab::Database::MigrationHelpers::ConvertToBigint
+
+  disable_ddl_transaction!
+
+  milestone '16.9'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  TABLE_NAME = 'geo_event_log'
+  COLUMN_NAME = 'geo_event_id'
+  BIGINT_COLUMN_NAME = 'geo_event_id_convert_to_bigint'
+
+  def up
+    ensure_batched_background_migration_is_finished(
+      job_class_name: 'CopyColumnUsingBackgroundMigrationJob',
+      table_name: TABLE_NAME,
+      column_name: COLUMN_NAME,
+      job_arguments: [[COLUMN_NAME], [BIGINT_COLUMN_NAME]]
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20240205084846_swap_bigint_geo_event_id.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class SwapBigintGeoEventId < Gitlab::Database::Migration[2.2]
+  include Gitlab::Database::MigrationHelpers::ConvertToBigint
+
+  milestone '16.9'
+
+  disable_ddl_transaction!
+
+  TABLE_NAME = 'geo_event_log'
+  COLUMN_NAME = 'geo_event_id'
+  INDEX_NAME = 'index_geo_event_log_on_geo_event_id'
+  BIGINT_COLUMN_NAME = 'geo_event_id_convert_to_bigint'
+
+  # For the FK from 'geo_event_log' table referencing 'geo_events'
+  FK_SOURCE_TABLE_NAME = 'geo_events'
+  FK_NAME = 'fk_geo_event_log_on_geo_event_id'
+  TEMP_FK_NAME = 'fk_geo_event_id_convert_to_bigint'
+
+  def up
+    swap
+  end
+
+  def down
+    swap
+  end
+
+  def swap
+    add_bigint_column_indexes TABLE_NAME, COLUMN_NAME
+
+    unless foreign_key_exists?(TABLE_NAME, name: TEMP_FK_NAME)
+      add_concurrent_foreign_key TABLE_NAME, FK_SOURCE_TABLE_NAME,
+        name: TEMP_FK_NAME,
+        on_delete: :cascade,
+        column: BIGINT_COLUMN_NAME
+    end
+
+    with_lock_retries(raise_on_exhaustion: true) do
+      # Lock the table to avoid deadlocks
+      execute "LOCK TABLE #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE"
+
+      # Swap the column names
+      temp_name = quote_column_name('id_tmp')
+      id_name = quote_column_name(COLUMN_NAME)
+      id_convert_to_bigint_name = quote_column_name(BIGINT_COLUMN_NAME)
+      execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_name} TO #{temp_name}"
+      execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_convert_to_bigint_name} TO #{id_name}"
+      execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{temp_name} TO #{id_convert_to_bigint_name}"
+
+      # Reset the trigger function
+      function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(
+        TABLE_NAME, connection: connection).name(
+          COLUMN_NAME,
+          BIGINT_COLUMN_NAME
+        )
+      execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL"
+    end
+
+    # Rename the temporary FK
+    execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT #{FK_NAME} CASCADE"
+    rename_constraint TABLE_NAME, TEMP_FK_NAME, FK_NAME
+
+    # Rename index
+    execute "DROP INDEX CONCURRENTLY #{INDEX_NAME}"
+    rename_index TABLE_NAME, bigint_index_name(INDEX_NAME), INDEX_NAME
+  end
+end

db/schema_migrations/20240202120723

@@ -0,0 +1 @@
+16c4e020edd4300dc246ccb8b2f101de8e9c8767dfae918659940e461776d598

db/schema_migrations/20240205084846

@@ -0,0 +1 @@
+b2d951df7d6f27ef267984f233e30e9263b3db5de0107f914e5999738b81cb87

db/structure.sql

@@ -17347,8 +17347,8 @@ CREATE TABLE geo_event_log (
     hashed_storage_attachments_event_id bigint,
     reset_checksum_event_id bigint,
     cache_invalidation_event_id bigint,
-    geo_event_id integer,
-    geo_event_id_convert_to_bigint bigint
+    geo_event_id_convert_to_bigint integer,
+    geo_event_id bigint
 );
 
 CREATE SEQUENCE geo_event_log_id_seq

doc/api/personal_access_tokens.md

@@ -211,9 +211,16 @@ Example response:
 
 ## Rotate a personal access token
 
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/403042) in GitLab 16.0
+Rotate a personal access token. Revokes the previous token and creates a new token that expires in one week
 
-Rotate a personal access token. Revokes the previous token and creates a new token that expires in one week.
+You can either:
+
+- Use the personal access token ID.
+- Pass the personal access token to the API in a request header.
+
+### Use a personal access token ID
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/403042) in GitLab 16.0
 
 In GitLab 16.6 and later, you can use the `expires_at` parameter to set a different expiry date. This non-default expiry date can be up to a maximum of one year from the rotation date.
 
@@ -250,7 +257,7 @@ Example response:
 }
 ```
 
-### Responses
+#### Responses
 
 - `200: OK` if the existing token is successfully revoked and the new token successfully created.
 - `400: Bad Request` if not rotated successfully.
@@ -259,6 +266,50 @@ Example response:
   - Token with the specified ID does not exist.
 - `404: Not Found` if the user is an administrator but the token with the specified ID does not exist.
 
+### Use a request header
+
+Requires:
+
+- `api` scope.
+
+You can use the `expires_at` parameter to set a different expiry date. This non-default expiry date can be up to a maximum of one year from the rotation date.
+
+```plaintext
+POST /personal_access_tokens/self/rotate
+```
+
+```shell
+curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens/self/rotate"
+```
+
+Example response:
+
+```json
+{
+    "id": 42,
+    "name": "Rotated Token",
+    "revoked": false,
+    "created_at": "2023-08-01T15:00:00.000Z",
+    "scopes": ["api"],
+    "user_id": 1337,
+    "last_used_at": null,
+    "active": true,
+    "expires_at": "2023-08-15",
+    "token": "s3cr3t"
+}
+```
+
+#### Responses
+
+- `200: OK` if the existing token is successfully revoked and the new token successfully created.
+- `400: Bad Request` if not rotated successfully.
+- `401: Unauthorized` if either:
+  - The token does not exist.
+  - The token has expired.
+  - The token has been revoked.
+- `403: Forbidden` if the token is not allowed to rotate itself.
+- `405: Method Not Allowed` if the token is not a personal access token.
+
 ### Automatic reuse detection
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/395352) in GitLab 16.3

doc/architecture/blueprints/organization/index.md

@@ -21,7 +21,7 @@ This document is a work in progress and represents the current state of the Orga
 - Cell: A Cell is a set of infrastructure components that contains multiple Organizations. The infrastructure components provided in a Cell are shared among Organizations, but not shared with other Cells. This isolation of infrastructure components means that Cells are independent from each other.
 - User: An Organization has many Users. Joining an Organization makes someone a User of that Organization.
 - Member: Adding a User to a Group or Project within an Organization makes them a Member. Members are always Users, but Users are not necessarily Members of a Group or Project within an Organization. For instance, a User could just have accepted the invitation to join an Organization, but not be a Member of any Group or Project it contains.
-- Non-User: A Non-User of an Organization means a User is not part of that specific Organization.
+- Non-User: A Non-User of an Organization means a User is not part of that specific Organization. Non-Users are able to interact with public Groups and Projects of an Organization, and can raise issues and comment on them.
 
 ## Summary
 
@@ -33,16 +33,16 @@ Organizations solve the following problems:
 1. Allows different Organizations to be isolated. Top-level Groups of the same Organization can interact with each other but not with Groups in other Organizations, providing clear boundaries for an Organization, similar to a self-managed instance. Isolation should have a positive impact on performance and availability as things like User dashboards can be scoped to Organizations.
 1. Allows integration with Cells. Isolating Organizations makes it possible to allocate and distribute them across different Cells.
 1. Removes the need to define hierarchies. An Organization is a container that could be filled with whatever hierarchy/entity set makes sense (Organization, top-level Groups, etc.)
-1. Enables centralized control of user profiles. With an Organization-specific user profile, administrators can control the user's role in a company, enforce user emails, or show a graphical indicator that a user as part of the Organization. An example could be adding a "GitLab employee" stamp on comments.
-1. Organizations bring an on-premise-like experience to SaaS (GitLab.com). The Organization admin will have access to instance-equivalent Admin Area settings with most of the configuration controlled at the Organization level.
+1. Enables centralized control of user profiles. With an Organization-specific user profile, administrators can control the user's role in a company, enforce user emails, or show a graphical indicator that a user is part of the Organization. An example could be adding a "GitLab employee" stamp on comments.
+1. Organizations bring an on-premise-like experience to GitLab.com. The Organization admin will have access to instance-equivalent Admin Area settings with most of the configuration controlled at the Organization level.
 
 ## Motivation
 
 ### Goals
 
-The Organization focuses on creating a better experience for Organizations to manage their GitLab experience. By introducing Organizations and [Cells](../cells/index.md) we can improve the reliability, performance and availability of our SaaS Platforms.
+The Organization focuses on creating a better experience for Organizations to manage their GitLab experience. By introducing Organizations and [Cells](../cells/index.md) we can improve the reliability, performance and availability of GitLab.com.
 
-- Wider audience: Many instance-level features are admin only. We do not want to lock out users of GitLab.com in that way. We want to make administrative capabilities that previously only existed for self-managed users available to our SaaS users as well. This also means we would give users of GitLab.com more independence from GitLab.com admins in the long run. Today, there are actions that self-managed admins can perform that GitLab.com users have to request from GitLab.com admins.
+- Wider audience: Many instance-level features are admin only. We do not want to lock out users of GitLab.com in that way. We want to make administrative capabilities that previously only existed for self-managed users available to our GitLab.com users as well. This also means we would give users of GitLab.com more independence from GitLab.com admins in the long run. Today, there are actions that self-managed admins can perform that GitLab.com users have to request from GitLab.com admins, for instance banning malicious actors.
 - Improved UX: Inconsistencies between the features available at the Project and Group levels create navigation and usability issues. Moreover, there isn't a dedicated place for Organization-level features.
 - Aggregation: Data from all Groups and Projects in an Organization can be aggregated.
 - An Organization includes settings, data, and features from all Groups and Projects under the same owner (including personal Namespaces).
@@ -55,7 +55,7 @@ Due to urgency of delivering Organizations as a prerequisite for Cells, it is cu
 
 ## Proposal
 
-We create Organizations as a new lightweight entity, with just the features and workflows which it requires. We already have much of the functionality present in Groups and Projects, and Groups themselves are essentially already the top-level entity. It is unlikely that we need to add significant features to Organizations outside of some key settings, as top-level Groups can continue to serve this purpose at least on SaaS. From an infrastructure perspective, cluster-wide shared data must be both minimal (small in volume) and infrequently written.
+We create Organizations as a new lightweight entity, with just the features and workflows which it requires. We already have much of the functionality present in Groups and Projects, and Groups themselves are essentially already the top-level entity. It is unlikely that we need to add significant features to Organizations outside of some key settings, as top-level Groups can continue to serve this purpose at least on GitLab.com. From an infrastructure perspective, cluster-wide shared data must be both minimal (small in volume) and infrequently written.
 
 ```mermaid
 graph TD
@@ -74,12 +74,12 @@ All instances would set a default Organization.
 
 - No changes to URL's for Groups moving under an Organization, which makes moving around top-level Groups very easy.
 - Low risk rollout strategy, as there is no conversion process for existing top-level Groups.
-- Organization becomes the key for identifying what is part of an Organization, which is on its own table for performance and clarity.
+- The Organization becomes the key for identifying what is part of an Organization, which is on its own table for performance and clarity.
 
 ### Drawbacks
 
-- It is unclear right now how we would avoid continuing to spend effort to build instance (or not Organization) features, in particular much of the reporting. This is not an issue on SaaS as top-level Groups already have this capability, however it is a challenge on self-managed. If we introduce a built-in Organization (or just none at all) for self-managed, it seems like we would need to continue to build instance/Organization level reporting features as we would not get that for free along with the work to add to Groups.
-- Billing may need to be moved from top-level Groups to Organization level.
+- It is unclear right now how we would avoid continuing to spend effort to build instance (or not Organization) features, in particular much of the reporting. This is not an issue on GitLab.com as top-level Groups already have this capability, however, it is a challenge on self-managed. If we introduce a built-in Organization (or just none at all) for self-managed, it seems like we would need to continue to build instance/Organization level reporting features as we would not get that for free along with the work to add to Groups.
+- Billing may need to be moved from top-level Groups to the Organization level.
 
 ## Data Exploration
 
@@ -99,138 +99,39 @@ Based on this analysis we expect to see similar behavior when rolling out Organi
 
 ## Design and Implementation Details
 
+Cells will be rolled out in three phases: Cells 1.0, Cells 1.5 and Cells 2.0.
+The Organization functionality available in each phase is described below.
+
 ### Organization MVC
 
-The Organization MVC will contain the following functionality:
+#### Organizations on Cells 1.0
+
+The Organization MVC for Cells 1.0 will contain the following functionality:
 
 - Instance setting to allow the creation of multiple Organizations. This will be enabled by default on GitLab.com, and disabled for self-managed GitLab.
 - Every instance will have a default Organization named `Default Organization`. Initially, all Users will be managed by this default Organization.
 - Organization Owner. The creation of an Organization appoints that User as the Organization Owner. Once established, the Organization Owner can appoint other Organization Owners.
-- Organization Users. A User is managed by one Organization, but can be part of multiple Organizations. Users are able to navigate between the different Organizations they are part of.
-- Setup settings. Containing the Organization name, ID, description, and avatar. Settings are editable by the Organization Owner.
-- Setup flow. Users are able to build new Organizations. They can also create new top-level Groups in an Organization.
-- Visibility. Initially, Organizations can only be `public`. Public Organizations can be seen by everyone. They can contain public and private Groups and Projects.
+- Organization Users. A User can only be part of one Organization for Cells 1.0. A new account needs to be created for each Organization a User wants to be part of.
+- Setup settings. Containing the Organization name, ID, description, and avatar. Organization settings are editable by the Organization Owner.
+- Setup flow. New Users are able to create new Organizations. They can also create new top-level Groups in an Organization.
+- Visibility. Initially, Organizations can only be `private`. Private Organizations can only be seen by the Users that are part of the private Organization. They can only contain private Groups and Projects.
 - Organization settings page with the added ability to remove an Organization. Deletion of the default Organization is prevented.
 - Groups. This includes the ability to create, edit, and delete Groups, as well as a Groups overview that can be accessed by the Organization Owner and Users.
 - Projects. This includes the ability to create, edit, and delete Projects, as well as a Projects overview that can be accessed by the Organization Owner and Users.
-- Personal Namespaces. Users get [a personal Namespace in each Organization](../cells/impacted_features/personal-namespaces.md) they interact with.
+- Personal Namespaces. Users get [a personal Namespace in each Organization](../cells/impacted_features/personal-namespaces.md) they are associated with.
 - User Profile. Each [User Profile will be scoped to the Organization](../cells/impacted_features/user-profile.md).
 
+#### Organizations on Cells 1.5
+
+Organizations in the context of Cells 1.5 will contain the following functionality:
+
+#### Organizations on Cells 2.0
+
+Organizations in the context of Cells 2.0 will contain the following functionality:
+
 ### Organization Access
 
-#### Organization Users
-
-Organization Users can get access to Groups and Projects as:
-
-- A Group Member: this grants access to the Group and all its Projects, regardless of their visibility.
-- A Project Member: this grants access to the Project, and limited access to parent Groups, regardless of their visibility.
-- A Non-Member: this grants access to public and internal Groups and Projects of that Organization. To access a private Group or Project in an Organization, a User must become a Member.
-
-Organization Users can be managed in the following ways:
-
-- As [Enterprise Users](../../../user/enterprise_user/index.md), managed by the Organization. This includes control over their User account and the ability to block the User.
-- As Non-Enterprise Users, managed by the default Organization. Non-Enterprise Users can be removed from an Organization, but the User keeps ownership of their User account.
-
-Enterprise Users are only available to Organizations with a Premium or Ultimate subscription. Organizations on the free tier will only be able to host Non-Enterprise Users.
-
-##### How do Users join an Organization?
-
-Users are visible across all Organizations. This allows Users to move between Organizations. Users can join an Organization by:
-
-1. Becoming a Member of a Namespace (Group, Subgroup, or Project) contained within an Organization. A User can become a Member of a Namespace by:
-
-   - Being invited by username
-   - Being invited by email address
-   - Requesting access. This requires visibility of the Organization and Namespace and must be accepted by the owner of the Namespace. Access cannot be requested to private Groups or Projects.
-
-1. Becoming an Enterprise User of an Organization. Bringing Enterprise Users to the Organization level is planned post MVC. For the Organization MVC Enterprise Users will remain at the top-level Group.
-
-The creator of an Organization automatically becomes the Organization Owner. It is not necessary to become a User of a specific Organization to comment on or create public issues, for example. All existing Users can create and comment on all public issues.
-
-##### When can Users see an Organization?
-
-For the MVC, an Organization can only be public. Public Organizations can be seen by everyone. They can contain public and private Groups and Projects.
-
-In the future, Organizations will get an additional internal visibility setting for Groups and Projects. This will allow us to introduce internal Organizations that can only be seen by the Users it contains. This would mean that only Users that are part of the Organization will see:
-
-- The Organization front page, instead of a 404 when navigating the Organization URL
-- Name of the organization
-- Description of the organization
-- Organization pages, such as the Activity page, Groups, Projects and Users overview
-
-Content of these pages will be determined by each User's access to specific Groups and Projects. For instance, private Projects would only be seen by the members of this Project in the Project overview.
-
-As an end goal, we plan to offer the following scenarios:
-
-| Organization visibility | Group/Project visibility | Who sees the Organization? | Who sees Groups/Projects? |
-| ------ | ------ | ------ | ------ |
-| public | public | Everyone | Everyone |
-| public | internal | Everyone | Organization Users |
-| public | private | Everyone | Group/Project members |
-| internal | internal | Organization Users | Organization Users |
-| internal | private | Organization Users | Group/Project members |
-
-##### What can Users see in an Organization?
-
-Users can see the things that they have access to in an Organization. For instance, an Organization User would be able to access only the private Groups and Projects that they are a Member of, but could see all public Groups and Projects. Actionable items such as issues, merge requests and the to-do list are seen in the context of the Organization. This means that a User might see 10 merge requests they created in `Organization A`, and 7 in `Organization B`, when in total they have created 17 merge requests across both Organizations.
-
-##### What is a Billable Member?
-
-How Billable Members are defined differs between GitLabs two main offerings:
-
-- Self-managed (SM): [Billable Members are Users who consume seats against the SM License](../../../subscriptions/self_managed/index.md#subscription-seats). Custom roles elevated above the Guest role are consuming seats.
-- GitLab.com (SaaS): [Billable Members are Users who are Members of a Namespace (Group or Project) that consume a seat against the SaaS subscription for the top-level Group](../../../subscriptions/gitlab_com/index.md#how-seat-usage-is-determined). Currently, [Users with Minimal Access](../../../user/permissions.md#users-with-minimal-access) and Users without a Group count towards a licensed seat, but [that's changing](https://gitlab.com/gitlab-org/gitlab/-/issues/330663#note_1133361094).
-
-These differences and how they are calculated and displayed often cause confusion. For both SM and SaaS, we evaluate whether a User consumes a seat against the same core rule set:
-
-1. They are active users
-1. They are not bot users
-1. For the Ultimate tier, they are not a Guest
-
-For (1) this is determined differently per offering, in terms of both what classifies as active and also due to the underlying model that we refer to (User vs Member).
-To help demonstrate the various associations used in GitLab relating to Billable Members, here is a relationship diagram:
-
-```mermaid
-graph TD
-        A[Group] <-.type of.- B[Namespace]
-        C[Project] -.belongs to.-> A
-
-        E[GroupMember] <-.type of.- D[Member]
-        G[User] -.has many.-> F
-        F -.belongs to.-> C
-        F[ProjectMember] <-.type of.- D
-        G -.has many.-> E -.belongs to.-> A
-
-        GGL[GroupGroupLink] -.belongs to.->A
-        PGL[ProjectGroupLink] -.belongs to.->A
-        PGL -.belongs to.->C
-```
-
-GroupGroupLink is the join table between two Group records, indicating that one Group has invited the other.
-ProjectGroupLink is the join table between a Group and a Project, indicating the Group has been invited to the Project.
-
-SaaS has some additional complexity when it comes to the relationships that determine whether or not a User is considered a Billable Member, particularly relating to Group/Project membership that can often lead to confusion. An example of that are Members of a Group that have been invited into another Group or Project and therewith become billable.
-There are two charts as the flow is different for each: [SaaS](https://mermaid.live/view#pako:eNqNVl1v2jAU_StXeS5M-3hCU6N2aB3SqKbSPkyAhkkuxFsSs9hpVUX899mxYxsnlOWFcH1877nnfkATJSzFaBLtcvaSZKQS8DhdlWCeijGxXBCygCeOFdzSPCfbHOGrRK9Ho2tlvUkEfcZmo97HXBCBG6AcSGuOj86ZA8No_BP5eHQNMz7HYovV8kuGyR-gOx1I3Qd9Ap-31btrtgORITxIPnBXsfoAGcWKVEn2uj4T4Z6pAPdMdKyX8t2mIG-5ex0LkCnBdO4OOrOhO-O3TDQzrkkSkN9izW-BCCUTCB-8hGU866Bl45FxKJ-GdGiDDYI7SOtOp7o0GW90rA20NYjXQxE6cWSaGr1Q2BnX9hCnIbZWc1reJAly3pisMsJ19vKEFiQHfQw5PmMenwqhPQ5Uxa-DjeAa5IJk_g3t-hvdZ8jFA8vxrpYvccfWHIA6aVmrLtMQj2rvuqPynSZYcnx8PWDzlAuZsay3MfouPJxl1c9hKFCIPedzSBuH5fV2X5FDBrT8Zadk2bbszJur_xsp9UznzZRWmIizV-Njx346X9TbPpwoVqO9xobebUZmF3gse0yk9wA-jDBkflTst2TS-EyMTcrTZmGz7hPrkG8HdChdv1n5TAWmGuxHLmXI9qgTza9aO93-TVfnobAh1M6V0VDtuk7E0w313tMUy3Swc_Tyll9VLUwMPcFxUJGBNdKYTTTwY-ByesC_qusx1Yk0bXtao9kk8Snzj8eLsX0lwqV2ujnUE5Bw7FT4g7QbQGM-4YWoXPRZ2C7BnT4TXZPSiAHFUIP3nVhGbiN3G9-OyKWsTvpSS60yMYZA5U_HtyQzdy7p7GCBon65OyXNWJwT9DSNMwF7YB3Xly1o--gqKrAqCE3l359GHa4iuQ8KXEUT-ZrijtS5WEWr8iihpBZs8Vom0WRHco5XUX1IZd9NKZETUxjr8R82ROYl) and [SM](https://mermaid.live/view#pako:eNqFk1FvwiAQx7_KhefVD-CDZo2JNdmcWe3DYpeI7alsLRgKLob0u48qtqxRx9Plz4-7-3NgSCZyJEOyLcRPtqdSwXKScnBLVyhXswrUHiGxMYSsKOimwPHnXwiCYNQAsaIKzXOm2BFh3ShrOGvjujvQghAMPrAaBCOITKRLyu9Rc9FAc6Gu9VPegVELLEKzkOILMwWhUH6yRdhCcWJilEeWXSz5VJzcqrWycWvc830rOmdwnmZ8KoU-vEnXU6-bf6noPmResdzYWxdboHDeAiHBbfqOuqifonX6Ym-CV7g8HfAhfZ0U2-2xUu-iwKm2wdg4BRoJWAUXufZH5JnqH-8ye42YpFCsbGbvRN-Tx7UmunfxqFCfvZfTNeS9AfJESpQlZbn9K6Y5lxL7KUpMydCGOZXfKUl5bTmqlYhPPCNDJTU-EX3IrZEJoztJy4tY_wJJwxFj).
-
-##### How can Users switch between different Organizations?
-
-Users can utilize a [context switcher](https://gitlab.com/gitlab-org/gitlab/-/issues/411637). This feature allows easy navigation and access to different Organizations' content and settings. By clicking on the context switcher and selecting a specific Organization from the provided list, Users can seamlessly transition their view and permissions, enabling them to interact with the resources and functionalities of the chosen Organization.
-
-##### What happens when a User is deleted?
-
-We've identified three different scenarios where a User can be removed from an Organization:
-
-1. Removal: The User is removed from the organization_users table. This is similar to the User leaving a company, but the User can join the Organization again after access approval.
-1. Banning: The User is banned. This can happen in case of misconduct but the User cannot be added again to the Organization until they are unbanned. In this case, we keep the organization_users entry and change the permission to none.
-1. Deleting: The User is deleted. We assign everything the User has authored to the Ghost User and delete the entry from the organization_users table.
-
-As part of the Organization MVC, Organization Owners can remove Organization Users. This means that the User's membership entries are deleted from all Groups and Projects that are contained within the Organization. In addition, the User entry is removed from the `organization_users` table.
-
-Actions such as banning and deleting a User will be added to the Organization at a later point.
-
-#### Organization Non-Users
-
-Non-Users are external to the Organization and can only access the public resources of an Organization, such as public Projects.
+See [Organization Users](organization-users.md). 
 
 ### Roles and Permissions
 
@@ -309,9 +210,10 @@ In iteration 1, we introduce the concept of an Organization as a way to group to
 - The creator of the Organization is assigned as the Organization Owner.
 - Groups can be created in an Organization. Groups are listed in the Groups overview. Every Organization User can access the Groups overview and see the Groups they have access to.
 - Projects can be created in a Group. Projects are listed in the Projects overview. Every Organization User can access the Projects overview and see the Projects they have access to.
+- Users are listed in the User overview. Every Organization User can access the User overview and see Users that are part of the Groups and Projects they have access to.
 - Both Enterprise and Non-Enterprise Users can be part of an Organization.
 - Enterprise Users are still managed by top-level Groups.
-- A User can be part of multiple Organizations.
+- A User can be part of one Organization.
 - Users can navigate between the different Organizations they are part of.
 - Any User within or outside of an Organization can be invited to Groups and Projects contained by the Organization.
 - Organizations are not fully isolated. We aim to complete [phase 1 of Organization isolation](https://gitlab.com/groups/gitlab-org/-/epics/11837), with the goal to `define sharding_key` and `desired_sharding_key` rules.
@@ -320,11 +222,10 @@ In iteration 1, we introduce the concept of an Organization as a way to group to
 
 In iteration 2, an Organization MVC Experiment will be released. We will test the functionality with a select set of customers and improve the MVC based on these learnings. The MVC Experiment contains the following functionality:
 
-- Users are listed in the User overview. Every Organization User can access the User overview and see Users that are part of the Groups and Projects they have access to.
 - Organizations can be deleted.
 - Organization Owners can access the Activity page for the Organization.
 - Forking across Organizations will be defined.
-- [Organization Isolation](isolation.md) will be finished to meet the requirements of the initial set of customers
+- [Organization Isolation](isolation.md) meets the requirements of the initial set of customers
 
 ### Iteration 3: [Organization MVC Beta](https://gitlab.com/groups/gitlab-org/-/epics/10651) (FY25Q3)
 

doc/architecture/blueprints/organization/organization-faq.md

@@ -20,7 +20,7 @@ GitLab Dedicated will be a much better fit for that at this moment.
 ## Do we expect Organizations to have visibility settings (public/private) of their own? Will visibility remain a property of top-level Groups?
 
 Organizations are public for now but will have their own independent visibility settings.
-See also [When can Users see an Organization?](index.md#when-can-users-see-an-organization).
+See also [When can Users see an Organization?](organization-users.md#when-can-users-see-an-organization).
 
 ## What would the migration of a feature from the top-level Group to the Organization look like?
 

doc/architecture/blueprints/organization/organization-users.md

@@ -0,0 +1,133 @@
+---
+stage: enablement
+group: Tenant Scale
+description: 'Organization Users'
+---
+
+# Organization Users
+
+Users can become an Organization User in the following way:
+
+- Organization Owners create an account on behalf of a user, and then share it with the user.
+
+Organization Users can get access to Groups and Projects in an Organization as:
+
+- A Group Member: this grants access to the Group and all its Projects, regardless of their visibility.
+- A Project Member: this grants access to the Project, and limited access to parent Groups, regardless of their visibility.
+- A Non-Member: this grants access to public and internal Groups and Projects of that Organization. To access a private Group or Project in an Organization, a User must become a Member. Internal visibility will not be available for Organization in Cells 1.0.
+
+Organization Users can be managed in the following ways:
+
+- As [Enterprise Users](../../../user/enterprise_user/index.md), managed by the Organization. This includes control over their User account and the ability to block the User. In the context of Cells 1.0, Organization Users will essentially function like Enterprise Users.
+- As Non-Enterprise Users, managed by the default Organization. Non-Enterprise Users can be removed from an Organization, but the User keeps ownership of their User account. This will only be considered post Cells 1.0.
+
+Enterprise Users are only available to Organizations with a Premium or Ultimate subscription. Organizations on the free tier will only be able to host Non-Enterprise Users.
+
+## How do Users join an Organization?
+
+Users are visible across all Organizations. This allows Users to move between Organizations. Users can join an Organization by:
+
+1. Being invited by an Organization Owner. Because Organizations are private on Cells 1.0, only the Organization Owner can add new Users to an Organization by iniviting them to create an account.
+
+1. Becoming a Member of a Namespace (Group, Subgroup, or Project) contained within an Organization. A User can become a Member of a Namespace by:
+
+   - Being invited by username
+   - Being invited by email address
+   - Requesting access. This requires visibility of the Organization and Namespace and must be accepted by the owner of the Namespace. Access cannot be requested to private Groups or Projects.
+
+1. Becoming an Enterprise User of an Organization. Bringing Enterprise Users to the Organization level is planned post MVC. For the Organization MVC Enterprise Users will remain at the top-level Group.
+
+The creator of an Organization automatically becomes the Organization Owner. It is not necessary to become a User of a specific Organization to comment on or create public issues, for example. All existing Users can create and comment on all public issues.
+
+## How do Users log into an Organization?
+
+TBD
+
+## When can Users see an Organization?
+
+For Cells 1.0, an Organization can only be private. Private Organizations can only be seen by their Organization Users. They can only contain private Groups and Projects.
+
+For Cells 1.5, Organizations can also be public. Public Organizations can be seen by everyone. They can contain public and private Groups and Projects.
+
+In the future, Organizations will get an additional internal visibility setting for Groups and Projects. This will allow us to introduce internal Organizations that can only be seen by the Users it contains. This would mean that only Users that are part of the Organization will see:
+
+- The Organization front page, instead of a 404 when navigating to the Organization URL
+- Name of the Organization
+- Description of the Organization
+- Organization pages, such as the Activity page, Groups, Projects, and Users overview. Content of these pages will be determined by each User's access to specific Groups and Projects. For instance, private Projects would only be seen by the members of this Project in the Project overview.
+- Internal Groups and Projects
+
+As an end goal, we plan to offer the following scenarios:
+
+| Organization visibility | Group/Project visibility | Who sees the Organization? | Who sees Groups/Projects? |
+| ------ | ------ | ------ | ------ |
+| public | public | Everyone | Everyone |
+| public | internal | Everyone | Organization Users |
+| public | private | Everyone | Group/Project members |
+| internal | internal | Organization Users | Organization Users |
+| internal | private | Organization Users | Group/Project members |
+| private | private | Organization Users | Group/Project members |
+
+## What can Users see in an Organization?
+
+Users can see the things that they have access to in an Organization. For instance, an Organization User would be able to access only the private Groups and Projects that they are a Member of, but could see all public Groups and Projects. Actionable items such as issues, merge requests and the to-do list are seen in the context of the Organization. This means that a User might see 10 merge requests they created in `Organization A`, and 7 in `Organization B`, when in total they have created 17 merge requests across both Organizations.
+
+## What is a Billable Member?
+
+How Billable Members are defined differs between GitLabs two main offerings:
+
+- Self-managed (SM): [Billable Members are Users who consume seats against the SM License](../../../subscriptions/self_managed/index.md#subscription-seats). Custom roles elevated above the Guest role are consuming seats.
+- GitLab.com (SaaS): [Billable Members are Users who are Members of a Namespace (Group or Project) that consume a seat against the SaaS subscription for the top-level Group](../../../subscriptions/gitlab_com/index.md#how-seat-usage-is-determined). Currently, [Users with Minimal Access](../../../user/permissions.md#users-with-minimal-access) and Users without a Group count towards a licensed seat, but [that's changing](https://gitlab.com/gitlab-org/gitlab/-/issues/330663#note_1133361094).
+
+These differences and how they are calculated and displayed often cause confusion. For both SM and SaaS, we evaluate whether a User consumes a seat against the same core rule set:
+
+1. They are active users
+1. They are not bot users
+1. For the Ultimate tier, they are not a Guest
+
+For (1) this is determined differently per offering, in terms of both what classifies as active and also due to the underlying model that we refer to (User vs Member).
+To help demonstrate the various associations used in GitLab relating to Billable Members, here is a relationship diagram:
+
+```mermaid
+graph TD
+        A[Group] <-.type of.- B[Namespace]
+        C[Project] -.belongs to.-> A
+
+        E[GroupMember] <-.type of.- D[Member]
+        G[User] -.has many.-> F
+        F -.belongs to.-> C
+        F[ProjectMember] <-.type of.- D
+        G -.has many.-> E -.belongs to.-> A
+
+        GGL[GroupGroupLink] -.belongs to.->A
+        PGL[ProjectGroupLink] -.belongs to.->A
+        PGL -.belongs to.->C
+```
+
+GroupGroupLink is the join table between two Group records, indicating that one Group has invited the other.
+ProjectGroupLink is the join table between a Group and a Project, indicating the Group has been invited to the Project.
+
+SaaS has some additional complexity when it comes to the relationships that determine whether or not a User is considered a Billable Member, particularly relating to Group/Project membership that can often lead to confusion. An example of that are Members of a Group that have been invited into another Group or Project and therewith become billable.
+There are two charts as the flow is different for each: [SaaS](https://mermaid.live/view#pako:eNqNVl1v2jAU_StXeS5M-3hCU6N2aB3SqKbSPkyAhkkuxFsSs9hpVUX899mxYxsnlOWFcH1877nnfkATJSzFaBLtcvaSZKQS8DhdlWCeijGxXBCygCeOFdzSPCfbHOGrRK9Ho2tlvUkEfcZmo97HXBCBG6AcSGuOj86ZA8No_BP5eHQNMz7HYovV8kuGyR-gOx1I3Qd9Ap-31btrtgORITxIPnBXsfoAGcWKVEn2uj4T4Z6pAPdMdKyX8t2mIG-5ex0LkCnBdO4OOrOhO-O3TDQzrkkSkN9izW-BCCUTCB-8hGU866Bl45FxKJ-GdGiDDYI7SOtOp7o0GW90rA20NYjXQxE6cWSaGr1Q2BnX9hCnIbZWc1reJAly3pisMsJ19vKEFiQHfQw5PmMenwqhPQ5Uxa-DjeAa5IJk_g3t-hvdZ8jFA8vxrpYvccfWHIA6aVmrLtMQj2rvuqPynSZYcnx8PWDzlAuZsay3MfouPJxl1c9hKFCIPedzSBuH5fV2X5FDBrT8Zadk2bbszJur_xsp9UznzZRWmIizV-Njx346X9TbPpwoVqO9xobebUZmF3gse0yk9wA-jDBkflTst2TS-EyMTcrTZmGz7hPrkG8HdChdv1n5TAWmGuxHLmXI9qgTza9aO93-TVfnobAh1M6V0VDtuk7E0w313tMUy3Swc_Tyll9VLUwMPcFxUJGBNdKYTTTwY-ByesC_qusx1Yk0bXtao9kk8Snzj8eLsX0lwqV2ujnUE5Bw7FT4g7QbQGM-4YWoXPRZ2C7BnT4TXZPSiAHFUIP3nVhGbiN3G9-OyKWsTvpSS60yMYZA5U_HtyQzdy7p7GCBon65OyXNWJwT9DSNMwF7YB3Xly1o--gqKrAqCE3l359GHa4iuQ8KXEUT-ZrijtS5WEWr8iihpBZs8Vom0WRHco5XUX1IZd9NKZETUxjr8R82ROYl) and [SM](https://mermaid.live/view#pako:eNqFk1FvwiAQx7_KhefVD-CDZo2JNdmcWe3DYpeI7alsLRgKLob0u48qtqxRx9Plz4-7-3NgSCZyJEOyLcRPtqdSwXKScnBLVyhXswrUHiGxMYSsKOimwPHnXwiCYNQAsaIKzXOm2BFh3ShrOGvjujvQghAMPrAaBCOITKRLyu9Rc9FAc6Gu9VPegVELLEKzkOILMwWhUH6yRdhCcWJilEeWXSz5VJzcqrWycWvc830rOmdwnmZ8KoU-vEnXU6-bf6noPmResdzYWxdboHDeAiHBbfqOuqifonX6Ym-CV7g8HfAhfZ0U2-2xUu-iwKm2wdg4BRoJWAUXufZH5JnqH-8ye42YpFCsbGbvRN-Tx7UmunfxqFCfvZfTNeS9AfJESpQlZbn9K6Y5lxL7KUpMydCGOZXfKUl5bTmqlYhPPCNDJTU-EX3IrZEJoztJy4tY_wJJwxFj).
+
+## How can Users switch between different Organizations?
+
+For Organizations in the context of Cells 1.0, Users will only be able to be part of a single Organization. If a user wants to be part of multiple Organizations, they have to join every additional Organization with a new user account.
+
+Later, in the context of Cells 1.5, Users can utilize a [context switcher](https://gitlab.com/gitlab-org/gitlab/-/issues/411637). This feature allows easy navigation and access to different Organizations' content and settings. By clicking on the context switcher and selecting a specific Organization from the provided list, Users can seamlessly transition their view and permissions, enabling them to interact with the resources and functionalities of the chosen Organization.
+
+## What happens when a User is deleted?
+
+We've identified three different scenarios where a User can be removed from an Organization:
+
+1. Removal: The User is removed from the organization_users table. This is similar to the User leaving a company, but the User can join the Organization again after access approval.
+1. Banning: The User is banned. This can happen in case of misconduct but the User cannot be added again to the Organization until they are unbanned. In this case, we keep the organization_users entry and change the permission to none.
+1. Deleting: The User is deleted. We assign everything the User has authored to the Ghost User and delete the entry from the organization_users table.
+
+As part of the Organization MVC, Organization Owners can remove Organization Users. This means that the User's membership entries are deleted from all Groups and Projects that are contained within the Organization. In addition, the User entry is removed from the `organization_users` table.
+
+Actions such as banning and deleting a User will be added to the Organization at a later point.
+
+## Organization Non-Users
+
+Non-Users are external to the Organization and can only access the public resources of an Organization, such as public Projects.

lib/api/api.rb

@@ -296,6 +296,7 @@ module API
         mount ::API::Pages
         mount ::API::PagesDomains
         mount ::API::PersonalAccessTokens::SelfInformation
+        mount ::API::PersonalAccessTokens::SelfRotation
         mount ::API::PersonalAccessTokens
         mount ::API::ProjectAvatar
         mount ::API::ProjectClusters

lib/api/helpers/personal_access_tokens_helpers.rb

@@ -33,6 +33,18 @@ module API
 
         service.success? ? no_content! : bad_request!(nil)
       end
+
+      def rotate_token(token, params)
+        service = ::PersonalAccessTokens::RotateService.new(current_user, token).execute(params)
+
+        if service.success?
+          status :ok
+
+          service.payload[:personal_access_token]
+        else
+          bad_request!(service.message)
+        end
+      end
     end
   end
 end

lib/api/personal_access_tokens.rb

@@ -82,16 +82,9 @@ module API
         token = PersonalAccessToken.find_by_id(params[:id])
 
         if Ability.allowed?(current_user, :manage_user_personal_access_token, token&.user)
-          response = ::PersonalAccessTokens::RotateService.new(current_user, token).execute(declared_params)
+          new_token = rotate_token(token, declared_params)
 
-          if response.success?
-            status :ok
-
-            new_token = response.payload[:personal_access_token]
-            present new_token, with: Entities::PersonalAccessTokenWithToken
-          else
-            bad_request!(response.message)
-          end
+          present new_token, with: Entities::PersonalAccessTokenWithToken
         else
           # Only admins should be informed if the token doesn't exist
           current_user.can_admin_all_resources? ? not_found! : unauthorized!

lib/api/personal_access_tokens/self_rotation.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module API
+  class PersonalAccessTokens
+    class SelfRotation < ::API::Base
+      include APIGuard
+
+      feature_category :system_access
+
+      helpers ::API::Helpers::PersonalAccessTokensHelpers
+
+      allow_access_with_scope :api
+
+      before { authenticate! }
+
+      resource :personal_access_tokens do
+        desc 'Rotate a personal access token' do
+          detail 'Rotates a personal access token by passing it to the API in a header'
+          success code: 200, model: Entities::PersonalAccessTokenWithToken
+          failure [
+            { code: 400, message: 'Bad Request' },
+            { code: 401, message: 'Unauthorized' },
+            { code: 403, message: 'Forbidden' },
+            { code: 405, message: 'Method not allowed' }
+          ]
+          tags %w[personal_access_tokens]
+        end
+        params do
+          optional :expires_at,
+            type: Date,
+            desc: "The expiration date of the token",
+            documentation: { example: '2021-01-31' }
+        end
+        post 'self/rotate' do
+          not_allowed! unless access_token.is_a? PersonalAccessToken
+          forbidden! if current_user.project_bot?
+
+          new_token = rotate_token(access_token, declared_params)
+
+          present new_token, with: Entities::PersonalAccessTokenWithToken
+        end
+      end
+    end
+  end
+end

lib/click_house/connection.rb

@@ -15,13 +15,16 @@ module ClickHouse
       ClickHouse::Client.execute(query, database, configuration)
     end
 
+    def database_name
+      configuration.databases[database]&.database
+    end
+
     def table_exists?(table_name)
       raw_query = <<~SQL.squish
         SELECT 1 FROM system.tables
         WHERE name = {table_name: String} AND database = {database_name: String}
       SQL
 
-      database_name = configuration.databases[database]&.database
       placeholders = { table_name: table_name, database_name: database_name }
 
       query = ClickHouse::Client::Query.new(raw_query: raw_query, placeholders: placeholders)

lib/gitlab/auth/auth_finders.rb

@@ -426,7 +426,8 @@ module Gitlab
       end
 
       def access_token_rotation_request?
-        current_request.path.match(%r{access_tokens/\d+/rotate$})
+        current_request.path.match(%r{access_tokens/\d+/rotate$}) ||
+          current_request.path.match(%r{/personal_access_tokens/self/rotate$})
       end
     end
   end

spec/lib/click_house/connection_spec.rb

@@ -5,6 +5,13 @@ require 'spec_helper'
 RSpec.describe ClickHouse::Connection, click_house: :without_migrations, feature_category: :database do
   let(:connection) { described_class.new(:main) }
 
+  describe '#database_name' do
+    it 'returns the configured database name' do
+      name = ClickHouse::Client.configuration.databases[:main].database
+      expect(connection.database_name).to eq(name)
+    end
+  end
+
   describe '#select' do
     it 'proxies select to client' do
       expect(

spec/models/project_export_job_spec.rb

@@ -13,54 +13,4 @@ RSpec.describe ProjectExportJob, feature_category: :importers, type: :model do
     it { is_expected.to validate_presence_of(:jid) }
     it { is_expected.to validate_presence_of(:status) }
   end
-
-  context 'when pruning expired jobs' do
-    let_it_be(:old_job_1) { create(:project_export_job, updated_at: 37.months.ago) }
-    let_it_be(:old_job_2) { create(:project_export_job, updated_at: 12.months.ago) }
-    let_it_be(:old_job_3) { create(:project_export_job, updated_at: 8.days.ago) }
-    let_it_be(:fresh_job_1) { create(:project_export_job, updated_at: 1.day.ago) }
-    let_it_be(:fresh_job_2) { create(:project_export_job, updated_at: 2.days.ago) }
-    let_it_be(:fresh_job_3) { create(:project_export_job, updated_at: 6.days.ago) }
-
-    let_it_be(:old_relation_export_1) { create(:project_relation_export, project_export_job_id: old_job_1.id) }
-    let_it_be(:old_relation_export_2) { create(:project_relation_export, project_export_job_id: old_job_2.id) }
-    let_it_be(:old_relation_export_3) { create(:project_relation_export, project_export_job_id: old_job_3.id) }
-    let_it_be(:fresh_relation_export_1) { create(:project_relation_export, project_export_job_id: fresh_job_1.id) }
-
-    let_it_be(:old_upload_1) { create(:relation_export_upload, project_relation_export_id: old_relation_export_1.id) }
-    let_it_be(:old_upload_2) { create(:relation_export_upload, project_relation_export_id: old_relation_export_2.id) }
-    let_it_be(:old_upload_3) { create(:relation_export_upload, project_relation_export_id: old_relation_export_3.id) }
-    let_it_be(:fresh_upload_1) do
-      create(
-        :relation_export_upload,
-        project_relation_export_id: fresh_relation_export_1.id
-      )
-    end
-
-    it 'prunes jobs and associations older than 7 days' do
-      expect { described_class.prune_expired_jobs }.to change { described_class.count }.by(-3)
-
-      expect(described_class.find_by(id: old_job_1.id)).to be_nil
-      expect(described_class.find_by(id: old_job_2.id)).to be_nil
-      expect(described_class.find_by(id: old_job_3.id)).to be_nil
-
-      expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_1.id)).to be_nil
-      expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_2.id)).to be_nil
-      expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_3.id)).to be_nil
-
-      expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_1.id)).to be_nil
-      expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_2.id)).to be_nil
-      expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_3.id)).to be_nil
-    end
-
-    it 'does not delete associated records for jobs younger than 7 days' do
-      described_class.prune_expired_jobs
-
-      expect(fresh_job_1.reload).to be_present
-      expect(fresh_job_2.reload).to be_present
-      expect(fresh_job_3.reload).to be_present
-      expect(fresh_relation_export_1.reload).to be_present
-      expect(fresh_upload_1.reload).to be_present
-    end
-  end
 end

spec/models/projects/import_export/relation_export_upload_spec.rb

@@ -11,6 +11,35 @@ RSpec.describe Projects::ImportExport::RelationExportUpload, type: :model do
     it { is_expected.to belong_to(:relation_export) }
   end
 
+  describe '.for_project_export_jobs' do
+    let_it_be(:project_export_job_1) { create(:project_export_job) }
+    let_it_be(:project_export_job_2) { create(:project_export_job) }
+
+    let_it_be(:relation_export_1) { create(:project_relation_export, project_export_job: project_export_job_1) }
+    let_it_be(:relation_export_2) { create(:project_relation_export, project_export_job: project_export_job_2) }
+    let_it_be(:relation_export_3) do
+      create(:project_relation_export, project_export_job: project_export_job_1, relation: 'milestones')
+    end
+
+    let_it_be(:relation_export_upload_1) { create(:relation_export_upload, relation_export: relation_export_1) }
+    let_it_be(:relation_export_upload_2) { create(:relation_export_upload, relation_export: relation_export_2) }
+    let_it_be(:relation_export_upload_3) { create(:relation_export_upload, relation_export: relation_export_1) }
+
+    it 'returns RelationExportUploads for a single ProjectExportUpload id' do
+      project_export_job_id = project_export_job_1.id
+
+      expect(described_class.for_project_export_jobs(project_export_job_id))
+        .to contain_exactly(relation_export_upload_1, relation_export_upload_3)
+    end
+
+    it 'returns RelationExportUploads for multiple ProjectExportUpload ids' do
+      project_export_job_ids = [project_export_job_1, project_export_job_2].map(&:id)
+
+      expect(described_class.for_project_export_jobs(project_export_job_ids))
+        .to contain_exactly(relation_export_upload_1, relation_export_upload_2, relation_export_upload_3)
+    end
+  end
+
   it 'stores export file' do
     stub_uploads_object_storage(ImportExportUploader, enabled: false)
 

spec/models/upload_spec.rb

@@ -61,6 +61,20 @@ RSpec.describe Upload do
     end
   end
 
+  describe 'scopes' do
+    describe '.for_model_type_and_id' do
+      let(:avatar_uploads) { create_list(:upload, 2) }
+      let(:attachment_uploads) { create_list(:upload, 2, :attachment_upload) }
+
+      it 'returns records matching the given model_type and ids' do
+        model_ids = [avatar_uploads, attachment_uploads].map { |uploads| uploads.first.model_id }
+
+        expect(described_class.for_model_type_and_id(Note, model_ids))
+          .to contain_exactly(attachment_uploads.first)
+      end
+    end
+  end
+
   describe '#absolute_path' do
     it 'returns the path directly when already absolute' do
       path = '/path/to/namespace/project/secret/file.jpg'

spec/requests/api/personal_access_tokens/self_rotation_spec.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::PersonalAccessTokens::SelfRotation, feature_category: :system_access do
+  let(:path) { '/personal_access_tokens/self/rotate' }
+  let(:token) { create(:personal_access_token, user: current_user) }
+  let(:expiry_date) { Date.today + 1.week }
+  let(:params) { {} }
+
+  let_it_be(:current_user) { create(:user) }
+
+  describe 'POST /personal_access_tokens/self/rotate' do
+    subject(:rotate_token) { post(api(path, personal_access_token: token), params: params) }
+
+    shared_examples 'rotating token succeeds' do
+      it 'rotate token', :aggregate_failures do
+        rotate_token
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response['token']).not_to eq(token.token)
+        expect(json_response['expires_at']).to eq(expiry_date.to_s)
+        expect(token.reload).to be_revoked
+      end
+    end
+
+    shared_examples 'rotating token denied' do |status|
+      it 'cannot rotate token' do
+        rotate_token
+
+        expect(response).to have_gitlab_http_status(status)
+      end
+    end
+
+    context 'when current_user is an administrator', :enable_admin_mode do
+      let(:current_user) { create(:admin) }
+
+      it_behaves_like 'rotating token succeeds'
+
+      context 'when expiry is defined' do
+        let(:expiry_date) { Date.today + 1.month }
+        let(:params) { { expires_at: expiry_date } }
+
+        it_behaves_like 'rotating token succeeds'
+      end
+
+      context 'with impersonated token' do
+        let(:token) { create(:personal_access_token, :impersonation, user: current_user) }
+
+        it_behaves_like 'rotating token succeeds'
+      end
+
+      Gitlab::Auth.all_available_scopes.each do |scope|
+        context "with a '#{scope}' scoped token" do
+          let(:current_user) { create(:admin) }
+          let(:token) { create(:personal_access_token, scopes: [scope], user: current_user) }
+
+          if [Gitlab::Auth::API_SCOPE].include? scope
+            it_behaves_like 'rotating token succeeds'
+          else
+            it_behaves_like 'rotating token denied', :forbidden
+          end
+        end
+      end
+    end
+
+    context 'when current_user is not an administrator' do
+      let(:current_user) { create(:user) }
+
+      it_behaves_like 'rotating token succeeds'
+
+      context 'when expiry is defined' do
+        let(:expiry_date) { Date.today + 1.month }
+        let(:params) { { expires_at: expiry_date } }
+
+        it_behaves_like 'rotating token succeeds'
+      end
+
+      context 'with impersonated token' do
+        let(:token) { create(:personal_access_token, :impersonation, user: current_user) }
+
+        it_behaves_like 'rotating token succeeds'
+      end
+
+      Gitlab::Auth.all_available_scopes.each do |scope|
+        context "with a '#{scope}' scoped token" do
+          let(:current_user) { create(:user) }
+          let(:token) { create(:personal_access_token, scopes: [scope], user: current_user) }
+
+          if [Gitlab::Auth::API_SCOPE].include? scope
+            it_behaves_like 'rotating token succeeds'
+          else
+            it_behaves_like 'rotating token denied', :forbidden
+          end
+        end
+      end
+    end
+
+    context 'when token is invalid' do
+      let(:current_user) { create(:user) }
+      let(:token) { instance_double(PersonalAccessToken, token: 'invalidtoken') }
+
+      it_behaves_like 'rotating token denied', :unauthorized
+    end
+
+    context 'with a revoked token' do
+      let(:token) { create(:personal_access_token, :revoked, user: current_user) }
+
+      it_behaves_like 'rotating token denied', :unauthorized
+    end
+
+    context 'with an expired token' do
+      let(:token) { create(:personal_access_token, expires_at: 1.day.ago, user: current_user) }
+
+      it_behaves_like 'rotating token denied', :unauthorized
+    end
+
+    context 'with a rotated token' do
+      let(:token) { create(:personal_access_token, :revoked, user: current_user) }
+      let!(:child_token) { create(:personal_access_token, previous_personal_access_token_id: token.id) }
+
+      it_behaves_like 'rotating token denied', :unauthorized
+
+      it 'revokes token family' do
+        rotate_token
+
+        expect(child_token.reload).to be_revoked
+      end
+    end
+
+    context 'with an OAuth token' do
+      subject(:rotate_token) { post(api(path, oauth_access_token: token), params: params) }
+
+      context 'with default scope' do
+        let(:token) { create(:oauth_access_token) }
+
+        it_behaves_like 'rotating token denied', :forbidden
+      end
+
+      Gitlab::Auth.all_available_scopes.each do |scope|
+        context "with a '#{scope}' scoped token" do
+          let(:token) { create(:oauth_access_token, scopes: [scope]) }
+
+          if [Gitlab::Auth::API_SCOPE].include? scope
+            it_behaves_like 'rotating token denied', :method_not_allowed
+          else
+            it_behaves_like 'rotating token denied', :forbidden
+          end
+        end
+      end
+    end
+
+    context 'with a deploy token' do
+      let(:token) { create(:deploy_token) }
+      let(:headers) { { Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => token.token } }
+
+      subject(:rotate_token) { post(api(path), params: params, headers: headers) }
+
+      it_behaves_like 'rotating token denied', :unauthorized
+    end
+
+    context 'with a job token' do
+      let(:job) { create(:ci_build, :running, user: current_user) }
+
+      subject(:rotate_token) { post(api(path, job_token: job.token), params: params) }
+
+      it_behaves_like 'rotating token denied', :unauthorized
+    end
+
+    context 'when current_user is a project bot' do
+      let(:current_user) { create(:user, :project_bot) }
+
+      it_behaves_like 'rotating token denied', :forbidden
+
+      context 'when expiry is defined' do
+        let(:expiry_date) { Date.today + 1.month }
+        let(:params) { { expires_at: expiry_date } }
+
+        it_behaves_like 'rotating token denied', :forbidden
+      end
+
+      context 'with impersonated token' do
+        let(:token) { create(:personal_access_token, :impersonation, user: current_user) }
+
+        it_behaves_like 'rotating token denied', :forbidden
+      end
+
+      Gitlab::Auth.resource_bot_scopes.each do |scope|
+        context "with a '#{scope}' scoped token" do
+          let(:token) { create(:personal_access_token, scopes: [scope], user: current_user) }
+
+          it_behaves_like 'rotating token denied', :forbidden
+        end
+      end
+    end
+  end
+end

spec/services/click_house/rebuild_materialized_view_service_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ClickHouse::RebuildMaterializedViewService, :click_house, feature_category: :database do
+  include ClickHouseHelpers
+
+  let_it_be(:event1) { create(:event, :pushed) }
+  let_it_be(:event2) { create(:event, :pushed) }
+  let_it_be(:event3) { create(:closed_issue_event) }
+
+  let(:connection) { ClickHouse::Connection.new(:main) }
+
+  before do
+    insert_events_into_click_house
+  end
+
+  def invoke_service
+    described_class.new(connection: connection, state: {
+      view_name: 'contributions_mv',
+      view_table_name: 'contributions',
+      tmp_view_name: 'tmp_contributions_mv',
+      tmp_view_table_name: 'tmp_contributions',
+      source_table_name: 'events'
+    }).execute
+  end
+
+  it 're-creates the materialized view with correct data from the source table' do
+    stub_const("#{described_class}::INSERT_BATCH_SIZE", 1)
+    # Delete two records from the contributions MV to create so we have inconsistency
+    connection.execute("DELETE FROM contributions WHERE id IN (#{event2.id}, #{event3.id})")
+
+    # The current MV should have one record left
+    ids = connection.select('SELECT id FROM contributions FINAL').pluck('id')
+    expect(ids).to eq([event1.id])
+
+    # Rebuild the MV so we get the inconsistency corrected
+    invoke_service
+
+    ids = connection.select('SELECT id FROM contributions FINAL').pluck('id')
+    expect(ids).to match_array([event1.id, event2.id, event3.id])
+  end
+
+  it 'does not leave temporary tables around' do
+    invoke_service
+
+    view_query = <<~SQL
+      SELECT view_definition FROM information_schema.views
+      WHERE table_name = 'tmp_contributions_mv' AND
+      table_schema = '#{connection.database_name}'
+    SQL
+
+    table_query = <<~SQL
+      SELECT view_definition FROM information_schema.tables
+      WHERE table_name = 'tmp_contributions' AND
+      table_schema = '#{connection.database_name}'
+    SQL
+
+    expect(connection.select(view_query)).to be_empty
+    expect(connection.select(table_query)).to be_empty
+  end
+end

spec/services/projects/import_export/prune_expired_export_jobs_service_spec.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::ImportExport::PruneExpiredExportJobsService, feature_category: :importers do
+  describe '#execute' do
+    context 'when pruning expired jobs' do
+      let_it_be(:old_job_1) { create(:project_export_job, updated_at: 37.months.ago) }
+      let_it_be(:old_job_2) { create(:project_export_job, updated_at: 12.months.ago) }
+      let_it_be(:old_job_3) { create(:project_export_job, updated_at: 8.days.ago) }
+      let_it_be(:fresh_job_1) { create(:project_export_job, updated_at: 1.day.ago) }
+      let_it_be(:fresh_job_2) { create(:project_export_job, updated_at: 2.days.ago) }
+      let_it_be(:fresh_job_3) { create(:project_export_job, updated_at: 6.days.ago) }
+
+      let_it_be(:old_relation_export_1) { create(:project_relation_export, project_export_job_id: old_job_1.id) }
+      let_it_be(:old_relation_export_2) { create(:project_relation_export, project_export_job_id: old_job_2.id) }
+      let_it_be(:old_relation_export_3) { create(:project_relation_export, project_export_job_id: old_job_3.id) }
+      let_it_be(:fresh_relation_export_1) { create(:project_relation_export, project_export_job_id: fresh_job_1.id) }
+
+      let_it_be(:old_upload_1) { create(:relation_export_upload, project_relation_export_id: old_relation_export_1.id) }
+      let_it_be(:old_upload_2) { create(:relation_export_upload, project_relation_export_id: old_relation_export_2.id) }
+      let_it_be(:old_upload_3) { create(:relation_export_upload, project_relation_export_id: old_relation_export_3.id) }
+      let_it_be(:fresh_upload_1) do
+        create(
+          :relation_export_upload,
+          project_relation_export_id: fresh_relation_export_1.id
+        )
+      end
+
+      it 'prunes jobs and associations older than 7 days' do
+        old_uploads = Upload.for_model_type_and_id(
+          Projects::ImportExport::RelationExportUpload,
+          [old_upload_1, old_upload_2, old_upload_3].map(&:id)
+        )
+        old_upload_file_paths = Uploads::Local.new.keys(old_uploads)
+
+        expect(DeleteStoredFilesWorker).to receive(:perform_async).with(Uploads::Local, old_upload_file_paths)
+
+        expect { described_class.execute }.to change { ProjectExportJob.count }.by(-3)
+
+        expect(ProjectExportJob.find_by(id: old_job_1.id)).to be_nil
+        expect(ProjectExportJob.find_by(id: old_job_2.id)).to be_nil
+        expect(ProjectExportJob.find_by(id: old_job_3.id)).to be_nil
+
+        expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_1.id)).to be_nil
+        expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_2.id)).to be_nil
+        expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_3.id)).to be_nil
+
+        expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_1.id)).to be_nil
+        expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_2.id)).to be_nil
+        expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_3.id)).to be_nil
+
+        expect(old_uploads.reload).to be_empty
+      end
+
+      it 'does not delete associated records for jobs younger than 7 days' do
+        described_class.execute
+
+        expect(fresh_job_1.reload).to be_present
+        expect(fresh_job_2.reload).to be_present
+        expect(fresh_job_3.reload).to be_present
+        expect(fresh_relation_export_1.reload).to be_present
+        expect(fresh_upload_1.reload).to be_present
+        expect(
+          Upload.for_model_type_and_id(Projects::ImportExport::RelationExportUpload, fresh_upload_1.id)
+        ).to be_present
+      end
+    end
+  end
+end

spec/support/helpers/click_house_helpers.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module ClickHouseHelpers
+  include ActiveRecord::ConnectionAdapters::Quoting
+
+  def format_event_row(event)
+    path = event.project.reload.project_namespace.traversal_ids.join('/')
+
+    action = Event.actions[event.action]
+    [
+      event.id,
+      "'#{path}/'",
+      event.author_id,
+      event.target_id,
+      "'#{event.target_type}'",
+      action,
+      event.created_at.to_f,
+      event.updated_at.to_f
+    ].join(',')
+  end
+
+  def insert_events_into_click_house(events = Event.all)
+    rows = events.map { |event| "(#{format_event_row(event)})" }.join(',')
+
+    insert_query = <<~SQL
+    INSERT INTO events
+    (id, path, author_id, target_id, target_type, action, created_at, updated_at)
+    VALUES
+    #{rows}
+    SQL
+
+    ClickHouse::Client.execute(insert_query, :main)
+  end
+
+  def insert_ci_builds_to_click_house(builds)
+    values = builds.map do |build|
+      <<~SQL.squish
+        (
+        #{quote(build.id)},
+        #{quote(build.project_id)},
+        #{quote(build.pipeline_id)},
+        #{quote(build.status)},
+        #{format_datetime(build.finished_at)},
+        #{format_datetime(build.created_at)},
+        #{format_datetime(build.started_at)},
+        #{format_datetime(build.queued_at)},
+        #{quote(build.runner_id)},
+        #{quote(build.runner_manager&.system_xid)},
+        #{quote(build.runner&.run_untagged)},
+        #{quote(Ci::Runner.runner_types[build.runner&.runner_type])},
+        #{quote(build.runner_manager&.version || '')},
+        #{quote(build.runner_manager&.revision || '')},
+        #{quote(build.runner_manager&.platform || '')},
+        #{quote(build.runner_manager&.architecture || '')}
+        )
+      SQL
+    end
+
+    values = values.join(', ')
+
+    query = <<~SQL
+      INSERT INTO ci_finished_builds
+          (id, project_id, pipeline_id, status, finished_at, created_at, started_at, queued_at,
+           runner_id, runner_manager_system_xid, runner_run_untagged, runner_type,
+           runner_manager_version, runner_manager_revision, runner_manager_platform, runner_manager_architecture)
+      VALUES #{values}
+    SQL
+
+    result = ClickHouse::Client.execute(query, :main)
+    expect(result).to eq(true)
+  end
+
+  def format_datetime(date)
+    quote(date&.utc&.to_f)
+  end
+end

spec/workers/click_house/rebuild_materialized_view_cron_worker_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ClickHouse::RebuildMaterializedViewCronWorker, feature_category: :database do
+  it 'invokes the RebuildMaterializedViewService' do
+    allow_next_instance_of(ClickHouse::RebuildMaterializedViewService) do |instance|
+      allow(instance).to receive(:execute)
+    end
+
+    described_class.new.perform
+  end
+end

spec/workers/gitlab/export/prune_project_export_jobs_worker_spec.rb

@@ -3,50 +3,15 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Export::PruneProjectExportJobsWorker, feature_category: :importers do
-  let_it_be(:old_job_1) { create(:project_export_job, updated_at: 37.months.ago) }
-  let_it_be(:old_job_2) { create(:project_export_job, updated_at: 12.months.ago) }
-  let_it_be(:old_job_3) { create(:project_export_job, updated_at: 8.days.ago) }
-  let_it_be(:fresh_job_1) { create(:project_export_job, updated_at: 1.day.ago) }
-  let_it_be(:fresh_job_2) { create(:project_export_job, updated_at: 2.days.ago) }
-  let_it_be(:fresh_job_3) { create(:project_export_job, updated_at: 6.days.ago) }
-
-  let_it_be(:old_relation_export_1) { create(:project_relation_export, project_export_job_id: old_job_1.id) }
-  let_it_be(:old_relation_export_2) { create(:project_relation_export, project_export_job_id: old_job_2.id) }
-  let_it_be(:old_relation_export_3) { create(:project_relation_export, project_export_job_id: old_job_3.id) }
-  let_it_be(:fresh_relation_export_1) { create(:project_relation_export, project_export_job_id: fresh_job_1.id) }
-
-  let_it_be(:old_upload_1) { create(:relation_export_upload, project_relation_export_id: old_relation_export_1.id) }
-  let_it_be(:old_upload_2) { create(:relation_export_upload, project_relation_export_id: old_relation_export_2.id) }
-  let_it_be(:old_upload_3) { create(:relation_export_upload, project_relation_export_id: old_relation_export_3.id) }
-  let_it_be(:fresh_upload_1) { create(:relation_export_upload, project_relation_export_id: fresh_relation_export_1.id) }
-
   subject(:worker) { described_class.new }
 
   describe '#perform' do
-    include_examples 'an idempotent worker' do
-      it 'prunes jobs and associations older than 7 days' do
-        expect { perform_multiple }.to change { ProjectExportJob.count }.by(-3)
-        expect(ProjectExportJob.find_by(id: old_job_1.id)).to be_nil
-        expect(ProjectExportJob.find_by(id: old_job_2.id)).to be_nil
-        expect(ProjectExportJob.find_by(id: old_job_3.id)).to be_nil
+    it_behaves_like 'an idempotent worker'
 
-        expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_1.id)).to be_nil
-        expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_2.id)).to be_nil
-        expect(Projects::ImportExport::RelationExport.find_by(id: old_relation_export_3.id)).to be_nil
+    it 'executes PruneExpiredExportJobsService' do
+      expect(Projects::ImportExport::PruneExpiredExportJobsService).to receive(:execute)
 
-        expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_1.id)).to be_nil
-        expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_2.id)).to be_nil
-        expect(Projects::ImportExport::RelationExportUpload.find_by(id: old_upload_3.id)).to be_nil
-      end
-
-      it 'leaves fresh jobs and associations' do
-        perform_multiple
-        expect(fresh_job_1.reload).to be_present
-        expect(fresh_job_2.reload).to be_present
-        expect(fresh_job_3.reload).to be_present
-        expect(fresh_relation_export_1.reload).to be_present
-        expect(fresh_upload_1.reload).to be_present
-      end
+      worker.perform
     end
   end
 end