root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-01-25 12:10:29 +00:00
parent 0388da1c55
commit 1811b6a8f9
58 changed files with 1596 additions and 637 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
config/feature_flags/development/ai_claude_2_1.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: ai_claude_2_1
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139946
rollout_issue_url:
milestone: '16.7'
type: development
group: group::ai framework
default_enabled: false

4
config/feature_flags/development/use_sync_service_token_worker.yml
View File

@ -2,7 +2,7 @@
name: use_sync_service_token_worker
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136078
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/431608
milestone: '16.7'
milestone: '16.9'
type: development
group: group::cloud connector
default_enabled: false
default_enabled: true

14
db/docs/audit_events_amazon_s3_configurations.yml
View File

@ -1,10 +1,18 @@
---
table_name: audit_events_amazon_s3_configurations
classes:
- AuditEvents::AmazonS3Configuration
- AuditEvents::AmazonS3Configuration
feature_categories:
- audit_events
- audit_events
description: Stores Amazon S3 configurations associated used for audit event streaming.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/333371
milestone: '16.4'
gitlab_schema: gitlab_main
gitlab_schema: gitlab_main_cell
allow_cross_joins:
- gitlab_main_clusterwide
allow_cross_transactions:
- gitlab_main_clusterwide
allow_cross_foreign_keys:
- gitlab_main_clusterwide
sharding_key:
namespace_id: namespaces

2
db/docs/audit_events_external_audit_event_destinations.yml
View File

@ -8,3 +8,5 @@ description: TODO
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70706
milestone: '14.4'
gitlab_schema: gitlab_main_cell
sharding_key:
namespace_id: namespaces

9
db/docs/audit_events_google_cloud_logging_configurations.yml
View File

@ -1,10 +1,13 @@
---
table_name: audit_events_google_cloud_logging_configurations
classes:
- AuditEvents::GoogleCloudLoggingConfiguration
- AuditEvents::GoogleCloudLoggingConfiguration
feature_categories:
- audit_events
description: Stores Google Cloud Logging configurations associated with IAM service accounts, used for generating access tokens.
- audit_events
description: Stores Google Cloud Logging configurations associated with IAM service
accounts, used for generating access tokens.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/409421
milestone: '16.0'
gitlab_schema: gitlab_main_cell
sharding_key:
namespace_id: namespaces

22
db/post_migrate/20240123101716_ensure_id_uniqueness_for_p_ci_pipeline_variables.rb Normal file
View File

@ -0,0 +1,22 @@
# frozen_string_literal: true
class EnsureIdUniquenessForPCiPipelineVariables < Gitlab::Database::Migration[2.2]
include Gitlab::Database::PartitioningMigrationHelpers::UniquenessHelpers
milestone '16.9'
enable_lock_retries!
TABLE_NAME = :p_ci_pipeline_variables
FUNCTION_NAME = :assign_p_ci_pipeline_variables_id_value
def up
ensure_unique_id(TABLE_NAME)
end
def down
execute(<<~SQL.squish)
ALTER TABLE #{TABLE_NAME}
ALTER COLUMN id SET DEFAULT nextval('ci_pipeline_variables_id_seq'::regclass);
DROP FUNCTION IF EXISTS #{FUNCTION_NAME} CASCADE;
SQL
end
end

1
db/schema_migrations/20240123101716 Normal file
View File

@ -0,0 +1 @@
3f759e6a9fbc40a196770a788be410a26c2cc3be7213885782831172d8d721c6

37
db/structure.sql
View File

@ -23,6 +23,19 @@ RETURN NEW;
END
$$;
CREATE FUNCTION assign_p_ci_pipeline_variables_id_value() RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
IF NEW."id" IS NOT NULL THEN
RAISE WARNING 'Manually assigning ids is not allowed, the value will be ignored';
END IF;
NEW."id" := nextval('ci_pipeline_variables_id_seq'::regclass);
RETURN NEW;
END
$$;
CREATE FUNCTION delete_associated_project_namespace() RETURNS trigger
LANGUAGE plpgsql
AS $$
@ -14722,15 +14735,6 @@ CREATE TABLE p_ci_pipeline_variables (
)
PARTITION BY LIST (partition_id);
CREATE SEQUENCE ci_pipeline_variables_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE ci_pipeline_variables_id_seq OWNED BY p_ci_pipeline_variables.id;
CREATE TABLE ci_pipeline_variables (
key character varying NOT NULL,
value text,
@ -14740,10 +14744,19 @@ CREATE TABLE ci_pipeline_variables (
variable_type smallint DEFAULT 1 NOT NULL,
partition_id bigint NOT NULL,
raw boolean DEFAULT false NOT NULL,
id bigint DEFAULT nextval('ci_pipeline_variables_id_seq'::regclass) NOT NULL,
id bigint NOT NULL,
pipeline_id bigint NOT NULL
);
CREATE SEQUENCE ci_pipeline_variables_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE ci_pipeline_variables_id_seq OWNED BY p_ci_pipeline_variables.id;
CREATE TABLE ci_pipelines (
id integer NOT NULL,
ref character varying,
@ -27573,8 +27586,6 @@ ALTER TABLE ONLY p_ci_builds_metadata ALTER COLUMN id SET DEFAULT nextval('ci_bu
ALTER TABLE ONLY p_ci_job_annotations ALTER COLUMN id SET DEFAULT nextval('p_ci_job_annotations_id_seq'::regclass);
ALTER TABLE ONLY p_ci_pipeline_variables ALTER COLUMN id SET DEFAULT nextval('ci_pipeline_variables_id_seq'::regclass);
ALTER TABLE ONLY packages_build_infos ALTER COLUMN id SET DEFAULT nextval('packages_build_infos_id_seq'::regclass);
ALTER TABLE ONLY packages_composer_cache_files ALTER COLUMN id SET DEFAULT nextval('packages_composer_cache_files_id_seq'::regclass);
@ -38048,6 +38059,8 @@ ALTER INDEX p_ci_builds_token_encrypted_partition_id_idx ATTACH PARTITION unique
CREATE TRIGGER assign_p_ci_builds_id_trigger BEFORE INSERT ON p_ci_builds FOR EACH ROW EXECUTE FUNCTION assign_p_ci_builds_id_value();
CREATE TRIGGER assign_p_ci_pipeline_variables_id_trigger BEFORE INSERT ON p_ci_pipeline_variables FOR EACH ROW EXECUTE FUNCTION assign_p_ci_pipeline_variables_id_value();
CREATE TRIGGER chat_names_loose_fk_trigger AFTER DELETE ON chat_names REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records();
CREATE TRIGGER ci_builds_loose_fk_trigger AFTER DELETE ON ci_builds REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records();

105
doc/administration/backup_restore/backup_large_reference_architectures.md
View File

@ -25,16 +25,82 @@ This document is intended for environments using:
## Configure daily backups
### Configure backup of PostgreSQL and object storage data
### Configure backup of PostgreSQL data
The [backup command](backup_gitlab.md) uses `pg_dump`, which is [not appropriate for databases over 100 GB](backup_gitlab.md#postgresql-databases). You must choose a PostgreSQL solution which has native, robust backup capabilities.
[Object storage](../object_storage.md), ([not NFS](../nfs.md)) is recommended for storing GitLab data, including [blobs](backup_gitlab.md#blobs) and [Container registry](backup_gitlab.md#container-registry).
::Tabs
1. [Configure AWS Backup](https://docs.aws.amazon.com/aws-backup/latest/devguide/creating-a-backup-plan.html) to back up both RDS and S3 data. For maximum protection, [configure continuous backups as well as snapshot backups](https://docs.aws.amazon.com/aws-backup/latest/devguide/point-in-time-recovery.html).
:::TabTitle AWS
1. [Configure AWS Backup](https://docs.aws.amazon.com/aws-backup/latest/devguide/creating-a-backup-plan.html) to back up RDS (and S3) data. For maximum protection, [configure continuous backups as well as snapshot backups](https://docs.aws.amazon.com/aws-backup/latest/devguide/point-in-time-recovery.html).
1. Configure AWS Backup to copy backups to a separate region. When AWS takes a backup, the backup can only be restored in the region the backup is stored.
1. After AWS Backup has run at least one scheduled backup, then you can [create an on-demand backup](https://docs.aws.amazon.com/aws-backup/latest/devguide/recov-point-create-on-demand-backup.html) as needed.
:::TabTitle Google
Schedule [automated daily backups of Google Cloud SQL data](https://cloud.google.com/sql/docs/postgres/backup-recovery/backing-up#schedulebackups).
Daily backups [can be retained](https://cloud.google.com/sql/docs/postgres/backup-recovery/backups#retention) for up to one year, and transaction logs can be retained for 7 days by default for point-in-time recovery.
::EndTabs
### Configure backup of object storage data
[Object storage](../object_storage.md), ([not NFS](../nfs.md)) is recommended for storing GitLab data, including [blobs](backup_gitlab.md#blobs) and [Container registry](backup_gitlab.md#container-registry).
::Tabs
:::TabTitle AWS
Configure AWS Backup to back up S3 data. This can be done at the same time when [configuring the backup of PostgreSQL data](#configure-backup-of-postgresql-data).
:::TabTitle Google
1. [Create a backup bucket in GCS](https://cloud.google.com/storage/docs/creating-buckets).
1. [Create Storage Transfer Service jobs](https://cloud.google.com/storage-transfer/docs/create-transfers) which copy each GitLab object storage bucket to a backup bucket. You can create these jobs once, and [schedule them to run daily](https://cloud.google.com/storage-transfer/docs/schedule-transfer-jobs). However this mixes new and old object storage data, so files that were deleted in GitLab will still exist in the backup. This wastes storage after restore, but it is otherwise not a problem. These files would be inaccessible to GitLab users since they do not exist in the GitLab database. You can delete [some of these orphaned files](../../raketasks/cleanup.md#clean-up-project-upload-files-from-object-storage) after restore, but this clean up Rake task only operates on a subset of files.
1. For `When to overwrite`, choose `Never`. GitLab object stored files are intended to be immutable. This selection could be helpful if a malicious actor succeeded at mutating GitLab files.
1. For `When to delete`, choose `Never`. If you sync the backup bucket to source, then you cannot recover if files are accidentally or maliciously deleted from source.
1. Alternatively, it is possible to backup object storage into buckets or subdirectories segregated by day. This avoids the problem of orphaned files after restore, and supports backup of file versions if needed. But it greatly increases backup storage costs. This can be done with [a Cloud Function triggered by Cloud Scheduler](https://cloud.google.com/scheduler/docs/tut-pub-sub), or with a script run by a cronjob. A partial example:
```shell
# Set GCP project so you don't have to specify it in every command
gcloud config set project example-gcp-project-name
# Grant the Storage Transfer Service's hidden service account permission to write to the backup bucket. The integer 123456789012 is the GCP project's ID.
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.objectAdmin gs://backup-bucket
# Grant the Storage Transfer Service's hidden service account permission to list and read objects in the source buckets. The integer 123456789012 is the GCP project's ID.
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-artifacts
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-ci-secure-files
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-dependency-proxy
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-lfs
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-mr-diffs
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-packages
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-pages
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-registry
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-terraform-state
gsutil iam ch serviceAccount:project-123456789012@storage-transfer-service.iam.gserviceaccount.com:roles/storage.legacyBucketReader,roles/storage.objectViewer gs://gitlab-bucket-uploads
# Create transfer jobs for each bucket, targeting a subdirectory in the backup bucket.
today=$(date +%F)
gcloud transfer jobs create gs://gitlab-bucket-artifacts/ gs://backup-bucket/$today/artifacts/ --name "$today-backup-artifacts"
gcloud transfer jobs create gs://gitlab-bucket-ci-secure-files/ gs://backup-bucket/$today/ci-secure-files/ --name "$today-backup-ci-secure-files"
gcloud transfer jobs create gs://gitlab-bucket-dependency-proxy/ gs://backup-bucket/$today/dependency-proxy/ --name "$today-backup-dependency-proxy"
gcloud transfer jobs create gs://gitlab-bucket-lfs/ gs://backup-bucket/$today/lfs/ --name "$today-backup-lfs"
gcloud transfer jobs create gs://gitlab-bucket-mr-diffs/ gs://backup-bucket/$today/mr-diffs/ --name "$today-backup-mr-diffs"
gcloud transfer jobs create gs://gitlab-bucket-packages/ gs://backup-bucket/$today/packages/ --name "$today-backup-packages"
gcloud transfer jobs create gs://gitlab-bucket-pages/ gs://backup-bucket/$today/pages/ --name "$today-backup-pages"
gcloud transfer jobs create gs://gitlab-bucket-registry/ gs://backup-bucket/$today/registry/ --name "$today-backup-registry"
gcloud transfer jobs create gs://gitlab-bucket-terraform-state/ gs://backup-bucket/$today/terraform-state/ --name "$today-backup-terraform-state"
gcloud transfer jobs create gs://gitlab-bucket-uploads/ gs://backup-bucket/$today/uploads/ --name "$today-backup-uploads"
```
1. These Transfer Jobs are not automatically deleted after running. You could implement clean up of old jobs in the script.
1. The example script does not delete old backups. You could implement clean up of old backups according to your desired retention policy.
1. Ensure that backups are performed at the same time or later than Cloud SQL backups, to reduce data inconsistencies.
::EndTabs
### Configure backup of Git repositories
NOTE:
@ -93,7 +159,7 @@ To back up the Git repositories:
- A 7 GB archive file of all Git data.
1. SSH into the GitLab Rails node.
1. [Configure uploading backups to remote cloud storage](backup_gitlab.md#upload-backups-to-a-remote-cloud-storage).
1. [Configure AWS Backup](https://docs.aws.amazon.com/aws-backup/latest/devguide/creating-a-backup-plan.html) for this bucket, or use a bucket in the same account and region as your production data object storage buckets, and ensure this bucket is included in your [preexisting AWS Backup](#configure-backup-of-postgresql-and-object-storage-data).
1. [Configure AWS Backup](https://docs.aws.amazon.com/aws-backup/latest/devguide/creating-a-backup-plan.html) for this bucket, or use a bucket in the same account and region as your production data object storage buckets, and ensure this bucket is included in your [preexisting AWS Backup](#configure-backup-of-object-storage-data).
1. Run the [backup command](backup_gitlab.md#backup-command), skipping PostgreSQL data:
```shell
@ -216,6 +282,10 @@ Before restoring a backup:
### Restore object storage data
::Tabs
:::TabTitle AWS
Each bucket exists as a separate backup within AWS and each backup can be restored to an existing or
new bucket.
@ -232,8 +302,20 @@ new bucket.
1. You can move on to [Restore PostgreSQL data](#restore-postgresql-data) while the restore job is
running.
:::TabTitle Google
1. [Create Storage Transfer Service jobs](https://cloud.google.com/storage-transfer/docs/create-transfers) to transfer backed up data to the GitLab buckets.
1. You can move on to [Restore PostgreSQL data](#restore-postgresql-data) while the transfer jobs are
running.
::EndTabs
### Restore PostgreSQL data
::Tabs
:::TabTitle AWS
1. [Restore the AWS RDS database using built-in tooling](https://docs.aws.amazon.com/aws-backup/latest/devguide/restoring-rds.html),
which creates a new RDS instance.
1. Because the new RDS instance has a different endpoint, you must reconfigure the destination GitLab instance
@ -247,6 +329,21 @@ new bucket.
1. Before moving on, wait until the new RDS instance is created and ready to use.
:::TabTitle Google
1. [Restore the Google Cloud SQL database using built-in tooling](https://cloud.google.com/sql/docs/postgres/backup-recovery/restoring).
1. If you restore to a new database instance, then reconfigure GitLab to point to the new database:
- For Linux package installations, follow
[Using a non-packaged PostgreSQL database management server](https://docs.gitlab.com/omnibus/settings/database.html#using-a-non-packaged-postgresql-database-management-server).
- For Helm chart (Kubernetes) installations, follow
[Configure the GitLab chart with an external database](https://docs.gitlab.com/charts/advanced/external-db/index.html).
1. Before moving on, wait until the Cloud SQL instance is ready to use.
::EndTabs
### Restore Git repositories
Select or create a node to restore:

6
doc/administration/geo/disaster_recovery/bring_primary_back.md
View File

@ -88,9 +88,10 @@ When a secondary site is added, if it contains data that would otherwise be sync
- Geo's container registry sync code compares tags and only pulls missing tags.
- [Blobs/files](#skipping-re-transfer-of-blobs-or-files) are skipped if they exist on the first sync.
This behavior is useful when:
Use-cases:
- You do a planned failover and demote the old primary site by attaching it as a secondary site without rebuilding it.
- You have multiple secondary Geo sites. You do a planned failover and reattach the other secondary Geo sites without rebuilding them.
- You do a failover test by promoting and demoting a secondary site and reattach it without rebuilding it.
- You restore a backup and attach the site as a secondary site.
- You manually copy data to a secondary site to workaround a sync problem.
@ -100,9 +101,10 @@ This behavior is useful when:
### Skipping re-transfer of blobs or files
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352530) in GitLab 16.8 [with a flag](../../feature_flags.md) named `geo_skip_download_if_exists`. Disabled by default.
> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/435788) in GitLab 16.9. Feature flag `geo_skip_download_if_exists` removed.
FLAG:
On self-managed GitLab, by default this feature is not available. To make it available, an administrator can [enable the feature flag](../../feature_flags.md) named `geo_skip_download_if_exists`.
On self-managed GitLab, by default this feature is available.
On GitLab.com, this feature is not available.
When you add a secondary site which has preexisting file data, then the secondary Geo site will avoid re-transferring that data. This applies to:

34
doc/administration/geo/index.md
View File

@ -18,10 +18,12 @@ Geo undergoes significant changes from release to release. Upgrades are
supported and [documented](#upgrading-geo), but you should ensure that you're
using the right version of the documentation for your installation.
Fetching large repositories can take a long time for teams located far from a single GitLab instance.
Fetching large repositories can take a long time for teams and runners located far from a single GitLab instance.
Geo provides local, read-only sites of your GitLab instances. This can reduce the time it takes
to clone and fetch large repositories, speeding up development.
Geo provides local caches that can be placed geographically close to remote teams which can serve read requests. This can reduce the time it takes
to clone and fetch large repositories, speeding up development and increasing the productivity of your remote teams.
Geo secondary sites transparently proxy write requests to the primary site. All Geo sites can be configured to respond to a single GitLab URL, to deliver a consistent, seamless, and comprehensive experience whichever site the user lands on.
To make sure you're using the right version of the documentation, go to [the Geo page on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/geo/index.md) and choose the appropriate release from the **Switch branch/tag** dropdown list. For example, [`v13.7.6-ee`](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.7.6-ee/doc/administration/geo/index.md).
@ -46,9 +48,8 @@ In addition, it:
Geo provides:
- Read-only **secondary** sites: Maintain one **primary** GitLab site while still enabling read-only **secondary** sites for each of your distributed teams.
- A complete GitLab experience on **Secondary** sites: Maintain one **primary** GitLab site while enabling **secondary** sites with full read and write and UI experience for each of your distributed teams.
- Authentication system hooks: **Secondary** sites receive all authentication data (like user accounts and logins) from the **primary** instance.
- An intuitive UI: **Secondary** sites use the same web interface your team has grown accustomed to. In addition, there are visual notifications that block write operations and make it clear that a user is on a **secondary** sites.
### Gitaly Cluster
@ -64,7 +65,7 @@ Your Geo instance can be used for cloning and fetching projects, in addition to
When Geo is enabled, the:
- Original instance is known as the **primary** site.
- Replicated read-only sites are known as **secondary** sites.
- Replicating sites are known as **secondary** sites.
Keep in mind that:
@ -86,7 +87,7 @@ In this diagram:
- There is the **primary** site and the details of one **secondary** site.
- Writes to the database can only be performed on the **primary** site. A **secondary** site receives database
updates via PostgreSQL streaming replication.
updates by using [PostgreSQL streaming replication](https://wiki.postgresql.org/wiki/Streaming_Replication).
- If present, the [LDAP server](#ldap) should be configured to replicate for [Disaster Recovery](disaster_recovery/index.md) scenarios.
- A **secondary** site performs different type of synchronizations against the **primary** site, using a special
authorization protected by JWT:
@ -97,6 +98,7 @@ From the perspective of a user performing Git operations:
- The **primary** site behaves as a full read-write GitLab instance.
- **Secondary** sites are read-only but proxy Git push operations to the **primary** site. This makes **secondary** sites appear to support push operations themselves.
- **Secondary** sites proxy web UI requests to the primary. This makes the **secondary** sites appear to support full UI read/write operations.
To simplify the diagram, some necessary components are omitted.
@ -106,7 +108,7 @@ To simplify the diagram, some necessary components are omitted.
A **secondary** site needs two different PostgreSQL databases:
- A read-only database instance that streams data from the main GitLab database.
- [Another database instance](#geo-tracking-database) used internally by the **secondary** site to record what data has been replicated.
- A [read/write database instance(tracking database)](#geo-tracking-database) used internally by the **secondary** site to record what data has been replicated.
In **secondary** sites, there is an additional daemon: [Geo Log Cursor](#geo-log-cursor).
@ -120,7 +122,10 @@ The following are required to run Geo:
- [CentOS](https://www.centos.org) 7.4 or later
- [Ubuntu](https://ubuntu.com) 16.04 or later
- [Supported PostgreSQL versions](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/postgresql-upgrade-cadence.html) for your GitLab releases with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication).
- Note,[PostgreSQL 12 is deprecated](../../update/deprecations.md#postgresql-12-deprecated) and is removed in GitLab 16.0.
NOTE:
[PostgreSQL Logical replication](https://www.postgresql.org/docs/current/logical-replication.html) is not supported.
- All sites must run [the same PostgreSQL versions](setup/database.md#postgresql-replication).
- Where possible, you should also use the same operating system version on all
Geo sites. If using different operating system versions between Geo sites, you
@ -171,7 +176,7 @@ To update the internal URL of the primary Geo site:
### Geo Tracking Database
The tracking database instance is used as metadata to control what needs to be updated on the disk of the local instance. For example:
The tracking database instance is used as metadata to control what needs to be updated on the local instance. For example:
- Download new assets.
- Fetch new LFS Objects.
@ -206,11 +211,12 @@ This list of limitations only reflects the latest version of GitLab. If you are
- GitLab Runners cannot register with a **secondary** site. Support for this is [planned for the future](https://gitlab.com/gitlab-org/gitlab/-/issues/3294).
- [Selective synchronization](replication/configuration.md#selective-synchronization) only limits what repositories and files are replicated. The entire PostgreSQL data is still replicated. Selective synchronization is not built to accommodate compliance / export control use cases.
- [Pages access control](../../user/project/pages/pages_access_control.md) doesn't work on secondaries. See [GitLab issue #9336](https://gitlab.com/gitlab-org/gitlab/-/issues/9336) for details.
- [Disaster recovery](disaster_recovery/index.md) for multi-secondary sites causes downtime due to the complete re-synchronization and re-configuration of all non-promoted secondaries.
- [Disaster recovery](disaster_recovery/index.md) for deployments that have multiple secondary sites causes downtime due to the need to perform complete re-synchronization and re-configuration of all non-promoted secondaries to follow the new primary site.
- For Git over SSH, to make the project clone URL display correctly regardless of which site you are browsing, secondary sites must use the same port as the primary. [GitLab issue #339262](https://gitlab.com/gitlab-org/gitlab/-/issues/339262) proposes to remove this limitation.
- Git push over SSH against a secondary site does not work for pushes over 1.86 GB. [GitLab issue #413109](https://gitlab.com/gitlab-org/gitlab/-/issues/413109) tracks this bug.
- Backups [cannot be run on secondaries](replication/troubleshooting.md#message-error-canceling-statement-due-to-conflict-with-recovery).
- Git clone and fetch requests with option `--depth` over SSH against a secondary site does not work and hangs indefinitely if the secondary site is not up to date at the time the request is initiated. For more information, see [issue 391980](https://gitlab.com/gitlab-org/gitlab/-/issues/391980).
- Git push with options over SSH against a secondary site does not work and terminates the connection. For more information, see [issue 417186](https://gitlab.com/gitlab-org/gitlab/-/issues/417186).
### Limitations on replication/verification
@ -236,12 +242,6 @@ For information on how to update your Geo sites to the latest GitLab version, se
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35913) in GitLab 13.2.
WARNING:
In GitLab 13.2 and 13.3, promoting a secondary site to a primary while the
secondary is paused fails. Do not pause replication before promoting a
secondary. If the site is paused, be sure to resume before promoting. This
issue has been fixed in GitLab 13.4 and later.
WARNING:
Pausing and resuming of replication is only supported for Geo installations using a
Linux package-managed database. External databases are not supported.

7
doc/administration/geo/replication/multiple_servers.md
View File

@ -135,11 +135,10 @@ recommended.
### Step 2: Configure the Geo tracking database on the Geo **secondary** site
If you want to run the Geo tracking database in a multi-node PostgreSQL cluster,
then follow [Configuring Patroni cluster for the tracking PostgreSQL database](../setup/database.md#configuring-patroni-cluster-for-the-tracking-postgresql-database).
The Geo tracking database cannot be run in a multi-node PostgreSQL cluster,
see [Configuring Patroni cluster for the tracking PostgreSQL database](../setup/database.md#configuring-patroni-cluster-for-the-tracking-postgresql-database).
If you want to run the Geo tracking database on a single node, then follow
the instructions below.
You can run the Geo tracking database on a single node as follows:
1. Generate an MD5 hash of the desired password for the database user that the
GitLab application uses to access the tracking database:

2
doc/administration/geo/replication/troubleshooting.md
View File

@ -342,7 +342,7 @@ sudo gitlab-rake gitlab:geo:check
- If you are running the secondary site on a single node for all services, then follow [Geo database replication - Configure the secondary server](../setup/database.md#step-2-configure-the-secondary-server).
- If you are running the secondary site's tracking database on its own node, then follow [Geo for multiple servers - Configure the Geo tracking database on the Geo secondary site](multiple_servers.md#step-2-configure-the-geo-tracking-database-on-the-geo-secondary-site)
- If you are running the secondary site's tracking database in a Patroni cluster, then follow [Geo database replication - Configure the tracking database on the secondary sites](../setup/database.md#step-3-configure-the-tracking-database-on-the-secondary-sites)
- If you are running the secondary site's tracking database in a Patroni cluster, then follow [Geo database replication - Configuring Patroni cluster for the tracking PostgreSQL database](../setup/database.md#configuring-patroni-cluster-for-the-tracking-postgresql-database)
- If you are running the secondary site's tracking database in an external database, then follow [Geo with external PostgreSQL instances](../setup/external_database.md#configure-the-tracking-database)
- If the Geo check task was run on a node which is not running a service which runs the GitLab Rails app (Puma, Sidekiq, or Geo Log Cursor), then this error can be ignored. The node does not need Rails to be configured.

187
doc/administration/geo/setup/database.md
View File

@ -947,187 +947,20 @@ synchronization is required.
### Configuring Patroni cluster for the tracking PostgreSQL database
**Secondary** sites use a separate PostgreSQL installation as a tracking database to
**Secondary** Geo sites use a separate PostgreSQL installation as a tracking database to
keep track of replication status and automatically recover from potential replication issues.
The Linux package automatically configures a tracking database when `roles(['geo_secondary_role'])` is set.
If you want to run this database in a highly available configuration, don't use the `geo_secondary_role` above.
Instead, follow the instructions below.
If you want to run the Geo tracking database on a single node, see
[Configure the Geo tracking database on the Geo secondary site](../replication/multiple_servers.md#step-2-configure-the-geo-tracking-database-on-the-geo-secondary-site).
If you want to run the Geo tracking database on a single node, see [Configure the Geo tracking database on the Geo secondary site](../replication/multiple_servers.md#step-2-configure-the-geo-tracking-database-on-the-geo-secondary-site).
The Linux package does not support running the Geo tracking database in a highly available configuration.
In particular, failover does not work properly. See the
[feature request issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292).
A production-ready and secure setup for the tracking PostgreSQL DB requires at least three Consul nodes: two
Patroni nodes, and one PgBouncer node on the secondary site.
Consul can track multiple services, so you can choose to reuse the nodes used for the Standby Cluster database, though the instructions below do not show how to combine configurations when reusing Consul nodes.
Be sure to use [password credentials](../../postgresql/replication_and_failover.md#database-authorization-for-patroni)
and other database best practices.
#### Step 1. Configure PgBouncer nodes on the secondary site
On each node running the PgBouncer service for the PostgreSQL tracking database:
1. SSH into your PgBouncer node and log in as root:
```shell
sudo -i
```
1. Edit `/etc/gitlab/gitlab.rb` and add the following:
```ruby
# Disable all components except Pgbouncer and Consul agent
roles(['pgbouncer_role'])
# PgBouncer configuration
pgbouncer['users'] = {
'pgbouncer': {
password: 'PGBOUNCER_PASSWORD_HASH'
}
}
pgbouncer['databases'] = {
gitlabhq_geo_production: {
user: 'pgbouncer',
password: 'PGBOUNCER_PASSWORD_HASH'
}
}
# Consul configuration
consul['watchers'] = %w(postgresql)
consul['configuration'] = {
retry_join: %w[CONSUL_TRACKINGDB1_IP CONSUL_TRACKINGDB2_IP CONSUL_TRACKINGDB3_IP]
}
consul['monitoring_service_discovery'] = true
# GitLab database settings
gitlab_rails['db_database'] = 'gitlabhq_geo_production'
gitlab_rails['db_username'] = 'gitlab_geo'
```
1. Reconfigure GitLab for the changes to take effect:
```shell
gitlab-ctl reconfigure
```
1. Create a `.pgpass` file so Consul is able to reload PgBouncer. Enter the `PLAIN_TEXT_PGBOUNCER_PASSWORD` twice when asked:
```shell
gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul
```
1. Restart the PgBouncer service:
```shell
gitlab-ctl restart pgbouncer
```
#### Step 2. Configure a Patroni cluster
On each node running a Patroni instance on the secondary site for the PostgreSQL tracking database:
1. SSH into your Patroni node and log in as root:
```shell
sudo -i
```
1. Edit `/etc/gitlab/gitlab.rb` and add the following:
```ruby
# Disable all components except PostgreSQL, Patroni, and Consul
roles(['patroni_role'])
# Consul configuration
consul['services'] = %w(postgresql)
consul['configuration'] = {
server: true,
retry_join: %w[CONSUL_TRACKINGDB1_IP CONSUL_TRACKINGDB2_IP CONSUL_TRACKINGDB3_IP]
}
# PostgreSQL configuration
postgresql['listen_address'] = '0.0.0.0'
postgresql['hot_standby'] = 'on'
postgresql['wal_level'] = 'replica'
postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH'
postgresql['sql_replication_password'] = 'POSTGRESQL_REPLICATION_PASSWORD_HASH'
postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH'
postgresql['md5_auth_cidr_addresses'] = [
'PATRONI_TRACKINGDB1_IP/32', 'PATRONI_TRACKINGDB2_IP/32', 'PATRONI_TRACKINGDB3_IP/32', 'PATRONI_TRACKINGDB_PGBOUNCER/32',
# Any other instance that needs access to the database as per documentation
]
# Add patroni nodes to the allowlist
patroni['allowlist'] = %w[
127.0.0.1/32
PATRONI_TRACKINGDB1_IP/32 PATRONI_TRACKINGDB2_IP/32 PATRONI_TRACKINGDB3_IP/32
]
# Patroni configuration
patroni['username'] = 'PATRONI_API_USERNAME'
patroni['password'] = 'PATRONI_API_PASSWORD'
patroni['replication_password'] = 'PLAIN_TEXT_POSTGRESQL_REPLICATION_PASSWORD'
patroni['postgresql']['max_wal_senders'] = 5 # A minimum of three for one replica, plus two for each additional replica
# GitLab database settings
gitlab_rails['db_database'] = 'gitlabhq_geo_production'
gitlab_rails['db_username'] = 'gitlab_geo'
gitlab_rails['enable'] = true
# Disable automatic database migrations
gitlab_rails['auto_migrate'] = false
```
1. Reconfigure GitLab for the changes to take effect.
This step is required to bootstrap PostgreSQL users and settings:
```shell
gitlab-ctl reconfigure
```
#### Step 3. Configure the tracking database on the secondary sites
For each node running the `gitlab-rails`, `sidekiq`, and `geo-logcursor` services:
1. SSH into your node and log in as root:
```shell
sudo -i
```
1. Edit `/etc/gitlab/gitlab.rb` and add the following attributes. You may have other attributes set, but the following must be set.
```ruby
# Tracking database settings
geo_secondary['db_username'] = 'gitlab_geo'
geo_secondary['db_password'] = 'PLAIN_TEXT_PGBOUNCER_PASSWORD'
geo_secondary['db_database'] = 'gitlabhq_geo_production'
geo_secondary['db_host'] = 'PATRONI_TRACKINGDB_PGBOUNCER_IP'
geo_secondary['db_port'] = 6432
geo_secondary['auto_migrate'] = false
# Disable the tracking database service
geo_postgresql['enable'] = false
```
1. Reconfigure GitLab for the changes to take effect.
```shell
gitlab-ctl reconfigure
```
1. Run the tracking database migrations:
```shell
gitlab-rake db:migrate:geo
```
If you want to run the Geo tracking database in a highly available configuration, you can connect the
secondary site to an external PostgreSQL database, such as a cloud-managed database, or a manually
configured [Patroni](https://patroni.readthedocs.io/en/latest/) cluster (not managed by the GitLab Linux package).
Follow [Geo with external PostgreSQL instances](external_database.md#configure-the-tracking-database).
## Troubleshooting

11
doc/administration/raketasks/storage.md
View File

@ -1,11 +0,0 @@
---
redirect_to: '../repository_storage_paths.md'
remove_date: '2024-01-11'
---
This document was moved to [another location](../repository_storage_paths.md).
<!-- This redirect file can be deleted after <2024-01-11>. -->
<!-- Redirects that point to other docs in the same project expire in three months. -->
<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. -->
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->

32
doc/administration/reference_architectures/10k_users.md
View File

@ -1397,11 +1397,11 @@ To configure the Praefect nodes, on each one:
NOTE:
You can't remove the `default` entry from `virtual_storages` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage).
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Praefect server
@ -1561,12 +1561,12 @@ On each node:
1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure
storage paths, enable the network listener, and to configure the token:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Gitaly server
@ -1791,11 +1791,11 @@ To configure the Sidekiq nodes, on each one:
on the page.
1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
```ruby
# https://docs.gitlab.com/omnibus/roles/#sidekiq-roles

32
doc/administration/reference_architectures/25k_users.md
View File

@ -1405,11 +1405,11 @@ To configure the Praefect nodes, on each one:
NOTE:
You can't remove the `default` entry from `virtual_storages` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage).
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Praefect server
@ -1569,12 +1569,12 @@ On each node:
1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure
storage paths, enable the network listener, and to configure the token:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Gitaly server
@ -1799,11 +1799,11 @@ To configure the Sidekiq nodes, on each one:
on the page.
1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
```ruby
# https://docs.gitlab.com/omnibus/roles/#sidekiq-roles

22
doc/administration/reference_architectures/2k_users.md
View File

@ -455,12 +455,12 @@ To configure the Gitaly server, on the server node you want to use for Gitaly:
NOTE:
You can't remove the `default` entry from `gitaly['configuration'][:storage]` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage).
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Gitaly server
@ -622,11 +622,11 @@ To configure the Sidekiq server, on the server node you want to use for Sidekiq:
on the page.
1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
```ruby
# https://docs.gitlab.com/omnibus/roles/#sidekiq-roles

32
doc/administration/reference_architectures/3k_users.md
View File

@ -1323,11 +1323,11 @@ To configure the Praefect nodes, on each one:
NOTE:
You can't remove the `default` entry from `virtual_storages` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage).
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Praefect server
@ -1487,12 +1487,12 @@ On each node:
1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure
storage paths, enable the network listener, and to configure the token:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Gitaly server
@ -1721,11 +1721,11 @@ To configure the Sidekiq nodes, on each one:
on the page.
1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
```ruby
# https://docs.gitlab.com/omnibus/roles/#sidekiq-roles

32
doc/administration/reference_architectures/50k_users.md
View File

@ -1411,11 +1411,11 @@ To configure the Praefect nodes, on each one:
NOTE:
You can't remove the `default` entry from `virtual_storages` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage).
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Praefect server
@ -1575,12 +1575,12 @@ On each node:
1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure
storage paths, enable the network listener, and to configure the token:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Gitaly server
@ -1806,11 +1806,11 @@ To configure the Sidekiq nodes, on each one:
on the page.
1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
```ruby
# https://docs.gitlab.com/omnibus/roles/#sidekiq-roles

32
doc/administration/reference_architectures/5k_users.md
View File

@ -1323,11 +1323,11 @@ To configure the Praefect nodes, on each one:
NOTE:
You can't remove the `default` entry from `virtual_storages` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage).
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Praefect server
@ -1487,12 +1487,12 @@ On each node:
1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure
storage paths, enable the network listener, and to configure the token:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration
- all reference architecture pages
-->
```ruby
# Avoid running unnecessary services on the Gitaly server
@ -1711,11 +1711,11 @@ To configure the Sidekiq nodes, on each one:
on the page.
1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration:
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
<!--
Updates to example must be made at:
- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md
- all reference architecture pages
-->
```ruby
# https://docs.gitlab.com/omnibus/roles/#sidekiq-roles

40
doc/architecture/blueprints/cells/decisions/001_routing_technology.md Normal file
View File

@ -0,0 +1,40 @@
---
owning-stage: "~devops::data stores"
description: 'Cells ADR 001: Routing Technology using Cloudflare Workers'
---
# Cells ADR 001: Routing Technology using Cloudflare Workers
## Context
In <https://gitlab.com/groups/gitlab-org/-/epics/11002> we first brainstormed [multiple options](https://gitlab.com/gitlab-org/gitlab/-/issues/428195#note_1664622245) and investigated our 2 top technologies,
[Cloudflare Worker](https://gitlab.com/gitlab-org/gitlab/-/issues/433471) & [Istio](https://gitlab.com/gitlab-org/gitlab/-/issues/433472).
We favored the Cloudflare Worker PoC and extended the PoC with the [Cell 1.0 proposal](https://gitlab.com/gitlab-org/gitlab/-/issues/437818) to have multiple routing rules.
These PoCs help validate the [routing service blueprint](../routing-service.md),
that got accepted in <https://gitlab.com/gitlab-org/gitlab/-/merge_requests/142397>,
and rejected the [request buffering](../proposal-stateless-router-with-buffering-requests.md),
and [routes learning](../proposal-stateless-router-with-routes-learning.md)
## Decision
Use [Cloudflare Workers](https://workers.cloudflare.com/) written in JavaScript/TypeScript to route the request to the right cell, following the accepted [routing service blueprint](../routing-service.md).
Cloudflare Workers meets all our [requirments](../routing-service.md#requirements) apart from the `self-managed`, which is a low priority requirment.
You can read a detailed analysis of Cloudflare workers in <https://gitlab.com/gitlab-org/gitlab/-/issues/433471#results>
## Consequences
- We will be choosing a technology stack knowing that it will not support all self-managed customers.
- More vendor locking with Cloudflare, but are already heavily dependent on them.
- Run compute in a new platform, outside of GCP, however we already use Cloudflare.
- We anticipate that we might to rewrite Routing Service if the decision changes.
We don't expect this to be big risk, since we expect Routing Service to be very small and simple (up to 1000 lines of code).
## Alternatives
- We considered [Istio](https://gitlab.com/gitlab-org/gitlab/-/issues/433472) but concluded that it's not the right fit.
- We considered [Request Buffering](../proposal-stateless-router-with-buffering-requests.md)
- We considered [Routes Learning](../proposal-stateless-router-with-routes-learning.md)
- Use WASM for Cloudflare workers which is the wrong choice: <https://blog.cloudflare.com/webassembly-on-cloudflare-workers#whentousewebassembly>

10
doc/architecture/blueprints/cells/index.md
View File

@ -20,7 +20,13 @@ For more information about Cells, see also:
## Cells Iterations
See Cell 1.0, Cell 1.5, and Cell 2.0.
- The [Cells 1.0](iterations/cells-1.0.md) target is to deliver a solution
for new enterprise customers using the SaaS GitLab.com offering.
- The [Cells 1.5](iterations/cells-1.5.md) target is to deliver a migration solution
for existing enterprise customers using the SaaS GitLab.com offering, built on top of architecture
of Cells 1.0.
- The [Cells 2.0](iterations/cells-2.0.md) target is to support a public and open source contribution
model in a cellular architecture.
## Goals
@ -424,7 +430,7 @@ The Tenant Scale team sees an opportunity to use GitLab Dedicated as a base for
## Decision log
- 2022-03-15: Google Cloud as the cloud service. For details, see [issue 396641](https://gitlab.com/gitlab-org/gitlab/-/issues/396641#note_1314932272).
- [ADR-001: Routing Technology using Cloudflare Workers](decisions/001_routing_technology.md)
## Links

671
doc/architecture/blueprints/cells/iterations/cells-1.0.md Normal file
View File

@ -0,0 +1,671 @@
---
stage: core platform
group: Tenant Scale
description: 'Cells: 1.0'
---
# Cells 1.0
This document describes a technical proposal for a Cells 1.0.
Cells 1.0 is a first iteration of the cellular architecture. The Cells 1.0 target is to deliver a solution
for new enterprise customers using the SaaS GitLab.com offering.
Cells 1.0 is a complete working feature that is meant to be deployed to GitLab.com SaaS.
Read more about [Cells 1.5](cells-1.5.md), which is meant to provide a mechanism to migrate existing customers
and is built on top of the Cells 1.0 architecture.
Read more about [Cells 2.0](cells-2.0.md), which is meant to support the public and open source
contribution model in a cellular architecture.
## Preamble
A Cells 1.0 is meant to target enterprise customers that have the following expectations:
1. They want to use our multi-tenant SaaS solution GitLab.com to serve their Organization.
1. They may receive updates later than the rest of GitLab.com.
1. They want use environment with higher degree of isolation to rest of the system.
1. They want to control all users that contribute to their Organization.
1. Their groups and projects are meant to be private.
1. Their users don't need to interact with many Organizations, or contribute to public projects with their account.
For example these authenticated users would receive a 404 if they navigated to any public project
like `gitlab.com/gitlab-org/gitlab`.
1. Are OK with not being able to switch Organizations with their account.
From a development and infrastructure perspective we want to achieve the following goals:
1. All Cells are accessible under a single domain.
1. Cells are mostly independent with minimal data sharing. All stateful data is segregated, and minimal data sharing is needed initially. This includes any database and cloud storage buckets.
1. Cells needs to be able to run independently with different versions.
1. The proposed architecture allows us to achieve a cluster-wide data sharing later.
1. We have a lightweight routing solution that is robust, but simple.
1. All identifiers (primary keys, user, group and project names) are unique across the cluster, so that we can perform logical re-balancing at a later time. This includes all database tables, except ones using schemas `gitlab_internal`, or `gitlab_shared`.
1. Since all users and groups are unique across the cluster, the same user will be able to access other Organizations and groups at GitLab.com in [Cells 2.0](cells-2.0.md).
1. The overhead of managing and upgrading Cells is minimal and similar to managing a GitLab Dedicated instance. Secondary Cells should not be a linear increase in operational burden.
1. The Cell should be deployed using the same tooling as GitLab Dedicated.
This proposal is designed to cut as much scope as possible but it must not make it impossible to meet the following long-term goals:
1. Users can interact with many Organizations.
1. Cells can be re-balanced by moving Organizations between Cells.
1. The routing solution can dynamically route customers.
## Proposal
The following statements describe a high-level proposal to achieve a Cells 1.0:
1. Terms used:
1. Primary Cell: The current GitLab.com deployment. A special purpose Cell that serves
as a cluster-wide service in this architecture.
1. Secondary Cells: A Cell that connects to the Primary Cell to ensure cluster-wide uniqueness.
1. Organization properties:
1. We allow customers to create a new Organization on a Secondary Cell.
The chosen Cell would be controlled by GitLab Administrators.
1. The Organization is private, and cannot be made public.
1. Groups and projects can be made private, but not public.
1. User properties:
1. Users are created on the Cell that contains the Organization.
1. Users are presented with the Organization navigation, but can only be part of a single Organization.
1. Users cannot join or interact with other Organizations.
1. User accounts cannot be migrated between Cells.
1. A user's personal namespace is created in that Organization.
The following statements describe a low-level development proposal to achieve the above goals:
1. Application properties:
1. Each secret token (personal access token, build token, runner token, etc.) generated by the application includes a unique identifier indicating the Cell, for example `us0`. The identifier should try to obfuscate information about the Cell.
1. The session cookie sent to the client is prefixed with a unique identifier indicating the Cell, for example `us0`.
1. The application configuration includes a Cell secret prefix, and information about the Primary Cell.
1. User always logs into the Cell on which the user was created.
1. Database properties:
1. Each primary key in the database is unique across the cluster. We use database sequences that are allocated from the Primary Cell.
1. We require each table to be classified: to be cluster-wide or Cell-local.
1. We follow a model of eventual consistency:
1. All cluster-wide tables are stored in a Cell-local database.
1. All cluster-wide tables retain unique constraints across the whole cluster.
1. Locally stored cluster-wide tables contain information required by this Cell only, unless it is the Primary Cell.
1. The cluster-wide tables are restricted to be modified by the Cell that is authoritative over the particular record:
1. The user record can be modified by the given Cell only if that Cell is the authoritative source of this record.
1. In Cells 1.0 we are likely to not be replicating data across cluster,
so the authoritative source is the Cell that contains the record.
1. The Primary Cell serves as a single source of truth for the uniqueness constraint (be it ID or user, group, project uniqueness).
1. Secondary Cells use APIs to claim usernames, groups or projects.
1. The Primary Cell holds information about all usernames, groups and projects, and the Cell holding the record.
1. The Primary Cell does not hold source information (actual user or project records), only references that are indicative of the Cell where that information is stored.
1. Routing properties:
1. We implement a static routing service that performs secret-based routing based on the prefix.
1. The routing service is implemented as a Cloudflare Worker and is run on edge. The routing service is run with a static list of Cells. Each Cell is described by a proxy URL, and a prefix.
1. Cells are exposed over the public internet, but might be guarded with Zero Trust.
### Overview (Architecture)
```plantuml
@startuml
cloud "Cloudflare" as CF {
[Routing Service Worker] as CF_RSW
}
node "GitLab Inc. Infrastructure" {
node "Primary Cell" as PC {
frame "GitLab Rails" as PC_Rails {
[Puma + Workhorse + LB] as PC_Puma
[Sidekiq] as PC_Sidekiq
}
[Container Registry] as PC_Registry
database DB as PC_DB {
frame "PostgreSQL Cluster" as PC_PSQL {
package "ci" as PC_PSQL_ci {
[gitlab_ci] as PC_PSQL_gitlab_ci
}
package "main" as PC_PSQL_main {
[gitlab_main_clusterwide] as PC_PSQL_gitlab_main_clusterwide
[gitlab_main_cell] as PC_PSQL_gitlab_main_cell
}
PC_PSQL_main -[hidden]-> PC_PSQL_ci
}
frame "Redis Cluster" as PC_Redis {
[Redis (many)] as PC_Redis_many
}
frame "Gitaly Cluster" as PC_Gitaly {
[Gitaly Nodes (many)] as PC_Gitaly_many
}
}
PC_Rails -[hidden]-> PC_DB
}
node "Secondary Cell" as SC {
frame "GitLab Rails" as SC_Rails {
[Puma + Workhorse + LB] as SC_Puma
[Sidekiq] as SC_Sidekiq
}
[Container Registry] as SC_Registry
database DB as SC_DB {
frame "PostgreSQL Cluster" as SC_PSQL {
package "ci" as SC_PSQL_ci {
[gitlab_ci] as SC_PSQL_gitlab_ci
}
package "main" as SC_PSQL_main {
[gitlab_main_clusterwide] as SC_PSQL_gitlab_main_clusterwide
[gitlab_main_cell] as SC_PSQL_gitlab_main_cell
}
SC_PSQL_main -[hidden]-> SC_PSQL_ci
}
frame "Redis Cluster" as SC_Redis {
[Redis (many)] as SC_Redis_many
}
frame "Gitaly Cluster" as SC_Gitaly {
[Gitaly Nodes (many)] as SC_Gitaly_many
}
}
SC_Rails -[hidden]-> SC_DB
}
}
CF_RSW --> PC_Puma
CF_RSW --> PC_Registry
CF_RSW --> SC_Puma
CF_RSW --> SC_Registry
@enduml
```
### Overview (API)
```plantuml
@startuml
skinparam FrameBackgroundcolor white
node "GitLab Inc. Infrastructure" {
node "Primary Cell" as PC {
frame "GitLab Rails" as PC_Rails {
[Puma + Workhorse + LB] as PC_Puma
[Sidekiq] as PC_Sidekiq
}
}
node "Secondary Cell" as SC {
frame SC_Rails [
{{
[Puma + Workhorse + LB] as SC_Puma
[Sidekiq] as SC_Sidekiq
}}
]
}
}
SC_Rails -up-> PC_Puma : "Primary Cell API:\n/api/v4/internal/cells/..."
@enduml
```
### Problems
The following technical problems have to be addressed:
1. All secrets are prefixed as this is required by a simple routing layer to perform secret-based routing.
1. All usernames, Organizations, top-level groups (and as result groups and projects) are unique across the cluster.
1. All primary key identifiers are unique across the cluster.
### GitLab Configuration
The GitLab configuration in `gitlab.yml` will be extended with the following parameters to:
```yaml
production:
gitlab:
primary_cell:
url: https://cell1.gitlab.com
token: abcdef
secrets_prefix: kPptz
```
1. `primary_cell:` will be configured on Secondary Cells, and will indicate the URL endpoint to access the Primary Cell API.
1. `secrets_prefix:` can be used on all Cells, and indicates that each secret and session cookie is prefixed with this identifier.
### Primary Cell
The Primary Cell serves a special purpose to ensure cluster uniqueness.
The Primary Cell exposes a set of API interfaces to be used by Secondary Cells.
The API is considered internal, and is guarded with a secret that is shared with Secondary Cells.
1. `POST /api/v4/internal/cells/database/claim`
1. Request:
- `table`: table name for the allocated sequence, for example `projects`
- `count`: number of IDs to claim that are guaranteed to be unique, for example 100_000
1. Response:
- `start`: the start sequence value
- `limit`: the allowed maximum sequence value
1. `POST /api/v4/internal/cells/routes`
1. Request:
- `path`: the full path to a resource, for example `my-company/my-project`
- `source_type`: the class name for the container holding the resource, for example `project`
- `source_id`: the source ID for the container holding the resource, for example `1000`
- `namespace_id`: the underlying namespace associated with the resource, for example `32`
- `name`: the display name for the resource
- `cell_id`: the identifier of the Cell holding the resource
1. Response:
- 201: Created: The resource was created.
- `id:`: The ID of the resource as stored on the Primary Cell.
- 409: Conflict: The resource already exists.
1. Behavior:
1. The endpoint returns an `id`. The same ID has to be used to insert a record in `routes` table for the calling cell.
1. `GET /api/v4/internal/cells/routes/:id`
1. Request:
- `id`: resource identifier
1. Response:
- 200: OK: The resource was found. For response parameters look at `POST /api/v4/internal/cells/routes` request parameters.
- 404: Not found: The resource does not exit.
1. `PUT /api/v4/internal/cells/routes/:id`
1. Request: For parameters look at `POST /api/v4/internal/cells/routes` request parameters.
1. Response:
- 200: OK: The resource was updated.
- 403: Forbidden: The resource cannot be modified, because it is owned by another `cell_id`.
- 404: Not found: The resource does not exit.
- 409: Conflict: The resource already exists. The uniqueness constraint on `path` failed.
1. Behavior:
1. The endpoint modifies the given resource as long the `cell_id` matches.
1. `DELETE /api/v4/internal/cells/routes/:id`
1. Request: For parameters look at `POST /api/v4/internal/cells/routes` request parameters.
1. Response:
- 200: OK: The resource with given parameters was successfully deleted.
- 404: Not found: The given resource was not found.
- 400: Bad request: The resource failed to be deleted.
1. `POST /api/v4/internal/cells/redirect_routes`
1. `GET /api/v4/internal/cells/redirect_routes/:id`
1. `PUT /api/v4/internal/cells/redirect_routes/:id`
1. `DELETE /api/v4/internal/cells/redirect_routes/:id`
### Secondary Cell
The Secondary Cell does not expose any specific API at this point.
The Secondary Cell implements a solution to guarantee uniqueness of primary database keys.
1. `ReplenishDatabaseSequencesWorker`: this worker will run periodically, check all sequences, and replenish them.
#### Simple uniqueness of Database Sequences
Currently our DDL schema uses ID generation in the form: `id bigint DEFAULT nextval('product_analytics_events_experimental_id_seq'::regclass) NOT NULL`.
The `/api/v4/internal/cells/database/claim` would execute the following sequence to claim range:
```ruby
def claim_table_seq(table, count)
# Example: Use trick to get increase sequence by specific number in transactional way
# Making the transaction to be "atomic".
sql = <<-SQL
begin;
ALTER SEQUENCE ? INCREMENT BY ?;
SELECT nextval(?);
ALTER SEQUENCE ? INCREMENT BY 1;
% TODO: insert into cells_sequence_claims(cell_id, table_name, start_id, end_id)
commit;
SQL
seq_name = "#{table}_id_seq"
last_id = select_one(sql, seq_name, limit, seq_name, seq_name).first
{ start: last_id - count, limit: count }
end
```
The ReplenishDatabaseSequencesWorker would check how much space is left in a sequence, and request a new range if the value goes below the threshold.
```ruby
def replenish_table_seq(table, lower_limit, count)
seq_name = "#{table}_id_seq"
# maxval is not existing, so it would have to be implemented different way, but this is to showcase purposes
last_id, max_id = select_one("SELECT currval(?), maxval(?)", seq_name, seq_name)
return if max_id - last_id > lower_limit
new_start, new_limit = post("/api/v4/internal/cells/database/claim", { table: table, count: count })
execute("ALTER SEQUENCE START ? MAXVALUE ?", new_start, new_limit)
end
```
This makes us lose the `lower_limit` of IDs of sequence creating gaps. However, in this model we need to replenish the
sequence ahead of time, otherwise we will have catastrophic failure on inserting new records.
#### Robust uniqueness of table sequences
This is very similar to simple uniqueness, with these additions:
- We use and alternate between two sequences per-table (A/B), fully utilizing allocated sequences.
- We change the `DEFAULT` to accept two arguments to function `nextval2(table_id_seq_a::regclass, table_id_seq_b::regclass)`.
- We replenish the sequence as soon as it runs out of free IDs.
- We don't create sequence gaps.
```sql
CREATE FUNCTION nextval2(seq_a oid, seq_b oid) RETURNS bigint
LANGUAGE plpgsql
AS $$
BEGIN
-- pick from sequence that has lower number
-- as we want to retain monotonically increasing numbers
-- when allocation fails (as the sequence is exhausted) switch to another one
SELECT last_value INTO seq_a_last_value FROM seq_a;
SELECT last_value INTO seq_b_last_value FROM seq_b;
IF seq_a_last_value < seq_b_last_value
BEGIN TRY
RETURN nextval(seq_a);
END TRY
BEGIN CATCH
RETURN nextval(seq_b);
END CATCH;
ELSE
BEGIN TRY
RETURN nextval(seq_b);
END TRY
BEGIN CATCH
RETURN nextval(seq_a);
END CATCH;
END
END
$$;
```
## Pros
- The proposal is lean:
- Each Cell holds only a fraction of the data that is required for the cluster to operate.
- The tables marked as `main_clusterwide` that are stored locally can be selectively replicated across Cells following a mesh-architecture.
- Based on the assumption that Cells require only a fraction of shared data (like users), it is expected that Secondary Cells might need a small percentage of records across the whole cluster.
- The primary Cell is a single point of failure only for a limited set of features:
- Uniqueness is enforced by the primary Cell.
- The temporary reliability of the primary Cell has a limited impact on Secondary Cells.
- Secondary Cells would not be able to enforce unique constraints: create group, project, or user.
- Other functionality of Secondary Cells would continue working as is: push, run CI.
- The routing layer makes this service very simple, because it is secret-based and uses prefix.
- Reliability of the service is not dependent on Cell availability, since at this stage no dynamic classification is required.
- We anticipate that the routing layer will evolve to perform regular classification at a later point.
- Mixed-deployment compatible by design.
- We do not share database connections. We expose APIs to interact with cluster-wide data.
- The application is responsible to support API compatibility across versions, allowing us to easily support many versions of the application running from day zero.
- Database migrations.
- Work out of the box, and are isolated to the Cell.
- Because we don't share databases across Cells, the Cell has full ownership of `main_clusterwide` tables.
## Cons
- We intentionally split data for tables marked as `main_clusterwide` across all cluster cells.
- These tables will be selectively replicated (likely) outside of the application to allow Secondary Cells to share data.
- We effectively reinvent database replication to some extent.
- We are severely limited by how many tables can be made `main_clusterwide`. Exposing many tables is a significant amount of additional code to allow cross-Cell interaction.
- We require all tables to be classified. We want to ensure data consistency across the cluster if records are replicated.
## Requirements
## Questions
1. How do we create new Organizations with the user on Secondary Cell?
To be defined.
1. How do we register new users for the existing Organization on Secondary Cell?
If an Organization is already created, users can be invited.
We can then serve the registration flow from Secondary Cell.
1. How would users log in?
- UI: The login to Organizations would be scoped to the Organization: `https://<GITLAB_DOMAIN>/users/sign_in?organization=gitlab-inc`.
- SAML: `https://<GITLAB_DOMAIN>/users/auth/saml/callback` would receive `?organization=gitlab-inc` which would be routed to the correct Cell.
- This would require using the dynamic routing method with a list of Organizations available using a solution with high availability.
1. How do we add a new table if it is initially deployed on Secondary Cell?
The Primary Cell is ensuring uniqueness of sequences, so it needs to have `sequence`.
1. Is Container Registry cluster-wide or cell-local?
The Container Registry can be run Cell-local, and if we follow the secret-based routing, it can follow the same model for filtering.
We would have to ensure that the JWT token signed by GitLab is in a form that can be statically routed by the routing layer.
1. Are GitLab Pages cluster-wide or Cell-local?
If GitLab Pages are not essential we would not support them in for Cells 1.0.
This is to be defined.
If GitLab Pages are meant to support the `.gitlab.io` domain:
- GitLab Pages need to be run as a single service that is not run as part of a Cell.
- Since GitLab Pages use the API we need to make them routable.
- Similar to `routes`, claim `pages_domain` on the Primary Cell
- Implement dynamic classification in the routing service, based on a sharding key.
- Cons: This adds another table that has to be kept unique cluster-wide.
Alternatively:
- Run GitLab Pages within a Cell, but provide separate a domain.
- Custom domains would use the separate domain.
- Cons: This creates a problem with having to manage a domain per Cell.
- Cons: We expose Cells to users.
1. Should we use a static secret for the internal endpoint or JWT?
To be defined. The static secret is used for simplicity of the presented solution.
1. How do we handle SSH cloning?
This is a separate problem tangential to routing, which is intentionally not solved by this proposal.
The problems associated with SSH cloning are:
- We need to validate a user public key to identify the Cell holding it.
- We need to ensure uniqueness of public keys across the cluster.
1. Are there other cluster-wide unique constraints?
- Authorized keys
- GPG keys
- Custom e-mails
- Pages domains
- TBD
1. How do we synchronize cluster-wide tables, such as broadcast messages or application settings?
We would very likely take a naive approach: expose those information using API, and synchronize it periodically.
Tables likely to be made synchronized are:
- Application Settings
- Broadcast Messages
- TBD
1. How do we dogfood this work?
To be defined.
1. How do we manage admin accounts?
Since we don't yet synchronize across the cluster, admin accounts would have to be provided per Cell.
This might be solved by GitLab Dedicated already?
1. Is it a Primary Cell or cluster-wide service?
The Primary Cell in fact serves as a cluster-wide service. Depending on our intent it could be named the following:
- Primary Cell: To clearly state that the Primary Cell has a special purpose today, but we rename it later.
- Cluster-wide Data Provider: This is the current name used in the [Deployment Architecture](../deployment-architecture.md).
- Global Service: Alternative name to Cluster-wide Data Provider, indicating that the Primary Cell would implement a Global Service today.
1. How are secrets are generated?
The Cell prefix is used to generate a secret in a way that encodes the prefix. The prefix is added to the generated secret.
Example:
- GitLab Runner Tokens are generated in the form: `glrt-2CR8_eVxiioB1QmzPZwa`
- For Cell prefix: `secrets_prefix: kPptz`
- We would generate Runner tokens in the form: `glrt-kPptz_2CR8_eVxiioB1QmzPZwa`
1. Why secrets-based routing instead of path-based routing?
Cells 1.0 are meant to provide a first approximation of the architecture allowing to deploy a multi-tenant and multi-Cell solution quickly.
Solving path-based routing requires significant effort:
- Prefixing all existing routes with `/org/<org-name>/-/` will require a
deliberate choice how to handle ambiguous or Cell-local or cluster-wide routes, such as
`/api/v4/jobs/request`. Changing routes also introduces breaking changes for all users
that we would migrate.
- Making all existing routes routable would be a significant effort to fix
for routes like `/-/autocomplete/users` and likely a multi-year effort. Some preliminary analysis
how many routes are already classified can be found [here](https://gitlab.com/gitlab-org/gitlab/-/issues/430330#note_1633125914).
- In each case the routing service needs to be able to dynamically classify existing routes
based on some defined criteria, requiring significant development effort, and increasing the
dependency on the Primary Cell or Cluster-wide service.
By following secret-based routing we can cut a lot of initial complexity, which allows us to
make the best decision at a later point:
- The injected prefix is a temporary measure to have a static routing mechanism.
- We will have to implement dynamic classification anyway at a later point.
- We can figure out what is the best way to approach path-based routing after Cells 1.0.
1. What about data migration between Cells?
Cells 1.0 is targeted towards **new** customers. Migrating existing customers is a big undertaking on its own:
- Customers to be migrated need to opt into the Organization model first.
- We expect most of the isolation features to work.
- We have to transparently move data from the source Cell to the target Cell. Initially we would
follow a model of Cell split. We would clone the Cell and mark where the given record is located.
Existing customers could still be migrated onto a Cells 1.0, but it would require to use import/export
features similar to how we migrate customers from GitLab.com to Dedicated. As a result, not all information
would be migrated. For example, the current project import/export does neither migrate CI traces, nor job artefacts.
1. Is secret-based routing a problem?
To be determined.
1. How would we synchronize `users` across Cells?
We build out-of-bounds replication of tables marked as `main_clusterwide`. We have yet to define
if this would be better to do via an `API` that is part of Rails, or using the Dedicated service.
However, using Rails would likely be the simplest and most reliable solution, because the
application knows the expected data structure.
Following the above proposal we would expose `users` and likely all adjacent tables using `API`:
`/api/v4/internal/cells/users/:id`. The API would serialize the `users` table into a `protobuf`
data model.
This information would be fetched by another Cell that would synchronize user entries.
1. How would the Cell find users or projects?
To be defined. However, we would need `Primary Cell`/`Cluster-wide Data Provider` access to the `routes` table that contains a mapping of all names into objects (group, project, user) and the Cell holding the information.
1. Would the User Profile be public if created for enterprise customer?
No. Users created on another Cell in a given Organization would be limited to this Organization only.
The User Profile would be available as long as you are logged in.
1. What is the resiliency of a Primary Cell exposing cluster-wide API?
The API would be responsible for ensuring uniqueness of: User, Groups, Projects, Organizations, SSH keys, Pages domains, e-mails.
The API would also be responsible for classifying a sharding key for the routing service.
We need to ensure that the Primary Cell cluster-wide API is highly available. The solution here could be to:
1. Run the Primary Cell API as an additional node that has dedicated database replica and can work while the main Cell is down.
1. Implement a Primary Cell API on top of another highly available database in a different technology than Rails. Forward the API write calls to this storage (claim), but make read API calls (classify) to use this storage.
1. How will instance wide CI runners be configured on the new cells?
To be defined.
1. Why not use FQDN (`mycorp.gitlab.com` or `mygitlab.mycorp.com`) instead?
We want the `gitlab.com` to feel like a single application, regardless how many organizations you interact with.
This means that we want to share users, and possibly some data between organizations at later point.
Following model of `mycorp.gitlab.com` or `mygitlab.mycorp.com` creates a feel of a strong isolation between instances,
and different perception of how system operates compared to how we the system to actually behaves.
1. Are feature-flags cell-local or cluster-wide?
To be defined.
1. How the Cells 1.0 differ to [GitLab Dedicated](https://about.gitlab.com/dedicated/)?
- The GitLab Dedicated is a single-tenant hosted solution provided by GitLab Inc.
to serve a single customer under a custom domain.
- The Cellular architecture is meant to be multi-tenant solution using `gitlab.com` domain.
- The GitLab Dedicated by design has all resources separate between tenants.
- The Cellular architecture does share resources between tenants to achieve greater
operational efficiency and lower cost.
1. Why the `Organization is private, and cannot be made public` in Cells 1.0?
- Private organizations on Secondary Cells do not require any data sharing
or isolation as this is achieved by the current system.
- We don't have to do extensive work to isolate organizations yet.
- Routing is simpler since we anticipate all requests to be authenticated,
making them easier to route.
- We don't have to "fix" routes to make them sharded yet.
1. Why not to prefix all endpoints with the `relative_path` for all organizations?
This breaks the main contract of what we want to achieve:
- Migration to use organizations is seamless.
- We do not want to break existing user workflows if user migrates their groups to organization,
or when we migrate the organization to another Cell.
- For the following reason this is why we don't want to force particular paths, or use of subdomains.
- If we choose the path to force to use `relative_path` it would break all cell-wide endpoints
This seems to be longer and more complex that approaching this by making existing to be shareded.
- If we choose to fix existing not sharded [can be made](https://gitlab.com/gitlab-org/gitlab/-/issues/430330)
at later point we will achieve much better API consistency, and likely much less amount of the work.
1. Why not use subdomains, like `mycorp.gitlab.com`?
Reasons to avoid using DNS for routing:
- Risks setting an expectation of full isolation, which is not the case.
- Care must be taken to prevent security issues with cookies leaking across subdomains.
- Complicates working across Organizations, with changing hostnames.
and a requirement to pass full hostname in all links.
- Complicates integrations and API interactions, as the URL and host will change based on the organization.
- We will need to build a common login service, which will redirect users to a valid organization/URL.
- Increased risk of naming collisions with higher impact. For example different divisions
or organizations across a large enterprise, or companies with similar names.
1. What about Geo support?
The Geo support is out of scope of Cells 1.0. However, it is fair to assume the following:
- Geo is per-Cell.
- Routing Service can direct to Geo replica of the Cell (if it knows it).
- We might have many routing services in many regions.

31
doc/architecture/blueprints/cells/iterations/cells-1.5.md Normal file
View File

@ -0,0 +1,31 @@
---
stage: core platform
group: Tenant Scale
description: 'Cell: 1.5'
---
# Cells 1.5
This document describes a technical proposal for a Cells 1.5 that builds on top of [Cells 1.0](cells-1.0.md).
The Cells 1.5 target is to deliver a migration solution for existing enterprise customers using the SaaS GitLab.com offering.
## Preamble
Cells 1.5 is meant to target existing enterprise customers:
1. They are existing customers of the GitLab.com Primary Cell and want to use the Organization model.
1. They want their Organization to be isolated from the rest of GitLab.com.
1. They are fine with limiting their users from accessing other Organizations (including public projects on other Organizations).
1. Their groups and projects that are meant to be private.
From a development and infrastructure perspective we want to achieve the following goals:
1. Customers that migrated to the Organization model are isolated from each other.
1. We can migrate Organizations from the Primary Cell to another Cell without user intervention or changing any user workflows.
1. The routing solution can dynamically route customers to the correct Cell once they are migrated.
Long-term we want to achieve the following goals:
1. The proposed architecture must not prevent us from implementing the ability for users to interact with many Organizations.
1. Cells can be re-balanced by moving Organizations between Cells.

24
doc/architecture/blueprints/cells/iterations/cells-2.0.md Normal file
View File

@ -0,0 +1,24 @@
---
stage: core platform
group: Tenant Scale
description: 'Cells: 2.0'
---
# Cells 2.0
This document describes a technical proposal for a Cells 2.0 that builds on top of [Cells 1.0](cells-1.0.md).
The Cells 2.0 target is to support a public and open source contribution model in a cellular architecture.
## Preamble
Cells 2.0 is meant to target public and open source Organizations on GitLab.com:
1. Existing users can create public Organizations that are isolated from the rest of GitLab.com.
1. A single user can be part of many Organizations that are on different Cells.
1. Users can contribute to public projects across Cells.
From a development and infrastructure perspective we want to achieve the following goals:
1. We can migrate public Organizations between Cells without user intervention or a user changing any of their workflows.
1. The routing solution allows seamless interaction with many Organizations at the same time.

6
doc/architecture/blueprints/cells/routing-service.md
View File

@ -30,6 +30,8 @@ For example:
If it is required to make the service multi-cloud it might be required to deploy it to the CDN provider.
Then the service needs to be written using a technology compatible with the CDN provider.
[ADR 001](decisions/001_routing_technology.md)
1. **Cell discovery.**
The routing service needs to be able to discover and monitor the health of all Cells.
@ -764,10 +766,6 @@ sequenceDiagram
due to low cardinality of sharding key. The sharding key would effectively be mapped
into resource (organization, group, or project), and there's a finite amount of those.
## Technology
TBD
## Alternatives
### Buffering requests

1
doc/ci/runners/runner_fleet_dashboard.md
View File

@ -10,6 +10,7 @@ info: >-
DETAILS:
**Tier:** Ultimate
**Offering:** Self-managed
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/424495) in GitLab 16.6

21
doc/development/ai_features/index.md
View File

@ -32,9 +32,10 @@ info: Any user with at least the Maintainer role can merge updates to this conte
## Feature flags
Apply the following two feature flags to any AI feature work:
Apply the following feature flags to any AI feature work:
- A general flag (`ai_global_switch`) that applies to all AI features.
- A general flag (`ai_duo_chat_switch`) that applies to all GitLab Duo Chat features.
- A general flag (`ai_global_switch`) that applies to all other AI features.
- A flag specific to that feature. The feature flag name [must be different](../feature_flags/index.md#feature-flags-for-licensed-features) than the licensed feature name.
See the [feature flag tracker epic](https://gitlab.com/groups/gitlab-org/-/epics/10524) for the list of all feature flags and how to use them.
@ -62,6 +63,7 @@ RAILS_ENV=development bundle exec rake gitlab:duo:setup['<test-group-name>']
1. Enable the required general feature flags:
```ruby
Feature.enable(:ai_duo_chat_switch, type: :ops)
Feature.enable(:ai_global_switch, type: :ops)
```
@ -220,15 +222,11 @@ Therefore, a different setup is required from the [SaaS-only AI features](#test-
```
Alternatively, you can create an `env.runit` file in the root of your GDK with the above snippet.
1. Enable the following feature flags via `gdk rails console`:
1. Enable all AI feature flags:
```ruby
# NOTE: This feature flag name might be changed. See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/140352.
::Feature.enable(:ai_global_switch)
# This is to request to AI Gateway instead of built-in Anthropic client. See https://gitlab.com/gitlab-org/gitlab/-/issues/433213 for more info.
::Feature.enable(:gitlab_duo_chat_requests_to_ai_gateway)
```
```shell
rake gitlab:duo:enable_feature_flags
```
1. Create a dummy access token via `gdk rails console` OR skip this step and setup GitLab or Customer Dot as OIDC provider (See the following section):
@ -462,7 +460,8 @@ end
We recommend to use [policies](../policies.md) to deal with authorization for a feature. Currently we need to make sure to cover the following checks:
1. General AI feature flag (`ai_global_switch`) is enabled
1. For GitLab Duo Chat feature, `ai_duo_chat_switch` is enabled
1. For other general AI features, `ai_global_switch` is enabled
1. Feature specific feature flag is enabled
1. The namespace has the required license for the feature
1. User is a member of the group/project

1
doc/development/database/partitioning/list.md
View File

@ -347,6 +347,7 @@ class EnsureIdUniquenessForPCiBuilds < Gitlab::Database::Migration[2.1]
DROP FUNCTION IF EXISTS #{FUNCTION_NAME} CASCADE;
SQL
end
end
```
### Step 9 - Analyze the partitioned table and create new partitions

11
doc/development/fe_guide/accessibility.md
View File

@ -1,11 +0,0 @@
---
redirect_to: 'accessibility/index.md'
remove_date: '2024-01-12'
---
This document was moved to [another location](accessibility/index.md).
<!-- This redirect file can be deleted after <2024-01-12>. -->
<!-- Redirects that point to other docs in the same project expire in three months. -->
<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. -->
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->

11
doc/development/internal_analytics/service_ping/implement.md
View File

@ -1,11 +0,0 @@
---
redirect_to: '../metrics/metrics_instrumentation.md'
remove_date: '2024-01-13'
---
This document was moved to [another location](../metrics/metrics_instrumentation.md).
<!-- This redirect file can be deleted after <2024-01-13>. -->
<!-- Redirects that point to other docs in the same project expire in three months. -->
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->

11
doc/development/internal_analytics/service_ping/metrics_instrumentation.md
View File

@ -1,11 +0,0 @@
---
redirect_to: '../metrics/metrics_instrumentation.md'
remove_date: '2024-01-13'
---
This document was moved to [another location](../metrics/metrics_instrumentation.md).
<!-- This redirect file can be deleted after <2024-01-13>. -->
<!-- Redirects that point to other docs in the same project expire in three months. -->
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->

2
doc/integration/facebook.md
View File

@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
DETAILS:
**Tier:** Free, Premium, Ultimate
**Offering:** SaaS, self-managed
**Offering:** Self-managed
You can use the Facebook OmniAuth provider to authenticate users with their Facebook account.

2
doc/integration/jira/dvcs/index.md
View File

@ -13,7 +13,7 @@ DETAILS:
WARNING:
The Jira DVCS connector for Jira Cloud was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/362168) in GitLab 15.1
and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118126) in 16.0. Use the [GitLab for Jira Cloud app](../connect-app.md) instead.
The Jira DVCS connector was also deprecated and removed for Jira 8.13 and earlier. You can only use the Jira DVCS connector with Jira Data Center or Jira Server in Jira 8.14 and later. Upgrade your Jira instance to Jira 8.14 or later, and reconfigure the Jira integration in your GitLab instance.
The Jira DVCS connector was also deprecated and removed for Jira 8.13 and earlier. You can only use the Jira DVCS connector with Jira Data Center or Jira Server in Jira 8.14 and later. Upgrade your Jira instance to Jira 8.14 or later and reconfigure the Jira integration on your GitLab instance.
Use the Jira DVCS (distributed version control system) connector if you self-host
your Jira instance with Jira Data Center or Jira Server and want to use the [Jira development panel](../development_panel.md).

4
doc/integration/jira/dvcs/troubleshooting.md
View File

@ -26,9 +26,9 @@ Error obtaining access token. Cannot access https://gitlab.example.com from Jira
When you use GitLab 15.0 and later with Jira Server, you might encounter a
[session token bug in Jira](https://jira.atlassian.com/browse/JSWSERVER-21389).
This bug affects Jira Server 8.20.8, 8.22.3, 8.22.4, 9.4.6, and 9.4.14.
This bug affects Jira Server versions 8.20.8, 8.22.3, 8.22.4, 9.4.6, and 9.4.14.
To resolve this issue, ensure you use Jira Server version 9.1.0 and later or 8.20.11 and later.
To resolve this issue, ensure you use Jira Server 8.20.11 and later or 9.1.0 and later.
## SSL and TLS problems

2
doc/integration/vault.md
View File

@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
DETAILS:
**Tier:** Free, Premium, Ultimate
**Offering:** SaaS, self-managed
**Offering:** Self-managed
[Vault](https://www.vaultproject.io/) is a secrets management application offered by HashiCorp.
It allows you to store and manage sensitive information such as secret environment

35
doc/raketasks/user_management.md
View File

@ -182,6 +182,41 @@ sudo /etc/init.d/gitlab start
```
## Bulk assign users to GitLab Duo Pro
To perform bulk user assignment for GitLab Duo Pro, you can use the following Rake task:
```shell
bundle exec rake duo_pro:bulk_user_assignment DUO_PRO_BULK_USER_FILE_PATH=path/to/your/file.csv
```
If you prefer to use square brackets in the file path, you can escape them or use double quotes:
```shell
bundle exec rake duo_pro:bulk_user_assignment\['path/to/your/file.csv'\]
# or
bundle exec rake "duo_pro:bulk_user_assignment['path/to/your/file.csv']"
```
The CSV file should have the following format:
```csv
username
user1
user2
user3
user4
etc..
```
Ensure that the file contains a header named `username`, and each subsequent row represents a username for user assignment.
The task might raise the following error messages:
- `User is not found`: The specified user was not found.
- `ERROR_NO_SEATS_AVAILABLE`: No more seats are available for user assignment.
- `ERROR_INVALID_USER_MEMBERSHIP`: The user is not eligible for assignment due to being inactive, a bot, or a ghost.
## Related topics
- [Reset a user's password](../security/reset_user_password.md#use-a-rake-task)

14
doc/user/enterprise_user/index.md
View File

@ -204,10 +204,18 @@ For more information, see how to [configure enterprise user settings from the SA
A top-level group Owner can [set up verified domains to bypass confirmation emails](../group/saml_sso/index.md#bypass-user-email-confirmation-with-verified-domains).
### Get users' email addresses through the API
### Get users' email addresses
A top-level group Owner can use the [group and project members API](../../api/members.md) to access
users' information. For enterprise users of the group this information includes users' email addresses.
A top-level group Owner can use the UI to access enterprise users' email addresses:
1. On the left sidebar, select **Search or go to** and find your project or group.
1. Select **Manage > Members**.
1. In the group or project members page, hover over the enterprise user's name to
see their email address.
A group Owner can also use the [group and project members API](../../api/members.md)
to access users' information. For enterprise users of the group, this information
includes users' email addresses.
### Remove enterprise management features from an account

37
gems/gitlab-housekeeper/lib/gitlab/housekeeper/change.rb Normal file
View File

@ -0,0 +1,37 @@
# frozen_string_literal: true
module Gitlab
module Housekeeper
class Change
attr_accessor :identifiers,
:title,
:description,
:changed_files,
:labels,
:reviewers
def mr_description
<<~MARKDOWN
#{description}
This change was generated by
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
MARKDOWN
end
def commit_message
<<~MARKDOWN
#{title}
#{mr_description}
Changelog: other
MARKDOWN
end
def valid?
@identifiers && @title && @description && @changed_files
end
end
end
end

10
gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb
View File

@ -46,15 +46,7 @@ module Gitlab
Shell.execute("git", "checkout", "-b", branch_name)
Shell.execute("git", "add", *change.changed_files)
commit_message = <<~MSG
#{change.title}
#{change.description}
MSG
Shell.execute("git", "commit", "-m", commit_message)
Shell.execute("git", "commit", "-m", change.commit_message)
ensure
Shell.execute("git", "checkout", current_branch)
end

167
gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb
View File

@ -43,6 +43,9 @@ module Gitlab
changes << :title if note['body'].start_with?("changed title from")
changes << :description if note['body'] == "changed the description"
changes << :code if note['body'].match?(/added \d+ commit/)
changes << :reviewers if note['body'].include?('requested review from')
changes << :reviewers if note['body'].include?('removed review request for')
end
resource_label_events = get_merge_request_resource_label_events(target_project_id: target_project_id, iid: iid)
@ -61,17 +64,17 @@ module Gitlab
changes.to_a
end
# rubocop:disable Metrics/ParameterLists
def create_or_update_merge_request(
change:,
source_project_id:,
title:,
description:,
labels:,
source_branch:,
target_branch:,
target_project_id:,
update_title:,
update_description:,
update_labels:
update_labels:,
update_reviewers:
)
existing_iid = get_existing_merge_request(
source_project_id: source_project_id,
@ -82,105 +85,48 @@ module Gitlab
if existing_iid
update_existing_merge_request(
change: change,
existing_iid: existing_iid,
title: title,
description: description,
labels: labels,
target_project_id: target_project_id,
update_title: update_title,
update_description: update_description,
update_labels: update_labels
update_labels: update_labels,
update_reviewers: update_reviewers
)
else
create_merge_request(
change: change,
source_project_id: source_project_id,
title: title,
description: description,
labels: labels,
source_branch: source_branch,
target_branch: target_branch,
target_project_id: target_project_id
)
end
end
# rubocop:enable Metrics/ParameterLists
private
def get_merge_request_notes(target_project_id:, iid:)
response = HTTParty.get(
"#{@base_uri}/projects/#{target_project_id}/merge_requests/#{iid}/notes",
query: {
per_page: 100
},
headers: {
"Private-Token" => @token
}
)
unless (200..299).cover?(response.code)
raise Error,
"Failed to get merge request notes with response code: #{response.code} and body:\n#{response.body}"
end
JSON.parse(response.body)
request(:get, "/projects/#{target_project_id}/merge_requests/#{iid}/notes", query: { per_page: 100 })
end
def get_merge_request_resource_label_events(target_project_id:, iid:)
response = HTTParty.get(
"#{@base_uri}/projects/#{target_project_id}/merge_requests/#{iid}/resource_label_events",
query: {
per_page: 100
},
headers: {
"Private-Token" => @token
}
)
unless (200..299).cover?(response.code)
raise Error,
"Failed to get merge request label events with response code: #{response.code} and body:\n#{response.body}"
end
JSON.parse(response.body)
request(:get, "/projects/#{target_project_id}/merge_requests/#{iid}/resource_label_events",
query: { per_page: 100 })
end
def current_user_id
@current_user_id = begin
response = HTTParty.get(
"#{@base_uri}/user",
headers: {
"Private-Token" => @token
}
)
unless (200..299).cover?(response.code)
raise Error,
"Failed with response code: #{response.code} and body:\n#{response.body}"
end
data = JSON.parse(response.body)
data['id']
end
@current_user_id ||= request(:get, "/user")['id']
end
def get_existing_merge_request(source_project_id:, source_branch:, target_branch:, target_project_id:)
response = HTTParty.get("#{@base_uri}/projects/#{target_project_id}/merge_requests",
query: {
state: :opened,
source_branch: source_branch,
target_branch: target_branch,
source_project_id: source_project_id
},
headers: {
'Private-Token' => @token
})
unless (200..299).cover?(response.code)
raise Error,
"Failed with response code: #{response.code} and body:\n#{response.body}"
end
data = JSON.parse(response.body)
data = request(:get, "/projects/#{target_project_id}/merge_requests", query: {
state: :opened,
source_branch: source_branch,
target_branch: target_branch,
source_project_id: source_project_id
})
return nil if data.empty?
@ -192,63 +138,64 @@ module Gitlab
end
def create_merge_request(
change:,
source_project_id:,
title:,
description:,
labels:,
source_branch:,
target_branch:,
target_project_id:
)
response = HTTParty.post("#{@base_uri}/projects/#{source_project_id}/merge_requests", body: {
title: title,
description: description,
labels: Array(labels).join(','),
request(:post, "/projects/#{source_project_id}/merge_requests", body: {
title: change.title,
description: change.mr_description,
labels: Array(change.labels).join(','),
source_branch: source_branch,
target_branch: target_branch,
target_project_id: target_project_id,
remove_source_branch: true
}.to_json,
headers: {
'Private-Token' => @token,
'Content-Type' => 'application/json'
})
return if (200..299).cover?(response.code)
raise Error,
"Failed with response code: #{response.code} and body:\n#{response.body}"
remove_source_branch: true,
reviewer_ids: usernames_to_ids(change.reviewers)
})
end
def update_existing_merge_request(
change:,
existing_iid:,
title:,
description:,
labels:,
target_project_id:,
update_title:,
update_description:,
update_labels:
update_labels:,
update_reviewers:
)
body = {}
body[:title] = title if update_title
body[:description] = description if update_description
body[:add_labels] = Array(labels).join(',') if update_labels
body[:title] = change.title if update_title
body[:description] = change.mr_description if update_description
body[:add_labels] = Array(change.labels).join(',') if update_labels
body[:reviewer_ids] = usernames_to_ids(change.reviewers) if update_reviewers
return if body.empty?
response = HTTParty.put("#{@base_uri}/projects/#{target_project_id}/merge_requests/#{existing_iid}",
body: body.to_json,
headers: {
'Private-Token' => @token,
'Content-Type' => 'application/json'
})
request(:put, "/projects/#{target_project_id}/merge_requests/#{existing_iid}", body: body)
end
return if (200..299).cover?(response.code)
def usernames_to_ids(usernames)
usernames.map do |username|
data = request(:get, "/users", query: { username: username })
data[0]['id']
end
end
raise Error,
"Failed with response code: #{response.code} and body:\n#{response.body}"
def request(method, path, query: {}, body: {})
response = HTTParty.public_send(method, "#{@base_uri}#{path}", query: query, body: body.to_json, headers: { # rubocop:disable GitlabSecurity/PublicSend
'Private-Token' => @token,
'Content-Type' => 'application/json'
})
unless (200..299).cover?(response.code)
raise Error,
"Failed with response code: #{response.code} and body:\n#{response.body}"
end
JSON.parse(response.body)
end
end
end

23
gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb
View File

@ -4,13 +4,12 @@ require 'active_support/core_ext/string'
require 'gitlab/housekeeper/keep'
require 'gitlab/housekeeper/gitlab_client'
require 'gitlab/housekeeper/git'
require 'gitlab/housekeeper/change'
require 'awesome_print'
require 'digest'
module Gitlab
module Housekeeper
Change = Struct.new(:identifiers, :title, :description, :changed_files, :labels)
class Runner
def initialize(max_mrs: 1, dry_run: false, require: [], keeps: nil)
@max_mrs = max_mrs
@ -32,6 +31,11 @@ module Gitlab
@keeps.each do |keep_class|
keep = keep_class.new
keep.each_change do |change|
unless change.valid?
puts "Ignoring invalid change from: #{keep_class}"
next
end
branch_name = git.commit_in_branch(change)
add_standard_change_data(change)
@ -54,14 +58,6 @@ module Gitlab
def add_standard_change_data(change)
change.labels ||= []
change.labels << 'automation:gitlab-housekeeper-authored'
change.description += <<~MARKDOWN
This change was generated by
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
Changelog: other
MARKDOWN
end
def git
@ -112,16 +108,15 @@ module Gitlab
end
gitlab_client.create_or_update_merge_request(
change: change,
source_project_id: housekeeper_fork_project_id,
title: change.title,
description: change.description,
labels: change.labels,
source_branch: branch_name,
target_branch: 'master',
target_project_id: housekeeper_target_project_id,
update_title: !non_housekeeper_changes.include?(:title),
update_description: !non_housekeeper_changes.include?(:description),
update_labels: !non_housekeeper_changes.include?(:labels)
update_labels: !non_housekeeper_changes.include?(:labels),
update_reviewers: !non_housekeeper_changes.include?(:reviewers)
)
end

54
gems/gitlab-housekeeper/spec/gitlab/housekeeper/change_spec.rb Normal file
View File

@ -0,0 +1,54 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe ::Gitlab::Housekeeper::Change do
let(:change) { described_class.new }
before do
change.title = 'The title'
change.description = 'The description'
end
describe '#mr_description' do
it 'includes standard content' do
expect(change.mr_description).to eq(
<<~MARKDOWN
The description
This change was generated by
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
MARKDOWN
)
end
end
describe '#commit_message' do
it 'includes standard content' do
expect(change.commit_message).to eq(
<<~MARKDOWN
The title
The description
This change was generated by
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
Changelog: other
MARKDOWN
)
end
end
describe '#valid?' do
it 'is not valid if missing required attributes' do
[:identifiers, :title, :description, :changed_files].each do |attribute|
change = create_change
expect(change).to be_valid
change.public_send("#{attribute}=", nil)
expect(change).not_to be_valid
end
end
end
end

21
gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb
View File

@ -50,27 +50,21 @@ RSpec.describe ::Gitlab::Housekeeper::Git do
let(:test_file2) { 'files/test_file2.txt' }
it 'commits the given change details to the given branch name' do
title = "The commit title"
description = <<~COMMIT
change = ::Gitlab::Housekeeper::Change.new
change.title = "The commit title"
change.description = <<~COMMIT
The commit description can be
split over multiple lines!
COMMIT
identifiers = %w[GitlabHousekeeper TestBranch]
change.identifiers = %w[GitlabHousekeeper TestBranch]
Dir.mkdir('files')
File.write(test_file1, "Content in file 1!")
File.write(test_file2, "Other content in file 2!")
File.write(file_not_to_commit, 'Do not commit!')
changed_files = [test_file1, test_file2]
change = ::Gitlab::Housekeeper::Change.new(
identifiers,
title,
description,
changed_files
)
change.changed_files = [test_file1, test_file2]
branch_name = nil
git.with_branch_from_branch do
@ -91,6 +85,11 @@ RSpec.describe ::Gitlab::Housekeeper::Git do
The commit description can be
split over multiple lines!
This change was generated by
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
Changelog: other
diff --git a/files/test_file2.txt b/files/test_file2.txt
new file mode 100644

152
gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb
View File

@ -23,6 +23,24 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
}
end
let(:added_reviewer_note) do
{
id: 1698248524,
body: "requested review from @bob",
author: { "id" => 1234 },
system: true
}
end
let(:removed_reviewer_note) do
{
id: 1698248524,
body: "removed review request for @bob",
author: { "id" => 1234 },
system: true
}
end
let(:irrelevant_note1) do
{
id: 1698248523,
@ -138,7 +156,7 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
context 'when all important things change' do
let(:notes) do
[not_a_system_note, updated_title_note, updated_description_note, added_commit_note]
[not_a_system_note, updated_title_note, updated_description_note, added_commit_note, added_reviewer_note]
end
let(:resource_label_events) do
@ -146,10 +164,7 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
end
it 'returns :title, :description, :code, :labels' do
expect(non_housekeeper_changes).to include(:title)
expect(non_housekeeper_changes).to include(:description)
expect(non_housekeeper_changes).to include(:code)
expect(non_housekeeper_changes).to include(:labels)
expect(non_housekeeper_changes).to match_array([:title, :description, :code, :labels, :reviewers])
end
end
@ -159,10 +174,7 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
end
it 'returns :title' do
expect(non_housekeeper_changes).to include(:title)
expect(non_housekeeper_changes).not_to include(:description)
expect(non_housekeeper_changes).not_to include(:code)
expect(non_housekeeper_changes).not_to include(:labels)
expect(non_housekeeper_changes).to match_array([:title])
end
end
@ -172,10 +184,7 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
end
it 'returns :description' do
expect(non_housekeeper_changes).not_to include(:title)
expect(non_housekeeper_changes).to include(:description)
expect(non_housekeeper_changes).not_to include(:code)
expect(non_housekeeper_changes).not_to include(:labels)
expect(non_housekeeper_changes).to match_array([:description])
end
end
@ -189,10 +198,27 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
end
it 'returns :labels' do
expect(non_housekeeper_changes).not_to include(:title)
expect(non_housekeeper_changes).not_to include(:description)
expect(non_housekeeper_changes).not_to include(:code)
expect(non_housekeeper_changes).to include(:labels)
expect(non_housekeeper_changes).to match_array([:labels])
end
end
context 'when reviewers are added' do
let(:notes) do
[not_a_system_note, added_reviewer_note]
end
it 'returns :reviewers' do
expect(non_housekeeper_changes).to match_array([:reviewers])
end
end
context 'when reviewers are removed' do
let(:notes) do
[not_a_system_note, removed_reviewer_note]
end
it 'returns :reviewers' do
expect(non_housekeeper_changes).to match_array([:reviewers])
end
end
@ -204,24 +230,39 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
end
describe '#create_or_update_merge_request' do
let(:reviewer_id) { 999 }
let(:change) do
create_change
end
let(:params) do
{
change: change,
source_project_id: 123,
title: 'A new merge request!',
labels: %w[label-1 label-2],
description: 'This merge request is pretty good.',
source_branch: 'the-source-branch',
target_branch: 'the-target-branch',
target_project_id: 456,
update_title: true,
update_description: true,
update_labels: true
update_labels: true,
update_reviewers: true
}
end
let(:existing_mrs) { [] }
before do
# Stub the user id of the reviewers
stub_request(:get, "https://gitlab.com/api/v4/users")
.with(
query: { username: 'thegitlabreviewer' },
headers: {
'Private-Token' => 'the-api-token'
}
)
.to_return(status: 200, body: [{ id: reviewer_id }].to_json)
# Stub the check to see if the merge request already exists
stub_request(:get, "https://gitlab.com/api/v4/projects/456/merge_requests?state=opened&source_branch=the-source-branch&target_branch=the-target-branch&source_project_id=123")
.with(
@ -236,19 +277,20 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
stub = stub_request(:post, "https://gitlab.com/api/v4/projects/123/merge_requests")
.with(
body: {
title: "A new merge request!",
description: "This merge request is pretty good.",
labels: "label-1,label-2",
title: "The change title",
description: change.mr_description,
labels: "some-label-1,some-label-2",
source_branch: "the-source-branch",
target_branch: "the-target-branch",
target_project_id: 456,
remove_source_branch: true
remove_source_branch: true,
reviewer_ids: [reviewer_id]
},
headers: {
'Content-Type' => 'application/json',
'Private-Token' => 'the-api-token'
})
.to_return(status: 200, body: "")
.to_return(status: 200, body: '{}')
client.create_or_update_merge_request(**params)
@ -264,15 +306,16 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
stub = stub_request(:put, "https://gitlab.com/api/v4/projects/456/merge_requests/1234")
.with(
body: {
title: "A new merge request!",
description: "This merge request is pretty good.",
add_labels: "label-1,label-2"
title: "The change title",
description: change.mr_description,
add_labels: "some-label-1,some-label-2",
reviewer_ids: [reviewer_id]
}.to_json,
headers: {
'Content-Type' => 'application/json',
'Private-Token' => 'the-api-token'
})
.to_return(status: 200, body: "")
.to_return(status: 200, body: '{}')
client.create_or_update_merge_request(**params)
expect(stub).to have_been_requested
@ -293,14 +336,15 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
stub = stub_request(:put, "https://gitlab.com/api/v4/projects/456/merge_requests/1234")
.with(
body: {
description: "This merge request is pretty good.",
add_labels: "label-1,label-2"
description: change.mr_description,
add_labels: "some-label-1,some-label-2",
reviewer_ids: [reviewer_id]
}.to_json,
headers: {
'Content-Type' => 'application/json',
'Private-Token' => 'the-api-token'
}
).to_return(status: 200, body: "")
).to_return(status: 200, body: '{}')
client.create_or_update_merge_request(**params.merge(update_title: false))
expect(stub).to have_been_requested
@ -312,14 +356,15 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
stub = stub_request(:put, "https://gitlab.com/api/v4/projects/456/merge_requests/1234")
.with(
body: {
title: "A new merge request!",
add_labels: "label-1,label-2"
title: "The change title",
add_labels: "some-label-1,some-label-2",
reviewer_ids: [reviewer_id]
}.to_json,
headers: {
'Content-Type' => 'application/json',
'Private-Token' => 'the-api-token'
}
).to_return(status: 200, body: "")
).to_return(status: 200, body: '{}')
client.create_or_update_merge_request(**params.merge(update_description: false))
expect(stub).to have_been_requested
@ -331,24 +376,49 @@ RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
stub = stub_request(:put, "https://gitlab.com/api/v4/projects/456/merge_requests/1234")
.with(
body: {
title: "A new merge request!",
description: "This merge request is pretty good."
title: "The change title",
description: change.mr_description,
reviewer_ids: [reviewer_id]
}.to_json,
headers: {
'Content-Type' => 'application/json',
'Private-Token' => 'the-api-token'
}
).to_return(status: 200, body: "")
).to_return(status: 200, body: '{}')
client.create_or_update_merge_request(**params.merge(update_labels: false))
expect(stub).to have_been_requested
end
end
context 'when update_reviewers: false' do
it 'does not update the reviewers' do
stub = stub_request(:put, "https://gitlab.com/api/v4/projects/456/merge_requests/1234")
.with(
body: {
title: "The change title",
description: change.mr_description,
add_labels: "some-label-1,some-label-2"
}.to_json,
headers: {
'Content-Type' => 'application/json',
'Private-Token' => 'the-api-token'
}
).to_return(status: 200, body: '{}')
client.create_or_update_merge_request(**params.merge(update_reviewers: false))
expect(stub).to have_been_requested
end
end
context 'when there is nothing to update' do
it 'does not make a request' do
client.create_or_update_merge_request(**params.merge(update_description: false, update_title: false,
update_labels: false))
client.create_or_update_merge_request(**params.merge(
update_description: false,
update_title: false,
update_labels: false,
update_reviewers: false
))
end
end
end

57
gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb
View File

@ -8,32 +8,23 @@ RSpec.describe ::Gitlab::Housekeeper::Runner do
let(:fake_keep) { instance_double(Class) }
let(:change1) do
::Gitlab::Housekeeper::Change.new(
%w[the identifier for the first change],
"The title of MR1",
"The description of the MR",
['change1.txt', 'change2.txt'],
['example-label']
create_change(
identifiers: %w[the identifier for the first change],
title: "The title of MR1"
)
end
let(:change2) do
::Gitlab::Housekeeper::Change.new(
%w[the identifier for the second change],
"The title of MR2",
"The description of the MR",
['change1.txt', 'change2.txt'],
['example-label']
create_change(
identifiers: %w[the identifier for the second change],
title: "The title of MR2"
)
end
let(:change3) do
::Gitlab::Housekeeper::Change.new(
%w[the identifier for the third change],
"The title of MR3",
"The description of the MR",
['change1.txt', 'change2.txt'],
['example-label']
create_change(
identifiers: %w[the identifier for the third change],
title: "The title of MR3"
)
end
@ -98,29 +89,27 @@ RSpec.describe ::Gitlab::Housekeeper::Runner do
# Merge requests get created
expect(gitlab_client).to receive(:create_or_update_merge_request)
.with(
change: change1,
source_project_id: '123',
title: 'The title of MR1',
description: /The description of the MR/,
labels: %w[example-label automation:gitlab-housekeeper-authored],
source_branch: 'the-identifier-for-the-first-change',
target_branch: 'master',
target_project_id: '456',
update_title: true,
update_description: true,
update_labels: true
update_labels: true,
update_reviewers: true
)
expect(gitlab_client).to receive(:create_or_update_merge_request)
.with(
change: change2,
source_project_id: '123',
title: 'The title of MR2',
description: /The description of the MR/,
labels: %w[example-label automation:gitlab-housekeeper-authored],
source_branch: 'the-identifier-for-the-second-change',
target_branch: 'master',
target_project_id: '456',
update_title: true,
update_description: true,
update_labels: true
update_labels: true,
update_reviewers: true
)
described_class.new(max_mrs: 2, keeps: [fake_keep]).run
@ -135,7 +124,7 @@ RSpec.describe ::Gitlab::Housekeeper::Runner do
source_branch: 'the-identifier-for-the-first-change',
target_branch: 'master',
target_project_id: '456'
).and_return([:code, :description])
).and_return([:code, :description, :reviewers])
# Second change has updated title and description so it should push the code
expect(gitlab_client).to receive(:non_housekeeper_changes)
@ -155,29 +144,27 @@ RSpec.describe ::Gitlab::Housekeeper::Runner do
expect(gitlab_client).to receive(:create_or_update_merge_request)
.with(
change: change1,
source_project_id: '123',
title: 'The title of MR1',
description: /The description of the MR/,
labels: %w[example-label automation:gitlab-housekeeper-authored],
source_branch: 'the-identifier-for-the-first-change',
target_branch: 'master',
target_project_id: '456',
update_title: true,
update_description: false,
update_labels: true
update_labels: true,
update_reviewers: false
)
expect(gitlab_client).to receive(:create_or_update_merge_request)
.with(
change: change2,
source_project_id: '123',
title: 'The title of MR2',
description: /The description of the MR/,
labels: %w[example-label automation:gitlab-housekeeper-authored],
source_branch: 'the-identifier-for-the-second-change',
target_branch: 'master',
target_project_id: '456',
update_title: false,
update_description: false,
update_labels: true
update_labels: true,
update_reviewers: true
)
described_class.new(max_mrs: 2, keeps: [fake_keep]).run

24
gems/gitlab-housekeeper/spec/spec_helper.rb
View File

@ -6,6 +6,28 @@ require "gitlab/housekeeper/git"
require 'webmock/rspec'
require 'gitlab/rspec/all'
module HousekeeperFactory
def create_change(
identifiers: %w[the identifier],
title: 'The change title',
description: 'The change description',
changed_files: ['change1.txt', 'change2.txt'],
labels: %w[some-label-1 some-label-2],
reviewers: ['thegitlabreviewer']
)
change = ::Gitlab::Housekeeper::Change.new
change.identifiers = identifiers
change.title = title
change.description = description
change.changed_files = changed_files
change.labels = labels
change.reviewers = reviewers
change
end
end
RSpec.configure do |config|
config.include StubENV
# Enable flags like --only-failures and --next-failure
@ -17,4 +39,6 @@ RSpec.configure do |config|
config.expect_with :rspec do |c|
c.syntax = :expect
end
config.include(HousekeeperFactory)
end

37
keeps/helpers/groups.rb Normal file
View File

@ -0,0 +1,37 @@
# frozen_string_literal: true
module Keeps
module Helpers
class Groups
Error = Class.new(StandardError)
def group_for_feature_category(category)
@groups ||= {}
@groups[category] ||= fetch_groups.find do |_, group|
group['categories'].present? && group['categories'].include?(category)
end&.last
end
def pick_reviewer(group, identifiers)
random_engineer = Digest::SHA256.hexdigest(identifiers.join).to_i(16) % group['backend_engineers'].size
group['backend_engineers'][random_engineer]
end
private
def fetch_groups
@groups_json ||= begin
response = Gitlab::HTTP.get('https://about.gitlab.com/groups.json')
unless (200..299).cover?(response.code)
raise Error,
"Failed to get group information with response code: #{response.code} and body:\n#{response.body}"
end
Gitlab::Json.parse(response.body)
end
end
end
end
end

16
keeps/overdue_finalize_background_migration.rb
View File

@ -41,15 +41,16 @@ module Keeps
next unless migration_record
# Finalize the migration
title = "Finalize migration #{job_name}"
change = ::Gitlab::Housekeeper::Change.new
change.title = "Finalize migration #{job_name}"
identifiers = [self.class.name.demodulize, job_name]
change.identifiers = [self.class.name.demodulize, job_name]
last_migration_file = last_migration_for_job(job_name)
next unless last_migration_file
# rubocop:disable Gitlab/DocUrl -- Not running inside rails application
description = <<~MARKDOWN
change.description = <<~MARKDOWN
This migration was finished at `#{migration_record.finished_at || migration_record.updated_at}`, you can confirm
the status using our
[batched background migration chatops commands](https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#monitor-the-progress-and-status-of-a-batched-background-migration).
@ -79,7 +80,7 @@ module Keeps
.source_root('generator_templates/post_deployment_migration/post_deployment_migration/')
generator = ::PostDeploymentMigration::PostDeploymentMigrationGenerator.new([migration_name], force: true)
migration_file = generator.invoke_all.first
changed_files = [migration_file]
change.changed_files = [migration_file]
add_ensure_call_to_migration(migration_file, queue_method_node, job_name)
::Gitlab::Housekeeper::Shell.execute('rubocop', '-a', migration_file)
@ -90,11 +91,10 @@ module Keeps
add_finalized_by_to_yaml(migration_yaml_file, generator.migration_number)
changed_files << digest_file
changed_files << migration_yaml_file
change.changed_files << digest_file
change.changed_files << migration_yaml_file
to_create = ::Gitlab::Housekeeper::Change.new(identifiers, title, description, changed_files)
yield(to_create)
yield(change)
end
end

4
lib/gitlab/database/partitioning_migration_helpers/uniqueness_helpers.rb
View File

@ -20,7 +20,7 @@ module Gitlab
IF NEW."id" IS NOT NULL THEN
RAISE WARNING 'Manually assigning ids is not allowed, the value will be ignored';
END IF;
NEW."id" := nextval('#{existing_sequence(table_name)}'::regclass);
NEW."id" := nextval(\'#{existing_sequence(table_name)}\'::regclass);
RETURN NEW;
SQL
end
@ -31,7 +31,7 @@ module Gitlab
private
def existing_sequence(table_name)
Gitlab::Database::PostgresSequence.by_table_name(table_name).first
Gitlab::Database::PostgresSequence.by_table_name(table_name).first.seq_name
end
end
end

8
lib/gitlab/gitaly_client.rb
View File

@ -170,10 +170,10 @@ module Gitlab
{ 'service': 'grpc.health.v1.Health', 'method': 'Check' }
],
'retryPolicy': {
'maxAttempts': 3, # Initial request, plus up to two retries.
'initialBackoff': '0.25s',
'maxBackoff': '1s',
'backoffMultiplier': 2, # Minimum retry duration is 750ms.
'maxAttempts': 4, # Initial request, plus up to three retries.
'initialBackoff': '0.4s',
'maxBackoff': '1.4s',
'backoffMultiplier': 2, # Maximum retry duration is 2400ms.
'retryableStatusCodes': ['UNAVAILABLE']
}
}

9
locale/gitlab.pot
View File

@ -51798,9 +51798,6 @@ msgstr ""
msgid "Total"
msgstr ""
msgid "Total Score"
msgstr ""
msgid "Total cores (CPUs)"
msgstr ""
@ -57906,17 +57903,11 @@ msgstr ""
msgid "ciReport|Browser performance test metrics results are being parsed"
msgstr ""
msgid "ciReport|Browser performance test metrics: "
msgstr ""
msgid "ciReport|Browser performance test metrics: %{strong_start}%{changesFound}%{strong_end} change"
msgid_plural "ciReport|Browser performance test metrics: %{strong_start}%{changesFound}%{strong_end} changes"
msgstr[0] ""
msgstr[1] ""
msgid "ciReport|Browser performance test metrics: No changes"
msgstr ""
msgid "ciReport|Checks"
msgstr ""

7
rubocop/cop/gitlab/mark_used_feature_flags.rb
View File

@ -12,6 +12,7 @@ module RuboCop
class MarkUsedFeatureFlags < RuboCop::Cop::Base
include RuboCop::CodeReuseHelpers
FEATURE_CALLERS = %w[Feature YamlProcessor::FeatureFlags].freeze
FEATURE_METHODS = %i[enabled? disabled?].freeze
EXPERIMENT_METHODS = %i[
experiment
@ -146,7 +147,11 @@ module RuboCop
end
def caller_is_feature?(node)
%w[Feature YamlProcessor::FeatureFlags].include?(class_caller(node))
FEATURE_CALLERS.detect do |caller|
class_caller(node) == caller ||
# Support detecting fully-defined callers based on nested detectable callers
(caller.include?('::') && class_caller(node).end_with?(caller))
end
end
def caller_is_feature_gitaly?(node)

4
spec/fixtures/duo_pro/bulk_user_assignment.csv vendored Normal file
View File

@ -0,0 +1,4 @@
username
user1
user2
user3
1 username
2 user1
3 user2
4 user3

60
spec/keeps/helpers/groups_spec.rb Normal file
View File

@ -0,0 +1,60 @@
# frozen_string_literal: true
require 'spec_helper'
require './keeps/helpers/groups'
RSpec.describe Keeps::Helpers::Groups, feature_category: :tooling do
let(:groups) do
{
'tenant_scale' => {
'name' => 'Tenant Scale',
'section' => 'core_platform',
'stage' => 'data_stores',
'categories' => %w[cell groups_and_projects user_profile organization],
'label' => 'group::tenant scale',
'extra_labels' => [],
'slack_channel' => 'g_tenant_scale',
'backend_engineers' => %w[be1 be2 be3 be4 be5],
'triage_ops_config' => nil
}
}
end
before do
stub_request(:get, "https://about.gitlab.com/groups.json").to_return(status: 200, body: groups.to_json)
end
describe '#group_for_feature_category' do
let(:category) { 'organization' }
subject(:group) { described_class.new.group_for_feature_category(category) }
it { is_expected.to eq(groups['tenant_scale']) }
context 'when the category does not exist' do
let(:category) { 'missing-category' }
it { is_expected.to eq(nil) }
end
context 'when the request to fetch groups fails' do
before do
stub_request(:get, "https://about.gitlab.com/groups.json").to_return(status: 404, body: '')
end
it 'raises an error' do
expect { group }.to raise_error(described_class::Error)
end
end
end
describe '#pick_reviewer' do
let(:group) { groups['tenant_scale'] }
let(:identifiers) { %w[example identifier] }
let(:expected_index) { Digest::SHA256.hexdigest(identifiers.join).to_i(16) % group['backend_engineers'].size }
subject { described_class.new.pick_reviewer(group, identifiers) }
it { is_expected.to eq(group['backend_engineers'][expected_index]) }
end
end

1
spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb
View File

@ -45,6 +45,7 @@ RSpec.describe RuboCop::Cop::Gitlab::MarkUsedFeatureFlags do
Feature.disabled?
push_frontend_feature_flag
YamlProcessor::FeatureFlags.enabled?
::Gitlab::Ci::YamlProcessor::FeatureFlags.enabled?
].each do |feature_flag_method|
context "#{feature_flag_method} method" do
context 'a string feature flag' do

14
workhorse/internal/lsif_transformer/parser/ranges_test.go
View File

@ -8,8 +8,7 @@ import (
)
func TestRangesRead(t *testing.T) {
r, cleanup := setup(t)
defer cleanup()
r := setup(t)
firstRange := Range{Line: 1, Character: 2, ResultSetId: 4}
rg, err := r.getRange(1)
@ -28,8 +27,7 @@ func TestRangesRead(t *testing.T) {
}
func TestSerialize(t *testing.T) {
r, cleanup := setup(t)
defer cleanup()
r := setup(t)
docs := map[Id]string{6: "def-path", 7: "ref-path"}
@ -41,7 +39,7 @@ func TestSerialize(t *testing.T) {
require.Equal(t, want, buf.String())
}
func setup(t *testing.T) (*Ranges, func()) {
func setup(t *testing.T) *Ranges {
r, err := NewRanges()
require.NoError(t, err)
@ -61,9 +59,9 @@ func setup(t *testing.T) (*Ranges, func()) {
require.NoError(t, r.Read("item", []byte(`{"id":"12","label":"item","outV":5,"inVs":["2"],"document":"7"}`)))
require.NoError(t, r.Read("item", []byte(`{"id":"13","label":"item","outV":5,"inVs":["3"],"document":"7"}`)))
cleanup := func() {
t.Cleanup(func() {
require.NoError(t, r.Close())
}
})
return r, cleanup
return r
}