root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-05-17 00:21:27 +00:00
parent 1e526a1ad1
commit 6cd36703ff
28 changed files with 569 additions and 426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab/ci/global.gitlab-ci.yml
View File

@ -66,7 +66,9 @@
- ${TMP_TEST_FOLDER}/gitaly/_build/bin/
- ${TMP_TEST_FOLDER}/gitaly/_build/deps/git/install/
- ${TMP_TEST_FOLDER}/gitaly/config.toml
- ${TMP_TEST_FOLDER}/gitaly/config.toml.transactions
- ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml
- ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml.transactions
- ${TMP_TEST_FOLDER}/gitaly/internal/
- ${TMP_TEST_FOLDER}/gitaly/run/
- ${TMP_TEST_FOLDER}/gitaly/run2/

30
.gitlab/ci/rails.gitlab-ci.yml
View File

@ -31,7 +31,9 @@ setup-test-env:
paths:
- ${TMP_TEST_FOLDER}/gitaly/_build/bin/
- ${TMP_TEST_FOLDER}/gitaly/config.toml
- ${TMP_TEST_FOLDER}/gitaly/config.toml.transactions
- ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml
- ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml.transactions
- ${TMP_TEST_FOLDER}/gitaly/internal/
- ${TMP_TEST_FOLDER}/gitaly/Makefile
- ${TMP_TEST_FOLDER}/gitaly/praefect.config.toml
@ -45,6 +47,7 @@ setup-test-env:
- ${TMP_TEST_FOLDER}/repositories/
- ${TMP_TEST_FOLDER}/second_storage/
- ${TMP_TEST_GITLAB_WORKHORSE_PATH}/
- log/*.log
when: always
setup-test-env-fips:
@ -303,6 +306,33 @@ rspec system pg14 praefect:
- .rspec-system-parallel
- .rails:rules:praefect-with-db
# Test jobs that run with Gitaly's transactions enabled. These will be removed once
# transactions are always in use in Gitaly.
rspec migration gitaly_transactions:
extends:
- rspec migration pg14
- .gitaly-with-transactions
rspec background_migration gitaly_transactions:
extends:
- rspec background_migration pg14
- .gitaly-with-transactions
rspec unit gitaly_transactions:
extends:
- rspec unit pg14
- .gitaly-with-transactions
rspec integration gitaly_transactions:
extends:
- rspec integration pg14
- .gitaly-with-transactions
rspec system gitaly_transactions:
extends:
- rspec system pg14
- .gitaly-with-transactions
# Dedicated job to test DB library code against PG13.
# Note that these are already tested against PG13 in the `rspec unit pg13` / `rspec-ee unit pg13` jobs.
rspec db-library-code pg13:

4
.gitlab/ci/rails/shared.gitlab-ci.yml
View File

@ -56,6 +56,10 @@ include:
variables:
GITALY_PRAEFECT_WITH_DB: '1'
.gitaly-with-transactions:
variables:
GITALY_TRANSACTIONS_ENABLED: "true"
.rspec-base-needs:
needs:
- job: "clone-gitlab-repo"

5
db/docs/fork_networks.yml
View File

@ -4,7 +4,10 @@ classes:
- ForkNetwork
feature_categories:
- source_code_management
description: When a project is first forked, a row is created in this table. Also referenced by the fork_network_members table. This is used to know which projects can send merge reqeusts to each other.
description: When a project is first forked, a row is created in this table. Also
referenced by the fork_network_members table. This is used to know which projects
can send merge reqeusts to each other.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3098
milestone: '10.1'
gitlab_schema: gitlab_main_cell
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/lfs_object_states.yml
View File

@ -8,3 +8,4 @@ description: Geo verification states for LFS objects
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63981
milestone: '14.6'
gitlab_schema: gitlab_main
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/lfs_objects.yml
View File

@ -8,3 +8,4 @@ description: LFS files
introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/1727
milestone: '8.2'
gitlab_schema: gitlab_main
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/protected_branch_merge_access_levels.yml
View File

@ -8,3 +8,4 @@ description: Stores merge access settings for protected branches
introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5081
milestone: '8.11'
gitlab_schema: gitlab_main_cell
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/protected_branch_push_access_levels.yml
View File

@ -8,3 +8,4 @@ description: Stores push access settings for protected branches
introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5081
milestone: '8.11'
gitlab_schema: gitlab_main_cell
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/protected_branch_unprotect_access_levels.yml
View File

@ -10,3 +10,4 @@ milestone: '10.7'
gitlab_schema: gitlab_main_cell
allow_cross_foreign_keys:
- gitlab_main_clusterwide
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/push_rules.yml
View File

@ -8,3 +8,4 @@ description: TODO
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/1b98b5ab97ce3e9997df542059cbf3c6ce0bf0e1
milestone: '8.10'
gitlab_schema: gitlab_main_cell
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/required_code_owners_sections.yml
View File

@ -8,3 +8,4 @@ description: Keeps required code owners sections
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43573
milestone: '13.5'
gitlab_schema: gitlab_main_cell
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/snippet_repositories.yml
View File

@ -8,3 +8,4 @@ description: Stores repository information used to version control snippets.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23796
milestone: '12.8'
gitlab_schema: gitlab_main
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

4
db/docs/snippet_statistics.yml
View File

@ -4,7 +4,9 @@ classes:
- SnippetStatistics
feature_categories:
- source_code_management
description: Stores the repository size, commit count, and file count regarding the snippet repository.
description: Stores the repository size, commit count, and file count regarding the
snippet repository.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35026
milestone: '13.2'
gitlab_schema: gitlab_main
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

4
db/docs/snippet_user_mentions.yml
View File

@ -4,7 +4,9 @@ classes:
- SnippetUserMention
feature_categories:
- source_code_management
description: For storing mentioned users, groups, projects referenced in a snippet description.
description: For storing mentioned users, groups, projects referenced in a snippet
description.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19009
milestone: '12.6'
gitlab_schema: gitlab_main
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

6
db/docs/snippets.yml
View File

@ -6,7 +6,9 @@ classes:
- Snippet
feature_categories:
- source_code_management
description: GitLab snippets allow you to store and share bits of code and text with other users.
description: GitLab snippets allow you to store and share bits of code and text with
other users.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/9265de3d25715aeafd38a4ef41596dca058dc18c
milestone: "1.0.1"
milestone: 1.0.1
gitlab_schema: gitlab_main
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/460829

1
db/docs/x509_certificates.yml
View File

@ -8,3 +8,4 @@ description: Stores data about X.509 certificate
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17773
milestone: '12.8'
gitlab_schema: gitlab_main
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

1
db/docs/x509_issuers.yml
View File

@ -8,3 +8,4 @@ description: Stores data about issuer of X.509 certificate
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17773
milestone: '12.8'
gitlab_schema: gitlab_main
sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

404
doc/administration/audit_event_streaming/index.md
View File

@ -4,24 +4,26 @@ group: Compliance
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
# Audit event streaming
# Audit event streaming for instances
DETAILS:
**Tier:** Ultimate
**Offering:** GitLab.com, Self-managed, GitLab Dedicated
**Offering:** Self-managed, GitLab Dedicated
> - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2.
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default.
> - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2.
> - Instance streaming destinations [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) in GitLab 16.4. [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/417708) removed.
> - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](../feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default.
> - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed.
> - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3.
> - HTTP destination **Name*** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3.
> - HTTP destination **Name** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3.
> - Functionality for the **Active** checkbox [added](https://gitlab.com/gitlab-org/gitlab/-/issues/415268) in GitLab 16.5.
Users can set a streaming destination for a top-level group or instance to receive all audit events about the group,
subgroups, and projects, as structured JSON.
Audit event streaming for instances, administrators can:
Top-level group owners and instance administrators can manage their audit logs in third-party systems. Any service that
can receive structured JSON data can be used as the streaming destination.
- Set a streaming destination for an entire instance to receive all audit events about that instance as structured JSON.
- Manage their audit logs in third-party systems. Any service that can receive structured JSON data can be used as the
streaming destination.
Each streaming destination can have up to 20 custom HTTP headers included with each streamed event.
@ -34,349 +36,9 @@ WARNING:
Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust
the streaming destination.
## Top-level group streaming destinations
Manage streaming destinations for top-level groups.
### HTTP destinations
Prerequisites:
- For better security, you should use an SSL certificate on the destination URL.
Manage HTTP streaming destinations for top-level groups.
#### Add a new HTTP destination
Add a new HTTP streaming destination to a top-level group.
Prerequisites:
- Owner role for a top-level group.
To add streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select **Add streaming destination** and select **HTTP endpoint** to show the section for adding destinations.
1. In the **Name** and **Destination URL** fields, add a destination name and URL.
1. Optional. Locate the **Custom HTTP headers** table.
1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event.
1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to
20 headers per streaming destination.
1. After all headers have been filled out, select **Add** to add the new streaming destination.
#### List HTTP destinations
Prerequisites:
- Owner role for a group.
To list the streaming destinations for a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand it and see all the custom HTTP headers.
#### Update an HTTP destination
Prerequisites:
- Owner role for a group.
To update a streaming destination's name:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. In the **Name** fields, add a destination name to update.
1. Select **Save** to update the streaming destination.
To update a streaming destination's custom HTTP headers:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Custom HTTP headers** table.
1. Locate the header that you wish to update.
1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event.
1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to
20 headers per streaming destination.
1. Select **Save** to update the streaming destination.
#### Delete an HTTP destination
Delete streaming destinations for a top-level group. When the last destination is successfully deleted, streaming is
disabled for the top-level group.
Prerequisites:
- Owner role for a group.
To delete a streaming destination:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Select **Delete destination**.
1. Confirm by selecting **Delete destination** in the dialog.
To delete only the custom HTTP headers for a streaming destination:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Custom HTTP headers** table.
1. Locate the header that you wish to remove.
1. To the right of the header, select **Delete** (**{remove}**).
1. Select **Save** to update the streaming destination.
#### Verify event authenticity
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2.
Each streaming destination has a unique verification token (`verificationToken`) that can be used to verify the authenticity of the event. This
token is either specified by the Owner or generated automatically when the event destination is created and cannot be changed.
Each streamed event contains the verification token in the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against
the destination's value when listing streaming destinations.
Prerequisites:
- Owner role for a group.
To list streaming destinations and see the verification tokens:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Verification token** input.
#### Update event filters
> - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/413581) in GitLab 16.1.
When this feature is enabled for a group, you can permit users to filter streamed audit events per destination.
If the feature is enabled with no filters, the destination receives all audit events.
A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label.
To update a streaming destination's event filters:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Filter by audit event type** dropdown list.
1. Select the dropdown list and select or clear the required event types.
1. Select **Save** to update the event filters.
#### Update namespace filters
> - Namespace filtering in the UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390133) in GitLab 16.7.
When this feature is enabled for a group, you can permit users to filter streamed audit events per destination.
If the feature is enabled with no filters, the destination receives all audit events.
A streaming destination that has a namespace filter set has a **filtered** (**{filter}**) label.
To update a streaming destination's namespace filters:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Filter by groups or projects** dropdown list.
1. Select the dropdown list and select or clear the required namespaces.
1. Select **Save** to update the namespace filter.
#### Override default content type header
By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you
might want to set the `content-type` header to something else. For example ,`application/json`.
To override the `content-type` header default value for a top-level group streaming destination, use either:
- The [GitLab UI](#update-an-http-destination).
- The [GraphQL API](graphql_api.md#update-streaming-destinations).
### Google Cloud Logging destinations
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124384) in GitLab 16.2.
Manage Google Cloud Logging destinations for top-level groups.
#### Prerequisites
Before setting up Google Cloud Logging streaming audit events, you must:
1. Enable [Cloud Logging API](https://console.cloud.google.com/marketplace/product/google/logging.googleapis.com) on your Google Cloud project.
1. Create a service account for Google Cloud with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication.
For more information, see [Creating and managing service accounts in the Google Cloud documentation](https://cloud.google.com/iam/docs/service-accounts-create#creating).
1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter).
1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).
#### Add a new Google Cloud Logging destination
Prerequisites:
- Owner role for a top-level group.
To add Google Cloud Logging streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select **Add streaming destination** and select **Google Cloud Logging** to show the section for adding destinations.
1. Enter a random string to use as a name for the new destination.
1. Enter the Google project ID, Google client email, and Google private key from previously-created Google Cloud service account key to add to the new destination.
1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud.
1. Select **Add** to add the new streaming destination.
#### List Google Cloud Logging destinations
Prerequisites:
- Owner role for a top-level group.
To list Google Cloud Logging streaming destinations for a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the Google Cloud Logging stream to expand and see all the fields.
#### Update a Google Cloud Logging destination
> - Button to add private key [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/419675) in GitLab 16.3.
Prerequisites:
- Owner role for a top-level group.
To update Google Cloud Logging streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the Google Cloud Logging stream to expand.
1. Enter a random string to use as a name for the destination.
1. Enter the Google project ID and Google client email from previously-created Google Cloud service account key to update the destination.
1. Enter a random string to update the log ID for the destination. You can use this later to filter log results in Google Cloud.
1. Select **Add a new private key** and enter a Google private key to update the private key.
1. Select **Save** to update the streaming destination.
#### Delete a Google Cloud Logging streaming destination
Prerequisites:
- Owner role for a top-level group.
To delete Google Cloud Logging streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the Google Cloud Logging stream to expand.
1. Select **Delete destination**.
1. Confirm by selecting **Delete destination** in the dialog.
### AWS S3 destinations
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132603) in GitLab 16.6 [with a flag](../feature_flags.md) named `allow_streaming_audit_events_to_amazon_s3`. Enabled by default.
> - [Feature flag `allow_streaming_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.7.
Manage AWS S3 destinations for top-level groups.
#### Prerequisites
Before setting up AWS S3 streaming audit events, you must:
1. Create a access key for AWS with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication.
For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey).
1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)
#### Add a new AWS S3 destination
Prerequisites:
- Owner role for a top-level group.
To add AWS S3 streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select **Add streaming destination** and select **AWS S3** to show the section for adding destinations.
1. Enter a random string to use as a name for the new destination.
1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination.
1. Select **Add** to add the new streaming destination.
#### List AWS S3 destinations
Prerequisites:
- Owner role for a top-level group.
To list AWS S3 streaming destinations for a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the AWS S3 stream to expand and see all the fields.
#### Update a AWS S3 destination
Prerequisites:
- Owner role for a top-level group.
To update AWS S3 streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the AWS S3 stream to expand.
1. Enter a random string to use as a name for the destination.
1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to update the destination.
1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key.
1. Select **Save** to update the streaming destination.
#### Delete a AWS S3 streaming destination
Prerequisites:
- Owner role for a top-level group.
To delete AWS S3 streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the AWS S3 stream to expand.
1. Select **Delete destination**.
1. Confirm by selecting **Delete destination** in the dialog.
## Instance streaming destinations
DETAILS:
**Tier:** Ultimate
**Offering:** Self-managed
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default.
> - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2.
> - Instance streaming destinations [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) in GitLab 16.4. [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/417708) removed.
Manage streaming destinations for an entire instance.
### HTTP destinations
## HTTP destinations
Prerequisites:
@ -384,7 +46,7 @@ Prerequisites:
Manage HTTP streaming destinations for an entire instance.
#### Add a new HTTP destination
### Add a new HTTP destination
Add a new HTTP streaming destination to an instance.
@ -405,7 +67,7 @@ To add a streaming destination for an instance:
20 headers per streaming destination.
1. After all headers have been filled out, select **Add** to add the new streaming destination.
#### List HTTP destinations
### List HTTP destinations
Prerequisites:
@ -418,7 +80,7 @@ To list the streaming destinations for an instance:
1. On the main area, select the **Streams** tab.
1. Select the stream to expand it and see all the custom HTTP headers.
#### Update an HTTP destination
### Update an HTTP destination
Prerequisites:
@ -446,7 +108,7 @@ To update a instance streaming destination's custom HTTP headers:
20 headers per streaming destination.
1. Select **Save** to update the streaming destination.
#### Delete an HTTP destination
### Delete an HTTP destination
Delete streaming destinations for an entire instance. When the last destination is successfully deleted, streaming is
disabled for the instance.
@ -475,7 +137,7 @@ To delete only the custom HTTP headers for a streaming destination:
1. To the right of the header, select **Delete** (**{remove}**).
1. Select **Save** to update the streaming destination.
#### Verify event authenticity
### Verify event authenticity
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default.
> - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2.
@ -498,7 +160,7 @@ To list streaming destinations for an instance and see the verification tokens:
1. On the main area, select the **Streams** tab.
1. View the verification token on the right side of each item.
#### Update event filters
### Update event filters
> - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/415013) in GitLab 16.3.
@ -517,23 +179,23 @@ To update a streaming destination's event filters:
1. Select the dropdown list and select or clear the required event types.
1. Select **Save** to update the event filters.
#### Override default content type header
### Override default content type header
By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you
might want to set the `content-type` header to something else. For example ,`application/json`.
To override the `content-type` header default value for an instance streaming destination, use either:
- The [GitLab UI](#update-an-http-destination-1).
- The [GitLab UI](#update-an-http-destination).
- The [GraphQL API](graphql_api.md#update-streaming-destinations).
### Google Cloud Logging destinations
## Google Cloud Logging destinations
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131851) in GitLab 16.5.
Manage Google Cloud Logging destinations for an entire instance.
#### Prerequisites
### Prerequisites
Before setting up Google Cloud Logging streaming audit events, you must:
@ -543,7 +205,7 @@ Before setting up Google Cloud Logging streaming audit events, you must:
1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter).
1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).
#### Add a new Google Cloud Logging destination
### Add a new Google Cloud Logging destination
Prerequisites:
@ -560,7 +222,7 @@ To add Google Cloud Logging streaming destinations to an instance:
1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud.
1. Select **Add** to add the new streaming destination.
#### List Google Cloud Logging destinations
### List Google Cloud Logging destinations
Prerequisites:
@ -573,7 +235,7 @@ To list Google Cloud Logging streaming destinations for an instance:
1. On the main area, select the **Streams** tab.
1. Select the Google Cloud Logging stream to expand and see all the fields.
#### Update a Google Cloud Logging destination
### Update a Google Cloud Logging destination
Prerequisites:
@ -591,7 +253,7 @@ To update Google Cloud Logging streaming destinations to an instance:
1. Select **Add a new private key** and enter a Google private key to update the private key.
1. Select **Save** to update the streaming destination.
#### Delete a Google Cloud Logging streaming destination
### Delete a Google Cloud Logging streaming destination
Prerequisites:
@ -606,14 +268,14 @@ To delete Google Cloud Logging streaming destinations to an instance:
1. Select **Delete destination**.
1. Confirm by selecting **Delete destination** in the dialog.
### AWS S3 destinations
## AWS S3 destinations
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138245) in GitLab 16.7 [with a flag](../feature_flags.md) named `allow_streaming_instance_audit_events_to_amazon_s3`. Disabled by default.
> - [Feature flag `allow_streaming_instance_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.8.
Manage AWS S3 destinations for entire instance.
#### Prerequisites
### Prerequisites
Before setting up AWS S3 streaming audit events, you must:
@ -621,7 +283,7 @@ Before setting up AWS S3 streaming audit events, you must:
For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey).
1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)
#### Add a new AWS S3 destination
### Add a new AWS S3 destination
Prerequisites:
@ -637,7 +299,7 @@ To add AWS S3 streaming destinations to an instance:
1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination.
1. Select **Add** to add the new streaming destination.
#### List AWS S3 destinations
### List AWS S3 destinations
Prerequisites:
@ -650,7 +312,7 @@ To list AWS S3 streaming destinations for an instance.
1. On the main area, select the **Streams** tab.
1. Select the AWS S3 stream to expand and see all the fields.
#### Update an AWS S3 destination
### Update an AWS S3 destination
Prerequisites:
@ -667,7 +329,7 @@ To update AWS S3 streaming destinations to an instance:
1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key.
1. Select **Save** to update the streaming destination.
#### Delete an AWS S3 streaming destination
### Delete an AWS S3 streaming destination
Prerequisites:
@ -681,3 +343,7 @@ To delete AWS S3 streaming destinations on an instance:
1. Select the AWS S3 stream to expand.
1. Select **Delete destination**.
1. Confirm by selecting **Delete destination** in the dialog.
## Related topics
- [Audit event streaming for top-level groups](../../user/compliance/audit_event_streaming.md)

2
doc/development/development_processes.md
View File

@ -19,10 +19,10 @@ Must-reads:
database-related changes and complex SQL queries, and having them reviewed
- [Secure coding guidelines](secure_coding_guidelines.md)
- [Pipelines for the GitLab project](pipelines/index.md)
- [Avoiding required stops](avoiding_required_stops.md)
Complementary reads:
- [Avoiding required stops](avoiding_required_stops.md)
- [Contribute to GitLab](contributing/index.md)
- [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/engineer.md#security-releases-critical-non-critical-as-a-developer)
- [Patch release process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/patch/process.md#process-for-developers)

366
doc/user/compliance/audit_event_streaming.md Normal file
View File

@ -0,0 +1,366 @@
---
stage: Govern
group: Compliance
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
# Audit event streaming for top-level groups
DETAILS:
**Tier:** Ultimate
**Offering:** GitLab.com, Self-managed, GitLab Dedicated
> - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2.
> - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](../feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default.
> - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed.
> - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3.
> - HTTP destination **Name** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3.
> - Functionality for the **Active** checkbox [added](https://gitlab.com/gitlab-org/gitlab/-/issues/415268) in GitLab 16.5.
With audit event streaming for top-level groups, group owners can:
- Set a streaming destination for a top-level group to receive all audit events about the group, subgroups, and projects
as structured JSON.
- Manage their audit logs in third-party systems. Any service that can receive structured JSON data can be used as the
streaming destination.
Each streaming destination can have up to 20 custom HTTP headers included with each streamed event.
GitLab can stream a single event more than once to the same destination. Use the `id` key in the payload to deduplicate
incoming data.
Audit events are sent using the POST request method protocol supported by HTTP.
WARNING:
Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust
the streaming destination.
## HTTP destinations
Prerequisites:
- For better security, you should use an SSL certificate on the destination URL.
Manage HTTP streaming destinations for top-level groups.
### Add a new HTTP destination
Add a new HTTP streaming destination to a top-level group.
Prerequisites:
- Owner role for a top-level group.
To add streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select **Add streaming destination** and select **HTTP endpoint** to show the section for adding destinations.
1. In the **Name** and **Destination URL** fields, add a destination name and URL.
1. Optional. Locate the **Custom HTTP headers** table.
1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event.
1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to
20 headers per streaming destination.
1. After all headers have been filled out, select **Add** to add the new streaming destination.
### List HTTP destinations
Prerequisites:
- Owner role for a group.
To list the streaming destinations for a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand it and see all the custom HTTP headers.
### Update an HTTP destination
Prerequisites:
- Owner role for a group.
To update a streaming destination's name:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. In the **Name** fields, add a destination name to update.
1. Select **Save** to update the streaming destination.
To update a streaming destination's custom HTTP headers:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Custom HTTP headers** table.
1. Locate the header that you wish to update.
1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event.
1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to
20 headers per streaming destination.
1. Select **Save** to update the streaming destination.
### Delete an HTTP destination
Delete streaming destinations for a top-level group. When the last destination is successfully deleted, streaming is
disabled for the top-level group.
Prerequisites:
- Owner role for a group.
To delete a streaming destination:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Select **Delete destination**.
1. Confirm by selecting **Delete destination** in the dialog.
To delete only the custom HTTP headers for a streaming destination:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Custom HTTP headers** table.
1. Locate the header that you wish to remove.
1. To the right of the header, select **Delete** (**{remove}**).
1. Select **Save** to update the streaming destination.
### Verify event authenticity
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2.
Each streaming destination has a unique verification token (`verificationToken`) that can be used to verify the authenticity of the event. This
token is either specified by the Owner or generated automatically when the event destination is created and cannot be changed.
Each streamed event contains the verification token in the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against
the destination's value when listing streaming destinations.
Prerequisites:
- Owner role for a group.
To list streaming destinations and see the verification tokens:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Verification token** input.
### Update event filters
> - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/413581) in GitLab 16.1.
When this feature is enabled for a group, you can permit users to filter streamed audit events per destination.
If the feature is enabled with no filters, the destination receives all audit events.
A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label.
To update a streaming destination's event filters:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Filter by audit event type** dropdown list.
1. Select the dropdown list and select or clear the required event types.
1. Select **Save** to update the event filters.
### Update namespace filters
> - Namespace filtering in the UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390133) in GitLab 16.7.
When this feature is enabled for a group, you can permit users to filter streamed audit events per destination.
If the feature is enabled with no filters, the destination receives all audit events.
A streaming destination that has a namespace filter set has a **filtered** (**{filter}**) label.
To update a streaming destination's namespace filters:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the stream to expand.
1. Locate the **Filter by groups or projects** dropdown list.
1. Select the dropdown list and select or clear the required namespaces.
1. Select **Save** to update the namespace filter.
### Override default content type header
By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you
might want to set the `content-type` header to something else. For example ,`application/json`.
To override the `content-type` header default value for a top-level group streaming destination, use either:
- The [GitLab UI](#update-an-http-destination).
- The [GraphQL API](../../administration/audit_event_streaming/graphql_api.md#update-streaming-destinations).
## Google Cloud Logging destinations
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124384) in GitLab 16.2.
Manage Google Cloud Logging destinations for top-level groups.
### Prerequisites
Before setting up Google Cloud Logging streaming audit events, you must:
1. Enable [Cloud Logging API](https://console.cloud.google.com/marketplace/product/google/logging.googleapis.com) on your Google Cloud project.
1. Create a service account for Google Cloud with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication.
For more information, see [Creating and managing service accounts in the Google Cloud documentation](https://cloud.google.com/iam/docs/service-accounts-create#creating).
1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter).
1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).
### Add a new Google Cloud Logging destination
Prerequisites:
- Owner role for a top-level group.
To add Google Cloud Logging streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select **Add streaming destination** and select **Google Cloud Logging** to show the section for adding destinations.
1. Enter a random string to use as a name for the new destination.
1. Enter the Google project ID, Google client email, and Google private key from previously-created Google Cloud service account key to add to the new destination.
1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud.
1. Select **Add** to add the new streaming destination.
### List Google Cloud Logging destinations
Prerequisites:
- Owner role for a top-level group.
To list Google Cloud Logging streaming destinations for a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the Google Cloud Logging stream to expand and see all the fields.
### Update a Google Cloud Logging destination
> - Button to add private key [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/419675) in GitLab 16.3.
Prerequisites:
- Owner role for a top-level group.
To update Google Cloud Logging streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the Google Cloud Logging stream to expand.
1. Enter a random string to use as a name for the destination.
1. Enter the Google project ID and Google client email from previously-created Google Cloud service account key to update the destination.
1. Enter a random string to update the log ID for the destination. You can use this later to filter log results in Google Cloud.
1. Select **Add a new private key** and enter a Google private key to update the private key.
1. Select **Save** to update the streaming destination.
### Delete a Google Cloud Logging streaming destination
Prerequisites:
- Owner role for a top-level group.
To delete Google Cloud Logging streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the Google Cloud Logging stream to expand.
1. Select **Delete destination**.
1. Confirm by selecting **Delete destination** in the dialog.
## AWS S3 destinations
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132603) in GitLab 16.6 [with a flag](../feature_flags.md) named `allow_streaming_audit_events_to_amazon_s3`. Enabled by default.
> - [Feature flag `allow_streaming_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.7.
Manage AWS S3 destinations for top-level groups.
### Prerequisites
Before setting up AWS S3 streaming audit events, you must:
1. Create a access key for AWS with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication.
For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey).
1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)
### Add a new AWS S3 destination
Prerequisites:
- Owner role for a top-level group.
To add AWS S3 streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select **Add streaming destination** and select **AWS S3** to show the section for adding destinations.
1. Enter a random string to use as a name for the new destination.
1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination.
1. Select **Add** to add the new streaming destination.
### List AWS S3 destinations
Prerequisites:
- Owner role for a top-level group.
To list AWS S3 streaming destinations for a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the AWS S3 stream to expand and see all the fields.
### Update a AWS S3 destination
Prerequisites:
- Owner role for a top-level group.
To update AWS S3 streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the AWS S3 stream to expand.
1. Enter a random string to use as a name for the destination.
1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to update the destination.
1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key.
1. Select **Save** to update the streaming destination.
### Delete a AWS S3 streaming destination
Prerequisites:
- Owner role for a top-level group.
To delete AWS S3 streaming destinations to a top-level group:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Secure > Audit events**.
1. On the main area, select the **Streams** tab.
1. Select the AWS S3 stream to expand.
1. Select **Delete destination**.
1. Confirm by selecting **Delete destination** in the dialog.
## Related topics
- [Audit event streaming for instances](../../administration/audit_event_streaming/index.md)

1
lib/gitlab/setup_helper.rb
View File

@ -113,6 +113,7 @@ module Gitlab
config[:bin_dir] = File.expand_path(File.join(gitaly_dir, '_build', 'bin')) # binaries by default are in `_build/bin`
config[:gitlab] = { url: Gitlab.config.gitlab.url }
config[:logging] = { dir: Rails.root.join('log').to_s }
config[:transactions] = { enabled: true } if options[:transactions_enabled]
TomlRB.dump(config)
end

4
scripts/gitaly-test-build
View File

@ -24,8 +24,8 @@ class GitalyTestBuild
ensure_gitlab_shell_secret!
# Starting gitaly further validates its configuration
gitaly_pid = start_gitaly
gitaly2_pid = start_gitaly2
gitaly_pid = start_gitaly(:gitaly)
gitaly2_pid = start_gitaly(:gitaly2)
praefect_pid = start_praefect
Process.kill('TERM', gitaly_pid)
Process.kill('TERM', gitaly2_pid)

2
spec/lib/gitlab/git_access_snippet_spec.rb
View File

@ -311,7 +311,7 @@ RSpec.describe Gitlab::GitAccessSnippet do
end
describe 'repository size restrictions' do
let_it_be(:snippet) { create(:personal_snippet, :public, :repository) }
let_it_be_with_refind(:snippet) { create(:personal_snippet, :public, :repository) }
let(:actor) { snippet.author }
let(:oldrev) { TestEnv::BRANCH_SHA["snippet/single-file"] }

10
spec/requests/api/ci/jobs_spec.rb
View File

@ -435,11 +435,11 @@ RSpec.describe API::Ci::Jobs, feature_category: :continuous_integration do
control = ActiveRecord::QueryRecorder.new(skip_cached: false) { go }
5.times do
second_pipeline = create(:ci_pipeline, project: project, sha: project.commit.id, ref: project.default_branch)
second_build = create(:ci_build, :trace_artifact, :artifacts, :test_reports, pipeline: second_pipeline)
second_build.runner = create(:ci_runner)
second_build.user = create(:user)
second_build.save!
another_pipeline = create(:ci_pipeline, project: project, sha: project.commit.id, ref: project.default_branch)
another_build = create(:ci_build, :trace_artifact, :artifacts, :test_reports, pipeline: another_pipeline)
another_build.runner = create(:ci_runner)
another_build.user = create(:user)
another_build.save!
end
expect { go }.not_to exceed_query_limit(control)

93
spec/support/helpers/gitaly_setup.rb
View File

@ -70,17 +70,21 @@ module GitalySetup
}
end
def config_path(service)
def config_name(service)
case service
when :gitaly
File.join(tmp_tests_gitaly_dir, 'config.toml')
'config.toml'
when :gitaly2
File.join(tmp_tests_gitaly_dir, 'gitaly2.config.toml')
'gitaly2.config.toml'
when :praefect
File.join(tmp_tests_gitaly_dir, 'praefect.config.toml')
'praefect.config.toml'
end
end
def config_path(service)
File.join(tmp_tests_gitaly_dir, config_name(service))
end
def service_cmd(service, toml = nil)
toml ||= config_path(service)
@ -100,12 +104,22 @@ module GitalySetup
run_command(%w[make all WITH_BUNDLED_GIT=YesPlease], env: env.merge('GIT_VERSION' => nil))
end
def start_gitaly(toml = nil)
start(:gitaly, toml)
def start_gitaly(service, toml = nil)
case service
when :gitaly
FileUtils.mkdir_p(GitalySetup.storage_path)
when :gitaly2
FileUtils.mkdir_p(GitalySetup.second_storage_path)
end
def start_gitaly2
start(:gitaly2)
if ENV['CI'] && gitaly_with_transactions?
# The configuration file with transactions is pre-generated in the CI. Here we check
# whether this job should actually run with transactions and choose the pre-generated
# configuration with transactions enabled if so.
toml = "#{config_path(service)}.transactions"
end
start(service, toml)
end
def start_praefect
@ -226,26 +240,55 @@ module GitalySetup
build_gitaly
end
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => storage_path },
force: true,
[
{
storages: { 'default' => storage_path },
options: {
runtime_dir: runtime_dir,
prometheus_listen_addr: 'localhost:9236'
prometheus_listen_addr: 'localhost:9236',
config_filename: config_name(:gitaly),
transactions_enabled: gitaly_with_transactions?
}
)
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'test_second_storage' => second_storage_path },
force: true,
},
{
storages: { 'test_second_storage' => second_storage_path },
options: {
runtime_dir: runtime_dir,
gitaly_socket: "gitaly2.socket",
config_filename: "gitaly2.config.toml"
config_filename: config_name(:gitaly2),
transactions_enabled: gitaly_with_transactions?
}
}
].each do |params|
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
params[:storages],
force: true,
options: params[:options]
)
# CI generates all of the configuration files in the setup-test-env job. When we eventually get
# to run the rspec jobs with transactions enabled, the configuration has already been created
# without transactions enabled.
#
# Similarly to the Praefect configuration, generate variant of the configuration file with
# transactions enabled in CI. Later when the rspec job runs, we decide whether to run Gitaly
# using the configuration with transactions enabled or not.
#
# These configuration files are only used in the CI.
next unless ENV['CI']
params[:options][:config_filename] = "#{params[:options][:config_filename]}.transactions"
params[:options][:transactions_enabled] = true
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
params[:storages],
force: true,
options: params[:options]
)
end
# In CI we need to pre-generate both config files.
# For local testing we'll create the correct file on-demand.
if ENV['CI'] || !praefect_with_db?
@ -301,10 +344,10 @@ module GitalySetup
pids = []
if toml
pids << start_gitaly(toml)
pids << start_gitaly(:gitaly, toml)
else
pids << start_gitaly
pids << start_gitaly2
pids << start_gitaly(:gitaly)
pids << start_gitaly(:gitaly2)
pids << start_praefect
end
@ -319,6 +362,8 @@ module GitalySetup
next if ENV['GITALY_PID_FILE']
pids.each { |pid| stop(pid) }
[storage_path, second_storage_path].each { |storage_dir| FileUtils.rm_rf(storage_dir) }
end
rescue StandardError
raise gitaly_failure_message
@ -356,4 +401,8 @@ module GitalySetup
def praefect_with_db?
Gitlab::Utils.to_boolean(ENV['GITALY_PRAEFECT_WITH_DB'], default: false)
end
def gitaly_with_transactions?
Gitlab::Utils.to_boolean(ENV['GITALY_TRANSACTIONS_ENABLED'], default: false)
end
end

24
spec/support/helpers/test_env.rb
View File

@ -173,8 +173,6 @@ module TestEnv
end
end
FileUtils.mkdir_p(GitalySetup.storage_path)
FileUtils.mkdir_p(GitalySetup.second_storage_path)
FileUtils.mkdir_p(backup_path)
FileUtils.mkdir_p(pages_path)
FileUtils.mkdir_p(artifacts_path)
@ -399,16 +397,18 @@ module TestEnv
# These are directories that should be preserved at cleanup time
def test_dirs
@test_dirs ||= %w[
frontend
gitaly
gitlab-shell
gitlab-test
gitlab-test.bundle
gitlab-test-fork
gitlab-test-fork.bundle
gitlab-workhorse
gitlab_workhorse_secret
@test_dirs ||= [
'frontend',
'gitaly',
'gitlab-shell',
'gitlab-test',
'gitlab-test.bundle',
'gitlab-test-fork',
'gitlab-test-fork.bundle',
'gitlab-workhorse',
'gitlab_workhorse_secret',
File.basename(GitalySetup.storage_path),
File.basename(GitalySetup.second_storage_path)
]
end

6
spec/support/shared_contexts/glfm/example_snapshot_fixtures.rb
View File

@ -24,4 +24,10 @@ RSpec.shared_context 'with GLFM example snapshot fixtures' do
stub_licensed_features(group_wikis: true)
sign_in(user)
end
after(:all) do
# We need to clean up the repository explicitly as we're using a static project ID. If two tests
# use this fixture, they'd attempt to create repositories with the same disk path and conflict.
::Gitlab::GitalyClient::RepositoryService.new(project.repository).remove
end
end

2
workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go
View File

@ -93,7 +93,7 @@ func ensureGitalyRepository(_ *testing.T, apiResponse *api.Response) error {
},
}); removeRepoErr != nil {
status, ok := status.FromError(removeRepoErr)
if !ok || !(status.Code() == codes.NotFound && status.Message() == "repository does not exist") {
if !ok || !(status.Code() == codes.NotFound && (status.Message() == "repository does not exist" || status.Message() == "repository not found")) {
return fmt.Errorf("remove repository: %w", removeRepoErr)
}