Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/global.gitlab-ci.yml

@@ -66,7 +66,9 @@
     - ${TMP_TEST_FOLDER}/gitaly/_build/bin/
     - ${TMP_TEST_FOLDER}/gitaly/_build/deps/git/install/
     - ${TMP_TEST_FOLDER}/gitaly/config.toml
+    - ${TMP_TEST_FOLDER}/gitaly/config.toml.transactions
     - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml
+    - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml.transactions
     - ${TMP_TEST_FOLDER}/gitaly/internal/
     - ${TMP_TEST_FOLDER}/gitaly/run/
     - ${TMP_TEST_FOLDER}/gitaly/run2/

.gitlab/ci/rails.gitlab-ci.yml

@@ -31,7 +31,9 @@ setup-test-env:
     paths:
       - ${TMP_TEST_FOLDER}/gitaly/_build/bin/
       - ${TMP_TEST_FOLDER}/gitaly/config.toml
+      - ${TMP_TEST_FOLDER}/gitaly/config.toml.transactions
       - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml
+      - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml.transactions
       - ${TMP_TEST_FOLDER}/gitaly/internal/
       - ${TMP_TEST_FOLDER}/gitaly/Makefile
       - ${TMP_TEST_FOLDER}/gitaly/praefect.config.toml
@@ -45,6 +47,7 @@ setup-test-env:
       - ${TMP_TEST_FOLDER}/repositories/
       - ${TMP_TEST_FOLDER}/second_storage/
       - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/
+      - log/*.log
     when: always
 
 setup-test-env-fips:
@@ -303,6 +306,33 @@ rspec system pg14 praefect:
     - .rspec-system-parallel
     - .rails:rules:praefect-with-db
 
+# Test jobs that run with Gitaly's transactions enabled. These will be removed once
+# transactions are always in use in Gitaly.
+rspec migration gitaly_transactions:
+  extends:
+    - rspec migration pg14
+    - .gitaly-with-transactions
+
+rspec background_migration gitaly_transactions:
+  extends:
+    - rspec background_migration pg14
+    - .gitaly-with-transactions
+
+rspec unit gitaly_transactions:
+  extends:
+    - rspec unit pg14
+    - .gitaly-with-transactions
+
+rspec integration gitaly_transactions:
+  extends:
+    - rspec integration pg14
+    - .gitaly-with-transactions
+
+rspec system gitaly_transactions:
+  extends:
+    - rspec system pg14
+    - .gitaly-with-transactions
+
 # Dedicated job to test DB library code against PG13.
 # Note that these are already tested against PG13 in the `rspec unit pg13` / `rspec-ee unit pg13` jobs.
 rspec db-library-code pg13:

.gitlab/ci/rails/shared.gitlab-ci.yml

@@ -56,6 +56,10 @@ include:
   variables:
     GITALY_PRAEFECT_WITH_DB: '1'
 
+.gitaly-with-transactions:
+  variables:
+    GITALY_TRANSACTIONS_ENABLED: "true"
+
 .rspec-base-needs:
   needs:
     - job: "clone-gitlab-repo"

db/docs/fork_networks.yml

@@ -4,7 +4,10 @@ classes:
 - ForkNetwork
 feature_categories:
 - source_code_management
-description: When a project is first forked, a row is created in this table. Also referenced by the fork_network_members table. This is used to know which projects can send merge reqeusts to each other.
+description: When a project is first forked, a row is created in this table. Also
+  referenced by the fork_network_members table. This is used to know which projects
+  can send merge reqeusts to each other.
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3098
 milestone: '10.1'
 gitlab_schema: gitlab_main_cell
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/lfs_object_states.yml

@@ -8,3 +8,4 @@ description: Geo verification states for LFS objects
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63981
 milestone: '14.6'
 gitlab_schema: gitlab_main
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/lfs_objects.yml

@@ -8,3 +8,4 @@ description: LFS files
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/1727
 milestone: '8.2'
 gitlab_schema: gitlab_main
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/protected_branch_merge_access_levels.yml

@@ -8,3 +8,4 @@ description: Stores merge access settings for protected branches
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5081
 milestone: '8.11'
 gitlab_schema: gitlab_main_cell
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/protected_branch_push_access_levels.yml

@@ -8,3 +8,4 @@ description: Stores push access settings for protected branches
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5081
 milestone: '8.11'
 gitlab_schema: gitlab_main_cell
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/protected_branch_unprotect_access_levels.yml

@@ -10,3 +10,4 @@ milestone: '10.7'
 gitlab_schema: gitlab_main_cell
 allow_cross_foreign_keys:
 - gitlab_main_clusterwide
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/push_rules.yml

@@ -8,3 +8,4 @@ description: TODO
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/1b98b5ab97ce3e9997df542059cbf3c6ce0bf0e1
 milestone: '8.10'
 gitlab_schema: gitlab_main_cell
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/required_code_owners_sections.yml

@@ -8,3 +8,4 @@ description: Keeps required code owners sections
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43573
 milestone: '13.5'
 gitlab_schema: gitlab_main_cell
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/snippet_repositories.yml

@@ -8,3 +8,4 @@ description: Stores repository information used to version control snippets.
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23796
 milestone: '12.8'
 gitlab_schema: gitlab_main
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/snippet_statistics.yml

@@ -4,7 +4,9 @@ classes:
 - SnippetStatistics
 feature_categories:
 - source_code_management
-description: Stores the repository size, commit count, and file count regarding the snippet repository.
+description: Stores the repository size, commit count, and file count regarding the
+  snippet repository.
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35026
 milestone: '13.2'
 gitlab_schema: gitlab_main
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/snippet_user_mentions.yml

@@ -4,7 +4,9 @@ classes:
 - SnippetUserMention
 feature_categories:
 - source_code_management
-description: For storing mentioned users, groups, projects referenced in a snippet description.
+description: For storing mentioned users, groups, projects referenced in a snippet
+  description.
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19009
 milestone: '12.6'
 gitlab_schema: gitlab_main
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/snippets.yml

@@ -6,7 +6,9 @@ classes:
 - Snippet
 feature_categories:
 - source_code_management
-description: GitLab snippets allow you to store and share bits of code and text with other users. 
+description: GitLab snippets allow you to store and share bits of code and text with
+  other users.
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/9265de3d25715aeafd38a4ef41596dca058dc18c
-milestone: "1.0.1"
+milestone: 1.0.1
 gitlab_schema: gitlab_main
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/460829

db/docs/x509_certificates.yml

@@ -8,3 +8,4 @@ description: Stores data about X.509 certificate
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17773
 milestone: '12.8'
 gitlab_schema: gitlab_main
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

db/docs/x509_issuers.yml

@@ -8,3 +8,4 @@ description: Stores data about issuer of X.509 certificate
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17773
 milestone: '12.8'
 gitlab_schema: gitlab_main
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136

doc/administration/audit_event_streaming/index.md

@@ -4,24 +4,26 @@ group: Compliance
 info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments
 ---
 
-# Audit event streaming
+# Audit event streaming for instances
 
 DETAILS:
 **Tier:** Ultimate
-**Offering:** GitLab.com, Self-managed, GitLab Dedicated
+**Offering:** Self-managed, GitLab Dedicated
 
-> - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default.
+> - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2.
+> - Instance streaming destinations [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) in GitLab 16.4. [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/417708) removed.
 > - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](../feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default.
 > - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed.
 > - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3.
-> - HTTP destination **Name*** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3.
+> - HTTP destination **Name** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3.
 > - Functionality for the **Active** checkbox [added](https://gitlab.com/gitlab-org/gitlab/-/issues/415268) in GitLab 16.5.
 
-Users can set a streaming destination for a top-level group or instance to receive all audit events about the group,
+Audit event streaming for instances, administrators can:
-subgroups, and projects, as structured JSON.
 
-Top-level group owners and instance administrators can manage their audit logs in third-party systems. Any service that
+- Set a streaming destination for an entire instance to receive all audit events about that instance as structured JSON.
-can receive structured JSON data can be used as the streaming destination.
+- Manage their audit logs in third-party systems. Any service that can receive structured JSON data can be used as the
+  streaming destination.
 
 Each streaming destination can have up to 20 custom HTTP headers included with each streamed event.
 
@@ -34,349 +36,9 @@ WARNING:
 Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust
 the streaming destination.
 
-## Top-level group streaming destinations
-
-Manage streaming destinations for top-level groups.
-
-### HTTP destinations
-
-Prerequisites:
-
-- For better security, you should use an SSL certificate on the destination URL.
-
-Manage HTTP streaming destinations for top-level groups.
-
-#### Add a new HTTP destination
-
-Add a new HTTP streaming destination to a top-level group.
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To add streaming destinations to a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select **Add streaming destination** and select **HTTP endpoint** to show the section for adding destinations.
-1. In the **Name** and **Destination URL** fields, add a destination name and URL.
-1. Optional. Locate the **Custom HTTP headers** table.
-1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event.
-1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to
-   20 headers per streaming destination.
-1. After all headers have been filled out, select **Add** to add the new streaming destination.
-
-#### List HTTP destinations
-
-Prerequisites:
-
-- Owner role for a group.
-
-To list the streaming destinations for a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the stream to expand it and see all the custom HTTP headers.
-
-#### Update an HTTP destination
-
-Prerequisites:
-
-- Owner role for a group.
-
-To update a streaming destination's name:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the stream to expand.
-1. In the **Name** fields, add a destination name to update.
-1. Select **Save** to update the streaming destination.
-
-To update a streaming destination's custom HTTP headers:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the stream to expand.
-1. Locate the **Custom HTTP headers** table.
-1. Locate the header that you wish to update.
-1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event.
-1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to
-   20 headers per streaming destination.
-1. Select **Save** to update the streaming destination.
-
-#### Delete an HTTP destination
-
-Delete streaming destinations for a top-level group. When the last destination is successfully deleted, streaming is
-disabled for the top-level group.
-
-Prerequisites:
-
-- Owner role for a group.
-
-To delete a streaming destination:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the stream to expand.
-1. Select **Delete destination**.
-1. Confirm by selecting **Delete destination** in the dialog.
-
-To delete only the custom HTTP headers for a streaming destination:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the stream to expand.
-1. Locate the **Custom HTTP headers** table.
-1. Locate the header that you wish to remove.
-1. To the right of the header, select **Delete** (**{remove}**).
-1. Select **Save** to update the streaming destination.
-
-#### Verify event authenticity
-
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2.
-
-Each streaming destination has a unique verification token (`verificationToken`) that can be used to verify the authenticity of the event. This
-token is either specified by the Owner or generated automatically when the event destination is created and cannot be changed.
-
-Each streamed event contains the verification token in the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against
-the destination's value when listing streaming destinations.
-
-Prerequisites:
-
-- Owner role for a group.
-
-To list streaming destinations and see the verification tokens:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the stream to expand.
-1. Locate the **Verification token** input.
-
-#### Update event filters
-
-> - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/413581) in GitLab 16.1.
-
-When this feature is enabled for a group, you can permit users to filter streamed audit events per destination.
-If the feature is enabled with no filters, the destination receives all audit events.
-
-A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label.
-
-To update a streaming destination's event filters:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the stream to expand.
-1. Locate the **Filter by audit event type** dropdown list.
-1. Select the dropdown list and select or clear the required event types.
-1. Select **Save** to update the event filters.
-
-#### Update namespace filters
-
-> - Namespace filtering in the UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390133) in GitLab 16.7.
-
-When this feature is enabled for a group, you can permit users to filter streamed audit events per destination.
-If the feature is enabled with no filters, the destination receives all audit events.
-
-A streaming destination that has a namespace filter set has a **filtered** (**{filter}**) label.
-
-To update a streaming destination's namespace filters:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the stream to expand.
-1. Locate the **Filter by groups or projects** dropdown list.
-1. Select the dropdown list and select or clear the required namespaces.
-1. Select **Save** to update the namespace filter.
-
-#### Override default content type header
-
-By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you
-might want to set the `content-type` header to something else. For example ,`application/json`.
-
-To override the `content-type` header default value for a top-level group streaming destination, use either:
-
-- The [GitLab UI](#update-an-http-destination).
-- The [GraphQL API](graphql_api.md#update-streaming-destinations).
-
-### Google Cloud Logging destinations
-
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124384) in GitLab 16.2.
-
-Manage Google Cloud Logging destinations for top-level groups.
-
-#### Prerequisites
-
-Before setting up Google Cloud Logging streaming audit events, you must:
-
-1. Enable [Cloud Logging API](https://console.cloud.google.com/marketplace/product/google/logging.googleapis.com) on your Google Cloud project.
-1. Create a service account for Google Cloud with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication.
-   For more information, see [Creating and managing service accounts in the Google Cloud documentation](https://cloud.google.com/iam/docs/service-accounts-create#creating).
-1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter).
-1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).
-
-#### Add a new Google Cloud Logging destination
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To add Google Cloud Logging streaming destinations to a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select **Add streaming destination** and select **Google Cloud Logging** to show the section for adding destinations.
-1. Enter a random string to use as a name for the new destination.
-1. Enter the Google project ID, Google client email, and Google private key from previously-created Google Cloud service account key to add to the new destination.
-1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud.
-1. Select **Add** to add the new streaming destination.
-
-#### List Google Cloud Logging destinations
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To list Google Cloud Logging streaming destinations for a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the Google Cloud Logging stream to expand and see all the fields.
-
-#### Update a Google Cloud Logging destination
-
-> - Button to add private key [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/419675) in GitLab 16.3.
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To update Google Cloud Logging streaming destinations to a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the Google Cloud Logging stream to expand.
-1. Enter a random string to use as a name for the destination.
-1. Enter the Google project ID and Google client email from previously-created Google Cloud service account key to update the destination.
-1. Enter a random string to update the log ID for the destination. You can use this later to filter log results in Google Cloud.
-1. Select **Add a new private key** and enter a Google private key to update the private key.
-1. Select **Save** to update the streaming destination.
-
-#### Delete a Google Cloud Logging streaming destination
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To delete Google Cloud Logging streaming destinations to a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the Google Cloud Logging stream to expand.
-1. Select **Delete destination**.
-1. Confirm by selecting **Delete destination** in the dialog.
-
-### AWS S3 destinations
-
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132603) in GitLab 16.6 [with a flag](../feature_flags.md) named `allow_streaming_audit_events_to_amazon_s3`. Enabled by default.
-> - [Feature flag `allow_streaming_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.7.
-
-Manage AWS S3 destinations for top-level groups.
-
-#### Prerequisites
-
-Before setting up AWS S3 streaming audit events, you must:
-
-1. Create a access key for AWS with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication.
-   For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey).
-1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)
-
-#### Add a new AWS S3 destination
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To add AWS S3 streaming destinations to a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select **Add streaming destination** and select **AWS S3** to show the section for adding destinations.
-1. Enter a random string to use as a name for the new destination.
-1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination.
-1. Select **Add** to add the new streaming destination.
-
-#### List AWS S3 destinations
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To list AWS S3 streaming destinations for a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the AWS S3 stream to expand and see all the fields.
-
-#### Update a AWS S3 destination
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To update AWS S3 streaming destinations to a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the AWS S3 stream to expand.
-1. Enter a random string to use as a name for the destination.
-1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to update the destination.
-1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key.
-1. Select **Save** to update the streaming destination.
-
-#### Delete a AWS S3 streaming destination
-
-Prerequisites:
-
-- Owner role for a top-level group.
-
-To delete AWS S3 streaming destinations to a top-level group:
-
-1. On the left sidebar, select **Search or go to** and find your group.
-1. Select **Secure > Audit events**.
-1. On the main area, select the **Streams** tab.
-1. Select the AWS S3 stream to expand.
-1. Select **Delete destination**.
-1. Confirm by selecting **Delete destination** in the dialog.
-
-## Instance streaming destinations
-
-DETAILS:
-**Tier:** Ultimate
-**Offering:** Self-managed
-
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default.
-> - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2.
-> - Instance streaming destinations [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) in GitLab 16.4. [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/417708) removed.
-
 Manage streaming destinations for an entire instance.
 
-### HTTP destinations
+## HTTP destinations
 
 Prerequisites:
 
@@ -384,7 +46,7 @@ Prerequisites:
 
 Manage HTTP streaming destinations for an entire instance.
 
-#### Add a new HTTP destination
+### Add a new HTTP destination
 
 Add a new HTTP streaming destination to an instance.
 
@@ -405,7 +67,7 @@ To add a streaming destination for an instance:
    20 headers per streaming destination.
 1. After all headers have been filled out, select **Add** to add the new streaming destination.
 
-#### List HTTP destinations
+### List HTTP destinations
 
 Prerequisites:
 
@@ -418,7 +80,7 @@ To list the streaming destinations for an instance:
 1. On the main area, select the **Streams** tab.
 1. Select the stream to expand it and see all the custom HTTP headers.
 
-#### Update an HTTP destination
+### Update an HTTP destination
 
 Prerequisites:
 
@@ -446,7 +108,7 @@ To update a instance streaming destination's custom HTTP headers:
    20 headers per streaming destination.
 1. Select **Save** to update the streaming destination.
 
-#### Delete an HTTP destination
+### Delete an HTTP destination
 
 Delete streaming destinations for an entire instance. When the last destination is successfully deleted, streaming is
 disabled for the instance.
@@ -475,7 +137,7 @@ To delete only the custom HTTP headers for a streaming destination:
 1. To the right of the header, select **Delete** (**{remove}**).
 1. Select **Save** to update the streaming destination.
 
-#### Verify event authenticity
+### Verify event authenticity
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default.
 > - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2.
@@ -498,7 +160,7 @@ To list streaming destinations for an instance and see the verification tokens:
 1. On the main area, select the **Streams** tab.
 1. View the verification token on the right side of each item.
 
-#### Update event filters
+### Update event filters
 
 > - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/415013) in GitLab 16.3.
 
@@ -517,23 +179,23 @@ To update a streaming destination's event filters:
 1. Select the dropdown list and select or clear the required event types.
 1. Select **Save** to update the event filters.
 
-#### Override default content type header
+### Override default content type header
 
 By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you
 might want to set the `content-type` header to something else. For example ,`application/json`.
 
 To override the `content-type` header default value for an instance streaming destination, use either:
 
-- The [GitLab UI](#update-an-http-destination-1).
+- The [GitLab UI](#update-an-http-destination).
 - The [GraphQL API](graphql_api.md#update-streaming-destinations).
 
-### Google Cloud Logging destinations
+## Google Cloud Logging destinations
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131851) in GitLab 16.5.
 
 Manage Google Cloud Logging destinations for an entire instance.
 
-#### Prerequisites
+### Prerequisites
 
 Before setting up Google Cloud Logging streaming audit events, you must:
 
@@ -543,7 +205,7 @@ Before setting up Google Cloud Logging streaming audit events, you must:
 1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter).
 1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).
 
-#### Add a new Google Cloud Logging destination
+### Add a new Google Cloud Logging destination
 
 Prerequisites:
 
@@ -560,7 +222,7 @@ To add Google Cloud Logging streaming destinations to an instance:
 1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud.
 1. Select **Add** to add the new streaming destination.
 
-#### List Google Cloud Logging destinations
+### List Google Cloud Logging destinations
 
 Prerequisites:
 
@@ -573,7 +235,7 @@ To list Google Cloud Logging streaming destinations for an instance:
 1. On the main area, select the **Streams** tab.
 1. Select the Google Cloud Logging stream to expand and see all the fields.
 
-#### Update a Google Cloud Logging destination
+### Update a Google Cloud Logging destination
 
 Prerequisites:
 
@@ -591,7 +253,7 @@ To update Google Cloud Logging streaming destinations to an instance:
 1. Select **Add a new private key** and enter a Google private key to update the private key.
 1. Select **Save** to update the streaming destination.
 
-#### Delete a Google Cloud Logging streaming destination
+### Delete a Google Cloud Logging streaming destination
 
 Prerequisites:
 
@@ -606,14 +268,14 @@ To delete Google Cloud Logging streaming destinations to an instance:
 1. Select **Delete destination**.
 1. Confirm by selecting **Delete destination** in the dialog.
 
-### AWS S3 destinations
+## AWS S3 destinations
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138245) in GitLab 16.7 [with a flag](../feature_flags.md) named `allow_streaming_instance_audit_events_to_amazon_s3`. Disabled by default.
 > - [Feature flag `allow_streaming_instance_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.8.
 
 Manage AWS S3 destinations for entire instance.
 
-#### Prerequisites
+### Prerequisites
 
 Before setting up AWS S3 streaming audit events, you must:
 
@@ -621,7 +283,7 @@ Before setting up AWS S3 streaming audit events, you must:
    For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey).
 1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)
 
-#### Add a new AWS S3 destination
+### Add a new AWS S3 destination
 
 Prerequisites:
 
@@ -637,7 +299,7 @@ To add AWS S3 streaming destinations to an instance:
 1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination.
 1. Select **Add** to add the new streaming destination.
 
-#### List AWS S3 destinations
+### List AWS S3 destinations
 
 Prerequisites:
 
@@ -650,7 +312,7 @@ To list AWS S3 streaming destinations for an instance.
 1. On the main area, select the **Streams** tab.
 1. Select the AWS S3 stream to expand and see all the fields.
 
-#### Update an AWS S3 destination
+### Update an AWS S3 destination
 
 Prerequisites:
 
@@ -667,7 +329,7 @@ To update AWS S3 streaming destinations to an instance:
 1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key.
 1. Select **Save** to update the streaming destination.
 
-#### Delete an AWS S3 streaming destination
+### Delete an AWS S3 streaming destination
 
 Prerequisites:
 
@@ -681,3 +343,7 @@ To delete AWS S3 streaming destinations on an instance:
 1. Select the AWS S3 stream to expand.
 1. Select **Delete destination**.
 1. Confirm by selecting **Delete destination** in the dialog.
+
+## Related topics
+
+- [Audit event streaming for top-level groups](../../user/compliance/audit_event_streaming.md)

doc/development/development_processes.md

@@ -19,10 +19,10 @@ Must-reads:
   database-related changes and complex SQL queries, and having them reviewed
 - [Secure coding guidelines](secure_coding_guidelines.md)
 - [Pipelines for the GitLab project](pipelines/index.md)
+- [Avoiding required stops](avoiding_required_stops.md)
 
 Complementary reads:
 
-- [Avoiding required stops](avoiding_required_stops.md)
 - [Contribute to GitLab](contributing/index.md)
 - [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/engineer.md#security-releases-critical-non-critical-as-a-developer)
 - [Patch release process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/patch/process.md#process-for-developers)

doc/user/compliance/audit_event_streaming.md

@@ -0,0 +1,366 @@
+---
+stage: Govern
+group: Compliance
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments
+---
+
+# Audit event streaming for top-level groups
+
+DETAILS:
+**Tier:** Ultimate
+**Offering:** GitLab.com, Self-managed, GitLab Dedicated
+
+> - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2.
+> - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](../feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default.
+> - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed.
+> - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3.
+> - HTTP destination **Name** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3.
+> - Functionality for the **Active** checkbox [added](https://gitlab.com/gitlab-org/gitlab/-/issues/415268) in GitLab 16.5.
+
+With audit event streaming for top-level groups, group owners can:
+
+- Set a streaming destination for a top-level group to receive all audit events about the group, subgroups, and projects
+  as structured JSON.
+- Manage their audit logs in third-party systems. Any service that can receive structured JSON data can be used as the
+  streaming destination.
+
+Each streaming destination can have up to 20 custom HTTP headers included with each streamed event.
+
+GitLab can stream a single event more than once to the same destination. Use the `id` key in the payload to deduplicate
+incoming data.
+
+Audit events are sent using the POST request method protocol supported by HTTP.
+
+WARNING:
+Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust
+the streaming destination.
+
+## HTTP destinations
+
+Prerequisites:
+
+- For better security, you should use an SSL certificate on the destination URL.
+
+Manage HTTP streaming destinations for top-level groups.
+
+### Add a new HTTP destination
+
+Add a new HTTP streaming destination to a top-level group.
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To add streaming destinations to a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select **Add streaming destination** and select **HTTP endpoint** to show the section for adding destinations.
+1. In the **Name** and **Destination URL** fields, add a destination name and URL.
+1. Optional. Locate the **Custom HTTP headers** table.
+1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event.
+1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to
+   20 headers per streaming destination.
+1. After all headers have been filled out, select **Add** to add the new streaming destination.
+
+### List HTTP destinations
+
+Prerequisites:
+
+- Owner role for a group.
+
+To list the streaming destinations for a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the stream to expand it and see all the custom HTTP headers.
+
+### Update an HTTP destination
+
+Prerequisites:
+
+- Owner role for a group.
+
+To update a streaming destination's name:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the stream to expand.
+1. In the **Name** fields, add a destination name to update.
+1. Select **Save** to update the streaming destination.
+
+To update a streaming destination's custom HTTP headers:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the stream to expand.
+1. Locate the **Custom HTTP headers** table.
+1. Locate the header that you wish to update.
+1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event.
+1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to
+   20 headers per streaming destination.
+1. Select **Save** to update the streaming destination.
+
+### Delete an HTTP destination
+
+Delete streaming destinations for a top-level group. When the last destination is successfully deleted, streaming is
+disabled for the top-level group.
+
+Prerequisites:
+
+- Owner role for a group.
+
+To delete a streaming destination:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the stream to expand.
+1. Select **Delete destination**.
+1. Confirm by selecting **Delete destination** in the dialog.
+
+To delete only the custom HTTP headers for a streaming destination:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the stream to expand.
+1. Locate the **Custom HTTP headers** table.
+1. Locate the header that you wish to remove.
+1. To the right of the header, select **Delete** (**{remove}**).
+1. Select **Save** to update the streaming destination.
+
+### Verify event authenticity
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2.
+
+Each streaming destination has a unique verification token (`verificationToken`) that can be used to verify the authenticity of the event. This
+token is either specified by the Owner or generated automatically when the event destination is created and cannot be changed.
+
+Each streamed event contains the verification token in the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against
+the destination's value when listing streaming destinations.
+
+Prerequisites:
+
+- Owner role for a group.
+
+To list streaming destinations and see the verification tokens:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the stream to expand.
+1. Locate the **Verification token** input.
+
+### Update event filters
+
+> - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/413581) in GitLab 16.1.
+
+When this feature is enabled for a group, you can permit users to filter streamed audit events per destination.
+If the feature is enabled with no filters, the destination receives all audit events.
+
+A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label.
+
+To update a streaming destination's event filters:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the stream to expand.
+1. Locate the **Filter by audit event type** dropdown list.
+1. Select the dropdown list and select or clear the required event types.
+1. Select **Save** to update the event filters.
+
+### Update namespace filters
+
+> - Namespace filtering in the UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390133) in GitLab 16.7.
+
+When this feature is enabled for a group, you can permit users to filter streamed audit events per destination.
+If the feature is enabled with no filters, the destination receives all audit events.
+
+A streaming destination that has a namespace filter set has a **filtered** (**{filter}**) label.
+
+To update a streaming destination's namespace filters:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the stream to expand.
+1. Locate the **Filter by groups or projects** dropdown list.
+1. Select the dropdown list and select or clear the required namespaces.
+1. Select **Save** to update the namespace filter.
+
+### Override default content type header
+
+By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you
+might want to set the `content-type` header to something else. For example ,`application/json`.
+
+To override the `content-type` header default value for a top-level group streaming destination, use either:
+
+- The [GitLab UI](#update-an-http-destination).
+- The [GraphQL API](../../administration/audit_event_streaming/graphql_api.md#update-streaming-destinations).
+
+## Google Cloud Logging destinations
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124384) in GitLab 16.2.
+
+Manage Google Cloud Logging destinations for top-level groups.
+
+### Prerequisites
+
+Before setting up Google Cloud Logging streaming audit events, you must:
+
+1. Enable [Cloud Logging API](https://console.cloud.google.com/marketplace/product/google/logging.googleapis.com) on your Google Cloud project.
+1. Create a service account for Google Cloud with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication.
+   For more information, see [Creating and managing service accounts in the Google Cloud documentation](https://cloud.google.com/iam/docs/service-accounts-create#creating).
+1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter).
+1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).
+
+### Add a new Google Cloud Logging destination
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To add Google Cloud Logging streaming destinations to a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select **Add streaming destination** and select **Google Cloud Logging** to show the section for adding destinations.
+1. Enter a random string to use as a name for the new destination.
+1. Enter the Google project ID, Google client email, and Google private key from previously-created Google Cloud service account key to add to the new destination.
+1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud.
+1. Select **Add** to add the new streaming destination.
+
+### List Google Cloud Logging destinations
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To list Google Cloud Logging streaming destinations for a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the Google Cloud Logging stream to expand and see all the fields.
+
+### Update a Google Cloud Logging destination
+
+> - Button to add private key [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/419675) in GitLab 16.3.
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To update Google Cloud Logging streaming destinations to a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the Google Cloud Logging stream to expand.
+1. Enter a random string to use as a name for the destination.
+1. Enter the Google project ID and Google client email from previously-created Google Cloud service account key to update the destination.
+1. Enter a random string to update the log ID for the destination. You can use this later to filter log results in Google Cloud.
+1. Select **Add a new private key** and enter a Google private key to update the private key.
+1. Select **Save** to update the streaming destination.
+
+### Delete a Google Cloud Logging streaming destination
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To delete Google Cloud Logging streaming destinations to a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the Google Cloud Logging stream to expand.
+1. Select **Delete destination**.
+1. Confirm by selecting **Delete destination** in the dialog.
+
+## AWS S3 destinations
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132603) in GitLab 16.6 [with a flag](../feature_flags.md) named `allow_streaming_audit_events_to_amazon_s3`. Enabled by default.
+> - [Feature flag `allow_streaming_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.7.
+
+Manage AWS S3 destinations for top-level groups.
+
+### Prerequisites
+
+Before setting up AWS S3 streaming audit events, you must:
+
+1. Create a access key for AWS with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication.
+   For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey).
+1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)
+
+### Add a new AWS S3 destination
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To add AWS S3 streaming destinations to a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select **Add streaming destination** and select **AWS S3** to show the section for adding destinations.
+1. Enter a random string to use as a name for the new destination.
+1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination.
+1. Select **Add** to add the new streaming destination.
+
+### List AWS S3 destinations
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To list AWS S3 streaming destinations for a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the AWS S3 stream to expand and see all the fields.
+
+### Update a AWS S3 destination
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To update AWS S3 streaming destinations to a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the AWS S3 stream to expand.
+1. Enter a random string to use as a name for the destination.
+1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to update the destination.
+1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key.
+1. Select **Save** to update the streaming destination.
+
+### Delete a AWS S3 streaming destination
+
+Prerequisites:
+
+- Owner role for a top-level group.
+
+To delete AWS S3 streaming destinations to a top-level group:
+
+1. On the left sidebar, select **Search or go to** and find your group.
+1. Select **Secure > Audit events**.
+1. On the main area, select the **Streams** tab.
+1. Select the AWS S3 stream to expand.
+1. Select **Delete destination**.
+1. Confirm by selecting **Delete destination** in the dialog.
+
+## Related topics
+
+- [Audit event streaming for instances](../../administration/audit_event_streaming/index.md)

lib/gitlab/setup_helper.rb

@@ -113,6 +113,7 @@ module Gitlab
           config[:bin_dir] = File.expand_path(File.join(gitaly_dir, '_build', 'bin')) # binaries by default are in `_build/bin`
           config[:gitlab] = { url: Gitlab.config.gitlab.url }
           config[:logging] = { dir: Rails.root.join('log').to_s }
+          config[:transactions] = { enabled: true } if options[:transactions_enabled]
 
           TomlRB.dump(config)
         end

scripts/gitaly-test-build

@@ -24,8 +24,8 @@ class GitalyTestBuild
     ensure_gitlab_shell_secret!
 
     # Starting gitaly further validates its configuration
-    gitaly_pid = start_gitaly
+    gitaly_pid = start_gitaly(:gitaly)
-    gitaly2_pid = start_gitaly2
+    gitaly2_pid = start_gitaly(:gitaly2)
     praefect_pid = start_praefect
     Process.kill('TERM', gitaly_pid)
     Process.kill('TERM', gitaly2_pid)

spec/lib/gitlab/git_access_snippet_spec.rb

@@ -311,7 +311,7 @@ RSpec.describe Gitlab::GitAccessSnippet do
   end
 
   describe 'repository size restrictions' do
-    let_it_be(:snippet) { create(:personal_snippet, :public, :repository) }
+    let_it_be_with_refind(:snippet) { create(:personal_snippet, :public, :repository) }
 
     let(:actor) { snippet.author }
     let(:oldrev) { TestEnv::BRANCH_SHA["snippet/single-file"] }

spec/requests/api/ci/jobs_spec.rb

@@ -435,11 +435,11 @@ RSpec.describe API::Ci::Jobs, feature_category: :continuous_integration do
         control = ActiveRecord::QueryRecorder.new(skip_cached: false) { go }
 
         5.times do
-          second_pipeline = create(:ci_pipeline, project: project, sha: project.commit.id, ref: project.default_branch)
+          another_pipeline = create(:ci_pipeline, project: project, sha: project.commit.id, ref: project.default_branch)
-          second_build = create(:ci_build, :trace_artifact, :artifacts, :test_reports, pipeline: second_pipeline)
+          another_build = create(:ci_build, :trace_artifact, :artifacts, :test_reports, pipeline: another_pipeline)
-          second_build.runner = create(:ci_runner)
+          another_build.runner = create(:ci_runner)
-          second_build.user = create(:user)
+          another_build.user = create(:user)
-          second_build.save!
+          another_build.save!
         end
 
         expect { go }.not_to exceed_query_limit(control)

spec/support/helpers/gitaly_setup.rb

@@ -70,17 +70,21 @@ module GitalySetup
     }
   end
 
-  def config_path(service)
+  def config_name(service)
     case service
     when :gitaly
-      File.join(tmp_tests_gitaly_dir, 'config.toml')
+      'config.toml'
     when :gitaly2
-      File.join(tmp_tests_gitaly_dir, 'gitaly2.config.toml')
+      'gitaly2.config.toml'
     when :praefect
-      File.join(tmp_tests_gitaly_dir, 'praefect.config.toml')
+      'praefect.config.toml'
     end
   end
 
+  def config_path(service)
+    File.join(tmp_tests_gitaly_dir, config_name(service))
+  end
+
   def service_cmd(service, toml = nil)
     toml ||= config_path(service)
 
@@ -100,12 +104,22 @@ module GitalySetup
     run_command(%w[make all WITH_BUNDLED_GIT=YesPlease], env: env.merge('GIT_VERSION' => nil))
   end
 
-  def start_gitaly(toml = nil)
+  def start_gitaly(service, toml = nil)
-    start(:gitaly, toml)
+    case service
-  end
+    when :gitaly
+      FileUtils.mkdir_p(GitalySetup.storage_path)
+    when :gitaly2
+      FileUtils.mkdir_p(GitalySetup.second_storage_path)
+    end
 
-  def start_gitaly2
+    if ENV['CI'] && gitaly_with_transactions?
-    start(:gitaly2)
+      # The configuration file with transactions is pre-generated in the CI. Here we check
+      # whether this job should actually run with transactions and choose the pre-generated
+      # configuration with transactions enabled if so.
+      toml = "#{config_path(service)}.transactions"
+    end
+
+    start(service, toml)
   end
 
   def start_praefect
@@ -226,25 +240,54 @@ module GitalySetup
       build_gitaly
     end
 
-    Gitlab::SetupHelper::Gitaly.create_configuration(
+    [
-      gitaly_dir,
+      {
-      { 'default' => storage_path },
+        storages: { 'default' => storage_path },
-      force: true,
+        options: {
-      options: {
+          runtime_dir: runtime_dir,
-        runtime_dir: runtime_dir,
+          prometheus_listen_addr: 'localhost:9236',
-        prometheus_listen_addr: 'localhost:9236'
+          config_filename: config_name(:gitaly),
+          transactions_enabled: gitaly_with_transactions?
+        }
+      },
+      {
+        storages: { 'test_second_storage' => second_storage_path },
+        options: {
+          runtime_dir: runtime_dir,
+          gitaly_socket: "gitaly2.socket",
+          config_filename: config_name(:gitaly2),
+          transactions_enabled: gitaly_with_transactions?
+        }
       }
-    )
+    ].each do |params|
-    Gitlab::SetupHelper::Gitaly.create_configuration(
+      Gitlab::SetupHelper::Gitaly.create_configuration(
-      gitaly_dir,
+        gitaly_dir,
-      { 'test_second_storage' => second_storage_path },
+        params[:storages],
-      force: true,
+        force: true,
-      options: {
+        options: params[:options]
-        runtime_dir: runtime_dir,
+      )
-        gitaly_socket: "gitaly2.socket",
+
-        config_filename: "gitaly2.config.toml"
+      # CI generates all of the configuration files in the setup-test-env job. When we eventually get
-      }
+      # to run the rspec jobs with transactions enabled, the configuration has already been created
-    )
+      # without transactions enabled.
+      #
+      # Similarly to the Praefect configuration, generate variant of the configuration file with
+      # transactions enabled in CI. Later when the rspec job runs, we decide whether to run Gitaly
+      # using the configuration with transactions enabled or not.
+      #
+      # These configuration files are only used in the CI.
+      next unless ENV['CI']
+
+      params[:options][:config_filename] = "#{params[:options][:config_filename]}.transactions"
+      params[:options][:transactions_enabled] = true
+
+      Gitlab::SetupHelper::Gitaly.create_configuration(
+        gitaly_dir,
+        params[:storages],
+        force: true,
+        options: params[:options]
+      )
+    end
 
     # In CI we need to pre-generate both config files.
     # For local testing we'll create the correct file on-demand.
@@ -301,10 +344,10 @@ module GitalySetup
     pids = []
 
     if toml
-      pids << start_gitaly(toml)
+      pids << start_gitaly(:gitaly, toml)
     else
-      pids << start_gitaly
+      pids << start_gitaly(:gitaly)
-      pids << start_gitaly2
+      pids << start_gitaly(:gitaly2)
       pids << start_praefect
     end
 
@@ -319,6 +362,8 @@ module GitalySetup
       next if ENV['GITALY_PID_FILE']
 
       pids.each { |pid| stop(pid) }
+
+      [storage_path, second_storage_path].each { |storage_dir| FileUtils.rm_rf(storage_dir) }
     end
   rescue StandardError
     raise gitaly_failure_message
@@ -356,4 +401,8 @@ module GitalySetup
   def praefect_with_db?
     Gitlab::Utils.to_boolean(ENV['GITALY_PRAEFECT_WITH_DB'], default: false)
   end
+
+  def gitaly_with_transactions?
+    Gitlab::Utils.to_boolean(ENV['GITALY_TRANSACTIONS_ENABLED'], default: false)
+  end
 end

spec/support/helpers/test_env.rb

@@ -173,8 +173,6 @@ module TestEnv
       end
     end
 
-    FileUtils.mkdir_p(GitalySetup.storage_path)
-    FileUtils.mkdir_p(GitalySetup.second_storage_path)
     FileUtils.mkdir_p(backup_path)
     FileUtils.mkdir_p(pages_path)
     FileUtils.mkdir_p(artifacts_path)
@@ -399,16 +397,18 @@ module TestEnv
 
   # These are directories that should be preserved at cleanup time
   def test_dirs
-    @test_dirs ||= %w[
+    @test_dirs ||= [
-      frontend
+      'frontend',
-      gitaly
+      'gitaly',
-      gitlab-shell
+      'gitlab-shell',
-      gitlab-test
+      'gitlab-test',
-      gitlab-test.bundle
+      'gitlab-test.bundle',
-      gitlab-test-fork
+      'gitlab-test-fork',
-      gitlab-test-fork.bundle
+      'gitlab-test-fork.bundle',
-      gitlab-workhorse
+      'gitlab-workhorse',
-      gitlab_workhorse_secret
+      'gitlab_workhorse_secret',
+      File.basename(GitalySetup.storage_path),
+      File.basename(GitalySetup.second_storage_path)
     ]
   end
 

spec/support/shared_contexts/glfm/example_snapshot_fixtures.rb

@@ -24,4 +24,10 @@ RSpec.shared_context 'with GLFM example snapshot fixtures' do
     stub_licensed_features(group_wikis: true)
     sign_in(user)
   end
+
+  after(:all) do
+    # We need to clean up the repository explicitly as we're using a static project ID. If two tests
+    # use this fixture, they'd attempt to create repositories with the same disk path and conflict.
+    ::Gitlab::GitalyClient::RepositoryService.new(project.repository).remove
+  end
 end

workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go

@@ -93,7 +93,7 @@ func ensureGitalyRepository(_ *testing.T, apiResponse *api.Response) error {
 		},
 	}); removeRepoErr != nil {
 		status, ok := status.FromError(removeRepoErr)
-		if !ok || !(status.Code() == codes.NotFound && status.Message() == "repository does not exist") {
+		if !ok || !(status.Code() == codes.NotFound && (status.Message() == "repository does not exist" || status.Message() == "repository not found")) {
 			return fmt.Errorf("remove repository: %w", removeRepoErr)
 		}