diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index b8853890951..9cc97c8c4e6 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -66,7 +66,9 @@ - ${TMP_TEST_FOLDER}/gitaly/_build/bin/ - ${TMP_TEST_FOLDER}/gitaly/_build/deps/git/install/ - ${TMP_TEST_FOLDER}/gitaly/config.toml + - ${TMP_TEST_FOLDER}/gitaly/config.toml.transactions - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml + - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml.transactions - ${TMP_TEST_FOLDER}/gitaly/internal/ - ${TMP_TEST_FOLDER}/gitaly/run/ - ${TMP_TEST_FOLDER}/gitaly/run2/ diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 6c9ec56319b..ce41f336e2d 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -31,7 +31,9 @@ setup-test-env: paths: - ${TMP_TEST_FOLDER}/gitaly/_build/bin/ - ${TMP_TEST_FOLDER}/gitaly/config.toml + - ${TMP_TEST_FOLDER}/gitaly/config.toml.transactions - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml + - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml.transactions - ${TMP_TEST_FOLDER}/gitaly/internal/ - ${TMP_TEST_FOLDER}/gitaly/Makefile - ${TMP_TEST_FOLDER}/gitaly/praefect.config.toml @@ -45,6 +47,7 @@ setup-test-env: - ${TMP_TEST_FOLDER}/repositories/ - ${TMP_TEST_FOLDER}/second_storage/ - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/ + - log/*.log when: always setup-test-env-fips: @@ -303,6 +306,33 @@ rspec system pg14 praefect: - .rspec-system-parallel - .rails:rules:praefect-with-db +# Test jobs that run with Gitaly's transactions enabled. These will be removed once +# transactions are always in use in Gitaly. +rspec migration gitaly_transactions: + extends: + - rspec migration pg14 + - .gitaly-with-transactions + +rspec background_migration gitaly_transactions: + extends: + - rspec background_migration pg14 + - .gitaly-with-transactions + +rspec unit gitaly_transactions: + extends: + - rspec unit pg14 + - .gitaly-with-transactions + +rspec integration gitaly_transactions: + extends: + - rspec integration pg14 + - .gitaly-with-transactions + +rspec system gitaly_transactions: + extends: + - rspec system pg14 + - .gitaly-with-transactions + # Dedicated job to test DB library code against PG13. # Note that these are already tested against PG13 in the `rspec unit pg13` / `rspec-ee unit pg13` jobs. rspec db-library-code pg13: diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml index 5ce7ed5db37..b9d39d2f9d0 100644 --- a/.gitlab/ci/rails/shared.gitlab-ci.yml +++ b/.gitlab/ci/rails/shared.gitlab-ci.yml @@ -56,6 +56,10 @@ include: variables: GITALY_PRAEFECT_WITH_DB: '1' +.gitaly-with-transactions: + variables: + GITALY_TRANSACTIONS_ENABLED: "true" + .rspec-base-needs: needs: - job: "clone-gitlab-repo" diff --git a/db/docs/fork_networks.yml b/db/docs/fork_networks.yml index 65938326da7..0977b27d182 100644 --- a/db/docs/fork_networks.yml +++ b/db/docs/fork_networks.yml @@ -4,7 +4,10 @@ classes: - ForkNetwork feature_categories: - source_code_management -description: When a project is first forked, a row is created in this table. Also referenced by the fork_network_members table. This is used to know which projects can send merge reqeusts to each other. +description: When a project is first forked, a row is created in this table. Also + referenced by the fork_network_members table. This is used to know which projects + can send merge reqeusts to each other. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3098 milestone: '10.1' gitlab_schema: gitlab_main_cell +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/lfs_object_states.yml b/db/docs/lfs_object_states.yml index a0a6d4345fb..2b258b31b70 100644 --- a/db/docs/lfs_object_states.yml +++ b/db/docs/lfs_object_states.yml @@ -8,3 +8,4 @@ description: Geo verification states for LFS objects introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63981 milestone: '14.6' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/lfs_objects.yml b/db/docs/lfs_objects.yml index 431aace668f..892f95cc837 100644 --- a/db/docs/lfs_objects.yml +++ b/db/docs/lfs_objects.yml @@ -8,3 +8,4 @@ description: LFS files introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/1727 milestone: '8.2' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/protected_branch_merge_access_levels.yml b/db/docs/protected_branch_merge_access_levels.yml index f0a11ef5489..cae623a9ee1 100644 --- a/db/docs/protected_branch_merge_access_levels.yml +++ b/db/docs/protected_branch_merge_access_levels.yml @@ -8,3 +8,4 @@ description: Stores merge access settings for protected branches introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5081 milestone: '8.11' gitlab_schema: gitlab_main_cell +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/protected_branch_push_access_levels.yml b/db/docs/protected_branch_push_access_levels.yml index e614c3d4838..654ee329bea 100644 --- a/db/docs/protected_branch_push_access_levels.yml +++ b/db/docs/protected_branch_push_access_levels.yml @@ -8,3 +8,4 @@ description: Stores push access settings for protected branches introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5081 milestone: '8.11' gitlab_schema: gitlab_main_cell +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/protected_branch_unprotect_access_levels.yml b/db/docs/protected_branch_unprotect_access_levels.yml index 0b631b392c2..3b596b7e592 100644 --- a/db/docs/protected_branch_unprotect_access_levels.yml +++ b/db/docs/protected_branch_unprotect_access_levels.yml @@ -10,3 +10,4 @@ milestone: '10.7' gitlab_schema: gitlab_main_cell allow_cross_foreign_keys: - gitlab_main_clusterwide +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/push_rules.yml b/db/docs/push_rules.yml index 1579268a9bb..10e2586fe05 100644 --- a/db/docs/push_rules.yml +++ b/db/docs/push_rules.yml @@ -8,3 +8,4 @@ description: TODO introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/1b98b5ab97ce3e9997df542059cbf3c6ce0bf0e1 milestone: '8.10' gitlab_schema: gitlab_main_cell +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/required_code_owners_sections.yml b/db/docs/required_code_owners_sections.yml index 6c7a56df6fc..608b106d8e2 100644 --- a/db/docs/required_code_owners_sections.yml +++ b/db/docs/required_code_owners_sections.yml @@ -8,3 +8,4 @@ description: Keeps required code owners sections introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43573 milestone: '13.5' gitlab_schema: gitlab_main_cell +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/snippet_repositories.yml b/db/docs/snippet_repositories.yml index e2fd54f7c58..3495aa790bb 100644 --- a/db/docs/snippet_repositories.yml +++ b/db/docs/snippet_repositories.yml @@ -8,3 +8,4 @@ description: Stores repository information used to version control snippets. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23796 milestone: '12.8' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/snippet_statistics.yml b/db/docs/snippet_statistics.yml index ad816a6e2e0..d77d9741378 100644 --- a/db/docs/snippet_statistics.yml +++ b/db/docs/snippet_statistics.yml @@ -4,7 +4,9 @@ classes: - SnippetStatistics feature_categories: - source_code_management -description: Stores the repository size, commit count, and file count regarding the snippet repository. +description: Stores the repository size, commit count, and file count regarding the + snippet repository. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35026 milestone: '13.2' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/snippet_user_mentions.yml b/db/docs/snippet_user_mentions.yml index 099d027d145..a2380bce672 100644 --- a/db/docs/snippet_user_mentions.yml +++ b/db/docs/snippet_user_mentions.yml @@ -4,7 +4,9 @@ classes: - SnippetUserMention feature_categories: - source_code_management -description: For storing mentioned users, groups, projects referenced in a snippet description. +description: For storing mentioned users, groups, projects referenced in a snippet + description. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19009 milestone: '12.6' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/snippets.yml b/db/docs/snippets.yml index 1a1719552a1..c0d616e97ab 100644 --- a/db/docs/snippets.yml +++ b/db/docs/snippets.yml @@ -6,7 +6,9 @@ classes: - Snippet feature_categories: - source_code_management -description: GitLab snippets allow you to store and share bits of code and text with other users. +description: GitLab snippets allow you to store and share bits of code and text with + other users. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/9265de3d25715aeafd38a4ef41596dca058dc18c -milestone: "1.0.1" +milestone: 1.0.1 gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/460829 diff --git a/db/docs/x509_certificates.yml b/db/docs/x509_certificates.yml index 364bd3615bb..7160ed85576 100644 --- a/db/docs/x509_certificates.yml +++ b/db/docs/x509_certificates.yml @@ -8,3 +8,4 @@ description: Stores data about X.509 certificate introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17773 milestone: '12.8' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/db/docs/x509_issuers.yml b/db/docs/x509_issuers.yml index 04253f903be..d872c3541b2 100644 --- a/db/docs/x509_issuers.yml +++ b/db/docs/x509_issuers.yml @@ -8,3 +8,4 @@ description: Stores data about issuer of X.509 certificate introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17773 milestone: '12.8' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/462136 diff --git a/doc/administration/audit_event_streaming/index.md b/doc/administration/audit_event_streaming/index.md index 79d809372c5..5e0b5fd7aaa 100644 --- a/doc/administration/audit_event_streaming/index.md +++ b/doc/administration/audit_event_streaming/index.md @@ -4,24 +4,26 @@ group: Compliance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Audit event streaming +# Audit event streaming for instances DETAILS: **Tier:** Ultimate -**Offering:** GitLab.com, Self-managed, GitLab Dedicated +**Offering:** Self-managed, GitLab Dedicated -> - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default. +> - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2. +> - Instance streaming destinations [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) in GitLab 16.4. [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/417708) removed. > - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](../feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default. > - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed. > - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3. -> - HTTP destination **Name*** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3. +> - HTTP destination **Name** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3. > - Functionality for the **Active** checkbox [added](https://gitlab.com/gitlab-org/gitlab/-/issues/415268) in GitLab 16.5. -Users can set a streaming destination for a top-level group or instance to receive all audit events about the group, -subgroups, and projects, as structured JSON. +Audit event streaming for instances, administrators can: -Top-level group owners and instance administrators can manage their audit logs in third-party systems. Any service that -can receive structured JSON data can be used as the streaming destination. +- Set a streaming destination for an entire instance to receive all audit events about that instance as structured JSON. +- Manage their audit logs in third-party systems. Any service that can receive structured JSON data can be used as the + streaming destination. Each streaming destination can have up to 20 custom HTTP headers included with each streamed event. @@ -34,349 +36,9 @@ WARNING: Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust the streaming destination. -## Top-level group streaming destinations - -Manage streaming destinations for top-level groups. - -### HTTP destinations - -Prerequisites: - -- For better security, you should use an SSL certificate on the destination URL. - -Manage HTTP streaming destinations for top-level groups. - -#### Add a new HTTP destination - -Add a new HTTP streaming destination to a top-level group. - -Prerequisites: - -- Owner role for a top-level group. - -To add streaming destinations to a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select **Add streaming destination** and select **HTTP endpoint** to show the section for adding destinations. -1. In the **Name** and **Destination URL** fields, add a destination name and URL. -1. Optional. Locate the **Custom HTTP headers** table. -1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event. -1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to - 20 headers per streaming destination. -1. After all headers have been filled out, select **Add** to add the new streaming destination. - -#### List HTTP destinations - -Prerequisites: - -- Owner role for a group. - -To list the streaming destinations for a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the stream to expand it and see all the custom HTTP headers. - -#### Update an HTTP destination - -Prerequisites: - -- Owner role for a group. - -To update a streaming destination's name: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the stream to expand. -1. In the **Name** fields, add a destination name to update. -1. Select **Save** to update the streaming destination. - -To update a streaming destination's custom HTTP headers: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the stream to expand. -1. Locate the **Custom HTTP headers** table. -1. Locate the header that you wish to update. -1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event. -1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to - 20 headers per streaming destination. -1. Select **Save** to update the streaming destination. - -#### Delete an HTTP destination - -Delete streaming destinations for a top-level group. When the last destination is successfully deleted, streaming is -disabled for the top-level group. - -Prerequisites: - -- Owner role for a group. - -To delete a streaming destination: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the stream to expand. -1. Select **Delete destination**. -1. Confirm by selecting **Delete destination** in the dialog. - -To delete only the custom HTTP headers for a streaming destination: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the stream to expand. -1. Locate the **Custom HTTP headers** table. -1. Locate the header that you wish to remove. -1. To the right of the header, select **Delete** (**{remove}**). -1. Select **Save** to update the streaming destination. - -#### Verify event authenticity - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2. - -Each streaming destination has a unique verification token (`verificationToken`) that can be used to verify the authenticity of the event. This -token is either specified by the Owner or generated automatically when the event destination is created and cannot be changed. - -Each streamed event contains the verification token in the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against -the destination's value when listing streaming destinations. - -Prerequisites: - -- Owner role for a group. - -To list streaming destinations and see the verification tokens: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the stream to expand. -1. Locate the **Verification token** input. - -#### Update event filters - -> - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/413581) in GitLab 16.1. - -When this feature is enabled for a group, you can permit users to filter streamed audit events per destination. -If the feature is enabled with no filters, the destination receives all audit events. - -A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label. - -To update a streaming destination's event filters: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the stream to expand. -1. Locate the **Filter by audit event type** dropdown list. -1. Select the dropdown list and select or clear the required event types. -1. Select **Save** to update the event filters. - -#### Update namespace filters - -> - Namespace filtering in the UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390133) in GitLab 16.7. - -When this feature is enabled for a group, you can permit users to filter streamed audit events per destination. -If the feature is enabled with no filters, the destination receives all audit events. - -A streaming destination that has a namespace filter set has a **filtered** (**{filter}**) label. - -To update a streaming destination's namespace filters: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the stream to expand. -1. Locate the **Filter by groups or projects** dropdown list. -1. Select the dropdown list and select or clear the required namespaces. -1. Select **Save** to update the namespace filter. - -#### Override default content type header - -By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you -might want to set the `content-type` header to something else. For example ,`application/json`. - -To override the `content-type` header default value for a top-level group streaming destination, use either: - -- The [GitLab UI](#update-an-http-destination). -- The [GraphQL API](graphql_api.md#update-streaming-destinations). - -### Google Cloud Logging destinations - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124384) in GitLab 16.2. - -Manage Google Cloud Logging destinations for top-level groups. - -#### Prerequisites - -Before setting up Google Cloud Logging streaming audit events, you must: - -1. Enable [Cloud Logging API](https://console.cloud.google.com/marketplace/product/google/logging.googleapis.com) on your Google Cloud project. -1. Create a service account for Google Cloud with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication. - For more information, see [Creating and managing service accounts in the Google Cloud documentation](https://cloud.google.com/iam/docs/service-accounts-create#creating). -1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter). -1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating). - -#### Add a new Google Cloud Logging destination - -Prerequisites: - -- Owner role for a top-level group. - -To add Google Cloud Logging streaming destinations to a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select **Add streaming destination** and select **Google Cloud Logging** to show the section for adding destinations. -1. Enter a random string to use as a name for the new destination. -1. Enter the Google project ID, Google client email, and Google private key from previously-created Google Cloud service account key to add to the new destination. -1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud. -1. Select **Add** to add the new streaming destination. - -#### List Google Cloud Logging destinations - -Prerequisites: - -- Owner role for a top-level group. - -To list Google Cloud Logging streaming destinations for a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the Google Cloud Logging stream to expand and see all the fields. - -#### Update a Google Cloud Logging destination - -> - Button to add private key [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/419675) in GitLab 16.3. - -Prerequisites: - -- Owner role for a top-level group. - -To update Google Cloud Logging streaming destinations to a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the Google Cloud Logging stream to expand. -1. Enter a random string to use as a name for the destination. -1. Enter the Google project ID and Google client email from previously-created Google Cloud service account key to update the destination. -1. Enter a random string to update the log ID for the destination. You can use this later to filter log results in Google Cloud. -1. Select **Add a new private key** and enter a Google private key to update the private key. -1. Select **Save** to update the streaming destination. - -#### Delete a Google Cloud Logging streaming destination - -Prerequisites: - -- Owner role for a top-level group. - -To delete Google Cloud Logging streaming destinations to a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the Google Cloud Logging stream to expand. -1. Select **Delete destination**. -1. Confirm by selecting **Delete destination** in the dialog. - -### AWS S3 destinations - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132603) in GitLab 16.6 [with a flag](../feature_flags.md) named `allow_streaming_audit_events_to_amazon_s3`. Enabled by default. -> - [Feature flag `allow_streaming_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.7. - -Manage AWS S3 destinations for top-level groups. - -#### Prerequisites - -Before setting up AWS S3 streaming audit events, you must: - -1. Create a access key for AWS with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication. - For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey). -1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) - -#### Add a new AWS S3 destination - -Prerequisites: - -- Owner role for a top-level group. - -To add AWS S3 streaming destinations to a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select **Add streaming destination** and select **AWS S3** to show the section for adding destinations. -1. Enter a random string to use as a name for the new destination. -1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination. -1. Select **Add** to add the new streaming destination. - -#### List AWS S3 destinations - -Prerequisites: - -- Owner role for a top-level group. - -To list AWS S3 streaming destinations for a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the AWS S3 stream to expand and see all the fields. - -#### Update a AWS S3 destination - -Prerequisites: - -- Owner role for a top-level group. - -To update AWS S3 streaming destinations to a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the AWS S3 stream to expand. -1. Enter a random string to use as a name for the destination. -1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to update the destination. -1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key. -1. Select **Save** to update the streaming destination. - -#### Delete a AWS S3 streaming destination - -Prerequisites: - -- Owner role for a top-level group. - -To delete AWS S3 streaming destinations to a top-level group: - -1. On the left sidebar, select **Search or go to** and find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. Select the AWS S3 stream to expand. -1. Select **Delete destination**. -1. Confirm by selecting **Delete destination** in the dialog. - -## Instance streaming destinations - -DETAILS: -**Tier:** Ultimate -**Offering:** Self-managed - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default. -> - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2. -> - Instance streaming destinations [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) in GitLab 16.4. [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/417708) removed. - Manage streaming destinations for an entire instance. -### HTTP destinations +## HTTP destinations Prerequisites: @@ -384,7 +46,7 @@ Prerequisites: Manage HTTP streaming destinations for an entire instance. -#### Add a new HTTP destination +### Add a new HTTP destination Add a new HTTP streaming destination to an instance. @@ -405,7 +67,7 @@ To add a streaming destination for an instance: 20 headers per streaming destination. 1. After all headers have been filled out, select **Add** to add the new streaming destination. -#### List HTTP destinations +### List HTTP destinations Prerequisites: @@ -418,7 +80,7 @@ To list the streaming destinations for an instance: 1. On the main area, select the **Streams** tab. 1. Select the stream to expand it and see all the custom HTTP headers. -#### Update an HTTP destination +### Update an HTTP destination Prerequisites: @@ -446,7 +108,7 @@ To update a instance streaming destination's custom HTTP headers: 20 headers per streaming destination. 1. Select **Save** to update the streaming destination. -#### Delete an HTTP destination +### Delete an HTTP destination Delete streaming destinations for an entire instance. When the last destination is successfully deleted, streaming is disabled for the instance. @@ -475,7 +137,7 @@ To delete only the custom HTTP headers for a streaming destination: 1. To the right of the header, select **Delete** (**{remove}**). 1. Select **Save** to update the streaming destination. -#### Verify event authenticity +### Verify event authenticity > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default. > - [Feature flag `ff_external_audit_events`](https://gitlab.com/gitlab-org/gitlab/-/issues/393772) enabled by default in GitLab 16.2. @@ -498,7 +160,7 @@ To list streaming destinations for an instance and see the verification tokens: 1. On the main area, select the **Streams** tab. 1. View the verification token on the right side of each item. -#### Update event filters +### Update event filters > - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/415013) in GitLab 16.3. @@ -517,23 +179,23 @@ To update a streaming destination's event filters: 1. Select the dropdown list and select or clear the required event types. 1. Select **Save** to update the event filters. -#### Override default content type header +### Override default content type header By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you might want to set the `content-type` header to something else. For example ,`application/json`. To override the `content-type` header default value for an instance streaming destination, use either: -- The [GitLab UI](#update-an-http-destination-1). +- The [GitLab UI](#update-an-http-destination). - The [GraphQL API](graphql_api.md#update-streaming-destinations). -### Google Cloud Logging destinations +## Google Cloud Logging destinations > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131851) in GitLab 16.5. Manage Google Cloud Logging destinations for an entire instance. -#### Prerequisites +### Prerequisites Before setting up Google Cloud Logging streaming audit events, you must: @@ -543,7 +205,7 @@ Before setting up Google Cloud Logging streaming audit events, you must: 1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter). 1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating). -#### Add a new Google Cloud Logging destination +### Add a new Google Cloud Logging destination Prerequisites: @@ -560,7 +222,7 @@ To add Google Cloud Logging streaming destinations to an instance: 1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud. 1. Select **Add** to add the new streaming destination. -#### List Google Cloud Logging destinations +### List Google Cloud Logging destinations Prerequisites: @@ -573,7 +235,7 @@ To list Google Cloud Logging streaming destinations for an instance: 1. On the main area, select the **Streams** tab. 1. Select the Google Cloud Logging stream to expand and see all the fields. -#### Update a Google Cloud Logging destination +### Update a Google Cloud Logging destination Prerequisites: @@ -591,7 +253,7 @@ To update Google Cloud Logging streaming destinations to an instance: 1. Select **Add a new private key** and enter a Google private key to update the private key. 1. Select **Save** to update the streaming destination. -#### Delete a Google Cloud Logging streaming destination +### Delete a Google Cloud Logging streaming destination Prerequisites: @@ -606,14 +268,14 @@ To delete Google Cloud Logging streaming destinations to an instance: 1. Select **Delete destination**. 1. Confirm by selecting **Delete destination** in the dialog. -### AWS S3 destinations +## AWS S3 destinations > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138245) in GitLab 16.7 [with a flag](../feature_flags.md) named `allow_streaming_instance_audit_events_to_amazon_s3`. Disabled by default. > - [Feature flag `allow_streaming_instance_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.8. Manage AWS S3 destinations for entire instance. -#### Prerequisites +### Prerequisites Before setting up AWS S3 streaming audit events, you must: @@ -621,7 +283,7 @@ Before setting up AWS S3 streaming audit events, you must: For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey). 1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) -#### Add a new AWS S3 destination +### Add a new AWS S3 destination Prerequisites: @@ -637,7 +299,7 @@ To add AWS S3 streaming destinations to an instance: 1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination. 1. Select **Add** to add the new streaming destination. -#### List AWS S3 destinations +### List AWS S3 destinations Prerequisites: @@ -650,7 +312,7 @@ To list AWS S3 streaming destinations for an instance. 1. On the main area, select the **Streams** tab. 1. Select the AWS S3 stream to expand and see all the fields. -#### Update an AWS S3 destination +### Update an AWS S3 destination Prerequisites: @@ -667,7 +329,7 @@ To update AWS S3 streaming destinations to an instance: 1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key. 1. Select **Save** to update the streaming destination. -#### Delete an AWS S3 streaming destination +### Delete an AWS S3 streaming destination Prerequisites: @@ -681,3 +343,7 @@ To delete AWS S3 streaming destinations on an instance: 1. Select the AWS S3 stream to expand. 1. Select **Delete destination**. 1. Confirm by selecting **Delete destination** in the dialog. + +## Related topics + +- [Audit event streaming for top-level groups](../../user/compliance/audit_event_streaming.md) diff --git a/doc/development/development_processes.md b/doc/development/development_processes.md index 730f7da6188..f35b49f6364 100644 --- a/doc/development/development_processes.md +++ b/doc/development/development_processes.md @@ -19,10 +19,10 @@ Must-reads: database-related changes and complex SQL queries, and having them reviewed - [Secure coding guidelines](secure_coding_guidelines.md) - [Pipelines for the GitLab project](pipelines/index.md) +- [Avoiding required stops](avoiding_required_stops.md) Complementary reads: -- [Avoiding required stops](avoiding_required_stops.md) - [Contribute to GitLab](contributing/index.md) - [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/engineer.md#security-releases-critical-non-critical-as-a-developer) - [Patch release process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/patch/process.md#process-for-developers) diff --git a/doc/user/compliance/audit_event_streaming.md b/doc/user/compliance/audit_event_streaming.md new file mode 100644 index 00000000000..a2c6da6d0e4 --- /dev/null +++ b/doc/user/compliance/audit_event_streaming.md @@ -0,0 +1,366 @@ +--- +stage: Govern +group: Compliance +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Audit event streaming for top-level groups + +DETAILS: +**Tier:** Ultimate +**Offering:** GitLab.com, Self-managed, GitLab Dedicated + +> - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2. +> - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](../feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default. +> - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed. +> - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3. +> - HTTP destination **Name** field [added](https://gitlab.com/gitlab-org/gitlab/-/issues/411357) in GitLab 16.3. +> - Functionality for the **Active** checkbox [added](https://gitlab.com/gitlab-org/gitlab/-/issues/415268) in GitLab 16.5. + +With audit event streaming for top-level groups, group owners can: + +- Set a streaming destination for a top-level group to receive all audit events about the group, subgroups, and projects + as structured JSON. +- Manage their audit logs in third-party systems. Any service that can receive structured JSON data can be used as the + streaming destination. + +Each streaming destination can have up to 20 custom HTTP headers included with each streamed event. + +GitLab can stream a single event more than once to the same destination. Use the `id` key in the payload to deduplicate +incoming data. + +Audit events are sent using the POST request method protocol supported by HTTP. + +WARNING: +Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust +the streaming destination. + +## HTTP destinations + +Prerequisites: + +- For better security, you should use an SSL certificate on the destination URL. + +Manage HTTP streaming destinations for top-level groups. + +### Add a new HTTP destination + +Add a new HTTP streaming destination to a top-level group. + +Prerequisites: + +- Owner role for a top-level group. + +To add streaming destinations to a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select **Add streaming destination** and select **HTTP endpoint** to show the section for adding destinations. +1. In the **Name** and **Destination URL** fields, add a destination name and URL. +1. Optional. Locate the **Custom HTTP headers** table. +1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event. +1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to + 20 headers per streaming destination. +1. After all headers have been filled out, select **Add** to add the new streaming destination. + +### List HTTP destinations + +Prerequisites: + +- Owner role for a group. + +To list the streaming destinations for a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the stream to expand it and see all the custom HTTP headers. + +### Update an HTTP destination + +Prerequisites: + +- Owner role for a group. + +To update a streaming destination's name: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the stream to expand. +1. In the **Name** fields, add a destination name to update. +1. Select **Save** to update the streaming destination. + +To update a streaming destination's custom HTTP headers: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the stream to expand. +1. Locate the **Custom HTTP headers** table. +1. Locate the header that you wish to update. +1. To make the header active, select the **Active** checkbox. The header will be sent with the audit event. +1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to + 20 headers per streaming destination. +1. Select **Save** to update the streaming destination. + +### Delete an HTTP destination + +Delete streaming destinations for a top-level group. When the last destination is successfully deleted, streaming is +disabled for the top-level group. + +Prerequisites: + +- Owner role for a group. + +To delete a streaming destination: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the stream to expand. +1. Select **Delete destination**. +1. Confirm by selecting **Delete destination** in the dialog. + +To delete only the custom HTTP headers for a streaming destination: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the stream to expand. +1. Locate the **Custom HTTP headers** table. +1. Locate the header that you wish to remove. +1. To the right of the header, select **Delete** (**{remove}**). +1. Select **Save** to update the streaming destination. + +### Verify event authenticity + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2. + +Each streaming destination has a unique verification token (`verificationToken`) that can be used to verify the authenticity of the event. This +token is either specified by the Owner or generated automatically when the event destination is created and cannot be changed. + +Each streamed event contains the verification token in the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against +the destination's value when listing streaming destinations. + +Prerequisites: + +- Owner role for a group. + +To list streaming destinations and see the verification tokens: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the stream to expand. +1. Locate the **Verification token** input. + +### Update event filters + +> - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/413581) in GitLab 16.1. + +When this feature is enabled for a group, you can permit users to filter streamed audit events per destination. +If the feature is enabled with no filters, the destination receives all audit events. + +A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label. + +To update a streaming destination's event filters: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the stream to expand. +1. Locate the **Filter by audit event type** dropdown list. +1. Select the dropdown list and select or clear the required event types. +1. Select **Save** to update the event filters. + +### Update namespace filters + +> - Namespace filtering in the UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390133) in GitLab 16.7. + +When this feature is enabled for a group, you can permit users to filter streamed audit events per destination. +If the feature is enabled with no filters, the destination receives all audit events. + +A streaming destination that has a namespace filter set has a **filtered** (**{filter}**) label. + +To update a streaming destination's namespace filters: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the stream to expand. +1. Locate the **Filter by groups or projects** dropdown list. +1. Select the dropdown list and select or clear the required namespaces. +1. Select **Save** to update the namespace filter. + +### Override default content type header + +By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you +might want to set the `content-type` header to something else. For example ,`application/json`. + +To override the `content-type` header default value for a top-level group streaming destination, use either: + +- The [GitLab UI](#update-an-http-destination). +- The [GraphQL API](../../administration/audit_event_streaming/graphql_api.md#update-streaming-destinations). + +## Google Cloud Logging destinations + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124384) in GitLab 16.2. + +Manage Google Cloud Logging destinations for top-level groups. + +### Prerequisites + +Before setting up Google Cloud Logging streaming audit events, you must: + +1. Enable [Cloud Logging API](https://console.cloud.google.com/marketplace/product/google/logging.googleapis.com) on your Google Cloud project. +1. Create a service account for Google Cloud with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication. + For more information, see [Creating and managing service accounts in the Google Cloud documentation](https://cloud.google.com/iam/docs/service-accounts-create#creating). +1. Enable the **Logs Writer** role for the service account to enable logging on Google Cloud. For more information, see [Access control with IAM](https://cloud.google.com/logging/docs/access-control#logging.logWriter). +1. Create a JSON key for the service account. For more information, see [Creating a service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating). + +### Add a new Google Cloud Logging destination + +Prerequisites: + +- Owner role for a top-level group. + +To add Google Cloud Logging streaming destinations to a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select **Add streaming destination** and select **Google Cloud Logging** to show the section for adding destinations. +1. Enter a random string to use as a name for the new destination. +1. Enter the Google project ID, Google client email, and Google private key from previously-created Google Cloud service account key to add to the new destination. +1. Enter a random string to use as a log ID for the new destination. You can use this later to filter log results in Google Cloud. +1. Select **Add** to add the new streaming destination. + +### List Google Cloud Logging destinations + +Prerequisites: + +- Owner role for a top-level group. + +To list Google Cloud Logging streaming destinations for a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the Google Cloud Logging stream to expand and see all the fields. + +### Update a Google Cloud Logging destination + +> - Button to add private key [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/419675) in GitLab 16.3. + +Prerequisites: + +- Owner role for a top-level group. + +To update Google Cloud Logging streaming destinations to a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the Google Cloud Logging stream to expand. +1. Enter a random string to use as a name for the destination. +1. Enter the Google project ID and Google client email from previously-created Google Cloud service account key to update the destination. +1. Enter a random string to update the log ID for the destination. You can use this later to filter log results in Google Cloud. +1. Select **Add a new private key** and enter a Google private key to update the private key. +1. Select **Save** to update the streaming destination. + +### Delete a Google Cloud Logging streaming destination + +Prerequisites: + +- Owner role for a top-level group. + +To delete Google Cloud Logging streaming destinations to a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the Google Cloud Logging stream to expand. +1. Select **Delete destination**. +1. Confirm by selecting **Delete destination** in the dialog. + +## AWS S3 destinations + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132603) in GitLab 16.6 [with a flag](../feature_flags.md) named `allow_streaming_audit_events_to_amazon_s3`. Enabled by default. +> - [Feature flag `allow_streaming_audit_events_to_amazon_s3`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137391) removed in GitLab 16.7. + +Manage AWS S3 destinations for top-level groups. + +### Prerequisites + +Before setting up AWS S3 streaming audit events, you must: + +1. Create a access key for AWS with the appropriate credentials and permissions. This account is used to configure audit log streaming authentication. + For more information, see [Managing access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html?icmpid=docs_iam_console#Using_CreateAccessKey). +1. Create a AWS S3 bucket. This bucket is used to store audit log streaming data. For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) + +### Add a new AWS S3 destination + +Prerequisites: + +- Owner role for a top-level group. + +To add AWS S3 streaming destinations to a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select **Add streaming destination** and select **AWS S3** to show the section for adding destinations. +1. Enter a random string to use as a name for the new destination. +1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to add to the new destination. +1. Select **Add** to add the new streaming destination. + +### List AWS S3 destinations + +Prerequisites: + +- Owner role for a top-level group. + +To list AWS S3 streaming destinations for a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the AWS S3 stream to expand and see all the fields. + +### Update a AWS S3 destination + +Prerequisites: + +- Owner role for a top-level group. + +To update AWS S3 streaming destinations to a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the AWS S3 stream to expand. +1. Enter a random string to use as a name for the destination. +1. Enter the Access Key ID, Secret Access Key, Bucket Name, and AWS Region from previously-created AWS access key and bucket to update the destination. +1. Select **Add a new Secret Access Key** and enter a AWS Secret Access Key to update the Secret Access Key. +1. Select **Save** to update the streaming destination. + +### Delete a AWS S3 streaming destination + +Prerequisites: + +- Owner role for a top-level group. + +To delete AWS S3 streaming destinations to a top-level group: + +1. On the left sidebar, select **Search or go to** and find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. Select the AWS S3 stream to expand. +1. Select **Delete destination**. +1. Confirm by selecting **Delete destination** in the dialog. + +## Related topics + +- [Audit event streaming for instances](../../administration/audit_event_streaming/index.md) diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index e25503ae7b9..67b8f06f0b6 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -113,6 +113,7 @@ module Gitlab config[:bin_dir] = File.expand_path(File.join(gitaly_dir, '_build', 'bin')) # binaries by default are in `_build/bin` config[:gitlab] = { url: Gitlab.config.gitlab.url } config[:logging] = { dir: Rails.root.join('log').to_s } + config[:transactions] = { enabled: true } if options[:transactions_enabled] TomlRB.dump(config) end diff --git a/scripts/gitaly-test-build b/scripts/gitaly-test-build index cfa089b327e..7c0d101f955 100755 --- a/scripts/gitaly-test-build +++ b/scripts/gitaly-test-build @@ -24,8 +24,8 @@ class GitalyTestBuild ensure_gitlab_shell_secret! # Starting gitaly further validates its configuration - gitaly_pid = start_gitaly - gitaly2_pid = start_gitaly2 + gitaly_pid = start_gitaly(:gitaly) + gitaly2_pid = start_gitaly(:gitaly2) praefect_pid = start_praefect Process.kill('TERM', gitaly_pid) Process.kill('TERM', gitaly2_pid) diff --git a/spec/lib/gitlab/git_access_snippet_spec.rb b/spec/lib/gitlab/git_access_snippet_spec.rb index 3a25bf81416..721fe75d18b 100644 --- a/spec/lib/gitlab/git_access_snippet_spec.rb +++ b/spec/lib/gitlab/git_access_snippet_spec.rb @@ -311,7 +311,7 @@ RSpec.describe Gitlab::GitAccessSnippet do end describe 'repository size restrictions' do - let_it_be(:snippet) { create(:personal_snippet, :public, :repository) } + let_it_be_with_refind(:snippet) { create(:personal_snippet, :public, :repository) } let(:actor) { snippet.author } let(:oldrev) { TestEnv::BRANCH_SHA["snippet/single-file"] } diff --git a/spec/requests/api/ci/jobs_spec.rb b/spec/requests/api/ci/jobs_spec.rb index f6b482b943b..adf822e3cc5 100644 --- a/spec/requests/api/ci/jobs_spec.rb +++ b/spec/requests/api/ci/jobs_spec.rb @@ -435,11 +435,11 @@ RSpec.describe API::Ci::Jobs, feature_category: :continuous_integration do control = ActiveRecord::QueryRecorder.new(skip_cached: false) { go } 5.times do - second_pipeline = create(:ci_pipeline, project: project, sha: project.commit.id, ref: project.default_branch) - second_build = create(:ci_build, :trace_artifact, :artifacts, :test_reports, pipeline: second_pipeline) - second_build.runner = create(:ci_runner) - second_build.user = create(:user) - second_build.save! + another_pipeline = create(:ci_pipeline, project: project, sha: project.commit.id, ref: project.default_branch) + another_build = create(:ci_build, :trace_artifact, :artifacts, :test_reports, pipeline: another_pipeline) + another_build.runner = create(:ci_runner) + another_build.user = create(:user) + another_build.save! end expect { go }.not_to exceed_query_limit(control) diff --git a/spec/support/helpers/gitaly_setup.rb b/spec/support/helpers/gitaly_setup.rb index 6272798aa15..dcb7f9afb0c 100644 --- a/spec/support/helpers/gitaly_setup.rb +++ b/spec/support/helpers/gitaly_setup.rb @@ -70,17 +70,21 @@ module GitalySetup } end - def config_path(service) + def config_name(service) case service when :gitaly - File.join(tmp_tests_gitaly_dir, 'config.toml') + 'config.toml' when :gitaly2 - File.join(tmp_tests_gitaly_dir, 'gitaly2.config.toml') + 'gitaly2.config.toml' when :praefect - File.join(tmp_tests_gitaly_dir, 'praefect.config.toml') + 'praefect.config.toml' end end + def config_path(service) + File.join(tmp_tests_gitaly_dir, config_name(service)) + end + def service_cmd(service, toml = nil) toml ||= config_path(service) @@ -100,12 +104,22 @@ module GitalySetup run_command(%w[make all WITH_BUNDLED_GIT=YesPlease], env: env.merge('GIT_VERSION' => nil)) end - def start_gitaly(toml = nil) - start(:gitaly, toml) - end + def start_gitaly(service, toml = nil) + case service + when :gitaly + FileUtils.mkdir_p(GitalySetup.storage_path) + when :gitaly2 + FileUtils.mkdir_p(GitalySetup.second_storage_path) + end - def start_gitaly2 - start(:gitaly2) + if ENV['CI'] && gitaly_with_transactions? + # The configuration file with transactions is pre-generated in the CI. Here we check + # whether this job should actually run with transactions and choose the pre-generated + # configuration with transactions enabled if so. + toml = "#{config_path(service)}.transactions" + end + + start(service, toml) end def start_praefect @@ -226,25 +240,54 @@ module GitalySetup build_gitaly end - Gitlab::SetupHelper::Gitaly.create_configuration( - gitaly_dir, - { 'default' => storage_path }, - force: true, - options: { - runtime_dir: runtime_dir, - prometheus_listen_addr: 'localhost:9236' + [ + { + storages: { 'default' => storage_path }, + options: { + runtime_dir: runtime_dir, + prometheus_listen_addr: 'localhost:9236', + config_filename: config_name(:gitaly), + transactions_enabled: gitaly_with_transactions? + } + }, + { + storages: { 'test_second_storage' => second_storage_path }, + options: { + runtime_dir: runtime_dir, + gitaly_socket: "gitaly2.socket", + config_filename: config_name(:gitaly2), + transactions_enabled: gitaly_with_transactions? + } } - ) - Gitlab::SetupHelper::Gitaly.create_configuration( - gitaly_dir, - { 'test_second_storage' => second_storage_path }, - force: true, - options: { - runtime_dir: runtime_dir, - gitaly_socket: "gitaly2.socket", - config_filename: "gitaly2.config.toml" - } - ) + ].each do |params| + Gitlab::SetupHelper::Gitaly.create_configuration( + gitaly_dir, + params[:storages], + force: true, + options: params[:options] + ) + + # CI generates all of the configuration files in the setup-test-env job. When we eventually get + # to run the rspec jobs with transactions enabled, the configuration has already been created + # without transactions enabled. + # + # Similarly to the Praefect configuration, generate variant of the configuration file with + # transactions enabled in CI. Later when the rspec job runs, we decide whether to run Gitaly + # using the configuration with transactions enabled or not. + # + # These configuration files are only used in the CI. + next unless ENV['CI'] + + params[:options][:config_filename] = "#{params[:options][:config_filename]}.transactions" + params[:options][:transactions_enabled] = true + + Gitlab::SetupHelper::Gitaly.create_configuration( + gitaly_dir, + params[:storages], + force: true, + options: params[:options] + ) + end # In CI we need to pre-generate both config files. # For local testing we'll create the correct file on-demand. @@ -301,10 +344,10 @@ module GitalySetup pids = [] if toml - pids << start_gitaly(toml) + pids << start_gitaly(:gitaly, toml) else - pids << start_gitaly - pids << start_gitaly2 + pids << start_gitaly(:gitaly) + pids << start_gitaly(:gitaly2) pids << start_praefect end @@ -319,6 +362,8 @@ module GitalySetup next if ENV['GITALY_PID_FILE'] pids.each { |pid| stop(pid) } + + [storage_path, second_storage_path].each { |storage_dir| FileUtils.rm_rf(storage_dir) } end rescue StandardError raise gitaly_failure_message @@ -356,4 +401,8 @@ module GitalySetup def praefect_with_db? Gitlab::Utils.to_boolean(ENV['GITALY_PRAEFECT_WITH_DB'], default: false) end + + def gitaly_with_transactions? + Gitlab::Utils.to_boolean(ENV['GITALY_TRANSACTIONS_ENABLED'], default: false) + end end diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index c9ea22a2f78..6bcc57008aa 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -173,8 +173,6 @@ module TestEnv end end - FileUtils.mkdir_p(GitalySetup.storage_path) - FileUtils.mkdir_p(GitalySetup.second_storage_path) FileUtils.mkdir_p(backup_path) FileUtils.mkdir_p(pages_path) FileUtils.mkdir_p(artifacts_path) @@ -399,16 +397,18 @@ module TestEnv # These are directories that should be preserved at cleanup time def test_dirs - @test_dirs ||= %w[ - frontend - gitaly - gitlab-shell - gitlab-test - gitlab-test.bundle - gitlab-test-fork - gitlab-test-fork.bundle - gitlab-workhorse - gitlab_workhorse_secret + @test_dirs ||= [ + 'frontend', + 'gitaly', + 'gitlab-shell', + 'gitlab-test', + 'gitlab-test.bundle', + 'gitlab-test-fork', + 'gitlab-test-fork.bundle', + 'gitlab-workhorse', + 'gitlab_workhorse_secret', + File.basename(GitalySetup.storage_path), + File.basename(GitalySetup.second_storage_path) ] end diff --git a/spec/support/shared_contexts/glfm/example_snapshot_fixtures.rb b/spec/support/shared_contexts/glfm/example_snapshot_fixtures.rb index fea42d1fa91..e963cc706e4 100644 --- a/spec/support/shared_contexts/glfm/example_snapshot_fixtures.rb +++ b/spec/support/shared_contexts/glfm/example_snapshot_fixtures.rb @@ -24,4 +24,10 @@ RSpec.shared_context 'with GLFM example snapshot fixtures' do stub_licensed_features(group_wikis: true) sign_in(user) end + + after(:all) do + # We need to clean up the repository explicitly as we're using a static project ID. If two tests + # use this fixture, they'd attempt to create repositories with the same disk path and conflict. + ::Gitlab::GitalyClient::RepositoryService.new(project.repository).remove + end end diff --git a/workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go b/workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go index 1a60cf51eb1..33d85a5783c 100644 --- a/workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go +++ b/workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go @@ -93,7 +93,7 @@ func ensureGitalyRepository(_ *testing.T, apiResponse *api.Response) error { }, }); removeRepoErr != nil { status, ok := status.FromError(removeRepoErr) - if !ok || !(status.Code() == codes.NotFound && status.Message() == "repository does not exist") { + if !ok || !(status.Code() == codes.NotFound && (status.Message() == "repository does not exist" || status.Message() == "repository not found")) { return fmt.Errorf("remove repository: %w", removeRepoErr) }