From fdb478e6f3ac1a370123df6b92dbd331b46a7558 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 14 Sep 2023 21:11:21 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/ci/rules.gitlab-ci.yml | 1 + Gemfile | 2 +- Gemfile.checksum | 2 +- Gemfile.lock | 6 +- app/models/x509_certificate.rb | 2 - app/models/x509_issuer.rb | 7 +- .../security/configuration_presenter.rb | 5 +- app/workers/x509_issuer_crl_check_worker.rb | 2 +- ...arch_milestones_hide_archived_projects.yml | 9 + config/sidekiq_queues.yml | 2 + ...s_merge_request_of_vulnerability_reads.yml | 6 + db/migrate/20230821101010_remove_crl_null.rb | 15 + ...as_merge_request_of_vulnerability_reads.rb | 26 + db/schema_migrations/20230821101010 | 1 + db/schema_migrations/20230907155247 | 1 + db/structure.sql | 6 +- .../package_information/supported_os.md | 2 +- doc/administration/silent_mode/index.md | 25 +- doc/development/ai_features/duo_chat.md | 12 +- doc/development/database/index.md | 1 + doc/development/database/poc_tree_iterator.md | 475 ++++++++++++++++++ doc/development/feature_flags/index.md | 9 + doc/user/project/issues/design_management.md | 86 ++-- .../project/repository/signed_commits/x509.md | 4 +- ...as_merge_request_of_vulnerability_reads.rb | 41 ++ lib/gitlab/search_results.rb | 19 +- lib/gitlab/x509/signature.rb | 5 +- locale/gitlab.pot | 3 + spec/fixtures/ci_secure_files/sample.p12 | Bin 3352 -> 3219 bytes ...rge_request_of_vulnerability_reads_spec.rb | 101 ++++ spec/lib/gitlab/group_search_results_spec.rb | 11 + spec/lib/gitlab/search_results_spec.rb | 47 +- spec/lib/gitlab/x509/certificate_spec.rb | 2 +- spec/lib/gitlab/x509/commit_sigstore_spec.rb | 53 ++ spec/lib/gitlab/x509/commit_spec.rb | 6 +- .../gitlab/x509/signature_sigstore_spec.rb | 453 +++++++++++++++++ spec/lib/gitlab/x509/signature_spec.rb | 2 +- spec/lib/gitlab/x509/tag_sigstore_spec.rb | 45 ++ spec/lib/gitlab/x509/tag_spec.rb | 27 +- ...rge_request_of_vulnerability_reads_spec.rb | 26 + spec/models/x509_certificate_spec.rb | 1 - spec/models/x509_issuer_spec.rb | 2 - .../security/configuration_presenter_spec.rb | 2 +- spec/support/helpers/x509_helpers.rb | 181 +++++++ workhorse/config_test.go | 2 + workhorse/internal/config/config.go | 15 +- workhorse/internal/goredis/goredis.go | 9 +- 47 files changed, 1628 insertions(+), 132 deletions(-) create mode 100644 config/feature_flags/development/search_milestones_hide_archived_projects.yml create mode 100644 db/docs/batched_background_migrations/backfill_has_merge_request_of_vulnerability_reads.yml create mode 100644 db/migrate/20230821101010_remove_crl_null.rb create mode 100644 db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb create mode 100644 db/schema_migrations/20230821101010 create mode 100644 db/schema_migrations/20230907155247 create mode 100644 doc/development/database/poc_tree_iterator.md create mode 100644 lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb create mode 100644 spec/lib/gitlab/x509/commit_sigstore_spec.rb create mode 100644 spec/lib/gitlab/x509/signature_sigstore_spec.rb create mode 100644 spec/lib/gitlab/x509/tag_sigstore_spec.rb create mode 100644 spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 98363b87f86..03cd4693991 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -1669,6 +1669,7 @@ .qa:rules:package-and-test-nightly: rules: + - !reference [".qa:rules:package-and-test-never-run", rules] - <<: *if-default-branch-schedule-nightly allow_failure: true variables: diff --git a/Gemfile b/Gemfile index 111131e04ff..c157a00b6f8 100644 --- a/Gemfile +++ b/Gemfile @@ -446,7 +446,7 @@ group :development, :test do end group :development, :test, :danger do - gem 'gitlab-dangerfiles', '~> 3.13.0', require: false + gem 'gitlab-dangerfiles', '~> 4.0.0', require: false end group :development, :test, :coverage do diff --git a/Gemfile.checksum b/Gemfile.checksum index 4bd3126c7d1..519c3402237 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -208,7 +208,7 @@ {"name":"gitaly","version":"16.3.0.pre.rc1","platform":"ruby","checksum":"55d9cc414a4f3859588f3770bd88d7c67c0f5454a1178b018b7a6f6913674c43"}, {"name":"gitlab","version":"4.19.0","platform":"ruby","checksum":"3f645e3e195dbc24f0834fbf83e8ccfb2056d8e9712b01a640aad418a6949679"}, {"name":"gitlab-chronic","version":"0.10.5","platform":"ruby","checksum":"f80f18dc699b708870a80685243331290bc10cfeedb6b99c92219722f729c875"}, -{"name":"gitlab-dangerfiles","version":"3.13.0","platform":"ruby","checksum":"2081eac7fe1f538427f8ebec1e8cd7c143a30d50e1470348cdec4f2d273ea1ad"}, +{"name":"gitlab-dangerfiles","version":"4.0.0","platform":"ruby","checksum":"e3abe81790388e6a686a2cfb248c9a46486c0efbf169a07b62df2dad740f4812"}, {"name":"gitlab-experiment","version":"0.8.0","platform":"ruby","checksum":"b4e2f73e0af19cdd899a745f5a846c1318d44054e068a8f4ac887f6b1017d3f9"}, {"name":"gitlab-fog-azure-rm","version":"1.8.0","platform":"ruby","checksum":"e4f24b174b273b88849d12fbcfecb79ae1c09f56cbd614998714c7f0a81e6c28"}, {"name":"gitlab-labkit","version":"0.34.0","platform":"ruby","checksum":"ca5c504201390cd07ba1029e6ca3059f4e2e6005eb121ba8a103af1e166a3ecd"}, diff --git a/Gemfile.lock b/Gemfile.lock index ee916862c50..124ac9ceed8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -644,8 +644,8 @@ GEM terminal-table (>= 1.5.1) gitlab-chronic (0.10.5) numerizer (~> 0.2) - gitlab-dangerfiles (3.13.0) - danger (>= 8.4.5) + gitlab-dangerfiles (4.0.0) + danger (>= 9.3.0) danger-gitlab (>= 8.0.0) rake gitlab-experiment (0.8.0) @@ -1818,7 +1818,7 @@ DEPENDENCIES gettext_i18n_rails_js (~> 1.3) gitaly (~> 16.3.0.pre.rc1) gitlab-chronic (~> 0.10.5) - gitlab-dangerfiles (~> 3.13.0) + gitlab-dangerfiles (~> 4.0.0) gitlab-experiment (~> 0.8.0) gitlab-fog-azure-rm (~> 1.8.0) gitlab-labkit (~> 0.34.0) diff --git a/app/models/x509_certificate.rb b/app/models/x509_certificate.rb index 7c2581b8bb2..90f3bd69c47 100644 --- a/app/models/x509_certificate.rb +++ b/app/models/x509_certificate.rb @@ -17,8 +17,6 @@ class X509Certificate < ApplicationRecord # rfc 5280 - 4.2.1.2 Subject Key Identifier validates :subject_key_identifier, presence: true, format: { with: Gitlab::Regex.x509_subject_key_identifier_regex } - # rfc 5280 - 4.1.2.6 Subject - validates :subject, presence: true # rfc 5280 - 4.1.2.6 Subject (subjectAltName contains the email address) validates :email, presence: true, format: { with: URI::MailTo::EMAIL_REGEXP } # rfc 5280 - 4.1.2.2 Serial number diff --git a/app/models/x509_issuer.rb b/app/models/x509_issuer.rb index 81491d8e507..769d56a9838 100644 --- a/app/models/x509_issuer.rb +++ b/app/models/x509_issuer.rb @@ -6,13 +6,16 @@ class X509Issuer < ApplicationRecord # rfc 5280 - 4.2.1.1 Authority Key Identifier validates :subject_key_identifier, presence: true, format: { with: Gitlab::Regex.x509_subject_key_identifier_regex } # rfc 5280 - 4.1.2.4 Issuer - validates :subject, presence: true # rfc 5280 - 4.2.1.13 CRL Distribution Points # cRLDistributionPoints extension using URI:http - validates :crl_url, presence: true, public_url: true + validates :crl_url, allow_nil: true, public_url: true def self.safe_create!(attributes) create_with(attributes) .safe_find_or_create_by!(subject_key_identifier: attributes[:subject_key_identifier]) end + + def self.with_crl_url + where.not(crl_url: nil) + end end diff --git a/app/presenters/projects/security/configuration_presenter.rb b/app/presenters/projects/security/configuration_presenter.rb index e1e729d28e3..f248652befc 100644 --- a/app/presenters/projects/security/configuration_presenter.rb +++ b/app/presenters/projects/security/configuration_presenter.rb @@ -25,7 +25,8 @@ module Projects auto_fix_enabled: autofix_enabled, can_toggle_auto_fix_settings: can_toggle_autofix, auto_fix_user_path: auto_fix_user_path, - security_training_enabled: project.security_training_available? + security_training_enabled: project.security_training_available?, + continuous_vulnerability_scans_enabled: continuous_vulnerability_scans_enabled } end @@ -95,6 +96,8 @@ module Projects def project_settings project.security_setting end + + def continuous_vulnerability_scans_enabled; end end end end diff --git a/app/workers/x509_issuer_crl_check_worker.rb b/app/workers/x509_issuer_crl_check_worker.rb index 58084405769..71d2c398ca7 100644 --- a/app/workers/x509_issuer_crl_check_worker.rb +++ b/app/workers/x509_issuer_crl_check_worker.rb @@ -18,7 +18,7 @@ class X509IssuerCrlCheckWorker def perform @logger = Gitlab::GitLogger.build - X509Issuer.all.find_each do |issuer| + X509Issuer.with_crl_url.find_each do |issuer| with_context(related_class: X509IssuerCrlCheckWorker) do update_certificates(issuer) end diff --git a/config/feature_flags/development/search_milestones_hide_archived_projects.yml b/config/feature_flags/development/search_milestones_hide_archived_projects.yml new file mode 100644 index 00000000000..859a92e5dec --- /dev/null +++ b/config/feature_flags/development/search_milestones_hide_archived_projects.yml @@ -0,0 +1,9 @@ +--- +name: search_milestones_hide_archived_projects +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/130937 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/424256 +milestone: '16.4' +type: development +group: group::global search +default_enabled: false + diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 7e037304d1d..2c0833ad5c7 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -593,6 +593,8 @@ - 1 - - search_wiki_elastic_delete_group_wiki - 1 +- - search_zoekt_default_branch_changed + - 1 - - search_zoekt_delete_project - 1 - - search_zoekt_namespace_indexer diff --git a/db/docs/batched_background_migrations/backfill_has_merge_request_of_vulnerability_reads.yml b/db/docs/batched_background_migrations/backfill_has_merge_request_of_vulnerability_reads.yml new file mode 100644 index 00000000000..c6bf73622f6 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_has_merge_request_of_vulnerability_reads.yml @@ -0,0 +1,6 @@ +--- +migration_job_name: BackfillHasMergeRequestOfVulnerabilityReads +description: Backfills has_merge_request column for vulnerability_reads table. +feature_category: database +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/130952 +milestone: 16.4 diff --git a/db/migrate/20230821101010_remove_crl_null.rb b/db/migrate/20230821101010_remove_crl_null.rb new file mode 100644 index 00000000000..16fa512a65a --- /dev/null +++ b/db/migrate/20230821101010_remove_crl_null.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveCrlNull < Gitlab::Database::Migration[2.1] + def up + change_column_null :x509_certificates, :subject, true + change_column_null :x509_issuers, :subject, true + change_column_null :x509_issuers, :crl_url, true + end + + def down + change_column_null :x509_certificates, :subject, false + change_column_null :x509_issuers, :subject, false + change_column_null :x509_issuers, :crl_url, false + end +end diff --git a/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb b/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb new file mode 100644 index 00000000000..fe939b4aaa0 --- /dev/null +++ b/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class QueueBackfillHasMergeRequestOfVulnerabilityReads < Gitlab::Database::Migration[2.1] + MIGRATION_NAME = 'BackfillHasMergeRequestOfVulnerabilityReads' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 200 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + def up + queue_batched_background_migration( + MIGRATION_NAME, + :vulnerability_reads, + :vulnerability_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION_NAME, :vulnerability_reads, :vulnerability_id, []) + end +end diff --git a/db/schema_migrations/20230821101010 b/db/schema_migrations/20230821101010 new file mode 100644 index 00000000000..32ce19db417 --- /dev/null +++ b/db/schema_migrations/20230821101010 @@ -0,0 +1 @@ +84586d94a586664bf049782d354b240998217fff131d3ab19b793da6333ee844 \ No newline at end of file diff --git a/db/schema_migrations/20230907155247 b/db/schema_migrations/20230907155247 new file mode 100644 index 00000000000..6d709e8c35c --- /dev/null +++ b/db/schema_migrations/20230907155247 @@ -0,0 +1 @@ +969028a44aa3e656595c2af113fab7a82f8f28514337b97bfb467a5c5550dfc3 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 5673abdd2be..0aa24e27b53 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -25375,7 +25375,7 @@ CREATE TABLE x509_certificates ( created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, subject_key_identifier character varying(255) NOT NULL, - subject character varying(512) NOT NULL, + subject character varying(512), email character varying(255) NOT NULL, serial_number bytea NOT NULL, certificate_status smallint DEFAULT 0 NOT NULL, @@ -25416,8 +25416,8 @@ CREATE TABLE x509_issuers ( created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, subject_key_identifier character varying(255) NOT NULL, - subject character varying(255) NOT NULL, - crl_url character varying(255) NOT NULL + subject character varying(255), + crl_url character varying(255) ); CREATE SEQUENCE x509_issuers_id_seq diff --git a/doc/administration/package_information/supported_os.md b/doc/administration/package_information/supported_os.md index eea4b8035b7..58e0d2a08c4 100644 --- a/doc/administration/package_information/supported_os.md +++ b/doc/administration/package_information/supported_os.md @@ -38,7 +38,7 @@ architecture. | Ubuntu 20.04 | GitLab CE / GitLab EE 13.2.0 | amd64, arm64 | [Ubuntu Install Documentation](https://about.gitlab.com/install/#ubuntu) | April 2025 | | | Ubuntu 22.04 | GitLab CE / GitLab EE 15.5.0 | amd64, arm64 | [Ubuntu Install Documentation](https://about.gitlab.com/install/#ubuntu) | April 2027 | | | Amazon Linux 2 | GitLab CE / GitLab EE 14.9.0 | amd64, arm64 | [Amazon Linux 2 Install Documentation](https://about.gitlab.com/install/#amazonlinux-2) | June 2025 | | -| Amazon Linux 2022 | GitLab CE / GitLab EE 15.9.0 | amd64, arm64 | [Amazon Linux 2022 Install Documentation](https://about.gitlab.com/install/#amazonlinux-2022) | October 2027 | | +| Amazon Linux 2023 | GitLab CE / GitLab EE 16.3.0 | amd64, arm64 | [Amazon Linux 2023 Install Documentation](https://about.gitlab.com/install/#amazonlinux-2023) | 2028 | | | Raspberry Pi OS (Buster) (formerly known as Raspbian Buster) | GitLab CE 12.2.0 | armhf | [Raspberry Pi Install Documentation](https://about.gitlab.com/install/#raspberry-pi-os) | 2024 | [Raspberry Pi Details](https://www.raspberrypi.com/news/new-old-functionality-with-raspberry-pi-os-legacy/) | | Raspberry Pi OS (Bullseye) | GitLab CE 15.5.0 | armhf | [Raspberry Pi Install Documentation](https://about.gitlab.com/install/#raspberry-pi-os) | 2026 | [Raspberry Pi Details](https://www.raspberrypi.com/news/raspberry-pi-os-debian-bullseye/) | diff --git a/doc/administration/silent_mode/index.md b/doc/administration/silent_mode/index.md index 84b994f00b6..b06f61d7efa 100644 --- a/doc/administration/silent_mode/index.md +++ b/doc/administration/silent_mode/index.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Silent Mode **(FREE SELF EXPERIMENT)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/9826) in GitLab 15.11. This feature is an [Experiment](../../policy/experiment-beta-support.md#experiment). +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/9826) in GitLab 15.11. This feature is an [Experiment](../../policy/experiment-beta-support.md#experiment). +> - Enabling and disabling Silent Mode through the web UI was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131090) in GitLab 16.4 Silent Mode allows you to suppress outbound communication, such as emails, from GitLab. Silent Mode is not intended to be used on environments which are in-use. Two use-cases are: @@ -19,7 +20,15 @@ Prerequisites: - You must have administrator access. -There are two ways to enable Silent Mode: +There are multiple ways to enable Silent Mode: + +- **Web UI** + + 1. On the left sidebar, select **Search or go to**. + 1. Select **Admin Area**. + 1. On the left sidebar, select **Settings > General**. + 1. Expand **Silent Mode**, and toggle **Enable Silent Mode**. + 1. Changes are saved immediately. - [**API**](../../api/settings.md): @@ -41,7 +50,15 @@ Prerequisites: - You must have administrator access. -There are two ways to disable Silent Mode: +There are multiple ways to disable Silent Mode: + +- **Web UI** + + 1. On the left sidebar, select **Search or go to**. + 1. Select **Admin Area**. + 1. On the left sidebar, select **Settings > General**. + 1. Expand **Silent Mode**, and toggle **Enable Silent Mode**. + 1. Changes are saved immediately. - [**API**](../../api/settings.md): @@ -61,6 +78,8 @@ It may take up to a minute to take effect. [Issue 405433](https://gitlab.com/git This section documents the current behavior of GitLab when Silent Mode is enabled. While Silent Mode is an Experiment, the behavior may change without notice. The work for the first iteration of Silent Mode is tracked by [Epic 9826](https://gitlab.com/groups/gitlab-org/-/epics/9826). +When Silent Mode is enabled, a banner is displayed at the top of the page for all users stating the setting is enabled and **All outbound communications are blocked.**. + ### Service Desk Incoming emails still raise issues, but the users who sent the emails to [Service Desk](../../user/project/service_desk/index.md) are not notified of issue creation or comments on their issues. diff --git a/doc/development/ai_features/duo_chat.md b/doc/development/ai_features/duo_chat.md index 33d50ab329d..5c7359eca9f 100644 --- a/doc/development/ai_features/duo_chat.md +++ b/doc/development/ai_features/duo_chat.md @@ -1,10 +1,10 @@ --- stage: AI-powered -group: AI Framework +group: Duo Chat info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Duo chat +# GitLab Duo Chat ## Set up GitLab Duo Chat @@ -98,7 +98,7 @@ To add a new tool: The key things to keep in mind are properly instructing the large language model through prompts and tool descriptions, keeping tools self-sufficient, and returning responses to the zero-shot agent. With some trial and error on prompts, -adding new tools can expand the capabilities of the chat feature. +adding new tools can expand the capabilities of the Chat feature. There are available short [videos](https://www.youtube.com/playlist?list=PL05JrBw4t0KoOK-bm_bwfHaOv-1cveh8i) covering this topic. @@ -130,10 +130,10 @@ make sure a new fixture is generated and committed together with the change. ## GraphQL Subscription -The GraphQL Subscription for chat behaves slightly different because it's user-centric. A user could have the chat open on multiple browser tabs, or also on their IDE. +The GraphQL Subscription for Chat behaves slightly different because it's user-centric. A user could have Chat open on multiple browser tabs, or also on their IDE. We therefore need to broadcast messages to multiple clients to keep them in sync. The `aiAction` mutation with the `chat` action behaves the following: -1. All complete chat messages (including messages from the user) are broadcasted with the `userId` and the `resourceId` from the mutation as identifier, ignoring the `clientSubscriptionId`. -1. Chunks from streamed chat messages are broadcasted with the `userId`, `resourceId`, and `clientSubscriptionId` as identifier. +1. All complete Chat messages (including messages from the user) are broadcasted with the `userId` and the `resourceId` from the mutation as identifier, ignoring the `clientSubscriptionId`. +1. Chunks from streamed Chat messages are broadcasted with the `userId`, `resourceId`, and `clientSubscriptionId` as identifier. To truly sync messages between all clients of a user, we need to remove the `resourceId` as well, which will be fixed by [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/420296). diff --git a/doc/development/database/index.md b/doc/development/database/index.md index 284c04d5f91..70681994229 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -119,3 +119,4 @@ including the major methods: - [Maintenance operations](maintenance_operations.md) - [Update multiple database objects](setting_multiple_values.md) +- [Batch iteration in a tree hierarchy proof of concept](poc_tree_iterator.md) diff --git a/doc/development/database/poc_tree_iterator.md b/doc/development/database/poc_tree_iterator.md new file mode 100644 index 00000000000..453f77f0cde --- /dev/null +++ b/doc/development/database/poc_tree_iterator.md @@ -0,0 +1,475 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Batch iteration in a tree hierarchy (proof of concept) + +The group hierarchy in GitLab is represented with a tree, where the root element +is the top-level namespace, and the child elements are the subgroups or the +recently introduced `Namespaces::ProjectNamespace` records. + +The tree is implemented in the `namespaces` table ,via the `parent_id` column. +The column points to the parent namespace record. The top level namespace has no +`parent_id`. + +Partial hierarchy of `gitlab-org`: + +```mermaid +flowchart TD + A("gitlab-org (9979)") --- B("quality (2750817)") + B --- C("engineering-productivity (16947798)") + B --- D("performance-testing (9453799)") + A --- F("charts (5032027)") + A --- E("ruby (14018648)") +``` + +Efficiently iterating over the group hierarchy has several potential use cases. +This is true especially in background jobs, which need to perform queries on the group hierarchy, +where stable and safe execution is more important than fast runtime. Batch iteration +requires more network round-trips, but each batch provides similar performance +characteristics. + +A few examples: + +- For each subgroup, do something. +- For each project in the hierarchy, do something. +- For each issue in the hierarchy, do something. + +## Problem statement + +A group hierarchy could grow so big that a single query would not be able to load +it in time. The query would fail with statement timeout error. + +Addressing scalability issues related to very large groups requires us to store +the same data in different formats (de-normalization). However, if we're unable +to load the group hierarchy, then de-normalization could not be implemented. + +One de-normalization technique would be to store all descendant group IDs for a +given group. This would speed up queries where we need to load the group and its +subgroups. Example: + +```mermaid +flowchart TD + A(1) --- B(2) + A --- C(3) + C --- D(4) +``` + +| GROUP_ID | DESCENDANT_GROUP_IDS | +|----------|------------------------| +| 1 | `[2,3,4]` | +| 2 | `[]` | +| 3 | `[4]` | +| 4 | `[]` | + +With this structure, determining all the subgroups would require us to read only +one row from the database, instead of 4 rows. For a hierarchy as big as 1000 groups, +this could make a huge difference. + +The reading of the hierarchy problem is solved with this de-normalization. However, +we still need to find a way to persist this data in a table. Because a group and +its hierarchy could grow very large, we cannot expect a single query to work here. + +```sql +SELECT id FROM namespaces WHERE traversal_ids && ARRAY[9970] +``` + +The query above could time out for large groups, so we need to process the data in batches. + +Implementing batching logic in a tree is not something we've looked at before, +and it's fairly complex to implement. An `EachBatch` or `find_in_batches` based +solution would not work because: + +- The data (group IDs) are not sorted in the hierarchy. +- Groups in sub groups don't know about the top-level group ID. + +## Algorithm + +The batching query is implemented as a recursive CTE SQL query, where one batch +would read a maximum of N rows. Due to the tree structure, reading N rows might +not necessarily mean that we're reading N group IDs. If the tree is structured in +a non-optimal way, a batch could return less (but never more) group IDs. + +The query implements a [depth-first](https://en.wikipedia.org/wiki/Depth-first_search) +tree walking logic, where the DB scans the first branch of the tree until the leaf +element. We're implementing depth-first algorithm because, when a batch is finished, +the query must return enough information for the next batch (cursor). In GitLab, +we limit the depth of the tree to 20, which means that in the worst case, the +query would return a cursor containing 19 elements. + +Implementing a [breadth-first](https://en.wikipedia.org/wiki/Breadth-first_search) +tree walking algorithm would be impractical, because a group can have unbounded +number of descendants, thus we might end up with a huge cursor. + +1. Create an initializer row that contains: + 1. The currently processed group ID (top-level group ID) + 1. Two arrays (tree depth and the collected IDs) + 1. A counter for tracking the number of row reads in the query. +1. Recursively process the row and do one of the following (whenever the condition matches): + - Load the first child namespace and update the currently processed namespace + ID if we're not at the leaf node. (Walking down a branch) + - Load the next namespace record on the current depth if there are any rows left. + - Walk up one node and process rows at one level higher. +1. Continue the processing until the number of reads reaches our `LIMIT` (batch size). +1. Find the last processed row which contains the data for the cursor, and all the collected record IDs. + +```sql +WITH RECURSIVE result AS ( + ( + SELECT + 9970 AS current_id, /* current namespace id we're processing */ + ARRAY[9970]::int[] AS depth, /* cursor */ + ARRAY[9970]::int[] AS ids, /* collected ids */ + 1::bigint AS reads, + 'initialize' AS action + ) UNION ALL + ( + WITH cte AS ( /* trick for referencing the result cte multiple times */ + select * FROM result + ) + SELECT * FROM ( + ( + SELECT /* walk down the branch */ + namespaces.id, + cte.depth || namespaces.id, + cte.ids || namespaces.id, + cte.reads + 1, + 'walkdown' + FROM namespaces, cte + WHERE + namespaces.parent_id = cte.current_id + ORDER BY namespaces.id ASC + LIMIT 1 + ) UNION ALL + ( + SELECT /* find next element on the same level */ + namespaces.id, + cte.depth[:array_length(cte.depth, 1) - 1] || namespaces.id, + cte.ids || namespaces.id, + cte.reads + 1, + 'next' + FROM namespaces, cte + WHERE + namespaces.parent_id = cte.depth[array_length(cte.depth, 1) - 1] AND + namespaces.id > cte.depth[array_length(cte.depth, 1)] + ORDER BY namespaces.id ASC + LIMIT 1 + ) UNION ALL + ( + SELECT /* jump up one node when finished with the current level */ + cte.current_id, + cte.depth[:array_length(cte.depth, 1) - 1], + cte.ids, + cte.reads + 1, + 'jump' + FROM cte + WHERE cte.depth <> ARRAY[]::int[] + LIMIT 1 + ) + ) next_row LIMIT 1 + ) +) +SELECT current_id, depth, ids, action +FROM result +``` + +```plaintext + current_id | depth | ids | action +------------+--------------+------------------------+------------ + 24 | {24} | {24} | initialize + 25 | {24,25} | {24,25} | walkdown + 26 | {24,26} | {24,25,26} | next + 112 | {24,112} | {24,25,26,112} | next + 113 | {24,113} | {24,25,26,112,113} | next + 114 | {24,113,114} | {24,25,26,112,113,114} | walkdown + 114 | {24,113} | {24,25,26,112,113,114} | jump + 114 | {24} | {24,25,26,112,113,114} | jump + 114 | {} | {24,25,26,112,113,114} | jump +``` + +NOTE: +Using this query to find all the namespace IDs in a group hierarchy is likely slower +than other querying methods, such as the current `self_and_descendants` implementation +based on the `traversal_ids` column. The query above should be only used when +implementing batch iteration over the group hierarchy. + +Rudimentary batching implementation in Ruby: + +```ruby +class NamespaceEachBatch + def initialize(namespace_id:, cursor: nil) + @namespace_id = namespace_id + @cursor = cursor || { current_id: namespace_id, depth: [namespace_id] } + end + + def each_batch(of: 500) + current_cursor = cursor.dup + + first_iteration = true + loop do + new_cursor, ids = load_batch(cursor: current_cursor, of: of, first_iteration: first_iteration) + first_iteration = false + current_cursor = new_cursor + + yield ids + + break if new_cursor[:depth].empty? + end + end + + private + + # yields array of namespace ids + def load_batch(cursor:, of:, first_iteration: false) + recursive_cte = Gitlab::SQL::RecursiveCTE.new(:result, + union_args: { remove_order: false, remove_duplicates: false }) + + ids = first_iteration ? namespace_id.to_s : "" + + recursive_cte << Namespace.select( + Arel.sql(Integer(cursor.fetch(:current_id)).to_s).as('current_id'), + Arel.sql("ARRAY[#{cursor.fetch(:depth).join(',')}]::int[]").as('depth'), + Arel.sql("ARRAY[#{ids}]::int[]").as('ids'), + Arel.sql("1::bigint AS count") + ).from('(VALUES (1)) AS does_not_matter').limit(1) + + cte = Gitlab::SQL::CTE.new(:cte, Namespace.select('*').from('result')) + + union_query = Namespace.with(cte.to_arel).from_union( + walk_down, + next_elements, + up_one_level, + remove_duplicates: false, + remove_order: false + ).select('current_id', 'depth', 'ids', 'count').limit(1) + + recursive_cte << union_query + + scope = Namespace.with + .recursive(recursive_cte.to_arel) + .from(recursive_cte.alias_to(Namespace.arel_table)) + .limit(of) + row = Namespace.from(scope.arel.as('namespaces')).order(count: :desc).limit(1).first + + [ + { current_id: row[:current_id], depth: row[:depth] }, + row[:ids] + ] + end + + attr_reader :namespace_id, :cursor + + def walk_down + Namespace.select( + Arel.sql('namespaces.id').as('current_id'), + Arel.sql('cte.depth || namespaces.id').as('depth'), + Arel.sql('cte.ids || namespaces.id').as('ids'), + Arel.sql('cte.count + 1').as('count') + ).from('cte, LATERAL (SELECT id FROM namespaces WHERE parent_id = cte.current_id ORDER BY id LIMIT 1) namespaces') + end + + def next_elements + Namespace.select( + Arel.sql('namespaces.id').as('current_id'), + Arel.sql('cte.depth[:array_length(cte.depth, 1) - 1] || namespaces.id').as('depth'), + Arel.sql('cte.ids || namespaces.id').as('ids'), + Arel.sql('cte.count + 1').as('count') + ).from('cte, LATERAL (SELECT id FROM namespaces WHERE namespaces.parent_id = cte.depth[array_length(cte.depth, 1) - 1] AND namespaces.id > cte.depth[array_length(cte.depth, 1)] ORDER BY id LIMIT 1) namespaces') + end + + def up_one_level + Namespace.select( + Arel.sql('cte.current_id').as('current_id'), + Arel.sql('cte.depth[:array_length(cte.depth, 1) - 1]').as('depth'), + Arel.sql('cte.ids').as('ids'), + Arel.sql('cte.count + 1').as('count') + ).from('cte') + .where('cte.depth <> ARRAY[]::int[]') + .limit(1) + end +end + +iterator = NamespaceEachBatch.new(namespace_id: 9970) +all_ids = [] +iterator.each_batch do |ids| + all_ids.concat(ids) +end + +# Test +puts all_ids.count +puts all_ids.sort == Namespace.where('traversal_ids && ARRAY[9970]').pluck(:id).sort +``` + +Example batch query: + +```sql +SELECT + "namespaces".* +FROM ( WITH RECURSIVE "result" AS (( + SELECT + 15847356 AS current_id, + ARRAY[9970, + 12061481, + 12128714, + 12445111, + 15847356]::int[] AS depth, + ARRAY[]::int[] AS ids, + 1::bigint AS count + FROM ( + VALUES (1)) AS does_not_matter + LIMIT 1) + UNION ALL ( WITH "cte" AS MATERIALIZED ( + SELECT + * + FROM + result +) + SELECT + current_id, + depth, + ids, + count + FROM (( + SELECT + namespaces.id AS current_id, + cte.depth || namespaces.id AS depth, + cte.ids || namespaces.id AS ids, + cte.count + 1 AS count + FROM + cte, + LATERAL ( + SELECT + id + FROM + namespaces + WHERE + parent_id = cte.current_id + ORDER BY + id + LIMIT 1 +) namespaces +) + UNION ALL ( + SELECT + namespaces.id AS current_id, + cte.depth[:array_length( + cte.depth, 1 +) - 1] || namespaces.id AS depth, + cte.ids || namespaces.id AS ids, + cte.count + 1 AS count + FROM + cte, + LATERAL ( + SELECT + id + FROM + namespaces + WHERE + namespaces.parent_id = cte.depth[array_length( + cte.depth, 1 +) - 1] + AND namespaces.id > cte.depth[array_length( + cte.depth, 1 +)] + ORDER BY + id + LIMIT 1 +) namespaces +) + UNION ALL ( + SELECT + cte.current_id AS current_id, + cte.depth[:array_length( + cte.depth, 1 +) - 1] AS depth, + cte.ids AS ids, + cte.count + 1 AS count + FROM + cte + WHERE ( + cte.depth <> ARRAY[]::int[] +) + LIMIT 1 +) +) namespaces + LIMIT 1 +)) +SELECT + "namespaces".* +FROM + "result" AS "namespaces" +LIMIT 500) namespaces +ORDER BY + "count" DESC +LIMIT 1 +``` + +Execution plan: + +```plaintext + Limit (cost=16.36..16.36 rows=1 width=76) (actual time=436.963..436.970 rows=1 loops=1) + Buffers: shared hit=3721 read=423 dirtied=8 + I/O Timings: read=412.590 write=0.000 + -> Sort (cost=16.36..16.39 rows=11 width=76) (actual time=436.961..436.968 rows=1 loops=1) + Sort Key: namespaces.count DESC + Sort Method: top-N heapsort Memory: 27kB + Buffers: shared hit=3721 read=423 dirtied=8 + I/O Timings: read=412.590 write=0.000 + -> Limit (cost=15.98..16.20 rows=11 width=76) (actual time=0.005..436.394 rows=500 loops=1) + Buffers: shared hit=3718 read=423 dirtied=8 + I/O Timings: read=412.590 write=0.000 + CTE result + -> Recursive Union (cost=0.00..15.98 rows=11 width=76) (actual time=0.003..432.924 rows=500 loops=1) + Buffers: shared hit=3718 read=423 dirtied=8 + I/O Timings: read=412.590 write=0.000 + -> Limit (cost=0.00..0.01 rows=1 width=76) (actual time=0.002..0.003 rows=1 loops=1) + I/O Timings: read=0.000 write=0.000 + -> Result (cost=0.00..0.01 rows=1 width=76) (actual time=0.001..0.002 rows=1 loops=1) + I/O Timings: read=0.000 write=0.000 + -> Limit (cost=0.76..1.57 rows=1 width=76) (actual time=0.862..0.862 rows=1 loops=499) + Buffers: shared hit=3718 read=423 dirtied=8 + I/O Timings: read=412.590 write=0.000 + CTE cte + -> WorkTable Scan on result (cost=0.00..0.20 rows=10 width=76) (actual time=0.000..0.000 rows=1 loops=499) + I/O Timings: read=0.000 write=0.000 + -> Append (cost=0.56..17.57 rows=21 width=76) (actual time=0.862..0.862 rows=1 loops=499) + Buffers: shared hit=3718 read=423 dirtied=8 + I/O Timings: read=412.590 write=0.000 + -> Nested Loop (cost=0.56..7.77 rows=10 width=76) (actual time=0.675..0.675 rows=0 loops=499) + Buffers: shared hit=1693 read=357 dirtied=1 + I/O Timings: read=327.812 write=0.000 + -> CTE Scan on cte (cost=0.00..0.20 rows=10 width=76) (actual time=0.001..0.001 rows=1 loops=499) + I/O Timings: read=0.000 write=0.000 + -> Limit (cost=0.56..0.73 rows=1 width=4) (actual time=0.672..0.672 rows=0 loops=499) + Buffers: shared hit=1693 read=357 dirtied=1 + I/O Timings: read=327.812 write=0.000 + -> Index Only Scan using index_namespaces_on_parent_id_and_id on public.namespaces namespaces_1 (cost=0.56..5.33 rows=29 width=4) (actual time=0.671..0.671 rows=0 loops=499) + Index Cond: (namespaces_1.parent_id = cte.current_id) + Heap Fetches: 7 + Buffers: shared hit=1693 read=357 dirtied=1 + I/O Timings: read=327.812 write=0.000 + -> Nested Loop (cost=0.57..9.45 rows=10 width=76) (actual time=0.208..0.208 rows=1 loops=442) + Buffers: shared hit=2025 read=66 dirtied=7 + I/O Timings: read=84.778 write=0.000 + -> CTE Scan on cte cte_1 (cost=0.00..0.20 rows=10 width=72) (actual time=0.000..0.000 rows=1 loops=442) + I/O Timings: read=0.000 write=0.000 + -> Limit (cost=0.57..0.89 rows=1 width=4) (actual time=0.203..0.203 rows=1 loops=442) + Buffers: shared hit=2025 read=66 dirtied=7 + I/O Timings: read=84.778 write=0.000 + -> Index Only Scan using index_namespaces_on_parent_id_and_id on public.namespaces namespaces_2 (cost=0.57..3.77 rows=10 width=4) (actual time=0.201..0.201 rows=1 loops=442) + Index Cond: ((namespaces_2.parent_id = (cte_1.depth)[(array_length(cte_1.depth, 1) - 1)]) AND (namespaces_2.id > (cte_1.depth)[array_length(cte_1.depth, 1)])) + Heap Fetches: 35 + Buffers: shared hit=2025 read=66 dirtied=6 + I/O Timings: read=84.778 write=0.000 + -> Limit (cost=0.00..0.03 rows=1 width=76) (actual time=0.003..0.003 rows=1 loops=59) + I/O Timings: read=0.000 write=0.000 + -> CTE Scan on cte cte_2 (cost=0.00..0.29 rows=9 width=76) (actual time=0.002..0.002 rows=1 loops=59) + Filter: (cte_2.depth <> '{}'::integer[]) + Rows Removed by Filter: 0 + I/O Timings: read=0.000 write=0.000 + -> CTE Scan on result namespaces (cost=0.00..0.22 rows=11 width=76) (actual time=0.005..436.240 rows=500 loops=1) + Buffers: shared hit=3718 read=423 dirtied=8 + I/O Timings: read=412.590 write=0.000 +``` diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 8c0f7faab28..af40fd8b945 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -420,9 +420,18 @@ The actor is a second parameter of the `Feature.enabled?` call. The same actor type must be used consistently for all invocations of `Feature.enabled?`. ```ruby +# Bad Feature.enabled?(:feature_flag, project) Feature.enabled?(:feature_flag, group) Feature.enabled?(:feature_flag, user) + +# Good +Feature.enabled?(:feature_flag, group_a) +Feature.enabled?(:feature_flag, group_b) + +# Also good - using separate flags for each actor type +Feature.enabled?(:feature_flag_group, group) +Feature.enabled?(:feature_flag_user, user) ``` See [Feature flags in the development of GitLab](controls.md#process) for details on how to use ChatOps diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index 65000965f7c..0ea49ff387f 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -6,12 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Design management **(FREE ALL)** -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/660) in GitLab 12.2. -> - Support for SVGs [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12771) in GitLab 12.4. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212566) from GitLab Premium to GitLab Free in 13.0. -> - Design Management section in issues [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223193) in GitLab 13.2, with a feature flag named `design_management_moved`. In earlier versions, designs were displayed in a separate tab. -> - Design Management section in issues [feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/223197) for new displays in GitLab 13.4. - With Design Management you can upload design assets (including wireframes and mockups) to GitLab issues and keep them stored in a single place. Product designers, product managers, and engineers can collaborate on designs with a single source of truth. @@ -68,7 +62,7 @@ Support for PDF files is tracked in [issue 32811](https://gitlab.com/gitlab-org/ - Design Management data isn't deleted when: - [A project is destroyed](https://gitlab.com/gitlab-org/gitlab/-/issues/13429). - [An issue is deleted](https://gitlab.com/gitlab-org/gitlab/-/issues/13427). -- In GitLab 12.7 and later, Design Management data [can be replicated](../../../administration/geo/replication/datatypes.md#limitations-on-replicationverification) +- Design Management data [can be replicated](../../../administration/geo/replication/datatypes.md#limitations-on-replicationverification) and in GitLab 16.1 and later it can be [verified by Geo as well](https://gitlab.com/gitlab-org/gitlab/-/issues/355660). ## View a design @@ -105,9 +99,6 @@ a blue icon (**{file-modified-solid}**) is displayed. ### Zoom in on a design -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13217) in GitLab 12.7. -> - Ability to drag a zoomed image to move it [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/197324) in GitLab 12.10. - You can explore a design in more detail by zooming in and out of the image: - To control the amount of zoom, select plus (`+`) and minus (`-`) @@ -123,7 +114,7 @@ To move around the image while zoomed in, drag the image. Prerequisites: - You must have at least the Developer role for the project. -- In GitLab 13.1 and later, the names of the uploaded files must be no longer than 255 characters. +- The names of the uploaded files must be no longer than 255 characters. To add a design to an issue: @@ -162,6 +153,7 @@ Prerequisites: To do so, [add a design](#add-a-design-to-an-issue) with the same filename. To browse all the design versions, use the dropdown list at the top of the **Designs** section. +It's shown as either **Showing latest version** or **Showing version #N**. ### Skipped designs @@ -171,14 +163,19 @@ When designs are skipped, a warning message is displayed. ## Archive a design -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11089) in GitLab 12.4. -> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/220964) the button from "Delete" to "Archive" in GitLab 13.3. - You can archive individual designs or select a few of them to archive at once. +Archived designs are not permanently lost. +You can browse [previous versions](#add-a-new-version-of-a-design). + +When you archive a design, its URL changes. +If the design isn't available in the latest version, you can link to it only with the version in the +URL. + Prerequisites: - You must have at least the Developer role for the project. +- You can archive only the latest version of a design. To archive a single design: @@ -191,15 +188,10 @@ To archive multiple designs at once: 1. Select the checkboxes on the designs you want to archive. 1. Select **Archive selected**. -NOTE: -Only the latest version of the designs can be archived. -Archived designs are not permanently lost. You can browse -[previous versions](#add-a-new-version-of-a-design). +## Markdown and rich text editors for descriptions - -## Markdown and rich text editors for descriptions + to "Add a design to an issue", update that topic, and delete this one. --> > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/388449) in GitLab 16.1 [with a flag](../../../administration/feature_flags.md) named `content_editor_on_issues`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/375172) in GitLab 16.2. @@ -213,30 +205,28 @@ It's the same editor you use for comments across GitLab. ## Reorder designs -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34382) in GitLab 13.3. - You can change the order of designs by dragging them to a new position. ## Add a comment to a design -> Adjusting a pin's position [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34353) in GitLab 12.8. - You can start [discussions](../../discussions/index.md) on uploaded designs. To do so: - 1. Go to an issue. 1. Select the design. + + 1. Click or tap the image. A pin is created in that spot, identifying the discussion's location. + 1. Enter your message. 1. Select **Comment**. - -You can adjust a pin's position by dragging it around the image. You can use this when your design's -layout has changed, or when you want to move a pin to add a new one in its place. +You can adjust a pin's position by dragging it around the image. +Use this when your design's layout has changed, or to move a pin so you can add a new one in +its place. New discussion threads get different pin numbers, which you can use to refer to them. -In GitLab 12.5 and later, new discussions are output to the issue activity, +New discussions are output to the issue activity, so that everyone involved can participate in the discussion. ## Delete a comment from a design @@ -254,8 +244,6 @@ To delete a comment from a design: ## Resolve a discussion thread on a design -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13049) in GitLab 13.1. - When you're done discussing part of a design, you can resolve the discussion thread. To mark a thread as resolved or unresolved, either: @@ -271,16 +259,10 @@ To revisit a resolved discussion, expand **Resolved Comments** below the visible ## Add a to-do item for a design -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198439) in GitLab 13.4. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/245074) in GitLab 13.5. - To add a [to-do item](../../todos.md) for a design, select **Add a to do** on the design sidebar. ## Refer to a design in Markdown -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217160) in GitLab 13.1. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/258662) in GitLab 13.5. - To refer to a design in a [Markdown](../../markdown.md) text box in GitLab, for example, in a comment or description, paste its URL. It's then displayed as a short reference. @@ -296,9 +278,6 @@ It's rendered as: ## Design activity records -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33051) in GitLab 13.1. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/225205) in GitLab 13.2. - User activity events on designs (creation, deletion, and updates) are tracked by GitLab and displayed on the [user profile](../../profile/index.md#access-your-user-profile), [group](../../group/manage.md#view-group-activity), @@ -306,8 +285,6 @@ and [project](../working_with_projects.md#view-project-activity) activity pages. ## GitLab-Figma plugin -> [Introduced](https://gitlab.com/gitlab-org/gitlab-figma-plugin/-/issues/2) in GitLab 13.2. - You can use the GitLab-Figma plugin to upload your designs from Figma directly to your issues in GitLab. @@ -315,3 +292,26 @@ To use the plugin in Figma, install it from the [Figma Directory](https://www.fi and connect to GitLab through a personal access token. For more information, see the [plugin documentation](https://gitlab.com/gitlab-org/gitlab-figma-plugin/-/wikis/home). + +## Troubleshooting + +When working with Design Management, you might encounter the following issues. + +### Could not find design + +You might get an error that states `Could not find design`. + +This issue occurs when a design has been [archived](#archive-a-design), +so it's not available in the latest version, and the link you've followed doesn't specify a version. + +When you archive a design, its URL changes. +If the design isn't available in the latest version, it can be linked to only with the version in the URL. + +For example, `https://gitlab.example.com/mygroup/myproject/-/issues/123456/designs/menu.png?version=503554`. +You can no longer access `menu.png` with `https://gitlab.example.com/mygroup/myproject/-/issues/123456/designs/menu.png`. + +The workaround is to select one of the previous versions from the dropdown list at the top of the +**Designs** section. +It's shown as either **Showing latest version** or **Showing version #N**. + +Issue [392540](https://gitlab.com/gitlab-org/gitlab/-/issues/392540) tracks improving this behavior. diff --git a/doc/user/project/repository/signed_commits/x509.md b/doc/user/project/repository/signed_commits/x509.md index 2d97925faa3..17767cbd8f4 100644 --- a/doc/user/project/repository/signed_commits/x509.md +++ b/doc/user/project/repository/signed_commits/x509.md @@ -43,8 +43,8 @@ GitLab checks certificate revocation lists on a daily basis with a background wo ## Limitations -- Self-signed certificates without `authorityKeyIdentifier`, - `subjectKeyIdentifier`, and `crlDistributionPoints` are not supported. We +- Certificates without `authorityKeyIdentifier`, + `subjectKeyIdentifier`, and `crlDistributionPoints` display as **Unverified**. We recommend using certificates from a PKI that are in line with [RFC 5280](https://www.rfc-editor.org/rfc/rfc5280). - If you have more than one email in the Subject Alternative Name list in diff --git a/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb new file mode 100644 index 00000000000..7f0ee54012b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Sets the `has_merge_request` of the existing `vulnerability_reads` records + class BackfillHasMergeRequestOfVulnerabilityReads < BatchedMigrationJob + operation_name :set_has_merge_request + feature_category :database + + UPDATE_SQL = <<~SQL + UPDATE + vulnerability_reads + SET + has_merge_request = true + FROM + (%s) as sub_query + WHERE + vulnerability_reads.vulnerability_id = sub_query.vulnerability_id + SQL + + def perform + each_sub_batch do |sub_batch| + update_query = update_query_for(sub_batch) + + connection.execute(update_query) + end + end + + private + + def update_query_for(sub_batch) + subquery = sub_batch.joins(" + INNER JOIN vulnerability_merge_request_links ON + vulnerability_reads.vulnerability_id = + vulnerability_merge_request_links.vulnerability_id") + + format(UPDATE_SQL, subquery: subquery.to_sql) + end + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index fe82129fb82..35e01101b3b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -235,22 +235,21 @@ module Gitlab # Filter milestones by authorized projects. # For performance reasons project_id is being plucked # to be used on a smaller query. - # - # rubocop: disable CodeReuse/ActiveRecord def filter_milestones_by_project(milestones) - project_ids = - milestones.where(project_id: project_ids_relation) - .select(:project_id).distinct - .pluck(:project_id) + candidate_project_ids = project_ids_relation + + if Feature.enabled?(:search_milestones_hide_archived_projects, current_user) && !filters[:include_archived] + candidate_project_ids = candidate_project_ids.non_archived + end + + project_ids = milestones.of_projects(candidate_project_ids).select(:project_id).distinct.pluck(:project_id) # rubocop: disable CodeReuse/ActiveRecord return Milestone.none if project_ids.nil? - authorized_project_ids_relation = - Project.where(id: project_ids).ids_with_issuables_available_for(current_user) + authorized_project_ids_relation = Project.id_in(project_ids).ids_with_issuables_available_for(current_user) - milestones.where(project_id: authorized_project_ids_relation) + milestones.of_projects(authorized_project_ids_relation) end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index 649e5379927..74d861c8701 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -39,6 +39,8 @@ module Gitlab return :unverified if x509_certificate.nil? || x509_certificate.revoked? || + certificate_subject.nil? || + certificate_crl.nil? || !verified_signature || signed_by_user.nil? @@ -127,6 +129,7 @@ module Gitlab def certificate_crl extension = get_certificate_extension('crlDistributionPoints') + return if extension.nil? crl_url = nil @@ -185,7 +188,7 @@ module Gitlab end def x509_issuer - return if verified_signature.nil? || issuer_subject_key_identifier.nil? || certificate_crl.nil? + return if verified_signature.nil? || issuer_subject_key_identifier.nil? || certificate_issuer.nil? attributes = { subject_key_identifier: issuer_subject_key_identifier, diff --git a/locale/gitlab.pot b/locale/gitlab.pot index adb500cb1d0..7143ec4a4ab 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -43445,6 +43445,9 @@ msgstr "" msgid "Service usage data" msgstr "" +msgid "ServiceAccount|No more seats are available to create Service Account User" +msgstr "" + msgid "ServiceAccount|User does not have permission to create a service account in this namespace." msgstr "" diff --git a/spec/fixtures/ci_secure_files/sample.p12 b/spec/fixtures/ci_secure_files/sample.p12 index c74df26a8d4dfb83c1b014ace3e4aa5094abb694..84c7bf6a2f5278566cad4201b1a7c11a96d7b0ae 100644 GIT binary patch literal 3219 zcmai$X*d)N_r@7Bld)$XyF7}qjU}XPl_AEyD}^FUWQnX3V;6($8jKmq7RfG4mPFKy zU9!z%H;iUflZ2)>lY15Qb5sYutY*I zAn|G2469U?;Hw!M6+P(9cb^q~=6;_mJy`E8MQw01R44@3L0@u=R_!E8(~JDlGWnK= zcmNt&MNj|!*?#@%v~A{d^-H&|NU_KJWC^MWPf3%;Rcb%9*b3#PEYe&#U61nnQHBhq zlhYG>RmV4Ywp*#EJp1vT%Ju9!@=6iUliF7s(|@e+Ylxd_Glh&xhs(cKOx`_4(fs^# zJ(p&z{DUa2k6AQoLhmA{UoU&L1Tpe+w$+T`y5} zXvBM!(;ubbGx5LDx7e6SE!B%_{ec^2QaVB{ENA-1iX{@k%z)MN6*KMKuD=o6lPGU% zx@>qj>kIbDORN-ms|J!(Pllf;yM>3l^|pT8uMsu6mZnX!h|Xt&$jpwq8)|@YJ0o$d zn7ISOh0)hislE^LmE3sPf4!V|)RrBS>d`-db0c5Hv`y5MfQja@zPHr|l3O?3Sf6_= zZa`hVE1&G=e!Z{P=Fq&wn{qKvnEpaYh`FGSPeM-{$2dOVV&l{;h)OFF+;i+ObD=K1j$C=U>P_g{MooxX@)gLQQ0s+eQdf6N1PUXUeK3`L z=gI*jsNPCLy;2IV&ONQxv=(Vo@z?tn>om{_Wy<|?^}nr%Rf)$jl6{aU z&Bg}XK^so4JRXt6k0T1`qU*>o#IJq+nR}{`;7m|h8STBh)@%0q z9xmg$6-1U2{O*p%6nQkUFenl4MXXxY;go(WZ6E9&yE|k>BP_#TDCx$R72D9YcM6uH zez*0>u{5nlc}>p7zIQM*;gdZPLENZ!YkGheUg#4pSDiVTOA-1C7rpMK>#V6z!=1cw zb=H1F-8|P5m9#QN1rtGSTeFAq*U)db=7)1?-w)B!ZY_l_q*!9yO}VnmU_+lS_MNRF zIrGgudk7AijpQG93vSt49;!NkpYx~QnS{s(*kpKDnh-{A^FHTX^&;?kT>5phw_Brb zXQ3#w(vfE~*;cp00X`hcY z>@{0L_v;j^Ld}*3c+4PXWn&XquF9~Lta2UGuO(Wi{*cGx*9wu`w0R>Nj|&A#UHa#P zc_=@tYztf#83bsKd(%k_%xuDD~+1d;^zB}B zbkN?Mi*RHbPso`fSF`#DgU>L=6YmU_$Zgk-T#x);Jr_NU_R8yH^}w*wCo9)38#Gd6 z&yCu+Nz;^Gu3pxDYv50gb42OoX3HOcSD~LJxf-I@zAtx5C)NkB>_i^NUKWemds}pP zkf5LNp+-C$gJ@OyvGv5>74r~Jp9r+Sk!)~Vr!@@^+LXlxBt@up~_X7aXci%x5%IO&TBtD zvsyKDS|BtgVlHRD<5L3vef|5>Hvj^k3had~CP6z!9F<-Nr;4_ASE`cXVq0pik7g}y zDYvo#KeS3!3+4ooi?7-@Be-Iy!^Vey4=Da#!Ti-p$ssY|AXyCZoF~q%EN6y`_?_+=_&x4$uYWOGI%o9JMM~%#yoK#>9qhGU_fVr|0P~ zS?lobLc-sh0VW6}ddsoxjiVTSJNCf|BQ$M7n%vk6;LuCHe-r-*864@--ZIL6i=Zff zILzg>i7f2}|MY6fUP+cWRA@1b5E=4PZlztlq(c>m{N^$-U_2PLvH}uK2iifvT$wZS^Em@o zeL%D-A413v3NgB#EBuZix~sikYeeZjHpSg5&E!N2^>L_V)#VzVxP4n-P9JonfOnK>zIAGJiiI~zg@!jEhl3ma&Be+vk`tzCQn|bJ( zrUbX6=S4qdoz=GD8v?6dR+RDDiOqF2Pqh9+8j|^>DNMg(`R97}HznK)h!T zQdt@<^-sV;_>}p9y!hxaf|wCrt~-C7IourVWS~1x2iJ!y!`c6N=K+i$2A~9=oQg#F sR?e}P!%-|*kMb$tFH^tA(0EYbs;zNbejyW(IrBS%-!^Co_TQ-e4@%4a?f?J) literal 3352 zcmZvdc{tQ<+sA*yAj?>WB1`r?jGZQ-lr4-cYsQ|X#1Lc2GPbN`t&n}}A=#G@MYg!@ zJBdMPn8^~uqx*TD_r2fac(3a?&g*-8uj@MhJ3n7oC>e+x7D`Di7Nr$+00B{fazh0| z!5H#&Sm^(6V<@s=7>W!^+W-8ZqXCiSV!*#)81Oz!gp%gJ_IX=&FiS@7Sy0*GT3G?Z zt5fP|yB6}(-X~;aU;vB(_ikx7OK#%|k>& zVU1Vt$>R`i*4a?4!b^iq7T#bltd)@9&# zhFr}JnVwQsSPxdAbj}KuCTF~0m`Dp^=F`-wLJ{1HBNR4Pmhg*044cSA=GZ_&jA&2h zcu*fY^XAG-#jJH_M96G%@5p)O)(JTUcyz2En3M`#VDv(?SWO9&q*sb^24tUEmHeJV zR@4pA+go_TU!ERl`o&GPwqpA-wNQ5}N(%51i6ZE<;frxqNIu=WCG>^H zB$4*3sIk3cKT@p{9@S_3kWjQtZOe9?ni1*S<+r=B7UFI+AQvugex$ui)M=Yu@JUI3 zfH_Mge6bE`dv8W!xaIeXf}#^qpU`~~$tAUW;G1GBFep_~E`P%mYh*YFF+YRa}8 z`Ik3(9k^!UYFK4n8}|k37#-6Gvwnp!d97RDTG|7vn%PZ;6CQA1zfxIr$Ld>a`^XgkxFZa;W!)7S`il* zSXYdTP5$)gvSMg{dxO;+i%s?HvT1E8EiGn$y#Ah_N(+MXX8|)opx26i{_JO%KS?UH`5nO;&$FC6J-?}wKIgUZcurVAxq`|qEq=$GE3R>lOm=%jEl16NC}c0} z^~kcW$k$BjjbXPd*yZ)t`Eac-;pRe=jKa0PeUA^YR___(lm5*5ZwGw|LG$GGY)8@c z@=fj@3JDG#$O%6f18VMAz*=Y=+C$#X&b2_S3Ka6 zVx+DYG+FS$Ni-v3wuJO8B3M|7#No-*>KWSX!z*7oq*BAi2!lOHyZI<|ihOFDHmqqa z!(9w5&tvQIZW67XZarEBN9$SEtLFhMN_E&Q_#$GxA0;2PSQ=bt%dz^ zq0=R_6lW=2G-F(@B1F6t-)M_U<6W4({)GvB$b{%3F07q?sw)+hQw3l8^nyf@g4|oY zmkk&M-Z5EH`k}j)RAMk%d$ueez3OJ#h&{8zG}5e0j+6Ap>^wiZr4?#OAddq#evZBU zEgcpQp9{_cJ+5=C{l`HTV@Bps;eP?&>+XMdR`p&s>`df(r`hX%}3|O4^(Fc?O z1Ymw%MS;K9KOs(;d(Lz+5b#eA^naL5`G?s8Cv6mW+B4WioziJFUIj_T4gSpk!|dF@ zbHSJ)szm{PI$Na4h#J=eVtL-u8;q)#DC!(Lihrv*(xRSF)#}i+a&%O7KT=?HiH>u( zQ&==CX}$ThD#Y7TdqbzP2b3zsbCwLx#5zulL?6^cNfpJr<@dB2DN%@rN&F~5CEHN} zvaO@NS>GGSC7f>^#AzTXTs!~u{N-VTt-J^<<#h>W)i8<5uzT-dyRl{nW zANQki_c~n&!7`B{v0FbSj3oGLdb@eN}WLMtPnONmxu=HNP$ ztc>Yb1gDpo*pFTHa``NKu}zKI_^h@}V)2NtFN$Jz^va4)WRyxfPL{JN4XIO>V`S)Q;%r*av93lK2uksH> z-k@^qw~y&86`8J@X%Ws;GliS!R6&^iJTv|6U)wUs=|TPXCShs;Kk0~A)&S~J*{deB z62g~Or;;xXsW~u~U@qtS*luuYrrvX`N&Z|Rm{8o35o9nQ%k)Cu!37il45cw{oK){R ziu8V1Rws~8cxHHP8YRCsn03pX0curuOw zg_r#md?>a%%CB4SNTilund*LDM0mGk)+=Yc%U3?4_V7x1|A@M3=eTWoPQp!? z3+;(AwiDsX)wS#iER~^)dV;olMUA`goO{ z^&VFk?qk#9_d2OUp-Dq9hAek*>(t0MFy2m)4j3>qZ|Zj;!5{Y$=qbcw>Wo5vy^4R9 z(one(sx;u$?3_BF#GzSx00x64Cl~B@_SxG$Gial$pXw!txzyWq3{g7jzC>HHI%j@>g3LB#P+VP!ywsF``hZ)OF<#?8C@PIPu$9&i$Fi~S>=MK zNQrWeDHW6nHEF}qK`o9otFK&hB(x#*__Dv;jpOe2p19bjr8dU0i(sWYN|EuaVcuql zG3kwC#|gC1Fx!>dWEc6A4DMfUP!3a75^$Kc$gb<6`#+J{3*ONT)cBDY49?Su9M;+axFU)23?ML`0v9ZprG(99zsxC+7{f?v@PlNxr?J`vxYrdf$%TzUWq+MNEOV zexTjlh#D>|is0sMsHWM~lzq~zXX$GetQD#BgvMuT;s~WLov8{P4A29I5eMsPq``IW z+*}L(+G%RGcWXxAohyW@RQE8)!Q5v5HjX`rOnoHm{Of&%RVwl1Xz0b^WBP)G}-e0;>{8Ft9Lr zVd$OPmpPy9VKrV9E>h;{%k`^X5#!F>%*QP#+S386-hTzg$dF}mL$=4n&c?DR$0BYN ziE2O^bx-I@<~XaaF@LG7um5+_{j90RIBwcrQ@^ diff --git a/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb new file mode 100644 index 00000000000..fc4597fbb96 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillHasMergeRequestOfVulnerabilityReads, schema: 20230907155247, feature_category: :database do # rubocop:disable Layout/LineLength + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:scanners) { table(:vulnerability_scanners) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_reads) { table(:vulnerability_reads) } + let(:merge_requests) { table(:merge_requests) } + let(:merge_request_links) { table(:vulnerability_merge_request_links) } + + let(:namespace) { namespaces.create!(name: 'user', path: 'user') } + let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } + let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'external_id', name: 'Test Scanner') } + + let(:vulnerability) do + vulnerabilities.create!( + project_id: project.id, + author_id: user.id, + title: 'test', + severity: 1, + confidence: 1, + report_type: 1 + ) + end + + let(:merge_request) do + merge_requests.create!( + target_project_id: project.id, + source_branch: "other", + target_branch: "main", + author_id: user.id, + title: 'Feedback Merge Request' + ) + end + + let!(:vulnerability_read) do + vulnerability_reads.create!( + project_id: project.id, + vulnerability_id: vulnerability.id, + scanner_id: scanner.id, + severity: 1, + report_type: 1, + state: 1, + uuid: SecureRandom.uuid + ) + end + + let!(:merge_request_link) do + merge_request_links.create!( + vulnerability_id: vulnerability.id, merge_request_id: merge_request.id) + end + + subject(:perform_migration) do + described_class.new( + start_id: vulnerability_reads.first.vulnerability_id, + end_id: vulnerability_reads.last.vulnerability_id, + batch_table: :vulnerability_reads, + batch_column: :vulnerability_id, + sub_batch_size: vulnerability_reads.count, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + before do + # Unset since the trigger already sets during merge_request_link creation. + vulnerability_reads.update_all(has_merge_request: false) + end + + it 'sets the has_merge_request of existing record' do + expect { perform_migration }.to change { vulnerability_read.reload.has_merge_request }.from(false).to(true) + end + + it 'does not modify has_merge_request of other vulnerabilities which do not have merge request' do + vulnerability_2 = vulnerabilities.create!( + project_id: project.id, + author_id: user.id, + title: 'test 2', + severity: 1, + confidence: 1, + report_type: 1 + ) + + vulnerability_read_2 = vulnerability_reads.create!( + project_id: project.id, + vulnerability_id: vulnerability_2.id, + scanner_id: scanner.id, + severity: 1, + report_type: 1, + state: 1, + uuid: SecureRandom.uuid + ) + + expect { perform_migration }.not_to change { vulnerability_read_2.reload.has_merge_request }.from(false) + end +end diff --git a/spec/lib/gitlab/group_search_results_spec.rb b/spec/lib/gitlab/group_search_results_spec.rb index aa9483ade86..314759fb8a4 100644 --- a/spec/lib/gitlab/group_search_results_spec.rb +++ b/spec/lib/gitlab/group_search_results_spec.rb @@ -51,6 +51,17 @@ RSpec.describe Gitlab::GroupSearchResults, feature_category: :global_search do include_examples 'search results filtered by archived', 'search_merge_requests_hide_archived_projects' end + describe 'milestones search' do + let!(:unarchived_project) { create(:project, :public, group: group) } + let!(:archived_project) { create(:project, :public, :archived, group: group) } + let!(:unarchived_result) { create(:milestone, project: unarchived_project, title: 'foo') } + let!(:archived_result) { create(:milestone, project: archived_project, title: 'foo') } + let(:query) { 'foo' } + let(:scope) { 'milestones' } + + include_examples 'search results filtered by archived', 'search_milestones_hide_archived_projects' + end + describe '#projects' do let(:scope) { 'projects' } let(:query) { 'Test' } diff --git a/spec/lib/gitlab/search_results_spec.rb b/spec/lib/gitlab/search_results_spec.rb index 62cc6376531..d1f19a5e1ba 100644 --- a/spec/lib/gitlab/search_results_spec.rb +++ b/spec/lib/gitlab/search_results_spec.rb @@ -16,8 +16,9 @@ RSpec.describe Gitlab::SearchResults, feature_category: :global_search do let(:query) { 'foo' } let(:filters) { {} } let(:sort) { nil } + let(:limit_projects) { Project.order(:id) } - subject(:results) { described_class.new(user, query, Project.order(:id), sort: sort, filters: filters) } + subject(:results) { described_class.new(user, query, limit_projects, sort: sort, filters: filters) } context 'as a user with access' do before do @@ -438,26 +439,32 @@ RSpec.describe Gitlab::SearchResults, feature_category: :global_search do end context 'milestones' do - it 'returns correct set of milestones' do - private_project_1 = create(:project, :private) - private_project_2 = create(:project, :private) - internal_project = create(:project, :internal) - public_project_1 = create(:project, :public) - public_project_2 = create(:project, :public, :issues_disabled, :merge_requests_disabled) + let_it_be(:archived_project) { create(:project, :public, :archived) } + let_it_be(:private_project_1) { create(:project, :private) } + let_it_be(:private_project_2) { create(:project, :private) } + let_it_be(:internal_project) { create(:project, :internal) } + let_it_be(:public_project_1) { create(:project, :public) } + let_it_be(:public_project_2) { create(:project, :public, :issues_disabled, :merge_requests_disabled) } + let_it_be(:hidden_milestone_1) { create(:milestone, project: private_project_2, title: 'Private project without access milestone') } + let_it_be(:hidden_milestone_2) { create(:milestone, project: public_project_2, title: 'Public project with milestones disabled milestone') } + let_it_be(:hidden_milestone_3) { create(:milestone, project: archived_project, title: 'Milestone from an archived project') } + let_it_be(:milestone_1) { create(:milestone, project: private_project_1, title: 'Private project with access milestone', state: 'closed') } + let_it_be(:milestone_2) { create(:milestone, project: internal_project, title: 'Internal project milestone') } + let_it_be(:milestone_3) { create(:milestone, project: public_project_1, title: 'Public project with milestones enabled milestone') } + let(:unarchived_result) { milestone_1 } + let(:archived_result) { hidden_milestone_3 } + let(:limit_projects) { ProjectsFinder.new(current_user: user).execute } + let(:query) { 'milestone' } + let(:scope) { 'milestones' } + + before do private_project_1.add_developer(user) - # milestones that should not be visible - create(:milestone, project: private_project_2, title: 'Private project without access milestone') - create(:milestone, project: public_project_2, title: 'Public project with milestones disabled milestone') - # milestones that should be visible - milestone_1 = create(:milestone, project: private_project_1, title: 'Private project with access milestone', state: 'closed') - milestone_2 = create(:milestone, project: internal_project, title: 'Internal project milestone') - milestone_3 = create(:milestone, project: public_project_1, title: 'Public project with milestones enabled milestone') - # Global search scope takes user authorized projects, internal projects and public projects. - limit_projects = ProjectsFinder.new(current_user: user).execute - - milestones = described_class.new(user, 'milestone', limit_projects).objects('milestones') - - expect(milestones).to match_array([milestone_1, milestone_2, milestone_3]) end + + it 'returns correct set of milestones' do + expect(results.objects(scope)).to match_array([milestone_1, milestone_2, milestone_3]) + end + + include_examples 'search results filtered by archived', 'search_milestones_hide_archived_projects' end end diff --git a/spec/lib/gitlab/x509/certificate_spec.rb b/spec/lib/gitlab/x509/certificate_spec.rb index d919b99de2a..a81bdfcbd42 100644 --- a/spec/lib/gitlab/x509/certificate_spec.rb +++ b/spec/lib/gitlab/x509/certificate_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::X509::Certificate do +RSpec.describe Gitlab::X509::Certificate, feature_category: :source_code_management do include SmimeHelper let(:sample_ca_certs_path) { Rails.root.join('spec/fixtures/clusters').to_s } diff --git a/spec/lib/gitlab/x509/commit_sigstore_spec.rb b/spec/lib/gitlab/x509/commit_sigstore_spec.rb new file mode 100644 index 00000000000..7079fa28108 --- /dev/null +++ b/spec/lib/gitlab/x509/commit_sigstore_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::X509::Commit, feature_category: :source_code_management do + let(:commit_sha) { '440bf5b2b499a90d9adcbebe3752f8c6f245a1aa' } + let_it_be(:user) { create(:user, email: X509Helpers::User2.certificate_email) } + let_it_be(:project) { create(:project, :repository, path: X509Helpers::User2.path, creator: user) } + let(:commit) { create(:commit, project: project) } + let(:signature) { described_class.new(commit).signature } + let(:store) { OpenSSL::X509::Store.new } + let(:certificate) { OpenSSL::X509::Certificate.new(X509Helpers::User2.trust_cert) } + + before do + store.add_cert(certificate) if certificate + allow(OpenSSL::X509::Store).to receive(:new).and_return(store) + end + + describe '#signature' do + context 'on second call' do + it 'returns the cached signature' do + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:new).and_call_original + end + expect_next_instance_of(described_class) do |instance| + expect(instance).to receive(:create_cached_signature!).and_call_original + end + + signature + + # consecutive call + expect(described_class).not_to receive(:create_cached_signature!).and_call_original + signature + end + end + end + + describe '#update_signature!' do + let(:certificate) { nil } + + it 'updates verification status' do + signature + + cert = OpenSSL::X509::Certificate.new(X509Helpers::User2.trust_cert) + store.add_cert(cert) + + # stored_signature = CommitSignatures::X509CommitSignature.find_by_commit_sha(commit_sha) + # expect { described_class.new(commit).update_signature!(stored_signature) }.to( + # change { signature.reload.verification_status }.from('unverified').to('verified') + # ) # TODO sigstore support pending + end + end +end diff --git a/spec/lib/gitlab/x509/commit_spec.rb b/spec/lib/gitlab/x509/commit_spec.rb index 412fa6e5a7f..2766a1a9bac 100644 --- a/spec/lib/gitlab/x509/commit_spec.rb +++ b/spec/lib/gitlab/x509/commit_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true require 'spec_helper' -RSpec.describe Gitlab::X509::Commit do +RSpec.describe Gitlab::X509::Commit, feature_category: :source_code_management do let(:commit_sha) { '189a6c924013fc3fe40d6f1ec1dc20214183bc97' } - let(:user) { create(:user, email: X509Helpers::User1.certificate_email) } - let(:project) { create(:project, :repository, path: X509Helpers::User1.path, creator: user) } + let_it_be(:user) { create(:user, email: X509Helpers::User1.certificate_email) } + let_it_be(:project) { create(:project, :repository, path: X509Helpers::User1.path, creator: user) } let(:commit) { project.commit_by(oid: commit_sha ) } let(:signature) { described_class.new(commit).signature } let(:store) { OpenSSL::X509::Store.new } diff --git a/spec/lib/gitlab/x509/signature_sigstore_spec.rb b/spec/lib/gitlab/x509/signature_sigstore_spec.rb new file mode 100644 index 00000000000..84962576ea2 --- /dev/null +++ b/spec/lib/gitlab/x509/signature_sigstore_spec.rb @@ -0,0 +1,453 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::X509::Signature, feature_category: :source_code_management do + let(:issuer_attributes) do + { + subject_key_identifier: X509Helpers::User2.issuer_subject_key_identifier, + subject: X509Helpers::User2.certificate_issuer + } + end + + it_behaves_like 'signature with type checking', :x509 do + subject(:signature) do + described_class.new( + X509Helpers::User2.signed_commit_signature, + X509Helpers::User2.signed_commit_base_data, + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ) + end + end + + shared_examples "a verified signature" do + let!(:user) { create(:user, email: X509Helpers::User2.certificate_email) } + + subject(:signature) do + described_class.new( + X509Helpers::User2.signed_commit_signature, + X509Helpers::User2.signed_commit_base_data, + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ) + end + + it 'returns a verified signature if email does match' do + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey # TODO sigstore support pending + expect(signature.verification_status).to eq(:unverified) # TODO sigstore support pending + end + + it 'returns a verified signature if email does match, case-insensitively' do + signature = described_class.new( + X509Helpers::User2.signed_commit_signature, + X509Helpers::User2.signed_commit_base_data, + X509Helpers::User2.certificate_email.upcase, + X509Helpers::User2.signed_commit_time + ) + + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey # TODO sigstore support pending + expect(signature.verification_status).to eq(:unverified) # TODO sigstore support pending + end + + context 'when the certificate contains multiple emails' do + before do + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:get_certificate_extension).and_call_original + allow(instance).to receive(:get_certificate_extension) + .with('subjectAltName') + .and_return("email:gitlab2@example.com, othername:, email:#{ + X509Helpers::User2.certificate_email + }") + end + end + + context 'and the email matches one of them' do + it 'returns a verified signature' do + expect(signature.x509_certificate).to have_attributes(certificate_attributes.except(:email, :emails)) + expect(signature.x509_certificate.email).to eq('gitlab2@example.com') + expect(signature.x509_certificate.emails).to contain_exactly('gitlab2@example.com', + X509Helpers::User2.certificate_email) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey # TODO sigstore support pending + expect(signature.verification_status).to eq(:unverified) # TODO sigstore support pending + end + end + end + + context "if the email matches but isn't confirmed" do + let!(:user) { create(:user, :unconfirmed, email: X509Helpers::User2.certificate_email) } + + it "returns an unverified signature" do + expect(signature.verification_status).to eq(:unverified) + end + end + + it 'returns an unverified signature if email does not match' do + signature = described_class.new( + X509Helpers::User2.signed_commit_signature, + X509Helpers::User2.signed_commit_base_data, + "gitlab@example.com", + X509Helpers::User2.signed_commit_time + ) + + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey # TODO sigstore support pending + expect(signature.verification_status).to eq(:unverified) + end + + it 'returns an unverified signature if email does match and time is wrong' do + signature = described_class.new( + X509Helpers::User2.signed_commit_signature, + X509Helpers::User2.signed_commit_base_data, + X509Helpers::User2.certificate_email, + Time.zone.local(2020, 2, 22) + ) + + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey + expect(signature.verification_status).to eq(:unverified) + end + + it 'returns an unverified signature if certificate is revoked' do + expect(signature.verification_status).to eq(:unverified) # TODO sigstore support pending + + signature.x509_certificate.revoked! + + expect(signature.verification_status).to eq(:unverified) + end + end + + context 'with commit signature' do + let(:certificate_attributes) do + { + subject_key_identifier: X509Helpers::User2.certificate_subject_key_identifier, + subject: X509Helpers::User2.certificate_subject, + email: X509Helpers::User2.certificate_email, + emails: [X509Helpers::User2.certificate_email], + serial_number: X509Helpers::User2.certificate_serial + } + end + + context 'with verified signature' do + context 'with trusted certificate store' do + before do + store = OpenSSL::X509::Store.new + certificate = OpenSSL::X509::Certificate.new(X509Helpers::User2.trust_cert) + store.add_cert(certificate) + allow(OpenSSL::X509::Store).to receive(:new).and_return(store) + end + + it_behaves_like "a verified signature" + end + + context 'with the certificate defined by OpenSSL::X509::DEFAULT_CERT_FILE' do + before do + store = OpenSSL::X509::Store.new + certificate = OpenSSL::X509::Certificate.new(X509Helpers::User2.trust_cert) + file_path = Rails.root.join("tmp/cert.pem").to_s + + File.open(file_path, "wb") do |f| + f.print certificate.to_pem + end + + allow(Gitlab::X509::Certificate).to receive(:default_cert_file).and_return(file_path) + + allow(OpenSSL::X509::Store).to receive(:new).and_return(store) + end + + it_behaves_like "a verified signature" + end + + context 'without trusted certificate within store' do + before do + store = OpenSSL::X509::Store.new + allow(OpenSSL::X509::Store).to receive(:new) + .and_return( + store + ) + end + + it 'returns an unverified signature' do + signature = described_class.new( + X509Helpers::User2.signed_commit_signature, + X509Helpers::User2.signed_commit_base_data, + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ) + + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey + expect(signature.verification_status).to eq(:unverified) + end + end + end + + context 'with invalid signature' do + it 'returns nil' do + signature = described_class.new( + X509Helpers::User2.signed_commit_signature.tr('A', 'B'), + X509Helpers::User2.signed_commit_base_data, + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ) + expect(signature.x509_certificate).to be_nil + expect(signature.verified_signature).to be_falsey + expect(signature.verification_status).to eq(:unverified) + end + end + + context 'with invalid commit message' do + it 'returns nil' do + signature = described_class.new( + X509Helpers::User2.signed_commit_signature, + 'x', + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ) + expect(signature.x509_certificate).to be_nil + expect(signature.verified_signature).to be_falsey + expect(signature.verification_status).to eq(:unverified) + end + end + end + + context 'with email' do + describe 'subjectAltName with email, othername' do + before do + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:get_certificate_extension).and_call_original + allow(instance).to receive(:get_certificate_extension) + .with('subjectAltName') + .and_return("email:gitlab@example.com, othername:") + end + end + + let(:signature) do + described_class.new( + X509Helpers::User2.signed_commit_signature, + X509Helpers::User2.signed_commit_base_data, + 'gitlab@example.com', + X509Helpers::User2.signed_commit_time + ) + end + + it 'extracts email' do + expect(signature.x509_certificate.email).to eq("gitlab@example.com") + expect(signature.x509_certificate.emails).to contain_exactly("gitlab@example.com") + end + + context 'when there are multiple emails' do + before do + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:get_certificate_extension).and_call_original + allow(instance).to receive(:get_certificate_extension) + .with('subjectAltName') + .and_return("email:gitlab@example.com, othername:, email:gitlab2@example.com") + end + end + + it 'extracts all the emails' do + expect(signature.x509_certificate.email).to eq("gitlab@example.com") + expect(signature.x509_certificate.emails).to contain_exactly("gitlab@example.com", "gitlab2@example.com") + end + end + end + + describe 'subjectAltName with othername, email' do + before do + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:get_certificate_extension).and_call_original + end + + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:get_certificate_extension).and_call_original + allow(instance).to receive(:get_certificate_extension) + .with('subjectAltName') + .and_return("othername:, email:gitlab@example.com") + end + end + + it 'extracts email' do + signature = described_class.new( + X509Helpers::User2.signed_commit_signature, + X509Helpers::User2.signed_commit_base_data, + 'gitlab@example.com', + X509Helpers::User2.signed_commit_time + ) + + expect(signature.x509_certificate.email).to eq("gitlab@example.com") + end + end + end + + describe '#signed_by_user' do + subject do + described_class.new( + X509Helpers::User2.signed_tag_signature, + X509Helpers::User2.signed_tag_base_data, + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ).signed_by_user + end + + context 'if email is assigned to a user' do + let!(:signed_by_user) { create(:user, email: X509Helpers::User2.certificate_email) } + + it 'returns user' do + is_expected.to eq(signed_by_user) + end + end + + it 'if email is not assigned to a user, return nil' do + is_expected.to be_nil + end + end + + context 'with tag signature' do + let(:certificate_attributes) do + { + subject_key_identifier: X509Helpers::User2.tag_certificate_subject_key_identifier, + subject: X509Helpers::User2.certificate_subject, + email: X509Helpers::User2.certificate_email, + emails: [X509Helpers::User2.certificate_email], + serial_number: X509Helpers::User2.tag_certificate_serial + } + end + + let(:issuer_attributes) do + { + subject_key_identifier: X509Helpers::User2.tag_issuer_subject_key_identifier, + subject: X509Helpers::User2.tag_certificate_issuer + } + end + + context 'with verified signature' do + let_it_be(:user) { create(:user, :unconfirmed, email: X509Helpers::User2.certificate_email) } + + subject(:signature) do + described_class.new( + X509Helpers::User2.signed_tag_signature, + X509Helpers::User2.signed_tag_base_data, + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ) + end + + context 'with trusted certificate store' do + before do + store = OpenSSL::X509::Store.new + certificate = OpenSSL::X509::Certificate.new X509Helpers::User2.trust_cert + store.add_cert(certificate) + allow(OpenSSL::X509::Store).to receive(:new).and_return(store) + end + + context 'when user email is confirmed' do + before_all do + user.confirm + end + + it 'returns a verified signature if email does match', :ggregate_failures do + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey # TODO sigstore support pending + expect(signature.verification_status).to eq(:unverified) # TODO sigstore support pending + end + + it 'returns an unverified signature if email does not match', :aggregate_failures do + signature = described_class.new( + X509Helpers::User2.signed_tag_signature, + X509Helpers::User2.signed_tag_base_data, + "gitlab@example.com", + X509Helpers::User2.signed_commit_time + ) + + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey # TODO sigstore support pending + expect(signature.verification_status).to eq(:unverified) + end + + it 'returns an unverified signature if email does match and time is wrong', :aggregate_failures do + signature = described_class.new( + X509Helpers::User2.signed_tag_signature, + X509Helpers::User2.signed_tag_base_data, + X509Helpers::User2.certificate_email, + Time.zone.local(2020, 2, 22) + ) + + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey + expect(signature.verification_status).to eq(:unverified) + end + + it 'returns an unverified signature if certificate is revoked' do + expect(signature.verification_status).to eq(:unverified) # TODO sigstore support pending + + signature.x509_certificate.revoked! + + expect(signature.verification_status).to eq(:unverified) + end + end + + it 'returns an unverified signature if the email matches but is not confirmed' do + expect(signature.verification_status).to eq(:unverified) + end + end + + context 'without trusted certificate within store' do + before do + store = OpenSSL::X509::Store.new + allow(OpenSSL::X509::Store).to receive(:new) + .and_return( + store + ) + end + + it 'returns an unverified signature' do + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_falsey + expect(signature.verification_status).to eq(:unverified) + end + end + end + + context 'with invalid signature' do + it 'returns nil' do + signature = described_class.new( + X509Helpers::User2.signed_tag_signature.tr('A', 'B'), + X509Helpers::User2.signed_tag_base_data, + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ) + expect(signature.x509_certificate).to be_nil + expect(signature.verified_signature).to be_falsey + expect(signature.verification_status).to eq(:unverified) + end + end + + context 'with invalid message' do + it 'returns nil' do + signature = described_class.new( + X509Helpers::User2.signed_tag_signature, + 'x', + X509Helpers::User2.certificate_email, + X509Helpers::User2.signed_commit_time + ) + expect(signature.x509_certificate).to be_nil + expect(signature.verified_signature).to be_falsey + expect(signature.verification_status).to eq(:unverified) + end + end + end +end diff --git a/spec/lib/gitlab/x509/signature_spec.rb b/spec/lib/gitlab/x509/signature_spec.rb index e0823aa8153..8043cefe888 100644 --- a/spec/lib/gitlab/x509/signature_spec.rb +++ b/spec/lib/gitlab/x509/signature_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::X509::Signature do +RSpec.describe Gitlab::X509::Signature, feature_category: :source_code_management do let(:issuer_attributes) do { subject_key_identifier: X509Helpers::User1.issuer_subject_key_identifier, diff --git a/spec/lib/gitlab/x509/tag_sigstore_spec.rb b/spec/lib/gitlab/x509/tag_sigstore_spec.rb new file mode 100644 index 00000000000..3cf864ea442 --- /dev/null +++ b/spec/lib/gitlab/x509/tag_sigstore_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::X509::Tag, feature_category: :source_code_management do + describe '#signature' do + let(:tag_id) { 'v1.1.2' } + let(:tag) { instance_double('Gitlab::Git::Tag') } + let_it_be(:user) { create(:user, email: X509Helpers::User2.tag_email) } + let_it_be(:project) { create(:project, path: X509Helpers::User2.path, creator: user) } + let(:signature) { described_class.new(project.repository, tag).signature } + + before do + allow(tag).to receive(:id).and_return(tag_id) + allow(tag).to receive(:has_signature?).and_return(true) + allow(tag).to receive(:user_email).and_return(user.email) + allow(tag).to receive(:date).and_return(X509Helpers::User2.signed_tag_time) + allow(Gitlab::Git::Tag).to receive(:extract_signature_lazily).with(project.repository, tag_id) + .and_return([X509Helpers::User2.signed_tag_signature, X509Helpers::User2.signed_tag_base_data]) + end + + describe 'signed tag' do + let(:certificate_attributes) do + { + subject_key_identifier: X509Helpers::User2.tag_certificate_subject_key_identifier, + subject: X509Helpers::User2.certificate_subject, + email: X509Helpers::User2.certificate_email, + serial_number: X509Helpers::User2.tag_certificate_serial + } + end + + let(:issuer_attributes) do + { + subject_key_identifier: X509Helpers::User2.tag_issuer_subject_key_identifier, + subject: X509Helpers::User2.tag_certificate_issuer + } + end + + it { expect(signature).not_to be_nil } + it { expect(signature.verification_status).to eq(:unverified) } + it { expect(signature.x509_certificate).to have_attributes(certificate_attributes) } + it { expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) } + end + end +end diff --git a/spec/lib/gitlab/x509/tag_spec.rb b/spec/lib/gitlab/x509/tag_spec.rb index e20ef688db5..4368c3d7a4b 100644 --- a/spec/lib/gitlab/x509/tag_spec.rb +++ b/spec/lib/gitlab/x509/tag_spec.rb @@ -1,15 +1,24 @@ # frozen_string_literal: true require 'spec_helper' -RSpec.describe Gitlab::X509::Tag do - subject(:signature) { described_class.new(project.repository, tag).signature } - +RSpec.describe Gitlab::X509::Tag, feature_category: :source_code_management do describe '#signature' do - let_it_be(:project) { create(:project, :repository) } - let_it_be(:repository) { project.repository.raw } + let(:tag_id) { 'v1.1.1' } + let(:tag) { instance_double('Gitlab::Git::Tag') } + let_it_be(:user) { create(:user, email: X509Helpers::User1.tag_email) } + let_it_be(:project) { create(:project, path: X509Helpers::User1.path, creator: user) } + let(:signature) { described_class.new(project.repository, tag).signature } + + before do + allow(tag).to receive(:id).and_return(tag_id) + allow(tag).to receive(:has_signature?).and_return(true) + allow(tag).to receive(:user_email).and_return(user.email) + allow(tag).to receive(:date).and_return(X509Helpers::User1.signed_tag_time) + allow(Gitlab::Git::Tag).to receive(:extract_signature_lazily).with(project.repository, tag_id) + .and_return([X509Helpers::User1.signed_tag_signature, X509Helpers::User1.signed_tag_base_data]) + end describe 'signed tag' do - let(:tag) { project.repository.find_tag('v1.1.1') } let(:certificate_attributes) do { subject_key_identifier: X509Helpers::User1.tag_certificate_subject_key_identifier, @@ -32,11 +41,5 @@ RSpec.describe Gitlab::X509::Tag do it { expect(signature.x509_certificate).to have_attributes(certificate_attributes) } it { expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) } end - - describe 'unsigned tag' do - let(:tag) { project.repository.find_tag('v1.0.0') } - - it { expect(signature).to be_nil } - end end end diff --git a/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb b/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb new file mode 100644 index 00000000000..7214e0114d4 --- /dev/null +++ b/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillHasMergeRequestOfVulnerabilityReads, feature_category: :database do + let!(:batched_migration) { described_class::MIGRATION_NAME } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :vulnerability_reads, + column_name: :vulnerability_id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end diff --git a/spec/models/x509_certificate_spec.rb b/spec/models/x509_certificate_spec.rb index 5723bd80739..a550b2caa44 100644 --- a/spec/models/x509_certificate_spec.rb +++ b/spec/models/x509_certificate_spec.rb @@ -5,7 +5,6 @@ require 'spec_helper' RSpec.describe X509Certificate do describe 'validation' do it { is_expected.to validate_presence_of(:subject_key_identifier) } - it { is_expected.to validate_presence_of(:subject) } it { is_expected.to validate_presence_of(:email) } it { is_expected.to validate_presence_of(:serial_number) } it { is_expected.to validate_presence_of(:x509_issuer_id) } diff --git a/spec/models/x509_issuer_spec.rb b/spec/models/x509_issuer_spec.rb index 3d04adf7e26..31470a443a2 100644 --- a/spec/models/x509_issuer_spec.rb +++ b/spec/models/x509_issuer_spec.rb @@ -5,8 +5,6 @@ require 'spec_helper' RSpec.describe X509Issuer do describe 'validation' do it { is_expected.to validate_presence_of(:subject_key_identifier) } - it { is_expected.to validate_presence_of(:subject) } - it { is_expected.to validate_presence_of(:crl_url) } end describe '.safe_create!' do diff --git a/spec/presenters/projects/security/configuration_presenter_spec.rb b/spec/presenters/projects/security/configuration_presenter_spec.rb index 714426ef0de..beabccf6639 100644 --- a/spec/presenters/projects/security/configuration_presenter_spec.rb +++ b/spec/presenters/projects/security/configuration_presenter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Projects::Security::ConfigurationPresenter do +RSpec.describe Projects::Security::ConfigurationPresenter, feature_category: :software_composition_analysis do include Gitlab::Routing.url_helpers using RSpec::Parameterized::TableSyntax diff --git a/spec/support/helpers/x509_helpers.rb b/spec/support/helpers/x509_helpers.rb index 1dc8b1d4845..aa5c360d953 100644 --- a/spec/support/helpers/x509_helpers.rb +++ b/spec/support/helpers/x509_helpers.rb @@ -173,6 +173,10 @@ module X509Helpers Time.at(1561027326) end + def signed_tag_time + Time.at(1574261780) + end + def signed_tag_signature <<~SIGNATURE -----BEGIN SIGNED MESSAGE----- @@ -337,6 +341,10 @@ module X509Helpers 'r.meier@siemens.com' end + def tag_email + 'dmitriy.zaporozhets@gmail.com' + end + def certificate_issuer 'CN=Siemens Issuing CA EE Auth 2016,OU=Siemens Trust Center,serialNumber=ZZZZZZA2,O=Siemens,L=Muenchen,ST=Bayern,C=DE' end @@ -357,4 +365,177 @@ module X509Helpers ['r.meier@siemens.com'] end end + + module User2 + extend self + + def commit + '440bf5b2b499a90d9adcbebe3752f8c6f245a1aa' + end + + def path + 'gitlab-test' + end + + def trust_cert + <<~TRUSTCERTIFICATE + -----BEGIN CERTIFICATE----- + MIICGjCCAaGgAwIBAgIUALnViVfnU0brJasmRkHrn/UnfaQwCgYIKoZIzj0EAwMw + KjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MREwDwYDVQQDEwhzaWdzdG9yZTAeFw0y + MjA0MTMyMDA2MTVaFw0zMTEwMDUxMzU2NThaMDcxFTATBgNVBAoTDHNpZ3N0b3Jl + LmRldjEeMBwGA1UEAxMVc2lnc3RvcmUtaW50ZXJtZWRpYXRlMHYwEAYHKoZIzj0C + AQYFK4EEACIDYgAE8RVS/ysH+NOvuDZyPIZtilgUF9NlarYpAd9HP1vBBH1U5CV7 + 7LSS7s0ZiH4nE7Hv7ptS6LvvR/STk798LVgMzLlJ4HeIfF3tHSaexLcYpSASr1kS + 0N/RgBJz/9jWCiXno3sweTAOBgNVHQ8BAf8EBAMCAQYwEwYDVR0lBAwwCgYIKwYB + BQUHAwMwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQU39Ppz1YkEZb5qNjp + KFWixi4YZD8wHwYDVR0jBBgwFoAUWMAeX5FFpWapesyQoZMi0CrFxfowCgYIKoZI + zj0EAwMDZwAwZAIwPCsQK4DYiZYDPIaDi5HFKnfxXx6ASSVmERfsynYBiX2X6SJR + nZU84/9DZdnFvvxmAjBOt6QpBlc4J/0DxvkTCqpclvziL6BCCPnjdlIB3Pu3BxsP + mygUY7Ii2zbdCdliiow= + -----END CERTIFICATE----- + TRUSTCERTIFICATE + end + + def signed_commit_signature + <<~SIGNATURE + -----BEGIN SIGNED MESSAGE----- + MIIEOQYJKoZIhvcNAQcCoIIEKjCCBCYCAQExDTALBglghkgBZQMEAgEwCwYJKoZI + hvcNAQcBoIIC2jCCAtYwggJdoAMCAQICFC5R9EXk+ljFhyCs4urRxmCuvQNAMAoG + CCqGSM49BAMDMDcxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEeMBwGA1UEAxMVc2ln + c3RvcmUtaW50ZXJtZWRpYXRlMB4XDTIzMDgxOTE3NTgwNVoXDTIzMDgxOTE4MDgw + NVowADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABBGajWb10Rt36IMxtJmjRDa7 + 5O6YCLhVq9+LNJSAx2M7p6netqW7W+lwym4z1Y1gXLdGHBshrbx/yr6Trhh2TCej + ggF8MIIBeDAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwMwHQYD + VR0OBBYEFBttEjGzNppCqA4tlZY4oaxkdmQbMB8GA1UdIwQYMBaAFN/T6c9WJBGW + +ajY6ShVosYuGGQ/MCUGA1UdEQEB/wQbMBmBF2dpdGxhYmdwZ3Rlc3RAZ21haWwu + Y29tMCwGCisGAQQBg78wAQEEHmh0dHBzOi8vZ2l0aHViLmNvbS9sb2dpbi9vYXV0 + aDAuBgorBgEEAYO/MAEIBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9sb2dpbi9vYXV0 + aDCBiwYKKwYBBAHWeQIEAgR9BHsAeQB3AN09MGrGxxEyYxkeHJlnNwKiSl643jyt + /4eKcoAvKe6OAAABig7ydOsAAAQDAEgwRgIhAMqJnFLAspeqfbK/gA/7zjceyExq + QN7qDXWKRLS01rTvAiEAp/uBShQb9tVa3P3fYVAMiXydvr5dqCpNiuudZiuYq0Yw + CgYIKoZIzj0EAwMDZwAwZAIwWKXYyP5FvbfhvfLkV0tN887ax1eg7TmF1Tzkugag + cLJ5MzK3xYNcUO/3AxO3H/b8AjBD9DF6R4kFO4cXoqnpsk2FTUeSPiUJ+0x2PDFG + gQZvoMWz7CnwjXml8XDEKNpYoPkxggElMIIBIQIBATBPMDcxFTATBgNVBAoTDHNp + Z3N0b3JlLmRldjEeMBwGA1UEAxMVc2lnc3RvcmUtaW50ZXJtZWRpYXRlAhQuUfRF + 5PpYxYcgrOLq0cZgrr0DQDALBglghkgBZQMEAgGgaTAYBgkqhkiG9w0BCQMxCwYJ + KoZIhvcNAQcBMBwGCSqGSIb3DQEJBTEPFw0yMzA4MTkxNzU4MDVaMC8GCSqGSIb3 + DQEJBDEiBCB4B7DeGk22WmBseJzjjRJcQsyYxu0PNDAFXq55uJ7MSzAKBggqhkjO + PQQDAgRHMEUCIQCNegIrK6m1xyGuu4lw06l22VQsmO74/k3H236jCFF+bAIgAX1N + rxBFWnjWboZmAV1NuduTD/YToShK6iRmJ/NpILA= + -----END SIGNED MESSAGE----- + SIGNATURE + end + + def signed_commit_base_data + <<~SIGNEDDATA + tree 7d5ee08cadaa161d731c56a9265feef130143b07 + parent 4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6 + author Mona Lisa 1692467872 +0000 + committer Mona Lisa 1692467872 +0000 + + Sigstore Signed Commit + SIGNEDDATA + end + + def signed_commit_time + Time.at(1692467872) + end + + def signed_tag_time + Time.at(1692467872) + end + + def signed_tag_signature + <<~SIGNATURE + -----BEGIN SIGNED MESSAGE----- + MIIEOgYJKoZIhvcNAQcCoIIEKzCCBCcCAQExDTALBglghkgBZQMEAgEwCwYJKoZI + hvcNAQcBoIIC2zCCAtcwggJdoAMCAQICFB5qFHBSNfcJDZecnHK5/tleuX3yMAoG + CCqGSM49BAMDMDcxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEeMBwGA1UEAxMVc2ln + c3RvcmUtaW50ZXJtZWRpYXRlMB4XDTIzMDgxOTE3NTgzM1oXDTIzMDgxOTE4MDgz + M1owADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABKJtbdL88PM8lE21CuyDYlZm + 0xZYCThoXZSGmULrgE5+hfroCIbLswOi5i6TyB8j4CCe0Jxeu94Jn+76SXF+lbej + ggF8MIIBeDAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwMwHQYD + VR0OBBYEFBkU3IBENVJYeyK9b56vbGGrjPwYMB8GA1UdIwQYMBaAFN/T6c9WJBGW + +ajY6ShVosYuGGQ/MCUGA1UdEQEB/wQbMBmBF2dpdGxhYmdwZ3Rlc3RAZ21haWwu + Y29tMCwGCisGAQQBg78wAQEEHmh0dHBzOi8vZ2l0aHViLmNvbS9sb2dpbi9vYXV0 + aDAuBgorBgEEAYO/MAEIBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9sb2dpbi9vYXV0 + aDCBiwYKKwYBBAHWeQIEAgR9BHsAeQB3AN09MGrGxxEyYxkeHJlnNwKiSl643jyt + /4eKcoAvKe6OAAABig7y4tYAAAQDAEgwRgIhAMUjWh8ayhjWDI3faFah3Du/7IuY + xzbUXaPQnCyUbvwwAiEAwHgWv8fmKMudbVu37Nbq/c1cdnQqDK9Y2UGtlmzaLrYw + CgYIKoZIzj0EAwMDaAAwZQIwZTKZlS4HNJH48km3pxG95JTbldSBhvFlrpIEVRUd + TEK6uGQJmpIm1WYQjbJbiVS8AjEA+2NoAdMuRpa2k13HUfWQEMtzQcxZMMNB7Yux + 9ZIADOlFp701ujtFSZAXgqGL3FYKMYIBJTCCASECAQEwTzA3MRUwEwYDVQQKEwxz + aWdzdG9yZS5kZXYxHjAcBgNVBAMTFXNpZ3N0b3JlLWludGVybWVkaWF0ZQIUHmoU + cFI19wkNl5yccrn+2V65ffIwCwYJYIZIAWUDBAIBoGkwGAYJKoZIhvcNAQkDMQsG + CSqGSIb3DQEHATAcBgkqhkiG9w0BCQUxDxcNMjMwODE5MTc1ODMzWjAvBgkqhkiG + 9w0BCQQxIgQgwpYCAlbS6KnfgxQD3SATWUbdUssLaBWkHwTkmtCye4wwCgYIKoZI + zj0EAwIERzBFAiB8y5bGhWJvWCHQyma7oF038ZPLzXmsDJyJffJHoAb6XAIhAOW3 + gxuYuJAKP86B1fY0vYCZHF8vU6SZAcE6teSDowwq + -----END SIGNED MESSAGE----- + SIGNATURE + end + + def signed_tag_base_data + <<~SIGNEDDATA + object 440bf5b2b499a90d9adcbebe3752f8c6f245a1aa + type commit + tag v1.1.2 + tagger Mona Lisa 1692467901 +0000 + + Sigstore Signed Tag + SIGNEDDATA + end + + def certificate_serial + 264441215000592123389532407734419590292801651520 + end + + def tag_certificate_serial + 173635382582380059990335547381753891120957980146 + end + + def certificate_subject_key_identifier + '1B:6D:12:31:B3:36:9A:42:A8:0E:2D:95:96:38:A1:AC:64:76:64:1B' + end + + def tag_certificate_subject_key_identifier + '19:14:DC:80:44:35:52:58:7B:22:BD:6F:9E:AF:6C:61:AB:8C:FC:18' + end + + def issuer_subject_key_identifier + 'DF:D3:E9:CF:56:24:11:96:F9:A8:D8:E9:28:55:A2:C6:2E:18:64:3F' + end + + def tag_issuer_subject_key_identifier + 'DF:D3:E9:CF:56:24:11:96:F9:A8:D8:E9:28:55:A2:C6:2E:18:64:3F' + end + + def certificate_email + 'gitlabgpgtest@gmail.com' + end + + def tag_email + 'gitlabgpgtest@gmail.com' + end + + def certificate_issuer + 'CN=sigstore-intermediate,O=sigstore.dev' + end + + def tag_certificate_issuer + 'CN=sigstore-intermediate,O=sigstore.dev' + end + + def certificate_subject + '' + end + + def names + ['Mona Lisa'] + end + + def emails + ['gitlabgpgtest@gmail.com'] + end + end end diff --git a/workhorse/config_test.go b/workhorse/config_test.go index a6a1bdd7187..64f0a24d148 100644 --- a/workhorse/config_test.go +++ b/workhorse/config_test.go @@ -34,6 +34,7 @@ trusted_cidrs_for_propagation = ["10.0.0.1/8"] [redis] password = "redis password" +SentinelPassword = "sentinel password" [object_storage] provider = "test provider" [image_resizer] @@ -68,6 +69,7 @@ key = "/path/to/private/key" // fields in each section; that should happen in the tests of the // internal/config package. require.Equal(t, "redis password", cfg.Redis.Password) + require.Equal(t, "sentinel password", cfg.Redis.SentinelPassword) require.Equal(t, "test provider", cfg.ObjectStorageCredentials.Provider) require.Equal(t, uint32(123), cfg.ImageResizerConfig.MaxScalerProcs, "image resizer max_scaler_procs") require.Equal(t, []string{"127.0.0.1/8", "192.168.0.1/8"}, cfg.TrustedCIDRsForXForwardedFor) diff --git a/workhorse/internal/config/config.go b/workhorse/internal/config/config.go index 687986974a3..3b928d42fe1 100644 --- a/workhorse/internal/config/config.go +++ b/workhorse/internal/config/config.go @@ -83,13 +83,14 @@ type GoogleCredentials struct { } type RedisConfig struct { - URL TomlURL - Sentinel []TomlURL - SentinelMaster string - Password string - DB *int - MaxIdle *int - MaxActive *int + URL TomlURL + Sentinel []TomlURL + SentinelMaster string + SentinelPassword string + Password string + DB *int + MaxIdle *int + MaxActive *int } type ImageResizerConfig struct { diff --git a/workhorse/internal/goredis/goredis.go b/workhorse/internal/goredis/goredis.go index cd25c7ca60e..13a9d4cc34f 100644 --- a/workhorse/internal/goredis/goredis.go +++ b/workhorse/internal/goredis/goredis.go @@ -157,10 +157,11 @@ func configureSentinel(cfg *config.RedisConfig) *redis.Client { } client := redis.NewFailoverClient(&redis.FailoverOptions{ - MasterName: cfg.SentinelMaster, - SentinelAddrs: sentinels, - Password: cfg.Password, - DB: getOrDefault(cfg.DB, 0), + MasterName: cfg.SentinelMaster, + SentinelAddrs: sentinels, + Password: cfg.Password, + SentinelPassword: cfg.SentinelPassword, + DB: getOrDefault(cfg.DB, 0), PoolSize: getOrDefault(cfg.MaxActive, defaultMaxActive), MaxIdleConns: getOrDefault(cfg.MaxIdle, defaultMaxIdle),