diff --git a/.gitlab/ci/test-on-cng/main.gitlab-ci.yml b/.gitlab/ci/test-on-cng/main.gitlab-ci.yml index a251f19a621..ccbd419628b 100644 --- a/.gitlab/ci/test-on-cng/main.gitlab-ci.yml +++ b/.gitlab/ci/test-on-cng/main.gitlab-ci.yml @@ -32,7 +32,9 @@ workflow: QA_DOCKER_NETWORK: host QA_GENERATE_ALLURE_REPORT: "true" QA_CAN_TEST_PRAEFECT: "false" + QA_SUITE_STATUS_ENV_FILE: $CI_PROJECT_DIR/suite_status.env before_script: + - echo "SUITE_RAN=true" > "$QA_SUITE_STATUS_ENV_FILE" - export GITLAB_DOMAIN="$(getent hosts docker | awk '{ print $1 }' | head -n1).nip.io" - export QA_GITLAB_URL="http://gitlab.${GITLAB_DOMAIN}" - source scripts/qa/cng_deploy/cng-kind.sh @@ -44,6 +46,10 @@ workflow: - echo "Running - '$QA_COMMAND'" - eval "$QA_COMMAND" after_script: + - | + if [ "$CI_JOB_STATUS" == "failed" ]; then + echo "SUITE_FAILED=true" >> "$QA_SUITE_STATUS_ENV_FILE" + fi - source scripts/qa/cng_deploy/cng-kind.sh - echo -e "\e[0Ksection_start:`date +%s`:log_deploy[collapsed=true]\r\e[0KDeployment info" - save_install_logs @@ -53,8 +59,10 @@ workflow: when: always reports: junit: qa/tmp/rspec-*.xml + dotenv: $QA_SUITE_STATUS_ENV_FILE paths: - "*.log" + - qa/tmp/test-metrics-*.json - qa/tmp/allure-results # ========================================== diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index aa8a5e932bb..c46ad1fb545 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -b6ff186eba35d03b2055c2a0b0e92c65762a6d06 +f508b901207c1e6d2d5bcb8da631b5ad815aa483 diff --git a/Gemfile b/Gemfile index adb5e93957c..8570ae96550 100644 --- a/Gemfile +++ b/Gemfile @@ -52,7 +52,7 @@ gem 'sprockets', '~> 3.7.0' # rubocop:todo Gemfile/MissingFeatureCategory gem 'view_component', '~> 3.11.0' # rubocop:todo Gemfile/MissingFeatureCategory # Supported DBs -gem 'pg', '~> 1.5.5' # rubocop:todo Gemfile/MissingFeatureCategory +gem 'pg', '~> 1.5.6' # rubocop:todo Gemfile/MissingFeatureCategory gem 'neighbor', '~> 0.2.3' # rubocop:todo Gemfile/MissingFeatureCategory @@ -525,7 +525,7 @@ group :test do # Moved in `test` because https://gitlab.com/gitlab-org/gitlab/-/issues/217527 gem 'derailed_benchmarks', require: false # rubocop:todo Gemfile/MissingFeatureCategory - gem 'gitlab_quality-test_tooling', '~> 1.15.0', require: false, feature_category: :tooling + gem 'gitlab_quality-test_tooling', '~> 1.17.0', require: false, feature_category: :tooling end gem 'octokit', '~> 8.0', feature_category: :importers diff --git a/Gemfile.checksum b/Gemfile.checksum index bfbd87304d0..6b8b1ec22fd 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -227,7 +227,7 @@ {"name":"gitlab-styles","version":"11.0.0","platform":"ruby","checksum":"0dd8ec066ce9955ac51d3616c6bfded30f75bb526f39ff392ece6f43d5b9406b"}, {"name":"gitlab_chronic_duration","version":"0.12.0","platform":"ruby","checksum":"0d766944d415b5c831f176871ee8625783fc0c5bfbef2d79a3a616f207ffc16d"}, {"name":"gitlab_omniauth-ldap","version":"2.2.0","platform":"ruby","checksum":"bb4d20acb3b123ed654a8f6a47d3fac673ece7ed0b6992edb92dca14bad2838c"}, -{"name":"gitlab_quality-test_tooling","version":"1.15.0","platform":"ruby","checksum":"33416dce2d6a686ea95643eb650a249e94f696731e7f7c5bc552b369b2ba0bb5"}, +{"name":"gitlab_quality-test_tooling","version":"1.17.0","platform":"ruby","checksum":"64d495e93b777bbc05d84fa54cf8752934d24a21fd40e9ceaf73230f5899a9b4"}, {"name":"globalid","version":"1.1.0","platform":"ruby","checksum":"b337e1746f0c8cb0a6c918234b03a1ddeb4966206ce288fbb57779f59b2d154f"}, {"name":"gon","version":"6.4.0","platform":"ruby","checksum":"e3a618d659392890f1aa7db420f17c75fd7d35aeb5f8fe003697d02c4b88d2f0"}, {"name":"google-apis-androidpublisher_v3","version":"0.34.0","platform":"ruby","checksum":"d7e1d7dd92f79c498fe2082222a1740d788e022e660c135564b3fd299cab5425"}, @@ -460,10 +460,7 @@ {"name":"parslet","version":"1.8.2","platform":"ruby","checksum":"08d1ab3721cd3f175bfbee8788b2ddff71f92038f2d69bd65454c22bb9fbd98a"}, {"name":"pastel","version":"0.8.0","platform":"ruby","checksum":"481da9fb7d2f6e6b1a08faf11fa10363172dc40fd47848f096ae21209f805a75"}, {"name":"peek","version":"1.1.0","platform":"ruby","checksum":"d6501ead8cde46d8d8ed0d59eb6f0ba713d0a41c11a2c4a81447b2dce37b3ecc"}, -{"name":"pg","version":"1.5.5","platform":"ruby","checksum":"7e4baa3395619424fe0e82d0b0489e54d3015c6ee5896dd007b3bce6d7d49b68"}, -{"name":"pg","version":"1.5.5","platform":"x64-mingw-ucrt","checksum":"1adad3a4b4631e3676891639bab5aed68ac7c6a379d8314b768f74e6bdf0375e"}, -{"name":"pg","version":"1.5.5","platform":"x64-mingw32","checksum":"98b1480a04e3f8aca9c7fc06dec5662117cec540e5c5058cb3a0812e8261adcc"}, -{"name":"pg","version":"1.5.5","platform":"x86-mingw32","checksum":"4fd1e309c5d227ecb1704fc2b3f1168b13748a8d8b0eb7c09d834b28069b9433"}, +{"name":"pg","version":"1.5.6","platform":"ruby","checksum":"4bc3ad2438825eea68457373555e3fd4ea1a82027b8a6be98ef57c0d57292b1c"}, {"name":"pg_query","version":"5.1.0","platform":"ruby","checksum":"b7f7f47c864f08ccbed46a8244906fb6ee77ee344fd27250717963928c93145d"}, {"name":"plist","version":"3.7.0","platform":"ruby","checksum":"703ca90a7cb00e8263edd03da2266627f6741d280c910abbbac07c95ffb2f073"}, {"name":"png_quantizator","version":"0.2.1","platform":"ruby","checksum":"6023d4d064125c3a7e02929c95b7320ed6ac0d7341f9e8de0c9ea6576ef3106b"}, diff --git a/Gemfile.lock b/Gemfile.lock index c66d08aae33..1808663e510 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -741,7 +741,7 @@ GEM omniauth (>= 1.3, < 3) pyu-ruby-sasl (>= 0.0.3.3, < 0.1) rubyntlm (~> 0.5) - gitlab_quality-test_tooling (1.15.0) + gitlab_quality-test_tooling (1.17.0) activesupport (>= 6.1, < 7.1) amatch (~> 0.4.1) gitlab (~> 4.19) @@ -1270,7 +1270,7 @@ GEM tty-color (~> 0.5) peek (1.1.0) railties (>= 4.0.0) - pg (1.5.5) + pg (1.5.6) pg_query (5.1.0) google-protobuf (>= 3.22.3) plist (3.7.0) @@ -1929,7 +1929,7 @@ DEPENDENCIES gitlab-utils! gitlab_chronic_duration (~> 0.12) gitlab_omniauth-ldap (~> 2.2.0) - gitlab_quality-test_tooling (~> 1.15.0) + gitlab_quality-test_tooling (~> 1.17.0) gon (~> 6.4.0) google-apis-androidpublisher_v3 (~> 0.34.0) google-apis-cloudbilling_v1 (~> 0.21.0) @@ -2042,7 +2042,7 @@ DEPENDENCIES parser (~> 3.3, >= 3.3.0.2) parslet (~> 1.8) peek (~> 1.1) - pg (~> 1.5.5) + pg (~> 1.5.6) pg_query (~> 5.1.0) png_quantizator (~> 0.2.1) premailer-rails (~> 1.10.3) diff --git a/app/controllers/organizations/organizations_controller.rb b/app/controllers/organizations/organizations_controller.rb index 0596441591d..11988da8104 100644 --- a/app/controllers/organizations/organizations_controller.rb +++ b/app/controllers/organizations/organizations_controller.rb @@ -3,10 +3,18 @@ module Organizations class OrganizationsController < ApplicationController include PreviewMarkdown + include FiltersEvents + + DEFAULT_RESOURCE_LIMIT = 1000 feature_category :cell - skip_before_action :authenticate_user!, only: [:show, :groups_and_projects] + before_action :event_filter, only: [:activity] + before_action :authorize_read_organization!, only: [:activity, :show, :groups_and_projects] + + skip_before_action :authenticate_user!, only: [:activity, :show, :groups_and_projects] + + urgency :low, [:activity] def index; end @@ -14,16 +22,37 @@ module Organizations authorize_create_organization! end - def show - authorize_read_organization! + def show; end + + def activity + respond_to do |format| + format.html + format.json do + load_events + @events = @events.select { |event| event.visible_to_user?(current_user) } + + render json: ::Profile::EventSerializer.new(current_user: current_user).represent(@events) + end + end end - def groups_and_projects - authorize_read_organization! - end + def groups_and_projects; end def users authorize_read_organization_user! end + + private + + def load_events + @events = EventCollection.new( + organization.projects.limit(DEFAULT_RESOURCE_LIMIT).sorted_by_activity, + offset: params[:offset].to_i, + filter: event_filter, + groups: organization.groups.limit(DEFAULT_RESOURCE_LIMIT) + ).to_a.map(&:present) + + Events::RenderService.new(current_user).execute(@events) + end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index d1887df4b42..79361bd3bec 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -115,43 +115,12 @@ module Ci validates :ref, presence: true scope :unstarted, -> { where(runner_id: nil) } - - scope :with_any_artifacts, -> do - where('EXISTS (?)', - Ci::JobArtifact.select(1).where("#{Ci::Build.quoted_table_name}.id = #{Ci::JobArtifact.quoted_table_name}.job_id") - ) - end - - scope :with_downloadable_artifacts, -> do - where('EXISTS (?)', - Ci::JobArtifact.select(1) - .where("#{Ci::Build.quoted_table_name}.id = #{Ci::JobArtifact.quoted_table_name}.job_id") - .where(file_type: Ci::JobArtifact::DOWNLOADABLE_TYPES) - ) - end - - scope :with_erasable_artifacts, -> do - where('EXISTS (?)', - Ci::JobArtifact.select(1) - .where("#{Ci::Build.quoted_table_name}.id = #{Ci::JobArtifact.quoted_table_name}.job_id") - .where(file_type: Ci::JobArtifact.erasable_file_types) - ) - end - - scope :in_pipelines, ->(pipelines) do - where(pipeline: pipelines) - end - - scope :with_existing_job_artifacts, ->(query) do - where('EXISTS (?)', ::Ci::JobArtifact.select(1).where("#{Ci::Build.quoted_table_name}.id = #{Ci::JobArtifact.quoted_table_name}.job_id").merge(query)) - end - + scope :with_any_artifacts, -> { where_exists(Ci::JobArtifact.scoped_build) } + scope :with_downloadable_artifacts, -> { where_exists(Ci::JobArtifact.scoped_build.downloadable) } + scope :with_erasable_artifacts, -> { where_exists(Ci::JobArtifact.scoped_build.erasable) } + scope :with_existing_job_artifacts, ->(query) { where_exists(Ci::JobArtifact.scoped_build.erasable.merge(query)) } scope :without_archived_trace, -> { where_not_exists(Ci::JobArtifact.scoped_build.trace) } - - scope :with_artifacts, ->(artifact_scope) do - with_existing_job_artifacts(artifact_scope) - .eager_load_job_artifacts - end + scope :with_artifacts, ->(artifact_scope) { with_existing_job_artifacts(artifact_scope).eager_load_job_artifacts } scope :eager_load_job_artifacts, -> { includes(:job_artifacts) } scope :eager_load_tags, -> { includes(:tags) } diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index ac3395ff440..2ae0a88b3d5 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -311,8 +311,8 @@ class IssuableBaseService < ::BaseContainerService GraphqlTriggers.issuable_description_updated(issuable) end - def update(issuable) # rubocop:disable Metrics/AbcSize -- remove with the FF `use_primary_for_update_computations` - ::Gitlab::Database::LoadBalancing::Session.current.use_primary! if Feature.enabled?(:use_primary_for_update_computations, issuable.try(:resource_parent)) + def update(issuable) + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! old_associations = associations_before_update(issuable) diff --git a/app/views/organizations/organizations/activity.html.haml b/app/views/organizations/organizations/activity.html.haml new file mode 100644 index 00000000000..57e47ae939b --- /dev/null +++ b/app/views/organizations/organizations/activity.html.haml @@ -0,0 +1 @@ +- page_title _("Activity") diff --git a/config/feature_flags/gitlab_com_derisk/use_primary_for_update_computations.yml b/config/feature_flags/gitlab_com_derisk/use_primary_for_update_computations.yml deleted file mode 100644 index 9b6608c3e20..00000000000 --- a/config/feature_flags/gitlab_com_derisk/use_primary_for_update_computations.yml +++ /dev/null @@ -1,10 +0,0 @@ - ---- -name: use_primary_for_update_computations -feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/416207 -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/145792 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/443293 -milestone: '16.10' -group: group::project management -type: gitlab_com_derisk -default_enabled: false diff --git a/config/routes/organizations.rb b/config/routes/organizations.rb index 00a00b25e9f..442ffaca37b 100644 --- a/config/routes/organizations.rb +++ b/config/routes/organizations.rb @@ -1,16 +1,12 @@ # frozen_string_literal: true -resources( - :organizations, - only: [:show, :index, :new], - param: :organization_path, - module: :organizations -) do +resources(:organizations, only: [:show, :index, :new], param: :organization_path, module: :organizations) do collection do post :preview_markdown end member do + get :activity get :groups_and_projects get :users diff --git a/db/docs/batched_background_migrations/backfill_archived_and_traversal_ids_to_vulnerability_reads.yml b/db/docs/batched_background_migrations/backfill_archived_and_traversal_ids_to_vulnerability_reads.yml new file mode 100644 index 00000000000..cddb591e186 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_archived_and_traversal_ids_to_vulnerability_reads.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillArchivedAndTraversalIdsToVulnerabilityReads +description: Backfill project.archived and project.namespace.traversal_ids values to the denormalized columns of the same name on vulnerability_reads +feature_category: vulnerability_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/144765 +milestone: '16.10' +queued_migration_version: 20240214163238 +finalize_after: '2024-03-15' +finalized_by: \ No newline at end of file diff --git a/db/post_migrate/20240210104125_ensure_member_roles_names_uniq.rb b/db/post_migrate/20240210104125_ensure_member_roles_names_uniq.rb new file mode 100644 index 00000000000..c219b4e5e0f --- /dev/null +++ b/db/post_migrate/20240210104125_ensure_member_roles_names_uniq.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class EnsureMemberRolesNamesUniq < Gitlab::Database::Migration[2.2] + milestone '16.10' + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + sql = <<~SQL + UPDATE member_roles SET name = CONCAT(name, ' (', id, ')') + WHERE id IN ( + SELECT mr.id FROM member_roles mr + WHERE EXISTS (SELECT mr_duplicates.id + FROM member_roles mr_duplicates + WHERE mr_duplicates.name = mr.name + AND ( + mr_duplicates.namespace_id = mr.namespace_id + OR (mr_duplicates.namespace_id IS NULL AND mr.namespace_id IS NULL) + ) + AND mr_duplicates.id < mr.id)) + SQL + + execute(sql) + end + + def down; end +end diff --git a/db/post_migrate/20240214163238_queue_backfill_archived_and_traversal_ids_to_vulnerability_reads.rb b/db/post_migrate/20240214163238_queue_backfill_archived_and_traversal_ids_to_vulnerability_reads.rb new file mode 100644 index 00000000000..f8e70a11b56 --- /dev/null +++ b/db/post_migrate/20240214163238_queue_backfill_archived_and_traversal_ids_to_vulnerability_reads.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueBackfillArchivedAndTraversalIdsToVulnerabilityReads < Gitlab::Database::Migration[2.2] + milestone '16.10' + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillArchivedAndTraversalIdsToVulnerabilityReads" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :vulnerability_reads, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :vulnerability_reads, :id, []) + end +end diff --git a/db/post_migrate/20240221134504_add_name_unique_index_to_member_roles.rb b/db/post_migrate/20240221134504_add_name_unique_index_to_member_roles.rb new file mode 100644 index 00000000000..594c5713121 --- /dev/null +++ b/db/post_migrate/20240221134504_add_name_unique_index_to_member_roles.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddNameUniqueIndexToMemberRoles < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '16.10' + + INDEX_WITH_NAMESPACE_NAME = 'index_member_roles_on_namespace_id_name_unique' + INDEX_NO_NAMESPACE_NAME = 'index_member_roles_on_name_unique' + + def up + add_concurrent_index :member_roles, [:namespace_id, :name], name: INDEX_WITH_NAMESPACE_NAME, + unique: true, where: 'namespace_id IS NOT NULL' + add_concurrent_index :member_roles, [:name], name: INDEX_NO_NAMESPACE_NAME, + unique: true, where: 'namespace_id IS NULL' + end + + def down + remove_concurrent_index_by_name :member_roles, INDEX_WITH_NAMESPACE_NAME + remove_concurrent_index_by_name :member_roles, INDEX_NO_NAMESPACE_NAME + end +end diff --git a/db/schema_migrations/20240210104125 b/db/schema_migrations/20240210104125 new file mode 100644 index 00000000000..ac9d7b9933f --- /dev/null +++ b/db/schema_migrations/20240210104125 @@ -0,0 +1 @@ +3eb5dbfdae669848eaf552cd6eb097fb139b41d18b614095ac8de67737b7c796 \ No newline at end of file diff --git a/db/schema_migrations/20240214163238 b/db/schema_migrations/20240214163238 new file mode 100644 index 00000000000..2d7efbec300 --- /dev/null +++ b/db/schema_migrations/20240214163238 @@ -0,0 +1 @@ +3b6c1f5d12ab7c97d8bc72064b1ef66b64ba8131c8a37a368d0ea58472e60fc8 \ No newline at end of file diff --git a/db/schema_migrations/20240221134504 b/db/schema_migrations/20240221134504 new file mode 100644 index 00000000000..7f35df05c15 --- /dev/null +++ b/db/schema_migrations/20240221134504 @@ -0,0 +1 @@ +e73649f6285a6ede1741169c62eecb474da9ddaa3c8c03e4f571d4621e3471d2 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 6493d38a72c..4ad48e8acde 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -25556,8 +25556,12 @@ CREATE INDEX index_member_approval_on_requested_by_id ON member_approvals USING CREATE INDEX index_member_approval_on_reviewed_by_id ON member_approvals USING btree (reviewed_by_id); +CREATE UNIQUE INDEX index_member_roles_on_name_unique ON member_roles USING btree (name) WHERE (namespace_id IS NULL); + CREATE INDEX index_member_roles_on_namespace_id ON member_roles USING btree (namespace_id); +CREATE UNIQUE INDEX index_member_roles_on_namespace_id_name_unique ON member_roles USING btree (namespace_id, name) WHERE (namespace_id IS NOT NULL); + CREATE INDEX index_member_roles_on_occupies_seat ON member_roles USING btree (occupies_seat); CREATE INDEX index_members_on_access_level ON members USING btree (access_level); diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 0d8bd737bd5..748dc40455f 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -1044,7 +1044,9 @@ See the limits in the [Add a design to an issue](../user/project/issues/design_m ### Max push size -The maximum allowed [push size](../administration/settings/account_and_limit_settings.md#max-push-size) is set to 5 GB. +The maximum allowed [push size](../administration/settings/account_and_limit_settings.md#max-push-size). + +Not set by default on self-managed instances. For GitLab.com, see [Account and limit settings](../user/gitlab_com/index.md#account-and-limit-settings) ### Webhooks and Project Services diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 039f938054f..0a807e0ebbd 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -254,6 +254,7 @@ the default value [is the same as for self-managed instances](../../administrati | [Maximum download file size when importing from source GitLab instances by direct transfer](../../administration/settings/import_and_export_settings.md#maximum-download-file-size-for-imports-by-direct-transfer) | 5 GiB | | Maximum attachment size | 100 MiB | | [Maximum decompressed file size for imported archives](../../administration/settings/import_and_export_settings.md#maximum-decompressed-file-size-for-imported-archives) | 25 GiB | +| [Maximum push size](../../administration/settings/account_and_limit_settings.md#max-push-size) | 5 GB | If you are near or over the repository size limit, you can either: diff --git a/lib/gitlab/background_migration/backfill_archived_and_traversal_ids_to_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_archived_and_traversal_ids_to_vulnerability_reads.rb new file mode 100644 index 00000000000..4a21e71b066 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_archived_and_traversal_ids_to_vulnerability_reads.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillArchivedAndTraversalIdsToVulnerabilityReads < BatchedMigrationJob + operation_name :backfill_archived_and_traversal_ids_in_vulnerability_reads_table + feature_category :vulnerability_management + + def perform + each_sub_batch do |sub_batch| + connection.exec_update(update_sql(sub_batch)) + end + end + + private + + def update_sql(sub_batch) + <<~SQL + UPDATE + vulnerability_reads + SET + traversal_ids = namespaces.traversal_ids, + archived = projects.archived + FROM + projects + INNER JOIN namespaces ON namespaces.id = projects.namespace_id + WHERE + vulnerability_reads.id IN (#{sub_batch.select(:id).to_sql}) AND + vulnerability_reads.project_id = projects.id + SQL + end + end + end +end diff --git a/lib/gitlab/click_house.rb b/lib/gitlab/click_house.rb index 81468ab2875..412c159adb0 100644 --- a/lib/gitlab/click_house.rb +++ b/lib/gitlab/click_house.rb @@ -9,3 +9,5 @@ module Gitlab end end end + +Gitlab::ClickHouse.prepend_mod_with('Gitlab::ClickHouse') diff --git a/lib/sidebars/organizations/menus/manage_menu.rb b/lib/sidebars/organizations/menus/manage_menu.rb index 1cea6ebe897..8e3306eca9e 100644 --- a/lib/sidebars/organizations/menus/manage_menu.rb +++ b/lib/sidebars/organizations/menus/manage_menu.rb @@ -21,12 +21,25 @@ module Sidebars override :configure_menu_items def configure_menu_items + activity_menu_item groups_and_projects_menu_item users_menu_item end private + def activity_menu_item + add_item( + ::Sidebars::MenuItem.new( + title: _('Activity'), + link: activity_organization_path(context.container), + super_sidebar_parent: ::Sidebars::Organizations::Menus::ManageMenu, + active_routes: { path: 'organizations/organizations#activity' }, + item_id: :organization_activity + ) + ) + end + def groups_and_projects_menu_item add_item( ::Sidebars::MenuItem.new( diff --git a/qa/Gemfile b/qa/Gemfile index 81d98bf869d..f749f8b3acb 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' gem 'gitlab-qa', '~> 14', '>= 14.2.1', require: 'gitlab/qa' -gem 'gitlab_quality-test_tooling', '~> 1.11.0', require: false +gem 'gitlab_quality-test_tooling', '~> 1.17.0', require: false gem 'gitlab-utils', path: '../gems/gitlab-utils' gem 'activesupport', '~> 7.0.8.1' # This should stay in sync with the root's Gemfile gem 'allure-rspec', '~> 2.24.1' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 13b69c8dae8..953c62484ed 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -128,14 +128,15 @@ GEM rainbow (>= 3, < 4) table_print (= 1.5.7) zeitwerk (>= 2, < 3) - gitlab_quality-test_tooling (1.11.0) - activesupport (>= 6.1, < 7.2) + gitlab_quality-test_tooling (1.17.0) + activesupport (>= 6.1, < 7.1) amatch (~> 0.4.1) gitlab (~> 4.19) http (~> 5.0) nokogiri (~> 1.10) parallel (>= 1, < 2) rainbow (>= 3, < 4) + rspec-parameterized (~> 1.0.0) table_print (= 1.5.7) zeitwerk (>= 2, < 3) google-apis-compute_v1 (0.51.0) @@ -296,7 +297,8 @@ GEM ruby-debug-ide (0.7.3) rake (>= 0.8.1) ruby2_keywords (0.0.5) - ruby_parser (3.20.3) + ruby_parser (3.21.0) + racc (~> 1.5) sexp_processor (~> 4.16) rubyzip (2.3.2) sawyer (0.9.2) @@ -307,7 +309,7 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - sexp_processor (4.17.0) + sexp_processor (4.17.1) signet (0.17.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) @@ -360,7 +362,7 @@ DEPENDENCIES fog-google (~> 1.19) gitlab-qa (~> 14, >= 14.2.1) gitlab-utils! - gitlab_quality-test_tooling (~> 1.11.0) + gitlab_quality-test_tooling (~> 1.17.0) influxdb-client (~> 3.1) knapsack (~> 4.0) nokogiri (~> 1.16, >= 1.16.2) diff --git a/qa/qa/page/project/web_ide/vscode.rb b/qa/qa/page/project/web_ide/vscode.rb index 013f2b9cc67..e52f1d54174 100644 --- a/qa/qa/page/project/web_ide/vscode.rb +++ b/qa/qa/page/project/web_ide/vscode.rb @@ -128,6 +128,14 @@ module QA page.within_frame(iframe, &block) end + def within_vscode_duo_chat(&block) + within_vscode_editor do + within_frame(all(:frame, class: 'webview', visible: false).last) do + within_frame(:frame, &block) + end + end + end + def switch_to_original_window page.driver.browser.switch_to.window(page.driver.browser.window_handles.first) end @@ -156,7 +164,7 @@ module QA end Support::WaitForRequests.wait_for_requests(finish_loading_wait: 30) - Support::Waiter.wait_until(max_duration: 10, reload_page: page, retry_on_exception: true) do + Support::Waiter.wait_until(max_duration: 60, reload_page: page, retry_on_exception: true) do within_vscode_editor do # Check for webide file_explorer element has_file_explorer? @@ -304,6 +312,12 @@ module QA end end + def open_duo_chat + within_vscode_editor do + click_element('a[aria-label="GitLab Duo Chat"]', wait: 60) + end + end + private def create_item(click_item, item_name) diff --git a/spec/lib/gitlab/background_migration/backfill_archived_and_traversal_ids_to_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/backfill_archived_and_traversal_ids_to_vulnerability_reads_spec.rb new file mode 100644 index 00000000000..f59ed81c4de --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_archived_and_traversal_ids_to_vulnerability_reads_spec.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillArchivedAndTraversalIdsToVulnerabilityReads, + feature_category: :vulnerability_management do + let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let(:vulnerability_findings) { table(:vulnerability_occurrences) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_reads) { table(:vulnerability_reads) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:namespaces) { table(:namespaces) } + let(:scanners) { table(:vulnerability_scanners) } + + let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 1) } + + let(:args) do + min, max = vulnerability_reads.pick('MIN(id)', 'MAX(id)') + + { + start_id: min, + end_id: max, + batch_table: 'vulnerability_reads', + batch_column: 'id', + sub_batch_size: 100, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let!(:group_namespace) do + namespaces.create!( + name: 'gitlab-org', + path: 'gitlab-org', + type: 'Group' + ).tap { |namespace| namespace.update!(traversal_ids: [namespace.id]) } + end + + let!(:other_group_namespace) do + namespaces.create!( + name: 'gitlab-com', + path: 'gitlab-com', + type: 'Group' + ).tap { |namespace| namespace.update!(traversal_ids: [namespace.id]) } + end + + let!(:project) { create_project('gitlab', group_namespace) } + let!(:other_project) { create_project('www-gitlab-com', other_group_namespace) } + + subject(:perform_migration) { described_class.new(**args).perform } + + before do + [project, other_project].each do |p| + create_vulnerability_read(project_id: p.id) + end + end + + it 'backfills traversal_ids and archived', :aggregate_failures do + perform_migration + + vulnerability_reads.find_each do |vulnerability_read| + project = projects.find(vulnerability_read.project_id) + namespace = namespaces.find(project.namespace_id) + + expect(vulnerability_read.traversal_ids).to eq(namespace.traversal_ids) + expect(vulnerability_read.archived).to eq(project.archived) + end + end + + def create_vulnerability_read(project_id:) + scanner = scanners.create!(project_id: project_id, external_id: 'foo', name: 'bar') + + identifier = vulnerability_identifiers.create!( + project_id: project_id, + external_id: "CVE-2018-1234", + external_type: "CVE", + name: "CVE-2018-1234", + fingerprint: SecureRandom.hex(20) + ) + + finding = vulnerability_findings.create!( + project_id: project_id, + scanner_id: scanner.id, + severity: 5, # medium + confidence: 2, # unknown, + report_type: 99, # generic + primary_identifier_id: identifier.id, + project_fingerprint: SecureRandom.hex(20), + location_fingerprint: SecureRandom.hex(20), + uuid: SecureRandom.uuid, + name: "CVE-2018-1234", + raw_metadata: "{}", + metadata_version: "test:1.0" + ) + + vulnerability = vulnerabilities.create!( + project_id: project_id, + author_id: user.id, + title: 'Vulnerability 1', + severity: 1, + confidence: 1, + report_type: 1, + state: 1, + finding_id: finding.id + ) + + vulnerability_reads.create!( + vulnerability_id: vulnerability.id, + namespace_id: project.namespace_id, + project_id: project_id, + scanner_id: scanner.id, + report_type: 1, + severity: 1, + state: 1, + uuid: SecureRandom.uuid, + archived: false, + traversal_ids: [] + ) + end + + def create_project(name, group) + project_namespace = namespaces.create!( + name: name, + path: name, + type: 'Project' + ) + + projects.create!( + namespace_id: group.id, + project_namespace_id: project_namespace.id, + name: name, + path: name, + archived: true + ) + end +end diff --git a/spec/lib/sidebars/organizations/menus/manage_menu_spec.rb b/spec/lib/sidebars/organizations/menus/manage_menu_spec.rb index 7f1dab6a8b4..6b311850411 100644 --- a/spec/lib/sidebars/organizations/menus/manage_menu_spec.rb +++ b/spec/lib/sidebars/organizations/menus/manage_menu_spec.rb @@ -17,6 +17,12 @@ RSpec.describe Sidebars::Organizations::Menus::ManageMenu, feature_category: :na describe 'Menu items' do subject(:item) { menu.renderable_items.find { |e| e.item_id == item_id } } + describe 'Activity' do + let(:item_id) { :organization_activity } + + it { is_expected.not_to be_nil } + end + describe 'Groups and projects' do let(:item_id) { :organization_groups_and_projects } diff --git a/spec/migrations/20240210104125_ensure_member_roles_names_uniq_spec.rb b/spec/migrations/20240210104125_ensure_member_roles_names_uniq_spec.rb new file mode 100644 index 00000000000..0f89e29b711 --- /dev/null +++ b/spec/migrations/20240210104125_ensure_member_roles_names_uniq_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe EnsureMemberRolesNamesUniq, feature_category: :permissions do + let(:migration) { described_class.new } + + let(:namespaces) { table(:namespaces) } + let(:member_roles) { table(:member_roles) } + + let(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } + + let!(:member_role_1) { member_roles.create!(name: 'foo', namespace_id: namespace.id, base_access_level: 10) } + let!(:member_role_2) { member_roles.create!(name: 'other', namespace_id: namespace.id, base_access_level: 10) } + let!(:member_role_3) { member_roles.create!(name: 'other', namespace_id: nil, base_access_level: 10) } + let!(:member_role_4) { member_roles.create!(name: 'no namespace', namespace_id: nil, base_access_level: 10) } + let!(:member_role_1_duplicated) do + member_roles.create!(name: 'foo', namespace_id: namespace.id, base_access_level: 10) + end + + let!(:member_role_4_duplicated) do + member_roles.create!(name: 'no namespace', namespace_id: nil, base_access_level: 10) + end + + describe '#up' do + it 'updates the duplicated names with higher id', :aggregate_failures do + migration.up + + expect(member_role_1.reload.name).to eq('foo') + expect(member_role_1_duplicated.reload.name).to eq("foo (#{member_role_1_duplicated.id})") + expect(member_role_2.reload.name).to eq('other') + expect(member_role_3.reload.name).to eq('other') + expect(member_role_4.reload.name).to eq('no namespace') + expect(member_role_4_duplicated.reload.name).to eq("no namespace (#{member_role_4_duplicated.id})") + + migration.down + migration.up + + expect(member_role_1.reload.name).to eq('foo') + expect(member_role_1_duplicated.reload.name).to eq("foo (#{member_role_1_duplicated.id})") + expect(member_role_2.reload.name).to eq('other') + expect(member_role_3.reload.name).to eq('other') + expect(member_role_4.reload.name).to eq('no namespace') + expect(member_role_4_duplicated.reload.name).to eq("no namespace (#{member_role_4_duplicated.id})") + end + end +end diff --git a/spec/migrations/20240214163238_queue_backfill_archived_and_traversal_ids_to_vulnerability_reads_spec.rb b/spec/migrations/20240214163238_queue_backfill_archived_and_traversal_ids_to_vulnerability_reads_spec.rb new file mode 100644 index 00000000000..8fc5c6b9c40 --- /dev/null +++ b/spec/migrations/20240214163238_queue_backfill_archived_and_traversal_ids_to_vulnerability_reads_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillArchivedAndTraversalIdsToVulnerabilityReads, feature_category: :vulnerability_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :vulnerability_reads, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end diff --git a/spec/requests/organizations/organizations_controller_spec.rb b/spec/requests/organizations/organizations_controller_spec.rb index fbe37442064..ba8edfcf05e 100644 --- a/spec/requests/organizations/organizations_controller_spec.rb +++ b/spec/requests/organizations/organizations_controller_spec.rb @@ -96,6 +96,47 @@ RSpec.describe Organizations::OrganizationsController, feature_category: :cell d it_behaves_like 'controller action that does not require authentication' end + describe 'GET #activity' do + subject(:gitlab_request) { get activity_organization_path(organization) } + + it_behaves_like 'controller action that does not require authentication' + + context 'when requested in json format' do + let_it_be(:user) { create(:organization_user, organization: organization).user } + + context 'without activities' do + before do + sign_in(user) + end + + it 'renders empty array' do + get activity_organization_path(organization, format: :json) + + expect(response.media_type).to eq('application/json') + expect(json_response).to be_an(Array) + expect(json_response.size).to eq(0) + end + end + + context 'with activities' do + let_it_be(:project) { create(:project, organization: organization) } + + before_all do + project.add_developer(user) + sign_in(user) + end + + it 'loads events' do + get activity_organization_path(organization, format: :json) + + expect(response.media_type).to eq('application/json') + expect(json_response).to be_an(Array) + expect(json_response.size).to eq(1) + end + end + end + end + describe 'GET #groups_and_projects' do subject(:gitlab_request) { get groups_and_projects_organization_path(organization) } diff --git a/spec/routing/organizations/organizations_controller_routing_spec.rb b/spec/routing/organizations/organizations_controller_routing_spec.rb index c5c7a0ae377..26ea94724dd 100644 --- a/spec/routing/organizations/organizations_controller_routing_spec.rb +++ b/spec/routing/organizations/organizations_controller_routing_spec.rb @@ -20,6 +20,11 @@ RSpec.describe Organizations::OrganizationsController, :routing, feature_categor .to route_to('organizations/organizations#index') end + it 'routes to #activity' do + expect(get("/-/organizations/#{organization.path}/activity")) + .to route_to('organizations/organizations#activity', organization_path: organization.path) + end + it 'routes to #groups_and_projects' do expect(get("/-/organizations/#{organization.path}/groups_and_projects")) .to route_to('organizations/organizations#groups_and_projects', organization_path: organization.path)