Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/rails.gitlab-ci.yml

@@ -1315,7 +1315,7 @@ fail-pipeline-early:
     - job: "detect-previous-failed-tests"
   script:
     - !reference [.base-script, script]
-    - rspec_section rspec_rerun_previous_failed_tests ${PREVIOUS_FAILED_TESTS_FILE}
+    - rspec_section rspec_rerun_previous_failed_tests "${PREVIOUS_FAILED_TESTS_FILE}"
 
 rspec rspec-pg14-rerun-previous-failed-tests:
   extends:

.gitlab/ci/rails/shared.gitlab-ci.yml

@@ -27,10 +27,7 @@ include:
     - section_start "gitaly-test-spawn" "Spawning Gitaly"; scripts/gitaly-test-spawn; section_end "gitaly-test-spawn"  # Do not use 'bundle exec' here
     - export RSPEC_SKIPPED_TESTS_REPORT_PATH="rspec/skipped_tests-${CI_JOB_ID}.txt"
     - export RSPEC_RETRIED_TESTS_REPORT_PATH="rspec/retried_tests-${CI_JOB_ID}.txt"
-    - |
-        section_start "job-metrics" "Create Job Metrics file"
-          tooling/bin/create_job_metrics_file || true
-        section_end "job-metrics"
+    - tooling/bin/create_job_metrics_file || true
 
 .no-redis-cluster:
   variables:

.gitlab/ci/rules.gitlab-ci.yml

@@ -2650,10 +2650,16 @@
         QA_SAVE_TEST_METRICS: "true"
         QA_EXPORT_TEST_METRICS: "false"  # on main runs, metrics are exported to separate bucket via rake task for better consistency
 
-# The following rules needs to be the same as the one for .review:rules:start-review-app-pipeline
-# except that:
-# - all rules have `when: manual` and `allow_failure: true` here
-.review:rules:review-stop-merge-requests:
+.review:rules:review-cleanup:
+  rules:
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *ci-review-patterns
+      when: manual
+      allow_failure: true
+    - <<: *if-dot-com-ee-schedule-default-branch-maintenance
+      allow_failure: true
+
+.review:rules:review-stop:
   rules:
     - <<: *if-not-ee
       when: never
@@ -2662,6 +2668,10 @@
     - <<: *if-merge-request-labels-run-review-app
       when: manual
       allow_failure: true
+    - if: '$CI_REVIEW_APPS_ENABLED != "true"'
+      when: never
+    - <<: *if-merge-request-not-approved
+      when: never
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *ci-review-patterns
       when: manual
@@ -2690,16 +2700,6 @@
       changes: *code-patterns
       when: manual
       allow_failure: true
-
-.review:rules:review-cleanup:
-  rules:
-    - !reference [".review:rules:review-stop-merge-requests", rules]
-    - <<: *if-dot-com-ee-schedule-default-branch-maintenance
-      allow_failure: true
-
-.review:rules:review-stop:
-  rules:
-    - !reference [".review:rules:review-stop-merge-requests", rules]
     - <<: *if-dot-com-gitlab-org-schedule
       when: manual
       allow_failure: true

app/controllers/import/github_controller.rb

@@ -50,7 +50,7 @@ class Import::GithubController < Import::BaseController
   end
 
   def status
-    @fine_grained = Gitlab::GithubImport.fine_grained_token?(session[access_token_key])
+    @fine_grained = Gitlab::GithubImport.fine_grained_personal_token?(session[access_token_key])
 
     client_repos
 

app/finders/namespaces/projects_finder.rb

@@ -84,7 +84,7 @@ module Namespaces
       return items.order_by_storage_size(:desc) if params[:sort] == :storage_size_desc
 
       if params[:sort] == :similarity && params[:search].present?
-        return items.sorted_by_similarity_desc(params[:search], include_in_select: true)
+        return items.sorted_by_similarity_desc(params[:search])
       end
 
       items.sort_by_attribute(params[:sort])

app/finders/projects_finder.rb

@@ -267,7 +267,7 @@ class ProjectsFinder < UnionFinder
     return items.projects_order_id_desc unless params[:sort]
 
     if params[:sort] == 'similarity' && params[:search].present?
-      return items.sorted_by_similarity_desc(params[:search], include_in_select: true)
+      return items.sorted_by_similarity_desc(params[:search])
     end
 
     items.sort_by_attribute(params[:sort])

app/models/project.rb

@@ -654,14 +654,21 @@ class Project < ApplicationRecord
     )
   end
 
-  scope :sorted_by_similarity_desc, -> (search, include_in_select: false) do
+  scope :sorted_by_similarity_desc, -> (search, full_path_only: false) do
+    rules = if full_path_only
+              [{ column: arel_table["path"], multiplier: 1 }]
+            else
+              [
+                { column: arel_table["path"], multiplier: 1 },
+                { column: arel_table["name"], multiplier: 0.7 },
+                { column: arel_table["description"], multiplier: 0.2 }
+              ]
+            end
+
     order_expression = Gitlab::Database::SimilarityScore.build_expression(
       search: search,
-      rules: [
-        { column: arel_table["path"], multiplier: 1 },
-        { column: arel_table["name"], multiplier: 0.7 },
-        { column: arel_table["description"], multiplier: 0.2 }
-      ])
+      rules: rules
+    )
 
     order = Gitlab::Pagination::Keyset::Order.build(
       [

app/services/import/github_service.rb

@@ -16,14 +16,15 @@ module Import
       return context_error if context_error
 
       if provider == :github # we skip scope validation for Gitea importer calls
+        token = access_params[:github_access_token]
 
-        if Gitlab::GithubImport.fine_grained_token?(access_params[:github_access_token])
+        if Gitlab::GithubImport.fine_grained_personal_token?(token)
           Gitlab::GithubImport::Logger.info(
             message: 'Fine grained GitHub personal access token used.'
           )
           warning = s_('GithubImport|Fine-grained personal access tokens are not officially supported. ' \
                        'It is recommended to use a classic token instead.')
-        else
+        elsif Gitlab::GithubImport.classic_personal_token?(token)
           scope_error = validate_scopes
           return scope_error if scope_error
         end

app/views/dashboard/todos/_todo.html.haml

@@ -64,6 +64,6 @@
           = gl_loading_icon(inline: true)
           = sprite_icon('redo', css_class: 'js-todo-button-icon')
       - else
-        = render Pajamas::ButtonComponent.new(button_options: { class: 'btn-loading btn-icon gl-display-flex js-add-todo has-tooltip', title: _('Add a to do')}, method: :patch, href: restore_dashboard_todo_path(todo)), 'aria-label' => _('Add a to do') do
+        = render Pajamas::ButtonComponent.new(button_options: { class: 'btn-loading btn-icon gl-display-flex js-add-todo has-tooltip', title: _('Re-add this to-do item')}, method: :patch, href: restore_dashboard_todo_path(todo)), 'aria-label' => _('Re-add this to-do item') do
           = gl_loading_icon(inline: true)
-          = sprite_icon('todo-add', css_class: 'js-todo-button-icon')
+          = sprite_icon('redo', css_class: 'js-todo-button-icon')

config/feature_flags/development/fetch_commits_for_bitbucket_server.yml

@@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/427699
 milestone: '16.5'
 type: development
 group: group::import and integrate
-default_enabled: false
+default_enabled: true

doc/api/graphql/reference/index.md

@@ -21038,6 +21038,23 @@ four standard [pagination arguments](#pagination-arguments):
 | ---- | ---- | ----------- |
 | <a id="groupscanresultpoliciesrelationship"></a>`relationship` | [`SecurityPolicyRelationType`](#securitypolicyrelationtype) | Filter policies by the given policy relationship. |
 
+##### `Group.securityPolicyProjectSuggestions`
+
+Security policy project suggestions.
+
+Returns [`ProjectConnection`](#projectconnection).
+
+This field returns a [connection](#connections). It accepts the
+four standard [pagination arguments](#pagination-arguments):
+`before: String`, `after: String`, `first: Int`, and `last: Int`.
+
+###### Arguments
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| <a id="groupsecuritypolicyprojectsuggestionsonlylinked"></a>`onlyLinked` | [`Boolean`](#boolean) | Whether to suggest only projects already linked as security policy projects. |
+| <a id="groupsecuritypolicyprojectsuggestionssearch"></a>`search` | [`String!`](#string) | Search query for projects' full paths. |
+
 ##### `Group.timelogs`
 
 Time logged on issues and merge requests in the group and its subgroups.
@@ -26609,6 +26626,23 @@ four standard [pagination arguments](#pagination-arguments):
 | ---- | ---- | ----------- |
 | <a id="projectscanresultpoliciesrelationship"></a>`relationship` | [`SecurityPolicyRelationType`](#securitypolicyrelationtype) | Filter policies by the given policy relationship. |
 
+##### `Project.securityPolicyProjectSuggestions`
+
+Security policy project suggestions.
+
+Returns [`ProjectConnection`](#projectconnection).
+
+This field returns a [connection](#connections). It accepts the
+four standard [pagination arguments](#pagination-arguments):
+`before: String`, `after: String`, `first: Int`, and `last: Int`.
+
+###### Arguments
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| <a id="projectsecuritypolicyprojectsuggestionsonlylinked"></a>`onlyLinked` | [`Boolean`](#boolean) | Whether to suggest only projects already linked as security policy projects. |
+| <a id="projectsecuritypolicyprojectsuggestionssearch"></a>`search` | [`String!`](#string) | Search query for projects' full paths. |
+
 ##### `Project.securityTrainingProviders`
 
 List of security training providers for the project.

doc/development/cloud_connector/index.md

@@ -30,6 +30,12 @@ To connect a feature using Cloud Connector:
 
 #### GitLab Rails
 
+1. Call `CloudConnector::AccessService.new.access_token(scopes: [...])` with the list of scopes your feature requires and include
+this token in the `Authorization` HTTP header field.
+The backend service must validate this token and any scopes it carries when receiving the request.
+If you need to embed additional claims in the token specific to your use case, you can pass these
+in the `extra_claims` argument. **Scopes and other claims passed here will only be included in self-issued tokens on GitLab.com.**
+Refer to [CustomersDot](#customersdot) to see how custom claims are handled for self-managed instances.
 1. Ensure your request sends the required headers to the [backend service](#backend-service).
 
    These headers are:
@@ -37,7 +43,7 @@ To connect a feature using Cloud Connector:
    - `X-Gitlab-Instance-Id`: A globally unique instance ID string.
    - `X-Gitlab-Global-User-Id`: A globally unique anonymous user ID string.
    - `X-Gitlab-Realm`: One of `saas`, `self-managed`.
-   - `Authorization`: Contains the Base64-encoded JWT as a `Bearer` token.
+   - `Authorization`: Contains the Base64-encoded JWT as a `Bearer` token obtained from the `access_token` method in step 1.
 
    Some of these headers can be injected by merging the result of the `API::Helpers::CloudConnector#cloud_connector_headers`
    method to your payload.
@@ -46,14 +52,11 @@ The following example is for a request that includes the `new_feature_scope` sco
 Here we assume your backend service is called `foo` and is already reachable at `https://cloud.gitlab.com/foo`.
 We also assume that the backend service exposes the feature using a `/new_feature_endpoint` endpoint.
 This allows clients to access the feature at `https://cloud.gitlab.com/foo/new_feature_endpoint`.
-Call `CloudConnector::AccessService.access_token` with the list of scopes your feature requires and include
-this token in the `Authorization` HTTP header field.
-The backend service must validate this token and any scopes it carries when receiving the request.
 
 ```ruby
 include API::Helpers::CloudConnector
 
-token = ::CloudConnector::AccessService.new.access_token([:new_feature_scope])
+token = ::CloudConnector::AccessService.new.access_token(scopes: [:new_feature_scope])
 
 Gitlab::HTTP.post(
   "https://cloud.gitlab.com/foo/new_feature_endpoint",
@@ -91,14 +94,18 @@ To add a new feature bound to a scope:
     For example:
 
     ```yaml
-   defaults: &defaults
-        services:
-            new_feature_scope:
-                service_start_time: 2024-02-15 00:00:00 UTC
-                min_gitlab_version: '16.8'
-                bundled_with: 'duo_pro'
+    defaults: &defaults
+      services:
+        new_feature_scope:
+          service_start_time: 2024-02-15 00:00:00 UTC
+            min_gitlab_version: '16.8'
+            bundled_with: 'duo_pro'
     ```
 
+1. **Optional:** If the backend service the token is used for requires additional claims to be embedded in the
+   service access token, contact [#g_cloud_connector](https://gitlab.enterprise.slack.com/archives/CGN8BUCKC) (Slack, internal only)
+   since we do not currently have interfaces in place to self-service this.
+
 #### Backend service
 
 GitLab Rails calls a backend service to deliver functionality that would otherwise be unavailable to Self-managed and

lib/gitlab/github_import.rb

@@ -2,10 +2,14 @@
 
 module Gitlab
   module GithubImport
-    def self.fine_grained_token?(token)
+    def self.fine_grained_personal_token?(token)
       token&.start_with?('github_pat')
     end
 
+    def self.classic_personal_token?(token)
+      token&.start_with?('ghp_')
+    end
+
     def self.refmap
       [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head']
     end

locale/gitlab.pot

@@ -40951,6 +40951,9 @@ msgstr ""
 msgid "Raw blob requests"
 msgstr ""
 
+msgid "Re-add this to-do item"
+msgstr ""
+
 msgid "Re-authentication period expired or never requested. Please try again"
 msgstr ""
 

spec/controllers/import/github_controller_spec.rb

@@ -214,7 +214,7 @@ RSpec.describe Import::GithubController, feature_category: :importers do
       end
     end
 
-    context 'with fine_grained access token' do
+    context 'with fine_grained personal access token' do
       let(:client_auth_success) { true }
       let(:provider_token) { 'github_pat_23542334' }
 
@@ -227,6 +227,32 @@ RSpec.describe Import::GithubController, feature_category: :importers do
       end
     end
 
+    context 'with classic personal access token' do
+      let(:client_auth_success) { true }
+      let(:provider_token) { 'ghp_23542334' }
+
+      it 'sets fine_grained to false' do
+        get :status, format: :json
+
+        expect(session[:"#{provider}_access_token"]).to be(provider_token)
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(assigns(:fine_grained)).to be false
+      end
+    end
+
+    context 'with non-classic personal access token' do
+      let(:client_auth_success) { true }
+      let(:provider_token) { 'ghu_23542334' }
+
+      it 'sets fine grained to false' do
+        get :status, format: :json
+
+        expect(session[:"#{provider}_access_token"]).to be(provider_token)
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(assigns(:fine_grained)).to be false
+      end
+    end
+
     context 'when a gitea import' do
       let(:provider) { :gitea }
       let(:provider_token) { 'github_pat_23542334' }

spec/dot_gitlab_ci/rules_spec.rb

@@ -51,6 +51,29 @@ RSpec.describe '.gitlab/ci/rules.gitlab-ci.yml', feature_category: :tooling do
     end
   end
 
+  describe '.review:rules:start-review-app-pipeline' do
+    let(:base_rules) { config.dig('.review:rules:start-review-app-pipeline', 'rules') }
+
+    context 'with .review:rules:review-stop' do
+      let(:derived_rules) { config.dig('.review:rules:review-stop', 'rules') }
+
+      it 'has the same rules as the base, but automatic jobs changed to manual' do
+        base_rules.zip(derived_rules).each do |(base, derived)|
+          # .review:rules:review-stop don't set variables
+          base.delete('variables')
+          base_with_manual_and_allowed_to_fail =
+            if base['when'] == 'never'
+              base
+            else
+              base.merge('when' => 'manual', 'allow_failure' => true)
+            end
+
+          expect(derived).to eq(base_with_manual_and_allowed_to_fail)
+        end
+      end
+    end
+  end
+
   describe '.rails:rules:ee-and-foss-default-rules' do
     let(:base_rules) { config.dig('.rails:rules:ee-and-foss-default-rules', 'rules') }
 

spec/features/dashboard/todos/todos_spec.rb

@@ -303,7 +303,7 @@ RSpec.describe 'Dashboard Todos', :js, feature_category: :team_planning do
     describe 'restoring the todo' do
       before do
         within first('.todo') do
-          find_by_testid('todo-add-icon').click
+          find_by_testid('redo-icon').click
         end
       end
 

spec/lib/gitlab/github_import_spec.rb

@@ -13,9 +13,9 @@ RSpec.describe Gitlab::GithubImport, feature_category: :importers do
     it 'returns a new Client with a custom token' do
       expect(described_class::Client)
         .to receive(:new)
-        .with('123', host: nil, parallel: true, per_page: 100)
+        .with('ghp_123', host: nil, parallel: true, per_page: 100)
 
-      described_class.new_client_for(project, token: '123')
+      described_class.new_client_for(project, token: 'ghp_123')
     end
 
     it 'returns a new Client with a token stored in the import data' do
@@ -54,9 +54,9 @@ RSpec.describe Gitlab::GithubImport, feature_category: :importers do
     it 'returns a new Client with a custom token' do
       expect(described_class::Client)
         .to receive(:new)
-        .with('123', host: 'http://github.another-domain.com/api/v3', parallel: true, per_page: 100)
+        .with('ghp_123', host: 'http://github.another-domain.com/api/v3', parallel: true, per_page: 100)
 
-      described_class.new_client_for(project, token: '123')
+      described_class.new_client_for(project, token: 'ghp_123')
     end
 
     it 'returns a new Client with a token stored in the import data' do
@@ -126,10 +126,17 @@ RSpec.describe Gitlab::GithubImport, feature_category: :importers do
     end
   end
 
-  describe '.fine_grained_token?' do
+  describe '.fine_grained_personal_token?' do
     it 'detects a fine-grained token' do
-      expect(described_class.fine_grained_token?('github_pat_235234')).to eq(true)
-      expect(described_class.fine_grained_token?('gh_235234ab234234')).to eq(false)
+      expect(described_class.fine_grained_personal_token?('github_pat_235234')).to eq(true)
+      expect(described_class.fine_grained_personal_token?('ghp_235234ab234234')).to eq(false)
+    end
+  end
+
+  describe '.classic_personal_token?' do
+    it 'detects a fine-grained token' do
+      expect(described_class.classic_personal_token?('github_pat_235234')).to eq(false)
+      expect(described_class.classic_personal_token?('ghp_235234ab234234')).to eq(true)
     end
   end
 end

spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb

@@ -290,7 +290,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do
 
         let_it_be(:nodes) do
           # Note: sorted_by_similarity_desc scope internally supports the generic keyset order.
-          Project.sorted_by_similarity_desc('test', include_in_select: true)
+          Project.sorted_by_similarity_desc('test')
         end
 
         let_it_be(:descending_nodes) { nodes.to_a }

spec/models/project_spec.rb

@@ -2181,6 +2181,32 @@ RSpec.describe Project, factory_default: :keep, feature_category: :groups_and_pr
     end
   end
 
+  describe '.sorted_by_similarity_desc' do
+    let_it_be(:project_a) { create(:project, path: 'similar-1', name: 'similar-1', description: 'A similar project') }
+    let_it_be_with_reload(:project_b) { create(:project, path: 'similar-2', name: 'similar-2', description: 'A related project') }
+    let_it_be(:project_c) { create(:project, path: 'different-path', name: 'different-name', description: 'A different project') }
+
+    let(:search_term) { 'similar' }
+
+    subject(:relation) { described_class.sorted_by_similarity_desc(search_term) }
+
+    context 'when sorting with full similarity' do
+      it 'sorts projects based on path, name, and description similarity' do
+        expect(relation).to eq([project_a, project_b, project_c])
+      end
+    end
+
+    context 'when sorting with path-only similarity' do
+      let(:search_term) { 'diff' }
+
+      subject(:relation) { described_class.sorted_by_similarity_desc(search_term, full_path_only: true) }
+
+      it 'sorts projects based on path similarity only' do
+        expect(relation).to eq([project_c, project_b, project_a])
+      end
+    end
+  end
+
   describe '.with_shared_runners_enabled' do
     subject { described_class.with_shared_runners_enabled }
 

spec/requests/api/import_github_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe API::ImportGithub, feature_category: :importers do
-  let(:token) { "asdasd12345" }
+  let(:token) { "ghp_asdasd12345" }
   let(:provider) { :github }
   let(:access_params) { { github_access_token: token } }
   let(:provider_username) { user.username }
@@ -163,6 +163,28 @@ RSpec.describe API::ImportGithub, feature_category: :importers do
       end
     end
 
+    context 'with a non-classic token' do
+      let(:token) { 'ghu_asdasd12345' }
+
+      it 'proceeds with the import' do
+        allow(Gitlab::LegacyGithubImport::ProjectCreator)
+          .to receive(:new).with(provider_repo, provider_repo[:name], user.namespace, user, type: provider, **access_params)
+          .and_return(double(execute: project))
+
+        post api("/import/github", user), params: {
+          target_namespace: user.namespace_path,
+          personal_access_token: token,
+          repo_id: non_existing_record_id,
+          optional_stages: { collaborators_import: true }
+        }
+
+        expect(response).to have_gitlab_http_status(:created)
+        expect(json_response).to be_a Hash
+        expect(json_response['name']).to eq(project.name)
+        expect(json_response['import_warning']).to eq(nil)
+      end
+    end
+
     context 'when collaborators_import option is not in params' do
       context 'with minimum scope token' do
         let(:scopes) { ['repo'] }

spec/services/import/github_service_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 RSpec.describe Import::GithubService, feature_category: :importers do
   let_it_be(:user) { create(:user) }
   let_it_be(:token) { 'complex-token' }
-  let_it_be(:access_params) { { github_access_token: 'github-complex-token' } }
+  let_it_be(:access_params) { { github_access_token: 'ghp_complex-token' } }
 
   let(:settings) { instance_double(Gitlab::GithubImport::Settings) }
   let(:user_namespace_path) { user.namespace_path }
@@ -328,6 +328,29 @@ RSpec.describe Import::GithubService, feature_category: :importers do
       end
     end
 
+    context 'when a non-classic access token is used' do
+      let(:access_params) { { github_access_token: 'ghu_token' } }
+
+      before do
+        allow(subject).to receive(:repo).and_return(repository_double)
+      end
+
+      it 'does not validate scopes' do
+        expect(subject).not_to receive(:validate_scopes)
+
+        subject.execute(access_params, :github)
+      end
+
+      it 'does not log or return a warning message' do
+        expect(Gitlab::Import::Logger).not_to receive(:info).with({
+          message: 'Fine grained GitHub personal access token used.'
+        }).and_call_original
+
+        expect(subject.execute(access_params, :github))
+          .to include(nil_warning)
+      end
+    end
+
     context 'when the collaborator import option is true' do
       let(:optional_stages) { { collaborators_import: true } }
       let(:scopes) { ['repo', 'read:user'] }
@@ -502,6 +525,14 @@ RSpec.describe Import::GithubService, feature_category: :importers do
     }
   end
 
+  def nil_warning
+    {
+      status: :success,
+      project: project_double,
+      warning: nil
+    }
+  end
+
   def blocked_url_error(url)
     {
       status: :error,