Add latest changes from gitlab-org/gitlab@14-9-stable-ee

app/services/ci/register_job_service.rb

@@ -156,6 +156,18 @@ module Ci
     def process_build(build, params)
       unless build.pending?
         @metrics.increment_queue_operation(:build_not_pending)
+
+        if Feature.enabled?(:ci_pending_builds_table_resiliency, default_enabled: :yaml)
+          ##
+          # If this build can not be picked because we had stale data in
+          # `ci_pending_builds` table, we need to respond with 409 to retry
+          # this operation.
+          #
+          if ::Ci::UpdateBuildQueueService.new.remove!(build)
+            return Result.new(nil, nil, false)
+          end
+        end
+
         return
       end
 

app/services/ci/update_build_queue_service.rb

@@ -37,14 +37,19 @@ module Ci
 
       raise InvalidQueueTransition unless transition.from == 'pending'
 
-      transition.within_transaction do
-        removed = build.all_queuing_entries.delete_all
+      transition.within_transaction { remove!(build) }
+    end
 
-        if removed > 0
-          metrics.increment_queue_operation(:build_queue_pop)
+    ##
+    # Force recemove build from the queue, without checking a transition state
+    #
+    def remove!(build)
+      removed = build.all_queuing_entries.delete_all
 
-          build.id
-        end
+      if removed > 0
+        metrics.increment_queue_operation(:build_queue_pop)
+
+        build.id
       end
     end
 

config/feature_flags/development/ci_pending_builds_table_resiliency.yml

@@ -0,0 +1,8 @@
+---
+name: ci_pending_builds_table_resiliency
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84359
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357988
+milestone: '14.10'
+type: development
+group: group::pipeline execution
+default_enabled: true

data/whats_new/202203210001_14_09.yml

@@ -0,0 +1,85 @@
+- title: "Link an epic to another epic"
+  body: |
+    GitLab now supports linking epics using "related", "blocking," or "blocked" relationships. This feature enables teams to better track and manage epic dependencies across GitLab groups. Effective dependency management is a key component of reducing variability and increasing predictability in value delivery.
+  stage: plan
+  self-managed: true
+  gitlab-com: true
+  packages: [Ultimate]
+  url: 'https://docs.gitlab.com/ee/user/group/epics/linked_epics.html'
+  image_url: 'https://about.gitlab.com/images/14_9/related_epics_add.png'
+  published_at: 2022-03-22
+  release: 14.9
+- title: "Rule mode for scan result policies"
+  body: |
+    With the GitLab 14.9 release, users can now use rule mode to design and edit scan result policies without needing to edit the policy's YAML directly. This new UI editor makes it easier for users who want to create and manage MR approval rules that are triggered when a given threshold of vulnerabilities are detected in the MR.
+
+    To get started with this new rule mode, navigate to **Security & Compliance > Policies** and create a new Scan Result policy.
+  stage: protect
+  self-managed: true
+  gitlab-com: true
+  packages: [Ultimate]
+  url: 'https://docs.gitlab.com/ee/user/application_security/policies/#policy-editor'
+  image_url: 'https://about.gitlab.com/images/14_9/protect-scan-result-policy-rule-mode.png'
+  published_at: 2022-03-22
+  release: 14.9
+- title: "Deployment Approval on the Environments page"
+  body: |
+    We are excited to introduce the Deployment Approval capability in the GitLab interface. In GitLab 14.8, we introduced the ability to approve deployments via the [API](https://docs.gitlab.com/ee/ci/environments/deployment_approvals.html#using-the-api). Now, deployment approvers can view a pending deployment and approve or reject it conveniently directly in the Environments page. This update continues our work to enable teams to create workflows for approving software to go to production or other protected environments. With this update, we are now upgrading the feature to beta.
+  stage: "Release"
+  self-managed: true
+  gitlab-com: true
+  packages: [Premium, Ultimate]
+  url: 'https://docs.gitlab.com/ee/ci/environments/deployment_approvals.html#approve-or-reject-a-deployment'
+  image_url: 'https://about.gitlab.com/images/growth/release.png'
+  published_at: 2022-03-22
+  release: 14.9
+- title: "New design for the Environments Page"
+  body: |
+    Previously, the Environments page enabled you to operate and understand deployments but the design hid some important information and was difficult to read. In GitLab 14.9, we made a comprehensive update to the page so that you can answer key questions about your environments and deployments. Now, you can easily see the status of the latest deployment, the status for various environments, and which commits have been deployed.
+  stage: "Release"
+  self-managed: true
+  gitlab-com: true
+  packages: [Free, Premium, Ultimate]
+  url: 'https://docs.gitlab.com/ee/ci/environments/#view-environments-and-deployments'
+  image_url: 'https://about.gitlab.com/images/14_9/release-enviroments-page-redesign.png'
+  published_at: 2022-03-22
+  release: 14.9
+- title: "Project Level Time to restore service API"
+  body: |
+    In this release, we added API support for Time to Restore Service. This is the 3rd of the 4 [DORA Metrics](https://docs.gitlab.com/ee/user/analytics/ci_cd_analytics.html#devops-research-and-assessment-dora-key-metrics). This data helps teams continuously improve in their stability metrics.
+  stage: manage
+  self-managed: true
+  gitlab-com: true
+  packages: [Ultimate]
+  url: 'https://docs.gitlab.com/ee/api/dora/metrics.html'
+  image_url: 'https://about.gitlab.com/images/14_9/ttr_api.png'
+  published_at: 2022-03-22
+  release: 14.9
+- title: "Integrated security training"
+  body: |
+    GitLab provides a comprehensive set of [security scanning tools](https://docs.gitlab.com/ee/user/application_security/#security-scanning-tools)
+    that can identify all manner of security issues. Scanner findings are presented
+    in merge requests, pipelines, and in a dedicated Vulnerability Report. When
+    available, a recommended solution is given. However, this is not possible for
+    all findings. Presenting security findings without guidance on how to fix identified
+    problems or explaining the problem’s potential impact can be challenging for
+    anyone not familiar with the specific security issue identified. This increases
+    the time and friction involved in assessing and ultimately fixing security issues — especially
+    in developer workflows.
+
+    We’re pleased to announce the launch of our new
+    integrated security training functionality. Two new partners are providing the
+    training content. GitLab is already where many developers are working, so we
+    designed a solution to provide context-aware security training options from
+    inside the GitLab experience.
+
+    Simply enable security training for your projects, select your preferred content sources, and view the results from a security scan. In the vulnerability finding, you'll find a direct link to the security training that most closely matches the particular security issue, and the specific language or framework in which it was detected. Now developers can spend a few quick minutes reviewing targeted, context-relevant training to address security issues as part of their
+    normal development workflow.
+  stage: secure
+  self-managed: true
+  gitlab-com: true
+  packages: [Ultimate]
+  url: 'https://docs.gitlab.com/ee/user/application_security/vulnerabilities/#enable-security-training-for-vulnerabilities'
+  image_url: 'https://about.gitlab.com/images/14_9/secure-enable-security-training.png'
+  published_at: 2022-03-22
+  release: 14.9

lib/gitlab/database/migration_helpers.rb

@@ -961,7 +961,7 @@ module Gitlab
             "\n\n" \
             "Finalize it manualy by running" \
             "\n\n" \
-            "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.inspect.gsub(',', '\,')}']" \
+            "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \
             "\n\n" \
             "For more information, check the documentation" \
             "\n\n" \

lib/gitlab/url_blocker.rb

@@ -289,9 +289,10 @@ module Gitlab
         ObjectStoreSettings::SUPPORTED_TYPES.collect do |type|
           section_setting = config.try(type)
 
-          next unless section_setting
+          next unless section_setting && section_setting['enabled']
 
-          object_store_setting = section_setting['object_store']
+          # Use #to_h to avoid Settingslogic bug: https://gitlab.com/gitlab-org/gitlab/-/issues/286873
+          object_store_setting = section_setting['object_store']&.to_h
 
           next unless object_store_setting && object_store_setting['enabled']
 

lib/tasks/gitlab/background_migrations.rake

@@ -19,7 +19,7 @@ namespace :gitlab do
         args[:job_class_name],
         args[:table_name],
         args[:column_name],
-        Gitlab::Json.parse(args[:job_arguments]),
+        args[:job_arguments],
         connection: main_model.connection
       )
     end
@@ -38,7 +38,7 @@ namespace :gitlab do
             args[:job_class_name],
             args[:table_name],
             args[:column_name],
-            Gitlab::Json.parse(args[:job_arguments]),
+            args[:job_arguments],
             connection: model.connection
           )
         end

spec/lib/gitlab/database/migration_helpers_spec.rb

@@ -2211,7 +2211,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
         job_class_name: 'CopyColumnUsingBackgroundMigrationJob',
         table_name: :events,
         column_name: :id,
-        job_arguments: [["id"], ["id_convert_to_bigint"]]
+        job_arguments: [["id"], ["id_convert_to_bigint"], nil]
       }
     end
 
@@ -2226,7 +2226,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
             "\n\n" \
             "Finalize it manualy by running" \
             "\n\n" \
-            "\tsudo gitlab-rake gitlab:background_migrations:finalize[CopyColumnUsingBackgroundMigrationJob,events,id,'[[\"id\"]\\, [\"id_convert_to_bigint\"]]']" \
+            "\tsudo gitlab-rake gitlab:background_migrations:finalize[CopyColumnUsingBackgroundMigrationJob,events,id,'[[\"id\"]\\,[\"id_convert_to_bigint\"]\\,null]']" \
             "\n\n" \
             "For more information, check the documentation" \
             "\n\n" \

spec/lib/gitlab/import_export/command_line_util_spec.rb

@@ -167,6 +167,7 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do
       context 'for object_storage uri' do
         let(:enabled_object_storage_setting) do
           {
+            'enabled' => true,
             'object_store' =>
             {
               'enabled' => true,

spec/lib/gitlab/url_blocker_spec.rb

@@ -43,6 +43,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do
       let(:import_url) { "#{host}/external-diffs/merge_request_diffs/mr-1/diff-1" }
       let(:enabled_object_storage_setting) do
         {
+          'enabled' => true,
           'object_store' =>
           {
             'enabled' => true,
@@ -81,6 +82,49 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do
             let(:expected_hostname) { nil }
           end
         end
+
+        context 'when LFS object storage is enabled' do
+          let(:lfs_config) do
+            {
+              'enabled' => lfs_enabled,
+              # This nesting of Settingslogic is necessary to trigger the bug
+              'object_store' => Settingslogic.new({ 'enabled' => true })
+            }
+          end
+
+          let(:config) do
+            {
+              'gitlab' => Gitlab.config.gitlab,
+              'repositories' => { 'storages' => { 'default' => 'test' } },
+              'lfs' => Settingslogic.new(lfs_config)
+            }
+          end
+
+          let(:host) { 'http://127.0.0.1:9000' }
+          let(:settings) { Settingslogic.new(config) }
+
+          before do
+            allow(Gitlab).to receive(:config).and_return(settings)
+            # Triggers Settingslogic bug: https://gitlab.com/gitlab-org/gitlab/-/issues/286873
+            settings.repositories.storages.default
+          end
+
+          context 'when LFS is disabled' do
+            let(:lfs_enabled) { false }
+
+            it 'raises an error' do
+              expect { subject }.to raise_error(described_class::BlockedUrlError)
+            end
+          end
+
+          context 'when LFS is enabled with no connection endpoint' do
+            let(:lfs_enabled) { true }
+
+            it 'raises an error' do
+              expect { subject }.to raise_error(described_class::BlockedUrlError)
+            end
+          end
+        end
       end
 
       context 'when allow_object_storage is false' do

spec/services/ci/register_job_service_spec.rb

@@ -771,6 +771,25 @@ module Ci
 
           include_examples 'handles runner assignment'
         end
+
+        context 'when a conflicting data is stored in denormalized table' do
+          let!(:specific_runner) { create(:ci_runner, :project, projects: [project], tag_list: %w[conflict]) }
+          let!(:pending_job) { create(:ci_build, :pending, :queued, pipeline: pipeline, tag_list: %w[conflict]) }
+
+          before do
+            pending_job.update_column(:status, :running)
+          end
+
+          it 'removes queuing entry upon build assignment attempt' do
+            expect(pending_job.reload).to be_running
+            expect(pending_job.queuing_entry).to be_present
+
+            result = described_class.new(specific_runner).execute
+
+            expect(result).not_to be_valid
+            expect(pending_job.reload.queuing_entry).not_to be_present
+          end
+        end
       end
 
       context 'when not using pending builds table' do

spec/services/ci/update_build_queue_service_spec.rb

@@ -103,6 +103,28 @@ RSpec.describe Ci::UpdateBuildQueueService do
         end
       end
     end
+
+    describe '#remove!' do
+      context 'when pending build exists' do
+        before do
+          create(:ci_pending_build, build: build, project: build.project)
+        end
+
+        it 'removes pending build in a transaction' do
+          dequeued = subject.remove!(build)
+
+          expect(dequeued).to eq build.id
+        end
+      end
+
+      context 'when pending build does not exist' do
+        it 'does nothing if there is no pending build to remove' do
+          dequeued = subject.remove!(build)
+
+          expect(dequeued).to be_nil
+        end
+      end
+    end
   end
 
   describe 'shared runner builds tracking' do

spec/tasks/gitlab/background_migrations_rake_spec.rb

@@ -42,6 +42,17 @@ RSpec.describe 'gitlab:background_migrations namespace rake tasks' do
       end
     end
 
+    context 'with a null parameter' do
+      let(:arguments) { %w[ProjectNamespaces::BackfillProjectNamespaces projects id] + ['[null\, "up"]'] }
+
+      it 'finalizes the matching migration' do
+        expect(Gitlab::Database::BackgroundMigration::BatchedMigrationRunner).to receive(:finalize)
+          .with('ProjectNamespaces::BackfillProjectNamespaces', 'projects', 'id', [nil, "up"], connection: connection)
+
+        expect { finalize_task }.to output(/Done/).to_stdout
+      end
+    end
+
     context 'when multiple database feature is enabled' do
       subject(:finalize_task) { run_rake_task("gitlab:background_migrations:finalize:#{ci_database_name}", *arguments) }