Add latest changes from gitlab-org/gitlab@master

app/services/wiki_pages/base_service.rb

@@ -47,7 +47,8 @@ module WikiPages
         project: project,
         namespace: group,
         additional_properties: {
-          label: label
+          label: label,
+          property: page[:format].to_s
         }
       )
     end

app/workers/delete_user_worker.rb

@@ -11,7 +11,8 @@ class DeleteUserWorker # rubocop:disable Scalability/IdempotentWorker
   loggable_arguments 2
 
   def perform(current_user_id, delete_user_id, options = {})
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/464672')
+    # Deleting a user deletes many different resources, so a higher threshold is OK for now
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/464672', new_threshold: 200)
 
     delete_user = User.find_by_id(delete_user_id)
     return unless delete_user.present?

config/events/create_wiki_page.yml

@@ -9,6 +9,8 @@ identifiers:
 additional_properties:
   label:
     description: "Is 'template' when the page is a template"
+  property:
+    description: "Name of the markup language used by the page"
 product_group: knowledge
 milestone: '17.0'
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/149271

config/events/delete_wiki_page.yml

@@ -9,6 +9,8 @@ identifiers:
 additional_properties:
   label:
     description: "Is 'template' when the page is a template"
+  property:
+    description: "Name of the markup language used by the page"
 product_group: knowledge
 milestone: '17.0'
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/149271

config/events/update_wiki_page.yml

@@ -9,6 +9,8 @@ identifiers:
 additional_properties:
   label:
     description: "Is 'template' when the page is a template"
+  property:
+    description: "Name of the markup language used by the page"
 product_group: knowledge
 milestone: '17.0'
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/149271

config/metrics/counts_all/count_total_asciidoc_wiki_page_created.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_asciidoc_wiki_page_created
+description: Count of all Wiki pages written in AsciiDoc created
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: create_wiki_page
+    filter:
+      property: asciidoc
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_asciidoc_wiki_page_deleted.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_asciidoc_wiki_page_deleted
+description: Count of all Wiki pages written in AsciiDoc deleted
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: delete_wiki_page
+    filter:
+      property: asciidoc
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_asciidoc_wiki_page_updated.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_asciidoc_wiki_page_updated
+description: Count of all Wiki pages written in AsciiDoc updated
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: update_wiki_page
+    filter:
+      property: asciidoc
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_markdown_wiki_page_created.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_markdown_wiki_page_created
+description: Count of all Wiki pages written in Markdown created
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: create_wiki_page
+    filter:
+      property: markdown
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_markdown_wiki_page_deleted.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_markdown_wiki_page_deleted
+description: Count of all Wiki pages written in Markdown deleted
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: delete_wiki_page
+    filter:
+      property: markdown
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_markdown_wiki_page_updated.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_markdown_wiki_page_updated
+description: Count of all Wiki pages written in Markdown updated
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: update_wiki_page
+    filter:
+      property: markdown
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_org_wiki_page_created.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_org_wiki_page_created
+description: Count of all Wiki pages written in Org created
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: create_wiki_page
+    filter:
+      property: org
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_org_wiki_page_deleted.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_org_wiki_page_deleted
+description: Count of all Wiki pages written in Org deleted
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: delete_wiki_page
+    filter:
+      property: org
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_org_wiki_page_updated.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_org_wiki_page_updated
+description: Count of all Wiki pages written in Org updated
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: update_wiki_page
+    filter:
+      property: org
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_rdoc_wiki_page_created.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_rdoc_wiki_page_created
+description: Count of all Wiki pages written in RDoc created
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: create_wiki_page
+    filter:
+      property: rdoc
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_rdoc_wiki_page_deleted.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_rdoc_wiki_page_deleted
+description: Count of all Wiki pages written in RDoc deleted
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: delete_wiki_page
+    filter:
+      property: rdoc
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

config/metrics/counts_all/count_total_rdoc_wiki_page_updated.yml

@@ -0,0 +1,22 @@
+---
+data_category: optional
+key_path: counts.count_total_rdoc_wiki_page_updated
+description: Count of all Wiki pages written in RDoc updated
+product_group: knowledge
+value_type: number
+status: active
+milestone: "17.3"
+time_frame: all
+data_source: internal_events
+events:
+  - name: update_wiki_page
+    filter:
+      property: rdoc
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+performance_indicator_type: []

db/docs/batched_background_migrations/resync_epic_dates_to_work_items_dates_sources.yml

@@ -0,0 +1,9 @@
+---
+migration_job_name: ResyncEpicDatesToWorkItemsDatesSources
+description: We backfilled work_item_dates_sources with epic dates data in 17.1,
+  but we now need to re-do the migration due to a fix in the syncing mechanism in 17.3.
+feature_category: team_planning
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/161847
+milestone: '17.3'
+queued_migration_version: 20240806193258
+finalize_after: '2024-08-20'

db/post_migrate/20240806193258_queue_resync_epic_dates_to_work_items_dates_sources.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class QueueResyncEpicDatesToWorkItemsDatesSources < Gitlab::Database::Migration[2.2]
+  milestone '17.3'
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+  disable_ddl_transaction!
+
+  MIGRATION = "ResyncEpicDatesToWorkItemsDatesSources"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :epics,
+      :id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :epics, :id, [])
+  end
+end

db/schema_migrations/20240806193258

@@ -0,0 +1 @@
+4eb9e90053cf4d89789a7672990c521a65fccbd0dd900f30eba01971d7a88ee9

lib/gitlab/background_migration/resync_epic_dates_to_work_items_dates_sources.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Migration/BatchedMigrationBaseClass -- BackfillEpicDatesToWorkItemDatesSources inherits from BatchedMigrationJob.
+    class ResyncEpicDatesToWorkItemsDatesSources < BackfillEpicDatesToWorkItemDatesSources
+      operation_name :resync_epic_dates_to_work_items_dates_sources
+      feature_category :team_planning
+    end
+    # rubocop: enable Migration/BatchedMigrationBaseClass
+  end
+end

spec/lib/gitlab/background_migration/resync_epic_dates_to_work_items_dates_sources_spec.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::ResyncEpicDatesToWorkItemsDatesSources, feature_category: :team_planning do
+  let!(:epic_type_id) { table(:work_item_types).find_by(base_type: 7).id }
+  let!(:author) { table(:users).create!(username: 'tester', projects_limit: 100) }
+  let!(:namespace) { table(:namespaces).create!(name: 'my test group1', path: 'my-test-group1') }
+
+  let(:milestone) do
+    table(:milestones).create!(
+      title: 'Milestone',
+      start_date: DateTime.parse('2024-01-01'),
+      due_date: DateTime.parse('2024-01-31')
+    )
+  end
+
+  let(:epics) { table(:epics) }
+  let(:issues) { table(:issues) }
+  let(:work_item_dates_sources) { table(:work_item_dates_sources) }
+  let(:start_id) { epics.minimum(:id) }
+  let(:end_id) { epics.maximum(:id) }
+
+  let!(:fixed_epic_1) do
+    create_epic_with_work_item(title: 'Epic 5', iid: 5, date_attrs: with_fixed_dates('2024-02-01', '2024-02-29'))
+  end
+
+  let!(:fixed_epic_2) do
+    create_epic_with_work_item(title: 'Epic 6', iid: 6, date_attrs: with_fixed_dates('2024-03-01', '2024-03-31'))
+  end
+
+  let!(:fixed_epic_3) do
+    create_epic_with_work_item(title: 'Epic 7', iid: 7, date_attrs: with_fixed_dates('2024-04-01', '2024-04-30'))
+  end
+
+  let!(:fixed_epic_4) do
+    create_epic_with_work_item(title: 'Epic 8', iid: 8, date_attrs: with_fixed_dates('2024-05-01', '2024-05-31'))
+  end
+
+  let!(:fixed_epic_5) do
+    create_epic_with_work_item(title: 'Epic 9', iid: 9, date_attrs: with_fixed_dates('2024-06-01', '2024-06-30'))
+  end
+
+  let!(:rolledup_epic_1) do
+    create_epic_with_work_item(
+      title: 'Epic 10',
+      iid: 10,
+      date_attrs: {
+        start_date_is_fixed: false,
+        due_date_is_fixed: false,
+        start_date: fixed_epic_1.start_date,
+        end_date: milestone.due_date,
+        start_date_sourcing_milestone_id: nil,
+        due_date_sourcing_milestone_id: milestone.id,
+        start_date_sourcing_epic_id: fixed_epic_1.id,
+        due_date_sourcing_epic_id: nil
+      }
+    )
+  end
+
+  let!(:rolledup_epic_2) do
+    create_epic_with_work_item(
+      title: 'Epic 11',
+      iid: 11,
+      date_attrs: {
+        start_date_is_fixed: false,
+        due_date_is_fixed: false,
+        start_date: fixed_epic_2.start_date,
+        end_date: fixed_epic_3.end_date,
+        start_date_sourcing_milestone_id: nil,
+        due_date_sourcing_milestone_id: nil,
+        start_date_sourcing_epic_id: fixed_epic_2.id,
+        due_date_sourcing_epic_id: fixed_epic_3.id
+      }
+    )
+  end
+
+  let!(:rolledup_epic_3) do
+    create_epic_with_work_item(
+      title: 'Epic 12',
+      iid: 12,
+      date_attrs: {
+        start_date_is_fixed: false,
+        due_date_is_fixed: nil,
+        start_date_fixed: DateTime.parse('2024-07-01'),
+        due_date_fixed: DateTime.parse('2024-07-31'),
+        start_date: fixed_epic_4.start_date,
+        end_date: fixed_epic_4.end_date,
+        start_date_sourcing_milestone_id: nil,
+        due_date_sourcing_milestone_id: nil,
+        start_date_sourcing_epic_id: fixed_epic_4.id,
+        due_date_sourcing_epic_id: fixed_epic_4.id
+      }
+    )
+  end
+
+  let!(:rolledup_epic_4) do
+    create_epic_with_work_item(
+      title: 'Epic 13',
+      iid: 13,
+      date_attrs: {
+        start_date_is_fixed: false,
+        due_date_is_fixed: false,
+        start_date_fixed: DateTime.parse('2024-08-01'),
+        due_date_fixed: DateTime.parse('2024-08-31'),
+        start_date: fixed_epic_5.start_date,
+        end_date: fixed_epic_5.end_date,
+        start_date_sourcing_milestone_id: nil,
+        due_date_sourcing_milestone_id: nil,
+        start_date_sourcing_epic_id: fixed_epic_5.id,
+        due_date_sourcing_epic_id: fixed_epic_5.id
+      }
+    )
+  end
+
+  let!(:rolledup_epic_5) do
+    create_epic_with_work_item(
+      title: 'Epic 14',
+      iid: 14,
+      date_attrs: {
+        start_date_is_fixed: nil,
+        due_date_is_fixed: true,
+        start_date_fixed: DateTime.parse('2024-09-01'),
+        due_date_fixed: DateTime.parse('2024-09-30'),
+        start_date: fixed_epic_5.start_date,
+        end_date: DateTime.parse('2024-09-30'),
+        start_date_sourcing_milestone_id: nil,
+        due_date_sourcing_milestone_id: nil,
+        start_date_sourcing_epic_id: fixed_epic_5.id,
+        due_date_sourcing_epic_id: nil
+      }
+    )
+  end
+
+  # Existing date_source for fixed_epic_1 that is not in sync
+  let!(:not_synced_date_source) do
+    work_item_dates_sources.create!(namespace_id: namespace.id, issue_id: fixed_epic_1.issue_id)
+  end
+
+  # Existing date_source for rolledup_epic_4 that is fully synced
+  let!(:synced_date_source) do
+    work_item_dates_sources.create!(
+      namespace_id: namespace.id,
+      issue_id: rolledup_epic_4.issue_id,
+      start_date_is_fixed: rolledup_epic_4.start_date_is_fixed,
+      due_date_is_fixed: rolledup_epic_4.due_date_is_fixed,
+      start_date_fixed: rolledup_epic_4.start_date_fixed,
+      due_date_fixed: rolledup_epic_4.due_date_fixed,
+      start_date: rolledup_epic_4.start_date,
+      due_date: rolledup_epic_4.end_date,
+      start_date_sourcing_work_item_id: fixed_epic_5.issue_id,
+      due_date_sourcing_work_item_id: fixed_epic_5.issue_id
+    )
+  end
+
+  context 'when backfilling all epics', :aggregate_failures do
+    subject(:migration) do
+      described_class.new(
+        start_id: start_id,
+        end_id: end_id,
+        batch_table: :epics,
+        batch_column: :id,
+        job_arguments: [nil],
+        sub_batch_size: 2,
+        pause_ms: 2,
+        connection: ::ApplicationRecord.connection
+      )
+    end
+
+    RSpec::Matchers.define :match_synced_work_item_dates do
+      match do |epic|
+        date_source = work_item_dates_sources.find_by_issue_id(epic.issue_id)
+
+        expect(date_source.start_date).to eq epic.start_date
+        expect(date_source.start_date_is_fixed).to eq epic.start_date_is_fixed.present?
+        expect(date_source.due_date_is_fixed).to eq epic.due_date_is_fixed.present?
+        expect(date_source.start_date_fixed).to eq epic.start_date_fixed
+        expect(date_source.due_date_fixed).to eq epic.due_date_fixed
+        expect(date_source.namespace_id).to eq(epic.group_id)
+        expect(date_source.due_date).to eq(epic.end_date)
+        expect(date_source.start_date_sourcing_milestone_id).to eq(epic.start_date_sourcing_milestone_id)
+        expect(date_source.due_date_sourcing_milestone_id).to eq(epic.due_date_sourcing_milestone_id)
+        expect(date_source.start_date_sourcing_work_item_id)
+          .to eq(epics.find_by_id(epic.start_date_sourcing_epic_id)&.issue_id)
+        expect(date_source.due_date_sourcing_work_item_id)
+          .to eq(epics.find_by_id(epic.due_date_sourcing_epic_id)&.issue_id)
+      end
+    end
+
+    it 'backfills data correctly' do
+      expect do
+        migration.perform
+      end.to change { work_item_dates_sources.count }.from(2).to(10).and not_change { synced_date_source }
+
+      expect(epics.all).to all(match_synced_work_item_dates)
+    end
+  end
+
+  def create_epic_with_work_item(iid:, title:, date_attrs: {})
+    wi = issues.create!(
+      iid: iid,
+      author_id: author.id,
+      work_item_type_id: epic_type_id,
+      namespace_id: namespace.id,
+      lock_version: 1,
+      title: title
+    )
+
+    epic_attributes = {
+      iid: iid,
+      title: title,
+      title_html: title,
+      group_id: namespace.id,
+      author_id: author.id,
+      issue_id: wi.id
+    }
+
+    epics.create!(epic_attributes.merge!(date_attrs))
+  end
+
+  def with_fixed_dates(start_date, due_date)
+    {
+      start_date: DateTime.parse(start_date),
+      end_date: DateTime.parse(due_date),
+      start_date_fixed: DateTime.parse(start_date),
+      due_date_fixed: DateTime.parse(due_date),
+      start_date_is_fixed: true,
+      due_date_is_fixed: true
+    }
+  end
+end

spec/migrations/20240806193258_queue_resync_epic_dates_to_work_items_dates_sources_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueResyncEpicDatesToWorkItemsDatesSources, feature_category: :team_planning do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :epics,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE
+        )
+      }
+    end
+  end
+end

spec/services/wiki_pages/base_service_spec.rb

@@ -7,6 +7,10 @@ RSpec.describe WikiPages::BaseService, feature_category: :wiki do
   let(:user) { double('user') }
   let(:page) { instance_double(WikiPage, template?: false) }
 
+  before do
+    allow(page).to receive(:[]).with(:format).and_return('markdown')
+  end
+
   describe '#increment_usage' do
     let(:subject) { bad_service_class.new(container: project, current_user: user) }
 

spec/support/shared_examples/services/wiki_pages/create_service_shared_examples.rb

@@ -49,6 +49,7 @@ RSpec.shared_examples 'WikiPages::CreateService#execute' do |container_type|
       let(:project) { container if container.is_a?(Project) }
       let(:namespace) { container.is_a?(Group) ? container : container.namespace }
       let(:label) { 'template' }
+      let(:property) { 'markdown' }
 
       subject(:track_event) { service.execute }
     end

spec/support/shared_examples/services/wiki_pages/destroy_service_shared_examples.rb

@@ -30,6 +30,7 @@ RSpec.shared_examples 'WikiPages::DestroyService#execute' do |container_type|
       let(:project) { container if container.is_a?(Project) }
       let(:namespace) { container.is_a?(Group) ? container : container.namespace }
       let(:label) { 'template' }
+      let(:property) { 'markdown' }
 
       subject(:track_event) { service.execute(page) }
     end

spec/support/shared_examples/services/wiki_pages/update_service_shared_examples.rb

@@ -52,6 +52,7 @@ RSpec.shared_examples 'WikiPages::UpdateService#execute' do |container_type|
       let(:project) { container if container.is_a?(Project) }
       let(:namespace) { container.is_a?(Group) ? container : container.namespace }
       let(:label) { 'template' }
+      let(:property) { 'markdown' }
 
       subject(:track_event) { service.execute(page) }
     end