diff --git a/.rubocop_todo/lint/redundant_cop_disable_directive.yml b/.rubocop_todo/lint/redundant_cop_disable_directive.yml index abba53fe393..020690ef61d 100644 --- a/.rubocop_todo/lint/redundant_cop_disable_directive.yml +++ b/.rubocop_todo/lint/redundant_cop_disable_directive.yml @@ -236,7 +236,6 @@ Lint/RedundantCopDisableDirective: - 'spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb' - 'spec/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy_spec.rb' - - 'spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb' - 'spec/lib/gitlab/ci/reports/security/scanner_spec.rb' - 'spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb' - 'spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb' diff --git a/.rubocop_todo/rspec/feature_category.yml b/.rubocop_todo/rspec/feature_category.yml index d2d984b4f36..5f73de31091 100644 --- a/.rubocop_todo/rspec/feature_category.yml +++ b/.rubocop_todo/rspec/feature_category.yml @@ -2748,7 +2748,6 @@ RSpec/FeatureCategory: - 'spec/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy_spec.rb' - 'spec/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy_spec.rb' - 'spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb' - - 'spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb' - 'spec/lib/gitlab/background_migration/copy_column_using_background_migration_job_spec.rb' - 'spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb' - 'spec/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules_spec.rb' @@ -2764,7 +2763,6 @@ RSpec/FeatureCategory: - 'spec/lib/gitlab/background_migration/mailers/unconfirm_mailer_spec.rb' - 'spec/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations_spec.rb' - 'spec/lib/gitlab/background_migration/populate_projects_star_count_spec.rb' - - 'spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb' - 'spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb' - 'spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb' - 'spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb' diff --git a/.rubocop_todo/rspec/named_subject.yml b/.rubocop_todo/rspec/named_subject.yml index e27d5942091..6e82a5563f1 100644 --- a/.rubocop_todo/rspec/named_subject.yml +++ b/.rubocop_todo/rspec/named_subject.yml @@ -1815,7 +1815,6 @@ RSpec/NamedSubject: - 'spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb' - 'spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb' - 'spec/lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl_spec.rb' - - 'spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb' - 'spec/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles_spec.rb' - 'spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb' - 'spec/lib/gitlab/background_task_spec.rb' diff --git a/.rubocop_todo/style/inline_disable_annotation.yml b/.rubocop_todo/style/inline_disable_annotation.yml index d8f4e088c3a..29d3ca150de 100644 --- a/.rubocop_todo/style/inline_disable_annotation.yml +++ b/.rubocop_todo/style/inline_disable_annotation.yml @@ -2805,7 +2805,6 @@ Style/InlineDisableAnnotation: - 'spec/lib/gitlab/background_migration/batched_migration_job_spec.rb' - 'spec/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy_spec.rb' - 'spec/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy_spec.rb' - - 'spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb' - 'spec/lib/gitlab/background_migration/convert_credit_card_validation_data_to_hashes_spec.rb' - 'spec/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2_spec.rb' - 'spec/lib/gitlab/background_migration/delete_orphans_approval_project_rules2_spec.rb' diff --git a/db/docs/ci_job_artifact_states.yml b/db/docs/ci_job_artifact_states.yml index 19cf5f5e63c..3e1b388db24 100644 --- a/db/docs/ci_job_artifact_states.yml +++ b/db/docs/ci_job_artifact_states.yml @@ -14,6 +14,6 @@ desired_sharding_key: backfill_via: parent: foreign_key: job_artifact_id - table: ci_job_artifacts + table: p_ci_job_artifacts sharding_key: project_id belongs_to: job_artifact diff --git a/doc/ci/ssh_keys/index.md b/doc/ci/ssh_keys/index.md index b056844430e..c85e51121c5 100644 --- a/doc/ci/ssh_keys/index.md +++ b/doc/ci/ssh_keys/index.md @@ -198,9 +198,10 @@ before_script: ## ## Alternatively, use ssh-keyscan to scan the keys of your private server. ## Replace example.com with your private server's domain name. Repeat that - ## command if you have more than one server to connect to. + ## command if you have more than one server to connect to. Include the -t + ## flag to specify the key type. ## - # - ssh-keyscan example.com >> ~/.ssh/known_hosts + # - ssh-keyscan -t rsa,ed25519 example.com >> ~/.ssh/known_hosts # - chmod 644 ~/.ssh/known_hosts ## diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/macro_reflection.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/macro_reflection.rb index 7ec7da44253..17c05645ac1 100644 --- a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/macro_reflection.rb +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/macro_reflection.rb @@ -5,13 +5,38 @@ module ActiveRecord module Partitioning module Reflection module MacroReflection + NO_OWNER = Struct.new(:partition_id).new(1..100_000) + + # We override the method to allow eager loading of partitioned records + # + # For eager loading the owner is always nil and we supply a benign + # object that allows the scope to be evaluated with a query like + # `where partition_id between 1 and 100000` + # and transforms it into + # `where partition_id is not null` + # to ensure that no partition is left out by the query. + # This is safe because `partition_id` columns are defined as `not null` + # def scope_for(relation, owner = nil) if scope.arity == 1 && owner.nil? && options.key?(:partition_foreign_key) - relation + relation = relation.instance_exec(NO_OWNER, &scope) + if relation_includes_partition_id_condition?(relation) + relation.rewhere(relation.table[:partition_id].not_eq(nil)) + else + relation + end else super end end + + def relation_includes_partition_id_condition?(relation) + relation + .where_clause + .extract_attributes + .map(&:name) + .include?('partition_id') + end end end end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/joins_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/joins_spec.rb index 038fae43644..67568a5a933 100644 --- a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/joins_spec.rb +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/joins_spec.rb @@ -9,7 +9,8 @@ RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Joins', join_statement = <<~SQL.squish SELECT \"pipelines\".* FROM \"pipelines\" - INNER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" + INNER JOIN \"jobs\" ON \"jobs\".\"partition_id\" IS NOT NULL + AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\" WHERE \"pipelines\".\"partition_id\" = #{pipeline.partition_id} SQL @@ -25,9 +26,11 @@ RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Joins', join_statement = <<~SQL.squish SELECT \"pipelines\".* FROM \"pipelines\" - INNER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" + INNER JOIN \"jobs\" ON \"jobs\".\"partition_id\" IS NOT NULL + AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\" - INNER JOIN \"metadata\" ON \"metadata\".\"job_id\" = \"jobs\".\"id\" + INNER JOIN \"metadata\" ON \"metadata\".\"partition_id\" IS NOT NULL + AND \"metadata\".\"job_id\" = \"jobs\".\"id\" AND \"metadata\".\"partition_id\" = \"jobs\".\"partition_id\" WHERE \"pipelines\".\"partition_id\" = #{pipeline.partition_id} SQL diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/preloads_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/preloads_spec.rb index f37a563fe9e..2bbd903e7ef 100644 --- a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/preloads_spec.rb +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/preloads_spec.rb @@ -216,9 +216,11 @@ RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Preload AS t2_r2, \"metadata\".\"test_flag\" AS t2_r3 FROM \"pipelines\" - LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" + LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"partition_id\" IS NOT NULL + AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\" - LEFT OUTER JOIN \"metadata\" ON \"metadata\".\"job_id\" = \"jobs\".\"id\" + LEFT OUTER JOIN \"metadata\" ON \"metadata\".\"partition_id\" IS NOT NULL + AND \"metadata\".\"job_id\" = \"jobs\".\"id\" AND \"metadata\".\"partition_id\" = \"jobs\".\"partition_id\" WHERE \"pipelines\".\"project_id\" = 1 AND \"pipelines\".\"id\" @@ -237,5 +239,75 @@ RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Preload expect(result).to include(preload_statement) end + + it 'keeps join conditions from scope' do + preload_statement = <<~SQL.squish + SELECT \"pipelines\".\"id\" + AS t0_r0, \"pipelines\".\"project_id\" + AS t0_r1, \"pipelines\".\"partition_id\" + AS t0_r2, \"jobs\".\"id\" + AS t1_r0, \"jobs\".\"pipeline_id\" + AS t1_r1, \"jobs\".\"partition_id\" + AS t1_r2, \"jobs\".\"name\" + AS t1_r3, \"metadata\".\"id\" + AS t2_r0, \"metadata\".\"job_id\" + AS t2_r1, \"metadata\".\"partition_id\" + AS t2_r2, \"metadata\".\"test_flag\" + AS t2_r3 + FROM \"pipelines\" + LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"partition_id\" IS NOT NULL + AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" + AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\" + LEFT OUTER JOIN \"metadata\" ON \"metadata\".\"test_flag\" = 1 + AND \"metadata\".\"partition_id\" IS NOT NULL + AND \"metadata\".\"job_id\" = \"jobs\".\"id\" + AND \"metadata\".\"partition_id\" = \"jobs\".\"partition_id\" + WHERE \"pipelines\".\"project_id\" = 1 + AND \"pipelines\".\"id\" + IN (#{pipeline.id}, #{other_pipeline.id}) + AND \"pipelines\".\"partition_id\" = 100 + SQL + + result = QueryRecorder.log do + project + .pipelines + .includes(jobs: :test_metadata) + .references(:jobs, :test_metadata) + .where(id: [1, 2], partition_id: 100) + .to_a + end + + expect(result).to include(preload_statement) + end + + it 'does rewhere the partition_id condition when missing' do + preload_statement = <<~SQL.squish + SELECT \"pipelines\".\"id\" + AS t0_r0, \"pipelines\".\"project_id\" + AS t0_r1, \"pipelines\".\"partition_id\" + AS t0_r2, \"jobs\".\"id\" + AS t1_r0, \"jobs\".\"pipeline_id\" + AS t1_r1, \"jobs\".\"partition_id\" + AS t1_r2, \"jobs\".\"name\" + AS t1_r3 FROM \"pipelines\" + LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = NULL + AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" + AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\" + WHERE \"pipelines\".\"project_id\" = 1 + AND \"pipelines\".\"id\" IN (1, 2) + AND \"pipelines\".\"partition_id\" = 100 + SQL + + result = QueryRecorder.log do + project + .pipelines + .includes(:unpartitioned_jobs) + .references(:unpartitioned_jobs) + .where(id: [1, 2], partition_id: 100) + .to_a + end + + expect(result).to include(preload_statement) + end end end diff --git a/gems/activerecord-gitlab/spec/support/models.rb b/gems/activerecord-gitlab/spec/support/models.rb index 445dc8b2ac2..8a1b37d5a75 100644 --- a/gems/activerecord-gitlab/spec/support/models.rb +++ b/gems/activerecord-gitlab/spec/support/models.rb @@ -22,6 +22,12 @@ class Pipeline < PartitionedRecord ->(pipeline) { where(partition_id: pipeline.partition_id) }, partition_foreign_key: :partition_id, dependent: :destroy + + has_many :unpartitioned_jobs, + ->(pipeline) { where(pipeline: pipeline).order(id: :desc) }, + partition_foreign_key: :partition_id, + dependent: :destroy, + class_name: 'Job' end class Job < PartitionedRecord @@ -38,6 +44,13 @@ class Job < PartitionedRecord inverse_of: :job, autosave: true + has_one :test_metadata, + ->(build) { where(partition_id: build.partition_id, test_flag: true) }, + foreign_key: :job_id, + partition_foreign_key: :partition_id, + inverse_of: :job, + class_name: 'Metadata' + accepts_nested_attributes_for :metadata end diff --git a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb deleted file mode 100644 index 49525479637..00000000000 --- a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module BatchingStrategies - # Used to apply additional filters to the batching table, migrated to - # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96478 - class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy - end - end - end -end diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb deleted file mode 100644 index 7b88e10f39c..00000000000 --- a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723. - # These job artifacts will not be deleted and will have their `expire_at` removed. - class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob - operation_name :update_all - feature_category :database - - # The migration would have backfilled `expire_at` - # to midnight on the 22nd of the month of the local timezone, - # storing it as UTC time in the database. - # - # If the timezone setting has changed since the migration, - # the `expire_at` stored in the database could have changed to a different local time other than midnight. - # For example: - # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00. - # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00 - # on the previous day (21st). - # - # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark - # on the dates 21, 22, 23 of the month will not be deleted. - # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets - EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL - EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23) - AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45) - AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0 - SQL - - scope_to ->(relation) { - relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE) - .or(relation.where(file_type: 3)) - } - - def perform - each_sub_batch do |sub_batch| - sub_batch.update_all(expire_at: nil) - end - end - end - end -end diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb index c2f94b7b0e6..ba2abe97576 100644 --- a/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb @@ -8,7 +8,13 @@ module Gitlab class PartitioningRoutingAnalyzer < Database::QueryAnalyzers::Base RoutingTableNotUsedError = Class.new(QueryAnalyzerError) - ENABLED_TABLES = %w[ci_builds ci_builds_metadata].freeze + ENABLED_TABLES = %w[ + ci_builds + ci_builds_metadata + ci_job_artifacts + ci_pipeline_variables + ci_stages + ].freeze class << self def enabled? diff --git a/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb deleted file mode 100644 index e296a46ea2f..00000000000 --- a/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy do # rubocop:disable Layout/LineLength - it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy } -end diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb deleted file mode 100644 index 4c989ba9cef..00000000000 --- a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt do - it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob } - - describe '#perform' do - let(:job_artifact) { table(:ci_job_artifacts, database: :ci) } - let(:jobs) { table(:p_ci_builds, database: :ci) { |model| model.primary_key = :id } } - - let(:test_worker) do - described_class.new( - start_id: 1, - end_id: 100, - batch_table: :ci_job_artifacts, - batch_column: :id, - sub_batch_size: 10, - pause_ms: 0, - connection: Ci::ApplicationRecord.connection - ) - end - - let!(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') } - let!(:project) do - table(:projects).create!( - id: 1, - name: 'gitlab1', - path: 'gitlab1', - project_namespace_id: 1, - namespace_id: namespace.id - ) - end - - subject { test_worker.perform } - - context 'with artifacts that has backfilled expire_at' do - let!(:created_on_00_30_45_minutes_on_21_22_23) do - create_job_artifact(id: 1, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) - create_job_artifact(id: 2, file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) - create_job_artifact(id: 3, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) - create_job_artifact(id: 4, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) - create_job_artifact(id: 5, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) - create_job_artifact(id: 6, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) - create_job_artifact(id: 7, file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) - end - - let!(:created_close_to_00_or_30_minutes) do - create_job_artifact(id: 8, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) - create_job_artifact(id: 9, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:30:00.999')) - end - - let!(:created_on_00_or_30_minutes_on_other_dates) do - create_job_artifact(id: 10, file_type: 1, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) - create_job_artifact(id: 11, file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) - create_job_artifact(id: 12, file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) - end - - let!(:created_at_other_times) do - create_job_artifact(id: 13, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:00:00.000')) - create_job_artifact(id: 14, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:30:00.000')) - create_job_artifact(id: 15, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:00:00.000')) - create_job_artifact(id: 16, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) - end - - it 'removes expire_at on job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do - expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(7) - end - - it 'keeps expire_at on other job artifacts' do - expect { subject }.to change { job_artifact.where.not(expire_at: nil).count }.from(16).to(9) - end - end - - context 'with trace artifacts that has backfilled expire_at' do - let!(:trace_artifacts) do - create_job_artifact(id: 1, file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) - create_job_artifact(id: 2, file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) - end - - it 'removes expire_at on trace job artifacts' do - expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(2) - end - end - - private - - def create_job_artifact(id:, file_type:, expire_at:) - job = jobs.create!(partition_id: 100) - job_artifact.create!( - id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, - file_type: file_type, partition_id: 100 - ) - end - end -end diff --git a/spec/migrations/20240108072318_fix_invalid_records_ci_build_trace_metadata_spec.rb b/spec/migrations/20240108072318_fix_invalid_records_ci_build_trace_metadata_spec.rb index 8749a304f19..6b3bffef4f1 100644 --- a/spec/migrations/20240108072318_fix_invalid_records_ci_build_trace_metadata_spec.rb +++ b/spec/migrations/20240108072318_fix_invalid_records_ci_build_trace_metadata_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' require_migration! -RSpec.describe FixInvalidRecordsCiBuildTraceMetadata, migration: :gitlab_ci, feature_category: :continuous_integration do +RSpec.describe FixInvalidRecordsCiBuildTraceMetadata, :suppress_partitioning_routing_analyzer, migration: :gitlab_ci, feature_category: :continuous_integration do let(:trace_metadata) { table(:ci_build_trace_metadata) } let(:job_artifacts) { table(:ci_job_artifacts) } let(:connection) { job_artifacts.connection } diff --git a/spec/migrations/20240108072545_fix_invalid_records_ci_job_artifact_states_spec.rb b/spec/migrations/20240108072545_fix_invalid_records_ci_job_artifact_states_spec.rb index fe1557c0fbb..44c23a1e361 100644 --- a/spec/migrations/20240108072545_fix_invalid_records_ci_job_artifact_states_spec.rb +++ b/spec/migrations/20240108072545_fix_invalid_records_ci_job_artifact_states_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' require_migration! -RSpec.describe FixInvalidRecordsCiJobArtifactStates, migration: :gitlab_ci, feature_category: :continuous_integration do +RSpec.describe FixInvalidRecordsCiJobArtifactStates, :suppress_partitioning_routing_analyzer, migration: :gitlab_ci, feature_category: :continuous_integration do let(:artifact_state) { table(:ci_job_artifact_states) } let(:job_artifacts) { table(:ci_job_artifacts) } let(:connection) { job_artifacts.connection }