diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb index c2ab8ca0929..3fdf07123e6 100644 --- a/app/models/ci/job_token/project_scope_link.rb +++ b/app/models/ci/job_token/project_scope_link.rb @@ -19,6 +19,11 @@ module Ci validates :target_project, presence: true validate :not_self_referential_link + enum direction: { + outbound: 0, + inbound: 1 + } + def self.for_source_and_target(source_project, target_project) self.find_by(source_project: source_project, target_project: target_project) end diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index 058cb7752f6..03e062a9855 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -131,14 +131,14 @@ module CounterAttribute end def update_counters_with_lease(increments) - detect_race_on_record(log_fields: increments.merge({ caller: __method__ })) do + detect_race_on_record(log_fields: { caller: __method__, attributes: increments.keys }) do self.class.update_counters(id, increments) end end def reset_counter!(attribute) if counter_attribute_enabled?(attribute) - detect_race_on_record(log_fields: { caller: __method__ }) do + detect_race_on_record(log_fields: { caller: __method__, attributes: attribute }) do update!(attribute => 0) clear_counter!(attribute) end @@ -219,13 +219,25 @@ module CounterAttribute def detect_race_on_record(log_fields: {}) return yield unless Feature.enabled?(:counter_attribute_db_lease_for_update, project) - in_lock(database_lock_key, retries: 2) do + # Ensure attributes is always an array before we log + log_fields[:attributes] = Array(log_fields[:attributes]) + + Gitlab::AppLogger.info( + message: 'Acquiring lease for project statistics update', + project_statistics_id: id, + project_id: project.id, + **log_fields, + **Gitlab::ApplicationContext.current + ) + + in_lock(database_lock_key, retries: 0) do yield end rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError Gitlab::AppLogger.warn( - message: 'Concurrent update to project statistics detected', + message: 'Concurrent project statistics update detected', project_statistics_id: id, + project_id: project.id, **log_fields, **Gitlab::ApplicationContext.current ) diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index b1753ce711e..e13f8d28c92 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -50,17 +50,16 @@ class ProjectStatistics < ApplicationRecord def refresh!(only: []) return if Gitlab::Database.read_only? - COLUMNS_TO_REFRESH.each do |column, generator| - if only.empty? || only.include?(column) - public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend - end + columns_to_update = only.empty? ? COLUMNS_TO_REFRESH : COLUMNS_TO_REFRESH & only + columns_to_update.each do |column| + public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend end if only.empty? || only.any? { |column| NAMESPACE_RELATABLE_COLUMNS.include?(column) } schedule_namespace_aggregation_worker end - detect_race_on_record(log_fields: { caller: __method__ }) do + detect_race_on_record(log_fields: { caller: __method__, attributes: columns_to_update }) do save! end end @@ -114,7 +113,7 @@ class ProjectStatistics < ApplicationRecord end def refresh_storage_size! - detect_race_on_record(log_fields: { caller: __method__ }) do + detect_race_on_record(log_fields: { caller: __method__, attributes: :storage_size }) do update!(storage_size: STORAGE_SIZE_SUM) end end diff --git a/db/migrate/20220922143612_add_inbound_ci_job_token_project_scope_links.rb b/db/migrate/20220922143612_add_inbound_ci_job_token_project_scope_links.rb new file mode 100644 index 00000000000..50b43ee4db8 --- /dev/null +++ b/db/migrate/20220922143612_add_inbound_ci_job_token_project_scope_links.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddInboundCiJobTokenProjectScopeLinks < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def up + add_column :ci_job_token_project_scope_links, :direction, :integer, limit: 2, default: 0, null: false + end + + def down + remove_column :ci_job_token_project_scope_links, :direction + end +end diff --git a/db/schema_migrations/20220922143612 b/db/schema_migrations/20220922143612 new file mode 100644 index 00000000000..4f93a218c74 --- /dev/null +++ b/db/schema_migrations/20220922143612 @@ -0,0 +1 @@ +4685b471f00f8ef5e8d8e521c50dc276c757c9f9caa50b1aa20c1f98b8b008c5 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 28732d9fe3f..cf56e01d569 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12833,7 +12833,8 @@ CREATE TABLE ci_job_token_project_scope_links ( source_project_id bigint NOT NULL, target_project_id bigint NOT NULL, added_by_id bigint, - created_at timestamp with time zone NOT NULL + created_at timestamp with time zone NOT NULL, + direction smallint DEFAULT 0 NOT NULL ); CREATE SEQUENCE ci_job_token_project_scope_links_id_seq diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 7cbc8e40b47..222f534387a 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -50,6 +50,8 @@ dependency_scanning: artifacts: paths: - "**/gl-sbom-*.cdx.json" + reports: + cyclonedx: "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml index 70f85382967..67057e916a8 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -50,6 +50,8 @@ dependency_scanning: artifacts: paths: - "**/gl-sbom-*.cdx.json" + reports: + cyclonedx: "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: diff --git a/spec/models/ci/job_token/project_scope_link_spec.rb b/spec/models/ci/job_token/project_scope_link_spec.rb index c000a3e29f7..4c1f11130a4 100644 --- a/spec/models/ci/job_token/project_scope_link_spec.rb +++ b/spec/models/ci/job_token/project_scope_link_spec.rb @@ -89,6 +89,12 @@ RSpec.describe Ci::JobToken::ProjectScopeLink do end end + describe 'enums' do + let(:directions) { { outbound: 0, inbound: 1 } } + + it { is_expected.to define_enum_for(:direction).with_values(directions) } + end + context 'loose foreign key on ci_job_token_project_scope_links.source_project_id' do it_behaves_like 'cleanup by a loose foreign key' do let!(:parent) { create(:project) } diff --git a/spec/services/ci/pipeline_artifacts/coverage_report_service_spec.rb b/spec/services/ci/pipeline_artifacts/coverage_report_service_spec.rb index b020479a78e..c4558bddc85 100644 --- a/spec/services/ci/pipeline_artifacts/coverage_report_service_spec.rb +++ b/spec/services/ci/pipeline_artifacts/coverage_report_service_spec.rb @@ -35,6 +35,7 @@ RSpec.describe Ci::PipelineArtifacts::CoverageReportService do end it 'logs relevant information' do + allow(Gitlab::AppLogger).to receive(:info).and_call_original expect(Gitlab::AppLogger).to receive(:info).with({ project_id: project.id, pipeline_id: pipeline.id, diff --git a/spec/support/shared_examples/models/concerns/counter_attribute_shared_examples.rb b/spec/support/shared_examples/models/concerns/counter_attribute_shared_examples.rb index ea4ba536403..a658d02f09a 100644 --- a/spec/support/shared_examples/models/concerns/counter_attribute_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/counter_attribute_shared_examples.rb @@ -104,7 +104,14 @@ RSpec.shared_examples_for CounterAttribute do |counter_attributes| model.delayed_increment_counter(incremented_attribute, -3) end - it 'updates the record and logs it' do + it 'updates the record and logs it', :aggregate_failures do + expect(Gitlab::AppLogger).to receive(:info).with( + hash_including( + message: 'Acquiring lease for project statistics update', + attributes: [incremented_attribute] + ) + ) + expect(Gitlab::AppLogger).to receive(:info).with( hash_including( message: 'Flush counter attribute to database',