From ca93dbd9bf57af7c00a3b9163329f1585bb5e91f Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 3 Jun 2025 06:07:40 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/CODEOWNERS | 114 +++++++++--------- Gemfile | 2 +- Gemfile.checksum | 2 +- Gemfile.lock | 4 +- Gemfile.next.checksum | 2 +- Gemfile.next.lock | 4 +- .../finished_pipelines_sync_service.rb | 9 +- ...70256_add_name_to_ci_finished_pipelines.rb | 17 +++ ...add_name_to_ci_finished_pipelines_daily.rb | 17 +++ ...dd_name_to_ci_finished_pipelines_hourly.rb | 17 +++ ...22_alter_ci_finished_pipelines_daily_mv.rb | 36 ++++++ ...0_alter_ci_finished_pipelines_hourly_mv.rb | 36 ++++++ .../run-smoke-test-suite.sh | 6 +- .../finished_pipelines_sync_service_spec.rb | 50 ++++---- 14 files changed, 227 insertions(+), 89 deletions(-) create mode 100644 db/click_house/migrate/main/20250509070256_add_name_to_ci_finished_pipelines.rb create mode 100644 db/click_house/migrate/main/20250509071921_add_name_to_ci_finished_pipelines_daily.rb create mode 100644 db/click_house/migrate/main/20250509071939_add_name_to_ci_finished_pipelines_hourly.rb create mode 100644 db/click_house/migrate/main/20250509075722_alter_ci_finished_pipelines_daily_mv.rb create mode 100644 db/click_house/migrate/main/20250509075750_alter_ci_finished_pipelines_hourly_mv.rb diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index a2919629a5a..149d62b28c6 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -188,15 +188,15 @@ Dangerfile /ee/spec/policies/vulnerability*.rb ^[Threat Insights frontend] @gitlab-org/govern/threat-insights-frontend-team -/app/assets/javascripts/vue_merge_request_widget/widgets/security_reports/** -/ee/app/assets/javascripts/vue_merge_request_widget/widgets/security_reports/** -/ee/app/assets/javascripts/security_dashboard/** -/ee/spec/frontend/security_dashboard/** -/ee/app/assets/javascripts/vulnerabilities/** -/spec/frontend/vue_merge_request_widget/widgets/security_reports/** -/ee/spec/frontend/vue_merge_request_widget/widgets/security_reports/** -/ee/app/assets/javascripts/dependencies/** -/ee/spec/frontend/dependencies/** +/app/assets/javascripts/vue_merge_request_widget/widgets/security_reports/ +/ee/app/assets/javascripts/vue_merge_request_widget/widgets/security_reports/ +/ee/app/assets/javascripts/security_dashboard/ +/ee/spec/frontend/security_dashboard/ +/ee/app/assets/javascripts/vulnerabilities/ +/spec/frontend/vue_merge_request_widget/widgets/security_reports/ +/ee/spec/frontend/vue_merge_request_widget/widgets/security_reports/ +/ee/app/assets/javascripts/dependencies/ +/ee/spec/frontend/dependencies/ ^[Composition Analysis backend] @gitlab-org/secure/composition-analysis-be /app/events/package_metadata/ @@ -229,74 +229,74 @@ Dangerfile /ee/app/services/app_sec/dast/ ^[Security Policies frontend] @gitlab-org/security-risk-management/security-policies/frontend -/ee/app/assets/javascripts/approvals/components/license_compliance/** -/ee/app/assets/javascripts/approvals/stores/modules/license_compliance/** -/ee/app/assets/javascripts/license_compliance/** -/ee/app/assets/javascripts/pages/projects/licenses/** -/ee/app/assets/javascripts/pages/projects/pipelines/licenses/** +/ee/app/assets/javascripts/approvals/components/license_compliance/ +/ee/app/assets/javascripts/approvals/stores/modules/license_compliance/ +/ee/app/assets/javascripts/license_compliance/ +/ee/app/assets/javascripts/pages/projects/licenses/ +/ee/app/assets/javascripts/pages/projects/pipelines/licenses/ /ee/app/assets/javascripts/pages/projects/pipelines/show/license_report.js -/ee/app/assets/javascripts/vue_merge_request_widget/widgets/license_compliance/** -/ee/app/assets/javascripts/vue_shared/license_compliance/** -/ee/app/views/projects/licenses/** -/ee/spec/frontend/approvals/components/license_compliance/** -/ee/spec/frontend/approvals/stores/modules/license_compliance/** -/ee/spec/frontend/license_compliance/** -/ee/spec/frontend/vue_merge_request_widget/widgets/license_compliance/** -/ee/spec/frontend/vue_shared/license_compliance/** +/ee/app/assets/javascripts/vue_merge_request_widget/widgets/license_compliance/ +/ee/app/assets/javascripts/vue_shared/license_compliance/ +/ee/app/views/projects/licenses/ +/ee/spec/frontend/approvals/components/license_compliance/ +/ee/spec/frontend/approvals/stores/modules/license_compliance/ +/ee/spec/frontend/license_compliance/ +/ee/spec/frontend/vue_merge_request_widget/widgets/license_compliance/ +/ee/spec/frontend/vue_shared/license_compliance/ -/ee/app/assets/javascripts/approvals/components/security_orchestration/** -/ee/app/assets/javascripts/approvals/stores/modules/security_orchestration/** -/ee/app/assets/javascripts/pages/groups/security/policies/** -/ee/app/assets/javascripts/pages/projects/security/policies/** -/ee/app/assets/javascripts/security_orchestration/** +/ee/app/assets/javascripts/approvals/components/security_orchestration/ +/ee/app/assets/javascripts/approvals/stores/modules/security_orchestration/ +/ee/app/assets/javascripts/pages/groups/security/policies/ +/ee/app/assets/javascripts/pages/projects/security/policies/ +/ee/app/assets/javascripts/security_orchestration/ /ee/app/views/groups/security/policies -/ee/app/views/projects/security/policies/** -/ee/spec/frontend/approvals/components/security_orchestration/** -/ee/spec/frontend/approvals/stores/modules/security_orchestration/** -/ee/spec/frontend/security_orchestration/** -/ee/spec/views/projects/security/policies/** +/ee/app/views/projects/security/policies/ +/ee/spec/frontend/approvals/components/security_orchestration/ +/ee/spec/frontend/approvals/stores/modules/security_orchestration/ +/ee/spec/frontend/security_orchestration/ +/ee/spec/views/projects/security/policies/ ^[Security Policies backend] @gitlab-org/security-risk-management/security-policies/backend /app/models/clusters/applications/cilium.rb /ee/app/controllers/groups/security/policies_controller.rb /ee/app/controllers/projects/security/policies_controller.rb /ee/app/graphql/mutations/concerns/mutations/finds_project_or_group_for_security_policies.rb -/ee/app/graphql/mutations/security_policy/** +/ee/app/graphql/mutations/security_policy/ /ee/app/graphql/resolvers/concerns/resolves_orchestration_policy.rb -/ee/app/graphql/resolvers/security_orchestration/** -/ee/app/graphql/types/security_orchestration/** +/ee/app/graphql/resolvers/security_orchestration/ +/ee/app/graphql/types/security_orchestration/ /ee/app/helpers/ee/security_orchestration_helper.rb /ee/app/models/security/orchestration_policy_configuration.rb /ee/app/models/security/orchestration_policy_rule_schedule.rb -/ee/app/services/security/orchestration/** -/ee/app/services/security/security_orchestration_policies/** +/ee/app/services/security/orchestration/ +/ee/app/services/security/security_orchestration_policies/ /ee/app/validators/json_schemas/security_orchestration_policy.json /ee/app/workers/concerns/update_orchestration_policy_configuration.rb /ee/app/workers/security/create_orchestration_policy_worker.rb /ee/app/workers/security/orchestration_policy_rule_schedule_namespace_worker.rb /ee/app/workers/security/orchestration_policy_rule_schedule_worker.rb /ee/lib/ee/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb -/ee/lib/gitlab/ci/config/security_orchestration_policies/** -/ee/lib/gitlab/graphql/aggregations/security_orchestration_policies/** +/ee/lib/gitlab/ci/config/security_orchestration_policies/ +/ee/lib/gitlab/graphql/aggregations/security_orchestration_policies/ /ee/spec/controllers/groups/security/policies_controller_spec.rb /ee/spec/factories/security_orchestration_policy_configurations.rb /ee/spec/factories/security_orchestration_policy_rule_schedules.rb /ee/spec/factories/security/policies.rb -/ee/spec/graphql/mutations/security_policy/** -/ee/spec/graphql/resolvers/security_orchestration/** -/ee/spec/graphql/types/security_orchestration/** +/ee/spec/graphql/mutations/security_policy/ +/ee/spec/graphql/resolvers/security_orchestration/ +/ee/spec/graphql/types/security_orchestration/ /ee/spec/helpers/ee/security_orchestration_helper_spec.rb /ee/spec/lib/ee/gitlab/ci/pipeline/chain/validate/security_orchestration_policy_spec.rb -/ee/spec/lib/gitlab/ci/config/security_orchestration_policies/** -/ee/spec/lib/gitlab/graphql/aggregations/security_orchestration_policies/** +/ee/spec/lib/gitlab/ci/config/security_orchestration_policies/ +/ee/spec/lib/gitlab/graphql/aggregations/security_orchestration_policies/ /ee/spec/models/security/orchestration_policy_configuration_spec.rb /ee/spec/models/security/orchestration_policy_rule_schedule_spec.rb -/ee/spec/requests/api/graphql/mutations/security_policy/** -/ee/spec/requests/api/graphql/project/security_orchestration/** +/ee/spec/requests/api/graphql/mutations/security_policy/ +/ee/spec/requests/api/graphql/project/security_orchestration/ /ee/spec/requests/projects/security/policies_controller_spec.rb -/ee/spec/services/security/orchestration/** -/ee/spec/services/security/security_orchestration_policies/** -/ee/spec/support/shared_contexts/graphql/resolvers/security_orchestration/** +/ee/spec/services/security/orchestration/ +/ee/spec/services/security/security_orchestration_policies/ +/ee/spec/support/shared_contexts/graphql/resolvers/security_orchestration/ /ee/spec/views/projects/security/policies/index.html.haml_spec.rb /ee/spec/workers/concerns/update_orchestration_policy_configuration_spec.rb /ee/spec/workers/security/create_orchestration_policy_worker_spec.rb @@ -312,14 +312,14 @@ Dangerfile /ee/app/models/sca/license_policy.rb /ee/app/models/software_license_policy.rb /ee/app/models/software_license.rb -/ee/app/serializers/license_compliance/** +/ee/app/serializers/license_compliance/ /ee/app/serializers/license_entity.rb /ee/app/serializers/licenses_list_entity.rb /ee/app/serializers/licenses_list_serializer.rb /ee/app/serializers/security/license_policy_entity.rb /ee/app/services/ci/compare_license_scanning_reports_collapsed_service.rb /ee/app/services/ci/compare_license_scanning_reports_service.rb -/ee/app/services/software_license_policies/** +/ee/app/services/software_license_policies/ /ee/app/workers/refresh_license_compliance_checks_worker.rb /ee/lib/api/managed_licenses.rb /ee/lib/ee/api/entities/managed_license.rb @@ -328,16 +328,16 @@ Dangerfile /ee/spec/factories/software_license.rb /ee/spec/factories/spdx_license.rb /ee/spec/finders/software_license_policies_finder_spec.rb -/ee/spec/lib/gitlab/ci/parsers/license_compliance/** +/ee/spec/lib/gitlab/ci/parsers/license_compliance/ /ee/spec/models/sca/license_compliance_spec.rb @gitlab-org/security-risk-management/security-policies/backend @gitlab-org/secure/composition-analysis-be /ee/spec/models/sca/license_policy_spec.rb /ee/spec/models/software_license_policy_spec.rb /ee/spec/models/software_license_spec.rb /ee/spec/requests/api/managed_licenses_spec.rb -/ee/spec/serializers/license_compliance/** +/ee/spec/serializers/license_compliance/ /ee/spec/services/ci/compare_license_scanning_reports_collapsed_service_spec.rb /ee/spec/services/ci/compare_license_scanning_reports_service_spec.rb -/ee/spec/services/software_license_policies/** +/ee/spec/services/software_license_policies/ /spec/finders/security/license_compliance_jobs_finder_spec.rb [Secure::Secret Detection] @gitlab-org/secure/secret-detection @@ -556,8 +556,8 @@ lib/gitlab/git_access.rb lib/gitlab/git_access_*.rb ee/lib/ee/gitlab/git_access.rb ee/lib/ee/gitlab/git_access_*.rb -ee/lib/ee/gitlab/checks/** -lib/gitlab/checks/** +ee/lib/ee/gitlab/checks/ +lib/gitlab/checks/ ^[Technical writing] @gitlab-org/technical-writing/tw-docops @gitlab-org/tw-leadership /.gitlab/ci/docs.gitlab-ci.yaml @@ -1210,7 +1210,7 @@ lib/gitlab/checks/** # i18n Docs [Localization Team] @gitlab-com/localization/maintainers -/doc-locale/** +/doc-locale/ /doc/development/i18n/proofreader.md /argo_translation.yml diff --git a/Gemfile b/Gemfile index 3faa3269c61..f13a51dd03c 100644 --- a/Gemfile +++ b/Gemfile @@ -306,7 +306,7 @@ gem 'gitlab-sidekiq-fetcher', gem 'fugit', '~> 1.11.1', feature_category: :continuous_integration # HTTP requests -gem 'httparty', '~> 0.22.0', feature_category: :shared +gem 'httparty', '~> 0.23.0', feature_category: :shared # Colored output to console gem 'rainbow', '~> 3.0', feature_category: :shared diff --git a/Gemfile.checksum b/Gemfile.checksum index d391418fd1b..17aab0e24b1 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -326,7 +326,7 @@ {"name":"http-accept","version":"1.7.0","platform":"ruby","checksum":"c626860682bfbb3b46462f8c39cd470fd7b0584f61b3cc9df5b2e9eb9972a126"}, {"name":"http-cookie","version":"1.0.5","platform":"ruby","checksum":"73756d46c7dbdc7023deecdb8a171348ea95a1b99810b31cfe8b4fb4e9a6318f"}, {"name":"http-form_data","version":"2.3.0","platform":"ruby","checksum":"cc4eeb1361d9876821e31d7b1cf0b68f1cf874b201d27903480479d86448a5f3"}, -{"name":"httparty","version":"0.22.0","platform":"ruby","checksum":"78652a5c9471cf0093d3b2083c2295c9c8f12b44c65112f1846af2b71430fa6c"}, +{"name":"httparty","version":"0.23.1","platform":"ruby","checksum":"3ac1dd62f2010f6ece551716f5ceec2b2012011d89f1751917ab7f724e966b55"}, {"name":"httpclient","version":"2.8.3","platform":"ruby","checksum":"2951e4991214464c3e92107e46438527d23048e634f3aee91c719e0bdfaebda6"}, {"name":"i18n","version":"1.14.4","platform":"ruby","checksum":"c7deedead0866ea9102975a4eab7968f53de50793a0c211a37808f75dd187551"}, {"name":"i18n_data","version":"0.13.1","platform":"ruby","checksum":"e5aa99b09a69b463bb0443fc1f9540351a49f3d1541c5e91316bafa035c63f66"}, diff --git a/Gemfile.lock b/Gemfile.lock index 79fe6e53502..158cda234d0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1027,7 +1027,7 @@ GEM http-cookie (1.0.5) domain_name (~> 0.5) http-form_data (2.3.0) - httparty (0.22.0) + httparty (0.23.1) csv mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) @@ -2223,7 +2223,7 @@ DEPENDENCIES health_check (~> 3.0) html-pipeline (~> 2.14.3) html2text - httparty (~> 0.22.0) + httparty (~> 0.23.0) i18n_data (~> 0.13.1) icalendar (~> 2.10.1) influxdb-client (~> 3.1) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index 8a936739258..78b9481ff84 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -326,7 +326,7 @@ {"name":"http-accept","version":"1.7.0","platform":"ruby","checksum":"c626860682bfbb3b46462f8c39cd470fd7b0584f61b3cc9df5b2e9eb9972a126"}, {"name":"http-cookie","version":"1.0.5","platform":"ruby","checksum":"73756d46c7dbdc7023deecdb8a171348ea95a1b99810b31cfe8b4fb4e9a6318f"}, {"name":"http-form_data","version":"2.3.0","platform":"ruby","checksum":"cc4eeb1361d9876821e31d7b1cf0b68f1cf874b201d27903480479d86448a5f3"}, -{"name":"httparty","version":"0.22.0","platform":"ruby","checksum":"78652a5c9471cf0093d3b2083c2295c9c8f12b44c65112f1846af2b71430fa6c"}, +{"name":"httparty","version":"0.23.1","platform":"ruby","checksum":"3ac1dd62f2010f6ece551716f5ceec2b2012011d89f1751917ab7f724e966b55"}, {"name":"httpclient","version":"2.8.3","platform":"ruby","checksum":"2951e4991214464c3e92107e46438527d23048e634f3aee91c719e0bdfaebda6"}, {"name":"i18n","version":"1.14.4","platform":"ruby","checksum":"c7deedead0866ea9102975a4eab7968f53de50793a0c211a37808f75dd187551"}, {"name":"i18n_data","version":"0.13.1","platform":"ruby","checksum":"e5aa99b09a69b463bb0443fc1f9540351a49f3d1541c5e91316bafa035c63f66"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index 0adc73c940b..cd463d916f2 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -1021,7 +1021,7 @@ GEM http-cookie (1.0.5) domain_name (~> 0.5) http-form_data (2.3.0) - httparty (0.22.0) + httparty (0.23.1) csv mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) @@ -2218,7 +2218,7 @@ DEPENDENCIES health_check (~> 3.0) html-pipeline (~> 2.14.3) html2text - httparty (~> 0.22.0) + httparty (~> 0.23.0) i18n_data (~> 0.13.1) icalendar (~> 2.10.1) influxdb-client (~> 3.1) diff --git a/app/services/ci/click_house/data_ingestion/finished_pipelines_sync_service.rb b/app/services/ci/click_house/data_ingestion/finished_pipelines_sync_service.rb index 4f546224a9a..339d386e041 100644 --- a/app/services/ci/click_house/data_ingestion/finished_pipelines_sync_service.rb +++ b/app/services/ci/click_house/data_ingestion/finished_pipelines_sync_service.rb @@ -19,11 +19,13 @@ module Ci PIPELINE_FIELD_NAMES = %i[id duration status source ref].freeze PIPELINE_EPOCH_FIELD_NAMES = %i[committed_at created_at started_at finished_at].freeze PIPELINE_COMPUTED_FIELD_NAMES = %i[path].freeze + PIPELINE_META_FIELD_NAMES = %i[name].freeze CSV_MAPPING = { **PIPELINE_FIELD_NAMES.index_with { |n| n }, **PIPELINE_EPOCH_FIELD_NAMES.index_with { |n| :"casted_#{n}" }, - **PIPELINE_COMPUTED_FIELD_NAMES.index_with { |n| n } + **PIPELINE_COMPUTED_FIELD_NAMES.index_with { |n| n }, + **PIPELINE_META_FIELD_NAMES.index_with { |n| n } }.freeze INSERT_FINISHED_PIPELINES_QUERY = <<~SQL.squish @@ -142,7 +144,7 @@ module Ci pipelines = Ci::Pipeline.id_in(pipeline_ids) pipelines - .left_outer_joins(project_mirror: :namespace_mirror) + .left_outer_joins(project_mirror: :namespace_mirror, pipeline_metadata: []) .select(:finished_at, *finished_pipeline_projections) .each do |pipeline| records_yielder << pipeline.attributes.symbolize_keys.tap do |record| @@ -160,7 +162,8 @@ module Ci *PIPELINE_FIELD_NAMES.map { |n| "#{::Ci::Pipeline.table_name}.#{n}" }, *PIPELINE_EPOCH_FIELD_NAMES .map { |n| "COALESCE(EXTRACT(epoch FROM #{::Ci::Pipeline.table_name}.#{n}), 0) AS casted_#{n}" }, - "ARRAY_TO_STRING(#{::Ci::NamespaceMirror.table_name}.traversal_ids, '/') || '/' AS path" + "ARRAY_TO_STRING(#{::Ci::NamespaceMirror.table_name}.traversal_ids, '/') || '/' AS path", + *PIPELINE_META_FIELD_NAMES.map { |n| "#{::Ci::PipelineMetadata.table_name}.#{n} AS #{n}" } ] end strong_memoize_attr :finished_pipeline_projections diff --git a/db/click_house/migrate/main/20250509070256_add_name_to_ci_finished_pipelines.rb b/db/click_house/migrate/main/20250509070256_add_name_to_ci_finished_pipelines.rb new file mode 100644 index 00000000000..8f2151c92e4 --- /dev/null +++ b/db/click_house/migrate/main/20250509070256_add_name_to_ci_finished_pipelines.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNameToCiFinishedPipelines < ClickHouse::Migration + def up + execute <<~SQL + ALTER TABLE ci_finished_pipelines + ADD COLUMN IF NOT EXISTS `name` String DEFAULT '' + SQL + end + + def down + execute <<~SQL + ALTER TABLE ci_finished_pipelines + DROP COLUMN IF EXISTS `name` + SQL + end +end diff --git a/db/click_house/migrate/main/20250509071921_add_name_to_ci_finished_pipelines_daily.rb b/db/click_house/migrate/main/20250509071921_add_name_to_ci_finished_pipelines_daily.rb new file mode 100644 index 00000000000..c3157577113 --- /dev/null +++ b/db/click_house/migrate/main/20250509071921_add_name_to_ci_finished_pipelines_daily.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNameToCiFinishedPipelinesDaily < ClickHouse::Migration + def up + execute <<~SQL + ALTER TABLE ci_finished_pipelines_daily + ADD COLUMN IF NOT EXISTS `name` String DEFAULT '' + SQL + end + + def down + execute <<~SQL + ALTER TABLE ci_finished_pipelines_daily + DROP COLUMN IF EXISTS `name` + SQL + end +end diff --git a/db/click_house/migrate/main/20250509071939_add_name_to_ci_finished_pipelines_hourly.rb b/db/click_house/migrate/main/20250509071939_add_name_to_ci_finished_pipelines_hourly.rb new file mode 100644 index 00000000000..2d0dc4913f6 --- /dev/null +++ b/db/click_house/migrate/main/20250509071939_add_name_to_ci_finished_pipelines_hourly.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNameToCiFinishedPipelinesHourly < ClickHouse::Migration + def up + execute <<~SQL + ALTER TABLE ci_finished_pipelines_hourly + ADD COLUMN IF NOT EXISTS `name` String DEFAULT '' + SQL + end + + def down + execute <<~SQL + ALTER TABLE ci_finished_pipelines_hourly + DROP COLUMN IF EXISTS `name` + SQL + end +end diff --git a/db/click_house/migrate/main/20250509075722_alter_ci_finished_pipelines_daily_mv.rb b/db/click_house/migrate/main/20250509075722_alter_ci_finished_pipelines_daily_mv.rb new file mode 100644 index 00000000000..f6d0a58fa7a --- /dev/null +++ b/db/click_house/migrate/main/20250509075722_alter_ci_finished_pipelines_daily_mv.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class AlterCiFinishedPipelinesDailyMv < ClickHouse::Migration + def up + execute <<~SQL + ALTER TABLE ci_finished_pipelines_daily_mv MODIFY QUERY + SELECT + path, + status, + source, + ref, + name, + toStartOfInterval(started_at, INTERVAL 1 day) AS started_at_bucket, + countState() AS count_pipelines, + quantileState(duration) AS duration_quantile + FROM ci_finished_pipelines + GROUP BY path, status, source, ref, name, started_at_bucket + SQL + end + + def down + execute <<~SQL + ALTER TABLE ci_finished_pipelines_daily_mv MODIFY QUERY + SELECT + path, + status, + source, + ref, + toStartOfInterval(started_at, INTERVAL 1 day) AS started_at_bucket, + countState() AS count_pipelines, + quantileState(duration) AS duration_quantile + FROM ci_finished_pipelines + GROUP BY path, status, source, ref, started_at_bucket + SQL + end +end diff --git a/db/click_house/migrate/main/20250509075750_alter_ci_finished_pipelines_hourly_mv.rb b/db/click_house/migrate/main/20250509075750_alter_ci_finished_pipelines_hourly_mv.rb new file mode 100644 index 00000000000..ac25c9d7d92 --- /dev/null +++ b/db/click_house/migrate/main/20250509075750_alter_ci_finished_pipelines_hourly_mv.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class AlterCiFinishedPipelinesHourlyMv < ClickHouse::Migration + def up + execute <<~SQL + ALTER TABLE ci_finished_pipelines_hourly_mv MODIFY QUERY + SELECT + path, + status, + source, + ref, + name, + toStartOfInterval(started_at, INTERVAL 1 hour) AS started_at_bucket, + countState() AS count_pipelines, + quantileState(duration) AS duration_quantile + FROM ci_finished_pipelines + GROUP BY path, status, source, ref, name, started_at_bucket + SQL + end + + def down + execute <<~SQL + ALTER TABLE ci_finished_pipelines_hourly_mv MODIFY QUERY + SELECT + path, + status, + source, + ref, + toStartOfInterval(started_at, INTERVAL 1 hour) AS started_at_bucket, + countState() AS count_pipelines, + quantileState(duration) AS duration_quantile + FROM ci_finished_pipelines + GROUP BY path, status, source, ref, started_at_bucket + SQL + end +end diff --git a/scripts/remote_development/run-smoke-test-suite.sh b/scripts/remote_development/run-smoke-test-suite.sh index e78a7103172..1edb5492888 100755 --- a/scripts/remote_development/run-smoke-test-suite.sh +++ b/scripts/remote_development/run-smoke-test-suite.sh @@ -93,9 +93,13 @@ function run_rspec_non_fast { files_for_non_fast=() + # Note that we do NOT exclude the fast_spec_helper specs here, because sometimes specs may pass + # when run with fast_spec_helper, but fail when run with the full spec_helper. This happens when + # they are run as part of a larger suite of mixed fast and slow files, for example, in CI jobs. + # Running all fast and slow specs here ensures that we catch those cases. while IFS='' read -r file; do files_for_non_fast+=("$file") - done < <(git grep -L -E '^require .fast_spec_helper' -- '**/remote_development/*_spec.rb' | grep -v 'qa/qa' | grep -v '/features/') + done < <(git ls-files -- '**/remote_development/*_spec.rb' | grep -v 'qa/qa' | grep -v '/features/') files_for_non_fast+=( "ee/spec/graphql/resolvers/clusters/agents_resolver_spec.rb" diff --git a/spec/services/ci/click_house/data_ingestion/finished_pipelines_sync_service_spec.rb b/spec/services/ci/click_house/data_ingestion/finished_pipelines_sync_service_spec.rb index a2832be1c09..e9641a48695 100644 --- a/spec/services/ci/click_house/data_ingestion/finished_pipelines_sync_service_spec.rb +++ b/spec/services/ci/click_house/data_ingestion/finished_pipelines_sync_service_spec.rb @@ -35,6 +35,8 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex let_it_be(:pipeline5) { create(:ci_pipeline, :pending, project: project1, ref: 'feature/b') } + let_it_be(:pipeline_with_name) { create(:ci_pipeline, :success, name: 'name', finished_at: 1.week.ago) } + before_all do create_sync_events(*Ci::Pipeline.finished.order(id: :desc)) end @@ -43,26 +45,26 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex it 'processes the pipelines' do expect(ClickHouse::Client).to receive(:insert_csv).once.and_call_original - expect { execute }.to change { ci_finished_pipelines_row_count }.by(4) + expect { execute }.to change { ci_finished_pipelines_row_count }.by(5) expect(execute).to have_attributes({ payload: { reached_end_of_table: true, - records_inserted: 4, + records_inserted: 5, worker_index: 0, total_workers: 1 } }) records = ci_finished_pipelines - expect(records.count).to eq 4 - expect(records).to contain_exactly_pipelines(pipeline1, pipeline2, pipeline3, pipeline4) + expect(records.count).to eq 5 + expect(records).to contain_exactly_pipelines(pipeline1, pipeline2, pipeline3, pipeline4, pipeline_with_name) end it 'processes only pipelines from Ci::FinishedPipelineChSyncEvent' do pipeline = create(:ci_pipeline, :failed, finished_at: 1.minute.ago) - expect { execute }.to change { ci_finished_pipelines_row_count }.by(4) + expect { execute }.to change { ci_finished_pipelines_row_count }.by(5) expect(execute).to have_attributes({ - payload: a_hash_including(reached_end_of_table: true, records_inserted: 4) + payload: a_hash_including(reached_end_of_table: true, records_inserted: 5) }) create_sync_events(pipeline) @@ -76,7 +78,7 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex .tap(&:save!) expect { execute } - .to change { ci_finished_pipelines_row_count }.by(4) + .to change { ci_finished_pipelines_row_count }.by(5) .and change { sync_event.reload.processed }.to(true) end end @@ -90,9 +92,9 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex it 'processes the pipelines' do expect(ClickHouse::Client).to receive(:insert_csv).once.and_call_original - expect { execute }.to change { ci_finished_pipelines_row_count }.by(4) + expect { execute }.to change { ci_finished_pipelines_row_count }.by(5) expect(execute).to have_attributes({ - payload: a_hash_including(reached_end_of_table: true, records_inserted: 4) + payload: a_hash_including(reached_end_of_table: true, records_inserted: 5) }) end end @@ -100,7 +102,7 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex context 'when multiple CSV uploads are required' do before do stub_const("#{described_class}::PIPELINES_BATCH_SIZE", 1) - stub_const("#{described_class}::PIPELINES_BATCH_COUNT", 2) + stub_const("#{described_class}::PIPELINES_BATCH_COUNT", 3) end it 'processes the pipelines' do @@ -110,9 +112,9 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex expect(ClickHouse::Client).to receive(:insert_csv).twice.and_call_original - expect { execute }.to change { ci_finished_pipelines_row_count }.by(4) + expect { execute }.to change { ci_finished_pipelines_row_count }.by(5) expect(execute).to have_attributes({ - payload: a_hash_including(reached_end_of_table: true, records_inserted: 4) + payload: a_hash_including(reached_end_of_table: true, records_inserted: 5) }) end @@ -154,9 +156,9 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex expect(iterator).to receive(:each_batch).once.with(of: described_class::PIPELINES_BATCH_SIZE).and_call_original end - expect { execute }.to change { ci_finished_pipelines_row_count }.by(4) + expect { execute }.to change { ci_finished_pipelines_row_count }.by(5) expect(execute).to have_attributes({ - payload: a_hash_including(reached_end_of_table: true, records_inserted: 4) + payload: a_hash_including(reached_end_of_table: true, records_inserted: 5) }) pipeline6 = create(:ci_pipeline, :failed, finished_at: 1.minute.ago) @@ -164,13 +166,15 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(1) records = ci_finished_pipelines - expect(records.count).to eq 5 - expect(records).to contain_exactly_pipelines(pipeline1, pipeline2, pipeline3, pipeline4, pipeline6) + expect(records.count).to eq 6 + expect(records).to contain_exactly_pipelines( + pipeline1, pipeline2, pipeline3, pipeline4, pipeline_with_name, pipeline6 + ) end context 'with same updated_at value' do it 'processes the pipelines' do - expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(4) + expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(5) pipeline6 = create(:ci_pipeline, :failed, finished_at: 1.second.ago, updated_at: 1.second.ago) pipeline7 = create(:ci_pipeline, :failed, finished_at: 1.second.ago, updated_at: 1.second.ago) @@ -179,14 +183,17 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex expect { execute }.to change { ci_finished_pipelines_row_count }.by(2) records = ci_finished_pipelines - expect(records.count).to eq 6 - expect(records).to contain_exactly_pipelines(pipeline1, pipeline2, pipeline3, pipeline4, pipeline6, pipeline7) + expect(records.count).to eq 7 + expect(records).to contain_exactly_pipelines( + pipeline1, pipeline2, pipeline3, pipeline4, pipeline_with_name, + pipeline6, pipeline7 + ) end end context 'with older finished_at value' do it 'does not process the pipeline' do - expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(4) + expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(5) create(:ci_pipeline, :failed, finished_at: 2.days.ago) @@ -264,7 +271,8 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex status: pipeline.status || '', source: pipeline.source || '', ref: pipeline.ref || '', - date: pipeline.finished_at.beginning_of_month + date: pipeline.finished_at.beginning_of_month, + name: pipeline.name || '' } end