Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
GitLab Bot
parent
cb6508e11f
commit
ca93dbd9bf
|
|
@ -188,15 +188,15 @@ Dangerfile
|
|||
/ee/spec/policies/vulnerability*.rb
|
||||
|
||||
^[Threat Insights frontend] @gitlab-org/govern/threat-insights-frontend-team
|
||||
/app/assets/javascripts/vue_merge_request_widget/widgets/security_reports/**
|
||||
/ee/app/assets/javascripts/vue_merge_request_widget/widgets/security_reports/**
|
||||
/ee/app/assets/javascripts/security_dashboard/**
|
||||
/ee/spec/frontend/security_dashboard/**
|
||||
/ee/app/assets/javascripts/vulnerabilities/**
|
||||
/spec/frontend/vue_merge_request_widget/widgets/security_reports/**
|
||||
/ee/spec/frontend/vue_merge_request_widget/widgets/security_reports/**
|
||||
/ee/app/assets/javascripts/dependencies/**
|
||||
/ee/spec/frontend/dependencies/**
|
||||
/app/assets/javascripts/vue_merge_request_widget/widgets/security_reports/
|
||||
/ee/app/assets/javascripts/vue_merge_request_widget/widgets/security_reports/
|
||||
/ee/app/assets/javascripts/security_dashboard/
|
||||
/ee/spec/frontend/security_dashboard/
|
||||
/ee/app/assets/javascripts/vulnerabilities/
|
||||
/spec/frontend/vue_merge_request_widget/widgets/security_reports/
|
||||
/ee/spec/frontend/vue_merge_request_widget/widgets/security_reports/
|
||||
/ee/app/assets/javascripts/dependencies/
|
||||
/ee/spec/frontend/dependencies/
|
||||
|
||||
^[Composition Analysis backend] @gitlab-org/secure/composition-analysis-be
|
||||
/app/events/package_metadata/
|
||||
|
|
@ -229,74 +229,74 @@ Dangerfile
|
|||
/ee/app/services/app_sec/dast/
|
||||
|
||||
^[Security Policies frontend] @gitlab-org/security-risk-management/security-policies/frontend
|
||||
/ee/app/assets/javascripts/approvals/components/license_compliance/**
|
||||
/ee/app/assets/javascripts/approvals/stores/modules/license_compliance/**
|
||||
/ee/app/assets/javascripts/license_compliance/**
|
||||
/ee/app/assets/javascripts/pages/projects/licenses/**
|
||||
/ee/app/assets/javascripts/pages/projects/pipelines/licenses/**
|
||||
/ee/app/assets/javascripts/approvals/components/license_compliance/
|
||||
/ee/app/assets/javascripts/approvals/stores/modules/license_compliance/
|
||||
/ee/app/assets/javascripts/license_compliance/
|
||||
/ee/app/assets/javascripts/pages/projects/licenses/
|
||||
/ee/app/assets/javascripts/pages/projects/pipelines/licenses/
|
||||
/ee/app/assets/javascripts/pages/projects/pipelines/show/license_report.js
|
||||
/ee/app/assets/javascripts/vue_merge_request_widget/widgets/license_compliance/**
|
||||
/ee/app/assets/javascripts/vue_shared/license_compliance/**
|
||||
/ee/app/views/projects/licenses/**
|
||||
/ee/spec/frontend/approvals/components/license_compliance/**
|
||||
/ee/spec/frontend/approvals/stores/modules/license_compliance/**
|
||||
/ee/spec/frontend/license_compliance/**
|
||||
/ee/spec/frontend/vue_merge_request_widget/widgets/license_compliance/**
|
||||
/ee/spec/frontend/vue_shared/license_compliance/**
|
||||
/ee/app/assets/javascripts/vue_merge_request_widget/widgets/license_compliance/
|
||||
/ee/app/assets/javascripts/vue_shared/license_compliance/
|
||||
/ee/app/views/projects/licenses/
|
||||
/ee/spec/frontend/approvals/components/license_compliance/
|
||||
/ee/spec/frontend/approvals/stores/modules/license_compliance/
|
||||
/ee/spec/frontend/license_compliance/
|
||||
/ee/spec/frontend/vue_merge_request_widget/widgets/license_compliance/
|
||||
/ee/spec/frontend/vue_shared/license_compliance/
|
||||
|
||||
/ee/app/assets/javascripts/approvals/components/security_orchestration/**
|
||||
/ee/app/assets/javascripts/approvals/stores/modules/security_orchestration/**
|
||||
/ee/app/assets/javascripts/pages/groups/security/policies/**
|
||||
/ee/app/assets/javascripts/pages/projects/security/policies/**
|
||||
/ee/app/assets/javascripts/security_orchestration/**
|
||||
/ee/app/assets/javascripts/approvals/components/security_orchestration/
|
||||
/ee/app/assets/javascripts/approvals/stores/modules/security_orchestration/
|
||||
/ee/app/assets/javascripts/pages/groups/security/policies/
|
||||
/ee/app/assets/javascripts/pages/projects/security/policies/
|
||||
/ee/app/assets/javascripts/security_orchestration/
|
||||
/ee/app/views/groups/security/policies
|
||||
/ee/app/views/projects/security/policies/**
|
||||
/ee/spec/frontend/approvals/components/security_orchestration/**
|
||||
/ee/spec/frontend/approvals/stores/modules/security_orchestration/**
|
||||
/ee/spec/frontend/security_orchestration/**
|
||||
/ee/spec/views/projects/security/policies/**
|
||||
/ee/app/views/projects/security/policies/
|
||||
/ee/spec/frontend/approvals/components/security_orchestration/
|
||||
/ee/spec/frontend/approvals/stores/modules/security_orchestration/
|
||||
/ee/spec/frontend/security_orchestration/
|
||||
/ee/spec/views/projects/security/policies/
|
||||
|
||||
^[Security Policies backend] @gitlab-org/security-risk-management/security-policies/backend
|
||||
/app/models/clusters/applications/cilium.rb
|
||||
/ee/app/controllers/groups/security/policies_controller.rb
|
||||
/ee/app/controllers/projects/security/policies_controller.rb
|
||||
/ee/app/graphql/mutations/concerns/mutations/finds_project_or_group_for_security_policies.rb
|
||||
/ee/app/graphql/mutations/security_policy/**
|
||||
/ee/app/graphql/mutations/security_policy/
|
||||
/ee/app/graphql/resolvers/concerns/resolves_orchestration_policy.rb
|
||||
/ee/app/graphql/resolvers/security_orchestration/**
|
||||
/ee/app/graphql/types/security_orchestration/**
|
||||
/ee/app/graphql/resolvers/security_orchestration/
|
||||
/ee/app/graphql/types/security_orchestration/
|
||||
/ee/app/helpers/ee/security_orchestration_helper.rb
|
||||
/ee/app/models/security/orchestration_policy_configuration.rb
|
||||
/ee/app/models/security/orchestration_policy_rule_schedule.rb
|
||||
/ee/app/services/security/orchestration/**
|
||||
/ee/app/services/security/security_orchestration_policies/**
|
||||
/ee/app/services/security/orchestration/
|
||||
/ee/app/services/security/security_orchestration_policies/
|
||||
/ee/app/validators/json_schemas/security_orchestration_policy.json
|
||||
/ee/app/workers/concerns/update_orchestration_policy_configuration.rb
|
||||
/ee/app/workers/security/create_orchestration_policy_worker.rb
|
||||
/ee/app/workers/security/orchestration_policy_rule_schedule_namespace_worker.rb
|
||||
/ee/app/workers/security/orchestration_policy_rule_schedule_worker.rb
|
||||
/ee/lib/ee/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb
|
||||
/ee/lib/gitlab/ci/config/security_orchestration_policies/**
|
||||
/ee/lib/gitlab/graphql/aggregations/security_orchestration_policies/**
|
||||
/ee/lib/gitlab/ci/config/security_orchestration_policies/
|
||||
/ee/lib/gitlab/graphql/aggregations/security_orchestration_policies/
|
||||
/ee/spec/controllers/groups/security/policies_controller_spec.rb
|
||||
/ee/spec/factories/security_orchestration_policy_configurations.rb
|
||||
/ee/spec/factories/security_orchestration_policy_rule_schedules.rb
|
||||
/ee/spec/factories/security/policies.rb
|
||||
/ee/spec/graphql/mutations/security_policy/**
|
||||
/ee/spec/graphql/resolvers/security_orchestration/**
|
||||
/ee/spec/graphql/types/security_orchestration/**
|
||||
/ee/spec/graphql/mutations/security_policy/
|
||||
/ee/spec/graphql/resolvers/security_orchestration/
|
||||
/ee/spec/graphql/types/security_orchestration/
|
||||
/ee/spec/helpers/ee/security_orchestration_helper_spec.rb
|
||||
/ee/spec/lib/ee/gitlab/ci/pipeline/chain/validate/security_orchestration_policy_spec.rb
|
||||
/ee/spec/lib/gitlab/ci/config/security_orchestration_policies/**
|
||||
/ee/spec/lib/gitlab/graphql/aggregations/security_orchestration_policies/**
|
||||
/ee/spec/lib/gitlab/ci/config/security_orchestration_policies/
|
||||
/ee/spec/lib/gitlab/graphql/aggregations/security_orchestration_policies/
|
||||
/ee/spec/models/security/orchestration_policy_configuration_spec.rb
|
||||
/ee/spec/models/security/orchestration_policy_rule_schedule_spec.rb
|
||||
/ee/spec/requests/api/graphql/mutations/security_policy/**
|
||||
/ee/spec/requests/api/graphql/project/security_orchestration/**
|
||||
/ee/spec/requests/api/graphql/mutations/security_policy/
|
||||
/ee/spec/requests/api/graphql/project/security_orchestration/
|
||||
/ee/spec/requests/projects/security/policies_controller_spec.rb
|
||||
/ee/spec/services/security/orchestration/**
|
||||
/ee/spec/services/security/security_orchestration_policies/**
|
||||
/ee/spec/support/shared_contexts/graphql/resolvers/security_orchestration/**
|
||||
/ee/spec/services/security/orchestration/
|
||||
/ee/spec/services/security/security_orchestration_policies/
|
||||
/ee/spec/support/shared_contexts/graphql/resolvers/security_orchestration/
|
||||
/ee/spec/views/projects/security/policies/index.html.haml_spec.rb
|
||||
/ee/spec/workers/concerns/update_orchestration_policy_configuration_spec.rb
|
||||
/ee/spec/workers/security/create_orchestration_policy_worker_spec.rb
|
||||
|
|
@ -312,14 +312,14 @@ Dangerfile
|
|||
/ee/app/models/sca/license_policy.rb
|
||||
/ee/app/models/software_license_policy.rb
|
||||
/ee/app/models/software_license.rb
|
||||
/ee/app/serializers/license_compliance/**
|
||||
/ee/app/serializers/license_compliance/
|
||||
/ee/app/serializers/license_entity.rb
|
||||
/ee/app/serializers/licenses_list_entity.rb
|
||||
/ee/app/serializers/licenses_list_serializer.rb
|
||||
/ee/app/serializers/security/license_policy_entity.rb
|
||||
/ee/app/services/ci/compare_license_scanning_reports_collapsed_service.rb
|
||||
/ee/app/services/ci/compare_license_scanning_reports_service.rb
|
||||
/ee/app/services/software_license_policies/**
|
||||
/ee/app/services/software_license_policies/
|
||||
/ee/app/workers/refresh_license_compliance_checks_worker.rb
|
||||
/ee/lib/api/managed_licenses.rb
|
||||
/ee/lib/ee/api/entities/managed_license.rb
|
||||
|
|
@ -328,16 +328,16 @@ Dangerfile
|
|||
/ee/spec/factories/software_license.rb
|
||||
/ee/spec/factories/spdx_license.rb
|
||||
/ee/spec/finders/software_license_policies_finder_spec.rb
|
||||
/ee/spec/lib/gitlab/ci/parsers/license_compliance/**
|
||||
/ee/spec/lib/gitlab/ci/parsers/license_compliance/
|
||||
/ee/spec/models/sca/license_compliance_spec.rb @gitlab-org/security-risk-management/security-policies/backend @gitlab-org/secure/composition-analysis-be
|
||||
/ee/spec/models/sca/license_policy_spec.rb
|
||||
/ee/spec/models/software_license_policy_spec.rb
|
||||
/ee/spec/models/software_license_spec.rb
|
||||
/ee/spec/requests/api/managed_licenses_spec.rb
|
||||
/ee/spec/serializers/license_compliance/**
|
||||
/ee/spec/serializers/license_compliance/
|
||||
/ee/spec/services/ci/compare_license_scanning_reports_collapsed_service_spec.rb
|
||||
/ee/spec/services/ci/compare_license_scanning_reports_service_spec.rb
|
||||
/ee/spec/services/software_license_policies/**
|
||||
/ee/spec/services/software_license_policies/
|
||||
/spec/finders/security/license_compliance_jobs_finder_spec.rb
|
||||
|
||||
[Secure::Secret Detection] @gitlab-org/secure/secret-detection
|
||||
|
|
@ -556,8 +556,8 @@ lib/gitlab/git_access.rb
|
|||
lib/gitlab/git_access_*.rb
|
||||
ee/lib/ee/gitlab/git_access.rb
|
||||
ee/lib/ee/gitlab/git_access_*.rb
|
||||
ee/lib/ee/gitlab/checks/**
|
||||
lib/gitlab/checks/**
|
||||
ee/lib/ee/gitlab/checks/
|
||||
lib/gitlab/checks/
|
||||
|
||||
^[Technical writing] @gitlab-org/technical-writing/tw-docops @gitlab-org/tw-leadership
|
||||
/.gitlab/ci/docs.gitlab-ci.yaml
|
||||
|
|
@ -1210,7 +1210,7 @@ lib/gitlab/checks/**
|
|||
|
||||
# i18n Docs
|
||||
[Localization Team] @gitlab-com/localization/maintainers
|
||||
/doc-locale/**
|
||||
/doc-locale/
|
||||
/doc/development/i18n/proofreader.md
|
||||
/argo_translation.yml
|
||||
|
||||
|
|
|
|||
2
Gemfile
2
Gemfile
|
|
@ -306,7 +306,7 @@ gem 'gitlab-sidekiq-fetcher',
|
|||
gem 'fugit', '~> 1.11.1', feature_category: :continuous_integration
|
||||
|
||||
# HTTP requests
|
||||
gem 'httparty', '~> 0.22.0', feature_category: :shared
|
||||
gem 'httparty', '~> 0.23.0', feature_category: :shared
|
||||
|
||||
# Colored output to console
|
||||
gem 'rainbow', '~> 3.0', feature_category: :shared
|
||||
|
|
|
|||
|
|
@ -326,7 +326,7 @@
|
|||
{"name":"http-accept","version":"1.7.0","platform":"ruby","checksum":"c626860682bfbb3b46462f8c39cd470fd7b0584f61b3cc9df5b2e9eb9972a126"},
|
||||
{"name":"http-cookie","version":"1.0.5","platform":"ruby","checksum":"73756d46c7dbdc7023deecdb8a171348ea95a1b99810b31cfe8b4fb4e9a6318f"},
|
||||
{"name":"http-form_data","version":"2.3.0","platform":"ruby","checksum":"cc4eeb1361d9876821e31d7b1cf0b68f1cf874b201d27903480479d86448a5f3"},
|
||||
{"name":"httparty","version":"0.22.0","platform":"ruby","checksum":"78652a5c9471cf0093d3b2083c2295c9c8f12b44c65112f1846af2b71430fa6c"},
|
||||
{"name":"httparty","version":"0.23.1","platform":"ruby","checksum":"3ac1dd62f2010f6ece551716f5ceec2b2012011d89f1751917ab7f724e966b55"},
|
||||
{"name":"httpclient","version":"2.8.3","platform":"ruby","checksum":"2951e4991214464c3e92107e46438527d23048e634f3aee91c719e0bdfaebda6"},
|
||||
{"name":"i18n","version":"1.14.4","platform":"ruby","checksum":"c7deedead0866ea9102975a4eab7968f53de50793a0c211a37808f75dd187551"},
|
||||
{"name":"i18n_data","version":"0.13.1","platform":"ruby","checksum":"e5aa99b09a69b463bb0443fc1f9540351a49f3d1541c5e91316bafa035c63f66"},
|
||||
|
|
|
|||
|
|
@ -1027,7 +1027,7 @@ GEM
|
|||
http-cookie (1.0.5)
|
||||
domain_name (~> 0.5)
|
||||
http-form_data (2.3.0)
|
||||
httparty (0.22.0)
|
||||
httparty (0.23.1)
|
||||
csv
|
||||
mini_mime (>= 1.0.0)
|
||||
multi_xml (>= 0.5.2)
|
||||
|
|
@ -2223,7 +2223,7 @@ DEPENDENCIES
|
|||
health_check (~> 3.0)
|
||||
html-pipeline (~> 2.14.3)
|
||||
html2text
|
||||
httparty (~> 0.22.0)
|
||||
httparty (~> 0.23.0)
|
||||
i18n_data (~> 0.13.1)
|
||||
icalendar (~> 2.10.1)
|
||||
influxdb-client (~> 3.1)
|
||||
|
|
|
|||
|
|
@ -326,7 +326,7 @@
|
|||
{"name":"http-accept","version":"1.7.0","platform":"ruby","checksum":"c626860682bfbb3b46462f8c39cd470fd7b0584f61b3cc9df5b2e9eb9972a126"},
|
||||
{"name":"http-cookie","version":"1.0.5","platform":"ruby","checksum":"73756d46c7dbdc7023deecdb8a171348ea95a1b99810b31cfe8b4fb4e9a6318f"},
|
||||
{"name":"http-form_data","version":"2.3.0","platform":"ruby","checksum":"cc4eeb1361d9876821e31d7b1cf0b68f1cf874b201d27903480479d86448a5f3"},
|
||||
{"name":"httparty","version":"0.22.0","platform":"ruby","checksum":"78652a5c9471cf0093d3b2083c2295c9c8f12b44c65112f1846af2b71430fa6c"},
|
||||
{"name":"httparty","version":"0.23.1","platform":"ruby","checksum":"3ac1dd62f2010f6ece551716f5ceec2b2012011d89f1751917ab7f724e966b55"},
|
||||
{"name":"httpclient","version":"2.8.3","platform":"ruby","checksum":"2951e4991214464c3e92107e46438527d23048e634f3aee91c719e0bdfaebda6"},
|
||||
{"name":"i18n","version":"1.14.4","platform":"ruby","checksum":"c7deedead0866ea9102975a4eab7968f53de50793a0c211a37808f75dd187551"},
|
||||
{"name":"i18n_data","version":"0.13.1","platform":"ruby","checksum":"e5aa99b09a69b463bb0443fc1f9540351a49f3d1541c5e91316bafa035c63f66"},
|
||||
|
|
|
|||
|
|
@ -1021,7 +1021,7 @@ GEM
|
|||
http-cookie (1.0.5)
|
||||
domain_name (~> 0.5)
|
||||
http-form_data (2.3.0)
|
||||
httparty (0.22.0)
|
||||
httparty (0.23.1)
|
||||
csv
|
||||
mini_mime (>= 1.0.0)
|
||||
multi_xml (>= 0.5.2)
|
||||
|
|
@ -2218,7 +2218,7 @@ DEPENDENCIES
|
|||
health_check (~> 3.0)
|
||||
html-pipeline (~> 2.14.3)
|
||||
html2text
|
||||
httparty (~> 0.22.0)
|
||||
httparty (~> 0.23.0)
|
||||
i18n_data (~> 0.13.1)
|
||||
icalendar (~> 2.10.1)
|
||||
influxdb-client (~> 3.1)
|
||||
|
|
|
|||
|
|
@ -19,11 +19,13 @@ module Ci
|
|||
PIPELINE_FIELD_NAMES = %i[id duration status source ref].freeze
|
||||
PIPELINE_EPOCH_FIELD_NAMES = %i[committed_at created_at started_at finished_at].freeze
|
||||
PIPELINE_COMPUTED_FIELD_NAMES = %i[path].freeze
|
||||
PIPELINE_META_FIELD_NAMES = %i[name].freeze
|
||||
|
||||
CSV_MAPPING = {
|
||||
**PIPELINE_FIELD_NAMES.index_with { |n| n },
|
||||
**PIPELINE_EPOCH_FIELD_NAMES.index_with { |n| :"casted_#{n}" },
|
||||
**PIPELINE_COMPUTED_FIELD_NAMES.index_with { |n| n }
|
||||
**PIPELINE_COMPUTED_FIELD_NAMES.index_with { |n| n },
|
||||
**PIPELINE_META_FIELD_NAMES.index_with { |n| n }
|
||||
}.freeze
|
||||
|
||||
INSERT_FINISHED_PIPELINES_QUERY = <<~SQL.squish
|
||||
|
|
@ -142,7 +144,7 @@ module Ci
|
|||
|
||||
pipelines = Ci::Pipeline.id_in(pipeline_ids)
|
||||
pipelines
|
||||
.left_outer_joins(project_mirror: :namespace_mirror)
|
||||
.left_outer_joins(project_mirror: :namespace_mirror, pipeline_metadata: [])
|
||||
.select(:finished_at, *finished_pipeline_projections)
|
||||
.each do |pipeline|
|
||||
records_yielder << pipeline.attributes.symbolize_keys.tap do |record|
|
||||
|
|
@ -160,7 +162,8 @@ module Ci
|
|||
*PIPELINE_FIELD_NAMES.map { |n| "#{::Ci::Pipeline.table_name}.#{n}" },
|
||||
*PIPELINE_EPOCH_FIELD_NAMES
|
||||
.map { |n| "COALESCE(EXTRACT(epoch FROM #{::Ci::Pipeline.table_name}.#{n}), 0) AS casted_#{n}" },
|
||||
"ARRAY_TO_STRING(#{::Ci::NamespaceMirror.table_name}.traversal_ids, '/') || '/' AS path"
|
||||
"ARRAY_TO_STRING(#{::Ci::NamespaceMirror.table_name}.traversal_ids, '/') || '/' AS path",
|
||||
*PIPELINE_META_FIELD_NAMES.map { |n| "#{::Ci::PipelineMetadata.table_name}.#{n} AS #{n}" }
|
||||
]
|
||||
end
|
||||
strong_memoize_attr :finished_pipeline_projections
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AddNameToCiFinishedPipelines < ClickHouse::Migration
|
||||
def up
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines
|
||||
ADD COLUMN IF NOT EXISTS `name` String DEFAULT ''
|
||||
SQL
|
||||
end
|
||||
|
||||
def down
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines
|
||||
DROP COLUMN IF EXISTS `name`
|
||||
SQL
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AddNameToCiFinishedPipelinesDaily < ClickHouse::Migration
|
||||
def up
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines_daily
|
||||
ADD COLUMN IF NOT EXISTS `name` String DEFAULT ''
|
||||
SQL
|
||||
end
|
||||
|
||||
def down
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines_daily
|
||||
DROP COLUMN IF EXISTS `name`
|
||||
SQL
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AddNameToCiFinishedPipelinesHourly < ClickHouse::Migration
|
||||
def up
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines_hourly
|
||||
ADD COLUMN IF NOT EXISTS `name` String DEFAULT ''
|
||||
SQL
|
||||
end
|
||||
|
||||
def down
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines_hourly
|
||||
DROP COLUMN IF EXISTS `name`
|
||||
SQL
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AlterCiFinishedPipelinesDailyMv < ClickHouse::Migration
|
||||
def up
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines_daily_mv MODIFY QUERY
|
||||
SELECT
|
||||
path,
|
||||
status,
|
||||
source,
|
||||
ref,
|
||||
name,
|
||||
toStartOfInterval(started_at, INTERVAL 1 day) AS started_at_bucket,
|
||||
countState() AS count_pipelines,
|
||||
quantileState(duration) AS duration_quantile
|
||||
FROM ci_finished_pipelines
|
||||
GROUP BY path, status, source, ref, name, started_at_bucket
|
||||
SQL
|
||||
end
|
||||
|
||||
def down
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines_daily_mv MODIFY QUERY
|
||||
SELECT
|
||||
path,
|
||||
status,
|
||||
source,
|
||||
ref,
|
||||
toStartOfInterval(started_at, INTERVAL 1 day) AS started_at_bucket,
|
||||
countState() AS count_pipelines,
|
||||
quantileState(duration) AS duration_quantile
|
||||
FROM ci_finished_pipelines
|
||||
GROUP BY path, status, source, ref, started_at_bucket
|
||||
SQL
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AlterCiFinishedPipelinesHourlyMv < ClickHouse::Migration
|
||||
def up
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines_hourly_mv MODIFY QUERY
|
||||
SELECT
|
||||
path,
|
||||
status,
|
||||
source,
|
||||
ref,
|
||||
name,
|
||||
toStartOfInterval(started_at, INTERVAL 1 hour) AS started_at_bucket,
|
||||
countState() AS count_pipelines,
|
||||
quantileState(duration) AS duration_quantile
|
||||
FROM ci_finished_pipelines
|
||||
GROUP BY path, status, source, ref, name, started_at_bucket
|
||||
SQL
|
||||
end
|
||||
|
||||
def down
|
||||
execute <<~SQL
|
||||
ALTER TABLE ci_finished_pipelines_hourly_mv MODIFY QUERY
|
||||
SELECT
|
||||
path,
|
||||
status,
|
||||
source,
|
||||
ref,
|
||||
toStartOfInterval(started_at, INTERVAL 1 hour) AS started_at_bucket,
|
||||
countState() AS count_pipelines,
|
||||
quantileState(duration) AS duration_quantile
|
||||
FROM ci_finished_pipelines
|
||||
GROUP BY path, status, source, ref, started_at_bucket
|
||||
SQL
|
||||
end
|
||||
end
|
||||
|
|
@ -93,9 +93,13 @@ function run_rspec_non_fast {
|
|||
|
||||
files_for_non_fast=()
|
||||
|
||||
# Note that we do NOT exclude the fast_spec_helper specs here, because sometimes specs may pass
|
||||
# when run with fast_spec_helper, but fail when run with the full spec_helper. This happens when
|
||||
# they are run as part of a larger suite of mixed fast and slow files, for example, in CI jobs.
|
||||
# Running all fast and slow specs here ensures that we catch those cases.
|
||||
while IFS='' read -r file; do
|
||||
files_for_non_fast+=("$file")
|
||||
done < <(git grep -L -E '^require .fast_spec_helper' -- '**/remote_development/*_spec.rb' | grep -v 'qa/qa' | grep -v '/features/')
|
||||
done < <(git ls-files -- '**/remote_development/*_spec.rb' | grep -v 'qa/qa' | grep -v '/features/')
|
||||
|
||||
files_for_non_fast+=(
|
||||
"ee/spec/graphql/resolvers/clusters/agents_resolver_spec.rb"
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
|
||||
let_it_be(:pipeline5) { create(:ci_pipeline, :pending, project: project1, ref: 'feature/b') }
|
||||
|
||||
let_it_be(:pipeline_with_name) { create(:ci_pipeline, :success, name: 'name', finished_at: 1.week.ago) }
|
||||
|
||||
before_all do
|
||||
create_sync_events(*Ci::Pipeline.finished.order(id: :desc))
|
||||
end
|
||||
|
|
@ -43,26 +45,26 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
it 'processes the pipelines' do
|
||||
expect(ClickHouse::Client).to receive(:insert_csv).once.and_call_original
|
||||
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(4)
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(5)
|
||||
expect(execute).to have_attributes({
|
||||
payload: {
|
||||
reached_end_of_table: true,
|
||||
records_inserted: 4,
|
||||
records_inserted: 5,
|
||||
worker_index: 0, total_workers: 1
|
||||
}
|
||||
})
|
||||
|
||||
records = ci_finished_pipelines
|
||||
expect(records.count).to eq 4
|
||||
expect(records).to contain_exactly_pipelines(pipeline1, pipeline2, pipeline3, pipeline4)
|
||||
expect(records.count).to eq 5
|
||||
expect(records).to contain_exactly_pipelines(pipeline1, pipeline2, pipeline3, pipeline4, pipeline_with_name)
|
||||
end
|
||||
|
||||
it 'processes only pipelines from Ci::FinishedPipelineChSyncEvent' do
|
||||
pipeline = create(:ci_pipeline, :failed, finished_at: 1.minute.ago)
|
||||
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(4)
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(5)
|
||||
expect(execute).to have_attributes({
|
||||
payload: a_hash_including(reached_end_of_table: true, records_inserted: 4)
|
||||
payload: a_hash_including(reached_end_of_table: true, records_inserted: 5)
|
||||
})
|
||||
|
||||
create_sync_events(pipeline)
|
||||
|
|
@ -76,7 +78,7 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
.tap(&:save!)
|
||||
|
||||
expect { execute }
|
||||
.to change { ci_finished_pipelines_row_count }.by(4)
|
||||
.to change { ci_finished_pipelines_row_count }.by(5)
|
||||
.and change { sync_event.reload.processed }.to(true)
|
||||
end
|
||||
end
|
||||
|
|
@ -90,9 +92,9 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
it 'processes the pipelines' do
|
||||
expect(ClickHouse::Client).to receive(:insert_csv).once.and_call_original
|
||||
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(4)
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(5)
|
||||
expect(execute).to have_attributes({
|
||||
payload: a_hash_including(reached_end_of_table: true, records_inserted: 4)
|
||||
payload: a_hash_including(reached_end_of_table: true, records_inserted: 5)
|
||||
})
|
||||
end
|
||||
end
|
||||
|
|
@ -100,7 +102,7 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
context 'when multiple CSV uploads are required' do
|
||||
before do
|
||||
stub_const("#{described_class}::PIPELINES_BATCH_SIZE", 1)
|
||||
stub_const("#{described_class}::PIPELINES_BATCH_COUNT", 2)
|
||||
stub_const("#{described_class}::PIPELINES_BATCH_COUNT", 3)
|
||||
end
|
||||
|
||||
it 'processes the pipelines' do
|
||||
|
|
@ -110,9 +112,9 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
|
||||
expect(ClickHouse::Client).to receive(:insert_csv).twice.and_call_original
|
||||
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(4)
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(5)
|
||||
expect(execute).to have_attributes({
|
||||
payload: a_hash_including(reached_end_of_table: true, records_inserted: 4)
|
||||
payload: a_hash_including(reached_end_of_table: true, records_inserted: 5)
|
||||
})
|
||||
end
|
||||
|
||||
|
|
@ -154,9 +156,9 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
expect(iterator).to receive(:each_batch).once.with(of: described_class::PIPELINES_BATCH_SIZE).and_call_original
|
||||
end
|
||||
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(4)
|
||||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(5)
|
||||
expect(execute).to have_attributes({
|
||||
payload: a_hash_including(reached_end_of_table: true, records_inserted: 4)
|
||||
payload: a_hash_including(reached_end_of_table: true, records_inserted: 5)
|
||||
})
|
||||
|
||||
pipeline6 = create(:ci_pipeline, :failed, finished_at: 1.minute.ago)
|
||||
|
|
@ -164,13 +166,15 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
|
||||
expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(1)
|
||||
records = ci_finished_pipelines
|
||||
expect(records.count).to eq 5
|
||||
expect(records).to contain_exactly_pipelines(pipeline1, pipeline2, pipeline3, pipeline4, pipeline6)
|
||||
expect(records.count).to eq 6
|
||||
expect(records).to contain_exactly_pipelines(
|
||||
pipeline1, pipeline2, pipeline3, pipeline4, pipeline_with_name, pipeline6
|
||||
)
|
||||
end
|
||||
|
||||
context 'with same updated_at value' do
|
||||
it 'processes the pipelines' do
|
||||
expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(4)
|
||||
expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(5)
|
||||
|
||||
pipeline6 = create(:ci_pipeline, :failed, finished_at: 1.second.ago, updated_at: 1.second.ago)
|
||||
pipeline7 = create(:ci_pipeline, :failed, finished_at: 1.second.ago, updated_at: 1.second.ago)
|
||||
|
|
@ -179,14 +183,17 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
expect { execute }.to change { ci_finished_pipelines_row_count }.by(2)
|
||||
|
||||
records = ci_finished_pipelines
|
||||
expect(records.count).to eq 6
|
||||
expect(records).to contain_exactly_pipelines(pipeline1, pipeline2, pipeline3, pipeline4, pipeline6, pipeline7)
|
||||
expect(records.count).to eq 7
|
||||
expect(records).to contain_exactly_pipelines(
|
||||
pipeline1, pipeline2, pipeline3, pipeline4, pipeline_with_name,
|
||||
pipeline6, pipeline7
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'with older finished_at value' do
|
||||
it 'does not process the pipeline' do
|
||||
expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(4)
|
||||
expect { service.execute }.to change { ci_finished_pipelines_row_count }.by(5)
|
||||
|
||||
create(:ci_pipeline, :failed, finished_at: 2.days.ago)
|
||||
|
||||
|
|
@ -264,7 +271,8 @@ RSpec.describe Ci::ClickHouse::DataIngestion::FinishedPipelinesSyncService, '#ex
|
|||
status: pipeline.status || '',
|
||||
source: pipeline.source || '',
|
||||
ref: pipeline.ref || '',
|
||||
date: pipeline.finished_at.beginning_of_month
|
||||
date: pipeline.finished_at.beginning_of_month,
|
||||
name: pipeline.name || ''
|
||||
}
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue