diff --git a/Gemfile.checksum b/Gemfile.checksum index 0c4f06de117..5e66212c82f 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -101,7 +101,7 @@ {"name":"cvss-suite","version":"3.3.0","platform":"ruby","checksum":"54199fc1e0d5d833b1cb161453439143665c644cd3be2d21c2eee57fa06ed843"}, {"name":"danger","version":"9.4.2","platform":"ruby","checksum":"43e552c6731030235a30fdeafe703d2e2ab9c30917154489cb0ecd9ad3259d80"}, {"name":"danger-gitlab","version":"8.0.0","platform":"ruby","checksum":"497dd7d0f6513913de651019223d8058cf494df10acbd17de92b175dfa04a3a8"}, -{"name":"database_cleaner-active_record","version":"2.2.0","platform":"ruby","checksum":"3228d6d8ec1f2103fd6ab468dae923424318bcfabcf5dd5b02e5fcb0c486e1c7"}, +{"name":"database_cleaner-active_record","version":"2.2.1","platform":"ruby","checksum":"be47005de91e48f97841f8220f7cbd57b13682073016caac7ce332e6c77bbfcb"}, {"name":"database_cleaner-core","version":"2.0.1","platform":"ruby","checksum":"8646574c32162e59ed7b5258a97a208d3c44551b854e510994f24683865d846c"}, {"name":"date","version":"3.4.1","platform":"java","checksum":"74740d914c65a922a15657c25ff0e203c16f1d0f7aa910a9ebed712afe9819c4"}, {"name":"date","version":"3.4.1","platform":"ruby","checksum":"bf268e14ef7158009bfeaec40b5fa3c7271906e88b196d958a89d4b408abe64f"}, diff --git a/Gemfile.lock b/Gemfile.lock index d162bd76dc9..aa0df090580 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -499,7 +499,7 @@ GEM danger-gitlab (8.0.0) danger gitlab (~> 4.2, >= 4.2.0) - database_cleaner-active_record (2.2.0) + database_cleaner-active_record (2.2.1) activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index 8c10e3bbf47..b67d6aa1aa5 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -101,7 +101,7 @@ {"name":"cvss-suite","version":"3.3.0","platform":"ruby","checksum":"54199fc1e0d5d833b1cb161453439143665c644cd3be2d21c2eee57fa06ed843"}, {"name":"danger","version":"9.4.2","platform":"ruby","checksum":"43e552c6731030235a30fdeafe703d2e2ab9c30917154489cb0ecd9ad3259d80"}, {"name":"danger-gitlab","version":"8.0.0","platform":"ruby","checksum":"497dd7d0f6513913de651019223d8058cf494df10acbd17de92b175dfa04a3a8"}, -{"name":"database_cleaner-active_record","version":"2.2.0","platform":"ruby","checksum":"3228d6d8ec1f2103fd6ab468dae923424318bcfabcf5dd5b02e5fcb0c486e1c7"}, +{"name":"database_cleaner-active_record","version":"2.2.1","platform":"ruby","checksum":"be47005de91e48f97841f8220f7cbd57b13682073016caac7ce332e6c77bbfcb"}, {"name":"database_cleaner-core","version":"2.0.1","platform":"ruby","checksum":"8646574c32162e59ed7b5258a97a208d3c44551b854e510994f24683865d846c"}, {"name":"date","version":"3.4.1","platform":"java","checksum":"74740d914c65a922a15657c25ff0e203c16f1d0f7aa910a9ebed712afe9819c4"}, {"name":"date","version":"3.4.1","platform":"ruby","checksum":"bf268e14ef7158009bfeaec40b5fa3c7271906e88b196d958a89d4b408abe64f"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index 8ed8113c68e..c7da29d17b0 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -493,7 +493,7 @@ GEM danger-gitlab (8.0.0) danger gitlab (~> 4.2, >= 4.2.0) - database_cleaner-active_record (2.2.0) + database_cleaner-active_record (2.2.1) activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) diff --git a/config/events/ci_template_included.yml b/config/events/ci_template_included.yml index ac706d8c56d..ff9d1d65b1e 100644 --- a/config/events/ci_template_included.yml +++ b/config/events/ci_template_included.yml @@ -14,3 +14,8 @@ tiers: - free - premium - ultimate +additional_properties: + label: + description: Template event name + property: + description: "Indicates if the template was included implicitly or not. Implicit includes happen due to certain features, like AutoDevOps, being configured – as opposed to explicit includes via the keyword in a YAML file." diff --git a/config/metrics/counts_28d/20210216184458_p_ci_templates_implicit_auto_devops_monthly.yml b/config/metrics/counts_28d/20210216184458_p_ci_templates_implicit_auto_devops_monthly.yml index 671d4f995cc..b3c6084fd71 100644 --- a/config/metrics/counts_28d/20210216184458_p_ci_templates_implicit_auto_devops_monthly.yml +++ b/config/metrics/counts_28d/20210216184458_p_ci_templates_implicit_auto_devops_monthly.yml @@ -8,11 +8,13 @@ product_categories: value_type: number status: active time_frame: 28d -data_source: redis_hll -instrumentation_class: RedisHLLMetric -options: - events: - - p_ci_templates_implicit_auto_devops +data_source: internal_events +events: + - name: ci_template_included + unique: project.id + filter: + label: auto_devops + property: 'true' tiers: - free - premium diff --git a/config/metrics/counts_7d/20210216184456_p_ci_templates_implicit_auto_devops_weekly.yml b/config/metrics/counts_7d/20210216184456_p_ci_templates_implicit_auto_devops_weekly.yml index af8f44e9415..7cd05c46689 100644 --- a/config/metrics/counts_7d/20210216184456_p_ci_templates_implicit_auto_devops_weekly.yml +++ b/config/metrics/counts_7d/20210216184456_p_ci_templates_implicit_auto_devops_weekly.yml @@ -8,11 +8,13 @@ product_categories: value_type: number status: active time_frame: 7d -data_source: redis_hll -instrumentation_class: RedisHLLMetric -options: - events: - - p_ci_templates_implicit_auto_devops +data_source: internal_events +events: + - name: ci_template_included + unique: project.id + filter: + label: auto_devops + property: 'true' tiers: - free - premium diff --git a/config/metrics/counts_all/p_ci_templates_5_minute_production_app.yml b/config/metrics/counts_all/p_ci_templates_5_minute_production_app.yml index 8e08fb4f9e1..ca4ac51a230 100644 --- a/config/metrics/counts_all/p_ci_templates_5_minute_production_app.yml +++ b/config/metrics/counts_all/p_ci_templates_5_minute_production_app.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_5_minute_production_app +events: + - name: ci_template_included + unique: project.id + filter: + label: 5_minute_production_app + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_android.yml b/config/metrics/counts_all/p_ci_templates_android.yml index 97e3a0d49fb..f6a473cdebf 100644 --- a/config/metrics/counts_all/p_ci_templates_android.yml +++ b/config/metrics/counts_all/p_ci_templates_android.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_android +events: + - name: ci_template_included + unique: project.id + filter: + label: android + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_android_fastlane.yml b/config/metrics/counts_all/p_ci_templates_android_fastlane.yml index 98be43d0df5..2d1d8f371b6 100644 --- a/config/metrics/counts_all/p_ci_templates_android_fastlane.yml +++ b/config/metrics/counts_all/p_ci_templates_android_fastlane.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_android_fastlane +events: + - name: ci_template_included + unique: project.id + filter: + label: android_fastlane + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_android_latest.yml b/config/metrics/counts_all/p_ci_templates_android_latest.yml index 6f9e1ec59b9..1705cf226cf 100644 --- a/config/metrics/counts_all/p_ci_templates_android_latest.yml +++ b/config/metrics/counts_all/p_ci_templates_android_latest.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_android_latest +events: + - name: ci_template_included + unique: project.id + filter: + label: android_latest + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_aws_cf_provision_and_deploy_ec2.yml b/config/metrics/counts_all/p_ci_templates_aws_cf_provision_and_deploy_ec2.yml index 8e5d6c3c40d..636a0f9fa1b 100644 --- a/config/metrics/counts_all/p_ci_templates_aws_cf_provision_and_deploy_ec2.yml +++ b/config/metrics/counts_all/p_ci_templates_aws_cf_provision_and_deploy_ec2.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_aws_cf_provision_and_deploy_ec2 +events: + - name: ci_template_included + unique: project.id + filter: + label: aws_cf_provision_and_deploy_ec2 + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_bash.yml b/config/metrics/counts_all/p_ci_templates_bash.yml index 4a4f549c52f..1dd361b7878 100644 --- a/config/metrics/counts_all/p_ci_templates_bash.yml +++ b/config/metrics/counts_all/p_ci_templates_bash.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_bash +events: + - name: ci_template_included + unique: project.id + filter: + label: bash + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_c.yml b/config/metrics/counts_all/p_ci_templates_c.yml index 415a8db7a7b..68f4bec580b 100644 --- a/config/metrics/counts_all/p_ci_templates_c.yml +++ b/config/metrics/counts_all/p_ci_templates_c.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_c +events: + - name: ci_template_included + unique: project.id + filter: + label: c + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_chef.yml b/config/metrics/counts_all/p_ci_templates_chef.yml index 13b5fa20851..ee7aab05b5e 100644 --- a/config/metrics/counts_all/p_ci_templates_chef.yml +++ b/config/metrics/counts_all/p_ci_templates_chef.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_chef +events: + - name: ci_template_included + unique: project.id + filter: + label: chef + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_clojure.yml b/config/metrics/counts_all/p_ci_templates_clojure.yml index 75918435a19..69700c53581 100644 --- a/config/metrics/counts_all/p_ci_templates_clojure.yml +++ b/config/metrics/counts_all/p_ci_templates_clojure.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_clojure +events: + - name: ci_template_included + unique: project.id + filter: + label: clojure + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_code_quality.yml b/config/metrics/counts_all/p_ci_templates_code_quality.yml index 6d3417e2627..69947fa508a 100644 --- a/config/metrics/counts_all/p_ci_templates_code_quality.yml +++ b/config/metrics/counts_all/p_ci_templates_code_quality.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_code_quality +events: + - name: ci_template_included + unique: project.id + filter: + label: code_quality + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_composer.yml b/config/metrics/counts_all/p_ci_templates_composer.yml index 5cb9ab1f981..ce2fe9ad41c 100644 --- a/config/metrics/counts_all/p_ci_templates_composer.yml +++ b/config/metrics/counts_all/p_ci_templates_composer.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_composer +events: + - name: ci_template_included + unique: project.id + filter: + label: composer + property: 'false' diff --git a/config/metrics/counts_all/p_ci_templates_implicit_jobs_dast_default_branch_deploy.yml b/config/metrics/counts_all/p_ci_templates_implicit_jobs_dast_default_branch_deploy.yml index 87ff12399ea..2f51a67354e 100644 --- a/config/metrics/counts_all/p_ci_templates_implicit_jobs_dast_default_branch_deploy.yml +++ b/config/metrics/counts_all/p_ci_templates_implicit_jobs_dast_default_branch_deploy.yml @@ -9,13 +9,15 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: - 28d - 7d -data_source: redis_hll +data_source: internal_events data_category: optional -instrumentation_class: RedisHLLMetric tiers: - free - premium - ultimate -options: - events: - - p_ci_templates_implicit_jobs_dast_default_branch_deploy +events: + - name: ci_template_included + unique: project.id + filter: + label: jobs_dast_default_branch_deploy + property: 'true' diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index b8a964be59a..5629f744e59 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -3,6 +3,21 @@ module Gitlab::UsageDataCounters class CiTemplateUniqueCounter PREFIX = 'ci_templates' + MIGRATED_REDIS_HLL_EVENTS = %w[ + p_ci_templates_5_minute_production_app + p_ci_templates_android + p_ci_templates_android_fastlane + p_ci_templates_android_latest + p_ci_templates_aws_cf_provision_and_deploy_ec2 + p_ci_templates_bash + p_ci_templates_c + p_ci_templates_chef + p_ci_templates_clojure + p_ci_templates_code_quality + p_ci_templates_composer + p_ci_templates_implicit_auto_devops + p_ci_templates_implicit_jobs_dast_default_branch_deploy + ].freeze class << self def track_unique_project_event(project:, template:, config_source:, user:) @@ -10,10 +25,23 @@ module Gitlab::UsageDataCounters return unless expanded_template_name event_name = ci_template_event_name(expanded_template_name, config_source) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: project.id) + unless MIGRATED_REDIS_HLL_EVENTS.include?(event_name) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: project.id) + end namespace = project.namespace - Gitlab::InternalEvents.track_event('ci_template_included', namespace: namespace, project: project, user: user) + implicit = config_source.to_s == 'auto_devops_source' + + Gitlab::InternalEvents.track_event( + 'ci_template_included', + namespace: namespace, + project: project, + user: user, + additional_properties: { + label: template_to_event_name(expanded_template_name), + property: implicit.to_s + } + ) end def ci_templates(relative_base = 'lib/gitlab/ci/templates') diff --git a/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml b/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml index 1601610190b..a66c8e5f715 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml +++ b/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml @@ -188,3 +188,16 @@ i_container_registry_delete_tag_user-user: i_container_registry_delete_tag_user i_container_registry_push_repository_user-user: i_container_registry_push_repository_user i_container_registry_push_tag_user-user: i_container_registry_push_tag_user i_code_review_user_searches_diff-user: i_code_review_user_searches_diff +'ci_template_included-filter:[label:jobs_dast_default_branch_deploy,property:true]-project': p_ci_templates_implicit_jobs_dast_default_branch_deploy +'ci_template_included-filter:[label:android,property:false]-project': p_ci_templates_android +'ci_template_included-filter:[label:auto_devops,property:true]-project': p_ci_templates_implicit_auto_devops +'ci_template_included-filter:[label:5_minute_production_app,property:false]-project': p_ci_templates_5_minute_production_app +'ci_template_included-filter:[label:android_fastlane,property:false]-project': p_ci_templates_android_fastlane +'ci_template_included-filter:[label:android_latest,property:false]-project': p_ci_templates_android_latest +'ci_template_included-filter:[label:aws_cf_provision_and_deploy_ec2,property:false]-project': p_ci_templates_aws_cf_provision_and_deploy_ec2 +'ci_template_included-filter:[label:bash,property:false]-project': p_ci_templates_bash +'ci_template_included-filter:[label:c,property:false]-project': p_ci_templates_c +'ci_template_included-filter:[label:chef,property:false]-project': p_ci_templates_chef +'ci_template_included-filter:[label:clojure,property:false]-project': p_ci_templates_clojure +'ci_template_included-filter:[label:code_quality,property:false]-project': p_ci_templates_code_quality +'ci_template_included-filter:[label:composer,property:false]-project': p_ci_templates_composer diff --git a/lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml b/lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml index bf0b377c1df..f3ba77c40dc 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml +++ b/lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml @@ -353,19 +353,8 @@ - p_analytics_pipelines - p_analytics_repo - p_analytics_valuestream -- p_ci_templates_5_minute_production_app -- p_ci_templates_android -- p_ci_templates_android_fastlane -- p_ci_templates_android_latest - p_ci_templates_auto_devops -- p_ci_templates_aws_cf_provision_and_deploy_ec2 - p_ci_templates_aws_deploy_ecs -- p_ci_templates_bash -- p_ci_templates_c -- p_ci_templates_chef -- p_ci_templates_clojure -- p_ci_templates_code_quality -- p_ci_templates_composer - p_ci_templates_cosign - p_ci_templates_crystal - p_ci_templates_dart @@ -381,13 +370,11 @@ - p_ci_templates_go - p_ci_templates_gradle - p_ci_templates_grails -- p_ci_templates_implicit_auto_devops - p_ci_templates_implicit_jobs_browser_performance_testing - p_ci_templates_implicit_jobs_build - p_ci_templates_implicit_jobs_code_intelligence - p_ci_templates_implicit_jobs_code_quality - p_ci_templates_implicit_jobs_container_scanning -- p_ci_templates_implicit_jobs_dast_default_branch_deploy - p_ci_templates_implicit_jobs_dependency_scanning - p_ci_templates_implicit_jobs_deploy - p_ci_templates_implicit_jobs_deploy_ec2