diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index ddcb8f2ffaf..d3f8f2f6f31 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -1024,7 +1024,7 @@ lib/gitlab/checks/** /doc/user/workspace/ @ashrafkhamis # End rake-managed-docs-block -[Authentication and Authorization] @gitlab-org/manage/authentication-and-authorization/approvers +[Authentication and Authorization] @gitlab-org/govern/authentication-and-authorization/approvers /app/assets/javascripts/access_tokens/ /app/assets/javascripts/alerts_settings/graphql/mutations/reset_http_token.mutation.graphql /app/assets/javascripts/authentication/ diff --git a/app/controllers/acme_challenges_controller.rb b/app/controllers/acme_challenges_controller.rb index 4a7706db94e..a187e43b3df 100644 --- a/app/controllers/acme_challenges_controller.rb +++ b/app/controllers/acme_challenges_controller.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -# rubocop:disable Rails/ApplicationController -class AcmeChallengesController < ActionController::Base +class AcmeChallengesController < BaseActionController def show if acme_order render plain: acme_order.challenge_file_content, content_type: 'text/plain' @@ -16,4 +15,3 @@ class AcmeChallengesController < ActionController::Base @acme_order ||= PagesDomainAcmeOrder.find_by_domain_and_token(params[:domain], params[:token]) end end -# rubocop:enable Rails/ApplicationController diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 7c69f43fa3d..bfd3388da1b 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,7 +3,7 @@ require 'gon' require 'fogbugz' -class ApplicationController < ActionController::Base +class ApplicationController < BaseActionController include Gitlab::GonHelper include Gitlab::NoCacheHeaders include GitlabRoutingHelper diff --git a/app/controllers/base_action_controller.rb b/app/controllers/base_action_controller.rb new file mode 100644 index 00000000000..af2c9e98778 --- /dev/null +++ b/app/controllers/base_action_controller.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +# GitLab lightweight base action controller +# +# This class should be limited to content that +# is desired/required for *all* controllers in +# GitLab. +# +# Most controllers inherit from `ApplicationController`. +# Some controllers don't want or need all of that +# logic and instead inherit from `ActionController::Base`. +# This makes it difficult to set security headers and +# handle other critical logic across *all* controllers. +# +# Between this controller and `ApplicationController` +# no controller should ever inherit directly from +# `ActionController::Base` +# +# rubocop:disable Rails/ApplicationController +# rubocop:disable Gitlab/NamespacedClass +class BaseActionController < ActionController::Base + before_action :security_headers + + private + + def security_headers + headers['Cross-Origin-Opener-Policy'] = 'same-origin' if ::Feature.enabled?(:coop_header) + end +end +# rubocop:enable Gitlab/NamespacedClass +# rubocop:enable Rails/ApplicationController diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index 7328b793b09..b61a8c5ff12 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -# rubocop:disable Rails/ApplicationController -class ChaosController < ActionController::Base +class ChaosController < BaseActionController before_action :validate_chaos_secret, unless: :development_or_test? def leakmem @@ -95,4 +94,3 @@ class ChaosController < ActionController::Base Rails.env.development? || Rails.env.test? end end -# rubocop:enable Rails/ApplicationController diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index 1381999ab4c..2b2db2f950c 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -# rubocop:disable Rails/ApplicationController -class HealthController < ActionController::Base +class HealthController < BaseActionController protect_from_forgery with: :exception, prepend: true include RequiresAllowlistedMonitoringClient @@ -40,4 +39,3 @@ class HealthController < ActionController::Base render json: result.json, status: result.http_status end end -# rubocop:enable Rails/ApplicationController diff --git a/app/controllers/metrics_controller.rb b/app/controllers/metrics_controller.rb index 9f41c092fa0..61851fd1c60 100644 --- a/app/controllers/metrics_controller.rb +++ b/app/controllers/metrics_controller.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -# rubocop:disable Rails/ApplicationController -class MetricsController < ActionController::Base +class MetricsController < BaseActionController include RequiresAllowlistedMonitoringClient protect_from_forgery with: :exception, prepend: true @@ -36,4 +35,3 @@ class MetricsController < ActionController::Base ) end end -# rubocop:enable Rails/ApplicationController diff --git a/app/helpers/diff_helper.rb b/app/helpers/diff_helper.rb index 9a78d4d9ad5..47a0d010424 100644 --- a/app/helpers/diff_helper.rb +++ b/app/helpers/diff_helper.rb @@ -270,11 +270,6 @@ module DiffHelper toggle_whitespace_link(url, options) end - def diff_merge_request_whitespace_link(project, merge_request, options) - url = diffs_project_merge_request_path(project, merge_request, params_with_whitespace) - toggle_whitespace_link(url, options) - end - def diff_compare_whitespace_link(project, from, to, options) url = project_compare_path(project, from, to, params_with_whitespace) toggle_whitespace_link(url, options) diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index e45b38f2266..11a964128bc 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -252,8 +252,6 @@ module ProjectsHelper end def show_mobile_devops_project_promo?(project) - return false unless ::Feature.enabled?(:mobile_devops_projects_promo, project) - return false unless (project.project_setting.target_platforms & ::ProjectSetting::ALLOWED_TARGET_PLATFORMS).any? cookies["hide_mobile_devops_promo_#{project.id}".to_sym].blank? diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 8ddb2ead37e..8f12c4e6dad 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -16,12 +16,6 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord ignore_columns %i[instance_administration_project_id instance_administrators_group_id], remove_with: '16.2', remove_after: '2023-06-22' ignore_column :database_apdex_settings, remove_with: '16.4', remove_after: '2023-08-22' - ignore_columns %i[ - dashboard_notification_limit - dashboard_enforcement_limit - dashboard_limit_new_namespace_creation_enforcement_date - ], remove_with: '16.5', remove_after: '2023-08-22' - ignore_column %i[ relay_state_domain_allowlist in_product_marketing_emails_enabled diff --git a/app/models/namespace.rb b/app/models/namespace.rb index b82757153b8..5aed1f4f5fb 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -18,6 +18,9 @@ class Namespace < ApplicationRecord include Ci::NamespaceSettings include Referable include CrossDatabaseIgnoredTables + include IgnorableColumns + + ignore_column :unlock_membership_to_ldap, remove_with: '16.7', remove_after: '2023-11-16' cross_database_ignore_tables %w[routes redirect_routes], url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/424277' diff --git a/app/services/update_container_registry_info_service.rb b/app/services/update_container_registry_info_service.rb index 7d79b257687..de90e7c71ad 100644 --- a/app/services/update_container_registry_info_service.rb +++ b/app/services/update_container_registry_info_service.rb @@ -24,7 +24,8 @@ class UpdateContainerRegistryInfoService Gitlab::CurrentSettings.update!( container_registry_vendor: info[:vendor] || '', container_registry_version: info[:version] || '', - container_registry_features: info[:features] || [] + container_registry_features: info[:features] || [], + container_registry_db_enabled: info[:db_enabled] || false ) end end diff --git a/config/feature_flags/development/mobile_devops_projects_promo.yml b/config/feature_flags/development/coop_header.yml similarity index 61% rename from config/feature_flags/development/mobile_devops_projects_promo.yml rename to config/feature_flags/development/coop_header.yml index c1fcca28858..9166f4c6819 100644 --- a/config/feature_flags/development/mobile_devops_projects_promo.yml +++ b/config/feature_flags/development/coop_header.yml @@ -1,8 +1,8 @@ --- -name: mobile_devops_projects_promo -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120144 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/410601 -milestone: '16.0' +name: coop_header +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131571 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/425701 +milestone: '16.5' type: development -group: group::mobile devops +group: group::authentication and authorization default_enabled: false diff --git a/db/migrate/20230922004506_add_latest_released_at_to_catalog_resources.rb b/db/migrate/20230922004506_add_latest_released_at_to_catalog_resources.rb new file mode 100644 index 00000000000..20cdef24e93 --- /dev/null +++ b/db/migrate/20230922004506_add_latest_released_at_to_catalog_resources.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddLatestReleasedAtToCatalogResources < Gitlab::Database::Migration[2.1] + enable_lock_retries! + + def change + add_column :catalog_resources, :latest_released_at, :datetime_with_timezone, null: true + end +end diff --git a/db/schema_migrations/20230922004506 b/db/schema_migrations/20230922004506 new file mode 100644 index 00000000000..266ff7f1562 --- /dev/null +++ b/db/schema_migrations/20230922004506 @@ -0,0 +1 @@ +f67f0a5aa43db625638dd0c2ab54c09586c182d54366dd780b250704e35ebb89 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index dc3222cde8d..97144891aad 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -13306,7 +13306,8 @@ CREATE TABLE catalog_resources ( id bigint NOT NULL, project_id bigint NOT NULL, created_at timestamp with time zone NOT NULL, - state smallint DEFAULT 0 NOT NULL + state smallint DEFAULT 0 NOT NULL, + latest_released_at timestamp with time zone ); CREATE SEQUENCE catalog_resources_id_seq diff --git a/doc/administration/audit_event_streaming/audit_event_types.md b/doc/administration/audit_event_streaming/audit_event_types.md index d3f123a1850..a91493c79be 100644 --- a/doc/administration/audit_event_streaming/audit_event_types.md +++ b/doc/administration/audit_event_streaming/audit_event_types.md @@ -232,6 +232,7 @@ audit events to external destinations. | [`project_packages_enabled_updated`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7962) | When the setting that controls packages for a project is toggled, this audit event is created | **{check-circle}** Yes | **{check-circle}** Yes | `groups_and_projects` | GitLab [11.5](https://gitlab.com/gitlab-org/gitlab/-/issues/369288) | | [`project_path_updated`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/100770) | Event triggered on updating a project's path | **{check-circle}** Yes | **{check-circle}** Yes | `groups_and_projects` | GitLab [15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/369271) | | [`project_printing_merge_request_link_enabled_updated`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106652) | Event triggered on updating setting for projects for enabling printing merge request link | **{check-circle}** Yes | **{check-circle}** Yes | `groups_and_projects` | GitLab [15.7](https://gitlab.com/gitlab-org/gitlab/-/issues/369283) | +| [`project_push_rules_commit_committer_check_updated`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132157) | Triggered when project push rule setting is updated for reject unverified users. | **{check-circle}** Yes | **{check-circle}** Yes | `source_code_management` | GitLab [16.5](https://gitlab.com/gitlab-org/gitlab/-/issues/268116) | | [`project_remove_source_branch_after_merge_updated`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83922) | Create this audit event whenever a project has its setting to remove branches after merges modified | **{check-circle}** Yes | **{check-circle}** Yes | `code_review_workflow` | GitLab [14.10](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) | | [`project_repository_size_limit_updated`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106652) | Event triggered on updating repository size limit of a project | **{check-circle}** Yes | **{check-circle}** Yes | `groups_and_projects` | GitLab [15.7](https://gitlab.com/gitlab-org/gitlab/-/issues/369274) | | [`project_require_password_to_approve_updated`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106652) | Event triggered on updating project setting for requiring user's password for approval of merge request | **{check-circle}** Yes | **{check-circle}** Yes | `groups_and_projects` | GitLab [15.7](https://gitlab.com/gitlab-org/gitlab/-/issues/369280) | diff --git a/doc/development/cicd/templates.md b/doc/development/cicd/templates.md index bf392a3f446..f0627caa5b3 100644 --- a/doc/development/cicd/templates.md +++ b/doc/development/cicd/templates.md @@ -429,7 +429,7 @@ To add a metric definition for a new template: - [`config/metrics/counts_28d/20210216184559_ci_templates_total_unique_counts_monthly.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_28d/20210216184559_ci_templates_total_unique_counts_monthly.yml) 1. Use the same event name as above as the last argument in the following command to - [add new metric definitions](../internal_analytics/service_ping/metrics_dictionary.md#metrics-added-dynamic-to-service-ping-payload): + [add new metric definitions](../internal_analytics/metrics/metrics_dictionary.md#metrics-added-dynamic-to-service-ping-payload): ```shell bundle exec rails generate gitlab:usage_metric_definition:redis_hll ci_templates @@ -450,7 +450,7 @@ To add a metric definition for a new template: - `data_source:`: Set to `redis_hll`. - `description`: Add a short description of what this metric counts, for example: `Count of pipelines using the latest Auto Deploy template` - `product_*`: Set to [section, stage, group, and feature category](https://about.gitlab.com/handbook/product/categories/#devops-stages) - as per the [metrics dictionary guide](../internal_analytics/service_ping/metrics_dictionary.md#metrics-definition-and-validation). + as per the [metrics dictionary guide](../internal_analytics/metrics/metrics_dictionary.md#metrics-definition-and-validation). If you are unsure what to use for these keywords, you can ask for help in the merge request. - Add the following to the end of each file: diff --git a/doc/development/internal_analytics/internal_event_tracking/index.md b/doc/development/internal_analytics/internal_event_tracking/index.md index e35d5f6f084..ba628baa379 100644 --- a/doc/development/internal_analytics/internal_event_tracking/index.md +++ b/doc/development/internal_analytics/internal_event_tracking/index.md @@ -13,5 +13,5 @@ This page is a work in progress. For any questions or clarifications, reach out - [Introduction to internal event tracking](introduction.md#internal-event-tracking) - [Quick start guide](quick_start.md#quick-start-for-internal-event-tracking) - [Event definition guide](event_definition_guide.md#internal-event-tracking-definition-guide) -- [Metrics dictionary guide](../service_ping/metrics_dictionary.md#metrics-dictionary-guide) +- [Metrics dictionary guide](../metrics/metrics_dictionary.md#metrics-dictionary-guide) - [Architecture](architecture.md#internal-event-tracking-architecture) diff --git a/doc/development/internal_analytics/metrics/metrics_dictionary.md b/doc/development/internal_analytics/metrics/metrics_dictionary.md new file mode 100644 index 00000000000..660a945d1b8 --- /dev/null +++ b/doc/development/internal_analytics/metrics/metrics_dictionary.md @@ -0,0 +1,239 @@ +--- +stage: Analyze +group: Analytics Instrumentation +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Metrics Dictionary Guide + +[Service Ping](../service_ping/index.md) metrics are defined in individual YAML files definitions from which the +[Metrics Dictionary](https://metrics.gitlab.com/) is built. Currently, the metrics dictionary is built automatically once a day. When a change to a metric is made in a YAML file, you can see the change in the dictionary within 24 hours. +This guide describes the dictionary and how it's implemented. + +## Metrics Definition and validation + +We are using [JSON Schema](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/schema.json) to validate the metrics definition. + +This process is meant to ensure consistent and valid metrics defined for Service Ping. All metrics *must*: + +- Comply with the defined [JSON schema](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/schema.json). +- Have a unique `key_path` . +- Have an owner. + +All metrics are stored in YAML files: + +- [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics) + +WARNING: +Only metrics with a metric definition YAML and whose status is not `removed` are added to the Service Ping JSON payload. + +Each metric is defined in a separate YAML file consisting of a number of fields: + +| Field | Required | Additional information | +|---------------------|----------|----------------------------------------------------------------| +| `key_path` | yes | JSON key path for the metric, location in Service Ping payload. | +| `description` | yes | | +| `product_section` | yes | The [section](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/data/sections.yml). | +| `product_stage` | yes | The [stage](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) for the metric. | +| `product_group` | yes | The [group](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) that owns the metric. | +| `value_type` | yes | `string`; one of [`string`, `number`, `boolean`, `object`](https://json-schema.org/understanding-json-schema/reference/type.html). | +| `status` | yes | `string`; [status](#metric-statuses) of the metric, may be set to `active`, `removed`, `broken`. | +| `time_frame` | yes | `string`; may be set to a value like `7d`, `28d`, `all`, `none`. | +| `data_source` | yes | `string`; may be set to a value like `database`, `redis`, `redis_hll`, `prometheus`, `system`, `license`, `internal_events`. | +| `data_category` | yes | `string`; [categories](#data-category) of the metric, may be set to `operational`, `optional`, `subscription`, `standard`. The default value is `optional`.| +| `instrumentation_class` | yes | `string`; [the class that implements the metric](../service_ping/metrics_instrumentation.md). | +| `distribution` | yes | `array`; may be set to one of `ce, ee` or `ee`. The [distribution](https://about.gitlab.com/handbook/marketing/brand-and-product-marketing/product-and-solution-marketing/tiers/#definitions) where the tracked feature is available. | +| `performance_indicator_type` | no | `array`; may be set to one of [`gmau`, `smau`, `paid_gmau`, `umau` or `customer_health_score`](https://about.gitlab.com/handbook/business-technology/data-team/data-catalog/xmau-analysis/). | +| `tier` | yes | `array`; may contain one or a combination of `free`, `premium` or `ultimate`. The [tier](https://about.gitlab.com/handbook/marketing/brand-and-product-marketing/product-and-solution-marketing/tiers/#definitions) where the tracked feature is available. This should be verbose and contain all tiers where a metric is available. | +| `milestone` | yes | The milestone when the metric is introduced and when it's available to self-managed instances with the official GitLab release. | +| `milestone_removed` | no | The milestone when the metric is removed. Required for removed metrics. | +| `introduced_by_url` | no | The URL to the merge request that introduced the metric to be available for self-managed instances. | +| `removed_by_url` | no | The URL to the merge request that removed the metric. Required for removed metrics. | +| `repair_issue_url` | no | The URL of the issue that was created to repair a metric with a `broken` status. | +| `options` | no | `object`: options information needed to calculate the metric value. | +| `skip_validation` | no | This should **not** be set. [Used for imported metrics until we review, update and make them valid](https://gitlab.com/groups/gitlab-org/-/epics/5425). | + +### Metric `key_path` + +The `key_path` of the metric is the location in the JSON Service Ping payload. + +The `key_path` could be composed from multiple parts separated by `.` and it must be unique. + +We recommend to add the metric in one of the top-level keys: + +- `settings`: for settings related metrics. +- `counts_weekly`: for counters that have data for the most recent 7 days. +- `counts_monthly`: for counters that have data for the most recent 28 days. +- `counts`: for counters that have data for all time. + +NOTE: +We can't control what the metric's `key_path` is, because some of them are generated dynamically in `usage_data.rb`. +For example, see [Redis HLL metrics](../service_ping/implement.md#redis-hll-counters). + +### Metric statuses + +Metric definitions can have one of the following statuses: + +- `active`: Metric is used and reports data. +- `broken`: Metric reports broken data (for example, -1 fallback), or does not report data at all. A metric marked as `broken` must also have the `repair_issue_url` attribute. +- `removed`: Metric was removed, but it may appear in Service Ping payloads sent from instances running on older versions of GitLab. + +### Metric `value_type` + +Metric definitions can have one of the following values for `value_type`: + +- `boolean` +- `number` +- `string` +- `object`: A metric with `value_type: object` must have `value_json_schema` with a link to the JSON schema for the object. +In general, we avoid complex objects and prefer one of the `boolean`, `number`, or `string` value types. +An example of a metric that uses `value_type: object` is `topology` (`/config/metrics/settings/20210323120839_topology.yml`), +which has a related schema in `/config/metrics/objects_schemas/topology_schema.json`. + +### Metric `time_frame` + +A metric's time frame is calculated based on the `time_frame` field and the `data_source` of the metric. + +| data_source | time_frame | Description | +|------------------------|------------|-------------------------------------------------| +| any | `none` | A type of data that's not tracked over time, such as settings and configuration information | +| `database` | `all` | The whole time the metric has been active (all-time interval) | +| `database` | `7d` | 9 days ago to 2 days ago | +| `database` | `28d` | 30 days ago to 2 days ago | +| `redis` | `all` | The whole time the metric has been active (all-time interval) | +| `redis_hll` | `7d` | Most recent complete week | +| `redis_hll` | `28d` | Most recent 4 complete weeks | + +### Data category + +We use the following categories to classify a metric: + +- `operational`: Required data for operational purposes. +- `optional`: Default value for a metric. Data that is optional to collect. This can be [enabled or disabled](../../../administration/settings/usage_statistics.md#enable-or-disable-usage-statistics) in the Admin Area. +- `subscription`: Data related to licensing. +- `standard`: Standard set of identifiers that are included when collecting data. + +An aggregate metric is a metric that is the sum of two or more child metrics. Service Ping uses the data category of +the aggregate metric to determine whether or not the data is included in the reported Service Ping payload. + +### Example YAML metric definition + +The linked [`uuid`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/license/uuid.yml) +YAML file includes an example metric definition, where the `uuid` metric is the GitLab +instance unique identifier. + +```yaml +key_path: uuid +description: GitLab instance unique identifier +product_section: analytics +product_stage: analytics +product_group: analytics_instrumentation +value_type: string +status: active +milestone: 9.1 +instrumentation_class: UuidMetric +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1521 +time_frame: none +data_source: database +distribution: +- ce +- ee +tier: +- free +- premium +- ultimate +``` + +### Create a new metric definition + +The GitLab codebase provides a dedicated [generator](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/generators/gitlab/usage_metric_definition_generator.rb) to create new metric definitions. + +For uniqueness, the generated files include a timestamp prefix in ISO 8601 format. + +The generator takes a list of key paths and 3 options as arguments. It creates metric YAML definitions in the corresponding location: + +- `--ee`, `--no-ee` Indicates if metric is for EE. +- `--dir=DIR` Indicates the metric directory. It must be one of: `counts_7d`, `7d`, `counts_28d`, `28d`, `counts_all`, `all`, `settings`, `license`. +- `--class_name=CLASS_NAME` Indicates the instrumentation class. For example `UsersCreatingIssuesMetric`, `UuidMetric` + +**Single metric example** + +```shell +bundle exec rails generate gitlab:usage_metric_definition counts.issues --dir=7d --class_name=CountIssues +// Creates 1 file +// create config/metrics/counts_7d/issues.yml +``` + +**Multiple metrics example** + +```shell +bundle exec rails generate gitlab:usage_metric_definition counts.issues counts.users --dir=7d --class_name=CountUsersCreatingIssues +// Creates 2 files +// create config/metrics/counts_7d/issues.yml +// create config/metrics/counts_7d/users.yml +``` + +NOTE: +To create a metric definition used in EE, add the `--ee` flag. + +```shell +bundle exec rails generate gitlab:usage_metric_definition counts.issues --ee --dir=7d --class_name=CountUsersCreatingIssues +// Creates 1 file +// create ee/config/metrics/counts_7d/issues.yml +``` + +### Metrics added dynamic to Service Ping payload + +The [Redis HLL metrics](../service_ping/implement.md#known-events-are-added-automatically-in-service-data-payload) are added automatically to Service Ping payload. + +A YAML metric definition is required for each metric. A dedicated generator is provided to create metric definitions for Redis HLL events. + +The generator takes `category` and `events` arguments, as the root key is `redis_hll_counters`, and creates two metric definitions for each of the events (for weekly and monthly time frames): + +**Single metric example** + +```shell +bundle exec rails generate gitlab:usage_metric_definition:redis_hll issues count_users_closing_issues +// Creates 2 files +// create config/metrics/counts_7d/count_users_closing_issues_weekly.yml +// create config/metrics/counts_28d/count_users_closing_issues_monthly.yml +``` + +**Multiple metrics example** + +```shell +bundle exec rails generate gitlab:usage_metric_definition:redis_hll issues count_users_closing_issues count_users_reopening_issues +// Creates 4 files +// create config/metrics/counts_7d/count_users_closing_issues_weekly.yml +// create config/metrics/counts_28d/count_users_closing_issues_monthly.yml +// create config/metrics/counts_7d/count_users_reopening_issues_weekly.yml +// create config/metrics/counts_28d/count_users_reopening_issues_monthly.yml +``` + +To create a metric definition used in EE, add the `--ee` flag. + +```shell +bundle exec rails generate gitlab:usage_metric_definition:redis_hll issues users_closing_issues --ee +// Creates 2 files +// create config/metrics/counts_7d/i_closed_weekly.yml +// create config/metrics/counts_28d/i_closed_monthly.yml +``` + +### Performance Indicator Metrics + +To use a metric definition to manage [performance indicator](https://about.gitlab.com/handbook/product/analytics-instrumentation-guide/#implementing-product-performance-indicators): + +1. Create a merge request that includes related changes. +1. Use labels `~"analytics instrumentation"`, `"~Data Warehouse::Impact Check"`. +1. Update the metric definition `performance_indicator_type` [field](metrics_dictionary.md#metrics-definition-and-validation). +1. Create an issue in GitLab Product Data Insights project with the [PI Chart Help template](https://gitlab.com/gitlab-data/product-analytics/-/issues/new?issuable_template=PI%20Chart%20Help) to have the new metric visualized. + +## Metrics Dictionary + +[Metrics Dictionary is a separate application](https://gitlab.com/gitlab-org/analytics-section/analytics-instrumentation/metric-dictionary). + +All metrics available in Service Ping are in the [Metrics Dictionary](https://metrics.gitlab.com/). + +### Copy query to clipboard + +To check if a metric has data in Sisense, use the copy query to clipboard feature. This copies a query that's ready to use in Sisense. The query gets the last five service ping data for GitLab.com for a given metric. For information about how to check if a Service Ping metric has data in Sisense, see this [demo](https://www.youtube.com/watch?v=n4o65ivta48). diff --git a/doc/development/internal_analytics/metrics/metrics_lifecycle.md b/doc/development/internal_analytics/metrics/metrics_lifecycle.md new file mode 100644 index 00000000000..1d992a56649 --- /dev/null +++ b/doc/development/internal_analytics/metrics/metrics_lifecycle.md @@ -0,0 +1,105 @@ +--- +stage: Analyze +group: Analytics Instrumentation +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Service Ping metric lifecycle + +The following guidelines explain the steps to follow at each stage of a metric's lifecycle. + +## Add a new metric + +Follow the [Implement Service Ping](../service_ping/implement.md) guide. + +## Change an existing metric + +WARNING: +We want to **PREVENT** changes to the calculation logic or important attributes on any metric as this invalidates comparisons of the same metric across different versions of GitLab. + +If you change a metric, you have to consider that not all instances of GitLab are running on the newest version. Old instances will still report the old version of the metric. +Additionally, a metric's reported numbers are primarily interesting compared to previously reported numbers. +As a result, if you need to change one of the following parts of a metric, you need to add a new metric instead. It's your choice whether to keep the old metric alongside the new one or [remove it](#remove-a-metric). + +- **calculation logic**: This means any changes that can produce a different value than the previous implementation +- **YAML attributes**: The following attributes are directly used for analysis or calculation: `key_path`, `time_frame`, `value_type`, `data_source`. + +If you change the `performance_indicator_type` attribute of a metric or think your case needs an exception from the outlined rules then please notify the Customer Success Ops team (`@csops-team`), Analytics Engineers (`@gitlab-data/analytics-engineers`), and Product Analysts (`@gitlab-data/product-analysts`) teams by `@` mentioning those groups in a comment on the merge request or issue. + +You can change any other attributes without impact to the calculation or analysis. See [this video tutorial](https://youtu.be/bYf3c01KCls) for help updating metric attributes. + +Currently, the [Metrics Dictionary](https://metrics.gitlab.com/) is built automatically once a day. You can see the change in the dictionary within 24 hours when you change the metric's YAML file. + +## Remove a metric + +WARNING: +If a metric is not used in Sisense or any other system after 6 months, the +Analytics Instrumentation team marks it as inactive and assigns it to the group owner for review. + +We are working on automating this process. See [this epic](https://gitlab.com/groups/gitlab-org/-/epics/8988) for details. + +Analytics Instrumentation removes metrics from Service Ping if they are not used in any Sisense dashboard. + +For an example of the metric removal process, see this [example issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388236). + +To remove a metric: + +1. Create an issue for removing the metric if none exists yet. The issue needs to outline why the metric should be deleted. You can use this issue to document the removal process. + +1. Verify the metric is not used to calculate the conversational index. The + conversational index is a measure that reports back to self-managed instances + to inform administrators of the progress of DevOps adoption for the instance. + + You can check + [`CalculateConvIndexService`](https://gitlab.com/gitlab-org/gitlab-services/version.gitlab.com/-/blob/main/app/services/calculate_conv_index_service.rb) + to view the metrics that are used. The metrics are represented + as the keys that are passed as a field argument into the `get_value` method. + +1. Verify that removing the metric from the Service Ping payload does not cause + errors in [Version App](https://gitlab.com/gitlab-org/gitlab-services/version.gitlab.com) + when the updated payload is collected and processed. Version App collects + and persists all Service Ping reports. To verify Service Ping processing in your local development environment, follow this [guide](https://www.youtube.com/watch?v=FS5emplabRU). + Alternatively, you can modify [fixtures](https://gitlab.com/gitlab-org/gitlab-services/version.gitlab.com/-/blob/main/spec/support/usage_data_helpers.rb) + used to test the [`UsageDataController#create`](https://gitlab.com/gitlab-org/gitlab-services/version.gitlab.com/-/blob/main/spec/controllers/usage_data_controller_spec.rb) + endpoint, and assure that test suite does not fail when metric that you wish to remove is not included into test payload. + +1. Remove data from Redis + + For [Ordinary Redis](../service_ping/implement.md#ordinary-redis-counters) counters remove data stored in Redis. + + - Add a migration to remove the data from Redis for the related Redis keys. For more details, see [this MR example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82604/diffs). + +1. Create an issue in the + [GitLab Data Team project](https://gitlab.com/gitlab-data/analytics/-/issues). + Ask for confirmation that the metric is not referred to in any SiSense dashboards and + can be safely removed from Service Ping. Use this + [example issue](https://gitlab.com/gitlab-data/analytics/-/issues/15266) for guidance. + +1. Notify the Customer Success Ops team (`@csops-team`), Analytics Engineers (`@gitlab-data/analytics-engineers`), and Product Analysts (`@gitlab-data/product-analysts`) by `@` mentioning those groups in a comment in the issue from step 1 regarding the deletion of the metric. + Many Service Ping metrics are relied upon for health score and XMAU reporting and unexpected changes to those metrics could break reporting. + +1. After you verify the metric can be safely removed, + update the attributes of the metric's YAML definition: + + - Set the `status:` to `removed`. + - Set `removed_by_url:` to the URL of the MR removing the metric + - Set `milestone_removed:` to the number of the + milestone in which the metric was removed. + + Do not remove the metric's YAML definition altogether. Some self-managed + instances might not immediately update to the latest version of GitLab, and + therefore continue to report the removed metric. The Analytics Instrumentation team + requires a record of all removed metrics to identify and filter them. + + For example please take a look at this [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60149/diffs#b01f429a54843feb22265100c0e4fec1b7da1240_10_10). + +1. After you verify the metric can be safely removed, + remove the metric's instrumentation from + [`lib/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb) + or + [`ee/lib/ee/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/gitlab/usage_data.rb). + + For example please take a look at this [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60149/diffs#6335dc533bd21df26db9de90a02dd66278c2390d_167_167). + +1. Remove any other records related to the metric: + - The feature flag YAML file at [`config/feature_flags/*/*.yaml`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/feature_flags). diff --git a/doc/development/internal_analytics/service_ping/implement.md b/doc/development/internal_analytics/service_ping/implement.md index c6da26f86c2..ae8c624a47c 100644 --- a/doc/development/internal_analytics/service_ping/implement.md +++ b/doc/development/internal_analytics/service_ping/implement.md @@ -16,7 +16,7 @@ Service Ping consists of two kinds of data: To implement a new metric in Service Ping, follow these steps: 1. [Implement the required counter](#types-of-counters) -1. [Name and place the metric](metrics_dictionary.md#metric-key_path) +1. [Name and place the metric](../metrics/metrics_dictionary.md#metric-key_path) 1. [Test counters manually using your Rails console](#test-counters-manually-using-your-rails-console) 1. [Generate the SQL query](#generate-the-sql-query) 1. [Optimize queries with Database Lab](#optimize-queries-with-database-lab) @@ -654,7 +654,7 @@ For more details, see the [database review guide](../../database_review.md#prepa ## Add the metric definition -See the [Metrics Dictionary guide](metrics_dictionary.md) for more information. +See the [Metrics Dictionary guide](../metrics/metrics_dictionary.md) for more information. ## Create a merge request diff --git a/doc/development/internal_analytics/service_ping/index.md b/doc/development/internal_analytics/service_ping/index.md index f532bb1ac31..afc3a27e097 100644 --- a/doc/development/internal_analytics/service_ping/index.md +++ b/doc/development/internal_analytics/service_ping/index.md @@ -31,7 +31,7 @@ We use the following terminology to describe the Service Ping components: - **Service Ping**: the process that collects and generates a JSON payload. - **Service Data**: the contents of the Service Ping JSON payload. This includes metrics. - **Metrics**: primarily made up of row counts for different tables in an instance's database. Each - metric has a corresponding [metric definition](metrics_dictionary.md#metrics-definition-and-validation) + metric has a corresponding [metric definition](../metrics/metrics_dictionary.md#metrics-definition-and-validation) in a YAML file. - **MAU**: monthly active users. - **WAU**: weekly active users. diff --git a/doc/development/internal_analytics/service_ping/metrics_dictionary.md b/doc/development/internal_analytics/service_ping/metrics_dictionary.md index e677118fff6..9adcaaf431d 100644 --- a/doc/development/internal_analytics/service_ping/metrics_dictionary.md +++ b/doc/development/internal_analytics/service_ping/metrics_dictionary.md @@ -1,230 +1,11 @@ --- -stage: Analyze -group: Analytics Instrumentation -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../metrics/metrics_dictionary.md' +remove_date: '2023-09-25' --- -# Metrics Dictionary Guide +This document was moved to [another location](../metrics/metrics_dictionary.md). -[Service Ping](index.md) metrics are defined in individual YAML files definitions from which the -[Metrics Dictionary](https://metrics.gitlab.com/) is built. Currently, the metrics dictionary is built automatically once a day. When a change to a metric is made in a YAML file, you can see the change in the dictionary within 24 hours. -This guide describes the dictionary and how it's implemented. - -## Metrics Definition and validation - -We are using [JSON Schema](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/schema.json) to validate the metrics definition. - -This process is meant to ensure consistent and valid metrics defined for Service Ping. All metrics *must*: - -- Comply with the defined [JSON schema](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/schema.json). -- Have a unique `key_path` . -- Have an owner. - -All metrics are stored in YAML files: - -- [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics) - -WARNING: -Only metrics with a metric definition YAML and whose status is not `removed` are added to the Service Ping JSON payload. - -Each metric is defined in a separate YAML file consisting of a number of fields: - -| Field | Required | Additional information | -|---------------------|----------|----------------------------------------------------------------| -| `key_path` | yes | JSON key path for the metric, location in Service Ping payload. | -| `description` | yes | | -| `product_section` | yes | The [section](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/data/sections.yml). | -| `product_stage` | yes | The [stage](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) for the metric. | -| `product_group` | yes | The [group](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) that owns the metric. | -| `value_type` | yes | `string`; one of [`string`, `number`, `boolean`, `object`](https://json-schema.org/understanding-json-schema/reference/type.html). | -| `status` | yes | `string`; [status](#metric-statuses) of the metric, may be set to `active`, `removed`, `broken`. | -| `time_frame` | yes | `string`; may be set to a value like `7d`, `28d`, `all`, `none`. | -| `data_source` | yes | `string`; may be set to a value like `database`, `redis`, `redis_hll`, `prometheus`, `system`, `license`, `internal_events`. | -| `data_category` | yes | `string`; [categories](#data-category) of the metric, may be set to `operational`, `optional`, `subscription`, `standard`. The default value is `optional`.| -| `instrumentation_class` | yes | `string`; [the class that implements the metric](metrics_instrumentation.md). | -| `distribution` | yes | `array`; may be set to one of `ce, ee` or `ee`. The [distribution](https://about.gitlab.com/handbook/marketing/brand-and-product-marketing/product-and-solution-marketing/tiers/#definitions) where the tracked feature is available. | -| `performance_indicator_type` | no | `array`; may be set to one of [`gmau`, `smau`, `paid_gmau`, `umau` or `customer_health_score`](https://about.gitlab.com/handbook/business-technology/data-team/data-catalog/xmau-analysis/). | -| `tier` | yes | `array`; may contain one or a combination of `free`, `premium` or `ultimate`. The [tier](https://about.gitlab.com/handbook/marketing/brand-and-product-marketing/product-and-solution-marketing/tiers/#definitions) where the tracked feature is available. This should be verbose and contain all tiers where a metric is available. | -| `milestone` | yes | The milestone when the metric is introduced and when it's available to self-managed instances with the official GitLab release. | -| `milestone_removed` | no | The milestone when the metric is removed. Required for removed metrics. | -| `introduced_by_url` | no | The URL to the merge request that introduced the metric to be available for self-managed instances. | -| `removed_by_url` | no | The URL to the merge request that removed the metric. Required for removed metrics. | -| `repair_issue_url` | no | The URL of the issue that was created to repair a metric with a `broken` status. | -| `options` | no | `object`: options information needed to calculate the metric value. | -| `skip_validation` | no | This should **not** be set. [Used for imported metrics until we review, update and make them valid](https://gitlab.com/groups/gitlab-org/-/epics/5425). | - -### Metric `key_path` - -The `key_path` of the metric is the location in the JSON Service Ping payload. - -The `key_path` could be composed from multiple parts separated by `.` and it must be unique. - -We recommend to add the metric in one of the top-level keys: - -- `settings`: for settings related metrics. -- `counts_weekly`: for counters that have data for the most recent 7 days. -- `counts_monthly`: for counters that have data for the most recent 28 days. -- `counts`: for counters that have data for all time. - -NOTE: -We can't control what the metric's `key_path` is, because some of them are generated dynamically in `usage_data.rb`. -For example, see [Redis HLL metrics](implement.md#redis-hll-counters). - -### Metric statuses - -Metric definitions can have one of the following statuses: - -- `active`: Metric is used and reports data. -- `broken`: Metric reports broken data (for example, -1 fallback), or does not report data at all. A metric marked as `broken` must also have the `repair_issue_url` attribute. -- `removed`: Metric was removed, but it may appear in Service Ping payloads sent from instances running on older versions of GitLab. - -### Metric `value_type` - -Metric definitions can have one of the following values for `value_type`: - -- `boolean` -- `number` -- `string` -- `object`: A metric with `value_type: object` must have `value_json_schema` with a link to the JSON schema for the object. -In general, we avoid complex objects and prefer one of the `boolean`, `number`, or `string` value types. -An example of a metric that uses `value_type: object` is `topology` (`/config/metrics/settings/20210323120839_topology.yml`), -which has a related schema in `/config/metrics/objects_schemas/topology_schema.json`. - -### Metric `time_frame` - -A metric's time frame is calculated based on the `time_frame` field and the `data_source` of the metric. - -| data_source | time_frame | Description | -|------------------------|------------|-------------------------------------------------| -| any | `none` | A type of data that's not tracked over time, such as settings and configuration information | -| `database` | `all` | The whole time the metric has been active (all-time interval) | -| `database` | `7d` | 9 days ago to 2 days ago | -| `database` | `28d` | 30 days ago to 2 days ago | -| `redis` | `all` | The whole time the metric has been active (all-time interval) | -| `redis_hll` | `7d` | Most recent complete week | -| `redis_hll` | `28d` | Most recent 4 complete weeks | - -### Data category - -We use the following categories to classify a metric: - -- `operational`: Required data for operational purposes. -- `optional`: Default value for a metric. Data that is optional to collect. This can be [enabled or disabled](../../../administration/settings/usage_statistics.md#enable-or-disable-usage-statistics) in the Admin Area. -- `subscription`: Data related to licensing. -- `standard`: Standard set of identifiers that are included when collecting data. - -An aggregate metric is a metric that is the sum of two or more child metrics. Service Ping uses the data category of -the aggregate metric to determine whether or not the data is included in the reported Service Ping payload. - -### Example YAML metric definition - -The linked [`uuid`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/license/uuid.yml) -YAML file includes an example metric definition, where the `uuid` metric is the GitLab -instance unique identifier. - -```yaml -key_path: uuid -description: GitLab instance unique identifier -product_section: analytics -product_stage: analytics -product_group: analytics_instrumentation -value_type: string -status: active -milestone: 9.1 -instrumentation_class: UuidMetric -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1521 -time_frame: none -data_source: database -distribution: -- ce -- ee -tier: -- free -- premium -- ultimate -``` - -### Create a new metric definition - -The GitLab codebase provides a dedicated [generator](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/generators/gitlab/usage_metric_definition_generator.rb) to create new metric definitions. - -For uniqueness, the generated files include a timestamp prefix in ISO 8601 format. - -The generator takes a list of key paths and 3 options as arguments. It creates metric YAML definitions in the corresponding location: - -- `--ee`, `--no-ee` Indicates if metric is for EE. -- `--dir=DIR` Indicates the metric directory. It must be one of: `counts_7d`, `7d`, `counts_28d`, `28d`, `counts_all`, `all`, `settings`, `license`. -- `--class_name=CLASS_NAME` Indicates the instrumentation class. For example `UsersCreatingIssuesMetric`, `UuidMetric` - -**Single metric example** - -```shell -bundle exec rails generate gitlab:usage_metric_definition counts.issues --dir=7d --class_name=CountIssues -// Creates 1 file -// create config/metrics/counts_7d/issues.yml -``` - -**Multiple metrics example** - -```shell -bundle exec rails generate gitlab:usage_metric_definition counts.issues counts.users --dir=7d --class_name=CountUsersCreatingIssues -// Creates 2 files -// create config/metrics/counts_7d/issues.yml -// create config/metrics/counts_7d/users.yml -``` - -NOTE: -To create a metric definition used in EE, add the `--ee` flag. - -```shell -bundle exec rails generate gitlab:usage_metric_definition counts.issues --ee --dir=7d --class_name=CountUsersCreatingIssues -// Creates 1 file -// create ee/config/metrics/counts_7d/issues.yml -``` - -### Metrics added dynamic to Service Ping payload - -The [Redis HLL metrics](implement.md#known-events-are-added-automatically-in-service-data-payload) are added automatically to Service Ping payload. - -A YAML metric definition is required for each metric. A dedicated generator is provided to create metric definitions for Redis HLL events. - -The generator takes `category` and `events` arguments, as the root key is `redis_hll_counters`, and creates two metric definitions for each of the events (for weekly and monthly time frames): - -**Single metric example** - -```shell -bundle exec rails generate gitlab:usage_metric_definition:redis_hll issues count_users_closing_issues -// Creates 2 files -// create config/metrics/counts_7d/count_users_closing_issues_weekly.yml -// create config/metrics/counts_28d/count_users_closing_issues_monthly.yml -``` - -**Multiple metrics example** - -```shell -bundle exec rails generate gitlab:usage_metric_definition:redis_hll issues count_users_closing_issues count_users_reopening_issues -// Creates 4 files -// create config/metrics/counts_7d/count_users_closing_issues_weekly.yml -// create config/metrics/counts_28d/count_users_closing_issues_monthly.yml -// create config/metrics/counts_7d/count_users_reopening_issues_weekly.yml -// create config/metrics/counts_28d/count_users_reopening_issues_monthly.yml -``` - -To create a metric definition used in EE, add the `--ee` flag. - -```shell -bundle exec rails generate gitlab:usage_metric_definition:redis_hll issues users_closing_issues --ee -// Creates 2 files -// create config/metrics/counts_7d/i_closed_weekly.yml -// create config/metrics/counts_28d/i_closed_monthly.yml -``` - -## Metrics Dictionary - -[Metrics Dictionary is a separate application](https://gitlab.com/gitlab-org/analytics-section/analytics-instrumentation/metric-dictionary). - -All metrics available in Service Ping are in the [Metrics Dictionary](https://metrics.gitlab.com/). - -### Copy query to clipboard - -To check if a metric has data in Sisense, use the copy query to clipboard feature. This copies a query that's ready to use in Sisense. The query gets the last five service ping data for GitLab.com for a given metric. For information about how to check if a Service Ping metric has data in Sisense, see this [demo](https://www.youtube.com/watch?v=n4o65ivta48). + + + + diff --git a/doc/development/internal_analytics/service_ping/metrics_instrumentation.md b/doc/development/internal_analytics/service_ping/metrics_instrumentation.md index ada7cc566a1..35215700931 100644 --- a/doc/development/internal_analytics/service_ping/metrics_instrumentation.md +++ b/doc/development/internal_analytics/service_ping/metrics_instrumentation.md @@ -489,7 +489,7 @@ This guide describes how to migrate a Service Ping metric from [`lib/gitlab/usag - Add tests for the individual metric [`spec/lib/gitlab/usage/metrics/instrumentations/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/usage/metrics/instrumentations). - Add tests for Service Ping. -1. [Generate the metric definition file](metrics_dictionary.md#create-a-new-metric-definition). +1. [Generate the metric definition file](../metrics/metrics_dictionary.md#create-a-new-metric-definition). 1. Remove the code from [`lib/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb) or [`ee/lib/ee/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/gitlab/usage_data.rb). diff --git a/doc/development/internal_analytics/service_ping/metrics_lifecycle.md b/doc/development/internal_analytics/service_ping/metrics_lifecycle.md index 4980a8cf63d..126bc3988ed 100644 --- a/doc/development/internal_analytics/service_ping/metrics_lifecycle.md +++ b/doc/development/internal_analytics/service_ping/metrics_lifecycle.md @@ -1,105 +1,11 @@ --- -stage: Analyze -group: Analytics Instrumentation -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../metrics/metrics_lifecycle.md' +remove_date: '2023-09-25' --- -# Service Ping metric lifecycle +This document was moved to [another location](../metrics/metrics_lifecycle.md). -The following guidelines explain the steps to follow at each stage of a metric's lifecycle. - -## Add a new metric - -Follow the [Implement Service Ping](implement.md) guide. - -## Change an existing metric - -WARNING: -We want to **PREVENT** changes to the calculation logic or important attributes on any metric as this invalidates comparisons of the same metric across different versions of GitLab. - -If you change a metric, you have to consider that not all instances of GitLab are running on the newest version. Old instances will still report the old version of the metric. -Additionally, a metric's reported numbers are primarily interesting compared to previously reported numbers. -As a result, if you need to change one of the following parts of a metric, you need to add a new metric instead. It's your choice whether to keep the old metric alongside the new one or [remove it](#remove-a-metric). - -- **calculation logic**: This means any changes that can produce a different value than the previous implementation -- **YAML attributes**: The following attributes are directly used for analysis or calculation: `key_path`, `time_frame`, `value_type`, `data_source`. - -If you change the `performance_indicator_type` attribute of a metric or think your case needs an exception from the outlined rules then please notify the Customer Success Ops team (`@csops-team`), Analytics Engineers (`@gitlab-data/analytics-engineers`), and Product Analysts (`@gitlab-data/product-analysts`) teams by `@` mentioning those groups in a comment on the merge request or issue. - -You can change any other attributes without impact to the calculation or analysis. See [this video tutorial](https://youtu.be/bYf3c01KCls) for help updating metric attributes. - -Currently, the [Metrics Dictionary](https://metrics.gitlab.com/) is built automatically once a day. You can see the change in the dictionary within 24 hours when you change the metric's YAML file. - -## Remove a metric - -WARNING: -If a metric is not used in Sisense or any other system after 6 months, the -Analytics Instrumentation team marks it as inactive and assigns it to the group owner for review. - -We are working on automating this process. See [this epic](https://gitlab.com/groups/gitlab-org/-/epics/8988) for details. - -Analytics Instrumentation removes metrics from Service Ping if they are not used in any Sisense dashboard. - -For an example of the metric removal process, see this [example issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388236). - -To remove a metric: - -1. Create an issue for removing the metric if none exists yet. The issue needs to outline why the metric should be deleted. You can use this issue to document the removal process. - -1. Verify the metric is not used to calculate the conversational index. The - conversational index is a measure that reports back to self-managed instances - to inform administrators of the progress of DevOps adoption for the instance. - - You can check - [`CalculateConvIndexService`](https://gitlab.com/gitlab-org/gitlab-services/version.gitlab.com/-/blob/main/app/services/calculate_conv_index_service.rb) - to view the metrics that are used. The metrics are represented - as the keys that are passed as a field argument into the `get_value` method. - -1. Verify that removing the metric from the Service Ping payload does not cause - errors in [Version App](https://gitlab.com/gitlab-org/gitlab-services/version.gitlab.com) - when the updated payload is collected and processed. Version App collects - and persists all Service Ping reports. To verify Service Ping processing in your local development environment, follow this [guide](https://www.youtube.com/watch?v=FS5emplabRU). - Alternatively, you can modify [fixtures](https://gitlab.com/gitlab-org/gitlab-services/version.gitlab.com/-/blob/main/spec/support/usage_data_helpers.rb) - used to test the [`UsageDataController#create`](https://gitlab.com/gitlab-org/gitlab-services/version.gitlab.com/-/blob/main/spec/controllers/usage_data_controller_spec.rb) - endpoint, and assure that test suite does not fail when metric that you wish to remove is not included into test payload. - -1. Remove data from Redis - - For [Ordinary Redis](implement.md#ordinary-redis-counters) counters remove data stored in Redis. - - - Add a migration to remove the data from Redis for the related Redis keys. For more details, see [this MR example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82604/diffs). - -1. Create an issue in the - [GitLab Data Team project](https://gitlab.com/gitlab-data/analytics/-/issues). - Ask for confirmation that the metric is not referred to in any SiSense dashboards and - can be safely removed from Service Ping. Use this - [example issue](https://gitlab.com/gitlab-data/analytics/-/issues/15266) for guidance. - -1. Notify the Customer Success Ops team (`@csops-team`), Analytics Engineers (`@gitlab-data/analytics-engineers`), and Product Analysts (`@gitlab-data/product-analysts`) by `@` mentioning those groups in a comment in the issue from step 1 regarding the deletion of the metric. - Many Service Ping metrics are relied upon for health score and XMAU reporting and unexpected changes to those metrics could break reporting. - -1. After you verify the metric can be safely removed, - update the attributes of the metric's YAML definition: - - - Set the `status:` to `removed`. - - Set `removed_by_url:` to the URL of the MR removing the metric - - Set `milestone_removed:` to the number of the - milestone in which the metric was removed. - - Do not remove the metric's YAML definition altogether. Some self-managed - instances might not immediately update to the latest version of GitLab, and - therefore continue to report the removed metric. The Analytics Instrumentation team - requires a record of all removed metrics to identify and filter them. - - For example please take a look at this [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60149/diffs#b01f429a54843feb22265100c0e4fec1b7da1240_10_10). - -1. After you verify the metric can be safely removed, - remove the metric's instrumentation from - [`lib/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb) - or - [`ee/lib/ee/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/gitlab/usage_data.rb). - - For example please take a look at this [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60149/diffs#6335dc533bd21df26db9de90a02dd66278c2390d_167_167). - -1. Remove any other records related to the metric: - - The feature flag YAML file at [`config/feature_flags/*/*.yaml`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/feature_flags). + + + + diff --git a/doc/development/internal_analytics/service_ping/performance_indicator_metrics.md b/doc/development/internal_analytics/service_ping/performance_indicator_metrics.md index 63177f093e2..7388415f384 100644 --- a/doc/development/internal_analytics/service_ping/performance_indicator_metrics.md +++ b/doc/development/internal_analytics/service_ping/performance_indicator_metrics.md @@ -1,16 +1,11 @@ --- -stage: Analyze -group: Analytics Instrumentation -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../metrics/metrics_dictionary.md#performance-indicator-metrics' +remove_date: '2023-09-25' --- -# Performance Indicator Metrics guide +This document was moved to [another location](../metrics/metrics_lifecycle.md). -This guide describes how to use metrics definitions to define [performance indicator](https://about.gitlab.com/handbook/product/analytics-instrumentation-guide/#implementing-product-performance-indicators) metrics. - -To use a metric definition to manage a performance indicator: - -1. Create a merge request that includes related changes. -1. Use labels `~"analytics instrumentation"`, `"~Data Warehouse::Impact Check"`. -1. Update the metric definition `performance_indicator_type` [field](metrics_dictionary.md#metrics-definition-and-validation). -1. Create an issue in GitLab Product Data Insights project with the [PI Chart Help template](https://gitlab.com/gitlab-data/product-analytics/-/issues/new?issuable_template=PI%20Chart%20Help) to have the new metric visualized. + + + + diff --git a/doc/development/internal_analytics/service_ping/review_guidelines.md b/doc/development/internal_analytics/service_ping/review_guidelines.md index c816c905097..9a324df2513 100644 --- a/doc/development/internal_analytics/service_ping/review_guidelines.md +++ b/doc/development/internal_analytics/service_ping/review_guidelines.md @@ -51,7 +51,7 @@ are regular backend changes. #### The Analytics Instrumentation **reviewer** should - Perform a first-pass review on the merge request and suggest improvements to the author. -- Check the [metric's location](metrics_dictionary.md#metric-key_path) in +- Check the [metric's location](../metrics/metrics_dictionary.md#metric-key_path) in the Service Ping JSON payload. - Add the `~database` label and ask for a [database review](../../database_review.md) for metrics that are based on Database. diff --git a/doc/development/internal_analytics/snowplow/index.md b/doc/development/internal_analytics/snowplow/index.md index 17d3f3f2cfc..dc2b7c18ddf 100644 --- a/doc/development/internal_analytics/snowplow/index.md +++ b/doc/development/internal_analytics/snowplow/index.md @@ -91,7 +91,7 @@ Each event provides attributes that describe the event. | --------- | ------- | -------- | ----------- | | category | text | true | The page or backend section of the application. Unless infeasible, use the Rails page attribute by default in the frontend, and namespace + class name on the backend, for example, `Notes::CreateService`. | | action | text | true | The action the user takes, or aspect that's being instrumented. The first word must describe the action or aspect. For example, clicks must be `click`, activations must be `activate`, creations must be `create`. Use underscores to describe what was acted on. For example, activating a form field is `activate_form_input`, an interface action like clicking on a dropdown list is `click_dropdown`, a behavior like creating a project record from the backend is `create_project`. | -| label | text | false | The specific element or object to act on. This can be one of the following: the label of the element, for example, a tab labeled 'Create from template' for `create_from_template`; a unique identifier if no text is available, for example, `groups_dropdown_close` for closing the Groups dropdown list; or the name or title attribute of a record being created. For Service Ping metrics adapted to Snowplow events, this should be the full metric [key path](../service_ping/metrics_dictionary.md#metric-key_path) taken from its definition file. | +| label | text | false | The specific element or object to act on. This can be one of the following: the label of the element, for example, a tab labeled 'Create from template' for `create_from_template`; a unique identifier if no text is available, for example, `groups_dropdown_close` for closing the Groups dropdown list; or the name or title attribute of a record being created. For Service Ping metrics adapted to Snowplow events, this should be the full metric [key path](../metrics/metrics_dictionary.md#metric-key_path) taken from its definition file. | | property | text | false | Any additional property of the element, or object being acted on. For Service Ping metrics adapted to Snowplow events, this should be additional information or context that can help analyze the event. For example, in the case of `usage_activity_by_stage_monthly.create.merge_requests_users`, there are four different possible merge request actions: "create", "merge", "comment", and "close". Each of these would be a possible property value. | | value | decimal | false | Describes a numeric value (decimal) directly related to the event. This could be the value of an input. For example, `10` when clicking `internal` visibility. | | context | vector | false | Additional data in the form of a [self-describing JSON](https://docs.snowplow.io/docs/pipeline-components-and-applications/iglu/common-architecture/self-describing-json-schemas/) to describe the event if the attributes are not sufficient. Each context must have its schema defined to assure data integrity. Refer to the list of GitLab-defined contexts for more details. | diff --git a/doc/user/application_security/dast/checks/611.1.md b/doc/user/application_security/dast/checks/611.1.md new file mode 100644 index 00000000000..49ef449f8b0 --- /dev/null +++ b/doc/user/application_security/dast/checks/611.1.md @@ -0,0 +1,31 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# External XML Entity Injection (XXE) + +## Description + +It is possible to cause the application's XML parser to include external resources. +This can include files or in some circumstances initiate requests to third party +servers. + +## Remediation + +Consult the documentation for the XML Parser used by the target application for security +guidelines and hardening steps. It is recommended that all XML parsers disable external +entity resolution and XML `xinclude` features. Most XML parsers based on `libxml` can also be +configured to disable network access. + +## Details + +| ID | Aggregated | CWE | Type | Risk | +|:---|:--------|:--------|:--------|:--------| +| 611.1 | false | 611 | Active | high | + +## Links + +- [OWASP](https://owasp.org/www-community/vulnerabilities/XML_External_Entity_(XXE)_Processing) +- [CWE](https://cwe.mitre.org/data/definitions/611.html) diff --git a/doc/user/application_security/dast/checks/index.md b/doc/user/application_security/dast/checks/index.md index 035f4a4b486..d23f1ccbd8b 100644 --- a/doc/user/application_security/dast/checks/index.md +++ b/doc/user/application_security/dast/checks/index.md @@ -169,3 +169,4 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne |:---|:------|:---------|:-----| | [113.1](113.1.md) | Improper Neutralization of CRLF Sequences in HTTP Headers | High | Active | | [22.1](22.1.md) | Improper limitation of a pathname to a restricted directory (Path traversal) | High | Active | +| [611.1](611.1.md) | External XML Entity Injection (XXE) | High | Active | diff --git a/doc/user/application_security/terminology/index.md b/doc/user/application_security/terminology/index.md index 5c2dbd8d728..0f0a61a2b02 100644 --- a/doc/user/application_security/terminology/index.md +++ b/doc/user/application_security/terminology/index.md @@ -35,6 +35,11 @@ The different places in an application that are vulnerable to attack. Secure pro search the attack surface during scans. Each product defines the attack surface differently. For example, SAST uses files and line numbers, and DAST uses URLs. +## Component + +A software component that makes up a portion of a software project. Examples include libraries, drivers, data, and +[many more](https://cyclonedx.org/docs/1.5/json/#components_items_type). + ## Corpus The set of meaningful test cases that are generated while the fuzzer is running. Each meaningful @@ -105,6 +110,12 @@ under development and tracked in issue [267588](https://gitlab.com/gitlab-org/gi A legitimate finding that a particular customer doesn't care about. +## Known affected component + +A component that matches the requirements for a vulnerability to be exploitable. For example, +`packageA@1.0.3` matches the name, package type, and one of the affected versions or version +ranges of `FAKECVE-2023-0001`. + ## Location fingerprint A finding's location fingerprint is a text value that's unique for each location on the attack @@ -217,6 +228,14 @@ table.package-managers-and-types ul { A page that displays findings discovered in the associated CI pipeline. +## Possibly affected component + +A software component that is possibly affected by vulnerability. For example, when scanning a +project for known vulnerabilities, components are first evaluated to see if they match the name +and [package type](https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst). +During this stage, they're _possibly_ affected by the vulnerability, and are only [known to be affected](#known-affected-component) +after it's confirmed that they fall in the affected version range. + ## Post-filter Post-filters help reduce noise in the scanner results and automate manual tasks. You can specify criteria that updates diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 695193f878a..9d789c27d1f 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -83,23 +83,11 @@ NPM_TOKEN=your_token npm publish Your package should now publish to the Package Registry. -## Publishing a package via a CI/CD pipeline +## Publishing a package by using a CI/CD pipeline -### Authenticating via the `.npmrc` +When publishing by using a CI/CD pipeline, you can use the [predefined variables](../../../ci/variables/predefined_variables.md) `${CI_PROJECT_ID}` and `${CI_JOB_TOKEN}` to authenticate with your project's Package Registry. We use these variables to create a `.npmrc` file [for authentication](#authenticating-via-the-npmrc) during execution of your CI/CD job. -Create or edit the `.npmrc` file in the same directory as your `package.json` in a GitLab project. Include the following lines in the `.npmrc` file: - -```shell -@scope:registry=https://your_domain_name/api/v4/projects/your_project_id/packages/npm/ -//your_domain_name/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN} -``` - -- Replace `@scope` with the [root level group](#naming-convention) of the project you're publishing to the package to. -- The `${CI_PROJECT_ID}` and `${CI_JOB_TOKEN}` are [predefined variables](../../../ci/variables/predefined_variables.md) that are available in the pipeline and do not need to be replaced. - -### Publishing a package via a CI/CD pipeline - -In the GitLab project that houses your `.npmrc` and `package.json`, edit or create a `.gitlab-ci.yml` file. For example: +In the GitLab project containing your `package.json`, edit or create a `.gitlab-ci.yml` file. For example: ```yaml image: node:latest @@ -107,14 +95,17 @@ image: node:latest stages: - deploy -deploy: +publish-npm: stage: deploy script: - - echo "//${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}">.npmrc + - echo "@scope:registry=https://${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/" > .npmrc + - echo "//${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}" >> .npmrc - npm publish ``` -Your package should now publish to the Package Registry when the pipeline runs. +- Replace `@scope` with the [scope](https://docs.npmjs.com/cli/v10/using-npm/scope) of the package that is being published. + +Your package is published to the Package Registry when the `publish-npm` job in your pipeline runs. ## Install a package diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index 59184b811d4..e9c019deefa 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -89,7 +89,7 @@ For more information about using the GitLab Package Registry with CI/CD, see: - [Conan](../conan_repository/index.md#publish-a-conan-package-by-using-cicd) - [Generic](../generic_packages/index.md#publish-a-generic-package-by-using-cicd) - [Maven](../maven_repository/index.md#create-maven-packages-with-gitlab-cicd) -- [npm](../npm_registry/index.md#publishing-a-package-via-a-cicd-pipeline) +- [npm](../npm_registry/index.md#publishing-a-package-by-using-a-cicd-pipeline) - [NuGet](../nuget_repository/index.md#publish-a-nuget-package-by-using-cicd) - [PyPI](../pypi_repository/index.md#authenticate-with-a-ci-job-token) - [RubyGems](../rubygems_registry/index.md#authenticate-with-a-ci-job-token) diff --git a/doc/user/project/repository/branches/index.md b/doc/user/project/repository/branches/index.md index f33d443cd7f..ef2e4cb0fef 100644 --- a/doc/user/project/repository/branches/index.md +++ b/doc/user/project/repository/branches/index.md @@ -286,6 +286,51 @@ To do this: 1. Select **Delete merged branches**. 1. In the dialog, enter the word `delete` to confirm, then select **Delete merged branches**. +## Configure rules for target branches **(PREMIUM ALL)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/127115) in GitLab 16.4 [with a flag](../../../../administration/feature_flags.md) named `target_branch_rules_flag`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, an administrator can [enable the feature flag](../../../../administration/feature_flags.md) named `target_branch_rules_flag`. +On GitLab.com, this feature is not available. + +Some projects use multiple long-term branches for development, like `develop` and `qa`. +In these projects, you might want to keep `main` as the default branch, but expect +merge requests to target `develop` or `qa` instead. Target branch rules help ensure +merge requests target the appropriate development branch for your project. + +When you create a merge request, the rule checks the name of the branch. If the +branch name matches the rule, the merge request targets the branch you specify +in the rule. If the branch name does not match, the merge request targets the +default branch of the project. + +Prerequisites: + +- You must have at least the Maintainer role. + +To create a target branch rule: + +1. On the left sidebar, select **Search or go to** and find your project. +1. Select **Settings > Merge requests**. +1. Select **Add target branch rule**. +1. For **Rule name**, provide a string or wild card to compare against branch names. +1. Select the **Target branch** to use when the branch name matches the **Rule name**. +1. Select **Save**. + +## Delete a target branch rule + +When you remove a target branch rule, existing merge requests remain unchanged. + +Prerequisites: + +- You must have at least the Maintainer role. + +To do this: + +1. On the left sidebar, select **Search or go to** and find your project. +1. Select **Settings > Merge requests**. +1. Select **Delete** on the rule you want to delete. + ## Related topics - [Protected branches](../../protected_branches.md) diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index c8520993b8e..91994c2fa95 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -3,8 +3,7 @@ # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab - # rubocop:disable Rails/ApplicationController - class BaseDoorkeeperController < ActionController::Base + class BaseDoorkeeperController < BaseActionController include Gitlab::Allowable include EnforcesTwoFactorAuthentication include SessionsHelper @@ -13,5 +12,4 @@ module Gitlab helper_method :can? end - # rubocop:enable Rails/ApplicationController end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index d5e80053772..3a389d3363f 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -6,8 +6,7 @@ module Gitlab module RequestForgeryProtection - # rubocop:disable Rails/ApplicationController - class Controller < ActionController::Base + class Controller < BaseActionController protect_from_forgery with: :exception, prepend: true def initialize @@ -40,6 +39,5 @@ module Gitlab rescue ActionController::InvalidAuthenticityToken false end - # rubocop:enable Rails/ApplicationController end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 9d742baa45c..f0d15618351 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4802,9 +4802,6 @@ msgstr "" msgid "Allow owners to manage default branch protection per group." msgstr "" -msgid "Allow owners to manually add users outside of LDAP" -msgstr "" - msgid "Allow password authentication for Git over HTTP(S)" msgstr "" @@ -13719,9 +13716,6 @@ msgstr "" msgid "Could not save configuration. Please refresh the page, or try again later." msgstr "" -msgid "Could not update the LDAP settings" -msgstr "" - msgid "Could not update wiki page" msgstr "" @@ -27302,15 +27296,9 @@ msgstr "" msgid "LDAP Synchronization" msgstr "" -msgid "LDAP group settings" -msgstr "" - msgid "LDAP settings" msgstr "" -msgid "LDAP settings updated" -msgstr "" - msgid "LDAP sync in progress. This could take a few minutes. Refresh the page to see the changes." msgstr "" diff --git a/spec/fixtures/lib/generators/gitlab/usage_metric_definition_generator/sample_metric.yml b/spec/fixtures/lib/generators/gitlab/usage_metric_definition_generator/sample_metric.yml index 29dd2ce0e84..64cf9371858 100644 --- a/spec/fixtures/lib/generators/gitlab/usage_metric_definition_generator/sample_metric.yml +++ b/spec/fixtures/lib/generators/gitlab/usage_metric_definition_generator/sample_metric.yml @@ -1,5 +1,5 @@ --- -# See Usage Ping metrics dictionary docs https://docs.gitlab.com/ee/development/usage_ping/metrics_dictionary.html +# See Usage Ping metrics dictionary docs https://docs.gitlab.com/ee/development/internal_analytics/metrics/metrics_dictionary.html key_path: counts_weekly.test_metric description: product_section: diff --git a/spec/frontend/clusters_list/components/clusters_spec.js b/spec/frontend/clusters_list/components/clusters_spec.js index 207bfddcb4f..71d3bf415d6 100644 --- a/spec/frontend/clusters_list/components/clusters_spec.js +++ b/spec/frontend/clusters_list/components/clusters_spec.js @@ -8,6 +8,11 @@ import ClustersEmptyState from '~/clusters_list/components/clusters_empty_state. import ClusterStore from '~/clusters_list/store'; import axios from '~/lib/utils/axios_utils'; import { HTTP_STATUS_OK } from '~/lib/utils/http_status'; +import { + SET_LOADING_NODES, + SET_CLUSTERS_DATA, + SET_LOADING_CLUSTERS, +} from '~/clusters_list/store/mutation_types'; import { apiData } from '../mock_data'; describe('Clusters', () => { @@ -82,7 +87,7 @@ describe('Clusters', () => { describe('clusters table', () => { describe('when data is loading', () => { beforeEach(() => { - wrapper.vm.$store.state.loadingClusters = true; + store.commit(SET_LOADING_CLUSTERS, true); }); it('displays a loader instead of the table while loading', () => { @@ -99,7 +104,12 @@ describe('Clusters', () => { describe('when there are no clusters', () => { beforeEach(() => { - wrapper.vm.$store.state.totalClusters = 0; + store.commit(SET_CLUSTERS_DATA, { + data: {}, + paginationInformation: { + total: 0, + }, + }); }); it('should render empty state', () => { expect(findEmptyState().exists()).toBe(true); @@ -175,7 +185,7 @@ describe('Clusters', () => { describe('nodes finish loading', () => { beforeEach(async () => { - wrapper.vm.$store.state.loadingNodes = false; + store.commit(SET_LOADING_NODES, false); await nextTick(); }); diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 9f9372f94cc..f415140b45d 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -753,26 +753,21 @@ RSpec.describe ProjectsHelper, feature_category: :source_code_management do describe '#show_mobile_devops_project_promo?' do using RSpec::Parameterized::TableSyntax - where(:hide_cookie, :feature_flag_enabled, :mobile_target_platform, :result) do - false | true | true | true - false | false | true | false - false | false | false | false - false | true | false | false - true | false | false | false - true | true | false | false - true | true | true | false - true | false | true | false + where(:hide_cookie, :mobile_target_platform, :result) do + false | true | true + false | false | false + true | false | false + true | true | false end with_them do before do allow(Gitlab).to receive(:com?) { gitlab_com } - Feature.enable(:mobile_devops_projects_promo, feature_flag_enabled) project.project_setting.target_platforms << 'ios' if mobile_target_platform helper.request.cookies["hide_mobile_devops_promo_#{project.id}"] = true if hide_cookie end - it 'resolves if the user can import members' do + it 'resolves if mobile devops promo banner should be displayed' do expect(helper.show_mobile_devops_project_promo?(project)).to eq result end end diff --git a/spec/requests/acme_challenges_controller_spec.rb b/spec/requests/acme_challenges_controller_spec.rb new file mode 100644 index 00000000000..f37aefed488 --- /dev/null +++ b/spec/requests/acme_challenges_controller_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe AcmeChallengesController, type: :request, feature_category: :pages do + it_behaves_like 'Base action controller' do + subject(:request) { get acme_challenge_path } + end +end diff --git a/spec/requests/application_controller_spec.rb b/spec/requests/application_controller_spec.rb new file mode 100644 index 00000000000..52fdf6bc69e --- /dev/null +++ b/spec/requests/application_controller_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ApplicationController, type: :request, feature_category: :shared do + let_it_be(:user) { create(:user) } + + before do + sign_in(user) + end + + it_behaves_like 'Base action controller' do + subject(:request) { get root_path } + end +end diff --git a/spec/requests/chaos_controller_spec.rb b/spec/requests/chaos_controller_spec.rb new file mode 100644 index 00000000000..d2ce618b041 --- /dev/null +++ b/spec/requests/chaos_controller_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ChaosController, type: :request, feature_category: :tooling do + it_behaves_like 'Base action controller' do + before do + # Stub leak_mem so we don't actually leak memory for the base action controller tests. + allow(Gitlab::Chaos).to receive(:leak_mem).with(100, 30.seconds) + end + + subject(:request) { get leakmem_chaos_path } + end +end diff --git a/spec/requests/health_controller_spec.rb b/spec/requests/health_controller_spec.rb index 639f6194af9..3ad1d8a75b4 100644 --- a/spec/requests/health_controller_spec.rb +++ b/spec/requests/health_controller_spec.rb @@ -73,7 +73,9 @@ RSpec.describe HealthController, feature_category: :database do end describe 'GET /-/readiness' do - subject { get '/-/readiness', params: params, headers: headers } + subject(:request) { get readiness_path, params: params, headers: headers } + + it_behaves_like 'Base action controller' shared_context 'endpoint responding with readiness data' do context 'when requesting instance-checks' do diff --git a/spec/requests/metrics_controller_spec.rb b/spec/requests/metrics_controller_spec.rb new file mode 100644 index 00000000000..ce96906e020 --- /dev/null +++ b/spec/requests/metrics_controller_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe MetricsController, type: :request, feature_category: :metrics do + it_behaves_like 'Base action controller' do + subject(:request) { get metrics_path } + end +end diff --git a/spec/requests/oauth/authorizations_controller_spec.rb b/spec/requests/oauth/authorizations_controller_spec.rb index 257f238d9ef..7887bf52542 100644 --- a/spec/requests/oauth/authorizations_controller_spec.rb +++ b/spec/requests/oauth/authorizations_controller_spec.rb @@ -20,6 +20,10 @@ RSpec.describe Oauth::AuthorizationsController, feature_category: :system_access end describe 'GET #new' do + it_behaves_like 'Base action controller' do + subject(:request) { get oauth_authorization_path } + end + context 'when application redirect URI has a custom scheme' do context 'when CSP is disabled' do before do diff --git a/spec/requests/registrations_controller_spec.rb b/spec/requests/registrations_controller_spec.rb index 8b857046a4d..71f2f347f0d 100644 --- a/spec/requests/registrations_controller_spec.rb +++ b/spec/requests/registrations_controller_spec.rb @@ -6,7 +6,9 @@ RSpec.describe RegistrationsController, type: :request, feature_category: :syste describe 'POST #create' do let_it_be(:user_attrs) { build_stubbed(:user).slice(:first_name, :last_name, :username, :email, :password) } - subject(:create_user) { post user_registration_path, params: { user: user_attrs } } + subject(:request) { post user_registration_path, params: { user: user_attrs } } + + it_behaves_like 'Base action controller' context 'when email confirmation is required' do before do @@ -15,7 +17,7 @@ RSpec.describe RegistrationsController, type: :request, feature_category: :syste end it 'redirects to the `users_almost_there_path`', unless: Gitlab.ee? do - create_user + request expect(response).to redirect_to(users_almost_there_path(email: user_attrs[:email])) end diff --git a/spec/requests/sessions_spec.rb b/spec/requests/sessions_spec.rb index 3428e607305..1a925969c5a 100644 --- a/spec/requests/sessions_spec.rb +++ b/spec/requests/sessions_spec.rb @@ -7,6 +7,10 @@ RSpec.describe 'Sessions', feature_category: :system_access do let(:user) { create(:user) } + it_behaves_like 'Base action controller' do + subject(:request) { get new_user_session_path } + end + context 'for authentication', :allow_forgery_protection do it 'logout does not require a csrf token' do login_as(user) diff --git a/spec/services/update_container_registry_info_service_spec.rb b/spec/services/update_container_registry_info_service_spec.rb index 416b08bd04b..59b4537a1d6 100644 --- a/spec/services/update_container_registry_info_service_spec.rb +++ b/spec/services/update_container_registry_info_service_spec.rb @@ -72,13 +72,14 @@ RSpec.describe UpdateContainerRegistryInfoService, feature_category: :container_ expect(application_settings.container_registry_vendor).to be_blank expect(application_settings.container_registry_version).to be_blank expect(application_settings.container_registry_features).to eq([]) + expect(application_settings.container_registry_db_enabled).to be_falsey end end context 'when able to detect the container registry type' do context 'when using the GitLab container registry' do it 'updates application settings accordingly' do - stub_registry_info(vendor: 'gitlab', version: '2.9.1-gitlab', features: %w[a b c]) + stub_registry_info(vendor: 'gitlab', version: '2.9.1-gitlab', features: %w[a b c], db_enabled: true) stub_supports_gitlab_api(true) subject @@ -88,12 +89,13 @@ RSpec.describe UpdateContainerRegistryInfoService, feature_category: :container_ expect(application_settings.container_registry_version).to eq('2.9.1-gitlab') expect(application_settings.container_registry_features) .to match_array(%W[a b c #{ContainerRegistry::GitlabApiClient::REGISTRY_GITLAB_V1_API_FEATURE}]) + expect(application_settings.container_registry_db_enabled).to be_truthy end end context 'when using a third-party container registry' do it 'updates application settings accordingly' do - stub_registry_info(vendor: 'other', version: nil, features: nil) + stub_registry_info(vendor: 'other', version: nil, features: nil, db_enabled: false) stub_supports_gitlab_api(false) subject @@ -102,6 +104,7 @@ RSpec.describe UpdateContainerRegistryInfoService, feature_category: :container_ expect(application_settings.container_registry_vendor).to eq('other') expect(application_settings.container_registry_version).to be_blank expect(application_settings.container_registry_features).to eq([]) + expect(application_settings.container_registry_db_enabled).to be_falsey end end end diff --git a/spec/support/shared_examples/config/metrics/every_metric_definition_shared_examples.rb b/spec/support/shared_examples/config/metrics/every_metric_definition_shared_examples.rb index f03a9302bca..53d80c64827 100644 --- a/spec/support/shared_examples/config/metrics/every_metric_definition_shared_examples.rb +++ b/spec/support/shared_examples/config/metrics/every_metric_definition_shared_examples.rb @@ -79,7 +79,7 @@ RSpec.shared_examples 'every metric definition' do end it 'is included in the Usage Ping hash structure' do - msg = "see https://docs.gitlab.com/ee/development/service_ping/metrics_dictionary.html#metrics-added-dynamic-to-service-ping-payload" + msg = "see https://docs.gitlab.com/ee/development/internal_analytics/metrics/metrics_dictionary.html#metrics-added-dynamic-to-service-ping-payload" expect(expected_metric_files_key_paths).to match_array(usage_ping_key_paths), msg end diff --git a/spec/support/shared_examples/controllers/base_action_controller_shared_examples.rb b/spec/support/shared_examples/controllers/base_action_controller_shared_examples.rb new file mode 100644 index 00000000000..5f236f25d35 --- /dev/null +++ b/spec/support/shared_examples/controllers/base_action_controller_shared_examples.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Requires `request` subject to be defined +# +# subject(:request) { get root_path } +RSpec.shared_examples 'Base action controller' do + describe 'security headers' do + describe 'Cross-Origin-Opener-Policy' do + it 'sets the header' do + request + + expect(response.headers['Cross-Origin-Opener-Policy']).to eq('same-origin') + end + + context 'when coop_header feature flag is disabled' do + it 'does not set the header' do + stub_feature_flags(coop_header: false) + + request + + expect(response.headers['Cross-Origin-Opener-Policy']).to be_nil + end + end + end + end +end diff --git a/tooling/config/CODEOWNERS.yml b/tooling/config/CODEOWNERS.yml index 07fddde056c..b7f8e5d4813 100644 --- a/tooling/config/CODEOWNERS.yml +++ b/tooling/config/CODEOWNERS.yml @@ -3,7 +3,7 @@ # And paste the contents into .gitlab/CODEOWNERS '[Authentication and Authorization]': - '@gitlab-org/manage/authentication-and-authorization/approvers': + '@gitlab-org/govern/authentication-and-authorization/approvers': allow: keywords: - 'password'