diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml index 0300eae6a7c..69917eef040 100644 --- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml @@ -84,6 +84,7 @@ name: ${QA_IMAGE} entrypoint: [""] stage: post-qa + allow_failure: true before_script: - cd qa script: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 0098ce5c887..cec2f732541 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -1648,22 +1648,27 @@ rules: - when: on_success -# The rule needs to be duplicated between `on_success` and `on_failure` -# because the jobs `needs` the previous job to complete. -# With `when: always`, and the `review-qa-*` jobs are manual, the `allure-report-qa-*` jobs -# would start running before the qa jobs have started. -# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 +# If the needed job isn't allowed to fail, we need to use `when: always` in +# order to keep the job always running after it. +# +# If the needed job is allowed to fail, we need to use both +# `when: on_success` and `when: on_failure` in order to keep +# the job always running after it. +# Not that if the needed job has `when: on_success` we can use `when: always` +# for the depending job. +# +# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76756 + +# Since `review-qa-smoke` isn't allowed to fail, we need to use `when: always` for `review-qa-smoke-report`. .review:rules:review-qa-smoke-report: rules: - - when: on_success - - when: on_failure + - when: always .review:rules:review-qa-reliable: rules: - when: on_success -# Since `review-qa-reliable ` isn't allowed to fail, we need to use `when: always` for `review-qa-reliable-report`. -# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76756#qa-job-that-run-on_success-and-not-allowed-to-fail-eg-report-qa-smoke +# Since `review-qa-reliable` isn't allowed to fail, we need to use `when: always`for `review-qa-reliable-report`. .review:rules:review-qa-reliable-report: rules: - when: always @@ -1678,17 +1683,11 @@ - when: on_success allow_failure: true -# The rule needs to be duplicated between `on_success` and `on_failure` -# because the jobs `needs` the previous job to complete. -# With `when: always`, and the `review-qa-*` jobs are manual, the `allure-report-qa-*` jobs -# would start running before the qa jobs have started. -# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 +# Since `review-qa-all` is allowed to fail (and potentially manual), we need to use `when: on_success` and `when: on_failure` for `review-qa-all-report`. .review:rules:review-qa-all-report: rules: - when: on_success - allow_failure: true - when: on_failure - allow_failure: true # Generate knapsack report on successful runs only # Reliable suite will pass most of the time so this should yield best distribution @@ -1696,13 +1695,14 @@ rules: - if: '$KNAPSACK_GENERATE_REPORT == "true"' when: on_success - allow_failure: true +# Since `review-qa-all` is allowed to fail (and potentially manual), we need to use `when: on_success` and `when: on_failure` for `knapsack-report-qa-all`. .review:rules:knapsack-report-qa-all: rules: - - if: '$KNAPSACK_GENERATE_REPORT == "true"' - when: always - allow_failure: true + - if: '$KNAPSACK_GENERATE_REPORT != "true"' + when: never + - when: on_success + - when: on_failure .review:rules:review-cleanup: rules: diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 1eb3bd2ea41..96ed18eff03 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -151,12 +151,15 @@ detect-previous-failed-tests: add-jh-folder: extends: .setup:rules:add-jh-folder - image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge + image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7 stage: prepare before_script: - - apk add --no-cache --update curl bash + - source ./scripts/utils.sh + - install_gitlab_gem script: - - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-jh/gitlab/-/archive/main-jh/gitlab-main-jh.tar.gz?path=jh" + - JH_BRANCH=$(./scripts/setup/find-jh-branch.rb) + - 'echo "JH_BRANCH: ${JH_BRANCH}"' + - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-jh/gitlab/-/archive/${JH_BRANCH}/gitlab-${JH_BRANCH}.tar.gz?path=jh" - tar -xf "jh-folder.tar.gz" - mv gitlab-main-jh-jh/jh/ ./ - cp Gemfile.lock jh/ diff --git a/.gitlab/issue_templates/Performance Indicator Metric.md b/.gitlab/issue_templates/Performance Indicator Metric.md index 294a617357c..f4d8885b119 100644 --- a/.gitlab/issue_templates/Performance Indicator Metric.md +++ b/.gitlab/issue_templates/Performance Indicator Metric.md @@ -15,7 +15,9 @@ Summary of the changes ## Tasks -[ ] [Link to metric definition]() -[ ] Create issue in GitLab Data Team project using [Product Performance Indicator template](https://gitlab.com/gitlab-data/analytics/-/issues/new?issuable_template=Product%20Performance%20Indicator%20Template) +- [ ] [Link to metric definition]() +- [ ] Create issue in GitLab Data Team project using [Product Performance Indicator template](https://gitlab.com/gitlab-data/analytics/-/issues/new?issuable_template=Product%20Performance%20Indicator%20Template) -/label ~"product intelligence" "~Data Warehouse::Impact Check" +See [Product Intelligence Guide](https://docs.gitlab.com/ee/development/service_ping/performance_indicator_metrics.html) for details + +/label ~"product intelligence" ~"Data Warehouse::Impact Check" diff --git a/app/assets/javascripts/blob/components/blob_header_filepath.vue b/app/assets/javascripts/blob/components/blob_header_filepath.vue index cb441a7e491..90d01358451 100644 --- a/app/assets/javascripts/blob/components/blob_header_filepath.vue +++ b/app/assets/javascripts/blob/components/blob_header_filepath.vue @@ -1,4 +1,5 @@ @@ -37,8 +42,6 @@ export default { > - {{ blobSize }} - + + {{ blobSize }} + + {{ __('LFS') }} diff --git a/app/assets/javascripts/repository/components/blob_content_viewer.vue b/app/assets/javascripts/repository/components/blob_content_viewer.vue index f3fa4526999..46b4c513109 100644 --- a/app/assets/javascripts/repository/components/blob_content_viewer.vue +++ b/app/assets/javascripts/repository/components/blob_content_viewer.vue @@ -105,6 +105,7 @@ export default { forkAndEditPath: '', ideForkAndEditPath: '', storedExternally: false, + externalStorage: '', canModifyBlob: false, canCurrentUserPushToBranch: false, rawPath: '', diff --git a/app/assets/javascripts/repository/queries/blob_info.query.graphql b/app/assets/javascripts/repository/queries/blob_info.query.graphql index 45d1ba80917..2754ed39988 100644 --- a/app/assets/javascripts/repository/queries/blob_info.query.graphql +++ b/app/assets/javascripts/repository/queries/blob_info.query.graphql @@ -36,6 +36,7 @@ query getBlobInfo($projectPath: ID!, $filePath: String!, $ref: String!) { canModifyBlob canCurrentUserPushToBranch storedExternally + externalStorage rawPath replacePath pipelineEditorPath diff --git a/app/assets/javascripts/security_configuration/components/training_provider_list.vue b/app/assets/javascripts/security_configuration/components/training_provider_list.vue index 509377a63e8..c574412096a 100644 --- a/app/assets/javascripts/security_configuration/components/training_provider_list.vue +++ b/app/assets/javascripts/security_configuration/components/training_provider_list.vue @@ -1,6 +1,7 @@ @@ -46,7 +80,13 @@ export default { >
- +

{{ name }}

diff --git a/app/assets/javascripts/security_configuration/graphql/configure_security_training_providers.mutation.graphql b/app/assets/javascripts/security_configuration/graphql/configure_security_training_providers.mutation.graphql new file mode 100644 index 00000000000..df8c7c9e30a --- /dev/null +++ b/app/assets/javascripts/security_configuration/graphql/configure_security_training_providers.mutation.graphql @@ -0,0 +1,8 @@ +mutation configureSecurityTrainingProviders($input: configureSecurityTrainingProvidersInput!) { + configureSecurityTrainingProviders(input: $input) @client { + securityTrainingProviders { + id + isEnabled + } + } +} diff --git a/app/assets/javascripts/security_configuration/index.js b/app/assets/javascripts/security_configuration/index.js index c86ff1a58f2..24c0585e077 100644 --- a/app/assets/javascripts/security_configuration/index.js +++ b/app/assets/javascripts/security_configuration/index.js @@ -2,38 +2,10 @@ import Vue from 'vue'; import VueApollo from 'vue-apollo'; import createDefaultClient from '~/lib/graphql'; import { parseBooleanDataAttributes } from '~/lib/utils/dom_utils'; -import { __ } from '~/locale'; import SecurityConfigurationApp from './components/app.vue'; import { securityFeatures, complianceFeatures } from './components/constants'; import { augmentFeatures } from './utils'; - -// Note: this is behind a feature flag and only a placeholder -// until the actual GraphQL fields have been added -// https://gitlab.com/gitlab-org/gi tlab/-/issues/346480 -export const tempResolvers = { - Query: { - securityTrainingProviders() { - return [ - { - __typename: 'SecurityTrainingProvider', - id: 101, - name: __('Kontra'), - description: __('Interactive developer security education.'), - url: 'https://application.security/', - isEnabled: false, - }, - { - __typename: 'SecurityTrainingProvider', - id: 102, - name: __('SecureCodeWarrior'), - description: __('Security training with guide and learning pathways.'), - url: 'https://www.securecodewarrior.com/', - isEnabled: true, - }, - ]; - }, - }, -}; +import tempResolvers from './resolver'; export const initSecurityConfiguration = (el) => { if (!el) { diff --git a/app/assets/javascripts/security_configuration/resolver.js b/app/assets/javascripts/security_configuration/resolver.js new file mode 100644 index 00000000000..93175d4a3d1 --- /dev/null +++ b/app/assets/javascripts/security_configuration/resolver.js @@ -0,0 +1,56 @@ +import produce from 'immer'; +import { __ } from '~/locale'; +import securityTrainingProvidersQuery from './graphql/security_training_providers.query.graphql'; + +// Note: this is behind a feature flag and only a placeholder +// until the actual GraphQL fields have been added +// https://gitlab.com/gitlab-org/gi tlab/-/issues/346480 +export default { + Query: { + securityTrainingProviders() { + return [ + { + __typename: 'SecurityTrainingProvider', + id: 101, + name: __('Kontra'), + description: __('Interactive developer security education.'), + url: 'https://application.security/', + isEnabled: false, + }, + { + __typename: 'SecurityTrainingProvider', + id: 102, + name: __('SecureCodeWarrior'), + description: __('Security training with guide and learning pathways.'), + url: 'https://www.securecodewarrior.com/', + isEnabled: true, + }, + ]; + }, + }, + + Mutation: { + configureSecurityTrainingProviders: ( + _, + { input: { enabledProviders, primaryProvider } }, + { cache }, + ) => { + const sourceData = cache.readQuery({ + query: securityTrainingProvidersQuery, + }); + + const data = produce(sourceData.securityTrainingProviders, (draftData) => { + /* eslint-disable no-param-reassign */ + draftData.forEach((provider) => { + provider.isPrimary = provider.id === primaryProvider; + provider.isEnabled = + provider.id === primaryProvider || enabledProviders.includes(provider.id); + }); + }); + return { + __typename: 'configureSecurityTrainingProvidersPayload', + securityTrainingProviders: data, + }; + }, + }, +}; diff --git a/app/assets/javascripts/vue_shared/issuable/list/components/issuable_bulk_edit_sidebar.vue b/app/assets/javascripts/vue_shared/issuable/list/components/issuable_bulk_edit_sidebar.vue index 5ca9e50d854..bcc889495bd 100644 --- a/app/assets/javascripts/vue_shared/issuable/list/components/issuable_bulk_edit_sidebar.vue +++ b/app/assets/javascripts/vue_shared/issuable/list/components/issuable_bulk_edit_sidebar.vue @@ -13,6 +13,7 @@ export default { if (layoutPageEl) { layoutPageEl.classList.toggle('right-sidebar-expanded', value); layoutPageEl.classList.toggle('right-sidebar-collapsed', !value); + layoutPageEl.classList.toggle('issuable-bulk-update-sidebar', !value); } }, }, diff --git a/app/graphql/types/repository/blob_type.rb b/app/graphql/types/repository/blob_type.rb index 3265c14bdca..c2159329e34 100644 --- a/app/graphql/types/repository/blob_type.rb +++ b/app/graphql/types/repository/blob_type.rb @@ -56,6 +56,9 @@ module Types field :stored_externally, GraphQL::Types::Boolean, null: true, method: :stored_externally?, description: "Whether the blob's content is stored externally (for instance, in LFS)." + field :external_storage, GraphQL::Types::String, null: true, method: :external_storage, + description: "External storage being used, if enabled (for instance, 'LFS')." + field :edit_blob_path, GraphQL::Types::String, null: true, description: 'Web path to edit the blob in the old-style editor.' diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index f8cb45dd00d..e4d1696ea93 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -179,3 +179,43 @@ Once saved, [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure NOTE: These are Omnibus GitLab settings. If an external database, such as a customer's PostgreSQL installation or Amazon RDS is being used, these values don't get set, and would have to be set externally. + +### Temporarily changing the statement timeout + +WARNING: +The following advice does not apply in case +[PgBouncer](../postgresql/pgbouncer.md) is enabled, +because the changed timeout might affect more transactions than intended. + +In some situations, it may be desirable to set a different statement timeout +without having to [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure), +which in this case would restart Puma and Sidekiq. + +For example, a backup may fail with the following errors in the output of the +[backup command](../../raketasks/backup_restore.md#back-up-gitlab) +because the statement timeout was too short: + +```plaintext +pg_dump: error: Error message from server: server closed the connection unexpectedly +``` + +You may also see errors in the [PostgreSQL logs](../logs.md#postgresql-logs): + +```plaintext +canceling statement due to statement timeout +``` + +To temporarily change the statement timeout: + +1. Open `/var/opt/gitlab/gitlab-rails/etc/database.yml` in an editor +1. Set the value of `statement_timeout` to `0`, which sets an unlimited statement timeout. +1. [Confirm in a new Rails console session](../operations/rails_console.md#using-the-rails-runner) + that this value is used: + + ```shell + sudo gitlab-rails runner "ActiveRecord::Base.connection_config[:variables]" + ``` + +1. Perform the action for which you need a different timeout + (for example the backup or the Rails command). +1. Revert the edit in `/var/opt/gitlab/gitlab-rails/etc/database.yml`. diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 9221559e5c0..8e0b39aaeae 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -14325,6 +14325,7 @@ Returns [`Tree`](#tree). | `canModifyBlob` | [`Boolean`](#boolean) | Whether the current user can modify the blob. | | `codeOwners` | [`[UserCore!]`](#usercore) | List of code owners for the blob. | | `editBlobPath` | [`String`](#string) | Web path to edit the blob in the old-style editor. | +| `externalStorage` | [`String`](#string) | External storage being used, if enabled (for instance, 'LFS'). | | `externalStorageUrl` | [`String`](#string) | Web path to download the raw blob via external storage, if enabled. | | `fileType` | [`String`](#string) | Expected format of the blob based on the extension. | | `forkAndEditPath` | [`String`](#string) | Web path to edit this blob using a forked project. | diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 8d25fd3bb0b..2fb88245665 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -327,21 +327,43 @@ You can find the schemas for these scanners here: - [Coverage Fuzzing](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/coverage-fuzzing-report-format.json) - [Secret Detection](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/secret-detection-report-format.json) -### Version +### Enable report validation + +In GitLab 14.10 and later, report validation against the schemas is enabled. To enable report validation for versions earlier than 14.10, +set [`VALIDATE_SCHEMA`](../../user/application_security/#enable-security-report-validation) to +`"true"`. + +Reports that don't pass validation are not ingested by GitLab, and an error message +displays on the corresponding pipeline. + +You should ensure that reports generated by the scanner pass validation against the schema version +declared in your reports. GitLab uses the +[`json_schemer`](https://www.rubydoc.info/gems/json_schemer) gem to perform validation. + +Ongoing improvements to report validation is tracked [in this epic](https://gitlab.com/groups/gitlab-org/-/epics/6968). + +### Report Fields + +#### Version This field specifies the version of the [Security Report Schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas) you are using. Please refer to the [releases](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) of the schemas for the specific versions to use. +This field specifies which [Security Report Schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas) version you are using. For information about the versions to use, see [releases](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases). -### Vulnerabilities +In GitLab 14.10 and later, GitLab validates your report against the version of the schema specified by this value. +The versions supported by GitLab can be found in +[`gitlab/ee/lib/ee/gitlab/ci/parsers/security/validators/schemas`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/lib/ee/gitlab/ci/parsers/security/validators/schemas). + +#### Vulnerabilities The `vulnerabilities` field of the report is an array of vulnerability objects. -#### ID +##### ID The `id` field is the unique identifier of the vulnerability. It is used to reference a fixed vulnerability from a [remediation objects](#remediations). We recommend that you generate a UUID and use it as the `id` field's value. -#### Category +##### Category The value of the `category` field matches the report type: @@ -351,12 +373,12 @@ The value of the `category` field matches the report type: - `sast` - `dast` -#### Scanner +##### Scanner The `scanner` field is an object that embeds a human-readable `name` and a technical `id`. The `id` should not collide with any other scanner another integrator would provide. -#### Name, message, and description +##### Name, message, and description The `name` and `message` fields contain a short description of the vulnerability. The `description` field provides more details. @@ -400,13 +422,13 @@ It should not repeat the other fields of the vulnerability object. In particular, the `description` should not repeat the `location` (what is affected) or the `solution` (how to mitigate the risk). -#### Solution +##### Solution You can use the `solution` field to instruct users how to fix the identified vulnerability or to mitigate the risk. End-users interact with this field, whereas GitLab automatically processes the `remediations` objects. -#### Identifiers +##### Identifiers The `identifiers` array describes the detected vulnerability. An identifier object's `type` and `value` fields are used to tell if two identifiers are the same. The user interface uses the @@ -443,11 +465,11 @@ The maximum number of identifiers for a vulnerability is set as 20. If a vulnera the system saves only the first 20 of them. Note that vulnerabilities in the [Pipeline Security](../../user/application_security/security_dashboard/#view-vulnerabilities-in-a-pipeline) tab do not enforce this limit and all identifiers present in the report artifact are displayed. -### Details +#### Details The `details` field is an object that supports many different content elements that are displayed when viewing vulnerability information. An example of the various data elements can be seen in the [security-reports repository](https://gitlab.com/gitlab-examples/security/security-reports/-/tree/master/samples/details-example). -### Location +#### Location The `location` indicates where the vulnerability has been detected. The format of the location depends on the type of scanning. @@ -457,7 +479,7 @@ which is used to track vulnerabilities as new commits are pushed to the repository. The attributes used to generate the location fingerprint also depend on the type of scanning. -#### Dependency Scanning +##### Dependency Scanning The `location` of a Dependency Scanning vulnerability is composed of a `dependency` and a `file`. The `dependency` object describes the affected `package` and the dependency `version`. @@ -487,7 +509,7 @@ combines the `file` and the package `name`, so these attributes are mandatory. All other attributes are optional. -#### Container Scanning +##### Container Scanning Similar to Dependency Scanning, the `location` of a Container Scanning vulnerability has a `dependency` and a `file`. @@ -518,7 +540,7 @@ so these attributes are mandatory. The `image` is also mandatory. All other attributes are optional. -#### Cluster Image Scanning +##### Cluster Image Scanning The `location` of a `cluster_image_scanning` vulnerability has a `dependency` field. It also has an `operating_system` field. For example, here is the `location` object for a vulnerability @@ -552,7 +574,7 @@ as well as the package `name`, so these fields are required. The `image` field i The `cluster_id` and `agent_id` are mutually exclusive, and one of them must be present. All other fields are optional. -#### SAST +##### SAST The `location` of a SAST vulnerability must have a `file` and a `start_line` field, giving the path of the affected file, and the affected line number, respectively. @@ -577,7 +599,7 @@ combines `file`, `start_line`, and `end_line`, so these attributes are mandatory. All other attributes are optional. -### Tracking and merging vulnerabilities +#### Tracking and merging vulnerabilities Users may give feedback on a vulnerability: @@ -605,7 +627,7 @@ and at least one [identifier](#identifiers). Two identifiers are the same if the CWE and WASC identifiers are not considered because they describe categories of vulnerability flaws, but not specific security flaws. -#### Severity and confidence +##### Severity and confidence The `severity` field describes how much the vulnerability impacts the software, whereas the `confidence` field describes how reliable the assessment of the vulnerability is. @@ -622,7 +644,7 @@ Valid values are: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, and needs to be investigated. We have [provided a chart](../../user/application_security/sast/analyzers.md#analyzers-data) of the available SAST Analyzers and what data is currently available. -### Remediations +#### Remediations The `remediations` field of the report is an array of remediation objects. Each remediation describes a patch that can be applied to @@ -666,16 +688,16 @@ Here is an example of a report that contains remediations. } ``` -#### Summary +##### Summary The `summary` field is an overview of how the vulnerabilities can be fixed. This field is required. -#### Fixed vulnerabilities +##### Fixed vulnerabilities The `fixes` field is an array of objects that reference the vulnerabilities fixed by the remediation. `fixes[].id` contains a fixed vulnerability's [unique identifier](#id). This field is required. -#### Diff +##### Diff The `diff` field is a base64-encoded remediation code diff, compatible with [`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). This field is required. diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index 02e86a8e34b..cde44a8eb01 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -222,6 +222,13 @@ The `* as-if-jh` jobs are run in addition to the regular EE-context jobs. The `j The intent is to ensure that a change doesn't introduce a failure after the `gitlab-org/gitlab` project is synced to the `gitlab-jh/gitlab` project. +### Corresponding JH branch + +You can create a corresponding JH branch on the `gitlab-jh/gitlab` project by +appending `-jh` to the branch name. If a corresponding JH branch is found, +`* as-if-jh` jobs grab the `jh` folder from the respective branch, +rather than from the default branch. + ## `undercover` RSpec test > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74859) in GitLab 14.6. diff --git a/doc/development/service_ping/performance_indicator_metrics.md b/doc/development/service_ping/performance_indicator_metrics.md index ee2d9dd923a..48c123171fa 100644 --- a/doc/development/service_ping/performance_indicator_metrics.md +++ b/doc/development/service_ping/performance_indicator_metrics.md @@ -10,7 +10,7 @@ This guide describes how to use metrics definitions to define [performance indic To use a metric definition to manage a performance indicator: -1. Create a new issue and use the [Performance Indicator Metric issue template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Performance%20Indicator%20Metric.md). +1. Create a new issue and use the [Performance Indicator Metric issue template](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Performance%20Indicator%20Metric). 1. Use labels `~"product intelligence"`, `"~Data Warehouse::Impact Check"`. 1. Create a merge request that includes changes related only to the metric performance indicator. 1. Update the metric definition `performance_indicator_type` [field](metrics_dictionary.md#metrics-definition-and-validation). diff --git a/doc/development/snowplow/schemas.md b/doc/development/snowplow/schemas.md index 890feabb43e..f66e0566a9c 100644 --- a/doc/development/snowplow/schemas.md +++ b/doc/development/snowplow/schemas.md @@ -19,7 +19,6 @@ The [`StandardContext`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/g | `project_id` | **{dotted-circle}** | integer | | | `namespace_id` | **{dotted-circle}** | integer | | | `user_id` | **{dotted-circle}** | integer | User database record ID attribute. This file undergoes a pseudonymization process at the collector level. | -| `context_generated_at` | **{dotted-circle}** | string (date time format) | Timestamp indicating when context was generated. | | `environment` | **{check-circle}** | string (max 32 chars) | Name of the source environment, such as `production` or `staging` | | `source` | **{check-circle}** | string (max 32 chars) | Name of the source application, such as `gitlab-rails` or `gitlab-javascript` | | `plan` | **{dotted-circle}** | string (max 32 chars) | Name of the plan for the namespace, such as `free`, `premium`, or `ultimate`. Automatically picked from the `namespace`. | diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7f45d626922..54fb1d19ea8 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -69,17 +69,6 @@ module Gitlab self.class.attempt_counter.increment end - # rubocop: disable CodeReuse/ActiveRecord - def jobs_running_for_project(job) - return '+Inf' unless runner.instance_type? - - # excluding currently started job - running_jobs_count = job.project.builds.running.where(runner: ::Ci::Runner.instance_type) - .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 - running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET ? running_jobs_count : "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" - end - # rubocop: enable CodeReuse/ActiveRecord - def increment_queue_operation(operation) self.class.increment_queue_operation(operation) end @@ -242,6 +231,32 @@ module Gitlab Gitlab::Metrics.histogram(name, comment, labels, buckets) end end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def jobs_running_for_project(job) + return '+Inf' unless runner.instance_type? + + # excluding currently started job + running_jobs_count = running_jobs_relation(job) + .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 + + if running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + running_jobs_count + else + "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" + end + end + + def running_jobs_relation(job) + if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) + else + job.project.builds.running.where(runner: ::Ci::Runner.instance_type) + end + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 542dc476526..837390b91fb 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,7 +3,7 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-8' + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-7' GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) @@ -46,8 +46,7 @@ module Gitlab extra: extra, user_id: user&.id, namespace_id: namespace&.id, - project_id: project_id, - context_generated_at: Time.current + project_id: project_id } end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index a2accb26e25..1d5eb73b0d2 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -563,7 +563,7 @@ msgstr "" msgid "%{description}- Sentry event: %{errorUrl}- First seen: %{firstSeen}- Last seen: %{lastSeen} %{countLabel}: %{count}%{userCountLabel}: %{userCount}" msgstr "" -msgid "%{doc_link_start}Advanced search%{doc_link_end} is disabled since %{ref_elem} is not the default branch; %{default_branch_link_start}search on %{default_branch} instead%{default_branch_link_end}." +msgid "%{doc_link_start}Advanced search%{doc_link_end} is disabled since %{ref_elem} is not the default branch. %{docs_link}" msgstr "" msgid "%{doc_link_start}Advanced search%{doc_link_end} is enabled." @@ -12832,6 +12832,9 @@ msgstr "" msgid "Edit environment" msgstr "" +msgid "Edit epics" +msgstr "" + msgid "Edit files in the editor and commit changes here" msgstr "" @@ -13773,6 +13776,9 @@ msgstr "" msgid "Epics|Something went wrong while removing issue from epic." msgstr "" +msgid "Epics|Something went wrong while updating epics." +msgstr "" + msgid "Epics|This epic and any containing child epics are confidential and should only be visible to team members with at least Reporter access." msgstr "" @@ -33562,9 +33568,6 @@ msgstr "" msgid "Start your free trial" msgstr "" -msgid "Start your trial" -msgstr "" - msgid "Started" msgstr "" diff --git a/scripts/setup/find-jh-branch.rb b/scripts/setup/find-jh-branch.rb new file mode 100755 index 00000000000..9c23b72e51d --- /dev/null +++ b/scripts/setup/find-jh-branch.rb @@ -0,0 +1,102 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# In spec/scripts/setup/find_jh_branch_spec.rb we completely stub it +require 'gitlab' unless Object.const_defined?(:Gitlab) + +require_relative '../api/default_options' + +class FindJhBranch + JH_DEFAULT_BRANCH = 'main-jh' + JH_PROJECT_PATH = 'gitlab-jh/gitlab' + BranchNotFound = Class.new(RuntimeError) + + def run + return JH_DEFAULT_BRANCH unless merge_request? + + jh_merge_request_ref_name || + default_branch_merge_request_ref_name || + stable_branch_merge_request_ref_name || + default_branch_for_non_stable + end + + private + + def merge_request? + !!merge_request_id + end + + def jh_merge_request_ref_name + branch_exist?(JH_PROJECT_PATH, jh_ref_name) && jh_ref_name + end + + def default_branch_merge_request_ref_name + target_default_branch? && JH_DEFAULT_BRANCH + end + + def stable_branch_merge_request_ref_name + target_stable_branch? && begin + jh_stable_branch_name = merge_request.target_branch.sub(/\-ee\z/, '-jh') + + branch_exist?(JH_PROJECT_PATH, jh_stable_branch_name) && + jh_stable_branch_name + end + end + + def default_branch_for_non_stable + if target_stable_branch? + raise(BranchNotFound, "Cannot find a suitable JH branch") + else + JH_DEFAULT_BRANCH + end + end + + def branch_exist?(project_path, branch_name) + !!gitlab.branch(project_path, branch_name) + rescue Gitlab::Error::NotFound + false + end + + def target_default_branch? + merge_request.target_branch == default_branch + end + + def target_stable_branch? + merge_request.target_branch.match?(/\A(?:\d+\-)+\d+\-stable\-ee\z/) + end + + def ref_name + ENV['CI_COMMIT_REF_NAME'] + end + + def default_branch + ENV['CI_DEFAULT_BRANCH'] + end + + def merge_request_id + ENV['CI_MERGE_REQUEST_IID'] + end + + def jh_ref_name + "#{ref_name}-jh" + end + + def project_id + API::DEFAULT_OPTIONS[:project] + end + + def merge_request + @merge_request ||= gitlab.merge_request(project_id, merge_request_id) + end + + def gitlab + @gitlab ||= Gitlab.client( + endpoint: API::DEFAULT_OPTIONS[:endpoint], + private_token: API::DEFAULT_OPTIONS[:api_token] || '' + ) + end +end + +if $0 == __FILE__ + puts FindJhBranch.new.run +end diff --git a/scripts/utils.sh b/scripts/utils.sh index ed27edcadb2..15047d35fc3 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -39,7 +39,7 @@ function bundle_install_script() { bundle --version bundle config set path "$(pwd)/vendor" bundle config set clean 'true' - test -d jh && bundle config set gemfile 'jh/Gemfile' + test -d jh && bundle config set --local gemfile 'jh/Gemfile' echo "${BUNDLE_WITHOUT}" bundle config diff --git a/spec/frontend/blob/components/__snapshots__/blob_header_filepath_spec.js.snap b/spec/frontend/blob/components/__snapshots__/blob_header_filepath_spec.js.snap index 46a5631b028..d698ee72ea4 100644 --- a/spec/frontend/blob/components/__snapshots__/blob_header_filepath_spec.js.snap +++ b/spec/frontend/blob/components/__snapshots__/blob_header_filepath_spec.js.snap @@ -20,12 +20,6 @@ exports[`Blob Header Filepath rendering matches the snapshot 1`] = ` foo/bar/dummy.md - - a lot - - + + + a lot + + +

`; diff --git a/spec/frontend/blob/components/blob_header_filepath_spec.js b/spec/frontend/blob/components/blob_header_filepath_spec.js index d935f73c0d1..8220b598ff6 100644 --- a/spec/frontend/blob/components/blob_header_filepath_spec.js +++ b/spec/frontend/blob/components/blob_header_filepath_spec.js @@ -1,3 +1,4 @@ +import { GlBadge } from '@gitlab/ui'; import { shallowMount } from '@vue/test-utils'; import BlobHeaderFilepath from '~/blob/components/blob_header_filepath.vue'; import { numberToHumanSize } from '~/lib/utils/number_utils'; @@ -24,6 +25,8 @@ describe('Blob Header Filepath', () => { wrapper.destroy(); }); + const findBadge = () => wrapper.find(GlBadge); + describe('rendering', () => { it('matches the snapshot', () => { createComponent(); @@ -54,6 +57,11 @@ describe('Blob Header Filepath', () => { expect(wrapper.vm.blobSize).toBe('a lot'); }); + it('renders LFS badge if LFS if enabled', () => { + createComponent({ storedExternally: true, externalStorage: 'lfs' }); + expect(findBadge().text()).toBe('LFS'); + }); + it('renders a slot and prepends its contents to the existing one', () => { const slotContent = 'Foo Bar'; createComponent( diff --git a/spec/frontend/repository/mock_data.js b/spec/frontend/repository/mock_data.js index 74d35daf578..3755bfb30d3 100644 --- a/spec/frontend/repository/mock_data.js +++ b/spec/frontend/repository/mock_data.js @@ -14,6 +14,7 @@ export const simpleViewerMock = { canModifyBlob: true, canCurrentUserPushToBranch: true, storedExternally: false, + externalStorage: 'lfs', rawPath: 'some_file.js', replacePath: 'some_file.js/replace', pipelineEditorPath: '', diff --git a/spec/frontend/security_configuration/components/training_provider_list_spec.js b/spec/frontend/security_configuration/components/training_provider_list_spec.js index 60cc36a634c..183deec0f10 100644 --- a/spec/frontend/security_configuration/components/training_provider_list_spec.js +++ b/spec/frontend/security_configuration/components/training_provider_list_spec.js @@ -4,8 +4,14 @@ import Vue from 'vue'; import VueApollo from 'vue-apollo'; import createMockApollo from 'helpers/mock_apollo_helper'; import TrainingProviderList from '~/security_configuration/components/training_provider_list.vue'; +import configureSecurityTrainingProvidersMutation from '~/security_configuration/graphql/configure_security_training_providers.mutation.graphql'; import waitForPromises from 'helpers/wait_for_promises'; -import { securityTrainingProviders, mockResolvers } from '../mock_data'; +import { + securityTrainingProviders, + mockResolvers, + testProjectPath, + textProviderIds, +} from '../mock_data'; Vue.use(VueApollo); @@ -18,6 +24,9 @@ describe('TrainingProviderList component', () => { mockApollo = createMockApollo([], mockResolvers); wrapper = shallowMount(TrainingProviderList, { + provide: { + projectPath: testProjectPath, + }, apolloProvider: mockApollo, }); }; @@ -85,4 +94,36 @@ describe('TrainingProviderList component', () => { }); }); }); + + describe('success mutation', () => { + const firstToggle = () => findToggles().at(0); + + beforeEach(async () => { + jest.spyOn(mockApollo.defaultClient, 'mutate'); + + await waitForQueryToBeLoaded(); + + firstToggle().vm.$emit('change'); + }); + + it('calls mutation when toggle is changed', () => { + expect(mockApollo.defaultClient.mutate).toHaveBeenCalledWith( + expect.objectContaining({ + mutation: configureSecurityTrainingProvidersMutation, + variables: { input: { enabledProviders: textProviderIds, fullPath: testProjectPath } }, + }), + ); + }); + + it.each` + loading | wait | desc + ${true} | ${false} | ${'enables loading of GlToggle when mutation is called'} + ${false} | ${true} | ${'disables loading of GlToggle when mutation is complete'} + `('$desc', async ({ loading, wait }) => { + if (wait) { + await waitForPromises(); + } + expect(firstToggle().props('isLoading')).toBe(loading); + }); + }); }); diff --git a/spec/frontend/security_configuration/mock_data.js b/spec/frontend/security_configuration/mock_data.js index cdb859c3800..b01dd1fc07b 100644 --- a/spec/frontend/security_configuration/mock_data.js +++ b/spec/frontend/security_configuration/mock_data.js @@ -1,13 +1,17 @@ +export const testProjectPath = 'foo/bar'; + +export const textProviderIds = [101, 102]; + export const securityTrainingProviders = [ { - id: 101, + id: textProviderIds[0], name: 'Kontra', description: 'Interactive developer security education.', url: 'https://application.security/', isEnabled: false, }, { - id: 102, + id: textProviderIds[1], name: 'SecureCodeWarrior', description: 'Security training with guide and learning pathways.', url: 'https://www.securecodewarrior.com/', diff --git a/spec/graphql/types/repository/blob_type_spec.rb b/spec/graphql/types/repository/blob_type_spec.rb index 21bc88e34c0..f23b621d58e 100644 --- a/spec/graphql/types/repository/blob_type_spec.rb +++ b/spec/graphql/types/repository/blob_type_spec.rb @@ -21,6 +21,7 @@ RSpec.describe Types::Repository::BlobType do :file_type, :edit_blob_path, :stored_externally, + :external_storage, :raw_path, :replace_path, :pipeline_editor_path, diff --git a/spec/lib/feature_spec.rb b/spec/lib/feature_spec.rb index 77a4880bf6e..01db06617dc 100644 --- a/spec/lib/feature_spec.rb +++ b/spec/lib/feature_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Feature, stub_feature_flags: false do + include StubVersion + before do # reset Flipper AR-engine Feature.reset @@ -583,7 +585,11 @@ RSpec.describe Feature, stub_feature_flags: false do end context 'when flag is new and not feature_flag_state_logs' do - let(:milestone) { "14.9" } + let(:milestone) { "14.6" } + + before do + stub_version('14.5.123', 'deadbeef') + end it { is_expected.to be_truthy } end diff --git a/spec/lib/gitlab/tracking/standard_context_spec.rb b/spec/lib/gitlab/tracking/standard_context_spec.rb index c88b0af30f6..7d678db5ec8 100644 --- a/spec/lib/gitlab/tracking/standard_context_spec.rb +++ b/spec/lib/gitlab/tracking/standard_context_spec.rb @@ -58,10 +58,6 @@ RSpec.describe Gitlab::Tracking::StandardContext do expect(snowplow_context.to_json.dig(:data, :source)).to eq(described_class::GITLAB_RAILS_SOURCE) end - it 'contains context_generated_at timestamp', :freeze_time do - expect(snowplow_context.to_json.dig(:data, :context_generated_at)).to eq(Time.current) - end - context 'plan' do context 'when namespace is not available' do it 'is nil' do diff --git a/spec/scripts/setup/find_jh_branch_spec.rb b/spec/scripts/setup/find_jh_branch_spec.rb new file mode 100644 index 00000000000..dfc3601ffa9 --- /dev/null +++ b/spec/scripts/setup/find_jh_branch_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +# NOTE: Under the context of fast_spec_helper, when we `require 'gitlab'` +# we do not load the Gitlab client, but our own Gitlab module. +# Keep this in mind and just stub anything which might touch it! +require_relative '../../../scripts/setup/find-jh-branch' + +RSpec.describe FindJhBranch do + subject { described_class.new } + + describe '#run' do + context 'when it is not a merge request' do + before do + expect(subject).to receive(:merge_request?).and_return(false) + end + + it 'returns JH_DEFAULT_BRANCH' do + expect(subject.run).to eq(described_class::JH_DEFAULT_BRANCH) + end + end + + context 'when it is a merge request' do + let(:branch_name) { 'branch-name' } + let(:jh_branch_name) { 'branch-name-jh' } + let(:default_branch) { 'main' } + let(:merge_request) { double(target_branch: target_branch) } + let(:target_branch) { default_branch } + + before do + expect(subject).to receive(:merge_request?).and_return(true) + + expect(subject) + .to receive(:branch_exist?) + .with(described_class::JH_PROJECT_PATH, jh_branch_name) + .and_return(jh_branch_exist) + + allow(subject).to receive(:ref_name).and_return(branch_name) + allow(subject).to receive(:default_branch).and_return(default_branch) + allow(subject).to receive(:merge_request).and_return(merge_request) + end + + context 'when there is a corresponding JH branch' do + let(:jh_branch_exist) { true } + + it 'returns the corresponding JH branch name' do + expect(subject.run).to eq(jh_branch_name) + end + end + + context 'when there is no corresponding JH branch' do + let(:jh_branch_exist) { false } + + it 'returns the default JH branch' do + expect(subject.run).to eq(described_class::JH_DEFAULT_BRANCH) + end + + context 'when it is targeting a default branch' do + let(:target_branch) { '14-6-stable-ee' } + let(:jh_stable_branch_name) { '14-6-stable-jh' } + + before do + expect(subject) + .to receive(:branch_exist?) + .with(described_class::JH_PROJECT_PATH, jh_stable_branch_name) + .and_return(jh_stable_branch_exist) + end + + context 'when there is a corresponding JH stable branch' do + let(:jh_stable_branch_exist) { true } + + it 'returns the corresponding JH stable branch' do + expect(subject.run).to eq(jh_stable_branch_name) + end + end + + context 'when there is no corresponding JH stable branch' do + let(:jh_stable_branch_exist) { false } + + it "raises #{described_class::BranchNotFound}" do + expect { subject.run }.to raise_error(described_class::BranchNotFound) + end + end + end + + context 'when it is not targeting the default branch' do + let(:target_branch) { default_branch.swapcase } + + it 'returns the default JH branch' do + expect(subject.run).to eq(described_class::JH_DEFAULT_BRANCH) + end + end + end + end + end +end diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index 866015aa523..251159864f5 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -827,11 +827,17 @@ module Ci end context 'when project already has running jobs' do - let!(:build2) { create(:ci_build, :running, pipeline: pipeline, runner: shared_runner) } - let!(:build3) { create(:ci_build, :running, pipeline: pipeline, runner: shared_runner) } + let(:build2) { create(:ci_build, :running, pipeline: pipeline, runner: shared_runner) } + let(:build3) { create(:ci_build, :running, pipeline: pipeline, runner: shared_runner) } + + before do + ::Ci::RunningBuild.upsert_shared_runner_build!(build2) + ::Ci::RunningBuild.upsert_shared_runner_build!(build3) + end it 'counts job queuing time histogram with expected labels' do allow(attempt_counter).to receive(:increment) + expect(job_queue_duration_seconds).to receive(:observe) .with({ shared_runner: expected_shared_runner, jobs_running_for_project: expected_jobs_running_for_project_third_job, @@ -845,6 +851,14 @@ module Ci shared_examples 'metrics collector' do it_behaves_like 'attempt counter collector' it_behaves_like 'jobs queueing time histogram collector' + + context 'when using denormalized data is disabled' do + before do + stub_feature_flags(ci_pending_builds_maintain_denormalized_data: false) + end + + it_behaves_like 'jobs queueing time histogram collector' + end end context 'when shared runner is used' do @@ -875,6 +889,16 @@ module Ci it_behaves_like 'metrics collector' end + context 'when max running jobs bucket size is exceeded' do + before do + stub_const('Gitlab::Ci::Queue::Metrics::JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET', 1) + end + + let(:expected_jobs_running_for_project_third_job) { '1+' } + + it_behaves_like 'metrics collector' + end + context 'when pending job with queued_at=nil is used' do before do pending_job.update!(queued_at: nil)