Add latest changes from gitlab-org/gitlab@master

app/assets/javascripts/admin/application_settings/setup_service_usage_data.js

@@ -1,9 +1,15 @@
 import PayloadPreviewer from '~/pages/admin/application_settings/payload_previewer';
+import PayloadDownloader from '~/pages/admin/application_settings/payload_downloader';
 
 export default () => {
   const payloadPreviewTrigger = document.querySelector('.js-payload-preview-trigger');
+  const payloadDownloadTrigger = document.querySelector('.js-payload-download-trigger');
 
   if (payloadPreviewTrigger) {
     new PayloadPreviewer(payloadPreviewTrigger).init();
   }
+
+  if (payloadDownloadTrigger) {
+    new PayloadDownloader(payloadDownloadTrigger).init();
+  }
 };

app/assets/javascripts/pages/admin/application_settings/payload_downloader.js

@@ -0,0 +1,52 @@
+import createFlash from '~/flash';
+import axios from '../../../lib/utils/axios_utils';
+import { __ } from '../../../locale';
+
+export default class PayloadDownloader {
+  constructor(trigger) {
+    this.trigger = trigger;
+  }
+
+  init() {
+    this.spinner = this.trigger.querySelector('.js-spinner');
+    this.text = this.trigger.querySelector('.js-text');
+
+    this.trigger.addEventListener('click', (event) => {
+      event.preventDefault();
+
+      return this.requestPayload();
+    });
+  }
+
+  requestPayload() {
+    this.spinner.classList.add('d-inline-flex');
+
+    return axios
+      .get(this.trigger.dataset.endpoint, {
+        responseType: 'json',
+      })
+      .then(({ data }) => {
+        PayloadDownloader.downloadFile(data);
+      })
+      .catch(() => {
+        createFlash({
+          message: __('Error fetching payload data.'),
+        });
+      })
+      .finally(() => {
+        this.spinner.classList.remove('d-inline-flex');
+      });
+  }
+
+  static downloadFile(data) {
+    const blob = new Blob([JSON.stringify(data)], { type: 'application/json' });
+
+    const link = document.createElement('a');
+    link.href = window.URL.createObjectURL(blob);
+    link.download = `${data.recorded_at.slice(0, 10)} payload.json`;
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+    window.URL.revokeObjectURL(link.href);
+  }
+}

app/presenters/projects/security/configuration_presenter.rb

@@ -80,7 +80,8 @@ module Projects
           type: scan.type,
           configured: scan.configured?,
           configuration_path: scan.configuration_path,
-          available: scan.available?
+          available: scan.available?,
+          can_enable_in_merge_request: scan.can_enable_in_merge_request?
         }
       end
 

app/views/admin/application_settings/service_usage_data.html.haml

@@ -10,4 +10,7 @@
 %button.gl-button.btn.btn-default.js-payload-preview-trigger{ type: 'button', data: { payload_selector: ".#{payload_class}" } }
   .gl-spinner.js-spinner.gl-display-none.gl-mr-2
   .js-text.gl-display-inline= _('Preview payload')
+%button.gl-button.btn.btn-default.js-payload-download-trigger{ type: 'button', data: { endpoint: usage_data_admin_application_settings_path(format: :json) } }
+  .gl-spinner.js-spinner.gl-display-none.gl-mr-2
+  .js-text.d-inline= _('Download payload')
 %pre.js-syntax-highlight.code.highlight.gl-mt-2.gl-display-none{ class: payload_class, data: { endpoint: usage_data_admin_application_settings_path(format: :html) } }

app/views/admin/hooks/edit.html.haml

@@ -13,7 +13,7 @@
       .form-actions
         %span>= f.submit _('Save changes'), class: 'btn gl-button btn-confirm gl-mr-3'
         = render 'shared/web_hooks/test_button', hook: @hook
-        = link_to _('Delete'), admin_hook_path(@hook), method: :delete, class: 'btn gl-button btn-danger float-right', data: { confirm: _('Are you sure?') }
+        = link_to _('Delete'), admin_hook_path(@hook), method: :delete, class: 'btn gl-button btn-danger float-right', aria: { label: s_('Webhooks|Delete webhook') }, data: { confirm: s_('Webhooks|Are you sure you want to delete this webhook?'), confirm_btn_variant: 'danger' }
 
     %hr
 

data/deprecations/14-8-geo-deprecate-db-rake-tasks.yml

@@ -0,0 +1,34 @@
+- name: "Deprecate custom Geo:db:* Rake tasks"
+  announcement_milestone: "14.8"
+  announcement_date: "2022-02-22"
+  removal_milestone: "15.0"
+  removal_date:  "2022-05-22"
+  breaking_change: false
+  reporter: nhxnguyen
+  body: |
+    In GitLab 14.8, we are [replacing the `geo:db:*` Rake tasks with built-in tasks](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77269/diffs) that are now possible after [switching the Geo tracking database to use Rails' 6 support of multiple databases](https://gitlab.com/groups/gitlab-org/-/epics/6458).
+    The following `geo:db:*` tasks will be replaced with their corresponding `db:*:geo` tasks:
+
+    - `geo:db:drop` -> `db:drop:geo`
+    - `geo:db:create` -> `db:create:geo`
+    - `geo:db:setup` -> `db:setup:geo`
+    - `geo:db:migrate` -> `db:migrate:geo`
+    - `geo:db:rollback` -> `db:rollback:geo`
+    - `geo:db:version` -> `db:version:geo`
+    - `geo:db:reset` -> `db:reset:geo`
+    - `geo:db:seed` -> `db:seed:geo`
+    - `geo:schema:load:geo` -> `db:schema:load:geo`
+    - `geo:db:schema:dump` -> `db:schema:dump:geo`
+    - `geo:db:migrate:up` -> `db:migrate:up:geo`
+    - `geo:db:migrate:down` -> `db:migrate:down:geo`
+    - `geo:db:migrate:redo` -> `db:migrate:redo:geo`
+    - `geo:db:migrate:status` -> `db:migrate:status:geo`
+    - `geo:db:test:prepare` -> `db:test:prepare:geo`
+    - `geo:db:test:load` -> `db:test:load:geo`
+    - `geo:db:test:purge` -> `db:test:purge:geo`
+  stage: "Enablement"
+  tiers: ["Premium", "Ultimate"]
+  issue_url:  "https://gitlab.com/gitlab-org/gitlab/-/issues/351945"
+  documentation_url:  # (optional) This is a link to the current documentation page
+  image_url:  # (optional) This is a link to a thumbnail image depicting the feature
+  video_url:  # (optional) Use the youtube thumbnail URL with the structure of https://img.youtube.com/vi/UNIQUEID/hqdefault.jpg

doc/administration/troubleshooting/diagnostics_tools.md

@@ -23,3 +23,7 @@ running on.
 
 [strace-parser](https://gitlab.com/wchandler/strace-parser) is a small tool to analyze
 and summarize raw `strace` data.
+
+## kubesos
+
+The [`kubesos`](https://gitlab.com/gitlab-com/support/toolbox/kubesos/) utiltity retrieves GitLab cluster configuration and logs from GitLab Cloud Native chart deployments.

doc/api/users.md

@@ -78,8 +78,8 @@ GET /users?external=true
 GitLab supports bot users such as the [alert bot](../operations/incident_management/integrations.md)
 or the [support bot](../user/project/service_desk.md#support-bot-user).
 You can exclude the following types of [internal users](../development/internal_users.md#internal-users)
-from the users' list, with the `exclude_internal=true` parameter,
-([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241144) in GitLab 13.4).
+from the users' list with the `exclude_internal=true` parameter
+([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241144) in GitLab 13.4):
 
 - Alert bot
 - Support bot

doc/ci/pipelines/schedules.md

@@ -17,7 +17,9 @@ Pipeline schedules can be used to also run [pipelines](index.md) at specific int
 - Every other Sunday at 0900 hours (cron example: `0 9 * * sun%2`).
 - Once every day (cron example: `0 0 * * *`).
 
-Schedule timing is configured with cron notation, parsed by [Fugit](https://github.com/floraison/fugit).
+Schedule timing is configured with [cron notation](../../topics/cron/index.md).
+You can use any cron value, but scheduled pipelines cannot run more frequently
+than the instance's [maximum frequency for scheduled pipelines](#advanced-configuration).
 
 In addition to using the GitLab UI, pipeline schedules can be maintained using the
 [Pipeline schedules API](../../api/pipeline_schedules.md).
@@ -82,20 +84,24 @@ job:
 
 ### Advanced configuration **(FREE SELF)**
 
-The pipelines are not executed exactly on schedule because schedules are handled by
-Sidekiq, which runs according to its interval.
+Scheduled pipelines can be configured with any [cron value](../../topics/cron/index.md),
+but they do not always run exactly when scheduled. An internal process, called the
+_pipeline schedule worker_, queues all the scheduled pipelines, but does not
+run continuously. The worker runs on its own schedule, and scheduled pipelines that
+are ready to start are only queued the next time the worker runs. Scheduled pipelines
+can't run more frequently than the worker.
 
-For example, only two pipelines are created per day if:
+The default frequency of the pipeline schedule worker is `3-59/10 * * * *` (every ten minutes,
+starting with `0:03`, `0:13`, `0:23`, and so on). The default frequency for GitLab.com
+is listed in the [GitLab.com settings](../../user/gitlab_com/index.md#gitlab-cicd).
 
-- You set a schedule to create a pipeline every minute (`* * * * *`).
-- The Sidekiq worker runs on 00:00 and 12:00 every day (`0 */12 * * *`).
-
-To change the Sidekiq worker's frequency:
+To change the frequency of the pipeline schedule worker:
 
 1. Edit the `gitlab_rails['pipeline_schedule_worker_cron']` value in your instance's `gitlab.rb` file.
 1. [Reconfigure GitLab](../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
 
-For GitLab.com, refer to the [dedicated settings page](../../user/gitlab_com/index.md#gitlab-cicd).
+For example, to set the maximum frequency of pipelines to twice a day, set `pipeline_schedule_worker_cron`
+to a cron value of `0 */12 * * *` (`00:00` and `12:00` every day).
 
 ## Working with scheduled pipelines
 

doc/development/export_csv.md

@@ -11,7 +11,7 @@ This document lists the different implementations of CSV export in GitLab codeba
 | Export type | How it works | Advantages | Disadvantages | Existing examples |
 |---|---|---|---|---|
 | Streaming | - Query and yield data in batches to a response stream.<br>- Download starts immediately. | - Report available immediately. | - No progress indicator.<br>- Requires a reliable connection. | [Export Audit Event Log](../administration/audit_events.md#export-to-csv) |
-| Downloading | - Query and write data in batches to a temporary file.<br>- Loads the file into memory.<br>- Sends the file to the client. | - Report available immediately. | - Large amount of data might cause request timeout.<br>- Memory intensive.<br>- Request expires when user navigates to a different page. | [Export Chain of Custody Report](../user/compliance/compliance_report/#chain-of-custody-report) |
+| Downloading | - Query and write data in batches to a temporary file.<br>- Loads the file into memory.<br>- Sends the file to the client. | - Report available immediately. | - Large amount of data might cause request timeout.<br>- Memory intensive.<br>- Request expires when user navigates to a different page. | - [Export Chain of Custody Report](../user/compliance/compliance_report/#chain-of-custody-report)<br>- [Export License Usage File](../subscriptions/self_managed/index.md#export-your-license-usage) |
 | As email attachment | - Asynchronously process the query with background job.<br>- Email uses the export as an attachment. | - Asynchronous processing. | - Requires users use a different app (email) to download the CSV.<br>- Email providers may limit attachment size. | - [Export issues](../user/project/issues/csv_export.md)<br>- [Export merge requests](../user/project/merge_requests/csv_export.md) |
 | As downloadable link in email (*) | - Asynchronously process the query with background job.<br>- Email uses an export link. | - Asynchronous processing.<br>- Bypasses email provider attachment size limit. | - Requires users use a different app (email).<br>- Requires additional storage and cleanup. | [Export User Permissions](https://gitlab.com/gitlab-org/gitlab/-/issues/1772) |
 | Polling (non-persistent state) | - Asynchronously processes the query with the background job.<br>- Frontend(FE) polls every few seconds to check if CSV file is ready. | - Asynchronous processing.<br>- Automatically downloads to local machine on completion.<br>- In-app solution. | - Non-persistable request - request expires when user navigates to a different page.<br>- API is processed for each polling request. | [Export Vulnerabilities](../user/application_security/vulnerability_report/#export-vulnerability-details) |

doc/development/service_ping/index.md

@@ -576,6 +576,24 @@ skip_db_write:
 ServicePing::SubmitService.new(skip_db_write: true).execute
 ```
 
+## Manually upload Service Ping payload 
+
+> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7388) in GitLab 14.8 with a flag named `admin_application_settings_service_usage_data_center`. Disabled by default.
+
+Service Ping payload can be uploaded to GitLab even if your application instance doesn't have access to the internet,
+or you don't have Service Ping [cron job](#how-service-ping-works) enabled.
+
+To upload payload manually:
+
+1. Sign in as a user with administrator access.
+1. On the top bar, select **Menu > Admin**.
+1. On the left sidebar, select **Settings > Service** usage data.
+1. Select **Download payload**.
+1. Save the JSON file.
+1. Visit [Service usage data center](https://version.gitlab.com/usage_data/new).
+1. Select **Choose file** and choose the file from p5.
+1. Select **Upload**.
+
 ## Monitoring
 
 Service Ping reporting process state is monitored with [internal SiSense dashboard](https://app.periscopedata.com/app/gitlab/968489/Product-Intelligence---Service-Ping-Health). 

doc/subscriptions/self_managed/index.md

@@ -309,6 +309,11 @@ The **License Usage** CSV includes the following details:
   - Date the count was recorded
   - Active user count
 
+NOTES:
+
+- All timestamps are displayed in UTC.
+- A custom format is used for [dates](https://gitlab.com/gitlab-org/gitlab/blob/3be39f19ac3412c089be28553e6f91b681e5d739/config/initializers/date_time_formats.rb#L7) and [times](https://gitlab.com/gitlab-org/gitlab/blob/3be39f19ac3412c089be28553e6f91b681e5d739/config/initializers/date_time_formats.rb#L13) in CSV files.
+
 ## Renew your subscription
 
 To renew your subscription,

doc/topics/cron/index.md

@@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 Cron syntax is used to schedule when jobs should run.
 
 You may need to use a cron syntax string to
-create a [pipeline schedule](../../api/pipeline_schedules.md#create-a-new-pipeline-schedule),
+create a [pipeline schedule](../../ci/pipelines/schedules.md),
 or to prevent unintentional releases by setting a
 [deploy freeze](../../user/project/releases/index.md#prevent-unintentional-releases-by-setting-a-deploy-freeze).
 

doc/update/deprecations.md

@@ -754,6 +754,31 @@ In GitLab 13.0, we introduced new project and design replication details routes
 
 **Planned removal milestone: 15.0 (2022-05-22)**
 
+### Deprecate custom Geo:db:* Rake tasks
+
+In GitLab 14.8, we are [replacing the `geo:db:*` Rake tasks with built-in tasks](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77269/diffs) that are now possible after [switching the Geo tracking database to use Rails' 6 support of multiple databases](https://gitlab.com/groups/gitlab-org/-/epics/6458).
+The following `geo:db:*` tasks will be replaced with their corresponding `db:*:geo` tasks:
+
+- `geo:db:drop` -> `db:drop:geo`
+- `geo:db:create` -> `db:create:geo`
+- `geo:db:setup` -> `db:setup:geo`
+- `geo:db:migrate` -> `db:migrate:geo`
+- `geo:db:rollback` -> `db:rollback:geo`
+- `geo:db:version` -> `db:version:geo`
+- `geo:db:reset` -> `db:reset:geo`
+- `geo:db:seed` -> `db:seed:geo`
+- `geo:schema:load:geo` -> `db:schema:load:geo`
+- `geo:db:schema:dump` -> `db:schema:dump:geo`
+- `geo:db:migrate:up` -> `db:migrate:up:geo`
+- `geo:db:migrate:down` -> `db:migrate:down:geo`
+- `geo:db:migrate:redo` -> `db:migrate:redo:geo`
+- `geo:db:migrate:status` -> `db:migrate:status:geo`
+- `geo:db:test:prepare` -> `db:test:prepare:geo`
+- `geo:db:test:load` -> `db:test:load:geo`
+- `geo:db:test:purge` -> `db:test:purge:geo`
+
+**Planned removal milestone: 15.0 (2022-05-22)**
+
 ### External status check API breaking changes
 
 WARNING:

doc/user/admin_area/license.md

@@ -128,6 +128,8 @@ the current date range is the active license.
 When you upload a future-dated license, it doesn't take effect until its applicable date.
 You can view all active subscriptions in the **Subscription history** table.
 
+You can also [export](../../subscriptions/self_managed/index.md) your license usage information to a CSV file.
+
 NOTE:
 In GitLab 13.6 and earlier, a banner about an expiring license may continue to display
 when you upload a new license. This happens when the start date of the new license

lib/gitlab/security/scan_configuration.rb

@@ -18,27 +18,25 @@ module Gitlab
         # SAST and Secret Detection are always available, but this isn't
         # reflected by our license model yet.
         # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/333113
-        %i[sast secret_detection].include?(type)
+        %i[sast sast_iac secret_detection].include?(type)
+      end
+
+      def can_enable_in_merge_request?
+        scans_configurable_in_merge_request.include?(type)
       end
 
       def configured?
         configured
       end
 
-      def configuration_path
-        configurable_scans[type]
-      end
+      def configuration_path; end
 
       private
 
       attr_reader :project, :configured
 
-      def configurable_scans
-        strong_memoize(:configurable_scans) do
-          {
-            sast: project_security_configuration_sast_path(project)
-          }
-        end
+      def scans_configurable_in_merge_request
+        %i[sast sast_iac secret_detection]
       end
     end
   end

locale/gitlab.pot

@@ -12935,6 +12935,9 @@ msgstr ""
 msgid "Download image"
 msgstr ""
 
+msgid "Download payload"
+msgstr ""
+
 msgid "Download raw data (.csv)"
 msgstr ""
 

package.json

@@ -114,7 +114,7 @@
     "codesandbox-api": "0.0.23",
     "compression-webpack-plugin": "^5.0.2",
     "copy-webpack-plugin": "^6.4.1",
-    "core-js": "^3.20.3",
+    "core-js": "^3.21.0",
     "cron-validator": "^1.1.1",
     "cronstrue": "^1.122.0",
     "cropper": "^2.3.0",

spec/features/admin/admin_settings_spec.rb

@@ -814,6 +814,16 @@ RSpec.describe 'Admin updates settings' do
         expect(page).to have_button 'Hide payload'
         expect(page).to have_content expected_payload_content
       end
+
+      it 'generates usage ping payload on button click', :js do
+        expect_next_instance_of(Admin::ApplicationSettingsController) do |instance|
+          expect(instance).to receive(:usage_data).and_call_original
+        end
+
+        click_button('Download payload')
+
+        wait_for_requests
+      end
     end
   end
 

spec/lib/gitlab/security/scan_configuration_spec.rb

@@ -3,6 +3,8 @@
 require 'spec_helper'
 
 RSpec.describe ::Gitlab::Security::ScanConfiguration do
+  using RSpec::Parameterized::TableSyntax
+
   let_it_be(:project) { create(:project, :repository) }
 
   let(:scan) { described_class.new(project: project, type: type, configured: configured) }
@@ -13,9 +15,11 @@ RSpec.describe ::Gitlab::Security::ScanConfiguration do
     let(:configured) { true }
 
     context 'with a core scanner' do
-      let(:type) { :sast }
+      where(type: %i(sast sast_iac secret_detection))
 
-      it { is_expected.to be_truthy }
+      with_them do
+        it { is_expected.to be_truthy }
+      end
     end
 
     context 'with custom scanner' do
@@ -38,27 +42,28 @@ RSpec.describe ::Gitlab::Security::ScanConfiguration do
     subject { scan.configuration_path }
 
     let(:configured) { true }
+    let(:type) { :sast }
 
-    context 'with a non configurable scanner' do
-      let(:type) { :secret_detection }
+    it { is_expected.to be_nil }
+  end
 
-      it { is_expected.to be_nil }
-    end
+  describe '#can_enable_in_merge_request?' do
+    subject { scan.can_enable_in_merge_request? }
 
-    context 'with licensed scanner for FOSS environment' do
-      let(:type) { :dast }
+    let(:configured) { true }
 
-      before do
-        stub_env('FOSS_ONLY', '1')
+    context 'with a core scanner' do
+      where(type: %i(sast sast_iac secret_detection))
+
+      with_them do
+        it { is_expected.to be_truthy }
       end
-
-      it { is_expected.to be_nil }
     end
 
-    context 'with custom scanner' do
+    context 'with a custom scanner' do
       let(:type) { :my_scanner }
 
-      it { is_expected.to be_nil }
+      it { is_expected.to be_falsey }
     end
   end
 end

spec/presenters/projects/security/configuration_presenter_spec.rb

@@ -86,8 +86,9 @@ RSpec.describe Projects::Security::ConfigurationPresenter do
 
         expect(feature['type']).to eq('sast')
         expect(feature['configured']).to eq(true)
-        expect(feature['configuration_path']).to eq(project_security_configuration_sast_path(project))
+        expect(feature['configuration_path']).to be_nil
         expect(feature['available']).to eq(true)
+        expect(feature['can_enable_in_merge_request']).to eq(true)
       end
 
       context 'when checking features configured status' do

yarn.lock

@@ -3773,10 +3773,10 @@ core-js-pure@^3.0.0:
   resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.6.5.tgz#c79e75f5e38dbc85a662d91eea52b8256d53b813"
   integrity sha512-lacdXOimsiD0QyNf9BC/mxivNJ/ybBGJXQFKzRekp1WTHoVUWsUHEn+2T8GJAzzIhyOuXA+gOxCVN3l+5PLPUA==
 
-core-js@^3.20.3:
-  version "3.20.3"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.20.3.tgz#c710d0a676e684522f3db4ee84e5e18a9d11d69a"
-  integrity sha512-vVl8j8ph6tRS3B8qir40H7yw7voy17xL0piAjlbBUsH7WIfzoedL/ZOr1OV9FyZQLWXsayOJyV4tnRyXR85/ag==
+core-js@^3.21.0:
+  version "3.21.0"
+  resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.21.0.tgz#f479dbfc3dffb035a0827602dd056839a774aa71"
+  integrity sha512-YUdI3fFu4TF/2WykQ2xzSiTQdldLB4KVuL9WeAy5XONZYt5Cun/fpQvctoKbCgvPhmzADeesTk/j2Rdx77AcKQ==
 
 core-js@~2.3.0:
   version "2.3.0"