root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-04-29 07:30:46 +00:00
parent 64522a5cb5
commit bc843478bc
40 changed files with 626 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/assets/javascripts/vue_shared/components/visibility_icon.vue
View File

@ -1,5 +1,6 @@
<script>
import { GlIcon, GlTooltipDirective } from '@gitlab/ui';
import { __ } from '~/locale';
import {
GROUP_VISIBILITY_TYPE,
PROJECT_VISIBILITY_TYPE,
@ -30,10 +31,17 @@ export default {
},
},
computed: {
isBannedProject() {
return !this.isGroup && this.visibilityLevel === 'banned';
},
visibilityIcon() {
return VISIBILITY_TYPE_ICON[this.visibilityLevel];
return this.isBannedProject ? 'spam' : VISIBILITY_TYPE_ICON[this.visibilityLevel];
},
visibilityTooltip() {
if (this.isBannedProject) {
return __('This project is hidden because its creator has been banned');
}
if (this.isGroup) {
return GROUP_VISIBILITY_TYPE[this.visibilityLevel];
}

2
app/graphql/types/ci/runner_manager_type.rb
View File

@ -47,7 +47,7 @@ module Types
field :version, GraphQL::Types::String, null: true, description: 'Version of the runner.'
def executor_name
::Ci::Runner::EXECUTOR_TYPE_TO_NAMES[runner_manager.executor_type&.to_sym]
::Ci::RunnerManager::EXECUTOR_TYPE_TO_NAMES[runner_manager.executor_type&.to_sym]
end
def job_execution_status

91
app/models/ci/runner.rb
View File

@ -90,6 +90,8 @@ module Ci
has_one :owner_runner_namespace, -> { order(:id) }, class_name: 'Ci::RunnerNamespace'
has_one :last_build, -> { order('id DESC') }, class_name: 'Ci::Build'
# TODO: Remove in 17.1 after hide_duplicate_runner_manager_fields_in_runner is removed
has_one :runner_version, primary_key: :version, foreign_key: :version, class_name: 'Ci::RunnerVersion'
belongs_to :creator, class_name: 'User', optional: true
@ -241,7 +243,25 @@ module Ci
after_destroy :cleanup_runner_queue
cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at, :executor_type
# NOTE: Remove in 17.1 only leaving :contacted_at once hide_duplicate_runner_manager_fields_in_runner is removed
def self.deprecated_cached_attr_reader(*attributes)
attributes.each do |attribute|
define_method(attribute) do
unless self.has_attribute?(attribute)
raise ArgumentError,
"`deprecated_cached_attr_reader` requires the #{self.class.name}\##{attribute} attribute to have a database column"
end
return if Feature.enabled?(:hide_duplicate_runner_manager_fields_in_runner)
cached_attribute(attribute) || read_attribute(attribute)
end
end
end
deprecated_cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :executor_type
cached_attr_reader :contacted_at
chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout,
error_message: 'Maximum job timeout has a value which could not be accepted'
@ -253,6 +273,7 @@ module Ci
allow_nil: false,
numericality: { greater_than_or_equal_to: 0.0, message: 'needs to be non-negative' }
# TODO: Remove in 17.1. See https://gitlab.com/gitlab-org/gitlab/-/issues/415185
validates :config, json_schema: { filename: 'ci_runner_config' }
validates :maintenance_note, length: { maximum: 1024 }
@ -458,37 +479,47 @@ module Ci
# database after heartbeat write happens.
#
::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do
values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {}
if Feature.enabled?(:hide_duplicate_runner_manager_fields_in_runner)
values = {}
else
values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {}
if values.include?(:executor)
values[:executor_type] = Ci::RunnerManager::EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown)
end
new_version = values[:version]
schedule_runner_version_update(new_version) if new_version && new_version != version
end
if update_contacted_at
values.merge!(contacted_at: Time.current, creation_state: :finished)
end
if values.include?(:executor)
values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown)
end
new_version = values[:version]
schedule_runner_version_update(new_version) if new_version && new_version != version
merge_cache_attributes(values)
# We save data without validation, it will always change due to `contacted_at`
update_columns(values) if persist_cached_data?
update_columns(values) if values.any? && persist_cached_data?
end
end
def clear_heartbeat
cleared_attributes = {
version: nil,
revision: nil,
platform: nil,
architecture: nil,
ip_address: nil,
executor_type: nil,
config: {},
contacted_at: nil
}
cleared_attributes =
if Feature.enabled?(:hide_duplicate_runner_manager_fields_in_runner)
{ contacted_at: nil }
else
{
version: nil,
revision: nil,
platform: nil,
architecture: nil,
ip_address: nil,
executor_type: nil,
config: {},
contacted_at: nil
}
end
merge_cache_attributes(cleared_attributes)
update_columns(cleared_attributes)
end
@ -542,25 +573,6 @@ module Ci
joins(:runner_managers).merge(RunnerManager.with_upgrade_status(upgrade_status))
end
EXECUTOR_NAME_TO_TYPES = {
'unknown' => :unknown,
'custom' => :custom,
'shell' => :shell,
'docker' => :docker,
'docker-windows' => :docker_windows,
'docker-ssh' => :docker_ssh,
'ssh' => :ssh,
'parallels' => :parallels,
'virtualbox' => :virtualbox,
'docker+machine' => :docker_machine,
'docker-ssh+machine' => :docker_ssh_machine,
'kubernetes' => :kubernetes,
'docker-autoscaler' => :docker_autoscaler,
'instance' => :instance
}.freeze
EXECUTOR_TYPE_TO_NAMES = EXECUTOR_NAME_TO_TYPES.invert.freeze
def compute_token_expiration_instance
return unless expiration_interval = Gitlab::CurrentSettings.runner_token_expiration_interval
@ -632,6 +644,7 @@ module Ci
# For now, heartbeats with version updates might result in two Sidekiq jobs being queued if a runner has a system_id
# This is not a problem since the jobs are deduplicated on the version
def schedule_runner_version_update(new_version)
return if Feature.enabled?(:hide_duplicate_runner_manager_fields_in_runner)
return unless new_version && Gitlab::Ci::RunnerReleases.instance.enabled?
Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(new_version)

21
app/models/ci/runner_manager.rb
View File

@ -15,6 +15,25 @@ module Ci
# The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner Machine DB entry can be updated
UPDATE_CONTACT_COLUMN_EVERY = (40.minutes)..(55.minutes)
EXECUTOR_NAME_TO_TYPES = {
'unknown' => :unknown,
'custom' => :custom,
'shell' => :shell,
'docker' => :docker,
'docker-windows' => :docker_windows,
'docker-ssh' => :docker_ssh,
'ssh' => :ssh,
'parallels' => :parallels,
'virtualbox' => :virtualbox,
'docker+machine' => :docker_machine,
'docker-ssh+machine' => :docker_ssh_machine,
'kubernetes' => :kubernetes,
'docker-autoscaler' => :docker_autoscaler,
'instance' => :instance
}.freeze
EXECUTOR_TYPE_TO_NAMES = EXECUTOR_NAME_TO_TYPES.invert.freeze
belongs_to :runner
enum creation_state: {
@ -119,7 +138,7 @@ module Ci
values.merge!(contacted_at: Time.current, creation_state: :finished) if update_contacted_at
if values.include?(:executor)
values[:executor_type] = Ci::Runner::EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown)
values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown)
end
new_version = values[:version]

8
app/services/ci/runners/register_runner_service.rb
View File

@ -7,7 +7,7 @@ module Ci
def initialize(registration_token, attributes)
@registration_token = registration_token
@attributes = attributes
@attributes = attributes.except(*deprecated_columns)
end
def execute
@ -38,6 +38,12 @@ module Ci
attr_reader :registration_token, :attributes
def deprecated_columns
return [] unless Feature.enabled?(:hide_duplicate_runner_manager_fields_in_runner)
%i[version revision platform architecture ip_address executor config]
end
def attrs_from_token
if runner_registration_token_valid?(registration_token)
# Create shared runner. Requires admin access

9
config/feature_flags/gitlab_com_derisk/hide_duplicate_runner_manager_fields_in_runner.yml Normal file
View File

@ -0,0 +1,9 @@
---
name: hide_duplicate_runner_manager_fields_in_runner
feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/415185
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/149997
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/456957
milestone: '17.0'
group: group::runner
type: gitlab_com_derisk
default_enabled: false

22
db/post_migrate/20240425133709_finalize_feedback_to_state_transition_migration.rb Normal file
View File

@ -0,0 +1,22 @@
# frozen_string_literal: true
class FinalizeFeedbackToStateTransitionMigration < Gitlab::Database::Migration[2.2]
disable_ddl_transaction!
restrict_gitlab_migration gitlab_schema: :gitlab_main
milestone '17.0'
MIGRATION_NAME = "MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition"
def up
ensure_batched_background_migration_is_finished(
job_class_name: MIGRATION_NAME,
table_name: :vulnerability_feedback,
column_name: :id,
job_arguments: []
)
end
def down
# no-op
end
end

22
db/post_migrate/20240425140717_finalize_vulnerability_links_creation.rb Normal file
View File

@ -0,0 +1,22 @@
# frozen_string_literal: true
class FinalizeVulnerabilityLinksCreation < Gitlab::Database::Migration[2.2]
disable_ddl_transaction!
restrict_gitlab_migration gitlab_schema: :gitlab_main
milestone '17.0'
MIGRATION_NAME = "CreateVulnerabilityLinks"
def up
ensure_batched_background_migration_is_finished(
job_class_name: MIGRATION_NAME,
table_name: :vulnerability_feedback,
column_name: :id,
job_arguments: []
)
end
def down
# no-op
end
end

1
db/schema_migrations/20240425133709 Normal file
View File

@ -0,0 +1 @@
68aeded0d56183c859a5fc6e7269dcf18beb3b76db543eca6057af910ff8cd8d

1
db/schema_migrations/20240425140717 Normal file
View File

@ -0,0 +1 @@
454061a9360af9a5d1115b0c381d3e7c897fd6ab02853f173f5d9081308799d2

2
doc/administration/system_hooks.md
View File

@ -60,7 +60,7 @@ To create a system hook:
1. Select **System Hooks**.
1. Select **Add new webhook**.
1. Enter the **URL**.
1. Optional. Enter a [**Secret Token**](../user/project/integrations/webhooks.md#validate-payloads-by-using-a-secret-token).
1. Optional. Enter a [**Secret Token**](../user/project/integrations/webhooks.md#validate-payloads-with-a-secret-token).
1. Select the checkbox next to each optional **Trigger** you want to enable.
1. Select **Enable SSL verification**, if desired.
1. Select **Add system hook**.

5
doc/architecture/blueprints/cells/goals.md
View File

@ -55,9 +55,8 @@ We should evaluate if the small to medium business and mid-market segment is int
### Self-managed
For reasons of consistency, it is expected that self-managed instances will
adopt the cells architecture as well. To expand, self-managed instances can
continue with just a single Cell while supporting the option of adding additional
Cells. Organizations, and possible User decomposition will also be adopted for
adopt the single cell architecture as well. Self-managed instances can
continue with just a single Cell without needing the additional Cell services like routing or topology. Organizations, and possible User decomposition will also be adopted for
self-managed instances.
## Requirements

2
doc/development/audit_event_guide/index.md
View File

@ -33,7 +33,7 @@ To instrument an audit event, the following attributes should be provided:
|:-------------|:------------------------------------|:----------|:------------------------------------------------------------------|
| `name` | String | false | Action name to be audited. Represents the [type of the event](#event-type-definitions). Used for error tracking |
| `author` | User | true | User who authors the change. Can be an [internal user](../internal_users.md). For example, [inactive project deletion](../../administration/inactive_project_deletion.md) audit events are authored by `GitLab-Admin-Bot`. |
| `scope` | User, Project, Group, or InstanceScope | true | Scope which the audit event belongs to |
| `scope` | User, Project, Group, or Instance | true | Scope which the audit event belongs to |
| `target` | Object | true | Target object being audited |
| `message` | String | true | Message describing the action ([not translated](#i18n-and-the-audit-event-message-attribute)) |
| `created_at` | DateTime | false | The time when the action occurred. Defaults to `DateTime.current` |

2
doc/integration/jenkins.md
View File

@ -157,7 +157,7 @@ If you cannot [provide GitLab with your Jenkins server URL and authentication in
1. Under **Secret Token**, select **Generate**.
1. Copy the token, and save the job configuration.
1. In GitLab:
- [Create a webhook for your project](../user/project/integrations/webhooks.md#configure-a-webhook-in-gitlab).
- [Create a webhook for your project](../user/project/integrations/webhooks.md#configure-webhooks-in-gitlab).
- Enter the trigger URL (such as `https://JENKINS_URL/project/YOUR_JOB`).
- Paste the token in **Secret Token**.
1. To test the webhook, select **Test**.

2
doc/policy/experiment-beta-support.md
View File

@ -76,6 +76,7 @@ In addition to the [Experiment details](#experiment) for users, Experiments shou
- Offer a way to opt in with minimal friction. For example, needing to flip a feature flag is too much friction,
but a group or project-level setting in the UI is not.
- Respect the group-level setting for [Experiment and Beta features](../user/group/manage.md#enable-experiment-and-beta-features).
- Link out to the [GitLab Testing Agreement](https://handbook.gitlab.com/handbook/legal/testing-agreement/) in the opt-in.
- Have documentation that reflects that the feature is subject to the [GitLab Testing Agreement](https://handbook.gitlab.com/handbook/legal/testing-agreement/).
- Have [UI that reflects the Experiment status](https://design.gitlab.com/usability/feature-management#highlighting-feature-versions).
@ -95,6 +96,7 @@ In addition to the [Beta details](#beta) for users, Beta features should:
- Not be required or necessary for most features.
- Have documentation that reflects the Beta status.
- Have [UI that reflects the Beta status](https://design.gitlab.com/usability/feature-management#highlighting-feature-versions).
- Respect the group-level setting for [Experiment and Beta features](../user/group/manage.md#enable-experiment-and-beta-features).
- Be behind a feature flag that is on by default.
- Be behind a toggle that is off by default.
- Be announced in a release post that reflects the Beta status, if desired.

2
doc/user/project/integrations/index.md
View File

@ -183,7 +183,7 @@ an internal list of certificate authorities. The SSL certificate cannot
be self-signed.
You can disable SSL verification when you configure
[webhooks](webhooks.md#configure-a-webhook-in-gitlab) and some integrations.
[webhooks](webhooks.md#configure-webhooks-in-gitlab) and some integrations.
## Related topics

203
doc/user/project/integrations/webhooks.md
View File

@ -11,31 +11,20 @@ DETAILS:
**Tier:** Free, Premium, Ultimate
**Offering:** GitLab.com, Self-managed, GitLab Dedicated
[Webhooks](https://en.wikipedia.org/wiki/Webhook) are custom HTTP callbacks
that you define. They are usually triggered by an
event, such as pushing code to a repository or posting a comment on an issue.
When the event occurs, the source app makes an HTTP request to the URI
configured for the webhook. The action to take may be anything. For example,
you can use webhooks to:
Webhooks are custom HTTP callbacks triggered by an event
such as pushing code to a repository or posting a comment on an issue.
Webhooks send JSON data about the event to the URI configured for the webhook.
For more information about these events and the JSON data sent in the webhook payload,
see [webhook events](webhook_events.md).
You can use webhooks to:
- Trigger continuous integration (CI) jobs, update external issue trackers,
update a backup mirror, or deploy to your production server.
- Send a notification to
[Slack](https://api.slack.com/incoming-webhooks) every time a job fails.
- [Integrate with Twilio to be notified via SMS](https://www.datadoghq.com/blog/send-alerts-sms-customizable-webhooks-twilio/)
every time an issue is created for a specific project or group in GitLab.
- [Automatically assign labels to merge requests](https://about.gitlab.com/blog/2016/08/19/applying-gitlab-labels-automatically/).
You can configure your GitLab project or [group](#group-webhooks) to trigger a
[percent-encoded](https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding) webhook URL
when an event occurs. For example, when new code is pushed or a new issue is created. The webhook
listens for specific [events](#events) and GitLab sends a POST request with data to the webhook URL.
Usually, you set up your own [webhook receiver](#create-an-example-webhook-receiver)
to receive information from GitLab and send it to another app, according to your requirements.
We have a [built-in receiver](gitlab_slack_application.md#slack-notifications)
for sending [Slack](https://api.slack.com/incoming-webhooks) notifications per project.
GitLab.com enforces [webhook limits](../../../user/gitlab_com/index.md#webhooks),
including:
@ -59,20 +48,22 @@ specific to a group, including:
- [Group member events](webhook_events.md#group-member-events)
- [Subgroup events](webhook_events.md#subgroup-events)
## Configure a webhook in GitLab
## Configure webhooks in GitLab
To configure a webhook for a project or group:
### Create a webhook
To create a webhook for a project or group:
1. In your project or group, on the left sidebar, select **Settings > Webhooks**.
1. Select **Add new webhook**.
1. In **URL**, enter the URL of the webhook endpoint.
The URL must be percent-encoded if it contains one or more special characters.
1. In **Secret token**, enter the [secret token](#validate-payloads-by-using-a-secret-token) to validate payloads.
1. In **Secret token**, enter the [secret token](#validate-payloads-with-a-secret-token) to validate payloads.
1. In the **Trigger** section, select the [events](webhook_events.md) to trigger the webhook.
1. Optional. Clear the **Enable SSL verification** checkbox to disable [SSL verification](index.md#ssl-verification).
1. Select **Add webhook**.
## Mask sensitive portions of webhook URLs
### Mask sensitive portions of webhook URLs
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/99995) in GitLab 15.5 [with a flag](../../../administration/feature_flags.md) named `webhook_form_mask_url`. Disabled by default.
> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/376106) in GitLab 15.6.
@ -107,7 +98,7 @@ You can define URL variables directly with the REST API.
The host portion of the URL (such as `webhook.example.com`) must remain valid without using a mask variable.
Otherwise, a `URI is invalid` error occurs.
## Custom headers
### Custom headers
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/146702) in GitLab 16.11 [with a flag](../../../administration/feature_flags.md) named `custom_webhook_headers`. Enabled by default.
> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/448604) in GitLab 17.0. Feature flag `custom_webhook_headers` removed.
@ -124,7 +115,7 @@ The header name must:
Custom headers appear in [recent deliveries](#recently-triggered-webhook-payloads-in-gitlab-settings) with masked values.
## Custom webhook template
### Custom webhook template
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/142738) in GitLab 16.10 [with a flag](../../../administration/feature_flags.md) named `custom_webhook_template`. Enabled by default.
> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/439610) in GitLab 17.0. Feature flag `custom_webhook_template` removed.
@ -153,7 +144,31 @@ You'll have this request payload that combines the template with a `push` event:
}
```
## Configure your webhook receiver endpoint
### Validate payloads with a secret token
You can specify a secret token to validate received payloads.
The token is sent with the hook request in the
`X-Gitlab-Token` HTTP header. Your webhook endpoint can check the token to verify
that the request is legitimate.
### Filter push events by branch
You can filter push events by branch. Use one of the following options to filter which push events are sent to your webhook endpoint:
- **All branches**: push events from all branches.
- **Wildcard pattern**: push events from a branch that matches a wildcard pattern (for example, `*-stable` or `production/*`).
- **Regular expression**: push events from a branch that matches a regular expression (regex).
The regex pattern must follow the [RE2 syntax](https://github.com/google/re2/wiki/Syntax).
For example, to exclude `main`, you can use:
```plaintext
\b(?:m(?!ain\b)|ma(?!in\b)|mai(?!n\b)|[a-l]|[n-z])\w*|\b\w{1,3}\b|\W+
```
To configure branch filtering for a project or group, see
[Configure a webhook in GitLab](#configure-webhooks-in-gitlab)
## Webhook receiver requirements
Webhook receiver endpoints should be fast and stable.
Slow and unstable receivers might be [disabled automatically](#auto-disabled-webhooks) to ensure system reliability. Webhooks that fail might lead to [duplicate events](#webhook-fails-or-multiple-webhook-requests-are-triggered).
@ -242,11 +257,66 @@ You can also test a webhook from its edit page.
![Webhook testing](img/webhook_testing.png)
## Create an example webhook receiver
## Delivery headers
To test how GitLab webhooks work, you can use
an echo script running in a console session. For the following script to
work you must have Ruby installed.
> - `X-Gitlab-Event-UUID` header [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329743) in GitLab 14.8.
> - `X-Gitlab-Instance` header [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31333) in GitLab 15.5.
> - `X-Gitlab-Webhook-UUID` header [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230830) in GitLab 16.2.
Webhook requests to your endpoint include the following headers:
| Header | Description | Example |
| ------ | ------ | ------ |
| `User-Agent` | User agent in the format `"Gitlab/<VERSION>"`. | `"GitLab/15.5.0-pre"` |
| `X-Gitlab-Instance` | Hostname of the GitLab instance that sent the webhook. | `"https://gitlab.com"` |
| `X-Gitlab-Webhook-UUID` | Unique ID per webhook. | `"02affd2d-2cba-4033-917d-ec22d5dc4b38"` |
| `X-Gitlab-Event` | Name of the webhook type. Corresponds to [event types](webhook_events.md) but in the format `"<EVENT> Hook"`. | `"Push Hook"` |
| `X-Gitlab-Event-UUID` | Unique ID per webhook that is not recursive. A hook is recursive if triggered by an earlier webhook that hit the GitLab instance. Recursive webhooks have the same value for this header. | `"13792a34-cac6-4fda-95a8-c58e00a3954e"` |
## Debug webhooks
To debug GitLab webhooks and capture payloads, you can use:
- [Public webhook inspection and testing tools](#public-webhook-inspection-and-testing-tools)
- [The GitLab Development Kit (GDK)](#gitlab-development-kit-gdk)
- [Recently triggered webhook payloads in GitLab settings](#recently-triggered-webhook-payloads-in-gitlab-settings)
- [A private webhook receiver](#create-a-private-webhook-receiver)
For information about webhook events and the JSON data sent in the webhook payload,
see [webhook events](webhook_events.md).
### Public webhook inspection and testing tools
You can use public tools to inspect and test webhook payloads. These tools act as catch-all endpoints for HTTP requests and respond with a `200 OK` HTTP status code. You can use these payloads to develop your webhook services.
You should exercise caution when using these tools as you might be sending sensitive data to external tools.
You should use test tokens with these tools and rotate any secrets inadvertently sent to a third party.
To keep your webhook payloads private, [create a private webhook receiver](#create-a-private-webhook-receiver) instead.
These public tools include:
- [Beeceptor](https://beeceptor.com) to create a temporary HTTPS endpoint and inspect incoming payloads
- [Webhook.site](https://webhook.site) to review incoming payloads
- [Webhook Tester](https://webhook-test.com) to inspect and debug incoming payloads
### GitLab Development Kit (GDK)
For a safer development environment, you can use the [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit) to develop against GitLab webhooks locally. With the GDK, you can send webhooks from your local GitLab instance to a webhook receiver running locally on your machine. To use this approach, you must install and configure the GDK.
### Recently triggered webhook payloads in GitLab settings
You can [review recently triggered webhook payloads](#troubleshooting) in GitLab settings. For each webhook event, a detail page exists with information about the data GitLab sends and receives from the webhook endpoint.
### Create a private webhook receiver
Prerequisites:
- You must have Ruby installed.
If you cannot send webhook payloads to a [public receiver](#public-webhook-inspection-and-testing-tools),
you can create your own private webhook receiver.
To create a private webhook receiver:
1. Save the following file as `print_http_body.rb`:
@ -270,7 +340,7 @@ work you must have Ruby installed.
ruby print_http_body.rb 8000
```
1. In GitLab, [configure the webhook](#configure-a-webhook-in-gitlab) and add your
1. In GitLab, [configure the webhook](#configure-webhooks-in-gitlab) and add your
receiver's URL, for example, `http://receiver.example.com:8000/`.
1. Select **Test**. You should see something like this in the console:
@ -285,30 +355,6 @@ NOTE:
You may need to [allow requests to the local network](../../../security/webhooks.md) for this
receiver to be added.
## Validate payloads by using a secret token
You can specify a secret token to validate received payloads.
The token is sent with the hook request in the
`X-Gitlab-Token` HTTP header. Your webhook endpoint can check the token to verify
that the request is legitimate.
## Filter push events by branch
You can filter push events by branch. Use one of the following options to filter which push events are sent to your webhook endpoint:
- **All branches**: push events from all branches.
- **Wildcard pattern**: push events from a branch that matches a wildcard pattern (for example, `*-stable` or `production/*`).
- **Regular expression**: push events from a branch that matches a regular expression (regex).
The regex pattern must follow the [RE2 syntax](https://github.com/google/re2/wiki/Syntax).
For example, to exclude `main`, you can use:
```plaintext
\b(?:m(?!ain\b)|ma(?!in\b)|mai(?!n\b)|[a-l]|[n-z])\w*|\b\w{1,3}\b|\W+
```
To configure branch filtering for a project or group, see
[Configure a webhook in GitLab](#configure-a-webhook-in-gitlab).
## How image URLs are displayed in the webhook body
Relative image references are rewritten to use an absolute URL
@ -337,54 +383,6 @@ Image URLs are not rewritten if:
protocol-relative URLs.
- They use advanced Markdown features like link labels.
## Events
For more information about supported events for webhooks, see [webhook events](webhook_events.md).
## Delivery headers
> - `X-Gitlab-Event-UUID` header [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329743) in GitLab 14.8.
> - `X-Gitlab-Instance` header [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31333) in GitLab 15.5.
> - `X-Gitlab-Webhook-UUID` header [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230830) in GitLab 16.2.
Webhook requests to your endpoint include the following headers:
| Header | Description | Example |
| ------ | ------ | ------ |
| `User-Agent` | User agent in the format `"Gitlab/<VERSION>"`. | `"GitLab/15.5.0-pre"` |
| `X-Gitlab-Instance` | Hostname of the GitLab instance that sent the webhook. | `"https://gitlab.com"` |
| `X-Gitlab-Webhook-UUID` | Unique ID per webhook. | `"02affd2d-2cba-4033-917d-ec22d5dc4b38"` |
| `X-Gitlab-Event` | Name of the webhook type. Corresponds to [event types](webhook_events.md) but in the format `"<EVENT> Hook"`. | `"Push Hook"` |
| `X-Gitlab-Event-UUID` | Unique ID per webhook that is not recursive. A hook is recursive if triggered by an earlier webhook that hit the GitLab instance. Recursive webhooks have the same value for this header. | `"13792a34-cac6-4fda-95a8-c58e00a3954e"` |
## Develop webhooks
If you don't have an existing HTTPS endpoint or URL for your webhook setup, you must deploy one on a server. This HTTPS endpoint is used in configuration. To develop against GitLab webhooks and capture the payloads, you can use:
- [Public webhook inspection and testing tools](#public-webhook-inspection-and-testing-tools)
- [The GitLab Development Kit (GDK)](#gitlab-development-kit-gdk)
- [Recently triggered webhook payloads in GitLab settings](#recently-triggered-webhook-payloads-in-gitlab-settings)
### Public webhook inspection and testing tools
You can use public tools to inspect and test webhook payloads. These tools act as catch-all endpoints for HTTP requests and respond with a `200 OK` HTTP status code. You can use these payloads to develop your webhook services.
You should exercise caution when using these tools as you might be sending sensitive data to external tools. You should use test tokens with these tools and rotate any secrets inadvertently sent to a third party.
These public tools include:
- [Beeceptor](https://beeceptor.com) to create a temporary HTTPS endpoint and inspect incoming payloads
- [Webhook.site](https://webhook.site) to review incoming payloads
- [Webhook Tester](https://webhook-test.com) to inspect and debug incoming payloads
### GitLab Development Kit (GDK)
For a safer development environment, you can use the [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit) to develop against GitLab webhooks locally. With the GDK, you can send webhooks from your local GitLab instance to a webhook receiver running locally on your machine. To use this approach, you must install and configure the GDK.
### Recently triggered webhook payloads in GitLab settings
You can [review recently triggered webhook payloads](#troubleshooting) in GitLab settings. For each webhook event, a detail page exists with information about the data GitLab sends and receives from the webhook endpoint.
## Configure webhooks to support mutual TLS
DETAILS:
@ -468,6 +466,7 @@ To configure the certificate, follow the instructions below.
## Related topics
- [Webhook events and webhook JSON payloads](webhook_events.md)
- [Project hooks API](../../../api/projects.md#hooks)
- [Group hooks API](../../../api/groups.md#hooks)
- [System hooks API](../../../api/system_hooks.md)

2
lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb
View File

@ -200,7 +200,7 @@ module Gitlab
scope_name = scope.class.name if scope
logger.info(message: 'Creating runner', scope: scope_name, name: name)
executor = ::Ci::Runner::EXECUTOR_NAME_TO_TYPES.keys.sample
executor = ::Ci::RunnerManager::EXECUTOR_NAME_TO_TYPES.keys.sample
args.merge!(additional_runner_args(name, executor))
runners_token = if scope.nil?

2
lib/tasks/gems.rake
View File

@ -91,7 +91,7 @@ namespace :gems do
end
def generated_by(task)
"Generated by `rake #{task.name}` on #{Time.now.strftime('%Y-%m-%d')}" # rubocop:disable Rails/TimeZone
"Generated by `rake #{task.name}` on #{Date.current.iso8601}"
end
def license

2
qa/qa/page/profile/ssh_keys.rb
View File

@ -37,7 +37,7 @@ module QA
end
def fill_expiry_date(date)
date = date.strftime('%Y-%m-%d') if date.is_a?(Date)
date = date.iso8601 if date.is_a?(Date)
begin
Date.strptime(date, '%Y-%m-%d')
rescue ArgumentError

9
spec/frontend/vue_shared/components/visibility_icon_spec.js
View File

@ -56,10 +56,11 @@ describe('Visibility icon', () => {
describe('if item represents project', () => {
it.each`
visibilityLevel | visibilityTooltip | visibilityIcon | tooltipPlacement
${VISIBILITY_LEVEL_PUBLIC_STRING} | ${PROJECT_VISIBILITY_TYPE[VISIBILITY_LEVEL_PUBLIC_STRING]} | ${VISIBILITY_TYPE_ICON[VISIBILITY_LEVEL_PUBLIC_STRING]} | ${'top'}
${VISIBILITY_LEVEL_INTERNAL_STRING} | ${PROJECT_VISIBILITY_TYPE[VISIBILITY_LEVEL_INTERNAL_STRING]} | ${VISIBILITY_TYPE_ICON[VISIBILITY_LEVEL_INTERNAL_STRING]} | ${'bottom'}
${VISIBILITY_LEVEL_PRIVATE_STRING} | ${PROJECT_VISIBILITY_TYPE[VISIBILITY_LEVEL_PRIVATE_STRING]} | ${VISIBILITY_TYPE_ICON[VISIBILITY_LEVEL_PRIVATE_STRING]} | ${'left'}
visibilityLevel | visibilityTooltip | visibilityIcon | tooltipPlacement
${VISIBILITY_LEVEL_PUBLIC_STRING} | ${PROJECT_VISIBILITY_TYPE[VISIBILITY_LEVEL_PUBLIC_STRING]} | ${VISIBILITY_TYPE_ICON[VISIBILITY_LEVEL_PUBLIC_STRING]} | ${'top'}
${VISIBILITY_LEVEL_INTERNAL_STRING} | ${PROJECT_VISIBILITY_TYPE[VISIBILITY_LEVEL_INTERNAL_STRING]} | ${VISIBILITY_TYPE_ICON[VISIBILITY_LEVEL_INTERNAL_STRING]} | ${'bottom'}
${VISIBILITY_LEVEL_PRIVATE_STRING} | ${PROJECT_VISIBILITY_TYPE[VISIBILITY_LEVEL_PRIVATE_STRING]} | ${VISIBILITY_TYPE_ICON[VISIBILITY_LEVEL_PRIVATE_STRING]} | ${'left'}
${'banned'} | ${'This project is hidden because its creator has been banned'} | ${'spam'} | ${'right'}
`(
'should return corresponding text when visibility level is $visibilityLevel',
({ visibilityLevel, visibilityTooltip, visibilityIcon, tooltipPlacement }) => {

6
spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb
View File

@ -42,13 +42,13 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do
end
def create_partition(name:, from:, to:, attached:, drop_after:, table: :_test_parent_table)
from = from.beginning_of_month
to = to.beginning_of_month
from = from.beginning_of_month.to_date
to = to.beginning_of_month.to_date
full_name = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{name}"
connection.execute(<<~SQL)
CREATE TABLE #{full_name}
PARTITION OF #{table}
FOR VALUES FROM ('#{from.strftime('%Y-%m-%d')}') TO ('#{to.strftime('%Y-%m-%d')}')
FOR VALUES FROM ('#{from.iso8601}') TO ('#{to.iso8601}')
SQL
unless attached

2
spec/lib/gitlab/dependency_linker/gemspec_linker_spec.rb
View File

@ -21,7 +21,7 @@ RSpec.describe Gitlab::DependencyLinker::GemspecLinker do
Gem::Specification.new do |s|
s.name = 'gitlab_git'
s.version = `cat VERSION`
s.date = Time.now.strftime('%Y-%m-%d')
s.date = Date.current.iso8601
s.summary = "Gitlab::Git library"
s.description = "GitLab wrapper around git objects"
s.authors = ["Dmitriy Zaporozhets"]

2
spec/models/ci/runner_manager_spec.rb
View File

@ -497,7 +497,7 @@ RSpec.describe Ci::RunnerManager, feature_category: :fleet_visibility, type: :mo
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).not_to have_received(:perform_async)
end
Ci::Runner::EXECUTOR_NAME_TO_TYPES.each_key do |executor|
described_class::EXECUTOR_NAME_TO_TYPES.each_key do |executor|
context "with #{executor} executor" do
let(:executor) { executor }

229
spec/models/ci/runner_spec.rb
View File

@ -575,10 +575,6 @@ RSpec.describe Ci::Runner, type: :model, feature_category: :runner do
before do
allow_any_instance_of(described_class).to receive(:cached_attribute).and_call_original
allow_any_instance_of(described_class).to receive(:cached_attribute)
.with(:platform).and_return("darwin")
allow_any_instance_of(described_class).to receive(:cached_attribute)
.with(:version).and_return("14.0.0")
allow(Ci::Runners::ProcessRunnerVersionUpdateWorker).to receive(:perform_async).once
end
@ -637,12 +633,6 @@ RSpec.describe Ci::Runner, type: :model, feature_category: :runner do
before do
allow_any_instance_of(described_class).to receive(:cached_attribute).and_call_original
allow_any_instance_of(described_class).to receive(:cached_attribute)
.with(:platform).and_return("darwin")
allow_any_instance_of(described_class).to receive(:cached_attribute)
.with(:version).and_return("14.0.0")
allow(Ci::Runners::ProcessRunnerVersionUpdateWorker).to receive(:perform_async).once
end
context 'no cache value' do
@ -1060,45 +1050,72 @@ RSpec.describe Ci::Runner, type: :model, feature_category: :runner do
expect(runner.runner_version).to be_nil
end
it 'schedules version information update' do
it 'does not schedule version information update' do
heartbeat
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).to have_received(:perform_async).with(version).once
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).not_to have_received(:perform_async)
end
context 'when fetching runner releases is disabled' do
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_application_setting(update_runner_versions_enabled: false)
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
it 'does not schedule version information update' do
it 'updates cache' do
expect_redis_update
heartbeat
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).not_to have_received(:perform_async)
expect(runner.runner_version).to be_nil
end
it 'schedules version information update' do
heartbeat
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).to have_received(:perform_async)
.with(version)
end
context 'when fetching runner releases is disabled' do
before do
stub_application_setting(update_runner_versions_enabled: false)
end
it 'does not schedule version information update' do
heartbeat
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).not_to have_received(:perform_async)
end
end
end
end
context 'with only ip_address specified', :freeze_time do
let(:values) do
{ ip_address: '1.1.1.1' }
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
it 'updates only ip_address' do
expect_redis_update(values.merge(contacted_at: Time.current, creation_state: :finished))
heartbeat
end
context 'with new version having been cached' do
let(:version) { '15.0.1' }
before do
runner.cache_attributes(version: version)
context 'with only ip_address specified', :freeze_time do
let(:values) do
{ ip_address: '1.1.1.1' }
end
it 'does not lose cached version value' do
expect { heartbeat }.not_to change { runner.version }.from(version)
it 'updates only ip_address' do
expect_redis_update(values.merge(contacted_at: Time.current, creation_state: :finished))
heartbeat
end
context 'with new version having been cached' do
let(:version) { '15.0.1' }
before do
runner.cache_attributes(version: version)
end
it 'does not lose cached version value' do
expect { heartbeat }.not_to change { runner.version }.from(version)
end
end
end
end
@ -1119,20 +1136,54 @@ RSpec.describe Ci::Runner, type: :model, feature_category: :runner do
runner.runner_projects.delete_all
end
it 'still updates redis cache and database' do
it 'still updates contacted at in redis cache and database' do
expect(runner).to be_invalid
expect_redis_update
does_db_update
expect_redis_update(contacted_at: Time.current, creation_state: :finished)
expect { heartbeat }.to change { runner.reload.read_attribute(:contacted_at) }
.and not_change { runner.reload.read_attribute(:config) }
.and not_change { runner.reload.runner_version }
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).to have_received(:perform_async).with(version).once
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).not_to have_received(:perform_async)
end
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
it 'still updates redis cache and database' do
expect(runner).to be_invalid
expect_redis_update
does_db_update
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).to have_received(:perform_async).with(version).once
end
end
end
it 'updates redis cache and database' do
expect_redis_update
does_db_update
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).to have_received(:perform_async).with(version).once
it 'only updates contacted at in redis cache and database' do
expect_redis_update(contacted_at: Time.current, creation_state: :finished)
expect { heartbeat }.to change { runner.reload.read_attribute(:contacted_at) }
.and not_change { runner.reload.read_attribute(:config) }
.and not_change { runner.reload.runner_version }
.and not_change { runner.reload.read_attribute(:architecture) }
.and not_change { runner.reload.read_attribute(:executor_type) }
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).not_to have_received(:perform_async)
end
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
it 'updates redis cache and database' do
expect_redis_update
does_db_update
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).to have_received(:perform_async).with(version).once
end
end
end
@ -1145,33 +1196,47 @@ RSpec.describe Ci::Runner, type: :model, feature_category: :runner do
expect(Ci::Runners::ProcessRunnerVersionUpdateWorker).not_to have_received(:perform_async)
end
Ci::Runner::EXECUTOR_NAME_TO_TYPES.each_key do |executor|
context "with #{executor} executor" do
let(:executor) { executor }
it 'does not update with the specified executor type' do
expect_redis_update
it 'updates with expected executor type' do
heartbeat
expect(runner.reload.read_attribute(:executor_type)).to be_nil
end
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
Ci::RunnerManager::EXECUTOR_NAME_TO_TYPES.each_key do |executor|
context "with #{executor} executor" do
let(:executor) { executor }
it 'updates with expected executor type' do
expect_redis_update
heartbeat
expect(runner.reload.read_attribute(:executor_type)).to eq(expected_executor_type)
end
def expected_executor_type
executor.gsub(/[+-]/, '_')
end
end
end
context 'with an unknown executor type' do
let(:executor) { 'some-unknown-type' }
it 'updates with unknown executor type' do
expect_redis_update
heartbeat
expect(runner.reload.read_attribute(:executor_type)).to eq(expected_executor_type)
expect(runner.reload.read_attribute(:executor_type)).to eq('unknown')
end
def expected_executor_type
executor.gsub(/[+-]/, '_')
end
end
end
context 'with an unknown executor type' do
let(:executor) { 'some-unknown-type' }
it 'updates with unknown executor type' do
expect_redis_update
heartbeat
expect(runner.reload.read_attribute(:executor_type)).to eq('unknown')
end
end
end
@ -1208,28 +1273,46 @@ RSpec.describe Ci::Runner, type: :model, feature_category: :runner do
}
end
let(:expected_attributes) { heartbeat_values.except(:executor).merge(executor_type: 'shell') }
let(:expected_cleared_attributes) { expected_attributes.to_h { |key, _| [key, nil] }.merge(config: {}) }
it 'clears contacted at and other attributes' do
it 'clears contacted at' do
expect do
runner.heartbeat(heartbeat_values)
end.to change { runner.reload.contacted_at }.from(nil).to(Time.current)
.and change { runner.reload.uncached_contacted_at }.from(nil).to(Time.current)
expected_attributes.each do |key, value|
expect(runner.public_send(key)).to eq(value)
expect(runner.read_attribute(key)).to eq(value)
end
expect do
runner.clear_heartbeat
end.to change { runner.reload.contacted_at }.from(Time.current).to(nil)
.and change { runner.reload.uncached_contacted_at }.from(Time.current).to(nil)
end
expected_cleared_attributes.each do |key, value|
expect(runner.public_send(key)).to eq(value)
expect(runner.read_attribute(key)).to eq(value)
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
let(:expected_attributes) { heartbeat_values.except(:executor).merge(executor_type: 'shell') }
let(:expected_cleared_attributes) { expected_attributes.to_h { |key, _| [key, nil] }.merge(config: {}) }
it 'clears contacted at and other attributes' do
expect do
runner.heartbeat(heartbeat_values)
end.to change { runner.reload.contacted_at }.from(nil).to(Time.current)
.and change { runner.reload.uncached_contacted_at }.from(nil).to(Time.current)
expected_attributes.each do |key, value|
expect(runner.public_send(key)).to eq(value)
expect(runner.read_attribute(key)).to eq(value)
end
expect do
runner.clear_heartbeat
end.to change { runner.reload.contacted_at }.from(Time.current).to(nil)
.and change { runner.reload.uncached_contacted_at }.from(Time.current).to(nil)
expected_cleared_attributes.each do |key, value|
expect(runner.public_send(key)).to eq(value)
expect(runner.read_attribute(key)).to eq(value)
end
end
end
end

132
spec/requests/api/ci/runner/jobs_request_post_spec.rb
View File

@ -270,7 +270,7 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state, feature_catego
expect(response).to have_gitlab_http_status(:created)
expect(response.headers['Content-Type']).to eq('application/json')
expect(response.headers).not_to have_key('X-GitLab-Last-Update')
expect(runner.reload.platform).to eq('darwin')
expect(runner.reload.runner_managers.last.platform).to eq('darwin')
expect(json_response['id']).to eq(job.id)
expect(json_response['token']).to eq(job.token)
expect(json_response['job_info']).to include(expected_job_info)
@ -475,52 +475,100 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state, feature_catego
end
end
it 'updates runner info' do
expect { request_job }.to change { runner.reload.contacted_at }
end
describe 'updates runner info' do
it { expect { request_job }.to change { runner.reload.contacted_at } }
%w[version revision platform architecture].each do |param|
context "when info parameter '#{param}' is present" do
let(:value) { "#{param}_value" }
%w[version revision platform architecture].each do |param|
context "when info parameter '#{param}' is present" do
let(:value) { "#{param}_value" }
it "updates provided Runner's parameter" do
request_job info: { param => value }
it "updates provided Runner's parameter" do
request_job info: { param => value }
expect(response).to have_gitlab_http_status(:created)
expect(runner.reload.read_attribute(param.to_sym)).to eq(value)
expect(response).to have_gitlab_http_status(:created)
expect(job.runner_manager.reload.read_attribute(param.to_sym)).to eq(value)
end
end
end
end
it "sets the runner's config" do
request_job info: { 'config' => { 'gpus' => 'all', 'ignored' => 'hello' } }
it "sets the runner's config" do
request_job info: { 'config' => { 'gpus' => 'all', 'ignored' => 'hello' } }
expect(response).to have_gitlab_http_status(:created)
expect(runner.reload.config).to eq({ 'gpus' => 'all' })
end
expect(response).to have_gitlab_http_status(:created)
expect(job.runner_manager.reload.config).to eq({ 'gpus' => 'all' })
end
it "sets the runner's ip_address" do
post api('/jobs/request'),
params: { token: runner.token },
headers: { 'User-Agent' => user_agent, 'X-Forwarded-For' => '123.222.123.222' }
it "sets the runner's ip_address" do
post api('/jobs/request'),
params: { token: runner.token },
headers: { 'User-Agent' => user_agent, 'X-Forwarded-For' => '123.222.123.222' }
expect(response).to have_gitlab_http_status(:created)
expect(runner.reload.ip_address).to eq('123.222.123.222')
end
expect(response).to have_gitlab_http_status(:created)
expect(job.runner_manager.reload.ip_address).to eq('123.222.123.222')
end
it "handles multiple X-Forwarded-For addresses" do
post api('/jobs/request'),
params: { token: runner.token },
headers: { 'User-Agent' => user_agent, 'X-Forwarded-For' => '123.222.123.222, 127.0.0.1' }
it "handles multiple X-Forwarded-For addresses" do
post api('/jobs/request'),
params: { token: runner.token },
headers: { 'User-Agent' => user_agent, 'X-Forwarded-For' => '123.222.123.222, 127.0.0.1' }
expect(response).to have_gitlab_http_status(:created)
expect(runner.reload.ip_address).to eq('123.222.123.222')
expect(response).to have_gitlab_http_status(:created)
expect(job.runner_manager.reload.ip_address).to eq('123.222.123.222')
end
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
%w[version revision platform architecture].each do |param|
context "when info parameter '#{param}' is present" do
let(:value) { "#{param}_value" }
it "updates provided Runner's parameter" do
request_job info: { param => value }
expect(response).to have_gitlab_http_status(:created)
expect(runner.reload.read_attribute(param.to_sym)).to eq(value)
expect(job.runner_manager.reload.read_attribute(param.to_sym)).to eq(value)
end
end
end
it "sets the runner's config" do
request_job info: { 'config' => { 'gpus' => 'all', 'ignored' => 'hello' } }
expect(response).to have_gitlab_http_status(:created)
expect(runner.reload.config).to eq({ 'gpus' => 'all' })
expect(job.runner_manager.reload.config).to eq({ 'gpus' => 'all' })
end
it "sets the runner's ip_address" do
post api('/jobs/request'),
params: { token: runner.token },
headers: { 'User-Agent' => user_agent, 'X-Forwarded-For' => '123.222.123.222' }
expect(response).to have_gitlab_http_status(:created)
expect(runner.reload.ip_address).to eq('123.222.123.222')
expect(job.runner_manager.reload.ip_address).to eq('123.222.123.222')
end
it "handles multiple X-Forwarded-For addresses" do
post api('/jobs/request'),
params: { token: runner.token },
headers: { 'User-Agent' => user_agent, 'X-Forwarded-For' => '123.222.123.222, 127.0.0.1' }
expect(response).to have_gitlab_http_status(:created)
expect(runner.reload.ip_address).to eq('123.222.123.222')
expect(job.runner_manager.reload.ip_address).to eq('123.222.123.222')
end
end
end
context 'when concurrently updating a job' do
before do
expect_any_instance_of(::Ci::Build).to receive(:run!)
.and_raise(ActiveRecord::StaleObjectError.new(nil, nil))
.and_raise(ActiveRecord::StaleObjectError.new(nil, nil))
end
it 'returns a conflict' do
@ -695,13 +743,33 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state, feature_catego
expect(response).to have_gitlab_http_status(:created)
expect(response.headers['Content-Type']).to eq('application/json')
expect(response.headers).not_to have_key('X-GitLab-Last-Update')
expect(runner.reload.platform).to eq('darwin')
expect(runner.reload.runner_managers.last.platform).to eq('darwin')
expect(json_response['id']).to eq(job.id)
expect(json_response['token']).to eq(job.token)
expect(json_response['job_info']).to include(expected_job_info)
expect(json_response['git_info']).to eq(expected_git_info)
expect(json_response['artifacts']).to eq(expected_artifacts)
end
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
it 'returns job with the correct artifact specification', :aggregate_failures do
request_job info: { platform: :darwin, features: { upload_multiple_artifacts: true } }
expect(response).to have_gitlab_http_status(:created)
expect(response.headers['Content-Type']).to eq('application/json')
expect(response.headers).not_to have_key('X-GitLab-Last-Update')
expect(runner.reload.platform).to eq('darwin')
expect(json_response['id']).to eq(job.id)
expect(json_response['token']).to eq(job.token)
expect(json_response['job_info']).to include(expected_job_info)
expect(json_response['git_info']).to eq(expected_git_info)
expect(json_response['artifacts']).to eq(expected_artifacts)
end
end
end
context 'when triggered job is available' do

40
spec/requests/api/ci/runner/runners_post_spec.rb
View File

@ -167,29 +167,35 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state, feature_catego
allow_any_instance_of(::Ci::Runner).to receive(:cache_attributes)
end
%w[name version revision platform architecture].each do |param|
context "when info parameter '#{param}' info is present" do
let(:value) { "#{param}_value" }
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
it "updates provided Runner's parameter" do
post api('/runners'), params: {
token: registration_token,
info: { param => value }
}
%w[name version revision platform architecture].each do |param|
context "when info parameter '#{param}' info is present" do
let(:value) { "#{param}_value" }
expect(response).to have_gitlab_http_status(:created)
expect(::Ci::Runner.last.read_attribute(param.to_sym)).to eq(value)
it "updates provided Runner's parameter" do
post api('/runners'), params: {
token: registration_token,
info: { param => value }
}
expect(response).to have_gitlab_http_status(:created)
expect(::Ci::Runner.last.read_attribute(param.to_sym)).to eq(value)
end
end
end
end
it "sets the runner's ip_address" do
post api('/runners'),
params: { token: registration_token },
headers: { 'X-Forwarded-For' => '123.111.123.111' }
it "sets the runner's ip_address" do
post api('/runners'),
params: { token: registration_token },
headers: { 'X-Forwarded-For' => '123.111.123.111' }
expect(response).to have_gitlab_http_status(:created)
expect(::Ci::Runner.last.ip_address).to eq('123.111.123.111')
expect(response).to have_gitlab_http_status(:created)
expect(::Ci::Runner.last.ip_address).to eq('123.111.123.111')
end
end
context 'when tags parameter is provided' do

7
spec/requests/api/ci/runners_spec.rb
View File

@ -152,12 +152,11 @@ RSpec.describe API::Ci::Runners, :aggregate_failures, feature_category: :fleet_v
it 'filters runners by scope' do
get api('/runners/all?scope=shared', admin, admin_mode: true)
shared = json_response.all? { |r| r['is_shared'] }
expect(response).to have_gitlab_http_status(:ok)
expect(response).to include_pagination_headers
expect(json_response).to be_an Array
expect(json_response[0]).to have_key('ip_address')
expect(shared).to be_truthy
expect(json_response).to contain_exactly(
a_hash_including('description' => 'Shared runner', 'is_shared' => true)
)
end
it 'filters runners by scope' do

2
spec/requests/api/graphql/mutations/issues/create_spec.rb
View File

@ -20,7 +20,7 @@ RSpec.describe 'Create an issue', feature_category: :team_planning do
'title' => 'new title',
'description' => 'new description',
'confidential' => true,
'dueDate' => Date.tomorrow.strftime('%Y-%m-%d'),
'dueDate' => Date.tomorrow.iso8601,
'type' => 'ISSUE'
}
end

2
spec/requests/api/graphql/mutations/issues/update_spec.rb
View File

@ -17,7 +17,7 @@ RSpec.describe 'Update of an existing issue', feature_category: :team_planning d
'title' => 'new title',
'description' => 'new description',
'confidential' => true,
'dueDate' => Date.tomorrow.strftime('%Y-%m-%d'),
'dueDate' => Date.tomorrow.iso8601,
'type' => 'ISSUE'
}
end

2
spec/requests/api/graphql/mutations/work_items/update_spec.rb
View File

@ -893,7 +893,7 @@ RSpec.describe 'Update a work item', feature_category: :team_planning do
expect(mutation_response['workItem']['title']).to eq('Foo')
expect(mutation_response['workItem']['widgets']).to include(
'type' => 'START_AND_DUE_DATE',
'dueDate' => Date.tomorrow.strftime('%Y-%m-%d'),
'dueDate' => Date.tomorrow.iso8601,
'startDate' => nil
)
end

2
spec/requests/api/issues/post_projects_issues_spec.rb
View File

@ -309,7 +309,7 @@ RSpec.describe API::Issues, :aggregate_failures, feature_category: :team_plannin
context 'with due date' do
it 'creates a new project issue' do
due_date = 2.weeks.from_now.strftime('%Y-%m-%d')
due_date = 2.weeks.from_now.to_date.iso8601
post api("/projects/#{project.id}/issues", user),
params: { title: 'new issue', due_date: due_date }

2
spec/requests/api/issues/put_projects_issues_spec.rb
View File

@ -449,7 +449,7 @@ RSpec.describe API::Issues, feature_category: :team_planning do
describe 'PUT /projects/:id/issues/:issue_iid to update due date' do
it 'creates a new project issue', :aggregate_failures do
due_date = 2.weeks.from_now.strftime('%Y-%m-%d')
due_date = 2.weeks.from_now.to_date.iso8601
put api_for_user, params: { due_date: due_date }

42
spec/services/ci/runners/register_runner_service_spec.rb
View File

@ -89,7 +89,7 @@ RSpec.describe ::Ci::Runners::RegisterRunnerService, '#execute', feature_categor
}
end
it 'creates runner with specified values', :aggregate_failures do
it 'creates runner with relevant values', :aggregate_failures do
expect(execute).to be_success
expect(runner).to be_an_instance_of(::Ci::Runner)
@ -103,15 +103,45 @@ RSpec.describe ::Ci::Runners::RegisterRunnerService, '#execute', feature_categor
expect(runner.access_level).to eq args[:access_level]
expect(runner.maximum_timeout).to eq args[:maximum_timeout]
expect(runner.name).to eq args[:name]
expect(runner.version).to eq args[:version]
expect(runner.revision).to eq args[:revision]
expect(runner.platform).to eq args[:platform]
expect(runner.architecture).to eq args[:architecture]
expect(runner.ip_address).to eq args[:ip_address]
expect(runner.version).to be_nil
expect(runner.revision).to be_nil
expect(runner.platform).to be_nil
expect(runner.architecture).to be_nil
expect(runner.ip_address).to be_nil
expect(Ci::Runner.tagged_with('tag1')).to include(runner)
expect(Ci::Runner.tagged_with('tag2')).to include(runner)
end
context 'when hide_duplicate_runner_manager_fields_in_runner FF is disabled' do
before do
stub_feature_flags(hide_duplicate_runner_manager_fields_in_runner: false)
end
it 'creates runner with specified values', :aggregate_failures do
expect(execute).to be_success
expect(runner).to be_an_instance_of(::Ci::Runner)
expect(runner.active).to eq args[:active]
expect(runner.locked).to eq args[:locked]
expect(runner.run_untagged).to eq args[:run_untagged]
expect(runner.tags).to contain_exactly(
an_object_having_attributes(name: 'tag1'),
an_object_having_attributes(name: 'tag2')
)
expect(runner.access_level).to eq args[:access_level]
expect(runner.maximum_timeout).to eq args[:maximum_timeout]
expect(runner.name).to eq args[:name]
expect(runner.version).to eq args[:version]
expect(runner.revision).to eq args[:revision]
expect(runner.platform).to eq args[:platform]
expect(runner.architecture).to eq args[:architecture]
expect(runner.ip_address).to eq args[:ip_address]
expect(Ci::Runner.tagged_with('tag1')).to include(runner)
expect(Ci::Runner.tagged_with('tag2')).to include(runner)
end
end
end
context 'with runner token expiration interval', :freeze_time do

2
spec/support/helpers/features/invite_members_modal_helpers.rb
View File

@ -70,7 +70,7 @@ module Features
toggle_listbox
select_listbox_item(role, exact_text: true)
end
fill_in 'YYYY-MM-DD', with: expires_at.strftime('%Y-%m-%d') if expires_at
fill_in 'YYYY-MM-DD', with: expires_at.to_date.iso8601 if expires_at
end
def click_groups_tab

17
workhorse/internal/upload/artifacts_store_test.go
View File

@ -1,3 +1,4 @@
// Package upload contains tests for artifact storage functionality.
package upload
import (
@ -21,6 +22,10 @@ import (
"gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test"
)
const (
putURL = "/url/put"
)
func createTestZipArchive(t *testing.T) (data []byte, md5Hash string) {
var buffer bytes.Buffer
archive := zip.NewWriter(&buffer)
@ -67,7 +72,7 @@ func TestUploadHandlerSendingToExternalStorage(t *testing.T) {
storeServerCalled := 0
storeServerMux := http.NewServeMux()
storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
storeServerMux.HandleFunc(putURL, func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, "PUT", r.Method)
receivedData, err := io.ReadAll(r.Body)
@ -103,7 +108,7 @@ func TestUploadHandlerSendingToExternalStorage(t *testing.T) {
name: "ObjectStore Upload",
preauth: &api.Response{
RemoteObject: api.RemoteObject{
StoreURL: storeServer.URL + "/url/put" + qs,
StoreURL: storeServer.URL + putURL + qs,
ID: "store-id",
GetURL: storeServer.URL + "/store-id",
},
@ -177,7 +182,7 @@ func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T)
putCalledTimes := 0
storeServerMux := http.NewServeMux()
storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
storeServerMux.HandleFunc(putURL, func(w http.ResponseWriter, r *http.Request) {
putCalledTimes++
require.Equal(t, "PUT", r.Method)
w.WriteHeader(510)
@ -192,7 +197,7 @@ func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T)
authResponse := &api.Response{
RemoteObject: api.RemoteObject{
StoreURL: storeServer.URL + "/url/put",
StoreURL: storeServer.URL + putURL,
ID: "store-id",
},
}
@ -208,7 +213,7 @@ func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T)
func TestUploadHandlerSendingToExternalStorageAndSupportRequestTimeout(t *testing.T) {
shutdown := make(chan struct{})
storeServerMux := http.NewServeMux()
storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
storeServerMux.HandleFunc(putURL, func(w http.ResponseWriter, r *http.Request) {
<-shutdown
})
@ -224,7 +229,7 @@ func TestUploadHandlerSendingToExternalStorageAndSupportRequestTimeout(t *testin
authResponse := &api.Response{
RemoteObject: api.RemoteObject{
StoreURL: storeServer.URL + "/url/put",
StoreURL: storeServer.URL + putURL,
ID: "store-id",
Timeout: 0.1,
},

13
workhorse/internal/upload/artifacts_upload_test.go
View File

@ -77,11 +77,9 @@ func testArtifactsUploadServer(t *testing.T, authResponse *api.Response, bodyPro
t.Fatal("Expected file to be readable")
return
}
} else {
if r.FormValue("file.remote_url") == "" {
t.Fatal("Expected file to be remote accessible")
return
}
} else if r.FormValue("file.remote_url") == "" {
t.Fatal("Expected file to be remote accessible")
return
}
if r.FormValue("metadata.path") != "" {
@ -107,7 +105,6 @@ func testArtifactsUploadServer(t *testing.T, authResponse *api.Response, bodyPro
}
w.Header().Set(MetadataHeaderKey, MetadataHeaderPresent)
} else {
w.Header().Set(MetadataHeaderKey, MetadataHeaderMissing)
}
@ -204,7 +201,7 @@ func TestUploadHandlerAddingMetadata(t *testing.T) {
require.NoError(t, err)
rewrittenFields := token.Claims.(*MultipartClaims).RewrittenFields
require.Equal(t, 2, len(rewrittenFields))
require.Len(t, rewrittenFields, 2)
require.Contains(t, rewrittenFields, "file")
require.Contains(t, rewrittenFields, "metadata")
@ -235,7 +232,7 @@ func TestUploadHandlerTarArtifact(t *testing.T) {
require.NoError(t, err)
rewrittenFields := token.Claims.(*MultipartClaims).RewrittenFields
require.Equal(t, 1, len(rewrittenFields))
require.Len(t, rewrittenFields, 1)
require.Contains(t, rewrittenFields, "file")
require.Contains(t, r.PostForm, "file.gitlab-workhorse-upload")

20
workhorse/internal/upload/artifacts_uploader.go
View File

@ -34,7 +34,7 @@ const (
var zipSubcommandsErrorsCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "gitlab_workhorse_zip_subcommand_errors_total",
Help: "Errors comming from subcommands used for processing ZIP archives",
Help: "Errors coming from subcommands used for processing ZIP archives",
}, []string{"error"})
type artifactsUploadProcessor struct {
@ -80,12 +80,20 @@ func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context,
if err != nil {
return nil, err
}
defer zipMdOut.Close()
defer func() {
if err = zipMdOut.Close(); err != nil {
log.ContextLogger(ctx).WithError(err).Error("Failed to close zip-metadata stdout")
}
}()
if err := zipMd.Start(); err != nil {
if err = zipMd.Start(); err != nil {
return nil, err
}
defer command.KillProcessGroup(zipMd)
defer func() {
if err = command.KillProcessGroup(zipMd); err != nil {
log.ContextLogger(ctx).WithError(err).Error("Failed to kill zip-metadata process group")
}
}()
fh, err := destination.Upload(ctx, zipMdOut, -1, "metadata.gz", metaOpts)
if err != nil {
@ -146,7 +154,9 @@ func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName str
}
for k, v := range fields {
writer.WriteField(k, v)
if err := writer.WriteField(k, v); err != nil {
return fmt.Errorf("write metadata field error: %v", err)
}
}
a.Track("metadata", metadata.LocalPath)

1
workhorse/internal/upload/body_uploader.go
View File

@ -1,3 +1,4 @@
// Package upload provides middleware for handling request bodies and uploading them to a destination.
package upload
import (

5
workhorse/internal/upload/body_uploader_test.go
View File

@ -29,6 +29,7 @@ func TestRequestBody(t *testing.T) {
body := strings.NewReader(fileContent)
resp := testUpload(&rails{}, &alwaysLocalPreparer{}, echoProxy(t, fileLen), body)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
uploadEcho, err := io.ReadAll(resp.Body)
@ -41,6 +42,7 @@ func TestRequestBodyCustomPreparer(t *testing.T) {
body := strings.NewReader(fileContent)
resp := testUpload(&rails{}, &alwaysLocalPreparer{}, echoProxy(t, fileLen), body)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
uploadEcho, err := io.ReadAll(resp.Body)
@ -73,6 +75,7 @@ func testNoProxyInvocation(t *testing.T, expectedStatus int, auth PreAuthorizer,
})
resp := testUpload(auth, preparer, proxy, nil)
defer resp.Body.Close()
require.Equal(t, expectedStatus, resp.StatusCode)
}
@ -128,7 +131,7 @@ func echoProxy(t *testing.T, expectedBodyLength int) http.Handler {
uploaded, err := os.Open(path)
require.NoError(t, err, "File not uploaded")
//sending back the file for testing purpose
// sending back the file for testing purpose
io.Copy(w, uploaded)
})
}