root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2023-04-26 15:17:18 +00:00
parent 9edf852c3a
commit 34283a71d9
75 changed files with 8136 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -43,7 +43,7 @@ We are hiring developers, support people, and production engineers all the time,
On [about.gitlab.com](https://about.gitlab.com/) you can find more information about:
- [Subscriptions](https://about.gitlab.com/pricing/)
- [Consultancy](https://about.gitlab.com/consultancy/)
- [Professional Services](https://about.gitlab.com/services/)
- [Community](https://about.gitlab.com/community/)
- [Hosted GitLab.com](https://about.gitlab.com/gitlab-com/) use GitLab as a free service
- [GitLab Enterprise Edition](https://about.gitlab.com/features/#enterprise) with additional features aimed at larger organizations.

5
app/assets/stylesheets/themes/dark_mode_overrides.scss
View File

@ -300,3 +300,8 @@ body.gl-dark {
// soften on darkmode
background-color: mix($gray-50, $orange-50, 75%) !important;
}
.tanuki-bot-chat-drawer .tanuki-bot-message {
// lightens chat bubble in darkmode as $gray-50 matches drawer background. See tanuki_bot_chat.scss
background-color: $gray-100;
}

4
app/controllers/jira_connect/subscriptions_controller.rb
View File

@ -19,10 +19,6 @@ class JiraConnect::SubscriptionsController < JiraConnect::ApplicationController
p.style_src(*style_src_values)
end
before_action do
push_frontend_feature_flag(:jira_connect_oauth, @user)
end
before_action :allow_rendering_in_iframe, only: :index
before_action :verify_qsh_claim!, only: :index
before_action :allow_self_managed_content_security_policy, only: :index

6
app/graphql/types/release_asset_link_type.rb
View File

@ -9,12 +9,6 @@ module Types
present_using Releases::LinkPresenter
field :external, GraphQL::Types::Boolean,
null: true,
method: :external?,
description: 'Indicates the link points to an external resource.',
deprecated: { reason: 'No longer used', milestone: '15.9' }
field :id, GraphQL::Types::ID, null: false,
description: 'ID of the link.'
field :link_type,

3
app/helpers/application_settings_helper.rb
View File

@ -484,7 +484,8 @@ module ApplicationSettingsHelper
:user_defaults_to_private_profile,
:deactivation_email_additional_text,
:projects_api_rate_limit_unauthenticated,
:gitlab_dedicated_instance
:gitlab_dedicated_instance,
:ci_max_includes
].tap do |settings|
next if Gitlab.com?

2
app/helpers/jira_connect_helper.rb
View File

@ -11,7 +11,7 @@ module JiraConnectHelper
subscriptions_path: jira_connect_subscriptions_path(format: :json),
users_path: current_user ? nil : jira_connect_users_path, # users_path is used to determine if user is signed in
gitlab_user_path: current_user ? user_path(current_user) : nil,
oauth_metadata: Feature.enabled?(:jira_connect_oauth, current_user) ? jira_connect_oauth_data(installation).to_json : nil,
oauth_metadata: jira_connect_oauth_data(installation).to_json,
public_key_storage_enabled: Gitlab::CurrentSettings.jira_connect_public_key_storage_enabled?
}
end

2
app/models/application_setting.rb
View File

@ -388,6 +388,8 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord
validates :max_yaml_size_bytes, numericality: { only_integer: true, greater_than: 0 }, presence: true
validates :max_yaml_depth, numericality: { only_integer: true, greater_than: 0 }, presence: true
validates :ci_max_includes, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, presence: true
validates :email_restrictions, untrusted_regexp: true
validates :hashed_storage_enabled, inclusion: { in: [true], message: N_("Hashed storage can't be disabled anymore for new projects") }

3
app/models/application_setting_implementation.rb
View File

@ -252,7 +252,8 @@ module ApplicationSettingImplementation
allow_runner_registration_token: true,
user_defaults_to_private_profile: false,
projects_api_rate_limit_unauthenticated: 400,
gitlab_dedicated_instance: false
gitlab_dedicated_instance: false,
ci_max_includes: 150
}.tap do |hsh|
hsh.merge!(non_production_defaults) unless Rails.env.production?
end

5
app/views/admin/application_settings/_ci_cd.html.haml
View File

@ -45,6 +45,11 @@
= link_to _('Learn more.'), help_page_path('user/admin_area/settings/continuous_integration', anchor: 'archive-jobs')
.form-group
= f.gitlab_ui_checkbox_component :protected_ci_variables, s_('AdminSettings|Protect CI/CD variables by default'), help_text: s_('AdminSettings|New CI/CD variables in projects and groups default to protected.')
.form-group
= f.label :ci_max_includes, s_('AdminSettings|Maximum includes'), class: 'label-bold'
= f.number_field :ci_max_includes, class: 'form-control gl-form-input'
.form-text.text-muted
= s_('AdminSettings|The maximum number of included files per pipeline.')
.form-group
= f.label :ci_config_path, _('Default CI/CD configuration file'), class: 'label-bold'
= f.text_field :default_ci_config_path, class: 'form-control gl-form-input', placeholder: '.gitlab-ci.yml'

9
app/workers/all_queues.yml
View File

@ -1245,15 +1245,6 @@
:weight: 1
:idempotent: false
:tags: []
- :name: github_importer:github_import_import_release_attachments
:worker_name: Gitlab::GithubImport::ImportReleaseAttachmentsWorker
:feature_category: :importers
:has_external_dependencies: true
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: github_importer:github_import_pull_requests_import_review_request
:worker_name: Gitlab::GithubImport::PullRequests::ImportReviewRequestWorker
:feature_category: :importers

23
app/workers/gitlab/github_import/import_release_attachments_worker.rb
View File

@ -1,23 +0,0 @@
# frozen_string_literal: true
# TODO: remove in 16.X milestone
# https://gitlab.com/gitlab-org/gitlab/-/issues/377059
module Gitlab
module GithubImport
class ImportReleaseAttachmentsWorker # rubocop:disable Scalability/IdempotentWorker
include ObjectImporter
def representation_class
Representation::NoteText
end
def importer_class
Importer::NoteAttachmentsImporter
end
def object_type
:release_attachment
end
end
end
end

10
config/feature_flags/development/jira_connect_oauth.yml → config/feature_flags/development/graphql_subs_lb.yml
View File

@ -1,8 +1,8 @@
---
name: jira_connect_oauth
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81126
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/355048
milestone: '14.9'
name: graphql_subs_lb
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117678
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/408178
milestone: '16.0'
type: development
group: group::integrations
group: group::application performance
default_enabled: false

18
db/migrate/20230419130952_remove_github_import_job_instances.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
# See https://docs.gitlab.com/ee/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class RemoveGithubImportJobInstances < Gitlab::Database::Migration[2.1]
DEPRECATED_JOB_CLASSES = %w[
Gitlab::GithubImport::ImportReleaseAttachmentsWorker
]
def up
sidekiq_remove_jobs(job_klasses: DEPRECATED_JOB_CLASSES)
end
def down
# This migration removes any instances of deprecated workers and cannot be undone.
end
end

7
db/migrate/20230421124302_epic_boards_show_colors.rb Normal file
View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
class EpicBoardsShowColors < Gitlab::Database::Migration[2.1]
def change
add_column :boards_epic_boards, :display_colors, :boolean, default: true, null: false
end
end

7
db/migrate/20230421165020_add_ci_max_includes_to_application_settings.rb Normal file
View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
class AddCiMaxIncludesToApplicationSettings < Gitlab::Database::Migration[2.1]
def change
add_column :application_settings, :ci_max_includes, :integer, default: 150, null: false
end
end

1
db/schema_migrations/20230419130952 Normal file
View File

@ -0,0 +1 @@
1172667d682c57cae72696030f3e5f58a57f5c71244190785a02c07e065173c6

1
db/schema_migrations/20230421124302 Normal file
View File

@ -0,0 +1 @@
3d8630d52fa7998ffd4775d0e070e778c8317416b241fd6a5149188ca7d2ff35

1
db/schema_migrations/20230421165020 Normal file
View File

@ -0,0 +1 @@
1e33e8da1f2f10f976e19bcf234f7ac5bdcc0eed7d60288ead2eaf87dfd678c9

2
db/structure.sql
View File

@ -11804,6 +11804,7 @@ CREATE TABLE application_settings (
encrypted_product_analytics_configurator_connection_string_iv bytea,
silent_mode_enabled boolean DEFAULT false NOT NULL,
package_metadata_purl_types smallint[] DEFAULT '{}'::smallint[],
ci_max_includes integer DEFAULT 150 NOT NULL,
CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
CONSTRAINT app_settings_container_registry_pre_import_tags_rate_positive CHECK ((container_registry_pre_import_tags_rate >= (0)::numeric)),
CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)),
@ -12596,6 +12597,7 @@ CREATE TABLE boards_epic_boards (
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
name text DEFAULT 'Development'::text NOT NULL,
display_colors boolean DEFAULT true NOT NULL,
CONSTRAINT check_bcbbffe601 CHECK ((char_length(name) <= 255))
);

15
doc/api/graphql/reference/index.md
View File

@ -973,6 +973,7 @@ Input type: `AiActionInput`
| <a id="mutationaiactionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
| <a id="mutationaiactionexplaincode"></a>`explainCode` | [`AiExplainCodeInput`](#aiexplaincodeinput) | Input for explain_code AI action. |
| <a id="mutationaiactionexplainvulnerability"></a>`explainVulnerability` | [`AiExplainVulnerabilityInput`](#aiexplainvulnerabilityinput) | Input for explain_vulnerability AI action. |
| <a id="mutationaiactionmarkupformat"></a>`markupFormat` | [`MarkupFormat`](#markupformat) | Indicates the response format. |
| <a id="mutationaiactionsummarizecomments"></a>`summarizeComments` | [`AiSummarizeCommentsInput`](#aisummarizecommentsinput) | Input for summarize_comments AI action. |
#### Fields
@ -3061,6 +3062,7 @@ Input type: `EpicBoardCreateInput`
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="mutationepicboardcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
| <a id="mutationepicboardcreatedisplaycolors"></a>`displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. Ignored unless `epic_color_highlight` flag is enabled. |
| <a id="mutationepicboardcreategrouppath"></a>`groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. |
| <a id="mutationepicboardcreatehidebackloglist"></a>`hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. |
| <a id="mutationepicboardcreatehideclosedlist"></a>`hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. |
@ -3127,6 +3129,7 @@ Input type: `EpicBoardUpdateInput`
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="mutationepicboardupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
| <a id="mutationepicboardupdatedisplaycolors"></a>`displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. Ignored unless `epic_color_highlight` flag is enabled. |
| <a id="mutationepicboardupdatehidebackloglist"></a>`hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. |
| <a id="mutationepicboardupdatehideclosedlist"></a>`hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. |
| <a id="mutationepicboardupdateid"></a>`id` | [`BoardsEpicBoardID!`](#boardsepicboardid) | Epic board global ID. |
@ -13951,6 +13954,7 @@ Represents an epic board.
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="epicboarddisplaycolors"></a>`displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. |
| <a id="epicboardhidebackloglist"></a>`hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. |
| <a id="epicboardhideclosedlist"></a>`hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. |
| <a id="epicboardid"></a>`id` | [`BoardsEpicBoardID!`](#boardsepicboardid) | Global ID of the epic board. |
@ -20424,7 +20428,6 @@ Represents an asset link associated with a release.
| ---- | ---- | ----------- |
| <a id="releaseassetlinkdirectassetpath"></a>`directAssetPath` | [`String`](#string) | Relative path for the direct asset link. |
| <a id="releaseassetlinkdirectasseturl"></a>`directAssetUrl` | [`String`](#string) | Direct asset URL of the link. |
| <a id="releaseassetlinkexternal"></a>`external` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated** in 15.9. No longer used. |
| <a id="releaseassetlinkid"></a>`id` | [`ID!`](#id) | ID of the link. |
| <a id="releaseassetlinklinktype"></a>`linkType` | [`ReleaseAssetLinkType`](#releaseassetlinktype) | Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`. |
| <a id="releaseassetlinkname"></a>`name` | [`String`](#string) | Name of the link. |
@ -24149,6 +24152,16 @@ List limit metric setting.
| <a id="listlimitmetricissue_count"></a>`issue_count` | Limit list by number of issues. |
| <a id="listlimitmetricissue_weights"></a>`issue_weights` | Limit list by total weight of issues. |
### `MarkupFormat`
List markup formats.
| Value | Description |
| ----- | ----------- |
| <a id="markupformathtml"></a>`HTML` | HTML format. |
| <a id="markupformatmarkdown"></a>`MARKDOWN` | Markdown format. |
| <a id="markupformatraw"></a>`RAW` | Raw format. |
### `MeasurementIdentifier`
Possible identifier types for a measurement.

12
doc/api/graphql/removed_items.md
View File

@ -10,6 +10,18 @@ GraphQL is a versionless API, unlike the REST API.
Occasionally, items have to be updated or removed from the GraphQL API.
According to our [process for removing items](index.md#deprecation-and-removal-process), here are the items that have been removed.
## GitLab 16.0
Fields removed in GitLab 16.0.
### GraphQL Fields
[Removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111750) in GitLab 16.0:
| Field name | GraphQL type | Deprecated in | Use instead |
|----------------------|--------------------------|---------------|----------------------------|
| `external` | `GraphQL::Types::Boolean`| 15.9 | None |
## GitLab 15.0
Fields removed in GitLab 15.0.

5
doc/api/jobs.md
View File

@ -867,8 +867,9 @@ POST /projects/:id/jobs/:job_id/play
Example request:
```shell
curl --request POST "https://gitlab.example.com/api/v4/projects/1/jobs/1/play
--header "PRIVATE-TOKEN: <your_access_token>"
curl --request POST "https://gitlab.example.com/api/v4/projects/1/jobs/1/play" \
--header "Content-Type: application/json" \
--header "PRIVATE-TOKEN: <your_access_token>" \
--data @variables.json
```

1
doc/api/settings.md
View File

@ -296,6 +296,7 @@ listed in the descriptions of the relevant settings.
| `bulk_import_enabled` | boolean | no | Enable migrating GitLab groups by direct transfer. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383268) in GitLab 15.8. Setting also [available](../user/admin_area/settings/visibility_and_access_controls.md#enable-migration-of-groups-and-projects-by-direct-transfer) in the Admin Area. |
| `can_create_group` | boolean | no | Indicates whether users can create top-level groups. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367754) in GitLab 15.5. Defaults to `true`. |
| `check_namespace_plan` **(PREMIUM)** | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. |
| `ci_max_includes` | integer | no | The maximum number of [includes](../ci/yaml/includes.md) per pipeline. Default is `150`. |
| `commit_email_hostname` | string | no | Custom hostname (for private commit emails). |
| `container_expiration_policies_enable_historic_entries` | boolean | no | Enable [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#enable-the-cleanup-policy) for all projects. |
| `container_registry_cleanup_tags_service_max_list_size` | integer | no | The maximum number of tags that can be deleted in a single execution of [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). |

172
doc/architecture/blueprints/gitlab_ml_experiments/index.md Normal file
View File

@ -0,0 +1,172 @@
---
status: proposed
creation-date: "2023-04-13"
authors: [ "@andrewn" ]
coach: "@grzesiek"
---
# GitLab Service-Integration: AI and Beyond
[[_TOC_]]
This document is an abbreviated proposal for Service-Integration to allow teams within GitLab to rapidly build new application features that leverage AI, ML, and data technologies.
## Executive Summary
This document proposes a service-integration approach to setting up infrastructure to allow teams within GitLab to build new application features that leverage AI, ML, and data technologies at a rapid pace. The scope of the document is limited specifically to internally hosted features, not third-party APIs. The current application architecture runs most GitLab application features in Ruby. However, many ML/AI experiments require different resources and tools, implemented in different languages, with huge libraries that do not always play nicely together, and have different hardware requirements. Adding all these features to the existing infrastructure will increase the size of the GitLab application container rapidly, resulting in slower startup times, increased number of dependencies, security risks, negatively impacting development velocity, and increasing complexity due to different hardware requirements. As an alternative, the proposal suggests adding services to avoid overloading GitLabs main workloads. These services will run independently with isolated resources and dependencies. By adding services, GitLab can maintain the availability and security of GitLab.com, and enable engineers to rapidly iterate on new ML/AI experiments.
## Scope
The infrastructure, platform, and other changes related to ML/AI experiments is broad. This blueprint is limited specifically to the following scope:
1. Production workloads, running (directly or indirectly) as a result of requests into the GitLab application (`gitlab.com`), or an associated subdomains (for example, `codesuggestions.gitlab.com`).
1. Excludes requests from the GitLab application, made to third-party APIs outside of our infrastructure. From an Infrastructure point-of-view, external AI/ML API requests are no different from other API (non ML/AI) requests and generally follow the existing guidelines that are in place for calling external APIs.
1. Excludes training and tuning workloads not _directly_ connected to our production workloads. Training and tuning workloads are distinct from production workloads and will be covered by their own blueprint(s).
## Running Production ML/AI experiment workloads
### Why Not Simply Continue To Use The Existing Application Architecture?
Let's start with some background on how the application is deployed:
1. Most GitLab application features are implemented in Ruby and run in one of two types of Ruby deployments: broadly Rails and Sidekiq (although we do partition this traffic further for different workloads).
1. These Ruby workloads have two main container images `gitlab-webservice-ee` and `gitlab-sidekiq-ee`. All the code, libraries, binaries, and other resources that we use to support the main Ruby part of the codebase are embedded within these images.
1. There are thousands of pods running these containers in production for GitLab.com at any moment in time. They are started up and shut down at a high rate throughout the day as traffic demands on the site fluctuate.
1. For _most_ new features developed, any new supporting resources need to be added to either one, or both of these containers.
![current containers](https://docs.google.com/drawings/d/e/2PACX-1vQh9ToJDy6ceKVMZxSJK5kjBjgKUKdnHcigqTz-Jte1G65aV9js5XZhCC-VYNtkJ_gnoNfob4z-DCui/pub?w=692&h=286)\
[source](https://docs.google.com/drawings/d/1RiTUnsDSkTGaMqK_RfUlCd_rQ6CgSInhfQJNewIKf1M/edit)
Many of the initial discussions focus on adding supporting resources to these existing containers ([example](https://gitlab.com/gitlab-org/gitlab/-/issues/403630#note_1345192671)). Choosing this approach would have many downsides, in terms of both the velocity at which new features can be iterated on, and in terms of the availability of GitLab.com.
Many of the AI experiments that GitLab is considering integrating into the application are substantially different from other libraries and tools that have been integrated in the past.
1. ML toolkits are **implemented in a plethora of languages**, each requiring separate runtimes. Python, C, C++ are the most common, but there is a long tail of languages used.
1. There are a very large number of tools that we're looking to integrate with and **no single tool will support all the features that are being investigated**. Tensorflow, PyTorch, Keras, Scikit-learn, Alpaca are just a few examples.
1. **These libraries are huge**. Tensorflow's container image with GPU support is 3GB, PyTorch is 5GB, Keras is 300MB. Prophet is ~250MB.
1. Many of these **libraries do not play nicely together**: they may have dependencies that are not compatible, or require different versions of Python, or GPU driver versions.
It's likely that in the next few months, GitLab will experiment with many different features, using many different libraries.
Trying to deploy all of these features into the existing infrastructure would have many downsides:
1. **The size of the GitLab application container would expand very rapidly** as each new experiment introduces a new set of supporting libraries, each library is as big, or bigger, than the existing GitLab application within the container.
1. **Startup times for new workloads would increase**, potentially impacting the availability of GitLab.com during high-traffic periods.
1. The number of dependencies within the container would increase rapidly, putting pressure on the engineering teams to **keep ahead of exploits and vulnerabilities**.
1. **The security attack surface within the container would be greatly increased** with each new dependency. These containers include secrets which, if leaked via an exploit would need costly application-wide secret rotation to be done.
1. **Development velocity will be negatively impacted** as engineers work to avoid dependency conflicts between libraries.
1. Additionally there may be **extra complexity due to different hardware requirements** for different libraries with appropriate drivers etc for GPUs, TPUs, CUDA versions, etc.
1. Our Kubernetes workloads have been tuned for the existing multithreaded Ruby request (Rails) and message (Sidekiq) processes. Adding extremely resource-intensive applications into these workloads would affect unrelated requests, **starving requests of CPU and memory and requiring complex tuning to ensure fairness**. Failure to do this would impact our availability of GitLab.com.
![fat containers](https://docs.google.com/drawings/d/e/2PACX-1vSW0Pm_7yZV-0JNmgfOHhQlvh6XsJYtrrzkPPhURf5sCbsQDKc0I0kCIbfios3ifD5tmcNvuchXSVUB/pub?w=686&h=364)
\
[source](https://docs.google.com/drawings/d/1aYffBzzea5QuZ-mTMteowefbV7VmsOuq2v4BqbPd6KE/edit)
### Proposal: Avoid Overfilling GitLabs Application Containers with Service-Integration
GitLab.com migrated to Kubernetes several years back, but for numerous good reasons, the application architecture deployed for GitLab.com remains fairly simple.
Instead of embedding these applications directly into the Rails and/or Sidekiq containers, we run them as small, independent Kubernetes deployments, isolated from the main workload.
![use services instead of fat containers](https://docs.google.com/drawings/d/e/2PACX-1vSRrPo0TNtXG8Yqj37TO2PaND9PojGZzNRs2rcTA37-vBZm5WZlfxLDCKVJD1vYHTbGy1KY1rDYHwlg/pub?w=1008&h=564)\
[source](https://docs.google.com/drawings/d/1ZPprcSYH5Oqp8T46I0p1Hhr-GD55iREDvFWcpQq9dTQ/edit)
The service-integration approach has already been used for the [Suggested Reviewers feature](https://gitlab.com/gitlab-com/gl-infra/readiness/-/merge_requests/114) that has been deployed to GitLab.com.
This approach would have many advantages:
1. **Componentization and Replaceability**: some of these AI feature experiments will likely be short-lived. Being able to shut them down (possibly quickly, in an emergency, such as a security breach) is important. If they are terminated, they are less likely to leave technical debt behind in our main application workloads.
1. **Security Isolation**: experimental services can run with access to a minimal set of secrets, or possibly none. Ideally, the services would be stateless, with data being passed in, processed, and returned to the caller without access to PostgreSQL or other data sources. In the event of a remote code exploit or other security breach, the attacker would have limited access to sensitive data.
1. In lieu of direct access to the main or CI Postgres clusters, services would be provided with access to the internal GitLab API through a predefined internal URL. The platform should provide instrumentation and monitoring on this address.
1. In future iterations, but out of scope for the initial delivery, the platform could facilitate automatic authentication against the internal API, for example by managing and injecting short-lived API tokens into internal API calls, or OIDC etc.
1. **Resource Isolation**: resource-intensive workloads would be isolated to individual containers. OOM failures would not impact requests outside of the experiment. CPU saturation would not slow down unrelated requests.
1. **Dependency Isolation**: different AI libraries will have conflicting dependencies. This will not be an issue if they're run as separate services in Kubernetes.
1. **Container Size**: the size of the main application containers is not drastically increased, placing a burden on the application.
1. **Distribution Team Bottleneck**: The Distribution team avoids becoming a bottleneck as demands for many different libraries to be included in the main application containers increase.
1. **Stronger Ownership of Workloads**: teams can better understand how their workloads are running as they run in isolation.
However, there are several outstanding questions:
1. **Availability Requirements**: would experimental services have the same availability requirements (and alerting requirements) as the main application?
1. **Oncall**: would teams be responsible for handling pager alerts for their services?
1. **Support for non-SAAS GitLab instances**: initially all experiments would target GitLab.com, but eventually we may need to consider how to support other instances.
1. There are three possible modes for services:
1. `M1`: GitLab.com only: only GitLab.com supports the service.
1. `M2`: SAAS-hosted for use with self-managed instance and instance-hosted: a singular SAAS-hosted service supports self-managed instances and GitLab.com. This is similar to the [GitLab Plus proposal](https://gitlab.com/groups/gitlab-org/-/epics/308).
1. `M3`: Instance-hosted: each instance has a copy of the service. GitLab.com has a copy for GitLab.com. Self-managed instances host their copy of the service. This is similar to the container registry or Gitaly today.
1. Initially, most experiments will probably be option 1 but may be promoted to 2 or 3 as they mature.
1. **Promotion Process**: ML/AI experimental features will need to be promoted to non-experimental status as they mature. A process for this will need to be established.
#### Proposed Guidelines for Building ML/AI Services
1. Avoid adding any large ML/AI libraries needed to support experimentation to the main application.
1. Create an platform to support individual ML/AI experiments.
1. Encourage supporting services to be stateless (excluding deployed models and other resources generated during ML training).
1. ML/AI experiment support services must not access main application datastores, including but not limited to main PostgreSQL, CI PostgreSQL, and main application Redis instances.
1. In the main application, client code for services should reside behind a feature-flag toggle, for fine-grained control of the feature.
#### Technical Details
Some points, in greater detail:
##### Traffic Access
1. Ideally these services should not be exposed externally to Internet traffic: only internally to our existing Rails and Sidekiq workloads should be routed.
1. For services intended to run at `M2`: "SAAS-hosted for use with self-managed instance and instance-hosted", we would expect to migrate the service to a public endpoint once sufficient security review has been performed.
##### Platform Requirements
In order to quickly deploy and manage experiments, an minimally viable platform will need to be provided to stage-group teams. The technical implementation details of this platform are out of scope for this blueprint and will require their own blueprint (to follow).
However, Service-Integration will establish certain necessary and optional requirements that the platform will need to satisfy.
###### Ease of Use, Ownership Requirements
1. <a name="R100">`R100`</a>: Required: the platform should be easy to use: imagine Heroku with [GitLab Production Readiness-approved](https://about.gitlab.com/handbook/engineering/infrastructure/production/readiness/) defaults.
1. <a name="R110">`R110`</a>: Required: with the exception of an Infrastructure-led onboarding process, services are owned, deployed and managed by stage-group teams. In other words,services follow a "You Build It, You Run It" model of ownership.
1. <a name="R120">`R120`</a>: Required: programming-language agnostic: no requirements for services. Services should be packaged as container images.
1. <a name="R130">`R130`</a>: Recommended: Each service should be evaluated against the GitLab.com [Service Maturity Model](https://about.gitlab.com/handbook/engineering/infrastructure/service-maturity-model/).
1. <a name="R140">`R140`</a>: Recommended: services using the platform have expedited production-readiness processes.
1. Production-readiness requirements graded by service maturity: low-traffic, low-maturity experimental services will have lower requirement thresholds than more mature services.
1. By default, the platform should provide services with defaults that would pass production-readiness review for the lowest service maturity-level.
1. At introduction, lowest maturity services can be deployed without production readiness, provided the meet certain automatically validated requirements. This removes Infrastructure gate-keeping from being a blocker to experimental service delivery.
###### Observability Requirements
1. <a name="R200">`R200`</a>: Required: the platform must provide SLIs for services out-of-the-box.
1. While it is recommended that services expose internal metrics, it is not mandatory. The platform will provide monitoring from the load-balancer. This is to speed up deployment by removing barriers to experimentation.
1. For services that provide internal metrics scrape endpoints, the platform must be configurable to collect these.
1. The platform must provide generic load-balancer level SLIs for all services. Service owners must be able to select from constructing SLIs from internal application metrics, the platform-provided external SLIs, or a combination of both.
1. <a name="R210">`R210`</a>: Required: Observability dashboards, rules, alerts (with per-term routing) must be generated from a manifest.
1. <a name="R220">`R220`</a>:Required: standardized logging infrastructure.
1. Mandate that all logging emitted from services must be Structured JSON. Text logs are permitted but not recommended.
1. See [Common Service Libraries](#common-service-libraries) for more details of building common SDKs for observability.
###### Deployment Requirements
1. <a name="R300">`R300`</a>: Required: No secrets stored in CI/CD.
1. Authentication with Cloud Provider Resources should be exclusively via OIDC, managed as part of the platform.
1. Secrets should be stored in the Infrastructure-provided Hashicorp Vault for the environment and passed to applications through files or environment variables.
1. Generation and management of service account tokens should be done declaratively, without manual interaction.
1. <a name="R310">`R310`</a>: Required: multiple environment should be supported, eg Staging and Production.
1. <a name="R320">`R320`</a>: Required: the platform should be cost-effective. Kubernetes clusters should support multiple services and teams.
1. <a name="R330">`R330`</a>: Recommended: gradual rollouts, rollbacks, blue-green deployments.
1. <a name="R340">`R340`</a>: Required: services should be isolated from one another.
1. <a name="R350">`R350`</a>: Recommended: services should have the ability to specify node characteristic requirements (eg, GPU).
1. <a name="R360">`R360`</a>: Required: Developers should not need knowledge of Helm, Kubernetes, Prometheus in order to deploy. All required values are configured and validated in project-hosted manifest before generating Kubernetes manifests, Prometheus rules, etc.
1. <a name="R370">`R370`</a>: Initially services should be synchronous only - using REST or GRPC requests.
1. This does not however preclude long-running HTTP(s) requests, for example long-polling or Websocket requests.
1. <a name="R390">`R390`</a>: Each service hosted in its own GitLab repository with deployment manifest stored in the repository.
1. Continuous deployments that are initiated from the CI pipeline of the corresponding GitLab repository.
##### Security Requirements
1. <a name="R400">`R400`</a>: stateful services deployed on the platform that utilize their own stateful storage (for example, custom deployed Postgres instance), must not store application security tokens, cloud-provider service keys or other long-lived security tokens in their stateful stores.
1. <a name="R410">`R410`</a>: long-lived shared secrets are discouraged, and should be referenced in the service manifest as such, to allow for accounting and monitoring.
1. <a name="R420">`R420`</a>: services using long-lived shared secrets should ensure that secret rotation can take place without downtime.
1. During a rotation, old and new generations of secrets should pass authentication, allowing gradual roll-out of new secrets.
##### Common Service Libraries
1. <a name="R500">`R500`</a>: Experimental services would be strongly encouraged to adopt and use [LabKit](https://gitlab.com/gitlab-org/labkit) (for Go services), or [LabKit-Ruby](https://gitlab.com/gitlab-org/ruby/gems/labkit-ruby) for observability, context, correlation, FIPs verification, etc.
1. At present, there is no LabKit-Python library, but some experiments will run in Python, so building a library to providing observability, context, correlation services in Python will be required.

3
doc/ci/yaml/includes.md
View File

@ -603,3 +603,6 @@ To help reduce the risk of this happening, edit the pipeline configuration file
with the [pipeline editor](../pipeline_editor/index.md), which validates if the
limit is reached. You can remove one included file at a time to try to narrow down
which configuration file is the source of the loop or excessive included files.
In [GitLab 16.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/207270) self-managed users can
change the [maximum includes](../../user/admin_area/settings/continuous_integration.md#maximum-includes) value.

20
doc/ci/yaml/index.md
View File

@ -136,16 +136,6 @@ The `include` files are:
- Always evaluated first and then merged with the content of the `.gitlab-ci.yml` file,
regardless of the position of the `include` keyword.
You can have up to 150 includes per pipeline, including [nested](includes.md#use-nested-includes) includes:
- In [GitLab 15.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367150) you can have up to 150 includes.
In nested includes, the same file can be included multiple times, but duplicated includes
count towards the limit.
- From [GitLab 14.9 to GitLab 15.9](https://gitlab.com/gitlab-org/gitlab/-/issues/28987), you can have up to 100 includes.
The same file can be included multiple times in nested includes, but duplicates are ignored.
- In GitLab 14.9 and earlier you can have up to 100 includes, but the same file can not
be included multiple times.
The time limit to resolve all files is 30 seconds.
**Keyword type**: Global keyword.
@ -171,6 +161,16 @@ The time limit to resolve all files is 30 seconds.
do not affect job reruns.
- Pipeline, the `include` files are fetched again. If they changed after the last
pipeline run, the new pipeline uses the changed configuration.
- You can have up to 150 includes per pipeline by default, including [nested](includes.md#use-nested-includes). Additionally:
- In [GitLab 16.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/207270) self-managed users can
change the [maximum includes](../../user/admin_area/settings/continuous_integration.md#maximum-includes) value.
- In [GitLab 15.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367150) you can have up to 150 includes.
In nested includes, the same file can be included multiple times, but duplicated includes
count towards the limit.
- From [GitLab 14.9 to GitLab 15.9](https://gitlab.com/gitlab-org/gitlab/-/issues/28987), you can have up to 100 includes.
The same file can be included multiple times in nested includes, but duplicates are ignored.
- In GitLab 14.9 and earlier you can have up to 100 includes, but the same file can not
be included multiple times.
**Related topics**:

8
doc/development/github_importer.md
View File

@ -146,10 +146,10 @@ comments.
This worker imports note attachments that are linked inside Markdown.
For each entity with Markdown text in the project, we schedule a job of:
- `Gitlab::GithubImport::ImportReleaseAttachmentsWorker` for every release.
- `Gitlab::GithubImport::ImportNoteAttachmentsWorker` for every note.
- `Gitlab::GithubImport::ImportIssueAttachmentsWorker` for every issue.
- `Gitlab::GithubImport::ImportMergeRequestAttachmentsWorker` for every merge request.
- `Gitlab::GithubImport::Attachments::ImportReleaseWorker` for every release.
- `Gitlab::GithubImport::Attachments::ImportNoteWorker` for every note.
- `Gitlab::GithubImport::Attachments::ImportIssueWorker` for every issue.
- `Gitlab::GithubImport::Attachments::ImportMergeRequestWorker` for every merge request.
Each job:

16
doc/development/integrations/jira_connect.md
View File

@ -74,7 +74,8 @@ To avoid external dependencies like Gitpod and a Jira Cloud instance, use the [J
## Test the GitLab OAuth authentication flow
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81126) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `jira_connect_oauth`. Disabled by default.
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81126) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `jira_connect_oauth`. Disabled by default.
> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117648) in GitLab 16.0. Feature flag `jira_connect_oauth` removed.
GitLab for Jira users can authenticate with GitLab using GitLab OAuth.
@ -83,18 +84,7 @@ This feature is not ready for production use. The feature flag should only be en
The following steps describe setting up an environment to test the GitLab OAuth flow:
1. Start a Gitpod session and open the rails console.
```shell
bundle exec rails console
```
1. Enable the feature flag.
```shell
Feature.enable(:jira_connect_oauth)
```
1. Start a Gitpod session.
1. On your GitLab instance, go to **Admin > Applications**.
1. Create a new application with the following settings:
- Name: `Jira Connect`

1
doc/development/pipelines/index.md
View File

@ -302,6 +302,7 @@ runs the GitLab test suite "as if JiHu", meaning as if the pipeline would run
in the context of [GitLab JH](../jh_features_review.md). These jobs are only
created in the following cases:
- when changes are made to feature flags
- when the `pipeline:run-as-if-jh` label is set on the merge request
This pipeline runs under the context of a generated branch in the

20
doc/development/testing_guide/frontend_testing.md
View File

@ -894,6 +894,26 @@ You can generate fixtures by running:
You can find generated fixtures are in `tmp/tests/frontend/fixtures-ee`.
### Download fixtures
We generate fixtures in GitLab CI, and store them in the package registry.
The `scripts/frontend/download_fixtures.sh` script is meant to download and extract those fixtures for local use:
```shell
# Checks if a frontend fixture package exists in the gitlab-org/gitlab
# package registry by looking at the commits on a local branch.
#
# The package is downloaded and extracted if it exists
$ scripts/frontend/download_fixtures.sh
# Same as above, but only looks at the last 10 commits of the currently checked-out branch
$ scripts/frontend/download_fixtures.sh --max-commits=10
# Looks at the commits on the local master branch instead of the currently checked-out branch
$ scripts/frontend/download_fixtures.sh --branch master
```
#### Creating new fixtures
For each fixture, you can find the content of the `response` variable in the output file.

25
doc/integration/jira/configure.md
View File

@ -4,22 +4,20 @@ group: Integrations
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
# Jira integration **(FREE)**
# Jira issue integration **(FREE)**
The Jira integration connects one or more GitLab projects to a Jira instance. You can host the Jira instance yourself or in [Atlassian Cloud](https://www.atlassian.com/migration/assess/why-cloud). The supported Jira versions are `6.x`, `7.x`, `8.x`, and `9.x`.
The Jira issue integration connects one or more GitLab projects to a Jira instance. You can host the Jira instance yourself or in [Jira Cloud](https://www.atlassian.com/migration/assess/why-cloud). The supported Jira versions are `6.x`, `7.x`, `8.x`, and `9.x`.
## Configure the integration
Prerequisites:
- Ensure your GitLab installation does not use a [relative URL](https://docs.gitlab.com/omnibus/settings/configuration.html#configure-a-relative-url-for-gitlab).
- For **Jira Server**, ensure you have a Jira username and password.
See [authentication in Jira](index.md#authentication-in-jira).
- For **Jira Cloud**, ensure you have an API token
and the email address you used to create the token.
See [authentication in Jira](index.md#authentication-in-jira).
- Your GitLab installation must not use a [relative URL](https://docs.gitlab.com/omnibus/settings/configuration.html#configure-a-relative-url-for-gitlab).
- **For Jira Server**, you must have a Jira username and password.
- **For Jira Cloud**, you must have a [Jira Cloud API token](#create-a-jira-cloud-api-token) and
the email address you used to create the token.
You can enable the Jira integration by configuring your project settings in GitLab.
You can enable the Jira issue integration by configuring your project settings in GitLab.
You can also configure these settings at the:
- [Group level](../../user/admin_area/settings/project_integration_management.md#manage-group-level-default-settings-for-a-project-integration)
@ -66,7 +64,7 @@ displays a Jira link that opens the Jira project.
## Create a Jira Cloud API token
To [integrate with Jira](index.md) in Atlassian Cloud, you must create an API token.
To configure the integration with Jira Cloud, you must create a Jira Cloud API token.
To create a Jira Cloud API token:
@ -81,11 +79,6 @@ To create a Jira Cloud API token:
To copy the API token, select **Copy** and paste the token somewhere safe.
To configure GitLab, you need:
- The newly created token
- The email address you used when you created the token
## Migrate from Jira Server to Jira Cloud in GitLab
To migrate from Jira Server to Jira Cloud in GitLab and maintain your Jira integration:
@ -95,7 +88,7 @@ To migrate from Jira Server to Jira Cloud in GitLab and maintain your Jira integ
1. Select **Jira**.
1. In **Web URL**, enter the new Jira site URL (for example, `https://myjirasite.atlassian.net`).
1. In **Username or Email**, enter the username or email registered on your Jira profile.
1. [Create an API token](#create-a-jira-cloud-api-token), and copy that value.
1. [Create a Jira Cloud API token](#create-a-jira-cloud-api-token), and copy the token value.
1. In **Password or API token**, paste the API token value.
1. Optional. Select **Test settings** to check if the connection is working.
1. Select **Save changes**.

3
doc/integration/jira/connect-app.md
View File

@ -79,7 +79,6 @@ To create an OAuth application:
1. Expand **GitLab for Jira App**.
1. Paste the **Application ID** value into **Jira Connect Application ID**.
1. Select **Save changes**.
1. Optional. Enable the `jira_connect_oauth` [feature flag](../../administration/feature_flags.md) to avoid [authentication problems in some browsers](#browser-displays-a-sign-in-message-when-already-signed-in).
## Connect the GitLab for Jira Cloud app for self-managed instances **(FREE SELF)**
@ -252,7 +251,7 @@ You need to sign in or sign up before continuing.
The GitLab for Jira Cloud app uses an iframe to add namespaces on the
settings page. Some browsers block cross-site cookies, which can lead to this issue.
To resolve this issue, set up [OAuth authentication](#set-up-oauth-authentication-for-self-managed-instances) and enable the `jira_connect_oauth` [feature flag](../../administration/feature_flags.md).
To resolve this issue, set up [OAuth authentication](#set-up-oauth-authentication-for-self-managed-instances).
### Manual installation fails

52
doc/integration/jira/index.md
View File

@ -6,38 +6,34 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Jira **(FREE)**
If your organization uses [Jira](https://www.atlassian.com/software/jira) issues,
you can [migrate your issues from Jira](../../user/project/import/jira.md) and work
exclusively in GitLab. However, if you'd like to continue to use Jira, you can
integrate it with GitLab. GitLab offers two types of Jira integrations, and you
can use one or both depending on the capabilities you need.
If your organization uses [Jira](https://www.atlassian.com/software/jira),
you can [migrate your issues from Jira to GitLab](../../user/project/import/jira.md).
If you want to continue to use Jira, you can integrate Jira with GitLab instead.
## Compare integrations
## Jira integrations
After you set up one or both of these integrations, you can cross-reference activity
in your GitLab project with any of your projects in Jira.
GitLab offers two types of Jira integrations. You can
use one or both depending on the capabilities you need.
### Jira integration
### Jira issue integration
The Jira integration connects one or more GitLab projects to a Jira instance. You can host
the Jira instance yourself or in [Atlassian Cloud](https://www.atlassian.com/migration/assess/why-cloud).
The supported Jira versions are `6.x`, `7.x`, `8.x`, and `9.x`.
You can use the [Jira issue integration](configure.md) developed by GitLab with Jira Cloud, Jira Data Center, or Jira Server. With this integration, you can:
For more information, see [Jira integration](configure.md).
- View and search Jira issues directly in GitLab.
- Refer to Jira issues by ID in GitLab commits and merge requests.
- Create Jira issues for vulnerabilities.
### Jira development panel
The [Jira development panel](development_panel.md) connects all GitLab projects in a group or personal namespace.
When you configure the Jira development panel, you can
[view GitLab activity for an issue](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/)
including related branches, commits, and merge requests.
You can use the [Jira development panel](development_panel.md) to [view GitLab activity for an issue](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/)
including related branches, commits, and merge requests. To configure the Jira development panel:
To set up the Jira development panel, use the GitLab for Jira Cloud app
or the Jira DVCS connector. For more information, see [Configure the panel](development_panel.md#configure-the-panel).
- **For Jira Cloud**, use the [GitLab for Jira Cloud app](connect-app.md) developed by GitLab.
- **For Jira Data Center or Jira Server**, use the [Jira DVCS connector](dvcs/index.md) developed by Atlassian.
### Direct feature comparison
## Jira integration capabilities
| Capability | Jira integration | Jira development panel |
| Capability | Jira issue integration | Jira development panel |
|-|-|-|
| Mention a Jira issue ID in a GitLab commit or merge request, and a link to the Jira issue is created. | Yes. | No. |
| Mention a Jira issue ID in GitLab and the Jira issue shows the GitLab issue or merge request. | Yes. A Jira comment with the GitLab issue or MR title links to GitLab. The first mention is also added to the Jira issue under **Web links**. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). |
@ -50,25 +46,13 @@ or the Jira DVCS connector. For more information, see [Configure the panel](deve
| Create a GitLab branch from a Jira issue. | No. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). |
| Mention a Jira issue ID in a GitLab merge request, and deployments are synced. | No. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). |
## Authentication in Jira
The authentication method in Jira depends on whether you host Jira on your own server or on
[Atlassian cloud](https://www.atlassian.com/migration/assess/why-cloud):
- **Jira Server** supports basic authentication. When connecting, a **username and password** are
required. Connecting to Jira Server using the Central Authentication Service (CAS) is not possible. For more information, read
how to [set up a user in Jira Server](jira_server_configuration.md).
- **Jira on Atlassian cloud** supports authentication through an API token. When connecting to Jira on
Atlassian cloud, an email and API token are required. For more information, see
[Create a Jira Cloud API token](configure.md#create-a-jira-cloud-api-token).
## Privacy considerations
All Jira integrations share data with Jira to make it visible outside of GitLab.
If you integrate a private GitLab project with Jira, the private data is
shared with users who have access to your Jira project.
The [Jira integration](#jira-integration) posts GitLab data in the form of comments in Jira issues.
The [Jira issue integration](configure.md) posts GitLab data in the form of comments in Jira issues.
The GitLab for Jira Cloud app and Jira DVCS connector share this data through the [Jira development panel](development_panel.md).
This method provides more fine-grained access control because access can be restricted to certain user groups or roles.

9
doc/integration/jira/issues.md
View File

@ -6,12 +6,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Jira issue management **(FREE)**
To integrate issue management with Jira, [configure Jira](index.md#jira-integration)
and [enable the integration](configure.md) in GitLab.
After you configure and enable the integration, you can reference and close Jira
issues by mentioning the Jira ID in GitLab commits and merge requests.
The Jira integration requires to you format any Jira issue IDs in uppercase.
You can [manage Jira issues directly in GitLab](configure.md).
You can then refer to Jira issues by ID in GitLab commits and merge requests.
The Jira issue IDs must be in uppercase.
## Reference Jira issues

10
doc/user/admin_area/settings/continuous_integration.md
View File

@ -194,6 +194,16 @@ To set all new [CI/CD variables](../../../ci/variables/index.md) as
1. On the left sidebar, select **Settings > CI/CD**.
1. Select **Protect CI/CD variables by default**.
## Maximum includes
The maximum number of [includes](../../../ci/yaml/includes.md) per pipeline can be set at the instance level.
The default is `150`.
1. On the top bar, select **Main menu > Admin**.
1. On the left sidebar, select **Settings > CI/CD**.
1. Change the value of **Maximum includes**.
1. Select **Save changes** for the changes to take effect.
## Default CI/CD configuration file
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18073) in GitLab 12.5.

2
doc/user/group/access_and_permissions.md
View File

@ -64,7 +64,7 @@ To change the permitted Git access protocols for a group:
To ensure only people from your organization can access particular resources, you can restrict access to groups by IP
address. This group-level setting applies to:
- The GitLab UI, including subgroups, projects, and issues.
- The GitLab UI, including subgroups, projects, and issues. It does not apply to GitLab Pages.
- [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/12874), the API.
- In self-managed installations of GitLab 15.1 and later, you can also configure
[globally-allowed IP address ranges](../admin_area/settings/visibility_and_access_controls.md#configure-globally-allowed-ip-address-ranges)

35
doc/user/product_analytics/index.md
View File

@ -100,7 +100,7 @@ On GitLab.com, this feature is not available.
This feature is not ready for production use.
Each project can have an unlimited number of dashboards.
These dashboards are defined using the GitLab YAML schema, and stored in the `.gitlab/product_analytics/dashboards/` directory of a project repository.
These dashboards are defined using the GitLab YAML schema, and stored in the `.gitlab/analytics/dashboards/` directory of a project repository.
The name of the file is the name of the dashboard.
Each dashboard can contain one or more visualizations (charts), which are shared across dashboards.
@ -126,13 +126,13 @@ To view a list of product analytics dashboards for a project:
To define a dashboard:
1. In `.gitlab/product_analytics/dashboards/`, create a directory named like the dashboard.
1. In `.gitlab/analytics/dashboards/`, create a directory named like the dashboard.
Each dashboard should have its own directory.
1. In the new directory, create a `.yaml` file with the same name as the directory.
This file contains the dashboard definition. It must conform to the JSON schema defined in `ee/app/validators/json_schemas/product_analytics_dashboard.json`.
1. In the `.gitlab/product_analytics/dashboards/visualizations/` directory, create a `.yaml` file.
1. In the `.gitlab/analytics/dashboards/visualizations/` directory, create a `.yaml` file.
This file defines the visualization type for the dashboard. It must conform to the schema in
`ee/app/validators/json_schemas/product_analytics_visualization.json`.
@ -141,7 +141,7 @@ For example, if you want to create three dashboards (Conversion funnels, Demogra
and one visualization (line chart) that applies to all dashboards, the file structure would be:
```plaintext
.gitlab/product_analytics/dashboards
.gitlab/analytics/dashboards
├── conversion_funnels
│ └── conversion_funnels.yaml
├── demographic_breakdown
@ -152,13 +152,38 @@ and one visualization (line chart) that applies to all dashboards, the file stru
│ └── example_line_chart.yaml
```
### Define a chart visualization
You can define different charts, and add visualization options to some of them:
- Line chart, with the options listed in the [ECharts documentation](https://echarts.apache.org/en/option.html).
- Column chart, with the options listed in the [ECharts documentation](https://echarts.apache.org/en/option.html).
- Data table, without options.
- Single stat, with the only option to set `decimalPlaces` (number, default value is 0).
To define a chart for your dashboards:
1. In the `.gitlab/product_analytics/dashboards/visualizations/` directory, create a `.yaml` file.
The filename should be descriptive of the visualization it defines.
1. In the `.yaml` file, define the visualization options, according to the schema in
`ee/app/validators/json_schemas/analytics_visualization.json`.
For [example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/assets/javascripts/analytics/analytics_dashboards/gl_dashboards/product_analytics/visualizations/events_over_time.json), to create a line chart that illustrates event count over time, in the `visualizations` folder
create a `line_chart.yaml` file with the following required fields:
- version
- title
- type
- data
- options
## Funnel analysis
Use funnel analysis to understand the flow of users through your application, and where
users drop out of a predefined flow (for example, a checkout process or ticket purchase).
Each product can also define an unlimited number of funnels.
Like dashboards, funnels are defined using the GitLab YAML schema, and stored in the `.gitlab/product_analytics/funnels/` directory of a project repository.
Like dashboards, funnels are defined using the GitLab YAML schema, and stored in the `.gitlab/analytics/funnels/` directory of a project repository.
Funnel definitions must include the keys `name` and `seconds_to_convert`, and an array of `steps`.

2
jest_resolver.js
View File

@ -9,7 +9,7 @@ module.exports = (request, options) => {
console.error(
'\x1b[1m\x1b[41m\x1b[30m %s \x1b[0m %s',
'!',
`Fixture file ${request} does not exist. Did you run bin/rake frontend:fixtures?`,
`Fixture file ${request} does not exist. Did you run bin/rake frontend:fixtures? You can also download fixtures from the gitlab-org/gitlab package registry. See the Fixtures docs for more info.`,
);
}
throw e;

1
lib/api/settings.rb
View File

@ -196,6 +196,7 @@ module API
optional :jira_connect_proxy_url, type: String, desc: "URL of the GitLab instance that should be used as a proxy for the GitLab for Jira Cloud app"
optional :bulk_import_enabled, type: Boolean, desc: 'Enable migrating GitLab groups and projects by direct transfer'
optional :allow_runner_registration_token, type: Boolean, desc: 'Allow registering runners using a registration token'
optional :ci_max_includes, type: Integer, desc: 'Maximum number of includes per pipeline'
Gitlab::SSHPublicKey.supported_types.each do |type|
optional :"#{type}_key_restriction",

5
lib/gitlab/ci/config/external/context.rb vendored
View File

@ -9,8 +9,7 @@ module Gitlab
TimeoutError = Class.new(StandardError)
MAX_INCLUDES = 150
TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/367150
TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/396776
include ::Gitlab::Utils::StrongMemoize
@ -32,7 +31,7 @@ module Gitlab
@expandset = []
@execution_deadline = 0
@logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project)
@max_includes = MAX_INCLUDES
@max_includes = Gitlab::CurrentSettings.current_application_settings.ci_max_includes
yield self if block_given?
end

16
lib/gitlab/ci/parsers/security/validators/schema_validator.rb
View File

@ -7,14 +7,14 @@ module Gitlab
module Validators
class SchemaValidator
SUPPORTED_VERSIONS = {
cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4]
cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4 15.0.6]
}.freeze
VERSIONS_TO_REMOVE_IN_16_0 = %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3].freeze

1035
lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json Normal file
View File

File diff suppressed because it is too large Load Diff

967
lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json Normal file
View File

@ -0,0 +1,967 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json",
"title": "Report format for GitLab Container Scanning",
"description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
"definitions": {
"detail_type": {
"oneOf": [
{
"$ref": "#/definitions/named_list"
},
{
"$ref": "#/definitions/list"
},
{
"$ref": "#/definitions/table"
},
{
"$ref": "#/definitions/text"
},
{
"$ref": "#/definitions/url"
},
{
"$ref": "#/definitions/code"
},
{
"$ref": "#/definitions/value"
},
{
"$ref": "#/definitions/diff"
},
{
"$ref": "#/definitions/markdown"
},
{
"$ref": "#/definitions/commit"
},
{
"$ref": "#/definitions/file_location"
},
{
"$ref": "#/definitions/module_location"
}
]
},
"text_value": {
"type": "string"
},
"named_field": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"$ref": "#/definitions/text_value",
"type": "string",
"minLength": 1
},
"description": {
"$ref": "#/definitions/text_value"
}
}
},
"named_list": {
"type": "object",
"description": "An object with named and typed fields",
"required": [
"type",
"items"
],
"properties": {
"type": {
"const": "named-list"
},
"items": {
"type": "object",
"patternProperties": {
"^.*$": {
"allOf": [
{
"$ref": "#/definitions/named_field"
},
{
"$ref": "#/definitions/detail_type"
}
]
}
}
}
}
},
"list": {
"type": "object",
"description": "A list of typed fields",
"required": [
"type",
"items"
],
"properties": {
"type": {
"const": "list"
},
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
}
}
},
"table": {
"type": "object",
"description": "A table of typed fields",
"required": [
"type",
"rows"
],
"properties": {
"type": {
"const": "table"
},
"header": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
},
"rows": {
"type": "array",
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
}
}
}
},
"text": {
"type": "object",
"description": "Raw text",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "text"
},
"value": {
"$ref": "#/definitions/text_value"
}
}
},
"url": {
"type": "object",
"description": "A single URL",
"required": [
"type",
"href"
],
"properties": {
"type": {
"const": "url"
},
"text": {
"$ref": "#/definitions/text_value"
},
"href": {
"type": "string",
"minLength": 1,
"examples": [
"http://mysite.com"
]
}
}
},
"code": {
"type": "object",
"description": "A codeblock",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "code"
},
"value": {
"type": "string"
},
"lang": {
"type": "string",
"description": "A programming language"
}
}
},
"value": {
"type": "object",
"description": "A field that can store a range of types of value",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "value"
},
"value": {
"type": [
"number",
"string",
"boolean"
]
}
}
},
"diff": {
"type": "object",
"description": "A diff",
"required": [
"type",
"before",
"after"
],
"properties": {
"type": {
"const": "diff"
},
"before": {
"type": "string"
},
"after": {
"type": "string"
}
}
},
"markdown": {
"type": "object",
"description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "markdown"
},
"value": {
"$ref": "#/definitions/text_value",
"examples": [
"Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
]
}
}
},
"commit": {
"type": "object",
"description": "A commit/tag/branch within the GitLab project",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "commit"
},
"value": {
"type": "string",
"description": "The commit SHA",
"minLength": 1
}
}
},
"file_location": {
"type": "object",
"description": "A location within a file in the project",
"required": [
"type",
"file_name",
"line_start"
],
"properties": {
"type": {
"const": "file-location"
},
"file_name": {
"type": "string",
"minLength": 1
},
"line_start": {
"type": "integer"
},
"line_end": {
"type": "integer"
}
}
},
"module_location": {
"type": "object",
"description": "A location within a binary module of the form module+relative_offset",
"required": [
"type",
"module_name",
"offset"
],
"properties": {
"type": {
"const": "module-location"
},
"module_name": {
"type": "string",
"minLength": 1,
"examples": [
"compiled_binary"
]
},
"offset": {
"type": "integer",
"examples": [
100
]
}
}
}
},
"self": {
"version": "15.0.6"
},
"type": "object",
"required": [
"scan",
"version",
"vulnerabilities"
],
"additionalProperties": true,
"properties": {
"scan": {
"type": "object",
"required": [
"analyzer",
"end_time",
"scanner",
"start_time",
"status",
"type"
],
"properties": {
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
"pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
},
"messages": {
"type": "array",
"items": {
"type": "object",
"description": "Communication intended for the initiator of a scan.",
"required": [
"level",
"value"
],
"properties": {
"level": {
"type": "string",
"description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
"enum": [
"info",
"warn",
"fatal"
],
"examples": [
"info"
]
},
"value": {
"type": "string",
"description": "The message to communicate.",
"minLength": 1,
"examples": [
"Permission denied, scanning aborted"
]
}
}
}
},
"options": {
"type": "array",
"items": {
"type": "object",
"description": "A configuration option used for this scan.",
"required": [
"name",
"value"
],
"properties": {
"name": {
"type": "string",
"description": "The configuration option name.",
"maxLength": 255,
"minLength": 1,
"examples": [
"DAST_FF_ENABLE_BAS",
"DOCKER_TLS_CERTDIR",
"DS_MAX_DEPTH",
"SECURE_LOG_LEVEL"
]
},
"source": {
"type": "string",
"description": "The source of this option.",
"enum": [
"argument",
"file",
"env_variable",
"other"
]
},
"value": {
"type": [
"boolean",
"integer",
"null",
"string"
],
"description": "The value used for this scan.",
"examples": [
true,
2,
null,
"fatal",
""
]
}
}
}
},
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
"required": [
"id",
"name",
"version",
"vendor"
],
"properties": {
"id": {
"type": "string",
"description": "Unique id that identifies the analyzer.",
"minLength": 1,
"examples": [
"gitlab-dast"
]
},
"name": {
"type": "string",
"description": "A human readable value that identifies the analyzer, not required to be unique.",
"minLength": 1,
"examples": [
"GitLab DAST"
]
},
"url": {
"type": "string",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
"https://docs.gitlab.com/ee/user/application_security/dast"
]
},
"vendor": {
"description": "The vendor/maintainer of the analyzer.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "The name of the vendor.",
"minLength": 1,
"examples": [
"GitLab"
]
}
}
},
"version": {
"type": "string",
"description": "The version of the analyzer.",
"minLength": 1,
"examples": [
"1.0.2"
]
}
}
},
"scanner": {
"type": "object",
"description": "Object defining the scanner used to perform the scan.",
"required": [
"id",
"name",
"version",
"vendor"
],
"properties": {
"id": {
"type": "string",
"description": "Unique id that identifies the scanner.",
"minLength": 1,
"examples": [
"my-sast-scanner"
]
},
"name": {
"type": "string",
"description": "A human readable value that identifies the scanner, not required to be unique.",
"minLength": 1,
"examples": [
"My SAST Scanner"
]
},
"url": {
"type": "string",
"description": "A link to more information about the scanner.",
"examples": [
"https://scanner.url"
]
},
"version": {
"type": "string",
"description": "The version of the scanner.",
"minLength": 1,
"examples": [
"1.0.2"
]
},
"vendor": {
"description": "The vendor/maintainer of the scanner.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "The name of the vendor.",
"minLength": 1,
"examples": [
"GitLab"
]
}
}
}
}
},
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
"pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
},
"status": {
"type": "string",
"description": "Result of the scan.",
"enum": [
"success",
"failure"
]
},
"type": {
"type": "string",
"description": "Type of the scan.",
"enum": [
"container_scanning"
]
},
"primary_identifiers": {
"type": "array",
"description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
"items": {
"type": "object",
"required": [
"type",
"name",
"value"
],
"properties": {
"type": {
"type": "string",
"description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
"minLength": 1
},
"name": {
"type": "string",
"description": "Human-readable name of the identifier.",
"minLength": 1
},
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
"pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
"description": "Value of the identifier, for matching purpose.",
"minLength": 1
}
}
}
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
"pattern": "^https?://.+"
},
"version": {
"type": "string",
"description": "The version of the schema to which the JSON report conforms.",
"pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
},
"vulnerabilities": {
"type": "array",
"description": "Array of vulnerability objects.",
"items": {
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
"id",
"identifiers",
"location"
],
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
"name": {
"type": "string",
"maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
"description": {
"type": "string",
"maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
"enum": [
"Info",
"Unknown",
"Low",
"Medium",
"High",
"Critical"
]
},
"solution": {
"type": "string",
"maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
"identifiers": {
"type": "array",
"minItems": 1,
"description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
"items": {
"type": "object",
"required": [
"type",
"name",
"value"
],
"properties": {
"type": {
"type": "string",
"description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
"minLength": 1
},
"name": {
"type": "string",
"description": "Human-readable name of the identifier.",
"minLength": 1
},
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
"pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
"description": "Value of the identifier, for matching purpose.",
"minLength": 1
}
}
}
},
"links": {
"type": "array",
"description": "An array of references to external documentation or articles that describe the vulnerability.",
"items": {
"type": "object",
"required": [
"url"
],
"properties": {
"name": {
"type": "string",
"description": "Name of the vulnerability details link."
},
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
"pattern": "^(https?|ftp)://.+"
}
}
}
},
"details": {
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
"type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
"description": "Declares that a series of items should be tracked using source-specific tracking methods.",
"required": [
"items"
],
"properties": {
"type": {
"const": "source"
},
"items": {
"type": "array",
"items": {
"description": "An item that should be tracked using source-specific tracking methods.",
"type": "object",
"required": [
"signatures"
],
"properties": {
"file": {
"type": "string",
"description": "Path to the file where the vulnerability is located."
},
"start_line": {
"type": "number",
"description": "The first line of the file that includes the vulnerability."
},
"end_line": {
"type": "number",
"description": "The last line of the file that includes the vulnerability."
},
"signatures": {
"type": "array",
"description": "An array of calculated tracking signatures for this tracking item.",
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
"type": "object",
"required": [
"algorithm",
"value"
],
"properties": {
"algorithm": {
"type": "string",
"description": "The algorithm used to generate the signature."
},
"value": {
"type": "string",
"description": "The result of this signature algorithm."
}
}
}
}
}
}
}
}
}
],
"properties": {
"type": {
"type": "string",
"description": "Each tracking type must declare its own type."
}
}
},
"flags": {
"description": "Flags that can be attached to vulnerabilities.",
"type": "array",
"items": {
"type": "object",
"description": "Informational flags identified and assigned to a vulnerability.",
"required": [
"type",
"origin",
"description"
],
"properties": {
"type": {
"type": "string",
"minLength": 1,
"description": "Result of the scan.",
"enum": [
"flagged-as-likely-false-positive"
]
},
"origin": {
"minLength": 1,
"description": "Tool that issued the flag.",
"type": "string"
},
"description": {
"minLength": 1,
"description": "What the flag is about.",
"type": "string"
}
}
}
},
"location": {
"type": "object",
"description": "Identifies the vulnerability's location.",
"required": [
"dependency",
"operating_system",
"image"
],
"properties": {
"dependency": {
"type": "object",
"description": "Describes the dependency of a project where the vulnerability is located.",
"required": [
"package",
"version"
],
"properties": {
"package": {
"type": "object",
"description": "Provides information on the package where the vulnerability is located.",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Name of the package where the vulnerability is located."
}
}
},
"version": {
"type": "string",
"description": "Version of the vulnerable package."
},
"iid": {
"description": "ID that identifies the dependency in the scope of a dependency file.",
"type": "number"
},
"direct": {
"type": "boolean",
"description": "Tells whether this is a direct, top-level dependency of the scanned project."
},
"dependency_path": {
"type": "array",
"description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
"items": {
"type": "object",
"required": [
"iid"
],
"properties": {
"iid": {
"type": "number",
"description": "ID that is unique in the scope of a parent object, and specific to the resource type."
}
}
}
}
}
},
"operating_system": {
"type": "string",
"minLength": 1,
"description": "The operating system that contains the vulnerable package."
},
"image": {
"type": "string",
"minLength": 1,
"description": "The analyzed Docker image."
},
"default_branch_image": {
"type": "string",
"maxLength": 255,
"description": "The name of the image on the default branch."
}
}
}
}
}
},
"remediations": {
"type": "array",
"description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
"items": {
"type": "object",
"required": [
"fixes",
"summary",
"diff"
],
"properties": {
"fixes": {
"type": "array",
"description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
"items": {
"type": "object",
"required": [
"id"
],
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
}
}
}
},
"summary": {
"type": "string",
"minLength": 1,
"description": "An overview of how the vulnerabilities were fixed."
},
"diff": {
"type": "string",
"minLength": 1,
"description": "A base64-encoded remediation code diff, compatible with git apply."
}
}
}
}
}
}

925
lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json Normal file
View File

@ -0,0 +1,925 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json",
"title": "Report format for GitLab Fuzz Testing",
"description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
"definitions": {
"detail_type": {
"oneOf": [
{
"$ref": "#/definitions/named_list"
},
{
"$ref": "#/definitions/list"
},
{
"$ref": "#/definitions/table"
},
{
"$ref": "#/definitions/text"
},
{
"$ref": "#/definitions/url"
},
{
"$ref": "#/definitions/code"
},
{
"$ref": "#/definitions/value"
},
{
"$ref": "#/definitions/diff"
},
{
"$ref": "#/definitions/markdown"
},
{
"$ref": "#/definitions/commit"
},
{
"$ref": "#/definitions/file_location"
},
{
"$ref": "#/definitions/module_location"
}
]
},
"text_value": {
"type": "string"
},
"named_field": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"$ref": "#/definitions/text_value",
"type": "string",
"minLength": 1
},
"description": {
"$ref": "#/definitions/text_value"
}
}
},
"named_list": {
"type": "object",
"description": "An object with named and typed fields",
"required": [
"type",
"items"
],
"properties": {
"type": {
"const": "named-list"
},
"items": {
"type": "object",
"patternProperties": {
"^.*$": {
"allOf": [
{
"$ref": "#/definitions/named_field"
},
{
"$ref": "#/definitions/detail_type"
}
]
}
}
}
}
},
"list": {
"type": "object",
"description": "A list of typed fields",
"required": [
"type",
"items"
],
"properties": {
"type": {
"const": "list"
},
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
}
}
},
"table": {
"type": "object",
"description": "A table of typed fields",
"required": [
"type",
"rows"
],
"properties": {
"type": {
"const": "table"
},
"header": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
},
"rows": {
"type": "array",
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
}
}
}
},
"text": {
"type": "object",
"description": "Raw text",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "text"
},
"value": {
"$ref": "#/definitions/text_value"
}
}
},
"url": {
"type": "object",
"description": "A single URL",
"required": [
"type",
"href"
],
"properties": {
"type": {
"const": "url"
},
"text": {
"$ref": "#/definitions/text_value"
},
"href": {
"type": "string",
"minLength": 1,
"examples": [
"http://mysite.com"
]
}
}
},
"code": {
"type": "object",
"description": "A codeblock",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "code"
},
"value": {
"type": "string"
},
"lang": {
"type": "string",
"description": "A programming language"
}
}
},
"value": {
"type": "object",
"description": "A field that can store a range of types of value",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "value"
},
"value": {
"type": [
"number",
"string",
"boolean"
]
}
}
},
"diff": {
"type": "object",
"description": "A diff",
"required": [
"type",
"before",
"after"
],
"properties": {
"type": {
"const": "diff"
},
"before": {
"type": "string"
},
"after": {
"type": "string"
}
}
},
"markdown": {
"type": "object",
"description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "markdown"
},
"value": {
"$ref": "#/definitions/text_value",
"examples": [
"Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
]
}
}
},
"commit": {
"type": "object",
"description": "A commit/tag/branch within the GitLab project",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "commit"
},
"value": {
"type": "string",
"description": "The commit SHA",
"minLength": 1
}
}
},
"file_location": {
"type": "object",
"description": "A location within a file in the project",
"required": [
"type",
"file_name",
"line_start"
],
"properties": {
"type": {
"const": "file-location"
},
"file_name": {
"type": "string",
"minLength": 1
},
"line_start": {
"type": "integer"
},
"line_end": {
"type": "integer"
}
}
},
"module_location": {
"type": "object",
"description": "A location within a binary module of the form module+relative_offset",
"required": [
"type",
"module_name",
"offset"
],
"properties": {
"type": {
"const": "module-location"
},
"module_name": {
"type": "string",
"minLength": 1,
"examples": [
"compiled_binary"
]
},
"offset": {
"type": "integer",
"examples": [
100
]
}
}
}
},
"self": {
"version": "15.0.6"
},
"type": "object",
"required": [
"scan",
"version",
"vulnerabilities"
],
"additionalProperties": true,
"properties": {
"scan": {
"type": "object",
"required": [
"analyzer",
"end_time",
"scanner",
"start_time",
"status",
"type"
],
"properties": {
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
"pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
},
"messages": {
"type": "array",
"items": {
"type": "object",
"description": "Communication intended for the initiator of a scan.",
"required": [
"level",
"value"
],
"properties": {
"level": {
"type": "string",
"description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
"enum": [
"info",
"warn",
"fatal"
],
"examples": [
"info"
]
},
"value": {
"type": "string",
"description": "The message to communicate.",
"minLength": 1,
"examples": [
"Permission denied, scanning aborted"
]
}
}
}
},
"options": {
"type": "array",
"items": {
"type": "object",
"description": "A configuration option used for this scan.",
"required": [
"name",
"value"
],
"properties": {
"name": {
"type": "string",
"description": "The configuration option name.",
"maxLength": 255,
"minLength": 1,
"examples": [
"DAST_FF_ENABLE_BAS",
"DOCKER_TLS_CERTDIR",
"DS_MAX_DEPTH",
"SECURE_LOG_LEVEL"
]
},
"source": {
"type": "string",
"description": "The source of this option.",
"enum": [
"argument",
"file",
"env_variable",
"other"
]
},
"value": {
"type": [
"boolean",
"integer",
"null",
"string"
],
"description": "The value used for this scan.",
"examples": [
true,
2,
null,
"fatal",
""
]
}
}
}
},
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
"required": [
"id",
"name",
"version",
"vendor"
],
"properties": {
"id": {
"type": "string",
"description": "Unique id that identifies the analyzer.",
"minLength": 1,
"examples": [
"gitlab-dast"
]
},
"name": {
"type": "string",
"description": "A human readable value that identifies the analyzer, not required to be unique.",
"minLength": 1,
"examples": [
"GitLab DAST"
]
},
"url": {
"type": "string",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
"https://docs.gitlab.com/ee/user/application_security/dast"
]
},
"vendor": {
"description": "The vendor/maintainer of the analyzer.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "The name of the vendor.",
"minLength": 1,
"examples": [
"GitLab"
]
}
}
},
"version": {
"type": "string",
"description": "The version of the analyzer.",
"minLength": 1,
"examples": [
"1.0.2"
]
}
}
},
"scanner": {
"type": "object",
"description": "Object defining the scanner used to perform the scan.",
"required": [
"id",
"name",
"version",
"vendor"
],
"properties": {
"id": {
"type": "string",
"description": "Unique id that identifies the scanner.",
"minLength": 1,
"examples": [
"my-sast-scanner"
]
},
"name": {
"type": "string",
"description": "A human readable value that identifies the scanner, not required to be unique.",
"minLength": 1,
"examples": [
"My SAST Scanner"
]
},
"url": {
"type": "string",
"description": "A link to more information about the scanner.",
"examples": [
"https://scanner.url"
]
},
"version": {
"type": "string",
"description": "The version of the scanner.",
"minLength": 1,
"examples": [
"1.0.2"
]
},
"vendor": {
"description": "The vendor/maintainer of the scanner.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "The name of the vendor.",
"minLength": 1,
"examples": [
"GitLab"
]
}
}
}
}
},
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
"pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
},
"status": {
"type": "string",
"description": "Result of the scan.",
"enum": [
"success",
"failure"
]
},
"type": {
"type": "string",
"description": "Type of the scan.",
"enum": [
"coverage_fuzzing"
]
},
"primary_identifiers": {
"type": "array",
"description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
"items": {
"type": "object",
"required": [
"type",
"name",
"value"
],
"properties": {
"type": {
"type": "string",
"description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
"minLength": 1
},
"name": {
"type": "string",
"description": "Human-readable name of the identifier.",
"minLength": 1
},
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
"pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
"description": "Value of the identifier, for matching purpose.",
"minLength": 1
}
}
}
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
"pattern": "^https?://.+"
},
"version": {
"type": "string",
"description": "The version of the schema to which the JSON report conforms.",
"pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
},
"vulnerabilities": {
"type": "array",
"description": "Array of vulnerability objects.",
"items": {
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
"id",
"identifiers",
"location"
],
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
"name": {
"type": "string",
"maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
"description": {
"type": "string",
"maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
"enum": [
"Info",
"Unknown",
"Low",
"Medium",
"High",
"Critical"
]
},
"solution": {
"type": "string",
"maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
"identifiers": {
"type": "array",
"minItems": 1,
"description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
"items": {
"type": "object",
"required": [
"type",
"name",
"value"
],
"properties": {
"type": {
"type": "string",
"description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
"minLength": 1
},
"name": {
"type": "string",
"description": "Human-readable name of the identifier.",
"minLength": 1
},
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
"pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
"description": "Value of the identifier, for matching purpose.",
"minLength": 1
}
}
}
},
"links": {
"type": "array",
"description": "An array of references to external documentation or articles that describe the vulnerability.",
"items": {
"type": "object",
"required": [
"url"
],
"properties": {
"name": {
"type": "string",
"description": "Name of the vulnerability details link."
},
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
"pattern": "^(https?|ftp)://.+"
}
}
}
},
"details": {
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
"type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
"description": "Declares that a series of items should be tracked using source-specific tracking methods.",
"required": [
"items"
],
"properties": {
"type": {
"const": "source"
},
"items": {
"type": "array",
"items": {
"description": "An item that should be tracked using source-specific tracking methods.",
"type": "object",
"required": [
"signatures"
],
"properties": {
"file": {
"type": "string",
"description": "Path to the file where the vulnerability is located."
},
"start_line": {
"type": "number",
"description": "The first line of the file that includes the vulnerability."
},
"end_line": {
"type": "number",
"description": "The last line of the file that includes the vulnerability."
},
"signatures": {
"type": "array",
"description": "An array of calculated tracking signatures for this tracking item.",
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
"type": "object",
"required": [
"algorithm",
"value"
],
"properties": {
"algorithm": {
"type": "string",
"description": "The algorithm used to generate the signature."
},
"value": {
"type": "string",
"description": "The result of this signature algorithm."
}
}
}
}
}
}
}
}
}
],
"properties": {
"type": {
"type": "string",
"description": "Each tracking type must declare its own type."
}
}
},
"flags": {
"description": "Flags that can be attached to vulnerabilities.",
"type": "array",
"items": {
"type": "object",
"description": "Informational flags identified and assigned to a vulnerability.",
"required": [
"type",
"origin",
"description"
],
"properties": {
"type": {
"type": "string",
"minLength": 1,
"description": "Result of the scan.",
"enum": [
"flagged-as-likely-false-positive"
]
},
"origin": {
"minLength": 1,
"description": "Tool that issued the flag.",
"type": "string"
},
"description": {
"minLength": 1,
"description": "What the flag is about.",
"type": "string"
}
}
}
},
"location": {
"description": "The location of the error",
"type": "object",
"properties": {
"crash_address": {
"type": "string",
"description": "The relative address in memory were the crash occurred.",
"examples": [
"0xabababab"
]
},
"stacktrace_snippet": {
"type": "string",
"description": "The stack trace recorded during fuzzing resulting the crash.",
"examples": [
"func_a+0xabcd\nfunc_b+0xabcc"
]
},
"crash_state": {
"type": "string",
"description": "Minimised and normalized crash stack-trace (called crash_state).",
"examples": [
"func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
]
},
"crash_type": {
"type": "string",
"description": "Type of the crash.",
"examples": [
"Heap-Buffer-overflow",
"Division-by-zero"
]
}
}
}
}
}
},
"remediations": {
"type": "array",
"description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
"items": {
"type": "object",
"required": [
"fixes",
"summary",
"diff"
],
"properties": {
"fixes": {
"type": "array",
"description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
"items": {
"type": "object",
"required": [
"id"
],
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
}
}
}
},
"summary": {
"type": "string",
"minLength": 1,
"description": "An overview of how the vulnerabilities were fixed."
},
"diff": {
"type": "string",
"minLength": 1,
"description": "A base64-encoded remediation code diff, compatible with git apply."
}
}
}
}
}
}

1330
lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json Normal file
View File

File diff suppressed because it is too large Load Diff

1033
lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json Normal file
View File

File diff suppressed because it is too large Load Diff

920
lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json Normal file
View File

@ -0,0 +1,920 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json",
"title": "Report format for GitLab SAST",
"description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
"definitions": {
"detail_type": {
"oneOf": [
{
"$ref": "#/definitions/named_list"
},
{
"$ref": "#/definitions/list"
},
{
"$ref": "#/definitions/table"
},
{
"$ref": "#/definitions/text"
},
{
"$ref": "#/definitions/url"
},
{
"$ref": "#/definitions/code"
},
{
"$ref": "#/definitions/value"
},
{
"$ref": "#/definitions/diff"
},
{
"$ref": "#/definitions/markdown"
},
{
"$ref": "#/definitions/commit"
},
{
"$ref": "#/definitions/file_location"
},
{
"$ref": "#/definitions/module_location"
}
]
},
"text_value": {
"type": "string"
},
"named_field": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"$ref": "#/definitions/text_value",
"type": "string",
"minLength": 1
},
"description": {
"$ref": "#/definitions/text_value"
}
}
},
"named_list": {
"type": "object",
"description": "An object with named and typed fields",
"required": [
"type",
"items"
],
"properties": {
"type": {
"const": "named-list"
},
"items": {
"type": "object",
"patternProperties": {
"^.*$": {
"allOf": [
{
"$ref": "#/definitions/named_field"
},
{
"$ref": "#/definitions/detail_type"
}
]
}
}
}
}
},
"list": {
"type": "object",
"description": "A list of typed fields",
"required": [
"type",
"items"
],
"properties": {
"type": {
"const": "list"
},
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
}
}
},
"table": {
"type": "object",
"description": "A table of typed fields",
"required": [
"type",
"rows"
],
"properties": {
"type": {
"const": "table"
},
"header": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
},
"rows": {
"type": "array",
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
}
}
}
},
"text": {
"type": "object",
"description": "Raw text",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "text"
},
"value": {
"$ref": "#/definitions/text_value"
}
}
},
"url": {
"type": "object",
"description": "A single URL",
"required": [
"type",
"href"
],
"properties": {
"type": {
"const": "url"
},
"text": {
"$ref": "#/definitions/text_value"
},
"href": {
"type": "string",
"minLength": 1,
"examples": [
"http://mysite.com"
]
}
}
},
"code": {
"type": "object",
"description": "A codeblock",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "code"
},
"value": {
"type": "string"
},
"lang": {
"type": "string",
"description": "A programming language"
}
}
},
"value": {
"type": "object",
"description": "A field that can store a range of types of value",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "value"
},
"value": {
"type": [
"number",
"string",
"boolean"
]
}
}
},
"diff": {
"type": "object",
"description": "A diff",
"required": [
"type",
"before",
"after"
],
"properties": {
"type": {
"const": "diff"
},
"before": {
"type": "string"
},
"after": {
"type": "string"
}
}
},
"markdown": {
"type": "object",
"description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "markdown"
},
"value": {
"$ref": "#/definitions/text_value",
"examples": [
"Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
]
}
}
},
"commit": {
"type": "object",
"description": "A commit/tag/branch within the GitLab project",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "commit"
},
"value": {
"type": "string",
"description": "The commit SHA",
"minLength": 1
}
}
},
"file_location": {
"type": "object",
"description": "A location within a file in the project",
"required": [
"type",
"file_name",
"line_start"
],
"properties": {
"type": {
"const": "file-location"
},
"file_name": {
"type": "string",
"minLength": 1
},
"line_start": {
"type": "integer"
},
"line_end": {
"type": "integer"
}
}
},
"module_location": {
"type": "object",
"description": "A location within a binary module of the form module+relative_offset",
"required": [
"type",
"module_name",
"offset"
],
"properties": {
"type": {
"const": "module-location"
},
"module_name": {
"type": "string",
"minLength": 1,
"examples": [
"compiled_binary"
]
},
"offset": {
"type": "integer",
"examples": [
100
]
}
}
}
},
"self": {
"version": "15.0.6"
},
"type": "object",
"required": [
"scan",
"version",
"vulnerabilities"
],
"additionalProperties": true,
"properties": {
"scan": {
"type": "object",
"required": [
"analyzer",
"end_time",
"scanner",
"start_time",
"status",
"type"
],
"properties": {
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
"pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
},
"messages": {
"type": "array",
"items": {
"type": "object",
"description": "Communication intended for the initiator of a scan.",
"required": [
"level",
"value"
],
"properties": {
"level": {
"type": "string",
"description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
"enum": [
"info",
"warn",
"fatal"
],
"examples": [
"info"
]
},
"value": {
"type": "string",
"description": "The message to communicate.",
"minLength": 1,
"examples": [
"Permission denied, scanning aborted"
]
}
}
}
},
"options": {
"type": "array",
"items": {
"type": "object",
"description": "A configuration option used for this scan.",
"required": [
"name",
"value"
],
"properties": {
"name": {
"type": "string",
"description": "The configuration option name.",
"maxLength": 255,
"minLength": 1,
"examples": [
"DAST_FF_ENABLE_BAS",
"DOCKER_TLS_CERTDIR",
"DS_MAX_DEPTH",
"SECURE_LOG_LEVEL"
]
},
"source": {
"type": "string",
"description": "The source of this option.",
"enum": [
"argument",
"file",
"env_variable",
"other"
]
},
"value": {
"type": [
"boolean",
"integer",
"null",
"string"
],
"description": "The value used for this scan.",
"examples": [
true,
2,
null,
"fatal",
""
]
}
}
}
},
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
"required": [
"id",
"name",
"version",
"vendor"
],
"properties": {
"id": {
"type": "string",
"description": "Unique id that identifies the analyzer.",
"minLength": 1,
"examples": [
"gitlab-dast"
]
},
"name": {
"type": "string",
"description": "A human readable value that identifies the analyzer, not required to be unique.",
"minLength": 1,
"examples": [
"GitLab DAST"
]
},
"url": {
"type": "string",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
"https://docs.gitlab.com/ee/user/application_security/dast"
]
},
"vendor": {
"description": "The vendor/maintainer of the analyzer.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "The name of the vendor.",
"minLength": 1,
"examples": [
"GitLab"
]
}
}
},
"version": {
"type": "string",
"description": "The version of the analyzer.",
"minLength": 1,
"examples": [
"1.0.2"
]
}
}
},
"scanner": {
"type": "object",
"description": "Object defining the scanner used to perform the scan.",
"required": [
"id",
"name",
"version",
"vendor"
],
"properties": {
"id": {
"type": "string",
"description": "Unique id that identifies the scanner.",
"minLength": 1,
"examples": [
"my-sast-scanner"
]
},
"name": {
"type": "string",
"description": "A human readable value that identifies the scanner, not required to be unique.",
"minLength": 1,
"examples": [
"My SAST Scanner"
]
},
"url": {
"type": "string",
"description": "A link to more information about the scanner.",
"examples": [
"https://scanner.url"
]
},
"version": {
"type": "string",
"description": "The version of the scanner.",
"minLength": 1,
"examples": [
"1.0.2"
]
},
"vendor": {
"description": "The vendor/maintainer of the scanner.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "The name of the vendor.",
"minLength": 1,
"examples": [
"GitLab"
]
}
}
}
}
},
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
"pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
},
"status": {
"type": "string",
"description": "Result of the scan.",
"enum": [
"success",
"failure"
]
},
"type": {
"type": "string",
"description": "Type of the scan.",
"enum": [
"sast"
]
},
"primary_identifiers": {
"type": "array",
"description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
"items": {
"type": "object",
"required": [
"type",
"name",
"value"
],
"properties": {
"type": {
"type": "string",
"description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
"minLength": 1
},
"name": {
"type": "string",
"description": "Human-readable name of the identifier.",
"minLength": 1
},
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
"pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
"description": "Value of the identifier, for matching purpose.",
"minLength": 1
}
}
}
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
"pattern": "^https?://.+"
},
"version": {
"type": "string",
"description": "The version of the schema to which the JSON report conforms.",
"pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
},
"vulnerabilities": {
"type": "array",
"description": "Array of vulnerability objects.",
"items": {
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
"id",
"identifiers",
"location"
],
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
"name": {
"type": "string",
"maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
"description": {
"type": "string",
"maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
"enum": [
"Info",
"Unknown",
"Low",
"Medium",
"High",
"Critical"
]
},
"solution": {
"type": "string",
"maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
"identifiers": {
"type": "array",
"minItems": 1,
"description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
"items": {
"type": "object",
"required": [
"type",
"name",
"value"
],
"properties": {
"type": {
"type": "string",
"description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
"minLength": 1
},
"name": {
"type": "string",
"description": "Human-readable name of the identifier.",
"minLength": 1
},
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
"pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
"description": "Value of the identifier, for matching purpose.",
"minLength": 1
}
}
}
},
"links": {
"type": "array",
"description": "An array of references to external documentation or articles that describe the vulnerability.",
"items": {
"type": "object",
"required": [
"url"
],
"properties": {
"name": {
"type": "string",
"description": "Name of the vulnerability details link."
},
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
"pattern": "^(https?|ftp)://.+"
}
}
}
},
"details": {
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
"type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
"description": "Declares that a series of items should be tracked using source-specific tracking methods.",
"required": [
"items"
],
"properties": {
"type": {
"const": "source"
},
"items": {
"type": "array",
"items": {
"description": "An item that should be tracked using source-specific tracking methods.",
"type": "object",
"required": [
"signatures"
],
"properties": {
"file": {
"type": "string",
"description": "Path to the file where the vulnerability is located."
},
"start_line": {
"type": "number",
"description": "The first line of the file that includes the vulnerability."
},
"end_line": {
"type": "number",
"description": "The last line of the file that includes the vulnerability."
},
"signatures": {
"type": "array",
"description": "An array of calculated tracking signatures for this tracking item.",
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
"type": "object",
"required": [
"algorithm",
"value"
],
"properties": {
"algorithm": {
"type": "string",
"description": "The algorithm used to generate the signature."
},
"value": {
"type": "string",
"description": "The result of this signature algorithm."
}
}
}
}
}
}
}
}
}
],
"properties": {
"type": {
"type": "string",
"description": "Each tracking type must declare its own type."
}
}
},
"flags": {
"description": "Flags that can be attached to vulnerabilities.",
"type": "array",
"items": {
"type": "object",
"description": "Informational flags identified and assigned to a vulnerability.",
"required": [
"type",
"origin",
"description"
],
"properties": {
"type": {
"type": "string",
"minLength": 1,
"description": "Result of the scan.",
"enum": [
"flagged-as-likely-false-positive"
]
},
"origin": {
"minLength": 1,
"description": "Tool that issued the flag.",
"type": "string"
},
"description": {
"minLength": 1,
"description": "What the flag is about.",
"type": "string"
}
}
}
},
"location": {
"type": "object",
"description": "Identifies the vulnerability's location.",
"properties": {
"file": {
"type": "string",
"description": "Path to the file where the vulnerability is located."
},
"start_line": {
"type": "number",
"description": "The first line of the code affected by the vulnerability."
},
"end_line": {
"type": "number",
"description": "The last line of the code affected by the vulnerability."
},
"class": {
"type": "string",
"description": "Provides the name of the class where the vulnerability is located."
},
"method": {
"type": "string",
"description": "Provides the name of the method where the vulnerability is located."
}
}
},
"raw_source_code_extract": {
"type": "string",
"description": "Provides an unsanitized excerpt of the affected source code."
}
}
}
},
"remediations": {
"type": "array",
"description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
"items": {
"type": "object",
"required": [
"fixes",
"summary",
"diff"
],
"properties": {
"fixes": {
"type": "array",
"description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
"items": {
"type": "object",
"required": [
"id"
],
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
}
}
}
},
"summary": {
"type": "string",
"minLength": 1,
"description": "An overview of how the vulnerabilities were fixed."
},
"diff": {
"type": "string",
"minLength": 1,
"description": "A base64-encoded remediation code diff, compatible with git apply."
}
}
}
}
}
}

944
lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json Normal file
View File

@ -0,0 +1,944 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json",
"title": "Report format for GitLab Secret Detection",
"description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
"definitions": {
"detail_type": {
"oneOf": [
{
"$ref": "#/definitions/named_list"
},
{
"$ref": "#/definitions/list"
},
{
"$ref": "#/definitions/table"
},
{
"$ref": "#/definitions/text"
},
{
"$ref": "#/definitions/url"
},
{
"$ref": "#/definitions/code"
},
{
"$ref": "#/definitions/value"
},
{
"$ref": "#/definitions/diff"
},
{
"$ref": "#/definitions/markdown"
},
{
"$ref": "#/definitions/commit"
},
{
"$ref": "#/definitions/file_location"
},
{
"$ref": "#/definitions/module_location"
}
]
},
"text_value": {
"type": "string"
},
"named_field": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"$ref": "#/definitions/text_value",
"type": "string",
"minLength": 1
},
"description": {
"$ref": "#/definitions/text_value"
}
}
},
"named_list": {
"type": "object",
"description": "An object with named and typed fields",
"required": [
"type",
"items"
],
"properties": {
"type": {
"const": "named-list"
},
"items": {
"type": "object",
"patternProperties": {
"^.*$": {
"allOf": [
{
"$ref": "#/definitions/named_field"
},
{
"$ref": "#/definitions/detail_type"
}
]
}
}
}
}
},
"list": {
"type": "object",
"description": "A list of typed fields",
"required": [
"type",
"items"
],
"properties": {
"type": {
"const": "list"
},
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
}
}
},
"table": {
"type": "object",
"description": "A table of typed fields",
"required": [
"type",
"rows"
],
"properties": {
"type": {
"const": "table"
},
"header": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
},
"rows": {
"type": "array",
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/detail_type"
}
}
}
}
},
"text": {
"type": "object",
"description": "Raw text",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "text"
},
"value": {
"$ref": "#/definitions/text_value"
}
}
},
"url": {
"type": "object",
"description": "A single URL",
"required": [
"type",
"href"
],
"properties": {
"type": {
"const": "url"
},
"text": {
"$ref": "#/definitions/text_value"
},
"href": {
"type": "string",
"minLength": 1,
"examples": [
"http://mysite.com"
]
}
}
},
"code": {
"type": "object",
"description": "A codeblock",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "code"
},
"value": {
"type": "string"
},
"lang": {
"type": "string",
"description": "A programming language"
}
}
},
"value": {
"type": "object",
"description": "A field that can store a range of types of value",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "value"
},
"value": {
"type": [
"number",
"string",
"boolean"
]
}
}
},
"diff": {
"type": "object",
"description": "A diff",
"required": [
"type",
"before",
"after"
],
"properties": {
"type": {
"const": "diff"
},
"before": {
"type": "string"
},
"after": {
"type": "string"
}
}
},
"markdown": {
"type": "object",
"description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "markdown"
},
"value": {
"$ref": "#/definitions/text_value",
"examples": [
"Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
]
}
}
},
"commit": {
"type": "object",
"description": "A commit/tag/branch within the GitLab project",
"required": [
"type",
"value"
],
"properties": {
"type": {
"const": "commit"
},
"value": {
"type": "string",
"description": "The commit SHA",
"minLength": 1
}
}
},
"file_location": {
"type": "object",
"description": "A location within a file in the project",
"required": [
"type",
"file_name",
"line_start"
],
"properties": {
"type": {
"const": "file-location"
},
"file_name": {
"type": "string",
"minLength": 1
},
"line_start": {
"type": "integer"
},
"line_end": {
"type": "integer"
}
}
},
"module_location": {
"type": "object",
"description": "A location within a binary module of the form module+relative_offset",
"required": [
"type",
"module_name",
"offset"
],
"properties": {
"type": {
"const": "module-location"
},
"module_name": {
"type": "string",
"minLength": 1,
"examples": [
"compiled_binary"
]
},
"offset": {
"type": "integer",
"examples": [
100
]
}
}
}
},
"self": {
"version": "15.0.6"
},
"type": "object",
"required": [
"scan",
"version",
"vulnerabilities"
],
"additionalProperties": true,
"properties": {
"scan": {
"type": "object",
"required": [
"analyzer",
"end_time",
"scanner",
"start_time",
"status",
"type"
],
"properties": {
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
"pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
},
"messages": {
"type": "array",
"items": {
"type": "object",
"description": "Communication intended for the initiator of a scan.",
"required": [
"level",
"value"
],
"properties": {
"level": {
"type": "string",
"description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
"enum": [
"info",
"warn",
"fatal"
],
"examples": [
"info"
]
},
"value": {
"type": "string",
"description": "The message to communicate.",
"minLength": 1,
"examples": [
"Permission denied, scanning aborted"
]
}
}
}
},
"options": {
"type": "array",
"items": {
"type": "object",
"description": "A configuration option used for this scan.",
"required": [
"name",
"value"
],
"properties": {
"name": {
"type": "string",
"description": "The configuration option name.",
"maxLength": 255,
"minLength": 1,
"examples": [
"DAST_FF_ENABLE_BAS",
"DOCKER_TLS_CERTDIR",
"DS_MAX_DEPTH",
"SECURE_LOG_LEVEL"
]
},
"source": {
"type": "string",
"description": "The source of this option.",
"enum": [
"argument",
"file",
"env_variable",
"other"
]
},
"value": {
"type": [
"boolean",
"integer",
"null",
"string"
],
"description": "The value used for this scan.",
"examples": [
true,
2,
null,
"fatal",
""
]
}
}
}
},
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
"required": [
"id",
"name",
"version",
"vendor"
],
"properties": {
"id": {
"type": "string",
"description": "Unique id that identifies the analyzer.",
"minLength": 1,
"examples": [
"gitlab-dast"
]
},
"name": {
"type": "string",
"description": "A human readable value that identifies the analyzer, not required to be unique.",
"minLength": 1,
"examples": [
"GitLab DAST"
]
},
"url": {
"type": "string",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
"https://docs.gitlab.com/ee/user/application_security/dast"
]
},
"vendor": {
"description": "The vendor/maintainer of the analyzer.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "The name of the vendor.",
"minLength": 1,
"examples": [
"GitLab"
]
}
}
},
"version": {
"type": "string",
"description": "The version of the analyzer.",
"minLength": 1,
"examples": [
"1.0.2"
]
}
}
},
"scanner": {
"type": "object",
"description": "Object defining the scanner used to perform the scan.",
"required": [
"id",
"name",
"version",
"vendor"
],
"properties": {
"id": {
"type": "string",
"description": "Unique id that identifies the scanner.",
"minLength": 1,
"examples": [
"my-sast-scanner"
]
},
"name": {
"type": "string",
"description": "A human readable value that identifies the scanner, not required to be unique.",
"minLength": 1,
"examples": [
"My SAST Scanner"
]
},
"url": {
"type": "string",
"description": "A link to more information about the scanner.",
"examples": [
"https://scanner.url"
]
},
"version": {
"type": "string",
"description": "The version of the scanner.",
"minLength": 1,
"examples": [
"1.0.2"
]
},
"vendor": {
"description": "The vendor/maintainer of the scanner.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "The name of the vendor.",
"minLength": 1,
"examples": [
"GitLab"
]
}
}
}
}
},
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
"pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
},
"status": {
"type": "string",
"description": "Result of the scan.",
"enum": [
"success",
"failure"
]
},
"type": {
"type": "string",
"description": "Type of the scan.",
"enum": [
"secret_detection"
]
},
"primary_identifiers": {
"type": "array",
"description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
"items": {
"type": "object",
"required": [
"type",
"name",
"value"
],
"properties": {
"type": {
"type": "string",
"description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
"minLength": 1
},
"name": {
"type": "string",
"description": "Human-readable name of the identifier.",
"minLength": 1
},
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
"pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
"description": "Value of the identifier, for matching purpose.",
"minLength": 1
}
}
}
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
"pattern": "^https?://.+"
},
"version": {
"type": "string",
"description": "The version of the schema to which the JSON report conforms.",
"pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
},
"vulnerabilities": {
"type": "array",
"description": "Array of vulnerability objects.",
"items": {
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
"id",
"identifiers",
"location"
],
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
"name": {
"type": "string",
"maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
"description": {
"type": "string",
"maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
"enum": [
"Info",
"Unknown",
"Low",
"Medium",
"High",
"Critical"
]
},
"solution": {
"type": "string",
"maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
"identifiers": {
"type": "array",
"minItems": 1,
"description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
"items": {
"type": "object",
"required": [
"type",
"name",
"value"
],
"properties": {
"type": {
"type": "string",
"description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
"minLength": 1
},
"name": {
"type": "string",
"description": "Human-readable name of the identifier.",
"minLength": 1
},
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
"pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
"description": "Value of the identifier, for matching purpose.",
"minLength": 1
}
}
}
},
"links": {
"type": "array",
"description": "An array of references to external documentation or articles that describe the vulnerability.",
"items": {
"type": "object",
"required": [
"url"
],
"properties": {
"name": {
"type": "string",
"description": "Name of the vulnerability details link."
},
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
"pattern": "^(https?|ftp)://.+"
}
}
}
},
"details": {
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
"type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
"description": "Declares that a series of items should be tracked using source-specific tracking methods.",
"required": [
"items"
],
"properties": {
"type": {
"const": "source"
},
"items": {
"type": "array",
"items": {
"description": "An item that should be tracked using source-specific tracking methods.",
"type": "object",
"required": [
"signatures"
],
"properties": {
"file": {
"type": "string",
"description": "Path to the file where the vulnerability is located."
},
"start_line": {
"type": "number",
"description": "The first line of the file that includes the vulnerability."
},
"end_line": {
"type": "number",
"description": "The last line of the file that includes the vulnerability."
},
"signatures": {
"type": "array",
"description": "An array of calculated tracking signatures for this tracking item.",
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
"type": "object",
"required": [
"algorithm",
"value"
],
"properties": {
"algorithm": {
"type": "string",
"description": "The algorithm used to generate the signature."
},
"value": {
"type": "string",
"description": "The result of this signature algorithm."
}
}
}
}
}
}
}
}
}
],
"properties": {
"type": {
"type": "string",
"description": "Each tracking type must declare its own type."
}
}
},
"flags": {
"description": "Flags that can be attached to vulnerabilities.",
"type": "array",
"items": {
"type": "object",
"description": "Informational flags identified and assigned to a vulnerability.",
"required": [
"type",
"origin",
"description"
],
"properties": {
"type": {
"type": "string",
"minLength": 1,
"description": "Result of the scan.",
"enum": [
"flagged-as-likely-false-positive"
]
},
"origin": {
"minLength": 1,
"description": "Tool that issued the flag.",
"type": "string"
},
"description": {
"minLength": 1,
"description": "What the flag is about.",
"type": "string"
}
}
}
},
"location": {
"required": [
"commit"
],
"type": "object",
"properties": {
"file": {
"type": "string",
"description": "Path to the file where the vulnerability is located"
},
"commit": {
"type": "object",
"description": "Represents the commit in which the vulnerability was detected",
"required": [
"sha"
],
"properties": {
"author": {
"type": "string"
},
"date": {
"type": "string"
},
"message": {
"type": "string"
},
"sha": {
"type": "string",
"minLength": 1
}
}
},
"start_line": {
"type": "number",
"description": "The first line of the code affected by the vulnerability"
},
"end_line": {
"type": "number",
"description": "The last line of the code affected by the vulnerability"
},
"class": {
"type": "string",
"description": "Provides the name of the class where the vulnerability is located"
},
"method": {
"type": "string",
"description": "Provides the name of the method where the vulnerability is located"
}
}
},
"raw_source_code_extract": {
"type": "string",
"description": "Provides an unsanitized excerpt of the affected source code."
}
}
}
},
"remediations": {
"type": "array",
"description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
"items": {
"type": "object",
"required": [
"fixes",
"summary",
"diff"
],
"properties": {
"fixes": {
"type": "array",
"description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
"items": {
"type": "object",
"required": [
"id"
],
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
}
}
}
},
"summary": {
"type": "string",
"minLength": 1,
"description": "An overview of how the vulnerabilities were fixed."
},
"diff": {
"type": "string",
"minLength": 1,
"description": "A base64-encoded remediation code diff, compatible with git apply."
}
}
}
}
}
}

22
lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb
View File

@ -5,6 +5,7 @@ module Gitlab
module LoadBalancing
class SidekiqClientMiddleware
include Gitlab::Utils::StrongMemoize
include WalTrackingSender
def call(worker_class, job, _queue, _redis_pool)
# Mailers can't be constantized
@ -30,15 +31,7 @@ module Gitlab
end
def set_data_consistency_locations!(job)
locations = {}
::Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
if (location = wal_location_for(lb))
locations[lb.name] = location
end
end
job['wal_locations'] = locations
job['wal_locations'] = wal_locations_by_db_name
job['wal_location_source'] = wal_location_source
end
@ -50,17 +43,6 @@ module Gitlab
end
end
def wal_location_for(load_balancer)
# When only using the primary there's no need for any WAL queries.
return if load_balancer.primary_only?
if uses_primary?
load_balancer.primary_write_location
else
load_balancer.host&.database_replica_location || load_balancer.primary_write_location
end
end
def uses_primary?
::Gitlab::Database::LoadBalancing::Session.current.use_primary?
end

15
lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb
View File

@ -4,6 +4,8 @@ module Gitlab
module Database
module LoadBalancing
class SidekiqServerMiddleware
include WalTrackingReceiver
JobReplicaNotUpToDate = Class.new(::Gitlab::SidekiqMiddleware::RetryError)
REPLICA_WAIT_SLEEP_SECONDS = 0.5
@ -92,19 +94,6 @@ module Gitlab
# the `0` indicates that this is a first retry
job['retry_count'].nil?
end
def databases_in_sync?(wal_locations)
::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb|
if (location = wal_locations.with_indifferent_access[lb.name])
lb.select_up_to_date_host(location)
else
# If there's no entry for a load balancer it means the Sidekiq
# job doesn't care for it. In this case we'll treat the load
# balancer as being in sync.
true
end
end
end
end
end
end

23
lib/gitlab/database/load_balancing/wal_tracking_receiver.rb Normal file
View File

@ -0,0 +1,23 @@
# frozen_string_literal: true
module Gitlab
module Database
module LoadBalancing
module WalTrackingReceiver
# NOTE: If there's no entry for a load balancer or no WAL locations were passed
# we assume the sender does not care about LB and we assume nodes are in-sync.
def databases_in_sync?(wal_locations)
return true unless wal_locations.present?
::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb|
if (location = wal_locations.with_indifferent_access[lb.name])
lb.select_up_to_date_host(location)
else
true
end
end
end
end
end
end
end

30
lib/gitlab/database/load_balancing/wal_tracking_sender.rb Normal file
View File

@ -0,0 +1,30 @@
# frozen_string_literal: true
module Gitlab
module Database
module LoadBalancing
module WalTrackingSender
def wal_locations_by_db_name
{}.tap do |locations|
::Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
if (location = wal_location_for(lb))
locations[lb.name] = location
end
end
end
end
def wal_location_for(load_balancer)
# When only using the primary there's no need for any WAL queries.
return if load_balancer.primary_only?
if Session.current.use_primary?
load_balancer.primary_write_location
else
load_balancer.host&.database_replica_location || load_balancer.primary_write_location
end
end
end
end
end
end

47
lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb
View File

@ -5,15 +5,56 @@ module Gitlab
module Subscriptions
class ActionCableWithLoadBalancing < ::GraphQL::Subscriptions::ActionCableSubscriptions
extend ::Gitlab::Utils::Override
include Gitlab::Database::LoadBalancing::WalTrackingReceiver
# When executing updates we are usually responding to a broadcast as a result of a DB update.
# We use the primary so that we are sure that we are returning the newly updated data.
def initialize(**options)
super(serializer: WalInjectingSerializer.new, **options)
end
# We fall back to the primary in case no replica is sufficiently caught up.
override :execute_update
def execute_update(subscription_id, event, object)
::Gitlab::Database::LoadBalancing::Session.current.use_primary!
::Gitlab::Database::LoadBalancing::Session.current.use_primary! if use_primary?
super
end
private
def use_primary?
@serializer.wal_locations.blank? ||
Feature.disabled?(:graphql_subs_lb) ||
!databases_in_sync?(@serializer.wal_locations)
end
end
class WalInjectingSerializer
include Gitlab::Database::LoadBalancing::WalTrackingSender
DEFAULT_SERIALIZER = GraphQL::Subscriptions::Serialize
attr_reader :wal_locations
# rubocop: disable GitlabSecurity/PublicSend
def load(str)
return DEFAULT_SERIALIZER.load(str) unless Feature.enabled?(:graphql_subs_lb)
value = Gitlab::Json.parse(str)
@wal_locations = value['wal_locations']
DEFAULT_SERIALIZER.send(:load_value, value['payload'])
end
def dump(obj)
return DEFAULT_SERIALIZER.dump(obj) unless Feature.enabled?(:graphql_subs_lb)
Gitlab::Json.dump({
'wal_locations' => wal_locations_by_db_name,
'payload' => DEFAULT_SERIALIZER.send(:dump_value, obj)
})
end
# rubocop: enable GitlabSecurity/PublicSend
end
end
end

24
locale/gitlab.pot
View File

@ -3081,6 +3081,9 @@ msgstr ""
msgid "AdminSettings|Maximum duration of a session for Git operations when 2FA is enabled."
msgstr ""
msgid "AdminSettings|Maximum includes"
msgstr ""
msgid "AdminSettings|Maximum number of DAG dependencies that a job can have"
msgstr ""
@ -3234,6 +3237,9 @@ msgstr ""
msgid "AdminSettings|The latest artifacts for all jobs in the most recent successful pipelines in each project are stored and do not expire."
msgstr ""
msgid "AdminSettings|The maximum number of included files per pipeline."
msgstr ""
msgid "AdminSettings|The template for the required pipeline configuration can be one of the GitLab-provided templates, or a custom template added to an instance template repository. %{link_start}How do I create an instance template repository?%{link_end}"
msgstr ""
@ -43695,9 +43701,27 @@ msgstr ""
msgid "Take a look at the documentation to discover all of GitLab’s capabilities."
msgstr ""
msgid "TanukiBot|Ask a question about GitLab"
msgstr ""
msgid "TanukiBot|For example, %{linkStart}what is a fork?%{linkEnd}"
msgstr ""
msgid "TanukiBot|Give feedback"
msgstr ""
msgid "TanukiBot|Sources"
msgstr ""
msgid "TanukiBot|Tanuki Bot"
msgstr ""
msgid "TanukiBot|There was an error communicating with Tanuki Bot. Please reach out to GitLab support for more assistance or try again later."
msgstr ""
msgid "TanukiBot|What is a fork?"
msgstr ""
msgid "Target"
msgstr ""

4
package.json
View File

@ -56,8 +56,8 @@
"@gitlab/cluster-client": "^1.2.0",
"@gitlab/favicon-overlay": "2.0.0",
"@gitlab/fonts": "^1.2.0",
"@gitlab/svgs": "3.40.0",
"@gitlab/ui": "62.4.0",
"@gitlab/svgs": "3.42.0",
"@gitlab/ui": "62.5.0",
"@gitlab/visual-review-tools": "1.7.3",
"@gitlab/web-ide": "0.0.1-dev-20230425040132",
"@mattiasbuelens/web-streams-adapter": "^0.1.0",

56
scripts/frontend/download_fixtures.sh Executable file
View File

@ -0,0 +1,56 @@
#!/usr/bin/env bash
#
# Downloads the most recent frontend fixtures for the current commit, going up the commit parent
# chain up to max-commits commits (defaults to 50 commits).
#
source scripts/packages/helpers.sh
print_help() {
echo "Usage: scripts/frontend/download_fixtures.sh [--branch <branch-name>] [--max-commits <number>]"
echo
echo "Looks for a frontend fixture package in the package registry for commits on a local branch."
echo
echo "If --branch isn't specified, the script will use the current branch as a commit reference."
echo "If --max-commits isn't specified, the default is 50 commits."
return
}
branch="HEAD"
max_commits_count=50
while [ $# -gt 0 ]; do
case "$1" in
--branch)
shift
branch="$1"
;;
--max-commits)
shift
max_commits_count="$1"
;;
*)
print_help
exit
;;
esac
shift
done
for commit_sha in $(git rev-list ${branch} --max-count="${max_commits_count}"); do
API_PACKAGES_BASE_URL=https://gitlab.com/api/v4/projects/278964/packages/generic
FIXTURES_PACKAGE="fixtures-${commit_sha}.tar.gz"
FIXTURES_PACKAGE_URL="${API_PACKAGES_BASE_URL}/fixtures/${commit_sha}/${FIXTURES_PACKAGE}"
echo "Looking for frontend fixtures for commit ${commit_sha}..."
if ! archive_doesnt_exist "${FIXTURES_PACKAGE_URL}" > /dev/null 2>&1; then
echo "We have found frontend fixtures at ${FIXTURES_PACKAGE_URL}!"
read_curl_package "${FIXTURES_PACKAGE_URL}" | extract_package
break
fi
done

60
scripts/gitlab_component_helpers.sh
View File

@ -2,6 +2,9 @@
set -euo pipefail
# Generic helper functions for archives/packages
source scripts/packages/helpers.sh
export CURL_TOKEN_HEADER="${CURL_TOKEN_HEADER:-"JOB-TOKEN"}"
export GITLAB_COM_CANONICAL_PROJECT_ID="278964" # https://gitlab.com/gitlab-org/gitlab
@ -54,63 +57,6 @@ export GITLAB_ASSETS_PACKAGE_URL="${API_PACKAGES_BASE_URL}/assets/${NODE_ENV}-${
# Fixtures constants
export FIXTURES_PATH="tmp/tests/frontend/**/*"
# Generic helper functions
function archive_doesnt_exist() {
local package_url="${1}"
status=$(curl -I --silent --retry 3 --output /dev/null -w "%{http_code}" "${package_url}")
if [[ "${status}" = "200" ]]; then
echoinfo "The archive was found. The server returned status ${status}."
return 1
else
echoinfo "The archive was not found. The server returned status ${status}."
return 0
fi
}
function create_package() {
local archive_filename="${1}"
local paths_to_archive="${2}"
local tar_working_folder="${3:-.}"
echoinfo "Running 'tar -czvf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}'"
tar -czf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}
du -h ${archive_filename}
}
function upload_package() {
local archive_filename="${1}"
local package_url="${2}"
local token_header="${CURL_TOKEN_HEADER}"
local token="${CI_JOB_TOKEN}"
if [[ "${UPLOAD_PACKAGE_FLAG}" = "false" ]]; then
echoerr "The archive ${archive_filename} isn't supposed to be uploaded for this instance (${CI_SERVER_HOST}) & project (${CI_PROJECT_PATH})!"
exit 1
fi
echoinfo "Uploading ${archive_filename} to ${package_url} ..."
curl --fail --silent --retry 3 --header "${token_header}: ${token}" --upload-file "${archive_filename}" "${package_url}"
}
function read_curl_package() {
local package_url="${1}"
echoinfo "Downloading from ${package_url} ..."
curl --fail --silent --retry 3 "${package_url}"
}
function extract_package() {
local tar_working_folder="${1:-.}"
mkdir -p "${tar_working_folder}"
echoinfo "Extracting archive to ${tar_working_folder}"
tar -xz -C ${tar_working_folder} < /dev/stdin
}
# Workhorse functions
function gitlab_workhorse_archive_doesnt_exist() {
archive_doesnt_exist "${GITLAB_WORKHORSE_PACKAGE_URL}"

59
scripts/packages/helpers.sh Normal file
View File

@ -0,0 +1,59 @@
#!/usr/bin/env bash
source scripts/utils.sh
function archive_doesnt_exist() {
local package_url="${1}"
status=$(curl -I --silent --retry 3 --output /dev/null -w "%{http_code}" "${package_url}")
if [[ "${status}" = "200" ]]; then
echoinfo "The archive was found. The server returned status ${status}."
return 1
else
echoinfo "The archive was not found. The server returned status ${status}."
return 0
fi
}
function create_package() {
local archive_filename="${1}"
local paths_to_archive="${2}"
local tar_working_folder="${3:-.}"
echoinfo "Running 'tar -czvf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}'"
tar -czf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}
du -h ${archive_filename}
}
function upload_package() {
local archive_filename="${1}"
local package_url="${2}"
local token_header="${CURL_TOKEN_HEADER}"
local token="${CI_JOB_TOKEN}"
if [[ "${UPLOAD_PACKAGE_FLAG}" = "false" ]]; then
echoerr "The archive ${archive_filename} isn't supposed to be uploaded for this instance (${CI_SERVER_HOST}) & project (${CI_PROJECT_PATH})!"
exit 1
fi
echoinfo "Uploading ${archive_filename} to ${package_url} ..."
curl --fail --silent --retry 3 --header "${token_header}: ${token}" --upload-file "${archive_filename}" "${package_url}"
}
function read_curl_package() {
local package_url="${1}"
echoinfo "Downloading from ${package_url} ..."
curl --fail --silent --retry 3 "${package_url}"
}
function extract_package() {
local tar_working_folder="${1:-.}"
mkdir -p "${tar_working_folder}"
echoinfo "Extracting archive to ${tar_working_folder}"
tar -xz -C ${tar_working_folder} < /dev/stdin
}

13
spec/controllers/admin/application_settings_controller_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Admin::ApplicationSettingsController, :do_not_mock_admin_mode_setting do
RSpec.describe Admin::ApplicationSettingsController, :do_not_mock_admin_mode_setting, feature_category: :shared do
include StubENV
include UsageDataHelpers
@ -398,6 +398,17 @@ RSpec.describe Admin::ApplicationSettingsController, :do_not_mock_admin_mode_set
expect(application_settings.reload.invitation_flow_enforcement).to eq(true)
end
end
context 'maximum includes' do
let(:application_settings) { ApplicationSetting.current }
it 'updates ci_max_includes setting' do
put :update, params: { application_setting: { ci_max_includes: 200 } }
expect(response).to redirect_to(general_admin_application_settings_path)
expect(application_settings.reload.ci_max_includes).to eq(200)
end
end
end
describe 'PUT #reset_registration_token', feature_category: :user_management do

2
spec/features/admin/admin_settings_spec.rb
View File

@ -426,6 +426,7 @@ RSpec.describe 'Admin updates settings', feature_category: :shared do
fill_in 'application_setting_auto_devops_domain', with: 'domain.com'
uncheck 'Keep the latest artifacts for all jobs in the latest successful pipelines'
uncheck 'Enable pipeline suggestion banner'
fill_in 'application_setting_ci_max_includes', with: 200
click_button 'Save changes'
end
@ -433,6 +434,7 @@ RSpec.describe 'Admin updates settings', feature_category: :shared do
expect(current_settings.auto_devops_domain).to eq('domain.com')
expect(current_settings.keep_latest_artifact).to be false
expect(current_settings.suggest_pipeline_enabled).to be false
expect(current_settings.ci_max_includes).to be 200
expect(page).to have_content "Application settings saved successfully"
end

20
spec/features/projects/settings/repository_settings_spec.rb
View File

@ -3,6 +3,8 @@
require 'spec_helper'
RSpec.describe 'Projects > Settings > Repository settings', feature_category: :projects do
include Features::MirroringHelpers
let(:project) { create(:project_empty_repo) }
let(:user) { create(:user) }
let(:role) { :developer }
@ -171,10 +173,8 @@ RSpec.describe 'Projects > Settings > Repository settings', feature_category: :p
end
it 'creates a push mirror that mirrors all branches', :js do
expect(page).to have_css('.js-mirror-protected-hidden[value="0"]', visible: false)
fill_in 'url', with: ssh_url
expect(page).to have_css(".js-mirror-url-hidden[value=\"#{ssh_url}\"]", visible: false)
wait_for_mirror_field_javascript('protected', '0')
fill_and_wait_for_mirror_url_javascript('url', ssh_url)
select 'SSH public key', from: 'Authentication method'
select_direction
@ -191,10 +191,8 @@ RSpec.describe 'Projects > Settings > Repository settings', feature_category: :p
it 'creates a push mirror that only mirrors protected branches', :js do
find('#only_protected_branches').click
expect(page).to have_css('.js-mirror-protected-hidden[value="1"]', visible: false)
fill_in 'url', with: ssh_url
expect(page).to have_css(".js-mirror-url-hidden[value=\"#{ssh_url}\"]", visible: false)
wait_for_mirror_field_javascript('protected', '1')
fill_and_wait_for_mirror_url_javascript('url', ssh_url)
select 'SSH public key', from: 'Authentication method'
select_direction
@ -211,8 +209,7 @@ RSpec.describe 'Projects > Settings > Repository settings', feature_category: :p
it 'creates a push mirror that keeps divergent refs', :js do
select_direction
fill_in 'url', with: ssh_url
expect(page).to have_css(".js-mirror-url-hidden[value=\"#{ssh_url}\"]", visible: false)
fill_and_wait_for_mirror_url_javascript('url', ssh_url)
fill_in 'Password', with: 'password'
check 'Keep divergent refs'
@ -231,8 +228,7 @@ RSpec.describe 'Projects > Settings > Repository settings', feature_category: :p
end
it 'generates an SSH public key on submission', :js do
fill_in 'url', with: ssh_url
expect(page).to have_css(".js-mirror-url-hidden[value=\"#{ssh_url}\"]", visible: false)
fill_and_wait_for_mirror_url_javascript('url', ssh_url)
select 'SSH public key', from: 'Authentication method'

5
spec/frontend/__helpers__/fixtures.js
View File

@ -12,7 +12,10 @@ export function getFixture(relativePath) {
throw new ErrorWithStack(
`Fixture file ${relativePath} does not exist.
Did you run bin/rake frontend:fixtures?`,
Did you run bin/rake frontend:fixtures? You can also download fixtures from the gitlab-org/gitlab package registry.
See https://docs.gitlab.com/ee/development/testing_guide/frontend_testing.html#download-fixtures for more info.
`,
getFixture,
);
}

2
spec/frontend_integration/README.md
View File

@ -22,6 +22,8 @@ We can generate the necessary fixtures and GraphQL schema by running:
bundle exec rake frontend:fixtures gitlab:graphql:schema:dump
```
You can also download those fixtures from the package registry: see [download fixtures](https://docs.gitlab.com/ee/development/testing_guide/frontend_testing.html#download-fixtures) for more info.
Then we can use [Jest](https://jestjs.io/) to run the frontend integration tests:
```shell

2
spec/graphql/types/release_asset_link_type_spec.rb
View File

@ -7,7 +7,7 @@ RSpec.describe GitlabSchema.types['ReleaseAssetLink'] do
it 'has the expected fields' do
expected_fields = %w[
id name url external link_type direct_asset_url direct_asset_path
id name url link_type direct_asset_url direct_asset_path
]
expect(described_class).to include_graphql_fields(*expected_fields)

10
spec/helpers/jira_connect_helper_spec.rb
View File

@ -69,16 +69,6 @@ RSpec.describe JiraConnectHelper, feature_category: :integrations do
)
end
context 'jira_connect_oauth feature is disabled' do
before do
stub_feature_flags(jira_connect_oauth: false)
end
it 'does not assign oauth_metadata' do
expect(oauth_metadata).to be_nil
end
end
context 'with self-managed instance' do
let_it_be(:installation) { create(:jira_connect_installation, instance_url: 'https://gitlab.example.com') }

18
spec/lib/gitlab/ci/config/external/context_spec.rb vendored
View File

@ -24,7 +24,6 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin
context 'with values' do
it { is_expected.to have_attributes(**attributes) }
it { expect(subject.expandset).to eq([]) }
it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) }
it { expect(subject.execution_deadline).to eq(0) }
it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) }
it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) }
@ -37,12 +36,27 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin
it { is_expected.to have_attributes(**attributes) }
it { expect(subject.expandset).to eq([]) }
it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) }
it { expect(subject.execution_deadline).to eq(0) }
it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) }
it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) }
it { expect(subject.pipeline_config).to be_nil }
end
describe 'max_includes' do
it 'returns the default value of application setting `ci_max_includes`' do
expect(subject.max_includes).to eq(150)
end
context 'when application setting `ci_max_includes` is changed' do
before do
stub_application_setting(ci_max_includes: 200)
end
it 'returns the new value of application setting `ci_max_includes`' do
expect(subject.max_includes).to eq(200)
end
end
end
end
describe '#set_deadline' do

42
spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb
View File

@ -64,47 +64,31 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature
shared_examples_for 'mark data consistency location' do |data_consistency|
include_context 'data consistency worker class', data_consistency, :load_balancing_for_test_data_consistency_worker
let(:location) { '0/D525E3A8' }
include_context 'when tracking WAL location reference'
context 'when write was not performed' do
before do
allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(false)
stub_no_writes_performed!
end
context 'when replica hosts are available' do
it 'passes database_replica_location' do
expected_location = {}
Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
expect(lb.host)
.to receive(:database_replica_location)
.and_return(location)
expected_location[lb.name] = location
end
expected_locations = expect_tracked_locations_when_replicas_available
run_middleware
expect(job['wal_locations']).to eq(expected_location)
expect(job['wal_locations']).to eq(expected_locations)
expect(job['wal_location_source']).to eq(:replica)
end
end
context 'when no replica hosts are available' do
it 'passes primary_write_location' do
expected_location = {}
Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
expect(lb).to receive(:host).at_least(:once).and_return(nil)
expect(lb).to receive(:primary_write_location).and_return(location)
expected_location[lb.name] = location
end
expected_locations = expect_tracked_locations_when_no_replicas_available
run_middleware
expect(job['wal_locations']).to eq(expected_location)
expect(job['wal_locations']).to eq(expected_locations)
expect(job['wal_location_source']).to eq(:replica)
end
end
@ -114,23 +98,15 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature
context 'when write was performed' do
before do
allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(true)
stub_write_performed!
end
it 'passes primary write location', :aggregate_failures do
expected_location = {}
Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
expect(lb)
.to receive(:primary_write_location)
.and_return(location)
expected_location[lb.name] = location
end
expected_locations = expect_tracked_locations_from_primary_only
run_middleware
expect(job['wal_locations']).to eq(expected_location)
expect(job['wal_locations']).to eq(expected_locations)
expect(job['wal_location_source']).to eq(:primary)
end

157
spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb
View File

@ -3,16 +3,159 @@
require 'spec_helper'
RSpec.describe Gitlab::Graphql::Subscriptions::ActionCableWithLoadBalancing, feature_category: :shared do
let(:session_class) { ::Gitlab::Database::LoadBalancing::Session }
let(:session) { instance_double(session_class) }
let(:event) { instance_double(::GraphQL::Subscriptions::Event) }
let(:field) { Types::SubscriptionType.fields.each_value.first }
let(:event) { ::GraphQL::Subscriptions::Event.new(name: 'test-event', arguments: {}, field: field) }
let(:object) { build(:project, id: 1) }
let(:action_cable) { instance_double(::ActionCable::Server::Broadcasting) }
subject(:subscriptions) { described_class.new(schema: GitlabSchema) }
it 'forces use of DB primary when executing subscription updates' do
expect(session_class).to receive(:current).and_return(session)
expect(session).to receive(:use_primary!)
include_context 'when tracking WAL location reference'
subscriptions.execute_update('sub:123', event, {})
before do
allow(::ActionCable).to receive(:server).and_return(action_cable)
end
context 'when triggering subscription' do
shared_examples_for 'injecting WAL locations' do
it 'injects correct WAL location into message' do
expect(action_cable).to receive(:broadcast) do |topic, payload|
expect(topic).to match(/^graphql-event/)
expect(Gitlab::Json.parse(payload)).to match({
'wal_locations' => expected_locations,
'payload' => { '__gid__' => be_instance_of(String) }
})
end
subscriptions.execute_all(event, object)
end
end
context 'when feature is disabled' do
before do
stub_feature_flags(graphql_subs_lb: false)
end
it 'does not inject WAL locations' do
expect(action_cable).to receive(:broadcast) do |topic, payload|
expect(topic).to match(/^graphql-event/)
expect(Gitlab::Json.parse(payload)).to match({ '__gid__' => be_instance_of(String) })
end
subscriptions.execute_all(event, object)
end
end
context 'when database load balancing is disabled' do
let!(:expected_locations) { {} }
before do
stub_load_balancing_disabled!
end
it_behaves_like 'injecting WAL locations'
end
context 'when database load balancing is enabled' do
before do
stub_load_balancing_enabled!
end
context 'when write was not performed' do
before do
stub_no_writes_performed!
end
context 'when replica hosts are available' do
let!(:expected_locations) { expect_tracked_locations_when_replicas_available.with_indifferent_access }
it_behaves_like 'injecting WAL locations'
end
context 'when no replica hosts are available' do
let!(:expected_locations) { expect_tracked_locations_when_no_replicas_available.with_indifferent_access }
it_behaves_like 'injecting WAL locations'
end
end
context 'when write was performed' do
let!(:expected_locations) { expect_tracked_locations_from_primary_only.with_indifferent_access }
before do
stub_write_performed!
end
it_behaves_like 'injecting WAL locations'
end
end
end
context 'when handling event' do
def handle_event!
subscriptions.execute_update('sub:123', event, object)
end
before do
allow(action_cable).to receive(:broadcast)
subscriptions.load_action_cable_message(Gitlab::Json.dump({
'wal_locations' => {
'main' => current_location
},
'payload' => {}
}), nil)
end
context 'when feature is disabled' do
before do
stub_feature_flags(graphql_subs_lb: false)
end
it 'uses the primary' do
expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!)
handle_event!
end
end
context 'when WAL locations are not present' do
it 'uses the primary' do
subscriptions.load_action_cable_message(Gitlab::Json.dump({}), nil)
expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!)
handle_event!
end
end
it 'strips out WAL location information before broadcasting payload' do
expect(action_cable).to receive(:broadcast) do |topic, payload|
expect(topic).to eq('graphql-subscription:sub:123')
expect(payload).to eq({ more: false })
end
handle_event!
end
context 'when database replicas are in sync' do
it 'does not use the primary' do
stub_replica_available!(true)
expect(::Gitlab::Database::LoadBalancing::Session.current).not_to receive(:use_primary!)
handle_event!
end
end
context 'when database replicas are not in sync' do
it 'uses the primary' do
stub_replica_available!(false)
expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!)
handle_event!
end
end
end
end

7
spec/models/application_setting_spec.rb
View File

@ -277,6 +277,13 @@ RSpec.describe ApplicationSetting, feature_category: :shared, type: :model do
it { is_expected.to allow_value([true, false]).for(:silent_mode_enabled) }
it { is_expected.not_to allow_value(nil).for(:silent_mode_enabled) }
it { is_expected.to allow_value(0).for(:ci_max_includes) }
it { is_expected.to allow_value(200).for(:ci_max_includes) }
it { is_expected.not_to allow_value('abc').for(:ci_max_includes) }
it { is_expected.not_to allow_value(nil).for(:ci_max_includes) }
it { is_expected.not_to allow_value(10.5).for(:ci_max_includes) }
it { is_expected.not_to allow_value(-1).for(:ci_max_includes) }
context 'when deactivate_dormant_users is enabled' do
before do
stub_application_setting(deactivate_dormant_users: true)

35
spec/requests/api/settings_spec.rb
View File

@ -70,6 +70,7 @@ RSpec.describe API::Settings, 'Settings', :do_not_mock_admin_mode_setting, featu
expect(json_response['projects_api_rate_limit_unauthenticated']).to eq(400)
expect(json_response['silent_mode_enabled']).to be(false)
expect(json_response['valid_runner_registrars']).to match_array(%w(project group))
expect(json_response['ci_max_includes']).to eq(150)
end
end
@ -819,6 +820,40 @@ RSpec.describe API::Settings, 'Settings', :do_not_mock_admin_mode_setting, featu
end
end
context 'with ci_max_includes' do
it 'updates the settings' do
put api("/application/settings", admin), params: {
ci_max_includes: 200
}
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to include(
'ci_max_includes' => 200
)
end
it 'allows a zero value' do
put api("/application/settings", admin), params: {
ci_max_includes: 0
}
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to include(
'ci_max_includes' => 0
)
end
it 'does not allow a nil value' do
put api("/application/settings", admin), params: {
ci_max_includes: nil
}
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response['message']['ci_max_includes'])
.to include(a_string_matching('is not a number'))
end
end
context 'with housekeeping enabled' do
it 'at least one of housekeeping_incremental_repack_period or housekeeping_optimize_repository_period is required' do
put api("/application/settings", admin), params: {

28
spec/support/helpers/features/mirroring_helpers.rb Normal file
View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
# These helpers allow you to set up mirroring.
#
# Usage:
# describe "..." do
# include Features::MirroringHelpers
# ...
#
# fill_and_wait_for_mirror_url_javascript("url", "ssh://user@localhost/project.git")
# wait_for_mirror_field_javascript("protected", "0")
#
module Features
module MirroringHelpers
# input_identifier - identifier of the input field, passed to `fill_in` (can be an ID or a label).
# url - the URL to fill the input field with.
def fill_and_wait_for_mirror_url_javascript(input_identifier, url)
fill_in input_identifier, with: url
wait_for_mirror_field_javascript('url', url)
end
# attribute - can be `url` or `protected`. It's used in the `.js-mirror-<field>-hidden` selector.
# expected_value - the expected value of the hidden field.
def wait_for_mirror_field_javascript(attribute, expected_value)
expect(page).to have_css(".js-mirror-#{attribute}-hidden[value=\"#{expected_value}\"]", visible: :hidden)
end
end
end

68
spec/support/shared_contexts/lib/gitlab/database/load_balancing/wal_tracking_shared_context.rb Normal file
View File

@ -0,0 +1,68 @@
# frozen_string_literal: true
RSpec.shared_context 'when tracking WAL location reference' do
let(:current_location) { '0/D525E3A8' }
around do |example|
Gitlab::Database::LoadBalancing::Session.clear_session
example.run
Gitlab::Database::LoadBalancing::Session.clear_session
end
def expect_tracked_locations_when_replicas_available
{}.tap do |locations|
Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
expect(lb.host).to receive(:database_replica_location).and_return(current_location)
locations[lb.name] = current_location
end
end
end
def expect_tracked_locations_when_no_replicas_available
{}.tap do |locations|
Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
expect(lb).to receive(:host).at_least(:once).and_return(nil)
expect(lb).to receive(:primary_write_location).and_return(current_location)
locations[lb.name] = current_location
end
end
end
def expect_tracked_locations_from_primary_only
{}.tap do |locations|
Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
expect(lb).to receive(:primary_write_location).and_return(current_location)
locations[lb.name] = current_location
end
end
end
def stub_load_balancing_disabled!
Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
allow(lb).to receive(:primary_only?).and_return(true)
end
end
def stub_load_balancing_enabled!
Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
allow(lb).to receive(:primary_only?).and_return(false)
end
end
def stub_no_writes_performed!
allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(false)
end
def stub_write_performed!
allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(true)
end
def stub_replica_available!(available)
::Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
allow(lb).to receive(:select_up_to_date_host).with(current_location).and_return(available)
end
end
end

1
spec/workers/every_sidekiq_worker_spec.rb
View File

@ -262,7 +262,6 @@ RSpec.describe 'Every Sidekiq worker', feature_category: :shared do
'GeoRepositoryDestroyWorker' => 3,
'GitGarbageCollectWorker' => false,
'Gitlab::GithubImport::AdvanceStageWorker' => 3,
'Gitlab::GithubImport::ImportReleaseAttachmentsWorker' => 5,
'Gitlab::GithubImport::Attachments::ImportReleaseWorker' => 5,
'Gitlab::GithubImport::Attachments::ImportNoteWorker' => 5,
'Gitlab::GithubImport::Attachments::ImportIssueWorker' => 5,

50
spec/workers/gitlab/github_import/import_release_attachments_worker_spec.rb
View File

@ -1,50 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::GithubImport::ImportReleaseAttachmentsWorker, feature_category: :importers do
subject(:worker) { described_class.new }
describe '#import' do
let(:import_state) { create(:import_state, :started) }
let(:project) do
instance_double('Project', full_path: 'foo/bar', id: 1, import_state: import_state)
end
let(:client) { instance_double('Gitlab::GithubImport::Client') }
let(:importer) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') }
let(:release_hash) do
{
'record_db_id' => rand(100),
'record_type' => 'Release',
'tag' => 'v1.0',
'text' => <<-TEXT
Some text...
![special-image](https://user-images.githubusercontent.com...)
TEXT
}
end
it 'imports an issue event' do
expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter)
.to receive(:new)
.with(
an_instance_of(Gitlab::GithubImport::Representation::NoteText),
project,
client
)
.and_return(importer)
expect(importer).to receive(:execute)
expect(Gitlab::GithubImport::ObjectCounter)
.to receive(:increment)
.and_call_original
worker.import(project, client, release_hash)
end
end
end

16
yarn.lock
View File

@ -1110,15 +1110,15 @@
stylelint-declaration-strict-value "1.8.0"
stylelint-scss "4.2.0"
"@gitlab/svgs@3.40.0":
version "3.40.0"
resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.40.0.tgz#f1ebb2fcdbb1181550d53f0db827eca1f5060af0"
integrity sha512-9CVkIbV0VnIFfVBjWcW8+nHzpMhHhC73C9mGPEktEPfpEbaaRws2UywgDEH+C2B8Ba1QdBo/aFr68RDu2VwvfA==
"@gitlab/svgs@3.42.0":
version "3.42.0"
resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.42.0.tgz#3168eb46ed9f87abc02b749cc1468902aee8e964"
integrity sha512-hOZtSWKndBP7WO7UxOe+dlMtWK+4J9srLBEAro8ZNcRufm+esF8N8HBPgFvx4ef6GVAW7Ej7qe1wW9kiQQGDyQ==
"@gitlab/ui@62.4.0":
version "62.4.0"
resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-62.4.0.tgz#827e2506a2880fb8d9899b9c18c1d5dfae79a9a7"
integrity sha512-FouJu0o3Za2Hz6WjnZSN+LTMSjpaBCE3SZ5HSBoz0be/WKjS3CX5WW3Qp2NMvNs59xe4V0xg3uWI2g/X0edjTw==
"@gitlab/ui@62.5.0":
version "62.5.0"
resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-62.5.0.tgz#f8038a27a742910418dc48b47020af75a5beaf02"
integrity sha512-JIWzFUV3n0LfqmR5fZ6mqFHqyZ+SlqnhnS74/3weqNos63PQ6CmXe58qW7LxMPj5+B9HWm76Hwd3KEHvv5Q1Aw==
dependencies:
"@popperjs/core" "^2.11.2"
bootstrap-vue "2.23.1"