root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2023-05-01 00:13:08 +00:00
parent ef9cb5bf1b
commit 9e10cf7117
24 changed files with 534 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
app/helpers/safe_format_helper.rb Normal file
View File

@ -0,0 +1,23 @@
# frozen_string_literal: true
module SafeFormatHelper
# Returns a HTML-safe string where +format+ and +args+ are escaped via
# `html_escape` if they are not marked as HTML-safe.
#
# Argument +format+ must not be marked as HTML-safe via `.html_safe`.
#
# Example:
# safe_format('Some %{open}bold%{close} text.', open: '<strong>'.html_safe, close: '</strong>'.html_safe)
# # => 'Some <strong>bold</strong>'
# safe_format('See %{user_input}', user_input: '<b>bold</b>')
# # => 'See &lt;b&gt;bold&lt;/b&gt;
#
def safe_format(format, **args)
raise ArgumentError, 'Argument `format` must not be marked as html_safe!' if format.html_safe?
format(
html_escape(format),
args.transform_values { |value| html_escape(value) }
).html_safe
end
end

2
app/models/user.rb
View File

@ -2237,7 +2237,7 @@ class User < ApplicationRecord
# override, from Devise::Validatable
def password_required?
return false if internal? || project_bot?
return false if internal? || project_bot? || security_policy_bot?
super
end

4
app/services/users/build_service.rb
View File

@ -2,6 +2,8 @@
module Users
class BuildService < BaseService
ALLOWED_USER_TYPES = %i[project_bot security_policy_bot].freeze
delegate :user_default_internal_regex_enabled?,
:user_default_internal_regex_instance,
to: :'Gitlab::CurrentSettings.current_application_settings'
@ -82,7 +84,7 @@ module Users
end
def allowed_user_type?
user_params[:user_type]&.to_sym == :project_bot
ALLOWED_USER_TYPES.include?(user_params[:user_type]&.to_sym)
end
def password_reset

8
config/feature_flags/development/scan_execution_bot_users.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: scan_execution_bot_users
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118089
rollout_issue_url:
milestone: '16.0'
type: development
group: group::security policies
default_enabled: false

3
config/initializers/postgres_partitioning.rb
View File

@ -14,7 +14,8 @@ if Gitlab.ee?
[
IncidentManagement::PendingEscalations::Alert,
IncidentManagement::PendingEscalations::Issue,
Security::Finding
Security::Finding,
Analytics::ValueStreamDashboard::Count
])
else
Gitlab::Database::Partitioning.register_tables(

2
config/sidekiq_queues.yml
View File

@ -511,6 +511,8 @@
- 1
- - security_auto_fix
- 1
- - security_orchestration_configuration_create_bot
- 1
- - security_orchestration_policy_rule_schedule_namespace
- 1
- - security_process_scan_result_policy

11
db/docs/value_stream_dashboard_counts.yml Normal file
View File

@ -0,0 +1,11 @@
---
table_name: value_stream_dashboard_counts
classes:
- Analytics::ValueStreamDashboard::Count
feature_categories:
- value_stream_management
description: Collects counts for various objects in GitLab for the Value Stream Dashboard
feature.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118220
milestone: '16.0'
gitlab_schema: gitlab_main

7
db/migrate/20230419192748_add_bot_user_id_to_security_orchestration_policy_configurations.rb Normal file
View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
class AddBotUserIdToSecurityOrchestrationPolicyConfigurations < Gitlab::Database::Migration[2.1]
def change
add_column :security_orchestration_policy_configurations, :bot_user_id, :integer
end
end

24
db/migrate/20230419193807_add_foreign_key_for_bot_user_id_to_security_orchestration_policy_configurations.rb Normal file
View File

@ -0,0 +1,24 @@
# frozen_string_literal: true
class AddForeignKeyForBotUserIdToSecurityOrchestrationPolicyConfigurations < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
INDEX_NAME = 'index_security_policy_configurations_on_bot_user_id'
def up
add_concurrent_foreign_key :security_orchestration_policy_configurations, :users, column: :bot_user_id,
on_delete: :nullify
add_concurrent_index :security_orchestration_policy_configurations, :bot_user_id,
where: "security_orchestration_policy_configurations.bot_user_id IS NOT NULL",
name: INDEX_NAME
end
def down
remove_concurrent_index_by_name :security_orchestration_policy_configurations, INDEX_NAME
with_lock_retries do
remove_foreign_key_if_exists :security_orchestration_policy_configurations, column: :bot_user_id
end
end
end

26
db/migrate/20230420131608_add_partitioned_vsd_counts_table.rb Normal file
View File

@ -0,0 +1,26 @@
# frozen_string_literal: true
class AddPartitionedVsdCountsTable < Gitlab::Database::Migration[2.1]
include Gitlab::Database::PartitioningMigrationHelpers
def up
execute(<<~SQL)
CREATE TABLE value_stream_dashboard_counts (
id bigserial NOT NULL,
namespace_id bigint NOT NULL,
count bigint NOT NULL,
recorded_at timestamp with time zone NOT NULL,
metric smallint NOT NULL,
PRIMARY KEY (namespace_id, metric, recorded_at, count, id)
) PARTITION BY RANGE (recorded_at);
SQL
min_date = Date.today
max_date = Date.today + 6.months
create_daterange_partitions('value_stream_dashboard_counts', 'recorded_at', min_date, max_date)
end
def down
drop_table :value_stream_dashboard_counts
end
end

1
db/schema_migrations/20230419192748 Normal file
View File

@ -0,0 +1 @@
732417f422b8f73df9e7a56ce7690ba8cc6a87b5e773fa356a0a50ed72dcace2

1
db/schema_migrations/20230419193807 Normal file
View File

@ -0,0 +1 @@
26904715659fd8d5bf3bf912781c9ae1cb61e8c990b46f12228aabdeb4f26ce7

1
db/schema_migrations/20230420131608 Normal file
View File

@ -0,0 +1 @@
91967e2d402dba78ae04cdfe27b0ad68e582b8daf23dd252ee8087f82ad3f39f

33
db/structure.sql
View File

@ -603,6 +603,15 @@ CREATE TABLE security_findings (
)
PARTITION BY LIST (partition_number);
CREATE TABLE value_stream_dashboard_counts (
id bigint NOT NULL,
namespace_id bigint NOT NULL,
count bigint NOT NULL,
recorded_at timestamp with time zone NOT NULL,
metric smallint NOT NULL
)
PARTITION BY RANGE (recorded_at);
CREATE TABLE verification_codes (
created_at timestamp with time zone DEFAULT now() NOT NULL,
visitor_id_code text NOT NULL,
@ -21168,13 +21177,13 @@ CREATE TABLE project_settings (
CONSTRAINT check_2981f15877 CHECK ((char_length(jitsu_key) <= 100)),
CONSTRAINT check_3a03e7557a CHECK ((char_length(previous_default_branch) <= 4096)),
CONSTRAINT check_3ca5cbffe6 CHECK ((char_length(issue_branch_template) <= 255)),
CONSTRAINT check_4b142e71f3 CHECK ((char_length(product_analytics_data_collector_host) <= 255)),
CONSTRAINT check_67292e4b99 CHECK ((char_length(mirror_branch_regex) <= 255)),
CONSTRAINT check_acb7fad2f9 CHECK ((char_length(product_analytics_instrumentation_key) <= 255)),
CONSTRAINT check_b09644994b CHECK ((char_length(squash_commit_template) <= 500)),
CONSTRAINT check_bde223416c CHECK ((show_default_award_emojis IS NOT NULL)),
CONSTRAINT check_eaf7cfb6a7 CHECK ((char_length(merge_commit_template) <= 500)),
CONSTRAINT check_4b142e71f3 CHECK ((char_length(product_analytics_data_collector_host) <= 255)),
CONSTRAINT check_ea15225016 CHECK ((char_length(jitsu_project_xid) <= 255)),
CONSTRAINT check_eaf7cfb6a7 CHECK ((char_length(merge_commit_template) <= 500)),
CONSTRAINT check_f4499c0fa4 CHECK ((char_length(jitsu_host) <= 255)),
CONSTRAINT check_f5495015f5 CHECK ((char_length(jitsu_administrator_email) <= 255)),
CONSTRAINT check_f9df7bcee2 CHECK ((char_length(cube_api_base_url) <= 512))
@ -22334,6 +22343,7 @@ CREATE TABLE security_orchestration_policy_configurations (
updated_at timestamp with time zone NOT NULL,
configured_at timestamp with time zone,
namespace_id bigint,
bot_user_id integer,
CONSTRAINT cop_configs_project_or_namespace_existence CHECK (((project_id IS NULL) <> (namespace_id IS NULL)))
);
@ -23774,6 +23784,15 @@ CREATE SEQUENCE users_statistics_id_seq
ALTER SEQUENCE users_statistics_id_seq OWNED BY users_statistics.id;
CREATE SEQUENCE value_stream_dashboard_counts_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE value_stream_dashboard_counts_id_seq OWNED BY value_stream_dashboard_counts.id;
CREATE TABLE vulnerabilities (
id bigint NOT NULL,
milestone_id bigint,
@ -25715,6 +25734,8 @@ ALTER TABLE ONLY users_star_projects ALTER COLUMN id SET DEFAULT nextval('users_
ALTER TABLE ONLY users_statistics ALTER COLUMN id SET DEFAULT nextval('users_statistics_id_seq'::regclass);
ALTER TABLE ONLY value_stream_dashboard_counts ALTER COLUMN id SET DEFAULT nextval('value_stream_dashboard_counts_id_seq'::regclass);
ALTER TABLE ONLY vulnerabilities ALTER COLUMN id SET DEFAULT nextval('vulnerabilities_id_seq'::regclass);
ALTER TABLE ONLY vulnerability_advisories ALTER COLUMN id SET DEFAULT nextval('vulnerability_advisories_id_seq'::regclass);
@ -28225,6 +28246,9 @@ ALTER TABLE ONLY users_star_projects
ALTER TABLE ONLY users_statistics
ADD CONSTRAINT users_statistics_pkey PRIMARY KEY (id);
ALTER TABLE ONLY value_stream_dashboard_counts
ADD CONSTRAINT value_stream_dashboard_counts_pkey PRIMARY KEY (namespace_id, metric, recorded_at, count, id);
ALTER TABLE ONLY verification_codes
ADD CONSTRAINT verification_codes_pkey PRIMARY KEY (created_at, visitor_id_code, code, phone);
@ -32307,6 +32331,8 @@ CREATE INDEX index_secure_ci_builds_on_user_id_name_created_at ON ci_builds USIN
CREATE INDEX index_security_ci_builds_on_name_and_id_parser_features ON ci_builds USING btree (name, id) WHERE (((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text, ('secret_detection'::character varying)::text, ('coverage_fuzzing'::character varying)::text, ('license_scanning'::character varying)::text, ('apifuzzer_fuzz'::character varying)::text, ('apifuzzer_fuzz_dnd'::character varying)::text])) AND ((type)::text = 'Ci::Build'::text));
CREATE INDEX index_security_policy_configurations_on_bot_user_id ON security_orchestration_policy_configurations USING btree (bot_user_id) WHERE (bot_user_id IS NOT NULL);
CREATE INDEX index_security_scans_for_non_purged_records ON security_scans USING btree (created_at, id) WHERE (status <> 6);
CREATE INDEX index_security_scans_on_created_at ON security_scans USING btree (created_at);
@ -34544,6 +34570,9 @@ ALTER TABLE ONLY epics
ALTER TABLE ONLY environments
ADD CONSTRAINT fk_01a033a308 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE SET NULL;
ALTER TABLE ONLY security_orchestration_policy_configurations
ADD CONSTRAINT fk_0247484b90 FOREIGN KEY (bot_user_id) REFERENCES users(id) ON DELETE SET NULL;
ALTER TABLE ONLY agent_user_access_project_authorizations
ADD CONSTRAINT fk_0250c0ad51 FOREIGN KEY (agent_id) REFERENCES cluster_agents(id) ON DELETE CASCADE;

9
doc/development/i18n/externalization.md
View File

@ -561,9 +561,8 @@ To include formatting in the translated string, you can do the following:
- In Ruby/HAML:
```ruby
html_escape(_('Some %{strongOpen}bold%{strongClose} text.')) % { strongOpen: '<strong>'.html_safe, strongClose: '</strong>'.html_safe }
# => 'Some <strong>bold</strong> text.'
safe_format(_('Some %{strongOpen}bold%{strongClose} text.'), strongOpen: '<strong>'.html_safe, strongClose: '</strong>'.html_safe)
# => 'Some <strong>bold</strong> text.'
```
- In JavaScript:
@ -801,8 +800,8 @@ translatable in certain languages.
```haml
- zones_link_url = 'https://cloud.google.com/compute/docs/regions-zones/regions-zones'
- zones_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: zones_link_url }
= html_escape(s_('ClusterIntegration|Learn more about %{zones_link_start}zones%{zones_link_end}')) % { zones_link_start: zones_link_start, zones_link_end: '</a>'.html_safe }
- zones_link_start = safe_format('<a href="%{url}" target="_blank" rel="noopener noreferrer">', url: zones_link_url)
= safe_format(s_('ClusterIntegration|Learn more about %{zones_link_start}zones%{zones_link_end}'), zones_link_start: zones_link_start, zones_link_end: '</a>'.html_safe)
```
- In Vue, instead of:

2
lib/api/api.rb
View File

@ -337,7 +337,7 @@ module API
mount ::API::Todos
mount ::API::UsageData
mount ::API::UsageDataNonSqlMetrics
mount ::API::Ml::Mlflow
mount ::API::Ml::Mlflow::Entrypoint
end
mount ::API::Internal::Base

297
lib/api/ml/mlflow.rb
View File

@ -1,297 +0,0 @@
# frozen_string_literal: true
require 'mime/types'
module API
# MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api
module Ml
class Mlflow < ::API::Base
include APIGuard
# The first part of the url is the namespace, the second part of the URL is what the MLFlow client calls
MLFLOW_API_PREFIX = ':id/ml/mlflow/api/2.0/mlflow/'
allow_access_with_scope :api
allow_access_with_scope :read_api, if: -> (request) { request.get? || request.head? }
feature_category :mlops
content_type :json, 'application/json'
default_format :json
before do
# MLFlow Client considers any status code different than 200 an error, even 201
status 200
authenticate!
not_found! unless Feature.enabled?(:ml_experiment_tracking, user_project)
end
rescue_from ActiveRecord::ActiveRecordError do |e|
invalid_parameter!(e.message)
end
helpers do
def resource_not_found!
render_structured_api_error!({ error_code: 'RESOURCE_DOES_NOT_EXIST' }, 404)
end
def resource_already_exists!
render_structured_api_error!({ error_code: 'RESOURCE_ALREADY_EXISTS' }, 400)
end
def invalid_parameter!(message = nil)
render_structured_api_error!({ error_code: 'INVALID_PARAMETER_VALUE', message: message }, 400)
end
def experiment_repository
::Ml::ExperimentTracking::ExperimentRepository.new(user_project, current_user)
end
def candidate_repository
::Ml::ExperimentTracking::CandidateRepository.new(user_project, current_user)
end
def experiment
@experiment ||= find_experiment!(params[:experiment_id], params[:experiment_name])
end
def candidate
@candidate ||= find_candidate!(params[:run_id])
end
def find_experiment!(iid, name)
experiment_repository.by_iid_or_name(iid: iid, name: name) || resource_not_found!
end
def find_candidate!(eid)
candidate_repository.by_eid(eid) || resource_not_found!
end
def packages_url
path = api_v4_projects_packages_generic_package_version_path(
id: user_project.id, package_name: '', file_name: ''
)
path = path.delete_suffix('/package_version')
"#{request.base_url}#{path}"
end
end
params do
requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'API to interface with MLFlow Client, REST API version 1.28.0' do
detail 'This feature is gated by :ml_experiment_tracking.'
end
namespace MLFLOW_API_PREFIX do
resource :experiments do
desc 'Fetch experiment by experiment_id' do
success Entities::Ml::Mlflow::GetExperiment
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment'
end
params do
optional :experiment_id, type: String, default: '', desc: 'Experiment ID, in reference to the project'
end
get 'get', urgency: :low do
present experiment, with: Entities::Ml::Mlflow::GetExperiment
end
desc 'Fetch experiment by experiment_name' do
success Entities::Ml::Mlflow::GetExperiment
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment-by-name'
end
params do
optional :experiment_name, type: String, default: '', desc: 'Experiment name'
end
get 'get-by-name', urgency: :low do
present experiment, with: Entities::Ml::Mlflow::GetExperiment
end
desc 'List experiments' do
success Entities::Ml::Mlflow::ListExperiment
detail 'https://www.mlflow.org/docs/latest/rest-api.html#list-experiments'
end
get 'list', urgency: :low do
response = { experiments: experiment_repository.all }
present response, with: Entities::Ml::Mlflow::ListExperiment
end
desc 'Create experiment' do
success Entities::Ml::Mlflow::NewExperiment
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#create-experiment'
end
params do
requires :name, type: String, desc: 'Experiment name'
optional :tags, type: Array, desc: 'Tags with information about the experiment'
optional :artifact_location, type: String, desc: 'This will be ignored'
end
post 'create', urgency: :low do
present experiment_repository.create!(params[:name], params[:tags]),
with: Entities::Ml::Mlflow::NewExperiment
rescue ActiveRecord::RecordInvalid
resource_already_exists!
end
desc 'Sets a tag for an experiment.' do
summary 'Sets a tag for an experiment. '
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-experiment-tag'
end
params do
requires :experiment_id, type: String, desc: 'ID of the experiment.'
requires :key, type: String, desc: 'Name for the tag.'
requires :value, type: String, desc: 'Value for the tag.'
end
post 'set-experiment-tag', urgency: :low do
bad_request! unless experiment_repository.add_tag!(experiment, params[:key], params[:value])
{}
end
end
resource :runs do
desc 'Creates a Run.' do
success Entities::Ml::Mlflow::Run
detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#create-run'
end
params do
requires :experiment_id, type: Integer,
desc: 'Id for the experiment, relative to the project'
optional :start_time, type: Integer,
desc: 'Unix timestamp in milliseconds of when the run started.',
default: 0
optional :user_id, type: String, desc: 'This will be ignored'
optional :tags, type: Array, desc: 'Tags are stored, but not displayed'
optional :run_name, type: String, desc: 'A name for this run'
end
post 'create', urgency: :low do
present candidate_repository.create!(experiment, params[:start_time], params[:tags], params[:run_name]),
with: Entities::Ml::Mlflow::Run, packages_url: packages_url
end
desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do
success Entities::Ml::Mlflow::Run
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-run'
end
params do
requires :run_id, type: String, desc: 'UUID of the candidate.'
optional :run_uuid, type: String, desc: 'This parameter is ignored'
end
get 'get', urgency: :low do
present candidate, with: Entities::Ml::Mlflow::Run, packages_url: packages_url
end
desc 'Updates a Run.' do
success Entities::Ml::Mlflow::UpdateRun
detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#update-run'
end
params do
requires :run_id, type: String, desc: 'UUID of the candidate.'
optional :status, type: String,
values: ::Ml::Candidate.statuses.keys.map(&:upcase),
desc: "Status of the run. Accepts: " \
"#{::Ml::Candidate.statuses.keys.map(&:upcase)}."
optional :end_time, type: Integer, desc: 'Ending time of the run'
end
post 'update', urgency: :low do
candidate_repository.update(candidate, params[:status], params[:end_time])
present candidate, with: Entities::Ml::Mlflow::UpdateRun, packages_url: packages_url
end
desc 'Logs a metric to a run.' do
summary 'Log a metric for a run. A metric is a key-value pair (string key, float value) with an '\
'associated timestamp. Examples include the various metrics that represent ML model accuracy. '\
'A metric can be logged multiple times.'
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-metric'
end
params do
requires :run_id, type: String, desc: 'UUID of the run.'
requires :key, type: String, desc: 'Name for the metric.'
requires :value, type: Float, desc: 'Value of the metric.'
requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded'
optional :step, type: Integer, desc: 'Step at which the metric was recorded'
end
post 'log-metric', urgency: :low do
candidate_repository.add_metric!(
candidate,
params[:key],
params[:value],
params[:timestamp],
params[:step]
)
{}
end
desc 'Logs a parameter to a run.' do
summary 'Log a param used for a run. A param is a key-value pair (string key, string value). '\
'Examples include hyperparameters used for ML model training and constant dates and values '\
'used in an ETL pipeline. A param can be logged only once for a run, duplicate will be .'\
'ignored'
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param'
end
params do
requires :run_id, type: String, desc: 'UUID of the run.'
requires :key, type: String, desc: 'Name for the parameter.'
requires :value, type: String, desc: 'Value for the parameter.'
end
post 'log-parameter', urgency: :low do
bad_request! unless candidate_repository.add_param!(candidate, params[:key], params[:value])
{}
end
desc 'Sets a tag for a run.' do
summary 'Sets a tag for a run. '
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-tag'
end
params do
requires :run_id, type: String, desc: 'UUID of the run.'
requires :key, type: String, desc: 'Name for the tag.'
requires :value, type: String, desc: 'Value for the tag.'
end
post 'set-tag', urgency: :low do
bad_request! unless candidate_repository.add_tag!(candidate, params[:key], params[:value])
{}
end
desc 'Logs multiple parameters and metrics.' do
summary 'Log a batch of metrics and params for a run. Validation errors will block the entire batch, '\
'duplicate errors will be ignored.'
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param'
end
params do
requires :run_id, type: String, desc: 'UUID of the run.'
optional :metrics, type: Array, default: [] do
requires :key, type: String, desc: 'Name for the metric.'
requires :value, type: Float, desc: 'Value of the metric.'
requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded'
optional :step, type: Integer, desc: 'Step at which the metric was recorded'
end
optional :params, type: Array, default: [] do
requires :key, type: String, desc: 'Name for the metric.'
requires :value, type: String, desc: 'Value of the metric.'
end
end
post 'log-batch', urgency: :low do
candidate_repository.add_metrics(candidate, params[:metrics])
candidate_repository.add_params(candidate, params[:params])
candidate_repository.add_tags(candidate, params[:tags])
{}
end
end
end
end
end
end
end

54
lib/api/ml/mlflow/api_helpers.rb Normal file
View File

@ -0,0 +1,54 @@
# frozen_string_literal: true
module API
module Ml
module Mlflow
module ApiHelpers
def resource_not_found!
render_structured_api_error!({ error_code: 'RESOURCE_DOES_NOT_EXIST' }, 404)
end
def resource_already_exists!
render_structured_api_error!({ error_code: 'RESOURCE_ALREADY_EXISTS' }, 400)
end
def invalid_parameter!(message = nil)
render_structured_api_error!({ error_code: 'INVALID_PARAMETER_VALUE', message: message }, 400)
end
def experiment_repository
::Ml::ExperimentTracking::ExperimentRepository.new(user_project, current_user)
end
def candidate_repository
::Ml::ExperimentTracking::CandidateRepository.new(user_project, current_user)
end
def experiment
@experiment ||= find_experiment!(params[:experiment_id], params[:experiment_name])
end
def candidate
@candidate ||= find_candidate!(params[:run_id])
end
def find_experiment!(iid, name)
experiment_repository.by_iid_or_name(iid: iid, name: name) || resource_not_found!
end
def find_candidate!(eid)
candidate_repository.by_eid(eid) || resource_not_found!
end
def packages_url
path = api_v4_projects_packages_generic_package_version_path(
id: user_project.id, package_name: '', file_name: ''
)
path = path.delete_suffix('/package_version')
"#{request.base_url}#{path}"
end
end
end
end
end

51
lib/api/ml/mlflow/entrypoint.rb Normal file
View File

@ -0,0 +1,51 @@
# frozen_string_literal: true
module API
# MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api
module Ml
module Mlflow
class Entrypoint < ::API::Base
include APIGuard
# The first part of the url is the namespace, the second part of the URL is what the MLFlow client calls
MLFLOW_API_PREFIX = ':id/ml/mlflow/api/2.0/mlflow/'
helpers ::API::Ml::Mlflow::ApiHelpers
allow_access_with_scope :api
allow_access_with_scope :read_api, if: ->(request) { request.get? || request.head? }
feature_category :mlops
content_type :json, 'application/json'
default_format :json
before do
# MLFlow Client considers any status code different than 200 an error, even 201
status 200
authenticate!
not_found! unless Feature.enabled?(:ml_experiment_tracking, user_project)
end
rescue_from ActiveRecord::ActiveRecordError do |e|
invalid_parameter!(e.message)
end
params do
requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'API to interface with MLFlow Client, REST API version 1.28.0' do
detail 'This feature is gated by :ml_experiment_tracking.'
end
namespace MLFLOW_API_PREFIX do
mount ::API::Ml::Mlflow::Experiments
mount ::API::Ml::Mlflow::Runs
end
end
end
end
end
end

80
lib/api/ml/mlflow/experiments.rb Normal file
View File

@ -0,0 +1,80 @@
# frozen_string_literal: true
require 'mime/types'
module API
# MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api
module Ml
module Mlflow
class Experiments < ::API::Base
feature_category :mlops
resource :experiments do
desc 'Fetch experiment by experiment_id' do
success Entities::Ml::Mlflow::GetExperiment
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment'
end
params do
optional :experiment_id, type: String, default: '', desc: 'Experiment ID, in reference to the project'
end
get 'get', urgency: :low do
present experiment, with: Entities::Ml::Mlflow::GetExperiment
end
desc 'Fetch experiment by experiment_name' do
success Entities::Ml::Mlflow::GetExperiment
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment-by-name'
end
params do
optional :experiment_name, type: String, default: '', desc: 'Experiment name'
end
get 'get-by-name', urgency: :low do
present experiment, with: Entities::Ml::Mlflow::GetExperiment
end
desc 'List experiments' do
success Entities::Ml::Mlflow::ListExperiment
detail 'https://www.mlflow.org/docs/latest/rest-api.html#list-experiments'
end
get 'list', urgency: :low do
response = { experiments: experiment_repository.all }
present response, with: Entities::Ml::Mlflow::ListExperiment
end
desc 'Create experiment' do
success Entities::Ml::Mlflow::NewExperiment
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#create-experiment'
end
params do
requires :name, type: String, desc: 'Experiment name'
optional :tags, type: Array, desc: 'Tags with information about the experiment'
optional :artifact_location, type: String, desc: 'This will be ignored'
end
post 'create', urgency: :low do
present experiment_repository.create!(params[:name], params[:tags]),
with: Entities::Ml::Mlflow::NewExperiment
rescue ActiveRecord::RecordInvalid
resource_already_exists!
end
desc 'Sets a tag for an experiment.' do
summary 'Sets a tag for an experiment. '
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-experiment-tag'
end
params do
requires :experiment_id, type: String, desc: 'ID of the experiment.'
requires :key, type: String, desc: 'Name for the tag.'
requires :value, type: String, desc: 'Value for the tag.'
end
post 'set-experiment-tag', urgency: :low do
bad_request! unless experiment_repository.add_tag!(experiment, params[:key], params[:value])
{}
end
end
end
end
end
end

152
lib/api/ml/mlflow/runs.rb Normal file
View File

@ -0,0 +1,152 @@
# frozen_string_literal: true
require 'mime/types'
module API
# MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api
module Ml
module Mlflow
class Runs < ::API::Base
feature_category :mlops
resource :runs do
desc 'Creates a Run.' do
success Entities::Ml::Mlflow::Run
detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#create-run'
end
params do
requires :experiment_id, type: Integer,
desc: 'Id for the experiment, relative to the project'
optional :start_time, type: Integer,
desc: 'Unix timestamp in milliseconds of when the run started.',
default: 0
optional :user_id, type: String, desc: 'This will be ignored'
optional :tags, type: Array, desc: 'Tags are stored, but not displayed'
optional :run_name, type: String, desc: 'A name for this run'
end
post 'create', urgency: :low do
present candidate_repository.create!(experiment, params[:start_time], params[:tags], params[:run_name]),
with: Entities::Ml::Mlflow::Run, packages_url: packages_url
end
desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do
success Entities::Ml::Mlflow::Run
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-run'
end
params do
requires :run_id, type: String, desc: 'UUID of the candidate.'
optional :run_uuid, type: String, desc: 'This parameter is ignored'
end
get 'get', urgency: :low do
present candidate, with: Entities::Ml::Mlflow::Run, packages_url: packages_url
end
desc 'Updates a Run.' do
success Entities::Ml::Mlflow::UpdateRun
detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#update-run'
end
params do
requires :run_id, type: String, desc: 'UUID of the candidate.'
optional :status, type: String,
values: ::Ml::Candidate.statuses.keys.map(&:upcase),
desc: "Status of the run. Accepts: " \
"#{::Ml::Candidate.statuses.keys.map(&:upcase)}."
optional :end_time, type: Integer, desc: 'Ending time of the run'
end
post 'update', urgency: :low do
candidate_repository.update(candidate, params[:status], params[:end_time])
present candidate, with: Entities::Ml::Mlflow::UpdateRun, packages_url: packages_url
end
desc 'Logs a metric to a run.' do
summary 'Log a metric for a run. A metric is a key-value pair (string key, float value) with an ' \
'associated timestamp. Examples include the various metrics that represent ML model accuracy. ' \
'A metric can be logged multiple times.'
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-metric'
end
params do
requires :run_id, type: String, desc: 'UUID of the run.'
requires :key, type: String, desc: 'Name for the metric.'
requires :value, type: Float, desc: 'Value of the metric.'
requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded'
optional :step, type: Integer, desc: 'Step at which the metric was recorded'
end
post 'log-metric', urgency: :low do
candidate_repository.add_metric!(
candidate,
params[:key],
params[:value],
params[:timestamp],
params[:step]
)
{}
end
desc 'Logs a parameter to a run.' do
summary 'Log a param used for a run. A param is a key-value pair (string key, string value). ' \
'Examples include hyperparameters used for ML model training and constant dates and values ' \
'used in an ETL pipeline. A param can be logged only once for a run, duplicate will be .' \
'ignored'
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param'
end
params do
requires :run_id, type: String, desc: 'UUID of the run.'
requires :key, type: String, desc: 'Name for the parameter.'
requires :value, type: String, desc: 'Value for the parameter.'
end
post 'log-parameter', urgency: :low do
bad_request! unless candidate_repository.add_param!(candidate, params[:key], params[:value])
{}
end
desc 'Sets a tag for a run.' do
summary 'Sets a tag for a run. '
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-tag'
end
params do
requires :run_id, type: String, desc: 'UUID of the run.'
requires :key, type: String, desc: 'Name for the tag.'
requires :value, type: String, desc: 'Value for the tag.'
end
post 'set-tag', urgency: :low do
bad_request! unless candidate_repository.add_tag!(candidate, params[:key], params[:value])
{}
end
desc 'Logs multiple parameters and metrics.' do
summary 'Log a batch of metrics and params for a run. Validation errors will block the entire batch, ' \
'duplicate errors will be ignored.'
detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param'
end
params do
requires :run_id, type: String, desc: 'UUID of the run.'
optional :metrics, type: Array, default: [] do
requires :key, type: String, desc: 'Name for the metric.'
requires :value, type: Float, desc: 'Value of the metric.'
requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded'
optional :step, type: Integer, desc: 'Step at which the metric was recorded'
end
optional :params, type: Array, default: [] do
requires :key, type: String, desc: 'Name for the metric.'
requires :value, type: String, desc: 'Value of the metric.'
end
end
post 'log-batch', urgency: :low do
candidate_repository.add_metrics(candidate, params[:metrics])
candidate_repository.add_params(candidate, params[:params])
candidate_repository.add_tags(candidate, params[:tags])
{}
end
end
end
end
end
end

4
spec/db/schema_spec.rb
View File

@ -127,8 +127,8 @@ RSpec.describe 'Database schema', feature_category: :database do
# Fixes performance issues with the deletion of web-hooks with many log entries
web_hook_logs: %w[web_hook_id],
webauthn_registrations: %w[u2f_registration_id], # this column will be dropped
ml_candidates: %w[internal_id]
ml_candidates: %w[internal_id],
value_stream_dashboard_counts: %w[namespace_id]
}.with_indifferent_access.freeze
context 'for table' do

41
spec/helpers/safe_format_helper_spec.rb Normal file
View File

@ -0,0 +1,41 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe SafeFormatHelper, feature_category: :shared do
describe '#safe_format' do
shared_examples 'safe formatting' do |format, args:, result:|
subject { helper.safe_format(format, **args) }
it { is_expected.to eq(result) }
it { is_expected.to be_html_safe }
end
it_behaves_like 'safe formatting', '', args: {}, result: ''
it_behaves_like 'safe formatting', 'Foo', args: {}, result: 'Foo'
it_behaves_like 'safe formatting', '<b>strong</b>', args: {},
result: '&lt;b&gt;strong&lt;/b&gt;'
it_behaves_like 'safe formatting', '%{open}strong%{close}',
args: { open: '<b>'.html_safe, close: '</b>'.html_safe },
result: '<b>strong</b>'
it_behaves_like 'safe formatting', '%{open}strong%{close} %{user_input}',
args: { open: '<b>'.html_safe, close: '</b>'.html_safe,
user_input: '<a href="">link</a>' },
result: '<b>strong</b> &lt;a href=&quot;&quot;&gt;link&lt;/a&gt;'
context 'when format is marked as html_safe' do
let(:format) { '<b>strong</b>'.html_safe }
let(:args) { {} }
it 'raises an error' do
message = 'Argument `format` must not be marked as html_safe!'
expect { helper.safe_format(format, **args) }
.to raise_error ArgumentError, message
end
end
end
end

8
spec/models/user_spec.rb
View File

@ -7077,6 +7077,14 @@ RSpec.describe User, feature_category: :user_profile do
it_behaves_like 'does not require password to be present'
end
context 'when user is a security_policy bot user' do
before do
user.update!(user_type: 'security_policy_bot')
end
it_behaves_like 'does not require password to be present'
end
end
describe 'can_trigger_notifications?' do