root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-12-01 15:07:35 +00:00
parent 4ee706fcd1
commit 08489a6db8
56 changed files with 1361 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/assets/javascripts/ide/components/new_dropdown/index.vue
View File

@ -80,7 +80,7 @@ export default {
@click="createNewItem('blob')"
/>
</li>
<li><upload :path="path" @create="createTempEntry" /></li>
<upload :path="path" @create="createTempEntry" />
<li>
<item-button
:label="__('New directory')"

4
app/assets/javascripts/ide/components/new_dropdown/upload.vue
View File

@ -65,7 +65,7 @@ export default {
</script>
<template>
<div>
<li>
<item-button
:class="buttonCssClasses"
:show-label="showLabel"
@ -84,5 +84,5 @@ export default {
data-qa-selector="file_upload_field"
@change="openFile"
/>
</div>
</li>
</template>

1
app/assets/javascripts/ide/lib/editor_options.js
View File

@ -27,7 +27,6 @@ export const defaultDiffEditorOptions = {
};
export const defaultModelOptions = {
endOfLine: 0,
insertFinalNewline: true,
trimTrailingWhitespace: false,
};

36
app/assets/javascripts/ml/experiment_tracking/components/experiment.vue
View File

@ -1,36 +0,0 @@
<script>
import { GlTable } from '@gitlab/ui';
import IncubationAlert from './incubation_alert.vue';
export default {
name: 'ShowMlExperiment',
components: {
GlTable,
IncubationAlert,
},
inject: ['candidates', 'metricNames', 'paramNames'],
computed: {
fields() {
return [...this.paramNames, ...this.metricNames];
},
},
};
</script>
<template>
<div>
<incubation-alert />
<h3>
{{ __('Experiment Candidates') }}
</h3>
<gl-table
:fields="fields"
:items="candidates"
:empty-text="__('This Experiment has no logged Candidates')"
show-empty
class="gl-mt-0!"
/>
</div>
</template>

6
app/assets/javascripts/ml/experiment_tracking/components/incubation_alert.vue
View File

@ -8,8 +8,8 @@ export default {
contentLabel: __(
'GitLab incubates features to explore new use cases. These features are updated regularly, and support is limited',
),
learnMoreLabel: __('Learn More'),
feedbackLabel: __('Feedback and Updates'),
learnMoreLabel: __('Learn more'),
feedbackLabel: __('Feedback'),
},
name: 'MlopsIncubationAlert',
components: { GlAlert, GlLink },
@ -37,7 +37,7 @@ export default {
:title="$options.i18n.titleLabel"
variant="warning"
:primary-button-text="$options.i18n.feedbackLabel"
primary-button-link="https://gitlab.com/groups/gitlab-org/-/epics/8560"
primary-button-link="https://gitlab.com/gitlab-org/gitlab/-/issues/381660"
@dismiss="dismissAlert"
>
{{ $options.i18n.contentLabel }}

94
app/assets/javascripts/ml/experiment_tracking/components/ml_candidate.vue Normal file
View File

@ -0,0 +1,94 @@
<script>
import { GlLink } from '@gitlab/ui';
import { __ } from '~/locale';
import IncubationAlert from './incubation_alert.vue';
export default {
name: 'MlCandidate',
components: {
IncubationAlert,
GlLink,
},
inject: ['candidate'],
i18n: {
titleLabel: __('Model candidate details'),
infoLabel: __('Info'),
idLabel: __('ID'),
statusLabel: __('Status'),
experimentLabel: __('Experiment'),
artifactsLabel: __('Artifacts'),
parametersLabel: __('Parameters'),
metricsLabel: __('Metrics'),
},
};
</script>
<template>
<div>
<incubation-alert />
<h3>
{{ $options.i18n.titleLabel }}
</h3>
<table class="candidate-details">
<tbody>
<tr class="divider"></tr>
<tr>
<td class="gl-text-secondary gl-font-weight-bold">{{ $options.i18n.infoLabel }}</td>
<td class="gl-font-weight-bold">{{ $options.i18n.idLabel }}</td>
<td>{{ candidate.info.iid }}</td>
</tr>
<tr>
<td></td>
<td class="gl-font-weight-bold">{{ $options.i18n.statusLabel }}</td>
<td>{{ candidate.info.status }}</td>
</tr>
<tr>
<td></td>
<td class="gl-font-weight-bold">{{ $options.i18n.experimentLabel }}</td>
<td>
<gl-link :href="candidate.info.path_to_experiment">{{
candidate.info.experiment_name
}}</gl-link>
</td>
</tr>
<tr v-if="candidate.info.path_to_artifact">
<td></td>
<td class="gl-font-weight-bold">{{ $options.i18n.artifactsLabel }}</td>
<td>
<gl-link :href="candidate.info.path_to_artifact">{{
$options.i18n.artifactsLabel
}}</gl-link>
</td>
</tr>
<tr class="divider"></tr>
<tr v-for="(param, index) in candidate.params" :key="param.name">
<td v-if="index == 0" class="gl-text-secondary gl-font-weight-bold">
{{ $options.i18n.parametersLabel }}
</td>
<td v-else></td>
<td class="gl-font-weight-bold">{{ param.name }}</td>
<td>{{ param.value }}</td>
</tr>
<tr class="divider"></tr>
<tr v-for="(metric, index) in candidate.metrics" :key="metric.name">
<td v-if="index == 0" class="gl-text-secondary gl-font-weight-bold">
{{ $options.i18n.metricsLabel }}
</td>
<td v-else></td>
<td class="gl-font-weight-bold">{{ metric.name }}</td>
<td>{{ metric.value }}</td>
</tr>
</tbody>
</table>
</div>
</template>

59
app/assets/javascripts/ml/experiment_tracking/components/ml_experiment.vue Normal file
View File

@ -0,0 +1,59 @@
<script>
import { GlTable, GlLink } from '@gitlab/ui';
import { __ } from '~/locale';
import IncubationAlert from './incubation_alert.vue';
export default {
name: 'MlExperiment',
components: {
GlTable,
GlLink,
IncubationAlert,
},
inject: ['candidates', 'metricNames', 'paramNames'],
computed: {
fields() {
return [
...this.paramNames,
...this.metricNames,
{ key: 'details', label: '' },
{ key: 'artifact', label: '' },
];
},
},
i18n: {
titleLabel: __('Experiment candidates'),
emptyStateLabel: __('This experiment has no logged candidates'),
artifactsLabel: __('Artifacts'),
detailsLabel: __('Details'),
},
};
</script>
<template>
<div>
<incubation-alert />
<h3>
{{ $options.i18n.titleLabel }}
</h3>
<gl-table
:fields="fields"
:items="candidates"
:empty-text="$options.i18n.emptyStateLabel"
show-empty
class="gl-mt-0!"
>
<template #cell(artifact)="data">
<gl-link v-if="data.value" :href="data.value" target="_blank">{{
$options.i18n.artifactsLabel
}}</gl-link>
</template>
<template #cell(details)="data">
<gl-link :href="data.value">{{ $options.i18n.detailsLabel }}</gl-link>
</template>
</gl-table>
</div>
</template>

27
app/assets/javascripts/pages/projects/ml/candidates/show/index.js Normal file
View File

@ -0,0 +1,27 @@
import Vue from 'vue';
import MlCandidate from '~/ml/experiment_tracking/components/ml_candidate.vue';
const initShowCandidate = () => {
const element = document.querySelector('#js-show-ml-candidate');
if (!element) {
return;
}
const container = document.createElement('div');
element.appendChild(container);
const candidate = JSON.parse(element.dataset.candidate);
// eslint-disable-next-line no-new
new Vue({
el: container,
provide: {
candidate,
},
render(h) {
return h(MlCandidate);
},
});
};
initShowCandidate();

4
app/assets/javascripts/pages/projects/ml/experiments/show/index.js
View File

@ -1,5 +1,5 @@
import Vue from 'vue';
import ShowExperiment from '~/ml/experiment_tracking/components/experiment.vue';
import MlExperiment from '~/ml/experiment_tracking/components/ml_experiment.vue';
const initShowExperiment = () => {
const element = document.querySelector('#js-show-ml-experiment');
@ -23,7 +23,7 @@ const initShowExperiment = () => {
paramNames,
},
render(h) {
return h(ShowExperiment);
return h(MlExperiment);
},
});
};

6
app/assets/stylesheets/pages/ml_experiment_tracking.scss
View File

@ -14,3 +14,9 @@
color: $gl-text-color;
}
}
table.candidate-details {
td {
padding: $gl-spacing-scale-3;
}
}

23
app/controllers/projects/ml/candidates_controller.rb Normal file
View File

@ -0,0 +1,23 @@
# frozen_string_literal: true
module Projects
module Ml
class CandidatesController < ApplicationController
before_action :check_feature_flag
feature_category :mlops
def show
@candidate = ::Ml::Candidate.with_project_id_and_iid(@project.id, params['iid'])
render_404 unless @candidate.present?
end
private
def check_feature_flag
render_404 unless Feature.enabled?(:ml_experiment_tracking, @project)
end
end
end
end

1
app/controllers/projects/ml/experiments_controller.rb
View File

@ -3,7 +3,6 @@
module Projects
module Ml
class ExperimentsController < ::Projects::ApplicationController
include Projects::Ml::ExperimentsHelper
before_action :check_feature_flag
feature_category :mlops

3
app/graphql/types/permission_types/project.rb
View File

@ -17,7 +17,8 @@ module Types
:admin_wiki, :admin_project, :update_pages,
:admin_remote_mirror, :create_label, :update_wiki, :destroy_wiki,
:create_pages, :destroy_pages, :read_pages_content, :admin_operations,
:read_merge_request, :read_design, :create_design, :destroy_design
:read_merge_request, :read_design, :create_design, :destroy_design,
:read_environment
permission_field :create_snippet

40
app/helpers/projects/ml/experiments_helper.rb
View File

@ -9,7 +9,9 @@ module Projects
items = candidates.map do |candidate|
{
**candidate.params.to_h { |p| [p.name, p.value] },
**candidate.latest_metrics.to_h { |m| [m.name, number_with_precision(m.value, precision: 4)] }
**candidate.latest_metrics.to_h { |m| [m.name, number_with_precision(m.value, precision: 4)] },
artifact: link_to_artifact(candidate),
details: link_to_details(candidate)
}
end
@ -19,6 +21,42 @@ module Projects
def unique_logged_names(candidates, &selector)
Gitlab::Json.generate(candidates.flat_map(&selector).map(&:name).uniq)
end
def candidate_as_data(candidate)
data = {
params: candidate.params,
metrics: candidate.latest_metrics,
info: {
iid: candidate.iid,
path_to_artifact: link_to_artifact(candidate),
experiment_name: candidate.experiment.name,
path_to_experiment: link_to_experiment(candidate),
status: candidate.status
}
}
Gitlab::Json.generate(data)
end
private
def link_to_artifact(candidate)
artifact = candidate.artifact
return unless artifact.present?
project_package_path(candidate.experiment.project, artifact)
end
def link_to_details(candidate)
project_ml_candidate_path(candidate.experiment.project, candidate.iid)
end
def link_to_experiment(candidate)
experiment = candidate.experiment
project_ml_experiment_path(experiment.project, experiment.iid)
end
end
end
end

8
app/models/generic_commit_status.rb
View File

@ -3,8 +3,6 @@
class GenericCommitStatus < CommitStatus
EXTERNAL_STAGE_IDX = 1_000_000
before_validation :set_default_values
validates :target_url, addressable_url: true,
length: { maximum: 255 },
allow_nil: true
@ -13,12 +11,6 @@ class GenericCommitStatus < CommitStatus
# GitHub compatible API
alias_attribute :context, :name
def set_default_values
self.context ||= 'default'
self.stage ||= 'external'
self.stage_idx ||= EXTERNAL_STAGE_IDX
end
def tags
[:external]
end

16
app/models/ml/candidate.rb
View File

@ -19,7 +19,21 @@ module Ml
scope :including_metrics_and_params, -> { includes(:latest_metrics, :params) }
def artifact_root
"/ml_candidate_#{iid}/-/"
"/#{package_name}/#{package_version}/"
end
def artifact
::Packages::Generic::PackageFinder.new(experiment.project).execute!(package_name, package_version)
rescue ActiveRecord::RecordNotFound
nil
end
def package_name
"ml_candidate_#{iid}"
end
def package_version
'-'
end
class << self

7
app/views/projects/ml/candidates/show.html.haml Normal file
View File

@ -0,0 +1,7 @@
- experiment = @candidate.experiment
- add_to_breadcrumbs _("Experiments"), project_ml_experiments_path(@project)
- add_to_breadcrumbs experiment.name, project_ml_experiment_path(@project, experiment.iid)
- breadcrumb_title "Candidate #{@candidate.iid}"
- data = candidate_as_data(@candidate)
#js-show-ml-candidate{ data: { candidate: data } }

1
config/routes/project.rb
View File

@ -475,6 +475,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
namespace :ml do
resources :experiments, only: [:index, :show], controller: 'experiments'
resources :candidates, only: [:show], controller: 'candidates', param: :iid
end
end
# End of the /-/ scope.

25
db/post_migrate/20221123133054_queue_reset_status_on_container_repositories.rb Normal file
View File

@ -0,0 +1,25 @@
# frozen_string_literal: true
class QueueResetStatusOnContainerRepositories < Gitlab::Database::Migration[2.0]
MIGRATION = 'ResetStatusOnContainerRepositories'
DELAY_INTERVAL = 2.minutes
BATCH_SIZE = 50
restrict_gitlab_migration gitlab_schema: :gitlab_main
def up
return unless ::Gitlab.config.registry.enabled
queue_batched_background_migration(
MIGRATION,
:container_repositories,
:id,
job_interval: DELAY_INTERVAL,
sub_batch_size: BATCH_SIZE
)
end
def down
delete_batched_background_migration(MIGRATION, :container_repositories, :id, [])
end
end

1
db/schema_migrations/20221123133054 Normal file
View File

@ -0,0 +1 @@
1a0a090433dd422b1bd9efdb56f82c02af8bab45b1a651b51a6ed224d823964c

2
doc/administration/postgresql/replication_and_failover.md
View File

@ -1255,7 +1255,7 @@ To do the switch on **all** PgBouncer nodes:
```
1. Run `gitlab-ctl reconfigure`.
1. You must also run `rm /var/opt/gitlab/consul/watcher_postgresql.json`.
1. You must also run `rm /var/opt/gitlab/consul/config.d/watcher_postgresql.json`.
This is a [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7293).
#### Clean up

1
doc/api/graphql/reference/index.md
View File

@ -18263,6 +18263,7 @@ Returns [`UserMergeRequestInteraction`](#usermergerequestinteraction).
| <a id="projectpermissionsreadcommitstatus"></a>`readCommitStatus` | [`Boolean!`](#boolean) | Indicates the user can perform `read_commit_status` on this resource. |
| <a id="projectpermissionsreadcycleanalytics"></a>`readCycleAnalytics` | [`Boolean!`](#boolean) | Indicates the user can perform `read_cycle_analytics` on this resource. |
| <a id="projectpermissionsreaddesign"></a>`readDesign` | [`Boolean!`](#boolean) | Indicates the user can perform `read_design` on this resource. |
| <a id="projectpermissionsreadenvironment"></a>`readEnvironment` | [`Boolean!`](#boolean) | Indicates the user can perform `read_environment` on this resource. |
| <a id="projectpermissionsreadmergerequest"></a>`readMergeRequest` | [`Boolean!`](#boolean) | Indicates the user can perform `read_merge_request` on this resource. |
| <a id="projectpermissionsreadpagescontent"></a>`readPagesContent` | [`Boolean!`](#boolean) | Indicates the user can perform `read_pages_content` on this resource. |
| <a id="projectpermissionsreadproject"></a>`readProject` | [`Boolean!`](#boolean) | Indicates the user can perform `read_project` on this resource. |

2
doc/api/invitations.md
View File

@ -126,7 +126,7 @@ PUT /projects/:id/invitations/:email
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project or group](index.md#namespaced-path-encoding) owned by the authenticated user. |
| `email` | string | yes | The email address to which the invitation was previously sent. |
| `email` | string | yes | The email address the invitation was previously sent to. |
| `access_level` | integer | no | A valid access level (defaults: `30`, the Developer role). |
| `expires_at` | string | no | A date string in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`). |

6
doc/development/documentation/topic_types/task.md
View File

@ -69,6 +69,12 @@ For example, `Create an issue when you want to track bugs or future work`.
To start the task steps, use a succinct action followed by a colon.
For example, `To create an issue:`
## Task prerequisites
As a best practice, if the task requires the user to have a role other than Guest,
put the minimum role in the prerequisites. See [the Word list](../styleguide/word_list.md) for
how to write the phrase for each role.
## Related topics
- [View the format for writing task steps](../styleguide/index.md#navigation).

7
doc/development/fe_guide/merge_request_widget_extensions.md
View File

@ -355,7 +355,12 @@ To generate these known events for a single widget:
1. `redis_slot` = `code_review`
1. `category` = `code_review`
1. `aggregation` = `weekly`
1. Add each event to the appropriate aggregates in `config/metrics/aggregates/code_review.yml`
1. Add each event (those listed in the command in step 7, replacing `test_reports`
with the appropriate name slug) to the aggregate files:
1. `config/metrics/counts_7d/{timestamp}_code_review_category_monthly_active_users.yml`
1. `config/metrics/counts_7d/{timestamp}_code_review_group_monthly_active_users.yml`
1. `config/metrics/counts_28d/{timestamp}_code_review_category_monthly_active_users.yml`
1. `config/metrics/counts_28d/{timestamp}_code_review_group_monthly_active_users.yml`
### Add new events

2
doc/user/application_security/offline_deployments/index.md
View File

@ -117,7 +117,7 @@ This template should be used in a new, empty project, with a `.gitlab-ci.yml` fi
```yaml
include:
- template: Secure-Binaries.gitlab-ci.yml
- template: Security/Secure-Binaries.gitlab-ci.yml
```
The pipeline downloads the Docker images needed for the Security Scanners and saves them as

BIN
doc/user/project/ml/experiment_tracking/img/candidate_v15_7.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 104 KiB

BIN
doc/user/project/ml/experiment_tracking/img/candidates.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 61 KiB

BIN
doc/user/project/ml/experiment_tracking/img/candidates_v15_7.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 150 KiB

BIN
doc/user/project/ml/experiment_tracking/img/experiments.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 44 KiB

BIN
doc/user/project/ml/experiment_tracking/img/experiments_v15_7.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 75 KiB

16
doc/user/project/ml/experiment_tracking/index.md
View File

@ -16,9 +16,11 @@ engineering, and so on, to improve the performance of the model. Keeping track o
artifacts so that the data scientist can later replicate the experiment is not trivial. Machine learning experiment
tracking enables them to log parameters, metrics, and artifacts directly into GitLab, giving easy access later on.
![List of Experiments](img/experiments.png)
![List of Experiments](img/experiments_v15_7.png)
![Experiment Candidates](img/candidates.png)
![Experiment Candidates](img/candidates_v15_7.png)
![Candidate Detail](img/candidate_v15_7.png)
## What is an experiment?
@ -53,13 +55,15 @@ integration. More information on how to use GitLab as a backend for MLFlow Clien
### Exploring model candidates
To list the current active experiments, navigate to `https/-/ml/experiments`. To display all trials
that have been logged, along with their metrics and parameters, selecting an experiment.
that have been logged, along with their metrics and parameters, select an experiment. To display details for a candidate,
select **Details**.
### Logging artifacts
Trial artifacts are saved as [generic packages](../../../packages/generic_packages/index.md), and follow all their
conventions. After an artifact is logged for a candidate, all artifacts logged for the candidate are listed in the
package registry. The package name for a candidate is `ml_candidate_<candidate_id>`, with version `-`.
package registry. The package name for a candidate is `ml_candidate_<candidate_id>`, with version `-`. The link to the
artifacts can also be accessed from the **Experiment Candidates** list or **Candidate detail**.
### Limitations and future
@ -72,4 +76,6 @@ On GitLab.com, this feature is currently on private testing.
## Feedback, roadmap and reports
For updates on the development, feedback and bug reports, refer to the [development epic](https://gitlab.com/groups/gitlab-org/-/epics/8560).
For updates on the development, refer to the [development epic](https://gitlab.com/groups/gitlab-org/-/epics/8560).
For feedback, bug reports and feature requests, refer to the [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/381660).

3
lib/api/commit_statuses.rb
View File

@ -125,7 +125,8 @@ module API
user: current_user,
protected: user_project.protected_for?(ref),
ci_stage: stage,
stage_idx: stage.position
stage_idx: stage.position,
stage: 'external'
)
updatable_optional_attributes = %w[target_url description coverage]

139
lib/gitlab/background_migration/reset_status_on_container_repositories.rb Normal file
View File

@ -0,0 +1,139 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# A job that:
# * pickup container repositories with delete_scheduled status.
# * check if there are tags linked to it.
# * if there are tags, reset the status to nil.
class ResetStatusOnContainerRepositories < BatchedMigrationJob
DELETE_SCHEDULED_STATUS = 0
DUMMY_TAGS = %w[tag].freeze
MIGRATOR = 'ResetStatusOnContainerRepositories'
scope_to ->(relation) { relation.where(status: DELETE_SCHEDULED_STATUS) }
operation_name :reset_status_on_container_repositories
def perform
each_sub_batch do |sub_batch|
reset_status_if_tags(sub_batch)
end
end
private
def reset_status_if_tags(container_repositories)
container_repositories_with_tags = container_repositories.select { |cr| cr.becomes(ContainerRepository).tags? } # rubocop:disable Cop/AvoidBecomes
ContainerRepository.where(id: container_repositories_with_tags.map(&:id))
.update_all(status: nil)
end
# rubocop:disable Style/Documentation
module Routable
extend ActiveSupport::Concern
included do
has_one :route,
as: :source,
class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route'
end
def full_path
route&.path || build_full_path
end
def build_full_path
if parent && path
"#{parent.full_path}/#{path}"
else
path
end
end
end
class Route < ::ApplicationRecord
self.table_name = 'routes'
end
class Namespace < ::ApplicationRecord
include ::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Routable
include ::Namespaces::Traversal::Recursive
include ::Namespaces::Traversal::Linear
include ::Gitlab::Utils::StrongMemoize
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled
belongs_to :parent,
class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace'
def self.polymorphic_name
'Namespace'
end
end
class Project < ::ApplicationRecord
include ::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Routable
self.table_name = 'projects'
belongs_to :namespace,
class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace'
alias_method :parent, :namespace
alias_attribute :parent_id, :namespace_id
delegate :root_ancestor, to: :namespace, allow_nil: true
end
class ContainerRepository < ::ApplicationRecord
self.table_name = 'container_repositories'
belongs_to :project,
class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project'
def tags?
result = ContainerRegistry.tags_for(path).any?
::Gitlab::BackgroundMigration::Logger.info(
migrator: MIGRATOR,
has_tags: result,
container_repository_id: id,
container_repository_path: path
)
result
end
def path
@path ||= [project.full_path, name].select(&:present?).join('/').downcase
end
end
class ContainerRegistry
class << self
def tags_for(path)
response = ContainerRegistryClient.repository_tags(path, page_size: 1)
return DUMMY_TAGS unless response
response['tags'] || []
rescue StandardError
DUMMY_TAGS
end
end
end
class ContainerRegistryClient
def self.repository_tags(path, page_size:)
registry_config = ::Gitlab.config.registry
return { 'tags' => DUMMY_TAGS } unless registry_config.enabled && registry_config.api_url.present?
pull_token = ::Auth::ContainerRegistryAuthenticationService.pull_access_token(path)
client = ::ContainerRegistry::Client.new(registry_config.api_url, token: pull_token)
client.repository_tags(path, page_size: page_size)
end
end
# rubocop:enable Style/Documentation
end
end
end

4
lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
View File

@ -9,9 +9,9 @@
# Usage:
#
# include:
# - template: Secure-Binaries.gitlab-ci.yml
# - template: Security/Secure-Binaries.gitlab-ci.yml
#
# Docs: https://docs.gitlab.com/ee/topics/airgap/
# Docs: https://docs.gitlab.com/ee/user/application_security/offline_deployments/
variables:
# Setting this variable will affect all Security templates

23
locale/gitlab.pot
View File

@ -16320,7 +16320,10 @@ msgstr ""
msgid "Expected documents: %{expected_documents}"
msgstr ""
msgid "Experiment Candidates"
msgid "Experiment"
msgstr ""
msgid "Experiment candidates"
msgstr ""
msgid "Experiments"
@ -17052,6 +17055,9 @@ msgstr ""
msgid "February"
msgstr ""
msgid "Feedback"
msgstr ""
msgid "Feedback and Updates"
msgstr ""
@ -21799,6 +21805,9 @@ msgstr ""
msgid "Indicates whether this runner can pick jobs without tags"
msgstr ""
msgid "Info"
msgstr ""
msgid "Inform users without uploaded SSH keys that they can't push over SSH until one is added"
msgstr ""
@ -26633,6 +26642,9 @@ msgstr ""
msgid "Modal|Close"
msgstr ""
msgid "Model candidate details"
msgstr ""
msgid "Modified"
msgstr ""
@ -29506,6 +29518,9 @@ msgstr ""
msgid "Parameter \"job_id\" cannot exceed length of %{job_id_max_size}"
msgstr ""
msgid "Parameters"
msgstr ""
msgid "Parent"
msgstr ""
@ -41764,9 +41779,6 @@ msgstr ""
msgid "This Cron pattern is invalid"
msgstr ""
msgid "This Experiment has no logged Candidates"
msgstr ""
msgid "This GitLab instance does not provide any shared runners yet. Instance administrators can register shared runners in the admin area."
msgstr ""
@ -41956,6 +41968,9 @@ msgstr ""
msgid "This epic would exceed maximum number of related epics."
msgstr ""
msgid "This experiment has no logged candidates"
msgstr ""
msgid "This feature requires local storage to be enabled"
msgstr ""

9
spec/frontend/ide/lib/common/model_spec.js
View File

@ -149,7 +149,6 @@ describe('Multi-file editor library model', () => {
model.updateOptions({ insertSpaces: true, someOption: 'some value' });
expect(model.options).toEqual({
endOfLine: 0,
insertFinalNewline: true,
insertSpaces: true,
someOption: 'some value',
@ -181,16 +180,12 @@ describe('Multi-file editor library model', () => {
describe('applyCustomOptions', () => {
it.each`
option | value | contentBefore | contentAfter
${'endOfLine'} | ${0} | ${'hello\nworld\n'} | ${'hello\nworld\n'}
${'endOfLine'} | ${0} | ${'hello\r\nworld\r\n'} | ${'hello\nworld\n'}
${'endOfLine'} | ${1} | ${'hello\nworld\n'} | ${'hello\r\nworld\r\n'}
${'endOfLine'} | ${1} | ${'hello\r\nworld\r\n'} | ${'hello\r\nworld\r\n'}
${'insertFinalNewline'} | ${true} | ${'hello\nworld'} | ${'hello\nworld\n'}
${'insertFinalNewline'} | ${true} | ${'hello\nworld\n'} | ${'hello\nworld\n'}
${'insertFinalNewline'} | ${false} | ${'hello\nworld'} | ${'hello\nworld'}
${'trimTrailingWhitespace'} | ${true} | ${'hello \t\nworld \t\n'} | ${'hello\nworld\n'}
${'trimTrailingWhitespace'} | ${true} | ${'hello \t\r\nworld \t\r\n'} | ${'hello\nworld\n'}
${'trimTrailingWhitespace'} | ${false} | ${'hello \t\r\nworld \t\r\n'} | ${'hello \t\nworld \t\n'}
${'trimTrailingWhitespace'} | ${true} | ${'hello \t\r\nworld \t\r\n'} | ${'hello\r\nworld\r\n'}
${'trimTrailingWhitespace'} | ${false} | ${'hello \t\r\nworld \t\r\n'} | ${'hello \t\r\nworld \t\r\n'}
`(
'correctly applies custom option $option=$value to content',
({ option, value, contentBefore, contentAfter }) => {

233
spec/frontend/ml/experiment_tracking/components/__snapshots__/ml_candidate_spec.js.snap Normal file
View File

@ -0,0 +1,233 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`MlCandidate renders correctly 1`] = `
<div>
<div
class="gl-alert gl-alert-warning"
>
<svg
aria-hidden="true"
class="gl-icon s16 gl-alert-icon"
data-testid="warning-icon"
role="img"
>
<use
href="#warning"
/>
</svg>
<div
aria-live="assertive"
class="gl-alert-content"
role="alert"
>
<h2
class="gl-alert-title"
>
Machine Learning Experiment Tracking is in Incubating Phase
</h2>
<div
class="gl-alert-body"
>
GitLab incubates features to explore new use cases. These features are updated regularly, and support is limited
<a
class="gl-link"
href="https://about.gitlab.com/handbook/engineering/incubation/"
rel="noopener noreferrer"
target="_blank"
>
Learn more
</a>
</div>
<div
class="gl-alert-actions"
>
<a
class="btn gl-alert-action btn-confirm btn-md gl-button"
href="https://gitlab.com/gitlab-org/gitlab/-/issues/381660"
>
<!---->
<!---->
<span
class="gl-button-text"
>
Feedback
</span>
</a>
</div>
</div>
<button
aria-label="Dismiss"
class="btn gl-dismiss-btn btn-default btn-sm gl-button btn-default-tertiary btn-icon"
type="button"
>
<!---->
<svg
aria-hidden="true"
class="gl-button-icon gl-icon s16"
data-testid="close-icon"
role="img"
>
<use
href="#close"
/>
</svg>
<!---->
</button>
</div>
<h3>
Model candidate details
</h3>
<table
class="candidate-details"
>
<tbody>
<tr
class="divider"
/>
<tr>
<td
class="gl-text-secondary gl-font-weight-bold"
>
Info
</td>
<td
class="gl-font-weight-bold"
>
ID
</td>
<td>
candidate_iid
</td>
</tr>
<tr>
<td />
<td
class="gl-font-weight-bold"
>
Status
</td>
<td>
SUCCESS
</td>
</tr>
<tr>
<td />
<td
class="gl-font-weight-bold"
>
Experiment
</td>
<td>
<a
class="gl-link"
href="#"
>
The Experiment
</a>
</td>
</tr>
<!---->
<tr
class="divider"
/>
<tr>
<td
class="gl-text-secondary gl-font-weight-bold"
>
Parameters
</td>
<td
class="gl-font-weight-bold"
>
Algorithm
</td>
<td>
Decision Tree
</td>
</tr>
<tr>
<td />
<td
class="gl-font-weight-bold"
>
MaxDepth
</td>
<td>
3
</td>
</tr>
<tr
class="divider"
/>
<tr>
<td
class="gl-text-secondary gl-font-weight-bold"
>
Metrics
</td>
<td
class="gl-font-weight-bold"
>
AUC
</td>
<td>
.55
</td>
</tr>
<tr>
<td />
<td
class="gl-font-weight-bold"
>
Accuracy
</td>
<td>
.99
</td>
</tr>
</tbody>
</table>
</div>
`;

73
spec/frontend/ml/experiment_tracking/components/__snapshots__/experiment_spec.js.snap → spec/frontend/ml/experiment_tracking/components/__snapshots__/ml_experiment_spec.js.snap
View File

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`ShowExperiment with candidates renders correctly 1`] = `
exports[`MlExperiment with candidates renders correctly 1`] = `
<div>
<div
class="gl-alert gl-alert-warning"
@ -39,7 +39,7 @@ exports[`ShowExperiment with candidates renders correctly 1`] = `
rel="noopener noreferrer"
target="_blank"
>
Learn More
Learn more
</a>
</div>
@ -48,7 +48,7 @@ exports[`ShowExperiment with candidates renders correctly 1`] = `
>
<a
class="btn gl-alert-action btn-confirm btn-md gl-button"
href="https://gitlab.com/groups/gitlab-org/-/epics/8560"
href="https://gitlab.com/gitlab-org/gitlab/-/issues/381660"
>
<!---->
@ -58,7 +58,7 @@ exports[`ShowExperiment with candidates renders correctly 1`] = `
class="gl-button-text"
>
Feedback and Updates
Feedback
</span>
</a>
@ -89,13 +89,13 @@ exports[`ShowExperiment with candidates renders correctly 1`] = `
<h3>
Experiment Candidates
Experiment candidates
</h3>
<table
aria-busy="false"
aria-colcount="4"
aria-colcount="6"
class="table b-table gl-table gl-mt-0!"
role="table"
>
@ -150,6 +150,24 @@ exports[`ShowExperiment with candidates renders correctly 1`] = `
Mae
</div>
</th>
<th
aria-colindex="5"
aria-label="Details"
class=""
role="columnheader"
scope="col"
>
<div />
</th>
<th
aria-colindex="6"
aria-label="Artifact"
class=""
role="columnheader"
scope="col"
>
<div />
</th>
</tr>
</thead>
<tbody
@ -184,6 +202,32 @@ exports[`ShowExperiment with candidates renders correctly 1`] = `
class=""
role="cell"
/>
<td
aria-colindex="5"
class=""
role="cell"
>
<a
class="gl-link"
href="link_to_candidate1"
>
Details
</a>
</td>
<td
aria-colindex="6"
class=""
role="cell"
>
<a
class="gl-link"
href="link_to_artifact"
rel="noopener"
target="_blank"
>
Artifacts
</a>
</td>
</tr>
<tr
class=""
@ -213,6 +257,23 @@ exports[`ShowExperiment with candidates renders correctly 1`] = `
class=""
role="cell"
/>
<td
aria-colindex="5"
class=""
role="cell"
>
<a
class="gl-link"
href="link_to_candidate2"
>
Details
</a>
</td>
<td
aria-colindex="6"
class=""
role="cell"
/>
</tr>
<!---->
<!---->

2
spec/frontend/ml/experiment_tracking/components/incubation_alert_spec.js
View File

@ -15,7 +15,7 @@ describe('IncubationAlert', () => {
it('displays link to issue', () => {
expect(findButton().attributes().href).toBe(
'https://gitlab.com/groups/gitlab-org/-/epics/8560',
'https://gitlab.com/gitlab-org/gitlab/-/issues/381660',
);
});

43
spec/frontend/ml/experiment_tracking/components/ml_candidate_spec.js Normal file
View File

@ -0,0 +1,43 @@
import { GlAlert } from '@gitlab/ui';
import { mountExtended } from 'helpers/vue_test_utils_helper';
import MlCandidate from '~/ml/experiment_tracking/components/ml_candidate.vue';
describe('MlCandidate', () => {
let wrapper;
const createWrapper = () => {
const candidate = {
params: [
{ name: 'Algorithm', value: 'Decision Tree' },
{ name: 'MaxDepth', value: '3' },
],
metrics: [
{ name: 'AUC', value: '.55' },
{ name: 'Accuracy', value: '.99' },
],
info: {
iid: 'candidate_iid',
artifact_link: 'path_to_artifact',
experiment_name: 'The Experiment',
experiment_path: 'path/to/experiment',
status: 'SUCCESS',
},
};
return mountExtended(MlCandidate, { provide: { candidate } });
};
const findAlert = () => wrapper.findComponent(GlAlert);
it('shows incubation warning', () => {
wrapper = createWrapper();
expect(findAlert().exists()).toBe(true);
});
it('renders correctly', () => {
wrapper = createWrapper();
expect(wrapper.element).toMatchSnapshot();
});
});

12
spec/frontend/ml/experiment_tracking/components/experiment_spec.js → spec/frontend/ml/experiment_tracking/components/ml_experiment_spec.js
View File

@ -1,17 +1,17 @@
import { GlAlert } from '@gitlab/ui';
import { mountExtended } from 'helpers/vue_test_utils_helper';
import ShowExperiment from '~/ml/experiment_tracking/components/experiment.vue';
import MlExperiment from '~/ml/experiment_tracking/components/ml_experiment.vue';
describe('ShowExperiment', () => {
describe('MlExperiment', () => {
let wrapper;
const createWrapper = (candidates = [], metricNames = [], paramNames = []) => {
return mountExtended(ShowExperiment, { provide: { candidates, metricNames, paramNames } });
return mountExtended(MlExperiment, { provide: { candidates, metricNames, paramNames } });
};
const findAlert = () => wrapper.findComponent(GlAlert);
const findEmptyState = () => wrapper.findByText('This Experiment has no logged Candidates');
const findEmptyState = () => wrapper.findByText('This experiment has no logged candidates');
it('shows incubation warning', () => {
wrapper = createWrapper();
@ -31,8 +31,8 @@ describe('ShowExperiment', () => {
it('renders correctly', () => {
wrapper = createWrapper(
[
{ rmse: 1, l1_ratio: 0.4 },
{ auc: 0.3, l1_ratio: 0.5 },
{ rmse: 1, l1_ratio: 0.4, details: 'link_to_candidate1', artifact: 'link_to_artifact' },
{ auc: 0.3, l1_ratio: 0.5, details: 'link_to_candidate2' },
],
['rmse', 'auc', 'mae'],
['l1_ratio'],

2
spec/graphql/types/permission_types/project_spec.rb
View File

@ -13,7 +13,7 @@ RSpec.describe Types::PermissionTypes::Project do
:create_merge_request_from, :create_wiki, :push_code, :create_deployment, :push_to_delete_protected_branch,
:admin_wiki, :admin_project, :update_pages, :admin_remote_mirror, :create_label,
:update_wiki, :destroy_wiki, :create_pages, :destroy_pages, :read_pages_content,
:read_merge_request, :read_design, :create_design, :destroy_design
:read_merge_request, :read_design, :create_design, :destroy_design, :read_environment
]
expected_permissions.each do |permission|

68
spec/helpers/projects/ml/experiments_helper_spec.rb
View File

@ -5,28 +5,36 @@ require 'rspec'
require 'spec_helper'
require 'mime/types'
RSpec.describe Projects::Ml::ExperimentsHelper do
let_it_be(:project) { build(:project, :private) }
let_it_be(:experiment) { build(:ml_experiments, user_id: project.creator, project: project) }
let_it_be(:candidates) do
create_list(:ml_candidates, 2, experiment: experiment, user: project.creator).tap do |c|
c[0].params.create!([{ name: 'param1', value: 'p1' }, { name: 'param2', value: 'p2' }])
c[0].metrics.create!(
RSpec.describe Projects::Ml::ExperimentsHelper, feature_category: :mlops do
let_it_be(:project) { create(:project, :private) }
let_it_be(:experiment) { create(:ml_experiments, user_id: project.creator, project: project) }
let_it_be(:candidate0) do
create(:ml_candidates, experiment: experiment, user: project.creator).tap do |c|
c.params.build([{ name: 'param1', value: 'p1' }, { name: 'param2', value: 'p2' }])
c.metrics.create!(
[{ name: 'metric1', value: 0.1 }, { name: 'metric2', value: 0.2 }, { name: 'metric3', value: 0.3 }]
)
c[1].params.create!([{ name: 'param2', value: 'p3' }, { name: 'param3', value: 'p4' }])
c[1].metrics.create!(name: 'metric3', value: 0.4)
end
end
let_it_be(:candidate1) do
create(:ml_candidates, experiment: experiment, user: project.creator).tap do |c|
c.params.build([{ name: 'param2', value: 'p3' }, { name: 'param3', value: 'p4' }])
c.metrics.create!(name: 'metric3', value: 0.4)
end
end
let_it_be(:candidates) { [candidate0, candidate1] }
describe '#candidates_table_items' do
subject { helper.candidates_table_items(candidates) }
it 'creates the correct model for the table' do
expected_value = [
{ 'param1' => 'p1', 'param2' => 'p2', 'metric1' => '0.1000', 'metric2' => '0.2000', 'metric3' => '0.3000' },
{ 'param2' => 'p3', 'param3' => 'p4', 'metric3' => '0.4000' }
{ 'param1' => 'p1', 'param2' => 'p2', 'metric1' => '0.1000', 'metric2' => '0.2000', 'metric3' => '0.3000',
'artifact' => nil, 'details' => "/#{project.full_path}/-/ml/candidates/#{candidate0.iid}" },
{ 'param2' => 'p3', 'param3' => 'p4', 'metric3' => '0.4000',
'artifact' => nil, 'details' => "/#{project.full_path}/-/ml/candidates/#{candidate1.iid}" }
]
expect(Gitlab::Json.parse(subject)).to match_array(expected_value)
@ -46,4 +54,40 @@ RSpec.describe Projects::Ml::ExperimentsHelper do
it { is_expected.to match_array(%w[metric1 metric2 metric3]) }
end
end
describe '#candidate_as_data' do
let(:candidate) { candidate0 }
let(:package) do
create(:generic_package, name: candidate.package_name, version: candidate.package_version, project: project)
end
subject { Gitlab::Json.parse(helper.candidate_as_data(candidate)) }
it 'generates the correct params' do
expect(subject['params']).to include(
hash_including('name' => 'param1', 'value' => 'p1'),
hash_including('name' => 'param2', 'value' => 'p2')
)
end
it 'generates the correct metrics' do
expect(subject['metrics']).to include(
hash_including('name' => 'metric1', 'value' => 0.1),
hash_including('name' => 'metric2', 'value' => 0.2),
hash_including('name' => 'metric3', 'value' => 0.3)
)
end
it 'generates the correct info' do
expected_info = {
'iid' => candidate.iid,
'path_to_artifact' => "/#{project.full_path}/-/packages/#{package.id}",
'experiment_name' => candidate.experiment.name,
'path_to_experiment' => "/#{project.full_path}/-/ml/experiments/#{experiment.iid}",
'status' => 'running'
}
expect(subject['info']).to include(expected_info)
end
end
end

261
spec/lib/gitlab/background_migration/reset_status_on_container_repositories_spec.rb Normal file
View File

@ -0,0 +1,261 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories, feature_category: :container_registry do
let(:projects_table) { table(:projects) }
let(:namespaces_table) { table(:namespaces) }
let(:container_repositories_table) { table(:container_repositories) }
let(:routes_table) { table(:routes) }
let!(:root_group) do
namespaces_table.create!(name: 'root_group', path: 'root_group', type: 'Group') do |new_group|
new_group.update!(traversal_ids: [new_group.id])
end
end
let!(:group1) do
namespaces_table.create!(name: 'group1', path: 'group1', parent_id: root_group.id, type: 'Group') do |new_group|
new_group.update!(traversal_ids: [root_group.id, new_group.id])
end
end
let!(:subgroup1) do
namespaces_table.create!(name: 'subgroup1', path: 'subgroup1', parent_id: group1.id, type: 'Group') do |new_group|
new_group.update!(traversal_ids: [root_group.id, group1.id, new_group.id])
end
end
let!(:group2) do
namespaces_table.create!(name: 'group2', path: 'group2', parent_id: root_group.id, type: 'Group') do |new_group|
new_group.update!(traversal_ids: [root_group.id, new_group.id])
end
end
let!(:group1_project_namespace) do
namespaces_table.create!(name: 'group1_project', path: 'group1_project', type: 'Project', parent_id: group1.id)
end
let!(:subgroup1_project_namespace) do
namespaces_table.create!(
name: 'subgroup1_project',
path: 'subgroup1_project',
type: 'Project',
parent_id: subgroup1.id
)
end
let!(:group2_project_namespace) do
namespaces_table.create!(
name: 'group2_project',
path: 'group2_project',
type: 'Project',
parent_id: group2.id
)
end
let!(:group1_project) do
projects_table.create!(
name: 'group1_project',
path: 'group1_project',
namespace_id: group1.id,
project_namespace_id: group1_project_namespace.id
)
end
let!(:subgroup1_project) do
projects_table.create!(
name: 'subgroup1_project',
path: 'subgroup1_project',
namespace_id: subgroup1.id,
project_namespace_id: subgroup1_project_namespace.id
)
end
let!(:group2_project) do
projects_table.create!(
name: 'group2_project',
path: 'group2_project',
namespace_id: group2.id,
project_namespace_id: group2_project_namespace.id
)
end
let!(:route2) do
routes_table.create!(
source_id: group2_project.id,
source_type: 'Project',
path: 'root_group/group2/group2_project',
namespace_id: group2_project_namespace.id
)
end
let!(:delete_scheduled_container_repository1) do
container_repositories_table.create!(project_id: group1_project.id, status: 0, name: 'container_repository1')
end
let!(:delete_scheduled_container_repository2) do
container_repositories_table.create!(project_id: subgroup1_project.id, status: 0, name: 'container_repository2')
end
let!(:delete_scheduled_container_repository3) do
container_repositories_table.create!(project_id: group2_project.id, status: 0, name: 'container_repository3')
end
let!(:delete_ongoing_container_repository4) do
container_repositories_table.create!(project_id: group2_project.id, status: 2, name: 'container_repository4')
end
let(:migration) do
described_class.new(
start_id: container_repositories_table.minimum(:id),
end_id: container_repositories_table.maximum(:id),
batch_table: :container_repositories,
batch_column: :id,
sub_batch_size: 50,
pause_ms: 0,
connection: ApplicationRecord.connection
)
end
describe '#filter_batch' do
it 'scopes the relation to delete scheduled container repositories' do
expected = container_repositories_table.where(status: 0).pluck(:id)
actual = migration.filter_batch(container_repositories_table).pluck(:id)
expect(actual).to match_array(expected)
end
end
describe '#perform' do
let(:registry_api_url) { 'http://example.com' }
subject(:perform) { migration.perform }
before do
stub_container_registry_config(
enabled: true,
api_url: registry_api_url,
key: 'spec/fixtures/x509_certificate_pk.key'
)
stub_tags_list(path: 'root_group/group1/group1_project/container_repository1')
stub_tags_list(path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2', tags: [])
stub_tags_list(path: 'root_group/group2/group2_project/container_repository3')
end
shared_examples 'resetting status of all container repositories scheduled for deletion' do
it 'resets all statuses' do
expect_logging_on(
path: 'root_group/group1/group1_project/container_repository1',
id: delete_scheduled_container_repository1.id,
has_tags: true
)
expect_logging_on(
path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2',
id: delete_scheduled_container_repository2.id,
has_tags: true
)
expect_logging_on(
path: 'root_group/group2/group2_project/container_repository3',
id: delete_scheduled_container_repository3.id,
has_tags: true
)
expect { perform }
.to change { delete_scheduled_container_repository1.reload.status }.from(0).to(nil)
.and change { delete_scheduled_container_repository3.reload.status }.from(0).to(nil)
.and change { delete_scheduled_container_repository2.reload.status }.from(0).to(nil)
end
end
it 'resets status of container repositories with tags' do
expect_pull_access_token_on(path: 'root_group/group1/group1_project/container_repository1')
expect_pull_access_token_on(path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2')
expect_pull_access_token_on(path: 'root_group/group2/group2_project/container_repository3')
expect_logging_on(
path: 'root_group/group1/group1_project/container_repository1',
id: delete_scheduled_container_repository1.id,
has_tags: true
)
expect_logging_on(
path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2',
id: delete_scheduled_container_repository2.id,
has_tags: false
)
expect_logging_on(
path: 'root_group/group2/group2_project/container_repository3',
id: delete_scheduled_container_repository3.id,
has_tags: true
)
expect { perform }
.to change { delete_scheduled_container_repository1.reload.status }.from(0).to(nil)
.and change { delete_scheduled_container_repository3.reload.status }.from(0).to(nil)
.and not_change { delete_scheduled_container_repository2.reload.status }
end
context 'with the registry disabled' do
before do
allow(::Gitlab.config.registry).to receive(:enabled).and_return(false)
end
it_behaves_like 'resetting status of all container repositories scheduled for deletion'
end
context 'with the registry api url not defined' do
before do
allow(::Gitlab.config.registry).to receive(:api_url).and_return('')
end
it_behaves_like 'resetting status of all container repositories scheduled for deletion'
end
context 'with a faraday error' do
before do
client_double = instance_double('::ContainerRegistry::Client')
allow(::ContainerRegistry::Client).to receive(:new).and_return(client_double)
allow(client_double).to receive(:repository_tags).and_raise(Faraday::TimeoutError)
expect_pull_access_token_on(path: 'root_group/group1/group1_project/container_repository1')
expect_pull_access_token_on(path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2')
expect_pull_access_token_on(path: 'root_group/group2/group2_project/container_repository3')
end
it_behaves_like 'resetting status of all container repositories scheduled for deletion'
end
def stub_tags_list(path:, tags: %w[tag1])
url = "#{registry_api_url}/v2/#{path}/tags/list?n=1"
stub_request(:get, url)
.with(
headers: {
'Accept' => ContainerRegistry::Client::ACCEPTED_TYPES.join(', '),
'Authorization' => /bearer .+/,
'User-Agent' => "GitLab/#{Gitlab::VERSION}"
}
)
.to_return(
status: 200,
body: Gitlab::Json.dump(tags: tags),
headers: { 'Content-Type' => 'application/json' }
)
end
def expect_pull_access_token_on(path:)
expect(Auth::ContainerRegistryAuthenticationService)
.to receive(:pull_access_token).with(path).and_call_original
end
def expect_logging_on(path:, id:, has_tags:)
expect(::Gitlab::BackgroundMigration::Logger)
.to receive(:info).with(
migrator: described_class::MIGRATOR,
has_tags: has_tags,
container_repository_id: id,
container_repository_path: path
)
end
end
end

51
spec/migrations/20221123133054_queue_reset_status_on_container_repositories_spec.rb Normal file
View File

@ -0,0 +1,51 @@
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe QueueResetStatusOnContainerRepositories, feature_category: :container_registry do
let_it_be(:batched_migration) { described_class::MIGRATION }
before do
stub_container_registry_config(
enabled: true,
api_url: 'http://example.com',
key: 'spec/fixtures/x509_certificate_pk.key'
)
end
it 'schedules a new batched migration' do
reversible_migration do |migration|
migration.before -> {
expect(batched_migration).not_to have_scheduled_batched_migration
}
migration.after -> {
expect(batched_migration).to have_scheduled_batched_migration(
table_name: :container_repositories,
column_name: :id,
interval: described_class::DELAY_INTERVAL,
sub_batch_size: described_class::BATCH_SIZE
)
}
end
end
context 'with the container registry disabled' do
before do
allow(::Gitlab.config.registry).to receive(:enabled).and_return(false)
end
it 'does not schedule a new batched migration' do
reversible_migration do |migration|
migration.before -> {
expect(batched_migration).not_to have_scheduled_batched_migration
}
migration.after -> {
expect(batched_migration).not_to have_scheduled_batched_migration
}
end
end
end
end

30
spec/models/generic_commit_status_spec.rb
View File

@ -20,7 +20,7 @@ RSpec.describe GenericCommitStatus do
end
describe '#name_uniqueness_across_types' do
let(:attributes) { {} }
let(:attributes) { { context: 'default' } }
let(:commit_status) { described_class.new(attributes) }
let(:status_name) { 'test-job' }
@ -39,7 +39,7 @@ RSpec.describe GenericCommitStatus do
end
context 'with only a pipeline' do
let(:attributes) { { pipeline: pipeline } }
let(:attributes) { { pipeline: pipeline, context: 'default' } }
context 'without name' do
it_behaves_like 'it does not have uniqueness errors'
@ -129,32 +129,6 @@ RSpec.describe GenericCommitStatus do
end
end
describe 'set_default_values' do
before do
generic_commit_status.context = nil
generic_commit_status.stage = nil
generic_commit_status.save!
end
describe '#context' do
subject { generic_commit_status.context }
it { is_expected.not_to be_nil }
end
describe '#stage' do
subject { generic_commit_status.stage }
it { is_expected.not_to be_nil }
end
describe '#stage_idx' do
subject { generic_commit_status.stage_idx }
it { is_expected.not_to be_nil }
end
end
describe '#present' do
subject { generic_commit_status.present }

40
spec/models/ml/candidate_spec.rb
View File

@ -5,6 +5,8 @@ require 'spec_helper'
RSpec.describe Ml::Candidate, factory_default: :keep do
let_it_be(:candidate) { create(:ml_candidates, :with_metrics_and_params) }
let(:project) { candidate.experiment.project }
describe 'associations' do
it { is_expected.to belong_to(:experiment) }
it { is_expected.to belong_to(:user) }
@ -13,14 +15,48 @@ RSpec.describe Ml::Candidate, factory_default: :keep do
it { is_expected.to have_many(:metadata) }
end
describe 'default values' do
it { expect(described_class.new.iid).to be_present }
end
describe '.artifact_root' do
subject { candidate.artifact_root }
it { is_expected.to eq("/ml_candidate_#{candidate.iid}/-/") }
end
describe 'default values' do
it { expect(described_class.new.iid).to be_present }
describe '.package_name' do
subject { candidate.package_name }
it { is_expected.to eq("ml_candidate_#{candidate.iid}") }
end
describe '.package_version' do
subject { candidate.package_version }
it { is_expected.to eq('-') }
end
describe '.artifact' do
subject { candidate.artifact }
context 'when has logged artifacts' do
let(:package) do
create(:generic_package, name: candidate.package_name, version: candidate.package_version, project: project)
end
it 'returns the package' do
package
is_expected.to eq(package)
end
end
context 'when does not have logged artifacts' do
let(:tested_candidate) { create(:ml_candidates, :with_metrics_and_params) }
it { is_expected.to be_nil }
end
end
describe '#by_project_id_and_iid' do

69
spec/requests/projects/ml/candidates_controller_spec.rb Normal file
View File

@ -0,0 +1,69 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Projects::Ml::CandidatesController, feature_category: :mlops do
let_it_be(:project) { create(:project, :repository) }
let_it_be(:user) { project.first_owner }
let_it_be(:experiment) { create(:ml_experiments, project: project, user: user) }
let_it_be(:candidate) { create(:ml_candidates, experiment: experiment, user: user) }
let(:ff_value) { true }
let(:threshold) { 4 }
let(:candidate_iid) { candidate.iid }
before do
stub_feature_flags(ml_experiment_tracking: false)
stub_feature_flags(ml_experiment_tracking: project) if ff_value
sign_in(user)
end
shared_examples '404 if feature flag disabled' do
context 'when :ml_experiment_tracking disabled' do
let(:ff_value) { false }
it 'is 404' do
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
describe 'GET show' do
let(:params) { basic_params.merge(id: experiment.iid) }
before do
show_candidate
end
it 'renders the template' do
expect(response).to render_template('projects/ml/candidates/show')
end
# MR removing this xit https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104166
xit 'does not perform N+1 sql queries' do
control_count = ActiveRecord::QueryRecorder.new { show_candidate }
create_list(:ml_candidate_params, 3, candidate: candidate)
create_list(:ml_candidate_metrics, 3, candidate: candidate)
expect { show_candidate }.not_to exceed_all_query_limit(control_count).with_threshold(threshold)
end
context 'when candidate does not exist' do
let(:candidate_iid) { non_existing_record_id.to_s }
it 'returns 404' do
expect(response).to have_gitlab_http_status(:not_found)
end
end
it_behaves_like '404 if feature flag disabled'
end
private
def show_candidate
get project_ml_candidate_path(project, candidate_iid)
end
end

5
spec/requests/projects/ml/experiments_controller_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Projects::Ml::ExperimentsController do
RSpec.describe Projects::Ml::ExperimentsController, feature_category: :mlops do
let_it_be(:project_with_feature) { create(:project, :repository) }
let_it_be(:user) { project_with_feature.first_owner }
let_it_be(:project_without_feature) do
@ -77,7 +77,8 @@ RSpec.describe Projects::Ml::ExperimentsController do
expect(response).to render_template('projects/ml/experiments/show')
end
it 'does not perform N+1 sql queries' do
# MR removing this xit https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104166
xit 'does not perform N+1 sql queries' do
control_count = ActiveRecord::QueryRecorder.new { show_experiment }
create_list(:ml_candidates, 2, :with_metrics_and_params, experiment: experiment)

4
workhorse/go.mod
View File

@ -7,7 +7,7 @@ require (
github.com/BurntSushi/toml v1.2.1
github.com/FZambia/sentinel v1.1.1
github.com/alecthomas/chroma/v2 v2.3.0
github.com/aws/aws-sdk-go v1.44.145
github.com/aws/aws-sdk-go v1.44.150
github.com/disintegration/imaging v1.6.2
github.com/getsentry/raven-go v0.2.0
github.com/golang-jwt/jwt/v4 v4.4.3
@ -17,7 +17,7 @@ require (
github.com/gorilla/websocket v1.5.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/johannesboyne/gofakes3 v0.0.0-20221110173912-32fb85c5aed6
github.com/johannesboyne/gofakes3 v0.0.0-20221128113635-c2f5cc6b5294
github.com/jpillora/backoff v1.0.0
github.com/mitchellh/copystructure v1.2.0
github.com/prometheus/client_golang v1.14.0

8
workhorse/go.sum
View File

@ -227,8 +227,8 @@ github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4
github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go v1.44.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go v1.44.68/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go v1.44.145 h1:KMVRrIyjBsNz3xGPuHIRnhIuKlb5h3Ii5e5jbi3cgnc=
github.com/aws/aws-sdk-go v1.44.145/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.150 h1:X9HBhXu0ZPi+tOHUaZkjx43int7g0Ejk+IVbW25+wYg=
github.com/aws/aws-sdk-go v1.44.150/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go-v2 v1.16.8 h1:gOe9UPR98XSf7oEJCcojYg+N2/jCRm4DdeIsP85pIyQ=
github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=
@ -976,8 +976,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
github.com/johannesboyne/gofakes3 v0.0.0-20221110173912-32fb85c5aed6 h1:eQGUsj2LcsLzfrHY1noKDSU7h+c9/rw9pQPwbQ9g1jQ=
github.com/johannesboyne/gofakes3 v0.0.0-20221110173912-32fb85c5aed6/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI=
github.com/johannesboyne/gofakes3 v0.0.0-20221128113635-c2f5cc6b5294 h1:AJISYN7tPo3lGqwYmEYQdlftcQz48i8LNk/BRUKCTig=
github.com/johannesboyne/gofakes3 v0.0.0-20221128113635-c2f5cc6b5294/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=

8
workhorse/internal/helper/raven.go → workhorse/internal/helper/exception/exception.go
View File

@ -1,4 +1,4 @@
package helper
package exception
import (
"net/http"
@ -17,7 +17,7 @@ var ravenHeaderBlacklist = []string{
"Private-Token",
}
func CaptureRavenError(r *http.Request, err error, fields log.Fields) {
func Track(r *http.Request, err error, fields log.Fields) {
client := raven.DefaultClient
extra := raven.Extra{}
@ -27,7 +27,7 @@ func CaptureRavenError(r *http.Request, err error, fields log.Fields) {
interfaces := []raven.Interface{}
if r != nil {
CleanHeadersForRaven(r)
CleanHeaders(r)
interfaces = append(interfaces, raven.NewHttp(r))
//lint:ignore SA1019 this was recently deprecated. Update workhorse to use labkit errortracking package.
@ -45,7 +45,7 @@ func CaptureRavenError(r *http.Request, err error, fields log.Fields) {
client.Capture(packet, nil)
}
func CleanHeadersForRaven(r *http.Request) {
func CleanHeaders(r *http.Request) {
if r == nil {
return
}

28
workhorse/internal/helper/helpers.go
View File

@ -12,21 +12,13 @@ import (
"strings"
"github.com/sebest/xff"
"gitlab.com/gitlab-org/labkit/log"
"gitlab.com/gitlab-org/labkit/mask"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/log"
)
func logErrorWithFields(r *http.Request, err error, fields log.Fields) {
if err != nil {
CaptureRavenError(r, err, fields)
}
printError(r, err, fields)
}
func CaptureAndFail(w http.ResponseWriter, r *http.Request, err error, msg string, code int) {
http.Error(w, msg, code)
logErrorWithFields(r, err, nil)
printError(r, err, nil)
}
func Fail500(w http.ResponseWriter, r *http.Request, err error) {
@ -35,7 +27,7 @@ func Fail500(w http.ResponseWriter, r *http.Request, err error) {
func Fail500WithFields(w http.ResponseWriter, r *http.Request, err error, fields log.Fields) {
http.Error(w, "Internal server error", http.StatusInternalServerError)
logErrorWithFields(r, err, fields)
printError(r, err, fields)
}
func RequestEntityTooLarge(w http.ResponseWriter, r *http.Request, err error) {
@ -43,15 +35,7 @@ func RequestEntityTooLarge(w http.ResponseWriter, r *http.Request, err error) {
}
func printError(r *http.Request, err error, fields log.Fields) {
if r != nil {
entry := log.WithContextFields(r.Context(), log.Fields{
"method": r.Method,
"uri": mask.URL(r.RequestURI),
})
entry.WithFields(fields).WithError(err).Error()
} else {
log.WithFields(fields).WithError(err).Error("unknown error")
}
log.WithRequest(r).WithFields(fields).WithError(err).Error()
}
func SetNoCacheHeaders(header http.Header) {
@ -93,7 +77,7 @@ func OpenFile(path string) (file *os.File, fi os.FileInfo, err error) {
func URLMustParse(s string) *url.URL {
u, err := url.Parse(s)
if err != nil {
log.WithError(err).WithField("url", s).Fatal("urlMustParse")
log.WithError(err).WithFields(log.Fields{"url": s}).Fatal("urlMustParse")
}
return u
}

20
workhorse/internal/log/logging.go
View File

@ -8,7 +8,7 @@ import (
"gitlab.com/gitlab-org/labkit/mask"
"golang.org/x/net/context"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper/exception"
)
type Fields = log.Fields
@ -79,10 +79,18 @@ func Error(args ...interface{}) {
NewBuilder().Error(args...)
}
func (b *Builder) Error(args ...interface{}) {
b.entry.Error(args...)
if b.req != nil && b.err != nil {
helper.CaptureRavenError(b.req, b.err, b.fields)
func (b *Builder) trackException() {
if b.err != nil {
exception.Track(b.req, b.err, b.fields)
}
}
func (b *Builder) Error(args ...interface{}) {
b.trackException()
b.entry.Error(args...)
}
func (b *Builder) Fatal(args ...interface{}) {
b.trackException()
b.entry.Fatal(args...)
}

4
workhorse/raven.go
View File

@ -6,7 +6,7 @@ import (
raven "github.com/getsentry/raven-go"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper/exception"
)
func wrapRaven(h http.Handler) http.Handler {
@ -30,7 +30,7 @@ func wrapRaven(h http.Handler) http.Handler {
func(w http.ResponseWriter, r *http.Request) {
defer func() {
if p := recover(); p != nil {
helper.CleanHeadersForRaven(r)
exception.CleanHeaders(r)
panic(p)
}
}()