root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-11-08 21:27:41 +00:00
parent 9261646b8c
commit 4b231b78b4
63 changed files with 844 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
.gitlab/ci/observability-backend.gitlab-ci.yml
View File

@ -3,14 +3,50 @@
# of opstrace being brought in through an acquisition.
.e2e-observability-backend-base:
stage: test
image: alpine:latest
needs: []
inherit:
variables: false
variables:
TEST_GITLAB_COMMIT: $CI_COMMIT_SHA
trigger:
project: gitlab-org/opstrace/opstrace
strategy: depend
before_script:
- apk add jq curl
script:
- |
OPSTRACE_PROJECT_ID="32149347"
echo "Triggering pipeline on opstrace/opstrace project with: ref=${OPSTRACE_REF} TEST_GITLAB_COMMIT=${CI_COMMIT_SHA}"
response=$(curl -s -X POST --fail \
-F "token=${GITLAB_OBSERVABILITY_BACKEND_PIPELINE_TRIGGER_TOKEN}" \
-F "ref=${OPSTRACE_REF}" \
-F "variables[TEST_GITLAB_COMMIT]=${CI_COMMIT_SHA}" \
"https://gitlab.com/api/v4/projects/${OPSTRACE_PROJECT_ID}/trigger/pipeline")
# Extract the pipeline ID from the response
pipeline_id=$(echo "$response" | jq -r '.id')
if [ -z "$pipeline_id" ] || [ "$pipeline_id" == "null" ]; then
echo "Failed to trigger pipeline."
exit 1
else
web_url=$(echo "$response" | jq -r '.web_url')
echo "Pipeline $pipeline_id created: $web_url"
fi
# Poll the pipeline status until it succeeds or fails
status=""
while true; do
status=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_OBSERVABILITY_BACKEND_TOKEN_FOR_CI_SCRIPTS}" \
"https://gitlab.com/api/v4/projects/${OPSTRACE_PROJECT_ID}/pipelines/$pipeline_id" | jq -r '.status')
if [ -z "$status" ] || [ "$status" == "null" ]; then
echo "Failed to get pipeline status"
exit 1
fi
echo "Pipeline status: $status"
if [ "$status" == "success" ]; then
echo "Triggered pipeline succeeded."
exit 0
elif [ "$status" == "failed" ]; then
echo "Triggered pipeline failed."
exit 1
elif [ "$status" == "canceled" ] || [ "$status" == "canceling" ]; then
echo "Triggered pipeline was canceled."
exit 1
fi
sleep 60
done
# e2e:observability-backend uses $CI_COMMIT_REF_NAME to
# checkout a branch in gitlab-org/opstrace/opstrace with
@ -21,21 +57,17 @@ e2e:observability-backend:
extends:
- .e2e-observability-backend-base
- .observability-backend-current-branch:rules
trigger:
project: gitlab-org/opstrace/opstrace
branch: $CI_COMMIT_REF_NAME
variables:
OPSTRACE_REF: $CI_COMMIT_REF_NAME
# e2e:observability-backend-main-branch will trigger
# an e2e test pipeline that checks out GitLab to
# $CI_COMMIT_SHA and Opstrace to the latest commit
# on main branch. Devs run this manually on local
# installs today periodically during development
# and this manual job increases dev velocity
# and testing reliablity.
# on main branch.
e2e:observability-backend-main-branch:
extends:
- .e2e-observability-backend-base
- .observability-backend-main-branch:rules
trigger:
project: gitlab-org/opstrace/opstrace
branch: main
variables:
OPSTRACE_REF: main

18
.gitlab/ci/rules.gitlab-ci.yml
View File

@ -213,6 +213,9 @@
.if-ruby-branch: &if-ruby-branch
if: '$CI_COMMIT_BRANCH =~ /^ruby\d+(_\d)*$/ || (($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_EVENT_TYPE != "merge_train") && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby\d+(_\d)*/)'
.if-observability-skip-e2e-jobs: &if-observability-skip-e2e-jobs
if: '$SKIP_GITLAB_OBSERVABILITY_BACKEND_TRIGGER || $GITLAB_OBSERVABILITY_BACKEND_PIPELINE_TRIGGER_TOKEN == null || $GITLAB_OBSERVABILITY_BACKEND_TOKEN_FOR_CI_SCRIPTS == null'
####################
# Changes patterns #
####################
@ -3384,28 +3387,21 @@
###############################
.observability-backend-main-branch:rules:
rules:
- <<: *if-observability-skip-e2e-jobs
when: never
- <<: *if-merge-request-labels-run-observability-e2e-tests-current-branch
when: never
- <<: *if-merge-request-labels-run-observability-e2e-tests-main-branch
allow_failure: true
- <<: *if-merge-request
changes: *observability-code-patterns
allow_failure: true
- <<: *if-merge-request
changes: *code-patterns
when: manual
allow_failure: true
.observability-backend-current-branch:rules:
rules:
- <<: *if-observability-skip-e2e-jobs
when: never
- <<: *if-merge-request-labels-run-observability-e2e-tests-main-branch
when: never
- <<: *if-merge-request-labels-run-observability-e2e-tests-current-branch
allow_failure: true
- <<: *if-merge-request
changes: *code-patterns
when: manual
allow_failure: true
##########################
# Pre-merge checks rules #

1
app/assets/javascripts/ml/model_registry/apps/show_ml_model.vue
View File

@ -263,6 +263,7 @@ export default {
<actions-dropdown>
<delete-model-disclosure-dropdown-item
v-if="canWriteModelRegistry"
:model="model"
@confirm-deletion="deleteModel"
/>
</actions-dropdown>

32
app/assets/javascripts/ml/model_registry/components/delete_model_disclosure_dropdown_item.vue
View File

@ -5,7 +5,7 @@ import {
GlDisclosureDropdownItem,
GlModalDirective,
} from '@gitlab/ui';
import { __, s__ } from '~/locale';
import { __, s__, sprintf } from '~/locale';
export default {
components: {
@ -16,11 +16,17 @@ export default {
GlTooltip: GlTooltipDirective,
GlModalDirective,
},
props: {
model: {
type: Object,
required: true,
},
},
data() {
return {
isDeleteModalVisible: false,
modal: {
id: 'ml-experiments-delete-modal',
id: 'ml-models-delete-modal',
deleteConfirmation: this.deleteConfirmationText,
actionPrimary: {
text: this.$options.i18n.actionPrimaryText,
@ -32,14 +38,26 @@ export default {
},
};
},
computed: {
modelName() {
return this.model.name;
},
modalId() {
return `ml-models-delete-modal-${this.model.id}`;
},
modalTitle() {
return sprintf(s__('MlModelRegistry|Delete model %{modelName}'), {
modelName: this.modelName,
});
},
},
methods: {
confirmDelete() {
this.$emit('confirm-deletion');
this.$emit('confirm-deletion', this.model.id);
},
},
i18n: {
actionPrimaryText: s__('MlModelRegistry|Delete model'),
modalTitle: s__('MlModelRegistry|Delete model'),
deleteConfirmationText: s__(
'MlExperimentTracking|Are you sure you would like to delete this model?',
),
@ -52,7 +70,7 @@ export default {
<template>
<gl-disclosure-dropdown-item
v-gl-modal-directive="modal.id"
v-gl-modal-directive="modalId"
:aria-label="$options.i18n.actionPrimaryText"
variant="danger"
>
@ -62,8 +80,8 @@ export default {
</span>
<gl-modal
:modal-id="modal.id"
:title="$options.i18n.modalTitle"
:modal-id="modalId"
:title="modalTitle"
:action-primary="modal.actionPrimary"
:action-cancel="modal.actionCancel"
@primary="confirmDelete"

47
app/assets/javascripts/ml/model_registry/components/models_table.vue
View File

@ -1,7 +1,17 @@
<script>
import { GlAvatarLink, GlAvatar, GlTable, GlLink, GlTooltip } from '@gitlab/ui';
import {
GlAvatarLink,
GlAvatar,
GlTable,
GlLink,
GlTooltip,
GlDisclosureDropdown,
} from '@gitlab/ui';
import TimeAgoTooltip from '~/vue_shared/components/time_ago_tooltip.vue';
import { s__ } from '~/locale';
import { convertToGraphQLId } from '~/graphql_shared/utils';
import DeleteModel from './functional/delete_model.vue';
import DeleteModelDisclosureDropdownItem from './delete_model_disclosure_dropdown_item.vue';
export default {
name: 'ModelsTable',
@ -11,6 +21,9 @@ export default {
TimeAgoTooltip,
GlAvatar,
GlLink,
DeleteModelDisclosureDropdownItem,
GlDisclosureDropdown,
DeleteModel,
},
directives: {
GlTooltip,
@ -20,6 +33,10 @@ export default {
type: Array,
required: true,
},
canWriteModelRegistry: {
type: Boolean,
required: true,
},
},
computed: {
tableFields() {
@ -28,6 +45,12 @@ export default {
{ key: 'latestVersion', label: s__('ModelRegistry|Latest version'), thClass: 'gl-w-1/4' },
{ key: 'author', label: s__('ModelRegistry|Author'), thClass: 'gl-w-1/4' },
{ key: 'createdAt', label: s__('ModelRegistry|Created'), thClass: 'gl-w-1/4' },
{
key: 'actions',
label: '',
tdClass: 'lg:gl-w-px gl-whitespace-nowrap',
thClass: 'lg:gl-w-px gl-whitespace-nowrap',
},
];
},
},
@ -38,6 +61,12 @@ export default {
showLatestVersion(item) {
return item.latestVersion && item.latestVersion._links;
},
modelGid(model) {
return convertToGraphQLId('Ml::Model', model.id);
},
modelDeleted() {
this.$emit('models-update');
},
},
};
</script>
@ -70,5 +99,21 @@ export default {
<template #cell(createdAt)="{ item: { createdAt } }">
<time-ago-tooltip v-if="createdAt" :time="createdAt" />
</template>
<template #cell(actions)="{ item }">
<delete-model :model-id="modelGid(item)" @model-deleted="modelDeleted">
<template #default="{ deleteModel }">
<gl-disclosure-dropdown
v-if="canWriteModelRegistry"
placement="bottom-end"
category="tertiary"
:aria-label="__('More actions')"
icon="ellipsis_v"
no-caret
>
<delete-model-disclosure-dropdown-item :model="item" @confirm-deletion="deleteModel" />
</gl-disclosure-dropdown>
</template>
</delete-model>
</template>
</gl-table>
</template>

7
app/assets/javascripts/ml/model_registry/components/searchable_table.vue
View File

@ -168,7 +168,12 @@ export default {
can-write-model-registry
@model-versions-update="submitFilters"
/>
<models-table v-else-if="!isModelsEmpty" :items="models" />
<models-table
v-else-if="!isModelsEmpty"
:items="models"
can-write-model-registry
@models-update="submitFilters"
/>
<gl-keyset-pagination
v-if="pageInfo.hasPreviousPage || pageInfo.hasNextPage"
v-bind="pageInfo"

10
db/migrate/20241107180528_add_cursors_to_batched_background_migration_jobs.rb Normal file
View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
class AddCursorsToBatchedBackgroundMigrationJobs < Gitlab::Database::Migration[2.2]
milestone '17.6'
def change
add_column :batched_background_migration_jobs, :min_cursor, :jsonb, null: true
add_column :batched_background_migration_jobs, :max_cursor, :jsonb, null: true
end
end

10
db/migrate/20241107180531_add_cursors_to_batched_background_migrations.rb Normal file
View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
class AddCursorsToBatchedBackgroundMigrations < Gitlab::Database::Migration[2.2]
milestone '17.6'
def change
add_column :batched_background_migrations, :min_cursor, :jsonb, null: true
add_column :batched_background_migrations, :max_cursor, :jsonb, null: true
end
end

18
db/migrate/20241107180533_add_ids_or_cursors_constraint_to_batched_background_migration_jobs.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class AddIdsOrCursorsConstraintToBatchedBackgroundMigrationJobs < Gitlab::Database::Migration[2.2]
milestone '17.6'
disable_ddl_transaction!
def up
add_check_constraint :batched_background_migration_jobs,
'num_nonnulls(min_value, max_value) = 2 OR num_nonnulls(min_cursor, max_cursor) = 2',
check_constraint_name(:batched_background_migration_jobs, 'ids_or_cursors', 'not_null')
end
def down
remove_check_constraint :batched_background_migration_jobs,
check_constraint_name(:batched_background_migration_jobs, 'ids_or_cursors', 'not_null')
end
end

18
db/migrate/20241107180537_add_ids_or_cursors_constraint_to_batched_background_migrations.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class AddIdsOrCursorsConstraintToBatchedBackgroundMigrations < Gitlab::Database::Migration[2.2]
milestone '17.6'
disable_ddl_transaction!
def up
add_check_constraint :batched_background_migrations,
'num_nonnulls(min_value, max_value) = 2 OR num_nonnulls(min_cursor, max_cursor) = 2',
check_constraint_name(:batched_background_migrations, 'ids_or_cursors', 'not_null')
end
def down
remove_check_constraint :batched_background_migrations,
check_constraint_name(:batched_background_migrations, 'ids_or_cursors', 'not_null')
end
end

17
db/migrate/20241107180541_remove_values_not_null_constraints_from_batched_background_migration_jobs.rb Normal file
View File

@ -0,0 +1,17 @@
# frozen_string_literal: true
class RemoveValuesNotNullConstraintsFromBatchedBackgroundMigrationJobs < Gitlab::Database::Migration[2.2]
milestone '17.6'
disable_ddl_transaction!
def up
change_column_null :batched_background_migration_jobs, :max_value, true
change_column_null :batched_background_migration_jobs, :min_value, true
end
def down
change_column_null :batched_background_migration_jobs, :max_value, false
change_column_null :batched_background_migration_jobs, :min_value, false
end
end

17
db/migrate/20241107180543_remove_values_not_null_constraints_from_batched_background_migrations.rb Normal file
View File

@ -0,0 +1,17 @@
# frozen_string_literal: true
class RemoveValuesNotNullConstraintsFromBatchedBackgroundMigrations < Gitlab::Database::Migration[2.2]
milestone '17.6'
disable_ddl_transaction!
def up
change_column_null :batched_background_migrations, :max_value, true
change_column_null :batched_background_migrations, :min_value, true
end
def down
change_column_null :batched_background_migrations, :max_value, false
change_column_null :batched_background_migrations, :min_value, false
end
end

18
db/post_migrate/20241107180535_add_jsonb_array_constraints_to_batched_background_migration_jobs.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class AddJsonbArrayConstraintsToBatchedBackgroundMigrationJobs < Gitlab::Database::Migration[2.2]
milestone '17.6'
disable_ddl_transaction!
def up
add_check_constraint :batched_background_migration_jobs,
"jsonb_typeof(min_cursor) = 'array' AND jsonb_typeof(max_cursor) = 'array'",
check_constraint_name(:batched_background_migration_jobs, 'cursors', 'jsonb_array')
end
def down
remove_check_constraint :batched_background_migration_jobs,
check_constraint_name(:batched_background_migration_jobs, 'cursors', 'jsonb_array')
end
end

18
db/post_migrate/20241107180539_add_jsonb_array_constraints_to_batched_background_migrations.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class AddJsonbArrayConstraintsToBatchedBackgroundMigrations < Gitlab::Database::Migration[2.2]
milestone '17.6'
disable_ddl_transaction!
def up
add_check_constraint :batched_background_migrations,
"jsonb_typeof(min_cursor) = 'array' AND jsonb_typeof(max_cursor) = 'array'",
check_constraint_name(:batched_background_migrations, 'cursors', 'jsonb_array')
end
def down
remove_check_constraint :batched_background_migrations,
check_constraint_name(:batched_background_migrations, 'cursors', 'jsonb_array')
end
end

20
db/post_migrate/20241107180545_index_background_migration_jobs_on_migration_id_and_max_cursor.rb Normal file
View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
class IndexBackgroundMigrationJobsOnMigrationIdAndMaxCursor < Gitlab::Database::Migration[2.2]
milestone '17.6'
disable_ddl_transaction!
INDEX_NAME = 'index_migration_jobs_on_migration_id_and_cursor_max_value'
def up
add_concurrent_index :batched_background_migration_jobs,
"batched_background_migration_id, max_cursor",
where: "max_cursor is not null",
name: INDEX_NAME
end
def down
remove_concurrent_index_by_name :batched_background_migration_jobs, name: INDEX_NAME
end
end

1
db/schema_migrations/20241107180528 Normal file
View File

@ -0,0 +1 @@
2cd53d0b40e3a40b63d1884dd6edd566e5f73daa94ae4cdf3bd0b7b2c06a410f

1
db/schema_migrations/20241107180531 Normal file
View File

@ -0,0 +1 @@
ad4cd463159888921a6282759970e383f4b5302ee312a802cc7b7c0934aa2c30

1
db/schema_migrations/20241107180533 Normal file
View File

@ -0,0 +1 @@
7e09202c2462efd09137564aa54c24117e7796057a870270ae2f1b7fb79c0836

1
db/schema_migrations/20241107180535 Normal file
View File

@ -0,0 +1 @@
bb493c349e1703ab3bf102841a94f18f1c75abf44503125db6b1340288f2ef73

1
db/schema_migrations/20241107180537 Normal file
View File

@ -0,0 +1 @@
a6f8299dc90cec44b163a11b1922e04580632a2f4e3259c595224fad948010ad

1
db/schema_migrations/20241107180539 Normal file
View File

@ -0,0 +1 @@
2f0dd7e0994e9f343a4da2a26020cd3137cd593fde4017e4c422a7cbccc5eb8f

1
db/schema_migrations/20241107180541 Normal file
View File

@ -0,0 +1 @@
7144e3fde08de141d3eeb5595ed4e72f7afecc03a727be3f5d9f3222748d21c7

1
db/schema_migrations/20241107180543 Normal file
View File

@ -0,0 +1 @@
89e25ea1719a78951ae6d47227f739802c28899e3b22b8bff73d4e3fa2524f80

1
db/schema_migrations/20241107180545 Normal file
View File

@ -0,0 +1 @@
dbfac17650b13fb4063080353082e53f54a097b5d8802eb4d2a318e5bf87d8cf

20
db/structure.sql
View File

@ -7842,14 +7842,18 @@ CREATE TABLE batched_background_migration_jobs (
started_at timestamp with time zone,
finished_at timestamp with time zone,
batched_background_migration_id bigint NOT NULL,
min_value bigint NOT NULL,
max_value bigint NOT NULL,
min_value bigint,
max_value bigint,
batch_size integer NOT NULL,
sub_batch_size integer NOT NULL,
status smallint DEFAULT 0 NOT NULL,
attempts smallint DEFAULT 0 NOT NULL,
metrics jsonb DEFAULT '{}'::jsonb NOT NULL,
pause_ms integer DEFAULT 100 NOT NULL
pause_ms integer DEFAULT 100 NOT NULL,
min_cursor jsonb,
max_cursor jsonb,
CONSTRAINT check_18d498ea58 CHECK (((jsonb_typeof(min_cursor) = 'array'::text) AND (jsonb_typeof(max_cursor) = 'array'::text))),
CONSTRAINT check_c1ce96fe3b CHECK (((num_nonnulls(min_value, max_value) = 2) OR (num_nonnulls(min_cursor, max_cursor) = 2)))
);
CREATE SEQUENCE batched_background_migration_jobs_id_seq
@ -7865,8 +7869,8 @@ CREATE TABLE batched_background_migrations (
id bigint NOT NULL,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
min_value bigint DEFAULT 1 NOT NULL,
max_value bigint NOT NULL,
min_value bigint DEFAULT 1,
max_value bigint,
batch_size integer NOT NULL,
sub_batch_size integer NOT NULL,
"interval" smallint NOT NULL,
@ -7884,12 +7888,16 @@ CREATE TABLE batched_background_migrations (
gitlab_schema text NOT NULL,
finished_at timestamp with time zone,
queued_migration_version text,
min_cursor jsonb,
max_cursor jsonb,
CONSTRAINT check_0406d9776f CHECK ((char_length(gitlab_schema) <= 255)),
CONSTRAINT check_122750e705 CHECK (((jsonb_typeof(min_cursor) = 'array'::text) AND (jsonb_typeof(max_cursor) = 'array'::text))),
CONSTRAINT check_5bb0382d6f CHECK ((char_length(column_name) <= 63)),
CONSTRAINT check_6b6a06254a CHECK ((char_length(table_name) <= 63)),
CONSTRAINT check_713f147aea CHECK ((char_length(queued_migration_version) <= 14)),
CONSTRAINT check_batch_size_in_range CHECK ((batch_size >= sub_batch_size)),
CONSTRAINT check_e6c75b1e29 CHECK ((char_length(job_class_name) <= 100)),
CONSTRAINT check_f5158baa12 CHECK (((num_nonnulls(min_value, max_value) = 2) OR (num_nonnulls(min_cursor, max_cursor) = 2))),
CONSTRAINT check_fe10674721 CHECK ((char_length(batch_class_name) <= 100)),
CONSTRAINT check_max_value_in_range CHECK ((max_value >= min_value)),
CONSTRAINT check_positive_min_value CHECK ((min_value > 0)),
@ -30673,6 +30681,8 @@ CREATE INDEX index_metrics_dashboard_annotations_on_timespan_end ON metrics_dash
CREATE INDEX index_metrics_users_starred_dashboards_on_project_id ON metrics_users_starred_dashboards USING btree (project_id);
CREATE INDEX index_migration_jobs_on_migration_id_and_cursor_max_value ON batched_background_migration_jobs USING btree (batched_background_migration_id, max_cursor) WHERE (max_cursor IS NOT NULL);
CREATE INDEX index_migration_jobs_on_migration_id_and_finished_at ON batched_background_migration_jobs USING btree (batched_background_migration_id, finished_at);
CREATE INDEX index_migration_jobs_on_migration_id_and_max_value ON batched_background_migration_jobs USING btree (batched_background_migration_id, max_value);

2
doc/user/application_security/vulnerability_report/index.md
View File

@ -11,6 +11,7 @@ DETAILS:
**Offering:** GitLab.com, Self-managed, GitLab Dedicated
> - Vulnerability Resolution activity icon [introduced](https://gitlab.com/groups/gitlab-org/-/epics/15036) in GitLab 17.5 with a flag named [`vulnerability_report_vr_badge`](https://gitlab.com/gitlab-org/gitlab/-/issues/486549). Disabled by default.
> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/171718) in GitLab 17.6.
FLAG:
The availability of Vulnerability Resolution activity icon is controlled by a feature flag.
@ -153,6 +154,7 @@ The content of the Project filter varies:
> - Introduced in GitLab 16.7 [with a flag](../../../administration/feature_flags.md) named `activity_filter_has_remediations`. Disabled by default.
> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/429262) in GitLab 16.9. Feature flag `activity_filter_has_remediations` removed.
> - Activity filter option **GitLab Duo (AI)** [introduced](https://gitlab.com/groups/gitlab-org/-/epics/15036) in GitLab 17.5 with a flag named [`vulnerability_report_vr_filter`](https://gitlab.com/gitlab-org/gitlab/-/issues/486534). Disabled by default.
> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/171718) in GitLab 17.6.
FLAG:
The availability of the activity filter option **GitLab Duo (AI)** is controlled by a feature flag.

41
doc/user/packages/go_proxy/index.md
View File

@ -11,7 +11,7 @@ DETAILS:
**Offering:** GitLab.com, Self-managed, GitLab Dedicated
**Status:** Experiment
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27376) in GitLab 13.1 [with a flag](../../../administration/feature_flags.md) named `go_proxy`. Disabled by default.
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27376) in GitLab 13.1 [with a flag](../../../administration/feature_flags.md) named `go_proxy`. Disabled by default. This feature is an [experiment](../../../policy/experiment-beta-support.md).
FLAG:
The availability of this feature is controlled by a feature flag.
@ -22,43 +22,14 @@ See [epic 3043](https://gitlab.com/groups/gitlab-org/-/epics/3043).
With the Go proxy for GitLab, every project in GitLab can be fetched with the
[Go proxy protocol](https://proxy.golang.org/).
The Go proxy for GitLab is an [experiment](../../../policy/experiment-beta-support.md), and isn't ready for production use
due to potential performance issues with large repositories. See [issue 218083](https://gitlab.com/gitlab-org/gitlab/-/issues/218083).
GitLab doesn't display Go modules in the package registry, even if the Go proxy is enabled. See [issue 213770](https://gitlab.com/gitlab-org/gitlab/-/issues/213770).
For documentation of the specific API endpoints that the Go Proxy uses, see the
[Go Proxy API documentation](../../../api/packages/go_proxy.md).
## Enable the Go proxy
The Go proxy for GitLab is under development, and isn't ready for production use
due to [potential performance issues with large repositories](https://gitlab.com/gitlab-org/gitlab/-/issues/218083).
It's deployed behind a feature flag that is _disabled by default_.
[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md)
can enable it for your instance.
To enable it:
```ruby
Feature.enable(:go_proxy) # or
```
To disable it:
```ruby
Feature.disable(:go_proxy)
```
To enable or disable it for specific projects:
```ruby
Feature.enable(:go_proxy, Project.find(1))
Feature.disable(:go_proxy, Project.find(2))
```
NOTE:
Even if it's enabled, GitLab doesn't display Go modules in the **package registry**.
Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213770) for
details.
## Add GitLab as a Go proxy
To use GitLab as a Go proxy, you must be using Go 1.13 or later.

81
lib/gitlab/background_migration/batched_migration_job.rb
View File

@ -59,15 +59,31 @@ module Gitlab
get_class_attribute(:feature_category) || DEFAULT_FEATURE_CATEGORY
end
end
def cursor_columns
[]
end
def cursor?
cursor_columns.count > 1
end
def cursor(*args)
define_singleton_method(:cursor_columns) do
args
end
end
end
def initialize(
start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:, job_arguments: [],
sub_batch_exception: nil
batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:, job_arguments: [],
start_id: nil, end_id: nil, start_cursor: nil, end_cursor: nil, sub_batch_exception: nil
)
@start_id = start_id
@end_id = end_id
@start_cursor = start_cursor
@end_cursor = end_cursor
@batch_table = batch_table
@batch_column = batch_column
@sub_batch_size = sub_batch_size
@ -91,18 +107,23 @@ module Gitlab
private
attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size,
attr_reader :start_id, :end_id, :start_cursor, :end_cursor, :batch_table, :batch_column, :sub_batch_size,
:pause_ms, :connection, :sub_batch_exception
delegate :cursor_columns, :cursor?, to: :'self.class'
def each_sub_batch(batching_arguments: {}, batching_scope: nil)
all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments)
base_batching_arguments = if cursor?
{ load_batch: false }
else
{ column: batch_column }
end
relation = filter_batch(base_relation)
sub_batch_relation = filter_sub_batch(relation, batching_scope)
all_batching_arguments = { of: sub_batch_size }.merge(base_batching_arguments, batching_arguments)
sub_batch_relation.each_batch(**all_batching_arguments) do |relation|
sub_batch_relation(batching_scope: batching_scope).each_batch(**all_batching_arguments) do |sub_batch|
batch_metrics.instrument_operation(operation_name) do
yield relation
yield sub_batch
rescue *Gitlab::Database::BackgroundMigration::BatchedJob::TIMEOUT_EXCEPTIONS => exception
exception_class = sub_batch_exception || exception.class
@ -130,8 +151,31 @@ module Gitlab
end
def base_relation
define_batchable_model(batch_table, connection: connection, primary_key: batch_column)
.where(batch_column => start_id..end_id)
if cursor?
model_class = define_batchable_model(batch_table, connection: connection)
cursor_expression = Arel::Nodes::Grouping.new(
cursor_columns.map { |column| model_class.arel_table[column] }
)
cursor_gteq_start = cursor_expression.gteq(arel_for_cursor(start_cursor, model_class.arel_table))
cursor_lteq_end = cursor_expression.lteq(arel_for_cursor(end_cursor, model_class.arel_table))
where_condition = Arel::Nodes::And.new([cursor_gteq_start, cursor_lteq_end])
model_class.where(where_condition)
else
define_batchable_model(batch_table, connection: connection, primary_key: batch_column)
.where(batch_column => start_id..end_id)
end
end
def arel_for_cursor(cursor, arel_table)
Arel::Nodes::Grouping.new(
cursor_columns.zip(cursor).map do |column, value|
Arel::Nodes.build_quoted(value, arel_table[column])
end
)
end
def filter_sub_batch(relation, batching_scope = nil)
@ -140,8 +184,23 @@ module Gitlab
batching_scope.call(relation)
end
def sub_batch_relation(batching_scope: nil)
if cursor?
base_class = Gitlab::Database.application_record_for_connection(connection)
model_class = define_batchable_model(batch_table, connection: connection, base_class: base_class)
order = model_class.order(cursor_columns)
keyset_order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(order)
sub_batch_relation = Gitlab::Pagination::Keyset::Iterator.new(scope: base_relation.order(keyset_order))
else
relation = filter_batch(base_relation)
sub_batch_relation = filter_sub_batch(relation, batching_scope)
end
sub_batch_relation
end
def operation_name
raise('Operation name is required, please define it with `operation_name`')
raise('Operation name is required, please define it with `operation_name`') unless cursor?
end
end
end

52
lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
View File

@ -19,31 +19,49 @@ module Gitlab
# batch_size - The size of the next batch
# job_arguments - The migration job arguments
# job_class - The migration job class
# rubocop:disable Metrics/AbcSize -- temporarily contains two branches for cursor and non-cursor batching
def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil)
model_class = define_batchable_model(table_name, connection: connection)
arel_column = model_class.arel_table[column_name]
relation = model_class.where(arel_column.gteq(batch_min_value))
reset_order = true
if job_class
relation = filter_batch(relation,
table_name: table_name, column_name: column_name,
job_class: job_class, job_arguments: job_arguments
)
reset_order = job_class.reset_order if job_class.respond_to?(:reset_order)
end
base_class = Gitlab::Database.application_record_for_connection(connection)
model_class = define_batchable_model(table_name, connection: connection, base_class: base_class)
next_batch_bounds = nil
relation.each_batch(of: batch_size, column: column_name, reset_order: reset_order) do |batch| # rubocop:disable Lint/UnreachableLoop
next_batch_bounds = batch.pick(arel_column.minimum, arel_column.maximum)
# rubocop:disable Lint/UnreachableLoop -- we need to use each_batch to pull one batch out
if job_class.cursor?
cursor_columns = job_class.cursor_columns
break
Gitlab::Pagination::Keyset::Iterator.new(
scope: model_class.order(cursor_columns),
cursor: cursor_columns.zip(batch_min_value).to_h
).each_batch(of: batch_size, load_batch: false) do |batch|
break unless batch.first && batch.last # skip if the batch is empty for some reason
next_batch_bounds = [batch.first.values_at(cursor_columns), batch.last.values_at(cursor_columns)]
break
end
else
arel_column = model_class.arel_table[column_name]
relation = model_class.where(arel_column.gteq(batch_min_value))
reset_order = true
if job_class
relation = filter_batch(relation,
table_name: table_name, column_name: column_name,
job_class: job_class, job_arguments: job_arguments
)
reset_order = job_class.reset_order if job_class.respond_to?(:reset_order)
end
relation.each_batch(of: batch_size, column: column_name, reset_order: reset_order) do |batch|
next_batch_bounds = batch.pick(arel_column.minimum, arel_column.maximum)
break
end
end
# rubocop:enable Lint/UnreachableLoop
next_batch_bounds
end
# rubocop:enable Metrics/AbcSize
private

10
lib/gitlab/database.rb
View File

@ -114,6 +114,16 @@ module Gitlab
.compact.with_indifferent_access.freeze
end
# Returns the application record that created the given connection.
# In single database mode, this always returns ApplicationRecord.
def self.application_record_for_connection(connection)
@gitlab_base_models ||=
database_base_models
.transform_values { |v| v == ActiveRecord::Base ? ApplicationRecord : v }
@gitlab_base_models[db_config_name(connection)]
end
# This returns a list of base models with connection associated for a given gitlab_schema
def self.schemas_to_base_models
@schemas_to_base_models ||=

19
lib/gitlab/database/background_migration/batched_job.rb
View File

@ -113,6 +113,24 @@ module Gitlab
[exception, from_sub_batch]
end
def job_attributes
{
batch_table: migration_table_name,
batch_column: migration_column_name,
sub_batch_size: sub_batch_size,
pause_ms: pause_ms,
job_arguments: migration_job_arguments
}.tap do |attributes|
if migration_job_class.cursor?
attributes[:start_cursor] = min_cursor
attributes[:end_cursor] = max_cursor
else
attributes[:start_id] = min_value
attributes[:end_id] = max_value
end
end
end
def time_efficiency
return unless succeeded?
return unless finished_at && started_at
@ -135,6 +153,7 @@ module Gitlab
def split_and_retry!
with_lock do
raise SplitAndRetryError, 'Split and retry not yet supported for cursor based jobs' unless max_cursor.nil?
raise SplitAndRetryError, 'Only failed jobs can be split' unless failed?
new_batch_size = batch_size / 2

35
lib/gitlab/database/background_migration/batched_migration.rb
View File

@ -13,7 +13,10 @@ module Gitlab
self.table_name = :batched_background_migrations
has_many :batched_jobs, foreign_key: :batched_background_migration_id
has_one :last_job, -> { order(max_value: :desc) },
has_one :last_job,
->(relation) {
relation.cursor? ? where.not(max_cursor: nil).order(max_cursor: :desc) : order(max_value: :desc)
},
class_name: 'Gitlab::Database::BackgroundMigration::BatchedJob',
foreign_key: :batched_background_migration_id
@ -124,6 +127,8 @@ module Gitlab
.sum(:batch_size)
end
delegate :cursor?, to: :job_class
def reset_attempts_of_blocked_jobs!
batched_jobs.blocked_by_max_attempts.each_batch(of: 100) do |batch|
batch.update_all(attempts: 0)
@ -138,13 +143,21 @@ module Gitlab
end
def create_batched_job!(min, max)
batched_jobs.create!(
min_value: min,
max_value: max,
job_arguments = {
batch_size: batch_size,
sub_batch_size: sub_batch_size,
pause_ms: pause_ms
)
}
if cursor?
job_arguments[:min_cursor] = min
job_arguments[:max_cursor] = max
else
job_arguments[:min_value] = min
job_arguments[:max_value] = max
end
batched_jobs.create!(job_arguments)
end
def retry_failed_jobs!
@ -171,7 +184,17 @@ module Gitlab
end
def next_min_value
last_job&.max_value&.next || min_value
if cursor?
# Cursors require a subtle off-by-one change: we return the end of the last batch instead
# of bumping it by 1 with .next because this class doesn't know what's in the cursor.
# This means that the min_cursor here must be logically before the beginning of the batch, not just
# equal to the first row (if it's equal it'll make batching skip the first row), this is because the
# KeysetIterator we use for cursor batching expects the cursor passed to it to be before the start of
# the iteration range.
last_job&.max_cursor || min_cursor
else
last_job&.max_value&.next || min_value
end
end
def job_class

25
lib/gitlab/database/background_migration/batched_migration_runner.rb
View File

@ -91,11 +91,11 @@ module Gitlab
attr_reader :connection, :migration_wrapper
def find_or_create_next_batched_job(active_migration)
if next_batch_range = find_next_batch_range(active_migration)
active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max)
else
active_migration.batched_jobs.retriable.first
end
next_batch_min, next_batch_max = find_next_batch_range(active_migration)
return active_migration.batched_jobs.retriable.first if next_batch_min.nil? || next_batch_max.nil?
active_migration.create_batched_job!(next_batch_min, next_batch_max)
end
def find_next_batch_range(active_migration)
@ -116,13 +116,19 @@ module Gitlab
end
def clamped_batch_range(active_migration, next_bounds)
min_value, max_value = next_bounds
next_min, next_max = next_bounds
return if min_value > active_migration.max_value
if active_migration.cursor?
return if (next_min <=> active_migration.max_cursor) > 0
max_value = max_value.clamp(min_value, active_migration.max_value)
next_max = active_migration.max_cursor if (next_max <=> active_migration.max_cursor) > 0
else
return if next_min > active_migration.max_value
(min_value..max_value)
next_max = next_max.clamp(next_min, active_migration.max_value)
end
[next_min, next_max]
end
def finish_active_migration(active_migration)
@ -138,7 +144,6 @@ module Gitlab
def run_migration_while(migration, status)
while migration.status_name == status
run_migration_job(migration)
migration.reload_last_job
end
end

14
lib/gitlab/database/background_migration/batched_migration_wrapper.rb
View File

@ -68,15 +68,11 @@ module Gitlab
def execute_batched_migration_job(job_class, tracking_record)
job_instance = job_class.new(
start_id: tracking_record.min_value,
end_id: tracking_record.max_value,
batch_table: tracking_record.migration_table_name,
batch_column: tracking_record.migration_column_name,
sub_batch_size: tracking_record.sub_batch_size,
pause_ms: tracking_record.pause_ms,
job_arguments: tracking_record.migration_job_arguments,
connection: connection,
sub_batch_exception: ::Gitlab::Database::BackgroundMigration::SubBatchTimeoutError)
**tracking_record.job_attributes.merge({
connection: connection,
sub_batch_exception: ::Gitlab::Database::BackgroundMigration::SubBatchTimeoutError
})
)
job_instance.perform

4
lib/gitlab/database/dynamic_model_helpers.rb
View File

@ -5,8 +5,8 @@ module Gitlab
module DynamicModelHelpers
BATCH_SIZE = 1_000
def define_batchable_model(table_name, connection:, primary_key: nil)
klass = Class.new(ActiveRecord::Base) do
def define_batchable_model(table_name, connection:, primary_key: nil, base_class: ActiveRecord::Base)
klass = Class.new(base_class) do
include EachBatch
self.table_name = table_name

10
lib/sidebars/concerns/has_pill.rb
View File

@ -13,9 +13,13 @@ module Sidebars
# In this method we will need to provide the query
# to retrieve the elements count
def pill_count
raise NotImplementedError
end
def pill_count; end
# The GraphQL field name from `SidebarType` that will be used
# as the pill count for this menu item.
# This is used when the count is expensive and we want to fetch it separately
# from GraphQL.
def pill_count_field; end
def pill_html_options
{}

6
lib/sidebars/groups/menus/issues_menu.rb
View File

@ -50,6 +50,11 @@ module Sidebars
nil
end
override :pill_count_field
def pill_count_field
'openIssuesCount' if Feature.enabled?(:async_sidebar_counts, context.group.root_ancestor)
end
override :pill_html_options
def pill_html_options
{
@ -62,6 +67,7 @@ module Sidebars
super.merge({
active_routes: list_menu_item.active_routes,
pill_count: pill_count,
pill_count_field: pill_count_field,
has_pill: has_pill?,
super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::PlanMenu,
item_id: :group_issue_list

6
lib/sidebars/groups/menus/merge_requests_menu.rb
View File

@ -43,6 +43,11 @@ module Sidebars
end
end
override :pill_count_field
def pill_count_field
'openMergeRequestsCount' if Feature.enabled?(:async_sidebar_counts, context.group.root_ancestor)
end
override :pill_html_options
def pill_html_options
{
@ -59,6 +64,7 @@ module Sidebars
def serialize_as_menu_item_args
super.merge({
pill_count: pill_count,
pill_count_field: pill_count_field,
has_pill: has_pill?,
super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::CodeMenu,
item_id: :group_merge_request_list

3
lib/sidebars/menu.rb
View File

@ -89,9 +89,10 @@ module Sidebars
link: link,
is_active: is_active,
pill_count: has_pill? ? pill_count : nil,
pill_count_field: has_pill? ? pill_count_field : nil,
items: items,
separated: separated?
}
}.compact
end
# Returns an array of renderable menu entries,

8
lib/sidebars/menu_item.rb
View File

@ -4,11 +4,11 @@ module Sidebars
class MenuItem
include ::Sidebars::Concerns::LinkWithHtmlOptions
attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :has_pill, :pill_count, :super_sidebar_parent, :avatar, :entity_id
attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :has_pill, :pill_count, :pill_count_field, :super_sidebar_parent, :avatar, :entity_id
alias_method :has_pill?, :has_pill
# rubocop: disable Metrics/ParameterLists
def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, has_pill: false, pill_count: nil, super_sidebar_parent: nil, avatar: nil, entity_id: nil)
def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, has_pill: false, pill_count: nil, pill_count_field: nil, super_sidebar_parent: nil, avatar: nil, entity_id: nil)
@title = title
@link = link
@active_routes = active_routes
@ -20,6 +20,7 @@ module Sidebars
@entity_id = entity_id
@has_pill = has_pill
@pill_count = pill_count
@pill_count_field = pill_count_field
@super_sidebar_parent = super_sidebar_parent
end
# rubocop: enable Metrics/ParameterLists
@ -38,13 +39,14 @@ module Sidebars
link: link,
active_routes: active_routes,
pill_count: has_pill ? pill_count : nil,
pill_count_field: has_pill ? pill_count_field : nil,
link_classes: container_html_options[:class]
# Check whether support is needed for the following properties,
# in order to get feature parity with the HAML renderer
# https://gitlab.com/gitlab-org/gitlab/-/issues/391864
#
# container_html_options
}
}.compact
end
end
end

6
lib/sidebars/projects/menus/issues_menu.rb
View File

@ -57,6 +57,11 @@ module Sidebars
end
end
override :pill_count_field
def pill_count_field
'openIssuesCount' if Feature.enabled?(:async_sidebar_counts, context.project.root_ancestor)
end
override :pill_html_options
def pill_html_options
{
@ -68,6 +73,7 @@ module Sidebars
def serialize_as_menu_item_args
super.merge({
pill_count: pill_count,
pill_count_field: pill_count_field,
has_pill: has_pill?,
super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu,
item_id: :project_issue_list

6
lib/sidebars/projects/menus/merge_requests_menu.rb
View File

@ -45,6 +45,11 @@ module Sidebars
format_cached_count(1000, count)
end
override :pill_count_field
def pill_count_field
'openMergeRequestsCount' if Feature.enabled?(:async_sidebar_counts, context.project.root_ancestor)
end
override :pill_html_options
def pill_html_options
{
@ -65,6 +70,7 @@ module Sidebars
def serialize_as_menu_item_args
super.merge({
pill_count: pill_count,
pill_count_field: pill_count_field,
has_pill: has_pill?,
super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
item_id: :project_merge_request_list

9
locale/gitlab.pot
View File

@ -10378,9 +10378,6 @@ msgstr ""
msgid "By quarter"
msgstr ""
msgid "By selecting Continue or registering through a third party, I agree that GitLab can contact me by email or telephone about its product, services, or events."
msgstr ""
msgid "By using a primary email tied to an Enterprise email address, you acknowledge that this account is an Enterprise User."
msgstr ""
@ -27729,6 +27726,9 @@ msgstr ""
msgid "HttpDestinationValidator validates only http external audit event destinations."
msgstr ""
msgid "I agree that GitLab can contact me by email or telephone about its product, services, or events."
msgstr ""
msgid "I forgot my password"
msgstr ""
@ -34972,6 +34972,9 @@ msgstr ""
msgid "MlModelRegistry|Delete model"
msgstr ""
msgid "MlModelRegistry|Delete model %{modelName}"
msgstr ""
msgid "MlModelRegistry|Delete version"
msgstr ""

6
qa/qa/fixtures/package_managers/maven/group/producer/settings.xml.erb
View File

@ -9,6 +9,12 @@ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.o
<name><%= maven_header_name %></name>
<value><%= token %></value>
</property>
<!-- Disable the expect header as some proxies don't play nice with it. See https://maven.apache.org/guides/mini/guide-http-settings.html#The_Basics. -->
<!-- See https://gitlab.com/gitlab-org/gitlab/-/issues/499660 -->
<property>
<name>Expect</name>
<value />
</property>
</httpHeaders>
</configuration>
</server>

6
qa/qa/fixtures/package_managers/maven/group/settings_with_pat.xml.erb
View File

@ -9,6 +9,12 @@
<name>Private-Token</name>
<value>${PERSONAL_ACCESS_TOKEN}</value>
</property>
<!-- Disable the expect header as some proxies don't play nice with it. See https://maven.apache.org/guides/mini/guide-http-settings.html#The_Basics. -->
<!-- See https://gitlab.com/gitlab-org/gitlab/-/issues/499660 -->
<property>
<name>Expect</name>
<value />
</property>
</httpHeaders>
</configuration>
</server>

6
qa/qa/fixtures/package_managers/maven/project/settings.xml.erb
View File

@ -9,6 +9,12 @@ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.o
<name><%= maven_header_name %></name>
<value><%= token %></value>
</property>
<!-- Disable the expect header as some proxies don't play nice with it. See https://maven.apache.org/guides/mini/guide-http-settings.html#The_Basics. -->
<!-- See https://gitlab.com/gitlab-org/gitlab/-/issues/499660 -->
<property>
<name>Expect</name>
<value />
</property>
</httpHeaders>
</configuration>
</server>

6
qa/qa/specs/features/browser_ui/5_package/package_registry/maven/maven_group_level_spec.rb
View File

@ -2,11 +2,7 @@
module QA
RSpec.describe 'Package', :object_storage, product_group: :package_registry do
describe 'Maven group level endpoint', :external_api_calls, quarantine: {
type: :investigating,
issue: "https://gitlab.com/gitlab-org/gitlab/-/issues/499660",
only: { job: /gdk-qa-.*/ }
} do
describe 'Maven group level endpoint', :external_api_calls do
include Runtime::Fixtures
include Support::Helpers::MaskToken
include_context 'packages registry qa scenario'

6
qa/qa/specs/features/browser_ui/5_package/package_registry/maven/maven_project_level_spec.rb
View File

@ -2,11 +2,7 @@
module QA
RSpec.describe 'Package', :object_storage, :external_api_calls do
describe 'Maven project level endpoint', product_group: :package_registry, quarantine: {
type: :investigating,
issue: "https://gitlab.com/gitlab-org/gitlab/-/issues/499660",
only: { job: /gdk-qa-.*/ }
} do
describe 'Maven project level endpoint', product_group: :package_registry do
include Runtime::Fixtures
include Support::Helpers::MaskToken

15
spec/frontend/ml/model_registry/components/delete_model_disclosure_dropdown_item_spec.js
View File

@ -12,20 +12,31 @@ describe('DeleteButton', () => {
const findNote = () => wrapper.findByTestId('confirmation-note');
beforeEach(() => {
wrapper = shallowMountExtended(DeleteModelDisclosureDropdownItem, {});
wrapper = shallowMountExtended(DeleteModelDisclosureDropdownItem, {
propsData: {
model: {
id: 1,
name: 'modelName',
},
},
});
});
it('mounts the modal', () => {
expect(findModal().exists()).toBe(true);
});
it('uses unique modal ids', () => {
expect(findModal().props('modalId')).toBe('ml-models-delete-modal-1');
});
it('mounts the button', () => {
expect(findDeleteButton().exists()).toBe(true);
});
describe('when modal is opened', () => {
it('displays modal title', () => {
expect(findModal().props('title')).toBe('Delete model');
expect(findModal().props('title')).toBe('Delete model modelName');
});
it('displays modal body', () => {

32
spec/frontend/ml/model_registry/components/models_table_spec.js
View File

@ -1,6 +1,7 @@
import { mount } from '@vue/test-utils';
import { GlTable, GlLink, GlAvatarLink, GlAvatar } from '@gitlab/ui';
import { GlTable, GlLink, GlAvatarLink, GlAvatar, GlDisclosureDropdown } from '@gitlab/ui';
import ModelsTable from '~/ml/model_registry/components/models_table.vue';
import DeleteModelDisclosureDropdownItem from '~/ml/model_registry/components/delete_model_disclosure_dropdown_item.vue';
import { modelWithOneVersion, modelWithoutVersion } from '../graphql_mock_data';
describe('ModelsTable', () => {
@ -8,10 +9,12 @@ describe('ModelsTable', () => {
const items = [modelWithOneVersion];
const createWrapper = (props = {}) => {
const createWrapper = (props = {}, canWriteModelRegistry = true) => {
wrapper = mount(ModelsTable, {
propsData: {
items,
canWriteModelRegistry,
...props,
},
stubs: {
@ -19,12 +22,17 @@ describe('ModelsTable', () => {
GlLink,
GlAvatarLink,
GlAvatar,
DeleteModelDisclosureDropdownItem,
},
provide: {
projectPath: 'projectPath',
},
});
};
const findGlTable = () => wrapper.findComponent(GlTable);
const findTableRows = () => findGlTable().findAll('tbody tr');
const findActionsDropdown = () => wrapper.findComponent(GlDisclosureDropdown);
beforeEach(() => {
createWrapper();
@ -40,9 +48,19 @@ describe('ModelsTable', () => {
{ key: 'latestVersion', label: 'Latest version', thClass: 'gl-w-1/4' },
{ key: 'author', label: 'Author', thClass: 'gl-w-1/4' },
{ key: 'createdAt', label: 'Created', thClass: 'gl-w-1/4' },
{
key: 'actions',
label: '',
tdClass: 'lg:gl-w-px gl-whitespace-nowrap',
thClass: 'lg:gl-w-px gl-whitespace-nowrap',
},
]);
});
it('renders actions dropdown if canWriteModelRegistry is true', () => {
expect(findActionsDropdown().exists()).toBe(true);
});
it('renders the correct number of rows', () => {
expect(findTableRows().length).toBe(1);
});
@ -115,4 +133,14 @@ describe('ModelsTable', () => {
expect(createdAtCell.text()).toBe('');
});
});
describe('when the user cannot write to the model registry', () => {
beforeEach(() => {
createWrapper({}, false);
});
it('does not render actions if canWriteModelRegistry is false', () => {
expect(findActionsDropdown().exists()).toBe(false);
});
});
});

9
spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb
View File

@ -5,6 +5,7 @@ require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy,
'#next_batch', feature_category: :database do
let(:batching_strategy) { described_class.new(connection: ActiveRecord::Base.connection) }
let(:job_class) { Class.new(Gitlab::BackgroundMigration::BatchedMigrationJob) }
let(:namespaces) { table(:namespaces) }
let!(:namespace1) { namespaces.create!(name: 'batchtest999', path: 'batch-test1') }
@ -16,7 +17,7 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchi
context 'when starting on the first batch' do
it 'returns the bounds of the next batch' do
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace1.id, batch_size: 3, job_arguments: [])
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace1.id, batch_size: 3, job_arguments: [], job_class: job_class)
expect(batch_bounds).to eq([namespace1.id, namespace3.id])
end
@ -24,7 +25,7 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchi
context 'when additional batches remain' do
it 'returns the bounds of the next batch' do
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace2.id, batch_size: 3, job_arguments: [])
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace2.id, batch_size: 3, job_arguments: [], job_class: job_class)
expect(batch_bounds).to eq([namespace2.id, namespace4.id])
end
@ -32,7 +33,7 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchi
context 'when on the final batch' do
it 'returns the bounds of the next batch' do
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace4.id, batch_size: 3, job_arguments: [])
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace4.id, batch_size: 3, job_arguments: [], job_class: job_class)
expect(batch_bounds).to eq([namespace4.id, namespace4.id])
end
@ -40,7 +41,7 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchi
context 'when no additional batches remain' do
it 'returns nil' do
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace4.id + 1, batch_size: 1, job_arguments: [])
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace4.id + 1, batch_size: 1, job_arguments: [], job_class: job_class)
expect(batch_bounds).to be_nil
end

96
spec/lib/gitlab/database/background_migration/cursor_based_background_migration_spec.rb Normal file
View File

@ -0,0 +1,96 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe 'Cursor based batched background migrations', feature_category: :database do
include Gitlab::Database::DynamicModelHelpers
let(:connection) { ApplicationRecord.connection }
let(:table_name) { :_test_cursor_batching }
let(:model) { define_batchable_model(table_name, connection: connection) }
let(:batching_strategy) { Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy }
let(:batching_strategy_name) { batching_strategy.name.demodulize }
let(:background_migration_job_class) do
stub_const('Gitlab::BackgroundMigration::TestCursorMigration',
Class.new(Gitlab::BackgroundMigration::BatchedMigrationJob) do
cursor :id_a, :id_b
def perform
each_sub_batch do |relation|
# Want to relation.update_all(backfilled: )
# But rails doesn't know what to use as the primary key when transforming that to
# UPDATE .. WHERE <pk> IN (subquery) because the primary key is composite
# So it generates invalid sql UPDATE ... WHERE <table_name>."" IN (subquery)
# Instead build our own
connection.execute(<<~SQL)
UPDATE #{batch_table}
SET backfilled = backfilled + 1
WHERE (id_a, id_b) IN (#{relation.select(:id_a, :id_b).to_sql})
SQL
end
end
end
)
end
let(:background_migration_name) { background_migration_job_class.name.demodulize }
before do
connection.execute(<<~SQL)
create table _test_cursor_batching (
id_a bigint not null,
id_b bigint not null,
backfilled int not null default 0,
primary key (id_a, id_b)
);
insert into _test_cursor_batching(id_a, id_b)
select i / 10, i % 10
from generate_series(1, 1000) g(i);
SQL
end
context 'when running a real cursor-based batched background migration' do
let(:strategy_instance) { batching_strategy.new(connection: connection) }
let(:migration) do
create(:batched_background_migration, :active,
job_class_name: background_migration_name,
batch_class_name: batching_strategy_name,
table_name: table_name,
batch_size: 10,
sub_batch_size: 5,
pause_ms: 0,
min_cursor: [0, 0],
max_cursor: [100, 9]
)
end
let(:runner) { Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new(connection: connection) }
it 'migrates correctly' do
runner.run_entire_migration(migration)
expect(model.where(backfilled: 1).count).to eq(model.count)
expect(migration.batched_jobs.count).to eq(100)
end
context 'when the last batch only has one row' do
let(:migration) do
create(:batched_background_migration, :active,
job_class_name: background_migration_name,
batch_class_name: batching_strategy_name,
table_name: table_name,
batch_size: 10,
sub_batch_size: 1,
pause_ms: 0,
min_cursor: [98, 9],
max_cursor: [100, 1]
)
end
it 'migrates correctly' do
runner.run_entire_migration(migration)
expect(model.where(backfilled: 1).count).to eq(11)
expect(migration.batched_jobs.count).to eq(2)
end
end
end
end

12
spec/lib/gitlab/database/migrations/test_background_runner_spec.rb
View File

@ -46,7 +46,7 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do
context 'running migrations', :freeze_time do
it 'runs the migration class correctly' do
calls = []
define_background_migration(migration_name) do |i|
define_background_migration(migration_name, with_base_class: false) do |i|
calls << i
end
described_class.new(result_dir: result_dir).run_jobs(for_duration: 1.second) # Any time would work here as we do not advance time
@ -54,7 +54,7 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do
end
it 'runs the migration for a uniform amount of time' do
migration = define_background_migration(migration_name) do |i|
migration = define_background_migration(migration_name, with_base_class: false) do |i|
travel(1.minute)
end
@ -73,11 +73,11 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do
end
it 'splits the time between migrations when all migrations use all their time' do
migration = define_background_migration(migration_name) do |i|
migration = define_background_migration(migration_name, with_base_class: false) do |i|
travel(1.minute)
end
other_migration = define_background_migration(other_migration_name) do |i|
other_migration = define_background_migration(other_migration_name, with_base_class: false) do |i|
travel(2.minutes)
end
@ -92,10 +92,10 @@ RSpec.describe Gitlab::Database::Migrations::TestBackgroundRunner, :redis do
it 'does not give leftover time to extra migrations' do
# This is currently implemented this way for simplicity, but it could make sense to change this behavior.
migration = define_background_migration(migration_name) do
migration = define_background_migration(migration_name, with_base_class: false) do
travel(1.second)
end
other_migration = define_background_migration(other_migration_name) do
other_migration = define_background_migration(other_migration_name, with_base_class: false) do
travel(1.minute)
end

6
spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb
View File

@ -171,9 +171,9 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez
it 'does not sample a job if there are zero rows to sample' do
calls = []
define_background_migration(migration_name, with_base_class: true, scoping: ->(relation) {
relation.none
}) do |*args|
define_background_migration(migration_name, scoping: ->(relation) {
relation.none
}) do |*args|
calls << args
end

14
spec/lib/gitlab/database_spec.rb
View File

@ -440,6 +440,20 @@ RSpec.describe Gitlab::Database, feature_category: :database do
end
end
describe '.application_record_for_connection' do
it 'returns ApplicationRecord for main database connection' do
connection = ApplicationRecord.retrieve_connection
expect(described_class.application_record_for_connection(connection)).to eq(ApplicationRecord)
end
it 'returns Ci::ApplicationRecord for ci database connection' do
skip_if_multiple_databases_not_setup(:ci)
connection = Ci::ApplicationRecord.retrieve_connection
expect(described_class.application_record_for_connection(connection)).to eq(Ci::ApplicationRecord)
end
end
describe '#true_value' do
it 'returns correct value' do
expect(described_class.true_value).to eq "'t'"

17
spec/lib/sidebars/groups/menus/issues_menu_spec.rb
View File

@ -52,6 +52,22 @@ RSpec.describe Sidebars::Groups::Menus::IssuesMenu, feature_category: :navigatio
let(:count_service) { ::Groups::OpenIssuesCountService }
end
describe '#pill_count_field' do
it 'returns the correct GraphQL field name' do
expect(menu.pill_count_field).to eq('openIssuesCount')
end
context 'when async_sidebar_counts feature flag is disabled' do
before do
stub_feature_flags(async_sidebar_counts: false)
end
it 'returns nil' do
expect(menu.pill_count_field).to be_nil
end
end
end
context 'when count query times out' do
let(:count_service) { ::Groups::OpenIssuesCountService }
@ -78,6 +94,7 @@ RSpec.describe Sidebars::Groups::Menus::IssuesMenu, feature_category: :navigatio
item_id: :group_issue_list,
active_routes: { path: 'groups#issues' },
pill_count: menu.pill_count,
pill_count_field: menu.pill_count_field,
has_pill: menu.has_pill?,
super_sidebar_parent: Sidebars::Groups::SuperSidebarMenus::PlanMenu
}

17
spec/lib/sidebars/groups/menus/merge_requests_menu_spec.rb
View File

@ -39,9 +39,26 @@ RSpec.describe Sidebars::Groups::Menus::MergeRequestsMenu, feature_category: :na
{
item_id: :group_merge_request_list,
pill_count: menu.pill_count,
pill_count_field: menu.pill_count_field,
has_pill: menu.has_pill?,
super_sidebar_parent: Sidebars::Groups::SuperSidebarMenus::CodeMenu
}
end
end
describe '#pill_count_field' do
it 'returns the correct GraphQL field name' do
expect(menu.pill_count_field).to eq('openMergeRequestsCount')
end
context 'when async_sidebar_counts feature flag is disabled' do
before do
stub_feature_flags(async_sidebar_counts: false)
end
it 'returns nil' do
expect(menu.pill_count_field).to be_nil
end
end
end
end

9
spec/lib/sidebars/menu_item_spec.rb
View File

@ -34,5 +34,14 @@ RSpec.describe Sidebars::MenuItem, feature_category: :navigation do
expect(subject[:avatar]).to be('/avatar.png')
expect(subject[:entity_id]).to be(123)
end
context 'with pill data' do
let(:extra) { { has_pill: true, pill_count: '5', pill_count_field: 'countField' } }
it 'includes pill count data' do
expect(subject[:pill_count]).to eq('5')
expect(subject[:pill_count_field]).to eq('countField')
end
end
end
end

34
spec/lib/sidebars/menu_spec.rb
View File

@ -52,37 +52,26 @@ RSpec.describe Sidebars::Menu, feature_category: :navigation do
expect(menu.serialize_for_super_sidebar).to eq(
{
title: "Title",
icon: nil,
id: 'menu',
avatar: nil,
avatar_shape: 'rect',
entity_id: nil,
link: "foo2",
is_active: true,
pill_count: nil,
separated: false,
items: [
{
id: 'id1',
title: "Is active",
icon: nil,
avatar: '/avatar.png',
entity_id: 123,
link: "foo2",
is_active: true,
pill_count: nil,
link_classes: nil
is_active: true
},
{
id: 'id2',
title: "Not active",
icon: nil,
avatar: nil,
entity_id: nil,
link: "foo3",
is_active: false,
pill_count: 10,
link_classes: nil
pill_count: 10
}
]
})
@ -94,18 +83,29 @@ RSpec.describe Sidebars::Menu, feature_category: :navigation do
expect(menu.serialize_for_super_sidebar).to eq(
{
title: "Title",
icon: nil,
id: 'menu',
avatar: nil,
avatar_shape: 'rect',
entity_id: nil,
link: nil,
is_active: false,
pill_count: 'foo',
separated: false,
items: []
})
end
it 'returns pill_count_field if defined' do
allow(menu).to receive(:has_pill?).and_return(true)
allow(menu).to receive(:pill_count_field).and_return('foo')
expect(menu.serialize_for_super_sidebar).to eq(
{
title: "Title",
id: 'menu',
avatar_shape: 'rect',
is_active: false,
pill_count_field: 'foo',
separated: false,
items: []
})
end
end
describe '#serialize_as_menu_item_args' do

17
spec/lib/sidebars/projects/menus/issues_menu_spec.rb
View File

@ -16,6 +16,7 @@ RSpec.describe Sidebars::Projects::Menus::IssuesMenu, feature_category: :navigat
item_id: :project_issue_list,
active_routes: { path: %w[projects/issues#index projects/issues#show projects/issues#new] },
pill_count: menu.pill_count,
pill_count_field: menu.pill_count_field,
has_pill: menu.has_pill?,
super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::PlanMenu
}
@ -100,6 +101,22 @@ RSpec.describe Sidebars::Projects::Menus::IssuesMenu, feature_category: :navigat
end
end
describe '#pill_count_field' do
it 'returns the correct GraphQL field name' do
expect(subject.pill_count_field).to eq('openIssuesCount')
end
context 'when async_sidebar_counts feature flag is disabled' do
before do
stub_feature_flags(async_sidebar_counts: false)
end
it 'returns nil' do
expect(subject.pill_count_field).to be_nil
end
end
end
describe 'Menu Items' do
subject { described_class.new(context).renderable_items.index { |e| e.item_id == item_id } }

17
spec/lib/sidebars/projects/menus/merge_requests_menu_spec.rb
View File

@ -16,6 +16,7 @@ RSpec.describe Sidebars::Projects::Menus::MergeRequestsMenu, feature_category: :
{
item_id: :project_merge_request_list,
pill_count: menu.pill_count,
pill_count_field: menu.pill_count_field,
has_pill: menu.has_pill?,
super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::CodeMenu
}
@ -98,4 +99,20 @@ RSpec.describe Sidebars::Projects::Menus::MergeRequestsMenu, feature_category: :
end
end
end
describe '#pill_count_field' do
it 'returns the correct GraphQL field name' do
expect(subject.pill_count_field).to eq('openMergeRequestsCount')
end
context 'when async_sidebar_counts feature flag is disabled' do
before do
stub_feature_flags(async_sidebar_counts: false)
end
it 'returns nil' do
expect(subject.pill_count_field).to be_nil
end
end
end
end

14
spec/lib/sidebars/static_menu_spec.rb
View File

@ -22,24 +22,14 @@ RSpec.describe Sidebars::StaticMenu, feature_category: :navigation do
{
id: 'id1',
title: "Is active",
icon: nil,
avatar: nil,
entity_id: nil,
link: "foo2",
is_active: true,
pill_count: nil,
link_classes: nil
is_active: true
},
{
id: 'id2',
title: "Not active",
icon: nil,
avatar: nil,
entity_id: nil,
link: "foo3",
is_active: false,
pill_count: nil,
link_classes: nil
is_active: false
}
]
)

26
spec/support/helpers/database/migration_testing_helpers.rb
View File

@ -2,24 +2,36 @@
module Database
module MigrationTestingHelpers
def define_background_migration(name, with_base_class: false, scoping: nil)
def define_background_migration(name, with_base_class: true, scoping: nil)
klass = Class.new(with_base_class ? Gitlab::BackgroundMigration::BatchedMigrationJob : Object) do
operation_name :update if with_base_class
# Can't simply def perform here as we won't have access to the block,
# similarly can't define_method(:perform, &block) here as it would change the block receiver
define_method(:perform) { |*args| yield(*args) }
scope_to(scoping) if scoping
end
stub_const("Gitlab::BackgroundMigration::#{name}", klass)
klass
end
def expect_migration_call_counts(migrations_to_calls)
migrations_to_calls.each do |migration, calls|
expect_next_instances_of(migration, calls) do |m|
expect(m).to receive(:perform).and_call_original
# Returns a hash of migration_class -> number of times perform was called.
# Sets up instrumentation of the provided array of migrations, as instances of their perform methods are called,
# the values in the hash update to count the calls.
def record_migration_call_counts(migrations)
call_counts = migrations.index_with { |_m| 0 }
migrations.each do |migration|
allow_next_instances_of(migration, nil) do |migration_instance|
allow(migration_instance).to receive(:perform).and_wrap_original do |method, *args, **kwargs|
call_counts[migration] += 1
method.call(*args, **kwargs)
end
end
end
call_counts
end
def expect_recorded_migration_runs(migrations_to_runs)
@ -35,10 +47,12 @@ module Database
end
def expect_migration_runs(migrations_to_run_counts)
expect_migration_call_counts(migrations_to_run_counts)
call_counts = record_migration_call_counts(migrations_to_run_counts.keys)
yield
expect(call_counts).to eq(migrations_to_run_counts)
expect_recorded_migration_runs(migrations_to_run_counts)
end
end