root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2025-02-18 18:07:29 +00:00
parent 897e444704
commit cf98c50ab8
51 changed files with 357 additions and 840 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
.gitlab/ci/rails.gitlab-ci.yml
View File

@ -761,49 +761,6 @@ rspec-ee:predictive:trigger single-db-ci-connection:
variables:
PIPELINE_NAME: 'rspec-ee:predictive single-db-ci-connection'
.rspec-ee-base-gitlab-duo:
extends:
- .rspec-ee-base-pg14
when: manual
variables:
REAL_AI_REQUEST: "true"
AI_GATEWAY_URL: http://ai-gateway:5052
rspec-ee unit gitlab-duo-chat-zeroshot pg14:
extends:
- .rspec-ee-base-gitlab-duo
- .rails:rules:ee-gitlab-duo-chat-optional
script:
- !reference [.base-script, script]
- rspec_parallelized_job "--tag zeroshot_executor"
rspec-ee unit gitlab-duo-chat-qa-fast pg14:
extends:
- .rspec-ee-base-gitlab-duo
- .rails:rules:ee-gitlab-duo-chat-always
script:
- !reference [.base-script, script]
- rspec_parallelized_job "--tag fast_chat_qa_evaluation"
rspec-ee unit gitlab-duo-chat-qa pg14:
variables:
QA_EVAL_REPORT_FILENAME: "qa_evaluation_report.md"
RSPEC_RETRY_RETRY_COUNT: 0
extends:
- .rspec-ee-base-gitlab-duo
- .rails:rules:ee-gitlab-duo-chat-qa-full
script:
- !reference [.base-script, script]
- source ./scripts/utils.sh
- install_gitlab_gem
- bundle exec rspec -Ispec -rspec_helper --failure-exit-code 0 --color --tag chat_qa_evaluation -- ee/spec/lib/gitlab/llm/chain/agents/zero_shot/qa_evaluation_spec.rb
- ./scripts/duo_chat/reporter.rb
artifacts:
expire_in: 5d
paths:
- tmp/duo_chat/qa*.json
- "${QA_EVAL_REPORT_FILENAME}"
rspec-ee migration pg14:
extends:
- .rspec-ee-base-pg14

1
.rubocop_todo/layout/class_structure.yml
View File

@ -243,7 +243,6 @@ Layout/ClassStructure:
- 'ee/lib/gitlab/geo/git_ssh_proxy.rb'
- 'ee/lib/gitlab/license_scanning/base_scanner.rb'
- 'ee/lib/gitlab/license_scanning/package_licenses.rb'
- 'ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb'
- 'ee/lib/gitlab/llm/chain/tools/tool.rb'
- 'ee/lib/gitlab/llm/chat_storage.rb'
- 'ee/lib/gitlab/llm/vertex_ai/configuration.rb'

1
.rubocop_todo/rspec/be_eq.yml
View File

@ -156,7 +156,6 @@ RSpec/BeEq:
- 'ee/spec/lib/gitlab/insights/project_insights_config_spec.rb'
- 'ee/spec/lib/gitlab/licenses/submit_license_usage_data_banner_spec.rb'
- 'ee/spec/lib/gitlab/llm/ai_gateway/docs_client_spec.rb'
- 'ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb'
- 'ee/spec/lib/gitlab/llm/chain/answer_spec.rb'
- 'ee/spec/lib/gitlab/llm/chain/requests/ai_gateway_spec.rb'
- 'ee/spec/lib/gitlab/llm/chain/tools/gitlab_documentation/executor_spec.rb'

1
.rubocop_todo/rspec/named_subject.yml
View File

@ -395,7 +395,6 @@ RSpec/NamedSubject:
- 'ee/spec/lib/gitlab/llm/ai_message_spec.rb'
- 'ee/spec/lib/gitlab/llm/anthropic/response_modifiers/tanuki_bot_spec.rb'
- 'ee/spec/lib/gitlab/llm/base_response_modifier_spec.rb'
- 'ee/spec/lib/gitlab/llm/chain/agents/zero_shot/prompts/anthropic_spec.rb'
- 'ee/spec/lib/gitlab/llm/chain/parsers/output_parser_spec.rb'
- 'ee/spec/lib/gitlab/llm/chain/response_modifier_spec.rb'
- 'ee/spec/lib/gitlab/llm/chain/tools/tool_spec.rb'

5
app/assets/javascripts/add_context_commits_modal/components/add_context_commits_modal_wrapper.vue
View File

@ -5,6 +5,7 @@ import { mapState, mapActions } from 'vuex';
import ReviewTabContainer from '~/add_context_commits_modal/components/review_tab_container.vue';
import { createAlert } from '~/alert';
import { BV_SHOW_MODAL } from '~/lib/utils/constants';
import { markRaw } from '~/lib/utils/vue3compat/mark_raw';
import { __, s__ } from '~/locale';
import {
OPERATORS_IS,
@ -51,7 +52,7 @@ export default {
},
data() {
return {
availableTokens: [
availableTokens: markRaw([
{
icon: 'pencil',
title: __('Author'),
@ -91,7 +92,7 @@ export default {
unique: true,
optionComponent: DateOption,
},
],
]),
};
},
computed: {

2
app/assets/javascripts/vue_shared/components/markdown/header.vue
View File

@ -164,7 +164,7 @@ export default {
this.restrictedToolBarItems.includes('attach-file') &&
!this.drawioEnabled &&
!this.supportsQuickActions &&
!this.newCommentTemplatePath)
!this.commentTemplatePaths.length)
);
},
showFindAndReplaceButton() {

9
app/models/container_registry/event.rb
View File

@ -10,6 +10,7 @@ module ContainerRegistry
DELETE_ACTION = 'delete'
EVENT_TRACKING_CATEGORY = 'container_registry:notification'
EVENT_PREFIX = 'i_container_registry'
INTERNAL_EVENTS_ORIGINATORS = %w[deploy_token].freeze
ALLOWED_ACTOR_TYPES = %w[
personal_access_token
@ -53,7 +54,13 @@ module ContainerRegistry
track_internal_event("delete_manifest_from_container_registry", project: project)
else
event = usage_data_event_for(tracking_action)
::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: originator.id) if event
return unless event
if originator_suffix.in? INTERNAL_EVENTS_ORIGINATORS
track_internal_event(event, additional_properties: { property: originator.id.to_s })
else
::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: originator.id)
end
end
end

15
config/events/i_container_registry_create_repository_deploy_token.yml Normal file
View File

@ -0,0 +1,15 @@
---
description: Tracks when a repository in container registry is created using a deploy token
internal_events: true
action: i_container_registry_create_repository_deploy_token
identifiers:
product_group: container_registry
milestone: '17.10'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/179408
tiers:
- free
- premium
- ultimate
additional_properties:
property:
description: Id of the deploy token used for authorization

15
config/events/i_container_registry_delete_repository_deploy_token.yml Normal file
View File

@ -0,0 +1,15 @@
---
description: Tracks when a repository in container registry is deleted using a deploy token
internal_events: true
action: i_container_registry_delete_repository_deploy_token
identifiers:
product_group: container_registry
milestone: '17.10'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/179408
tiers:
- free
- premium
- ultimate
additional_properties:
property:
description: Id of the deploy token used for authorization

15
config/events/i_container_registry_delete_tag_deploy_token.yml Normal file
View File

@ -0,0 +1,15 @@
---
description: Tracks when a tag in container registry is deleted using a deploy token
internal_events: true
action: i_container_registry_delete_tag_deploy_token
identifiers:
product_group: container_registry
milestone: '17.10'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/179408
tiers:
- free
- premium
- ultimate
additional_properties:
property:
description: Id of the deploy token used for authorization

15
config/events/i_container_registry_push_repository_deploy_token.yml Normal file
View File

@ -0,0 +1,15 @@
---
description: Tracks when a repository in container registry is pushed using a deploy token
internal_events: true
action: i_container_registry_push_repository_deploy_token
identifiers:
product_group: container_registry
milestone: '17.10'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/179408
tiers:
- free
- premium
- ultimate
additional_properties:
property:
description: Id of the deploy token used for authorization

15
config/events/i_container_registry_push_tag_deploy_token.yml Normal file
View File

@ -0,0 +1,15 @@
---
description: Tracks when a tag in container registry is pushed using a deploy token
internal_events: true
action: i_container_registry_push_tag_deploy_token
identifiers:
product_group: container_registry
milestone: '17.10'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/179408
tiers:
- free
- premium
- ultimate
additional_properties:
property:
description: Id of the deploy token used for authorization

9
config/metrics/counts_all/20231223170002_i_container_registry_push_repository_deploy_token.yml
View File

@ -9,12 +9,11 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131966
time_frame:
- 28d
- 7d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- i_container_registry_push_repository_deploy_token
events:
- name: i_container_registry_push_repository_deploy_token
unique: property
performance_indicator_type: []
tiers:
- free

9
config/metrics/counts_all/20231227165432_i_container_registry_push_tag_deploy_token.yml
View File

@ -9,12 +9,11 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131966
time_frame:
- 28d
- 7d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- i_container_registry_push_tag_deploy_token
events:
- name: i_container_registry_push_tag_deploy_token
unique: property
performance_indicator_type: []
tiers:
- free

9
config/metrics/counts_all/20231227165706_i_container_registry_delete_tag_deploy_token.yml
View File

@ -9,12 +9,11 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131966
time_frame:
- 28d
- 7d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- i_container_registry_delete_tag_deploy_token
events:
- name: i_container_registry_delete_tag_deploy_token
unique: property
performance_indicator_type: []
tiers:
- free

9
config/metrics/counts_all/20231227170213_i_container_registry_delete_repository_deploy_token.yml
View File

@ -9,12 +9,11 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131966
time_frame:
- 28d
- 7d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- i_container_registry_delete_repository_deploy_token
events:
- name: i_container_registry_delete_repository_deploy_token
unique: property
performance_indicator_type: []
tiers:
- free

9
config/metrics/counts_all/20231227170547_i_container_registry_create_repository_deploy_token.yml
View File

@ -9,12 +9,11 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131966
time_frame:
- 28d
- 7d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- i_container_registry_create_repository_deploy_token
events:
- name: i_container_registry_create_repository_deploy_token
unique: property
performance_indicator_type: []
tiers:
- free

13
db/docs/ai_active_context_connections.yml Normal file
View File

@ -0,0 +1,13 @@
---
table_name: ai_active_context_connections
classes:
- Ai::ActiveContext::Connection
feature_categories:
- global_search
description: Stores connection configurations for AI active context integrations.
Each connection represents a unique integration point with specific adapter settings and options.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181400
milestone: '17.10'
gitlab_schema: gitlab_main_cell
exempt_from_sharding: true
table_size: small

23
db/migrate/20250213093911_create_ai_active_context_connections.rb Normal file
View File

@ -0,0 +1,23 @@
# frozen_string_literal: true
class CreateAiActiveContextConnections < Gitlab::Database::Migration[2.2]
milestone '17.10'
def change
create_table :ai_active_context_connections do |t|
# Timestamps (8 bytes each)
t.timestamps_with_timezone
# Boolean (1 byte)
t.boolean :active, null: false, default: false
# Variable size columns (at the end)
t.text :name, null: false, limit: 255, index: { unique: true }
t.text :prefix, limit: 255
t.text :adapter_class, null: false, limit: 255
t.jsonb :options, null: false
t.index :active, unique: true, where: 'active = true', name: 'index_active_context_connections_single_active'
end
end
end

1
db/schema_migrations/20250213093911 Normal file
View File

@ -0,0 +1 @@
542ec7d6b7a648f9aa014876f410b37cf08c58908a4de99b96967e2d1ab3711a

32
db/structure.sql
View File

@ -7133,6 +7133,29 @@ CREATE SEQUENCE ai_active_context_collections_id_seq
ALTER SEQUENCE ai_active_context_collections_id_seq OWNED BY ai_active_context_collections.id;
CREATE TABLE ai_active_context_connections (
id bigint NOT NULL,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
active boolean DEFAULT false NOT NULL,
name text NOT NULL,
prefix text,
adapter_class text NOT NULL,
options jsonb NOT NULL,
CONSTRAINT check_7bb86d51bc CHECK ((char_length(name) <= 255)),
CONSTRAINT check_b1c9c4af95 CHECK ((char_length(adapter_class) <= 255)),
CONSTRAINT check_ffbd5301a3 CHECK ((char_length(prefix) <= 255))
);
CREATE SEQUENCE ai_active_context_connections_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE ai_active_context_connections_id_seq OWNED BY ai_active_context_connections.id;
CREATE TABLE ai_agent_version_attachments (
id bigint NOT NULL,
created_at timestamp with time zone NOT NULL,
@ -24927,6 +24950,8 @@ ALTER TABLE ONLY agent_user_access_project_authorizations ALTER COLUMN id SET DE
ALTER TABLE ONLY ai_active_context_collections ALTER COLUMN id SET DEFAULT nextval('ai_active_context_collections_id_seq'::regclass);
ALTER TABLE ONLY ai_active_context_connections ALTER COLUMN id SET DEFAULT nextval('ai_active_context_connections_id_seq'::regclass);
ALTER TABLE ONLY ai_agent_version_attachments ALTER COLUMN id SET DEFAULT nextval('ai_agent_version_attachments_id_seq'::regclass);
ALTER TABLE ONLY ai_agent_versions ALTER COLUMN id SET DEFAULT nextval('ai_agent_versions_id_seq'::regclass);
@ -26859,6 +26884,9 @@ ALTER TABLE ONLY agent_user_access_project_authorizations
ALTER TABLE ONLY ai_active_context_collections
ADD CONSTRAINT ai_active_context_collections_pkey PRIMARY KEY (id);
ALTER TABLE ONLY ai_active_context_connections
ADD CONSTRAINT ai_active_context_connections_pkey PRIMARY KEY (id);
ALTER TABLE ONLY ai_agent_version_attachments
ADD CONSTRAINT ai_agent_version_attachments_pkey PRIMARY KEY (id);
@ -31363,6 +31391,8 @@ CREATE INDEX index_ac3541b851 ON ci_pipelines USING btree (trigger_id);
CREATE UNIQUE INDEX "index_achievements_on_namespace_id_LOWER_name" ON achievements USING btree (namespace_id, lower(name));
CREATE UNIQUE INDEX index_active_context_connections_single_active ON ai_active_context_connections USING btree (active) WHERE (active = true);
CREATE UNIQUE INDEX index_activity_pub_releases_sub_on_project_id_inbox_url ON activity_pub_releases_subscriptions USING btree (project_id, lower(subscriber_inbox_url));
CREATE UNIQUE INDEX index_activity_pub_releases_sub_on_project_id_sub_url ON activity_pub_releases_subscriptions USING btree (project_id, lower(subscriber_url));
@ -31401,6 +31431,8 @@ CREATE INDEX index_agent_user_access_on_group_id ON agent_user_access_group_auth
CREATE INDEX index_agent_user_access_on_project_id ON agent_user_access_project_authorizations USING btree (project_id);
CREATE UNIQUE INDEX index_ai_active_context_connections_on_name ON ai_active_context_connections USING btree (name);
CREATE INDEX index_ai_agent_version_attachments_on_ai_agent_version_id ON ai_agent_version_attachments USING btree (ai_agent_version_id);
CREATE INDEX index_ai_agent_version_attachments_on_ai_vectorizable_file_id ON ai_agent_version_attachments USING btree (ai_vectorizable_file_id);

4
doc/.vale/gitlab_docs/RelativeLinks.yml
View File

@ -6,9 +6,9 @@
# For a list of all options, see https://vale.sh/docs/topics/styles/
extends: existence
message: "Use a relative link instead of a URL, and ensure the file name ends in .md and not .html."
link: https://docs.gitlab.com/ee/development/documentation/styleguide/#links
link: https://docs.gitlab.com/development/documentation/styleguide/#links
vocab: false
level: error
scope: raw
raw:
- '\[[^\]]+\]\(https?:\/\/docs\.gitlab\.com\/[ce]e.*?\)'
- '\[[^\]]+\]\(https?:\/\/docs\.gitlab\.com\/(?!runner|archives|omnibus|charts|operator|\))[^\)]*\)'

4
doc/administration/reference_architectures/10k_users.md
View File

@ -531,7 +531,7 @@ You can optionally use a [third party external service for PostgreSQL](../postgr
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.
@ -1329,7 +1329,7 @@ minimal.
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.

4
doc/administration/reference_architectures/25k_users.md
View File

@ -535,7 +535,7 @@ You can optionally use a [third party external service for PostgreSQL](../postgr
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.
@ -1337,7 +1337,7 @@ minimal.
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.

2
doc/administration/reference_architectures/2k_users.md
View File

@ -277,7 +277,7 @@ You can optionally use a [third party external service for PostgreSQL](../postgr
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.

4
doc/administration/reference_architectures/3k_users.md
View File

@ -518,7 +518,7 @@ You can optionally use a [third party external service for PostgreSQL](../postgr
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.
@ -1164,7 +1164,7 @@ minimal.
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.

4
doc/administration/reference_architectures/50k_users.md
View File

@ -539,7 +539,7 @@ You can optionally use a [third party external service for PostgreSQL](../postgr
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.
@ -1344,7 +1344,7 @@ minimal.
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
Examples of the above could include [Google's Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) or [Amazon RDS](https://aws.amazon.com/rds/).

4
doc/administration/reference_architectures/5k_users.md
View File

@ -521,7 +521,7 @@ You can optionally use a [third party external service for PostgreSQL](../postgr
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.
@ -1168,7 +1168,7 @@ minimal.
A reputable provider or solution should be used for this. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal)
and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. However, Amazon Aurora is **incompatible** with load balancing enabled by default from
[14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
[14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
See [Recommended cloud providers and services](_index.md#recommended-cloud-providers-and-services) for more information.

2
doc/administration/reference_architectures/_index.md
View File

@ -540,7 +540,7 @@ If you choose to use a third-party external service:
The following database cloud provider services are not recommended due to lack of support or known issues:
- [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is incompatible and not supported. For more details, see [14.4.0](https://docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes.html#1440).
- [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is incompatible and not supported. For more details, see [14.4.0](https://archives.docs.gitlab.com/17.3/ee/update/versions/gitlab_14_changes/#1440).
- [Azure Database for PostgreSQL Single Server](https://azure.microsoft.com/en-gb/products/postgresql/#overview) is not supported as the service is now deprecated and runs on an unsupported version of PostgreSQL. It also has notable performance and stability issues.
- [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) are not tested and are not recommended. Both solutions are not expected to work with GitLab Geo.
- [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported.

14
doc/development/ai_features/duo_chat.md
View File

@ -90,15 +90,11 @@ that Chat sends to assist troubleshooting.
From the code perspective, Chat is implemented in the similar fashion as other
AI features. Read more about GitLab [AI Abstraction layer](_index.md#feature-development-abstraction-layer).
The Chat feature uses a [zero-shot agent](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb)
that includes a system prompt explaining how the large language model should
interpret the question and provide an answer. The system prompt defines
available tools that can be used to gather information to answer the user's
question.
The Chat feature uses a [zero-shot agent](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/duo/chat/react_executor.rb)
that sends user question and relevant context to the [AI Gateway](https://gitlab.com/gitlab-org/modelops/applied-ml/code-suggestions/ai-assist)
which construct a prompt and sends the request to the large language model.
The zero-shot agent receives the user's question and decides which tools to use
to gather information to answer it. It then makes a request to the large
language model, which decides if it can answer directly or if it needs to use
Large language model decides if it can answer directly or if it needs to use
one of the defined tools.
The tools each have their own prompt that provides instructions to the large
@ -576,7 +572,7 @@ flow of how we construct a Chat prompt:
1. `Gitlab::Llm::Chain::Agents::SingleActionExecutor#execute` calls
`execute_streamed_request`, which calls `request`, a method defined in the
`AiDependent` concern
([code](https://gitlab.com/gitlab-org/gitlab/-/blob/d539f64ce6c5bed72ab65294da3bcebdc43f68c6/ee/lib/gitlab/llm/chain/agents/zero_shot/executor.rb#L85))
([code](https://gitlab.com/gitlab-org/gitlab/-/blob/7ac19f75bd0ba4db5cfe7030e56c3672e2ccdc88/ee/lib/gitlab/llm/chain/concerns/ai_dependent.rb#L14))
1. The `SingleActionExecutor#prompt_options` method assembles all prompt parameters for the AI gateway request
([code](https://gitlab.com/gitlab-org/gitlab/-/blob/971d07aa37d9f300b108ed66304505f2d7022841/ee/lib/gitlab/llm/chain/agents/single_action_executor.rb#L120-120))
1. `ai_request` is defined in `Llm::Completions::Chat` and evaluates to

2
doc/development/ai_features/logged_events.md
View File

@ -295,7 +295,7 @@ In addition to standard logging in the GitLab Rails Monolith instance, specializ
### Picked tool
- Description: information about tool picked by chat
- Class: `Gitlab::Llm::Chain::Agents::ZeroShot::Executor`
- Class: `Gitlab::Llm::Chain::Tools::Tool`
- Ai_event_name: picked_tool
- Level: info
- Arguments:

64
doc/development/architecture.md
View File

@ -332,38 +332,38 @@ graph LR
%% end subgraph External
end
click Alertmanager "./architecture.html#alertmanager"
click Praefect "./architecture.html#praefect"
click Geo "./architecture.html#gitlab-geo"
click NGINX "./architecture.html#nginx"
click Runner "./architecture.html#gitlab-runner"
click Registry "./architecture.html#registry"
click ObjectStorage "./architecture.html#minio"
click Mattermost "./architecture.html#mattermost"
click Gitaly "./architecture.html#gitaly"
click Jaeger "./architecture.html#jaeger"
click GitLabWorkhorse "./architecture.html#gitlab-workhorse"
click LDAP "./architecture.html#ldap-authentication"
click Puma "./architecture.html#puma"
click GitLabShell "./architecture.html#gitlab-shell"
click SSH "./architecture.html#ssh-request-22"
click Sidekiq "./architecture.html#sidekiq"
click Sentry "./architecture.html#sentry"
click GitLabExporter "./architecture.html#gitlab-exporter"
click Elasticsearch "./architecture.html#elasticsearch"
click Migrations "./architecture.html#database-migrations"
click PostgreSQL "./architecture.html#postgresql"
click Consul "./architecture.html#consul"
click PgBouncer "./architecture.html#pgbouncer"
click PgBouncerExporter "./architecture.html#pgbouncer-exporter"
click RedisExporter "./architecture.html#redis-exporter"
click Redis "./architecture.html#redis"
click Prometheus "./architecture.html#prometheus"
click Grafana "./architecture.html#grafana"
click GitLabPages "./architecture.html#gitlab-pages"
click PostgreSQLExporter "./architecture.html#postgresql-exporter"
click SMTP "./architecture.html#outbound-email"
click NodeExporter "./architecture.html#node-exporter"
click Alertmanager "#alertmanager"
click Praefect "#praefect"
click Geo "#gitlab-geo"
click NGINX "#nginx"
click Runner "#gitlab-runner"
click Registry "#registry"
click ObjectStorage "#minio"
click Mattermost "#mattermost"
click Gitaly "#gitaly"
click Jaeger "#jaeger"
click GitLabWorkhorse "#gitlab-workhorse"
click LDAP "#ldap-authentication"
click Puma "#puma"
click GitLabShell "#gitlab-shell"
click SSH "#ssh-request-22"
click Sidekiq "#sidekiq"
click Sentry "#sentry"
click GitLabExporter "#gitlab-exporter"
click Elasticsearch "#elasticsearch"
click Migrations "#database-migrations"
click PostgreSQL "#postgresql"
click Consul "#consul"
click PgBouncer "#pgbouncer"
click PgBouncerExporter "#pgbouncer-exporter"
click RedisExporter "#redis-exporter"
click Redis "#redis"
click Prometheus "#prometheus"
click Grafana "#grafana"
click GitLabPages "#gitlab-pages"
click PostgreSQLExporter "#postgresql-exporter"
click SMTP "#outbound-email"
click NodeExporter "#node-exporter"
```
### Component legend

4
doc/integration/omniauth.md
View File

@ -512,10 +512,6 @@ When authenticating using LDAP, the user's name and email are always synced.
With certain OmniAuth providers, users can sign in without using two-factor authentication (2FA).
Because of a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/196131) users must
[set up 2FA](../user/profile/account/two_factor_authentication.md#enable-two-factor-authentication) on their GitLab
account to bypass 2FA. Otherwise, they are prompted to set up 2FA when they sign in to GitLab.
To bypass 2FA, you can either:
- Define the allowed providers using an array (for example, `['saml', 'google_oauth2']`).

10
doc/subscriptions/self_managed/_index.md
View File

@ -431,6 +431,8 @@ you want to either:
- [Renew for fewer seats](#renew-for-fewer-seats).
- Increase or decrease the quantities of products being renewed.
- Remove add-on products no longer needed for the renewed subscription term.
- Upgrade the subscription tier.
Before your subscription renewal date, you should review your account.
@ -467,8 +469,7 @@ If you want to renew with fewer seats, you can do either of the following:
- [Manually renew your subscription](#renew-subscription-manually).
- [Cancel your subscription](#enable-or-disable-automatic-subscription-renewal)
and contact the [Sales team](https://customers.gitlab.com/contact_us) to specify how many seats you need in your subscription
going forward.
and contact the [Sales team](https://customers.gitlab.com/contact_us) to specify how many seats you need in your subscription going forward.
### Renew subscription manually
@ -505,7 +506,8 @@ To manually renew your subscription:
In the **Users over license** text box, enter the number of
[users over subscription](#users-over-subscription) for the user overage incurred.
1. Optional. If renewing additional products, review and update the desired quantity.
1. Optional. If renewing add-on products, review and update the desired quantity. You can also remove products.
1. Optional. If upgrading the subscription tier, select the desired option.
1. Review your renewal details and select **Renew subscription** to complete the
payment process.
1. On the [Subscriptions & purchases](https://customers.gitlab.com/subscriptions)
@ -513,7 +515,7 @@ To manually renew your subscription:
a copy of the renewal term activation code.
1. [Add the activation code](../../administration/license.md) to your instance.
To add or remove products from your subscription, or to upgrade your subscription tier, please [contact the sales team](https://customers.gitlab.com/contact_us).
To add products to your subscription, please [contact the sales team](https://customers.gitlab.com/contact_us).
### Automatic subscription renewal

2
doc/subscriptions/subscription-add-ons.md
View File

@ -45,7 +45,7 @@ Prerequisites:
1. On the left sidebar, select **Search or go to** and find your group.
1. Select **Settings > GitLab Duo**.
1. By **Seat utilization**, select **Assign seats**.
1. Select **Add seats**.
1. Select **Purchase seats**.
1. In the Customers Portal, in the **Add additional seats** field, enter the number of seats. The amount
cannot be higher than the number of seats in the subscription associated with your group namespace.
1. In the **Billing information** section, select the payment method from the dropdown list.

29
doc/user/clusters/agent/managed_kubernetes_resources.md
View File

@ -166,6 +166,35 @@ delete_resources: never
delete_resources: on_stop
```
### Managed resource labels and annotations
The resources created by GitLab use a series of labels and annotations for tracking and troubleshooting purposes.
The following labels are defined on every resource created by GitLab. The values are intentionally left empty:
- `agent.gitlab.com/id-<agent_id>: ""`
- `agent.gitlab.com/project_id-<project_id>: ""`
- `agent.gitlab.com/env-<kubernetes_namespace>: ""`
- `agent.gitlab.com/environment_slug-<gitlab_environment_slug>: ""`
On every resource created by GitLab, an `agent.gitlab.com/env-<kubernetes_namespace>` annotation is defined. The value of the annotation is a JSON object with the following keys:
| Key | Description |
|-----|-------------|
| `environment_id` | The GitLab environment ID. |
| `environment_name` | The GitLab environment name. |
| `environment_slug` | The GitLab environment slug. |
| `environment_page_url` | The link to the GitLab environment page. |
| `environment_tier` | The GitLab environment deployment tier. |
| `agent_id` | The agent ID. |
| `agent_name` | The agent name. |
| `agent_url` | The agent URL in the agent registration project. |
| `project_id` | The GitLab project ID. |
| `project_slug` | The GitLab project slug. |
| `project_path` | The full GitLab project path. |
| `project_url` | The link to the GitLab project. |
| `template_name` | The name of the template used. |
### Use managed resources in CI/CD pipelines
To use managed Kubernetes resources in your CI/CD pipelines, specify the agent and optionally the template name in your environment configuration:

8
doc/user/project/settings/migrate_projects.md
View File

@ -34,10 +34,10 @@ A project transfer includes:
- Membership invitations
{{< alert type="note" >}}
Members who inherited their access from the original group lose access
unless they are also members of the target group. The project inherits
new member permissions from the group you transfer it to.
Members with [inherited membership](../members/_index.md#membership-types)
in the project lose access unless they are also members of the target group.
The project inherits new member permissions from the group you transfer it to.
{{< /alert >}}

2
doc/user/workspace/settings.md
View File

@ -59,7 +59,7 @@ Updating any of these settings, except `enabled`, does not affect existing works
| [`max_active_hours_before_stop`](#max_active_hours_before_stop) | Maximum number of hours a workspace can be active before it is stopped. | Integer | `36` | No |
| [`max_stopped_hours_before_termination`](#max_stopped_hours_before_termination) | Maximum number of hours a workspace can be stopped before it is terminated. | Integer | `744` | No |
## `enabled`
### `enabled`
Use this setting to define whether:

5
lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml
View File

@ -176,3 +176,8 @@ visit_ci_cd_time_to_restore_service_tab-user: p_analytics_ci_cd_time_to_restore_
visit_compliance_credential_inventory-user: i_compliance_credential_inventory
commit_change_to_ciconfigfile-user: o_pipeline_authoring_unique_users_committing_ciconfigfile
performed_wiki_action-user: wiki_action
i_container_registry_create_repository_deploy_token-property: i_container_registry_create_repository_deploy_token
i_container_registry_delete_repository_deploy_token-property: i_container_registry_delete_repository_deploy_token
i_container_registry_delete_tag_deploy_token-property: i_container_registry_delete_tag_deploy_token
i_container_registry_push_repository_deploy_token-property: i_container_registry_push_repository_deploy_token
i_container_registry_push_tag_deploy_token-property: i_container_registry_push_tag_deploy_token

5
lib/gitlab/usage_data_counters/hll_redis_legacy_events.yml
View File

@ -169,15 +169,10 @@
- i_code_review_widget_nothing_merge_click_new_file
- i_compliance_audit_events
- i_compliance_credential_inventory
- i_container_registry_create_repository_deploy_token
- i_container_registry_create_repository_user
- i_container_registry_delete_repository_deploy_token
- i_container_registry_delete_repository_user
- i_container_registry_delete_tag_deploy_token
- i_container_registry_delete_tag_user
- i_container_registry_push_repository_deploy_token
- i_container_registry_push_repository_user
- i_container_registry_push_tag_deploy_token
- i_container_registry_push_tag_user
- i_ecosystem_jira_service_close_issue
- i_ecosystem_jira_service_create_issue

5
lib/sidebars/menu_item.rb
View File

@ -5,6 +5,7 @@ module Sidebars
include ::Sidebars::Concerns::LinkWithHtmlOptions
attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :has_pill, :pill_count, :pill_count_field, :super_sidebar_parent, :avatar, :entity_id
attr_accessor :render
alias_method :has_pill?, :has_pill
# rubocop: disable Metrics/ParameterLists
@ -26,7 +27,9 @@ module Sidebars
# rubocop: enable Metrics/ParameterLists
def render?
true
return true if @render.nil?
@render
end
def serialize_for_super_sidebar

27
locale/gitlab.pot
View File

@ -5451,9 +5451,6 @@ msgstr ""
msgid "AiPowered|Participate in the Early Access Program and help make GitLab better"
msgstr ""
msgid "AiPowered|Purchase seats"
msgstr ""
msgid "AiPowered|Seat assignment for GitLab Duo has moved"
msgstr ""
@ -51791,6 +51788,12 @@ msgstr[1] ""
msgid "SecurityOrchestration|%{state} and %{statuses}"
msgstr ""
msgid "SecurityOrchestration|%{strongStart}Run a DAST scan%{strongEnd} with Scan Profile A and Site Profile A %{strongStart}when a pipeline runs against the main branch%{strongEnd}."
msgstr ""
msgid "SecurityOrchestration|%{strongStart}Run customized GitLab security templates%{strongEnd} across my projects."
msgstr ""
msgid "SecurityOrchestration|(%{groups})"
msgstr ""
@ -51914,7 +51917,7 @@ msgstr ""
msgid "SecurityOrchestration|At least one framework label should be selected"
msgstr ""
msgid "SecurityOrchestration|Automate vulnerability management workflows."
msgid "SecurityOrchestration|Automate %{strongStart}vulnerability management%{strongEnd} workflows."
msgstr ""
msgid "SecurityOrchestration|Automatically selected runners"
@ -52154,10 +52157,10 @@ msgstr ""
msgid "SecurityOrchestration|If any of the following occur:"
msgstr ""
msgid "SecurityOrchestration|If any scanner finds a newly detected critical vulnerability in an open merge request targeting the main branch, then require two approvals from any two members of the application security team."
msgid "SecurityOrchestration|If any scanner finds a %{strongStart}newly detected critical vulnerability%{strongEnd} in an open %{strongStart}merge request%{strongEnd} targeting the main branch, then %{strongStart}require two approvals from any two members%{strongEnd} of the application security team."
msgstr ""
msgid "SecurityOrchestration|If any scanner finds a vulnerability that was previously detected but no longer found in a subsequent scan, then automatically set the status to Resolved."
msgid "SecurityOrchestration|If any scanner finds a %{strongStart}vulnerability%{strongEnd} that was %{strongStart}previously detected but no longer found%{strongEnd} in a subsequent scan, then automatically %{strongStart}set the status to Resolved%{strongEnd}."
msgstr ""
msgid "SecurityOrchestration|In each pipeline, a %{boldStart}maximum of 1000%{boldEnd} vulnerabilities that are %{italicStart}no longer detected%{italicEnd} will be set to status %{italicStart}Resolved%{italicEnd} until all have been auto-resolved."
@ -52402,12 +52405,6 @@ msgstr ""
msgid "SecurityOrchestration|Run a %{scannerStart}%{scanner}%{scannerEnd} scan with the following options:"
msgstr ""
msgid "SecurityOrchestration|Run a DAST scan with Scan Profile A and Site Profile A when a pipeline run against the main branch."
msgstr ""
msgid "SecurityOrchestration|Run customized Gitlab security templates across my projects."
msgstr ""
msgid "SecurityOrchestration|Save changes"
msgstr ""
@ -52624,13 +52621,13 @@ msgstr ""
msgid "SecurityOrchestration|Update via merge request"
msgstr ""
msgid "SecurityOrchestration|Use a merge request approval policy to create rules that check for security vulnerabilities and license compliance before merging a merge request."
msgid "SecurityOrchestration|Use a merge request approval policy to create rules that %{strongStart}check%{strongEnd} for %{strongStart}security vulnerabilities%{strongEnd} and %{strongStart}license compliance%{strongEnd} before %{strongStart}merging a merge request%{strongEnd}."
msgstr ""
msgid "SecurityOrchestration|Use a pipeline execution policy to enforce a custom CI/CD configuration to run in project pipelines."
msgid "SecurityOrchestration|Use a pipeline execution policy to %{strongStart}enforce a custom CI/CD configuration%{strongEnd} to run in project pipelines."
msgstr ""
msgid "SecurityOrchestration|Use a scan execution policy to create rules which enforce security scans for particular branches at a certain time. Supported types are SAST, SAST IaC, DAST, Secret detection, Container scanning, and Dependency scanning."
msgid "SecurityOrchestration|Use a scan execution policy to create rules which %{strongStart}enforce security scans%{strongEnd} for %{strongStart}particular branches%{strongEnd} at %{strongStart}certain times%{strongEnd}. Supported types are SAST, SAST IaC, DAST, Secret detection, Container scanning, and Dependency scanning."
msgstr ""
msgid "SecurityOrchestration|Use default mode for scoping"

308
scripts/duo_chat/reporter.rb
View File

@ -1,308 +0,0 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
# We need to take some precautions when using the `gitlab` gem in this project.
#
# See https://docs.gitlab.com/ee/development/pipelines/internals.html#using-the-gitlab-ruby-gem-in-the-canonical-project.
require 'gitlab'
require 'json'
class Reporter
GITLAB_COM_API_V4_ENDPOINT = "https://gitlab.com/api/v4"
QA_EVALUATION_PROJECT_ID = 52020045 # https://gitlab.com/gitlab-org/ai-powered/ai-framework/qa-evaluation
AGGREGATED_REPORT_ISSUE_IID = 1 # https://gitlab.com/gitlab-org/ai-powered/ai-framework/qa-evaluation/-/issues/1
IDENTIFIABLE_NOTE_TAG = 'gitlab-org/ai-powered/ai-framework:duo-chat-qa-evaluation'
GRADE_TO_EMOJI_MAPPING = {
correct: ":white_check_mark:",
incorrect: ":x:",
unexpected: ":warning:"
}.freeze
def run
if pipeline_running_on_master_branch?
snippet_web_url = upload_data_as_snippet
report_issue_url = create_report_issue
update_aggregation_issue(report_issue_url, snippet_web_url)
else
save_report_as_artifact
post_or_update_report_note
end
end
def markdown_report
@report ||= <<~MARKDOWN
<!-- #{IDENTIFIABLE_NOTE_TAG} -->
## GitLab Duo Chat QA evaluation
Report generated for "#{ENV['CI_JOB_NAME']}". This report is generated and refreshed automatically. Do not edit.
LLMs have been asked to evaluate GitLab Duo Chat's answers.
:white_check_mark: : LLM evaluated the answer as `CORRECT`.
:x: : LLM evaluated the answer as `INCORRECT`.
:warning: : LLM did not evaluate correctly or the evaluation request might have failed.
### Summary
- The total number of evaluations: #{summary_numbers[:total]}
- The number of evaluations in which all LLMs graded `CORRECT`: #{summary_numbers[:correct]} (#{summary_numbers[:correct_ratio]}%)
- Note: if an evaluation request failed or its response was not parsable, it was ignored. For example, :white_check_mark: :warning: would count as `CORRECT`.
- The number of evaluations in which all LLMs graded `INCORRECT`: #{summary_numbers[:incorrect]} (#{summary_numbers[:incorrect_ratio]}%)
- Note: if an evaluation request failed or its response was not parsable, it was ignored. For example, :x: :warning: would count as `INCORRECT`.
- The number of evaluations in which LLMs disagreed: #{summary_numbers[:disagreed]} (#{summary_numbers[:disagreed_ratio]}%)
### Evaluations
#{eval_content}
MARKDOWN
# Do this to avoid pinging users in notes/issues.
quote_usernames(@report)
end
private
def quote_usernames(text)
text.gsub(/(@\w+)/, '`\\1`')
end
def pipeline_running_on_master_branch?
ENV['CI_COMMIT_BRANCH'] == ENV['CI_DEFAULT_BRANCH']
end
def utc_timestamp
@utc_timestamp ||= Time.now.utc
end
def upload_data_as_snippet
filename = "#{utc_timestamp.to_i}.json"
title = utc_timestamp.to_s
snippet_content = ::JSON.pretty_generate({
commit: ENV["CI_COMMIT_SHA"],
pipeline_url: ENV["CI_PIPELINE_URL"],
data: report_data
})
puts "Creating a snippet #{filename}."
snippet = qa_evaluation_project_client.create_snippet(
QA_EVALUATION_PROJECT_ID,
{
title: title,
files: [{ file_path: filename, content: snippet_content }],
visibility: 'private'
}
)
snippet.web_url
end
def create_report_issue
puts "Creating a report issue."
issue_title = "Report #{utc_timestamp}"
new_issue = qa_evaluation_project_client.create_issue(
QA_EVALUATION_PROJECT_ID, issue_title, { description: markdown_report }
)
new_issue.web_url
end
def update_aggregation_issue(report_issue_url, snippet_web_url)
puts "Updating the aggregated report issue."
new_line = ["\n|"]
new_line << "#{utc_timestamp} |"
new_line << "#{summary_numbers[:total]} |"
new_line << "#{summary_numbers[:correct_ratio]}% |"
new_line << "#{summary_numbers[:incorrect_ratio]}% |"
new_line << "#{summary_numbers[:disagreed_ratio]}% |"
new_line << "#{report_issue_url} |"
new_line << "#{snippet_web_url} |"
new_line = new_line.join(' ')
aggregated_report_issue = qa_evaluation_project_client.issue(QA_EVALUATION_PROJECT_ID, AGGREGATED_REPORT_ISSUE_IID)
updated_description = aggregated_report_issue.description + new_line
qa_evaluation_project_client.edit_issue(
QA_EVALUATION_PROJECT_ID, AGGREGATED_REPORT_ISSUE_IID, { description: updated_description }
)
end
def save_report_as_artifact
artifact_path = File.join(base_dir, ENV['QA_EVAL_REPORT_FILENAME'])
puts "Saving #{artifact_path}"
File.write(artifact_path, markdown_report)
end
def post_or_update_report_note
note = existing_report_note
if note && note.type != 'DiscussionNote'
# The latest note has not led to a discussion. Update it.
gitlab_project_client.edit_merge_request_note(ci_project_id, merge_request_iid, note.id, markdown_report)
puts "Updated comment."
else
# This is the first note or the latest note has been discussed on the MR.
# Don't update, create new note instead.
gitlab_project_client.create_merge_request_note(ci_project_id, merge_request_iid, markdown_report)
puts "Posted comment."
end
end
def existing_report_note
# Look for an existing note using `IDENTIFIABLE_NOTE_TAG`
gitlab_project_client
.merge_request_notes(ci_project_id, merge_request_iid)
.auto_paginate
.select { |note| note.body.include? IDENTIFIABLE_NOTE_TAG }
.max_by { |note| Time.parse(note.created_at) }
end
def gitlab_project_client
@gitlab_project_client ||= Gitlab.client(
endpoint: GITLAB_COM_API_V4_ENDPOINT,
private_token: ENV['PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE']
)
end
def qa_evaluation_project_client
@qa_evaluation_project_client ||= Gitlab.client(
endpoint: GITLAB_COM_API_V4_ENDPOINT,
private_token: ENV['CHAT_QA_EVALUATION_PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE']
)
end
def base_dir
ENV['CI_PROJECT_DIR'] || "./"
end
def merge_request_iid
ENV['CI_MERGE_REQUEST_IID']
end
def ci_project_id
ENV['CI_PROJECT_ID']
end
def report_data
@report_data ||= Dir[File.join(base_dir, "tmp/duo_chat/qa*.json")]
.flat_map { |file| JSON.parse(File.read(file)) }
end
def eval_content
report_data
.sort_by { |a| a["question"] }
.map do |data|
<<~MARKDOWN
<details>
<summary>
#{correctness_indicator(data)}
`"#{data['question']}"`
(context: `#{data['resource']}`)
</summary>
#### Resource
`#{data['resource']}`
#### Answer
#{data['answer']}
#### LLM Evaluation
Tools used: #{data['tools_used']}
#{evalutions(data)}
</details>
MARKDOWN
end
.join
end
def summary_numbers
@graded_evaluations ||= report_data
.map { |data| data["evaluations"].map { |eval| parse_grade(eval) } }
.reject { |grades| !(grades.include? :correct) && !(grades.include? :incorrect) }
total = @graded_evaluations.size
correct = @graded_evaluations.count { |grades| !(grades.include? :incorrect) }
incorrect = @graded_evaluations.count { |grades| !(grades.include? :correct) }
disagreed = @graded_evaluations.count { |grades| (grades.include? :correct) && (grades.include? :incorrect) }
{
total: total,
correct: correct,
correct_ratio: (correct.to_f / total * 100).round(1),
incorrect: incorrect,
incorrect_ratio: (incorrect.to_f / total * 100).round(1),
disagreed: disagreed,
disagreed_ratio: (disagreed.to_f / total * 100).round(1)
}
end
def parse_grade(eval)
return :correct if eval["response"].match?(/Grade: CORRECT/i)
return :incorrect if eval["response"].match?(/Grade: INCORRECT/i)
# If the LLM's evaluation includes neither CORRECT nor CORRECT, flag it.
:unexpected
end
def correctness_indicator(data)
data["evaluations"].map { |eval| parse_grade(eval) }.map { |grade| GRADE_TO_EMOJI_MAPPING[grade] }.join(' ')
end
def evalutions(data)
rows = data["evaluations"].map do |eval|
grade = parse_grade(eval)
<<~MARKDOWN
<tr>
<td>#{eval['model']}</td>
<td>
#{GRADE_TO_EMOJI_MAPPING[grade]}
</td>
<td>
#{eval['response']}
</td
</tr>
MARKDOWN
end
.join
<<~MARKDOWN
<table>
<tr>
<td>Model</td>
<td>Grade</td>
<td>Details</td>
</tr>
#{rows}
</table>
MARKDOWN
end
end
Reporter.new.run if $PROGRAM_NAME == __FILE__

10
scripts/frontend/quarantined_vue3_specs.txt
View File

@ -100,7 +100,6 @@ ee/spec/frontend/vulnerabilities/generic_report/report_item_graphql_spec.js
ee/spec/frontend/vulnerabilities/related_issues_spec.js
spec/frontend/__helpers__/vue_test_utils_helper_spec.js
spec/frontend/access_tokens/index_spec.js
spec/frontend/add_context_commits_modal/components/add_context_commits_modal_spec.js
spec/frontend/admin/abuse_report/components/reported_content_spec.js
spec/frontend/admin/abuse_reports/components/abuse_reports_filtered_search_bar_spec.js
spec/frontend/admin/broadcast_messages/components/base_spec.js
@ -176,10 +175,6 @@ spec/frontend/issues/service_desk/components/service_desk_list_app_spec.js
spec/frontend/issues/show/components/app_spec.js
spec/frontend/issues/show/components/fields/description_spec.js
spec/frontend/issues/show/components/header_actions_spec.js
spec/frontend/issues/show/components/incidents/create_timeline_events_form_spec.js
spec/frontend/issues/show/components/incidents/edit_timeline_event_spec.js
spec/frontend/issues/show/components/incidents/timeline_events_form_spec.js
spec/frontend/issues/show/components/incidents/timeline_events_tab_spec.js
spec/frontend/jira_import/components/jira_import_form_spec.js
spec/frontend/lib/utils/breadcrumbs_spec.js
spec/frontend/lib/utils/confirm_via_gl_modal/confirm_action_spec.js
@ -188,7 +183,6 @@ spec/frontend/members/components/modals/leave_modal_spec.js
spec/frontend/members/components/table/max_role_spec.js
spec/frontend/members/components/table/members_pagination_spec.js
spec/frontend/members/components/table/members_table_spec.js
spec/frontend/merge_request_dashboard/components/assigned_users_spec.js
spec/frontend/milestones/components/milestone_combobox_spec.js
spec/frontend/ml/model_registry/apps/show_ml_model_spec.js
spec/frontend/ml/model_registry/components/candidate_detail_spec.js
@ -198,7 +192,6 @@ spec/frontend/notebook/cells/markdown_spec.js
spec/frontend/notebook/cells/output/html_spec.js
spec/frontend/notes/components/discussion_notes_spec.js
spec/frontend/notes/components/notes_app_spec.js
spec/frontend/organizations/shared/components/new_edit_form_spec.js
spec/frontend/packages_and_registries/container_registry/explorer/components/list_page/image_list_row_spec.js
spec/frontend/packages_and_registries/dependency_proxy/app_spec.js
spec/frontend/packages_and_registries/infrastructure_registry/components/details/components/app_spec.js
@ -219,7 +212,6 @@ spec/frontend/profile/preferences/components/profile_preferences_spec.js
spec/frontend/projects/commit/components/form_modal_spec.js
spec/frontend/projects/commits/components/author_select_spec.js
spec/frontend/projects/compare/components/revision_dropdown_spec.js
spec/frontend/projects/components/new_edit_form_spec.js
spec/frontend/projects/new/components/new_project_url_select_spec.js
spec/frontend/projects/report_abuse/components/report_abuse_dropdown_item_spec.js
spec/frontend/projects/settings/components/branch_rule_modal_spec.js
@ -267,9 +259,7 @@ spec/frontend/vue_shared/components/design_management/design_note_pin_spec.js
spec/frontend/vue_shared/components/diff_stats_dropdown_spec.js
spec/frontend/vue_shared/components/file_finder/index_spec.js
spec/frontend/vue_shared/components/file_tree_spec.js
spec/frontend/vue_shared/components/filtered_search_bar/filtered_search_bar_root_spec.js
spec/frontend/vue_shared/components/filtered_search_bar/tokens/date_token_spec.js
spec/frontend/vue_shared/components/markdown/header_spec.js
spec/frontend/vue_shared/components/markdown/markdown_editor_spec.js
spec/frontend/vue_shared/components/metric_images/metric_image_details_modal_spec.js
spec/frontend/vue_shared/components/page_heading_spec.js

62
spec/frontend/merge_request_dashboard/components/__snapshots__/assigned_users_spec.js.snap
View File

@ -17,65 +17,3 @@ exports[`Merge request dashboard assigned users component as reviewer list rende
</svg>
</span>
`;
exports[`Merge request dashboard assigned users component renders user avatars 1`] = `
<div
class="gl-flex gl-justify-center mr-users-list"
>
<div
class="gl-avatars-inline gl-avatars-inline-lg"
>
<div
class="gl-avatars-inline-child"
>
<a
class="gl-avatar-link gl-link gl-relative js-user-link"
data-name="Admin"
data-testid="assigned-user"
data-user-id="2"
href="/root"
target="blank"
>
<img
alt="avatar"
class="!gl-bg-white gl-avatar gl-avatar-circle gl-avatar-s32"
src="/root"
/>
<span
class="-gl-bottom-2 -gl-right-2 gl-absolute gl-bg-status-danger gl-flex gl-h-5 gl-items-center gl-justify-center gl-p-1 gl-rounded-full gl-w-5"
data-testid="review-state-icon"
>
<svg
aria-hidden="true"
class="gl-block gl-fill-current gl-fill-status-danger gl-icon s12"
data-testid="error-icon"
role="img"
>
<use
href="file-mock#error"
/>
</svg>
</span>
</a>
</div>
<div
class="gl-avatars-inline-child"
>
<a
class="gl-avatar-link gl-link gl-relative js-user-link"
data-name="Admin"
data-testid="assigned-user"
data-user-id="1"
href="/root"
target="blank"
>
<img
alt="avatar"
class="!gl-bg-white gl-avatar gl-avatar-circle gl-avatar-s32"
src="/root"
/>
</a>
</div>
</div>
</div>
`;

1
spec/frontend/merge_request_dashboard/components/assigned_users_spec.js
View File

@ -46,7 +46,6 @@ describe('Merge request dashboard assigned users component', () => {
createComponent();
expect(findAllUsers()).toHaveLength(2);
expect(wrapper.html()).toMatchSnapshot();
});
describe('current user avatar', () => {

14
spec/frontend/vue_shared/components/filtered_search_bar/filtered_search_bar_root_spec.js
View File

@ -4,6 +4,8 @@ import { nextTick } from 'vue';
import { useLocalStorageSpy } from 'helpers/local_storage_helper';
import waitForPromises from 'helpers/wait_for_promises';
import { markRaw } from '~/lib/utils/vue3compat/mark_raw';
import RecentSearchesService from '~/filtered_search/services/recent_searches_service';
import {
FILTERED_SEARCH_TERM,
@ -60,7 +62,7 @@ jest.mock('~/filtered_search/services/recent_searches_service', () => {
const defaultProps = {
namespace: 'gitlab-org/gitlab-test',
recentSearchesStorageKey: 'issues',
tokens: mockAvailableTokens,
tokens: markRaw(mockAvailableTokens),
initialFilterValue: [],
showCheckbox: false,
checkboxChecked: false,
@ -255,7 +257,7 @@ describe('FilteredSearchBarRoot', () => {
findGlSorting().vm.$emit('sortByChange', mockSortOptions[1].id);
await nextTick();
expect(wrapper.vm.selectedSortOption).toBe(mockSortOptions[1]);
expect(wrapper.vm.selectedSortOption).toEqual(mockSortOptions[1]);
expect(wrapper.emitted('onSort')[0]).toEqual([mockSortOptions[1].sortDirection.descending]);
});
});
@ -386,7 +388,7 @@ describe('FilteredSearchBarRoot', () => {
it('renders search history items dropdown with formatting done using token symbols', async () => {
createComponent({ propsData: { sortOptions: mockSortOptions }, shallow: false });
wrapper.vm.recentSearchesStore.addRecentSearch(mockHistoryItems[0]);
await nextTick();
await waitForPromises();
expect(findGlDisclosureDropdownItems().at(0).text()).toBe(
'Author := @rootLabel := ~bugMilestone := %v1.0"duo"',
@ -404,12 +406,12 @@ describe('FilteredSearchBarRoot', () => {
});
wrapper.vm.recentSearchesStore.addRecentSearch([tokenValueMembership]);
await nextTick();
await waitForPromises();
expect(findGlDisclosureDropdownItem().text()).toBe('Membership := Direct');
});
});
describe('when token options have do not have `title` attribute defined', () => {
describe('when token options do not have `title` attribute defined', () => {
it('renders search history items using the provided `value` attribute', async () => {
createComponent({
propsData: {
@ -419,7 +421,7 @@ describe('FilteredSearchBarRoot', () => {
shallow: false,
});
wrapper.vm.recentSearchesStore.addRecentSearch([tokenValueMembership]);
await nextTick();
await waitForPromises();
expect(findGlDisclosureDropdownItem().text()).toBe('Membership := exclude');
});
});

32
spec/lib/sidebars/menu_item_spec.rb
View File

@ -44,4 +44,36 @@ RSpec.describe Sidebars::MenuItem, feature_category: :navigation do
end
end
end
describe '#render?' do
subject(:menu_item) do
described_class.new(
title: 'Test Item',
link: '/test',
active_routes: []
)
end
context 'when render is not set' do
it { is_expected.to be_render }
end
context 'when render is explicitly set' do
context 'when set to false' do
before do
menu_item.render = false
end
it { is_expected.not_to be_render }
end
context 'when set to true' do
before do
menu_item.render = true
end
it { is_expected.to be_render }
end
end
end
end

41
spec/models/container_registry/event_spec.rb
View File

@ -137,6 +137,16 @@ RSpec.describe ContainerRegistry::Event, feature_category: :container_registry d
end
end
shared_examples 'tracking an internal event' do
it 'sends a tracking event' do
event_name = "i_container_registry_#{event}_deploy_token"
expect { subject }
.to trigger_internal_events(event_name)
.with(additional_properties: { property: originator.id.to_s })
.exactly(count).time
end
end
shared_examples 'event originator is fetched based on ID' do |originator_class|
it 'fetches the event originator based on id' do
count.times do
@ -147,7 +157,7 @@ RSpec.describe ContainerRegistry::Event, feature_category: :container_registry d
end
end
context 'with a respository target' do
context 'with a repository target' do
let(:target) do
{
'mediaType' => ContainerRegistry::Client::DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE,
@ -199,13 +209,9 @@ RSpec.describe ContainerRegistry::Event, feature_category: :container_registry d
context 'with a deploy token as the actor' do
let!(:originator) { create(:deploy_token, username: 'username', id: 3) }
shared_examples 'no tracking of a deploy token is sent to HLLRedisCounter' do
it 'does not send a tracking event to HLLRedisCounter' do
expect(DeployToken).not_to receive(:find)
expect(DeployToken).not_to receive(:find_by_username)
expect(::Gitlab::UsageDataCounters::HLLRedisCounter).not_to receive(:track_event)
expect { subject }.not_to raise_error
shared_examples 'no tracking of a deploy token is sent' do
it 'does not send a tracking event' do
expect { subject }.not_to trigger_internal_events
end
end
@ -218,13 +224,13 @@ RSpec.describe ContainerRegistry::Event, feature_category: :container_registry d
}
end
it_behaves_like 'no tracking of a deploy token is sent to HLLRedisCounter'
it_behaves_like 'no tracking of a deploy token is sent'
end
context 'when no username or deploy_token_id is given' do
let(:raw_event) { { 'action' => 'push', 'target' => {}, 'actor' => { 'user_type' => 'deploy_token' } } }
it_behaves_like 'no tracking of a deploy token is sent to HLLRedisCounter'
it_behaves_like 'no tracking of a deploy token is sent'
end
context 'when deploy_token_id is given' do
@ -255,12 +261,19 @@ RSpec.describe ContainerRegistry::Event, feature_category: :container_registry d
{ 'tag' => 'latest' } | 'delete' | 'delete_tag' | 1
{ 'repository' => 'foo/bar' } | 'push' | 'create_repository' | 1
{ 'repository' => 'foo/bar' } | 'delete' | 'delete_repository' | 1
{ 'tag' => 'latest' } | 'copy' | '' | 0
end
with_them do
it_behaves_like 'event originator is fetched based on ID', DeployToken
it_behaves_like 'tracking event is sent to HLLRedisCounter with event and originator ID', :deploy_token
it_behaves_like 'tracking an internal event'
end
context 'when the event is not a valid trackable event' do
let(:action) { 'copy' }
let(:target) { { 'tag' => 'latest' } }
it_behaves_like 'no tracking of a deploy token is sent'
end
context "when there are errors" do
@ -273,7 +286,7 @@ RSpec.describe ContainerRegistry::Event, feature_category: :container_registry d
allow(File).to receive(:read).with(key_file.path).and_raise(Errno::ENOENT)
end
it_behaves_like 'no tracking of a deploy token is sent to HLLRedisCounter'
it_behaves_like 'no tracking of a deploy token is sent'
end
[JWT::VerificationError, JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature].each do |error|
@ -284,7 +297,7 @@ RSpec.describe ContainerRegistry::Event, feature_category: :container_registry d
.and_raise(error)
end
it_behaves_like 'no tracking of a deploy token is sent to HLLRedisCounter'
it_behaves_like 'no tracking of a deploy token is sent'
end
end
end

270
spec/scripts/duo_chat/reporter_spec.rb
View File

@ -1,270 +0,0 @@
# frozen_string_literal: true
require 'fast_spec_helper'
require 'gitlab'
require 'json'
require_relative '../../../scripts/duo_chat/reporter'
RSpec.describe Reporter, feature_category: :ai_abstraction_layer do
subject(:reporter) { described_class.new }
describe '#run', :freeze_time do
let(:ci_commit_sha) { 'commitsha' }
let(:ci_pipeline_url) { 'https://gitlab.com/pipeline/url' }
let(:client) { double }
before do
stub_env('CI_COMMIT_SHA', ci_commit_sha)
stub_env('CI_PIPELINE_URL', ci_pipeline_url)
stub_env('CI_COMMIT_BRANCH', ci_commit_branch)
stub_env('CI_DEFAULT_BRANCH', ci_default_branch)
allow(Gitlab).to receive(:client).and_return(client)
end
context 'when the CI pipeline is running with the commit in `master` branch' do
let(:ci_commit_branch) { 'master' }
let(:ci_default_branch) { 'master' }
let(:snippet_web_url) { 'https://gitlab.com/snippet/url' }
let(:issue_web_url) { 'https://gitlab.com/issue/url' }
let(:mock_data) do
[
{
"question" => "question1",
"resource" => "resource",
"answer" => "answer1",
"tools_used" => ["foobar tool"],
"evaluations" => [
{ "model" => "claude-2", "response" => "Grade: CORRECT" },
{ "model" => "text-bison", "response" => "Grade: CORRECT" }
]
}
]
end
before do
allow(reporter).to receive(:report_data).and_return(mock_data)
end
it 'uploads snippet, creates a report issue and updates the tracking issue' do
# Uploads the test data as a snippet along with commit sha and pipeline url
snippet = double(web_url: snippet_web_url) # rubocop: disable RSpec/VerifiedDoubles -- an internal detail of Gitlab gem.
snippet_content = ::JSON.pretty_generate({
commit: ci_commit_sha,
pipeline_url: ci_pipeline_url,
data: mock_data
})
expect(client).to receive(:create_snippet).with(
described_class::QA_EVALUATION_PROJECT_ID,
{
title: Time.now.utc.to_s,
files: [{ file_path: "#{Time.now.utc.to_i}.json", content: snippet_content }],
visibility: 'private'
}
).and_return(snippet)
# Create a new issue for the report
issue_title = "Report #{Time.now.utc}"
issue = double(web_url: issue_web_url) # rubocop: disable RSpec/VerifiedDoubles -- an internal detail of Gitlab gem.
expect(client).to receive(:create_issue).with(
described_class::QA_EVALUATION_PROJECT_ID,
issue_title,
{ description: reporter.markdown_report }
).and_return(issue)
# Updates the tracking issue by adding a row that links to the snippet and the issue just created.
aggregated_report_issue = double(description: "") # rubocop: disable RSpec/VerifiedDoubles -- an internal detail of Gitlab gem.
allow(client).to receive(:issue).with(
described_class::QA_EVALUATION_PROJECT_ID,
described_class::AGGREGATED_REPORT_ISSUE_IID
).and_return(aggregated_report_issue)
row = "\n| #{Time.now.utc} | 1 | 100.0% | 0.0% | 0.0%"
row << " | #{issue_web_url} | #{snippet_web_url} |"
expect(client).to receive(:edit_issue).with(
described_class::QA_EVALUATION_PROJECT_ID,
described_class::AGGREGATED_REPORT_ISSUE_IID,
{ description: aggregated_report_issue.description + row }
)
reporter.run
end
end
context 'when the CI pipeline is not running with the commit in `master` branch' do
let(:ci_commit_branch) { 'foobar' }
let(:ci_default_branch) { 'master' }
let(:qa_eval_report_filename) { 'report.md' }
let(:merge_request_iid) { "123" }
let(:ci_project_id) { "456" }
let(:ci_project_dir) { "/builds/gitlab-org/gitlab" }
let(:base_dir) { "#{ci_project_dir}/#{qa_eval_report_filename}" }
before do
stub_env('QA_EVAL_REPORT_FILENAME', qa_eval_report_filename)
stub_env('CI_MERGE_REQUEST_IID', merge_request_iid)
stub_env('CI_PROJECT_ID', ci_project_id)
stub_env('CI_PROJECT_DIR', ci_project_dir)
end
context 'when a note does not already exist' do
let(:note) { nil }
it 'saves the report as a markdown file and creates a new MR note containing the report content' do
expect(File).to receive(:write).with(base_dir, reporter.markdown_report)
allow(reporter).to receive(:existing_report_note).and_return(note)
expect(client).to receive(:create_merge_request_note).with(
ci_project_id,
merge_request_iid,
reporter.markdown_report
)
reporter.run
end
end
context 'when a note exists' do
let(:note_id) { "1" }
let(:note) { double(id: note_id, type: "Note") } # rubocop: disable RSpec/VerifiedDoubles -- an internal detail of Gitlab gem.
it 'saves the report as a markdown file and updates the existing MR note containing the report content' do
expect(File).to receive(:write).with(base_dir, reporter.markdown_report)
allow(reporter).to receive(:existing_report_note).and_return(note)
expect(client).to receive(:edit_merge_request_note).with(
ci_project_id,
merge_request_iid,
note_id,
reporter.markdown_report
)
reporter.run
end
end
end
end
describe '#markdown_report' do
let(:mock_data) do
[
{
"question" => "question1",
"resource" => "resource",
"answer" => "answer1",
"tools_used" => ["foobar tool"],
"evaluations" => [
{ "model" => "claude-2", "response" => "Grade: CORRECT" },
{ "model" => "text-bison", "response" => "Grade: CORRECT" }
]
},
{
"question" => "question2",
"resource" => "resource",
"answer" => "answer2",
"tools_used" => [],
"evaluations" => [
{ "model" => "claude-2", "response" => " Grade: INCORRECT" },
{ "model" => "text-bison", "response" => "Grade: INCORRECT" }
]
},
{
"question" => "question3",
"resource" => "resource",
"answer" => "answer3",
"tools_used" => [],
"evaluations" => [
{ "model" => "claude-2", "response" => " Grade: CORRECT" },
{ "model" => "text-bison", "response" => "Grade: INCORRECT" }
]
},
{
"question" => "question4",
"resource" => "resource",
"answer" => "answer4",
"tools_used" => [],
# Note: The first evaluation (claude-2) is considered invalid and ignored.
"evaluations" => [
{ "model" => "claude-2", "response" => "???" },
{ "model" => "text-bison", "response" => "Grade: CORRECT" }
]
},
{
"question" => "question5",
"resource" => "resource",
"answer" => "answer5",
"tools_used" => [],
# Note: The second evaluation (text-bison) is considered invalid and ignored.
"evaluations" => [
{ "model" => "claude-2", "response" => " Grade: INCORRECT" },
{ "model" => "text-bison", "response" => "???" }
]
},
{
"question" => "question6",
"resource" => "resource",
"answer" => "answer6",
"tools_used" => [],
# Note: Both evaluations are invalid as they contain neither `CORRECT` nor `INCORRECT`.
# It should be ignored in the report.
"evaluations" => [
{ "model" => "claude-2", "response" => "???" },
{ "model" => "text-bison", "response" => "???" }
]
}
]
end
before do
allow(reporter).to receive(:report_data).and_return(mock_data)
end
it "generates the correct summary stats and uses the correct emoji indicators" do
expect(reporter.markdown_report).to include "The total number of evaluations: 5"
expect(reporter.markdown_report).to include "all LLMs graded `CORRECT`: 2 (40.0%)"
expect(reporter.markdown_report).to include ":white_check_mark: :white_check_mark:"
expect(reporter.markdown_report).to include ":warning: :white_check_mark:"
expect(reporter.markdown_report).to include "all LLMs graded `INCORRECT`: 2 (40.0%)"
expect(reporter.markdown_report).to include ":x: :x:"
expect(reporter.markdown_report).to include ":x: :warning:"
expect(reporter.markdown_report).to include "in which LLMs disagreed: 1 (20.0%)"
expect(reporter.markdown_report).to include ":white_check_mark: :x:"
end
it "includes the tools used" do
expect(reporter.markdown_report).to include "[\"foobar tool\"]"
end
context 'when usernames are present' do
let(:mock_data) do
[
{
"question" => "@user's @root?",
"resource" => "resource",
"answer" => "@user2 and @user3",
"tools_used" => ["foobar tool"],
"evaluations" => [
{ "model" => "claude-2", "response" => "Grade: CORRECT\n\n@user4" },
{ "model" => "text-bison", "response" => "Grade: CORRECT\n\n@user5" }
]
}
]
end
it 'quotes the usernames with backticks' do
expect(reporter.markdown_report).to include "`@root`"
expect(reporter.markdown_report).to include "`@user`"
expect(reporter.markdown_report).to include "`@user2`"
expect(reporter.markdown_report).to include "`@user3`"
expect(reporter.markdown_report).to include "`@user4`"
expect(reporter.markdown_report).to include "`@user5`"
end
end
end
end

3
spec/support/known_rspec_metadata_keys.yml
View File

@ -10,7 +10,6 @@
- :block
- :broken_storage
- :capybara_ignore_server_errors
- :chat_qa_evaluation
- :ci_config_validation
- :clean_gitlab_redis_buffered_counter
- :clean_gitlab_redis_cache
@ -54,7 +53,6 @@
- :experiment
- :factory_default
- :fails_if_sidekiq_not_configured
- :fast_chat_qa_evaluation
- :feature
- :feature_category
- :features
@ -168,6 +166,5 @@
- :with_sidekiq_context
- :without_license
- :yaml_processor_feature_flag_corectness
- :zeroshot_executor
- :zoekt
- :zoekt_settings_enabled