From dde89bf569336ffedbea98c1ebc2280276e24f12 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 1 Jul 2024 15:22:58 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../concerns/import/has_import_source.rb | 9 ++- doc/.vale/gitlab/Level.yml | 18 ++++++ .../blueprints/ai_gateway/index.md | 31 ++++------ .../legacy_github_import/base_formatter.rb | 7 +++ .../legacy_github_import/comment_formatter.rb | 3 +- .../legacy_github_import/issue_formatter.rb | 3 +- .../pull_request_formatter.rb | 3 +- package.json | 2 +- .../base_formatter_spec.rb | 61 +++++++++++++++++++ .../comment_formatter_spec.rb | 37 +++++++++-- .../issue_formatter_spec.rb | 17 ++++-- .../pull_request_formatter_spec.rb | 23 ++++--- .../concerns/import/has_import_source_spec.rb | 8 +++ yarn.lock | 8 +-- 14 files changed, 182 insertions(+), 48 deletions(-) create mode 100644 doc/.vale/gitlab/Level.yml create mode 100644 spec/lib/gitlab/legacy_github_import/base_formatter_spec.rb diff --git a/app/models/concerns/import/has_import_source.rb b/app/models/concerns/import/has_import_source.rb index fcbf6fe32a2..73c9c25156c 100644 --- a/app/models/concerns/import/has_import_source.rb +++ b/app/models/concerns/import/has_import_source.rb @@ -1,24 +1,27 @@ # frozen_string_literal: true module Import + SOURCE_NONE = :none SOURCE_DIRECT_TRANSFER = :gitlab_migration # aka BulkImports SOURCE_PROJECT_EXPORT_IMPORT = :gitlab_project SOURCE_GROUP_EXPORT_IMPORT = :gitlab_group + SOURCE_GITHUB = :github + SOURCE_GITEA = :gitea SOURCE_BITBUCKET_SERVER = :bitbucket_server module HasImportSource extend ActiveSupport::Concern IMPORT_SOURCES = { - none: 0, # not imported + SOURCE_NONE => 0, # not imported SOURCE_DIRECT_TRANSFER => 1, SOURCE_PROJECT_EXPORT_IMPORT => 2, SOURCE_GROUP_EXPORT_IMPORT => 3, - github: 4, + SOURCE_GITHUB => 4, bitbucket: 5, # aka bitbucket cloud SOURCE_BITBUCKET_SERVER => 6, fogbugz: 7, - gitea: 8, + SOURCE_GITEA => 8, git: 9, # aka repository by url manifest: 10, # aka manifest file custom_template: 11 # aka gitlab custom project template export diff --git a/doc/.vale/gitlab/Level.yml b/doc/.vale/gitlab/Level.yml new file mode 100644 index 00000000000..5eba926cee2 --- /dev/null +++ b/doc/.vale/gitlab/Level.yml @@ -0,0 +1,18 @@ +--- +# Suggestion: gitlab.Level +# +# Avoid variations on the phrase "instance level" and "group level" +# +# For a list of all options, see https://vale.sh/docs/topics/styles/ +extends: existence +message: "Avoid using 'level' when referring to groups, instances, or projects: '%s'" +link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html#level +level: suggestion +ignorecase: true +tokens: + - 'instance level' + - 'instance-level' + - 'group level' + - 'group-level' + - 'project level' + - 'project-level' diff --git a/doc/architecture/blueprints/ai_gateway/index.md b/doc/architecture/blueprints/ai_gateway/index.md index dac5a921c87..44153516f02 100644 --- a/doc/architecture/blueprints/ai_gateway/index.md +++ b/doc/architecture/blueprints/ai_gateway/index.md @@ -453,31 +453,24 @@ different. ## Authentication & Authorization -GitLab provides the first layer of authorization: It authenticates -the user and checks if the license allows using the feature the user is -trying to use. This can be done using the authentication, policy and license -checks that are already built into GitLab. +GitLab provides the first layer of authorization by authenticating the user and checking if the license permits using the requested feature. This is accomplished using the existing authentication, policy, and license checks built into GitLab. -Authenticating the GitLab-instance on the AI-gateway was discussed -in: +Authenticating the GitLab instance on the AI-gateway was discussed in: - [Issue 177](https://gitlab.com/gitlab-org/modelops/applied-ml/code-suggestions/ai-assist/-/issues/177) - [Epic 10808](https://gitlab.com/groups/gitlab-org/-/epics/10808) -The specific mechanism by which trust is delegated between end-users, GitLab instances, -and the AI-gateway is covered in the [Cloud Connector access control documentation](../../../development/cloud_connector/architecture.md#access-control). +The specific mechanism by which trust is delegated between end-users, GitLab instances, and the AI-gateway is detailed in the [Cloud Connector access control documentation](../../../development/cloud_connector/architecture.md#access-control). -AI Gateway is accessed only through Cloud connector which handles instance -authentication (among others). Cloud connector will need to support also end-user -authentication because some requests (e.g. code completion) will be sent -directly by clients instead of sending all requests indirectly through -GitLab Rails. A possible solution in -which short-term user tokens are used is described in [Epic 13252](https://gitlab.com/groups/gitlab-org/-/epics/13252). -AI Gateway needs to be able to distinguish between requests proxied by -GitLab Rails and direct client requests because some endpoints or parameters -may not be available for direct requests (for example clients should not be -able to send final prompt, but rather only sub-components from which the final -prompt is built by AI Gateway). +The AI Gateway, as a backend service, handles instance authentication among other tasks. It is accessed through the Cloud Connector Load Balancer (currently implemented as Cloudflare), which acts as a Web Application Firewall (WAF) layer but does not perform authentication. The AI Gateway also needs to support end-user authentication because some requests, such as code completion, will be sent directly by clients rather than indirectly through GitLab Rails. A possible solution involving short-term user tokens is described in [Epic 13252](https://gitlab.com/groups/gitlab-org/-/epics/13252). The AI Gateway must distinguish between requests proxied by GitLab Rails and direct client requests, as some endpoints or parameters may not be available for direct requests (e.g., clients should only send sub-components of the final prompt, which the AI Gateway will build). + +The AI Gateway uses JSON-based API communication and relies on JWTs for authentication. These JWTs are generally scoped to the GitLab instance and are obtained using a PAT or OAuth token. For Code Completions, a short-lived user-bound JWT is used, allowing direct communication between the client and the AI Gateway, bypassing the need to go through the GitLab instance for each request. This short-lived JWT is valid for one hour, reducing the number of JWTs needed for multiple prompt requests. + +When a client initiates a prompt in GitLab Duo Chat or an IDE, the prompts, telemetry, context, and JWT token are packaged and transmitted using TLS. It is important to note that the payload itself is not encrypted; it is passed as plain text in JSON format within the request body. This approach is consistent for calls to third-party models where only tunnel-level TLS encryption is used via HTTPS. + +The AI Gateway operates within GitLab hosted infrastructure, interfacing with APIs hosted in GitLab accounts for providers like Anthropic and Vertex. Models are hosted as frozen versions of Google's models within GitLab GCP tenancy, using private endpoints within the security perimeter. While prompts from different customers use the same shared model, each session maintains a user-level connection to ensure isolation. + +The AI Gateway is designed as a stateless service, meaning it does not store any customer-specific data. Decryption of payloads occurs at the network layer rather than the application layer, and encryption keys are generated per request using GKE native GCP processes. Requests to Anthropic APIs are made via the public internet, whereas requests to Vertex AI models are optimized by being co-located within the same GCP region. All connections are secured with TLS/HTTPS to ensure encrypted communication throughout the data flow. ## Embeddings diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index 7bb33cd474b..9bbf68cd9c9 100644 --- a/lib/gitlab/legacy_github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -25,6 +25,13 @@ module Gitlab def url raw_data[:url] || '' end + + def imported_from + return ::Import::SOURCE_GITEA if project.gitea_import? + return ::Import::SOURCE_GITHUB if project.github_import? + + ::Import::SOURCE_NONE + end end end end diff --git a/lib/gitlab/legacy_github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb index ffd9da604ca..acf1729ff09 100644 --- a/lib/gitlab/legacy_github_import/comment_formatter.rb +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -14,7 +14,8 @@ module Gitlab author_id: author_id, type: type, created_at: raw_data[:created_at], - updated_at: raw_data[:updated_at] + updated_at: raw_data[:updated_at], + imported_from: imported_from } end diff --git a/lib/gitlab/legacy_github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb index e5c568207e3..8b94307337e 100644 --- a/lib/gitlab/legacy_github_import/issue_formatter.rb +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -14,7 +14,8 @@ module Gitlab author_id: author_id, assignee_ids: Array(assignee_id), created_at: raw_data[:created_at], - updated_at: raw_data[:updated_at] + updated_at: raw_data[:updated_at], + imported_from: imported_from } end diff --git a/lib/gitlab/legacy_github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb index 10ace7a09f8..9afd859b995 100644 --- a/lib/gitlab/legacy_github_import/pull_request_formatter.rb +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -22,7 +22,8 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data[:created_at], - updated_at: raw_data[:updated_at] + updated_at: raw_data[:updated_at], + imported_from: imported_from } end diff --git a/package.json b/package.json index 85dd89cf777..5f102191b03 100644 --- a/package.json +++ b/package.json @@ -153,7 +153,7 @@ "gettext-parser": "^6.0.0", "graphql": "^15.7.2", "graphql-tag": "^2.11.0", - "gridstack": "^10.2.1", + "gridstack": "^10.3.0", "highlight.js": "^11.8.0", "immer": "^9.0.15", "ipaddr.js": "^1.9.1", diff --git a/spec/lib/gitlab/legacy_github_import/base_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/base_formatter_spec.rb new file mode 100644 index 00000000000..88279b8a210 --- /dev/null +++ b/spec/lib/gitlab/legacy_github_import/base_formatter_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::LegacyGithubImport::BaseFormatter, feature_category: :importers do + let_it_be(:project) { create(:project, import_type: 'gitea', namespace: create(:namespace, path: 'octocat')) } + let(:client) { double } + let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } + let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } + let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } + let(:imported_from) { ::Import::SOURCE_GITEA } + + let(:raw_data) do + { + number: 1347, + milestone: nil, + state: 'open', + title: 'Found a bug', + body: "I'm having a problem with this.", + assignee: nil, + user: octocat, + comments: 0, + pull_request: nil, + created_at: created_at, + updated_at: updated_at, + closed_at: nil + } + end + + subject(:base) { described_class.new(project, raw_data, client) } + + before do + allow(client).to receive(:user).and_return(octocat) + end + + describe '#imported_from' do + it 'returns the correct value for a gitea import' do + expect(base.imported_from).to eq(:gitea) + end + + context 'when the import type is github' do + before do + project.import_type = 'github' + end + + it 'returns the correct value for a github import' do + expect(base.imported_from).to eq(:github) + end + end + + context 'when the import type is unknown' do + before do + project.import_type = nil + end + + it 'returns the correct value for a unknown import' do + expect(base.imported_from).to eq(:none) + end + end + end +end diff --git a/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb index 8b6d628833e..780a9d1d19e 100644 --- a/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb @@ -3,11 +3,12 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: :importers do - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, import_type: 'gitea') } let(:client) { double } let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:created_at) { DateTime.strptime('2013-04-10T20:09:31Z') } let(:updated_at) { DateTime.strptime('2014-03-03T18:58:10Z') } + let(:imported_from) { ::Import::SOURCE_GITEA } let(:base) do { body: "I'm having a problem with this.", @@ -15,7 +16,8 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: : commit_id: nil, diff_hunk: nil, created_at: created_at, - updated_at: updated_at + updated_at: updated_at, + imported_from: imported_from } end @@ -38,7 +40,8 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: : author_id: project.creator_id, type: nil, created_at: created_at, - updated_at: updated_at + updated_at: updated_at, + imported_from: imported_from } expect(comment.attributes).to eq(expected) @@ -66,7 +69,8 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: : author_id: project.creator_id, type: 'LegacyDiffNote', created_at: created_at, - updated_at: updated_at + updated_at: updated_at, + imported_from: imported_from } expect(comment.attributes).to eq(expected) @@ -88,5 +92,30 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: : expect(comment.attributes.fetch(:note)).to eq("I'm having a problem with this.") end end + + context 'when importing a GitHub project' do + let(:imported_from) { ::Import::SOURCE_GITHUB } + let(:raw) { base } + + before do + project.import_type = 'github' + end + + it 'returns formatted attributes' do + expected = { + project: project, + note: "*Created by: octocat*\n\nI'm having a problem with this.", + commit_id: nil, + line_code: nil, + author_id: project.creator_id, + type: nil, + created_at: created_at, + updated_at: updated_at, + imported_from: imported_from + } + + expect(comment.attributes).to eq(expected) + end + end end end diff --git a/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb index 9baf234b14b..4427f0163c8 100644 --- a/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb @@ -3,11 +3,12 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :importers do - let_it_be(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) } + let_it_be(:project) { create(:project, import_type: 'gitea', namespace: create(:namespace, path: 'octocat')) } let(:client) { double } let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } + let(:imported_from) { ::Import::SOURCE_GITEA } let(:base_data) do { @@ -47,7 +48,8 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im author_id: project.creator_id, assignee_ids: [], created_at: created_at, - updated_at: updated_at + updated_at: updated_at, + imported_from: imported_from } expect(issue.attributes).to eq(expected) @@ -68,7 +70,8 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im author_id: project.creator_id, assignee_ids: [], created_at: created_at, - updated_at: updated_at + updated_at: updated_at, + imported_from: imported_from } expect(issue.attributes).to eq(expected) @@ -133,14 +136,16 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im end end - context 'when importing a GitHub project' do + context 'when importing a Gitea project' do it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#attributes' it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#number' end - context 'when importing a Gitea project' do + context 'when importing a GitHub project' do + let(:imported_from) { ::Import::SOURCE_GITHUB } + before do - project.update!(import_type: 'gitea') + project.import_type = 'github' end it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#attributes' diff --git a/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb index a437a6b1b57..49856c946de 100644 --- a/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_category: :importers do - let_it_be(:project) { create(:project, :repository) } + let_it_be(:project) { create(:project, :repository, import_type: 'gitea') } let(:client) { double } let(:source_sha) { create(:commit, project: project).id } let(:target_commit) { create(:commit, project: project, git_commit: RepoHelpers.another_sample_commit) } @@ -21,6 +21,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } + let(:imported_from) { ::Import::SOURCE_GITEA } let(:base_data) do { number: 1347, @@ -36,7 +37,8 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor updated_at: updated_at, closed_at: nil, merged_at: nil, - url: 'https://api.github.com/repos/octocat/Hello-World/pulls/1347' + url: 'https://api.github.com/repos/octocat/Hello-World/pulls/1347', + imported_from: imported_from } end @@ -66,7 +68,8 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor author_id: project.creator_id, assignee_id: nil, created_at: created_at, - updated_at: updated_at + updated_at: updated_at, + imported_from: imported_from } expect(pull_request.attributes).to eq(expected) @@ -92,7 +95,8 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor author_id: project.creator_id, assignee_id: nil, created_at: created_at, - updated_at: updated_at + updated_at: updated_at, + imported_from: imported_from } expect(pull_request.attributes).to eq(expected) @@ -119,7 +123,8 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor author_id: project.creator_id, assignee_id: nil, created_at: created_at, - updated_at: updated_at + updated_at: updated_at, + imported_from: imported_from } expect(pull_request.attributes).to eq(expected) @@ -236,16 +241,18 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor end end - context 'when importing a GitHub project' do + context 'when importing a Gitea project' do it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#attributes' it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#number' it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#source_branch_name' it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#target_branch_name' end - context 'when importing a Gitea project' do + context 'when importing a GitHub project' do + let(:imported_from) { ::Import::SOURCE_GITHUB } + before do - project.update!(import_type: 'gitea') + project.import_type = 'github' end it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#attributes' diff --git a/spec/models/concerns/import/has_import_source_spec.rb b/spec/models/concerns/import/has_import_source_spec.rb index 4a9f56138d2..92a6aa1b81a 100644 --- a/spec/models/concerns/import/has_import_source_spec.rb +++ b/spec/models/concerns/import/has_import_source_spec.rb @@ -6,12 +6,16 @@ RSpec.describe Import::HasImportSource, feature_category: :importers do let_it_be(:snippet_not_imported) { create(:snippet, :repository) } let_it_be(:snippet_imported) { create(:snippet, imported_from: :bitbucket) } let_it_be(:merge_request_imported) { create(:snippet, imported_from: :fogbugz) } + let_it_be(:merge_request_imported_github) { create(:snippet, imported_from: :github) } + let_it_be(:merge_request_imported_gitea) { create(:snippet, imported_from: :gitea) } describe '#imported?' do it 'returns the correct imported state' do expect(snippet_not_imported.imported?).to eq(false) expect(snippet_imported.imported?).to eq(true) expect(merge_request_imported.imported?).to eq(true) + expect(merge_request_imported_github.imported?).to eq(true) + expect(merge_request_imported_gitea.imported?).to eq(true) end end @@ -20,6 +24,8 @@ RSpec.describe Import::HasImportSource, feature_category: :importers do expect(snippet_not_imported.imported_from).to eq('none') expect(snippet_imported.imported_from).to eq('bitbucket') expect(merge_request_imported.imported_from).to eq('fogbugz') + expect(merge_request_imported_github.imported_from).to eq('github') + expect(merge_request_imported_gitea.imported_from).to eq('gitea') end end @@ -28,6 +34,8 @@ RSpec.describe Import::HasImportSource, feature_category: :importers do expect(snippet_not_imported.imported_from_github?).to eq(false) expect(snippet_imported.imported_from_bitbucket?).to eq(true) expect(merge_request_imported.imported_from_gitlab_migration?).to eq(false) + expect(merge_request_imported_github.imported_from_gitlab_project?).to eq(false) + expect(merge_request_imported_gitea.imported_from_gitea?).to eq(true) end end end diff --git a/yarn.lock b/yarn.lock index 513da1e0be1..176f68b6516 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7637,10 +7637,10 @@ graphql@^15.7.2: resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.7.2.tgz#85ab0eeb83722977151b3feb4d631b5f2ab287ef" integrity sha512-AnnKk7hFQFmU/2I9YSQf3xw44ctnSFCfp3zE0N6W174gqe9fWG/2rKaKxROK7CcI3XtERpjEKFqts8o319Kf7A== -gridstack@^10.2.1: - version "10.2.1" - resolved "https://registry.yarnpkg.com/gridstack/-/gridstack-10.2.1.tgz#3ce6119ae86cfb0a533c5f0d15b03777a55384ca" - integrity sha512-UAPKnIvd9sIqPDFMtKMqj0G5GDj8MUFPcelRJq7FzQFSxSYBblKts/Gd52iEJg0EvTFP51t6ZuMWGx0pSSFBdw== +gridstack@^10.3.0: + version "10.3.0" + resolved "https://registry.yarnpkg.com/gridstack/-/gridstack-10.3.0.tgz#8fa065f896d0a880c5c54c24d189f3197184488a" + integrity sha512-eGKsmU2TppV4coyDu9IIdIkm4qjgLLdjlEOFwQyQMuSwfOpzSfLdPc8du0HuebGr7CvAIrJxN4lBOmGrWSBg9g== gzip-size@^6.0.0: version "6.0.0"