root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2025-05-12 06:10:52 +00:00
parent e2ea448cb4
commit c0f79a5e14
31 changed files with 937 additions and 430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Gemfile
View File

@ -645,7 +645,7 @@ gem 'spamcheck', '~> 1.3.0', feature_category: :insider_threat
gem 'gitaly', '~> 17.8.0', feature_category: :gitaly
# KAS GRPC protocol definitions
gem 'gitlab-kas-grpc', '~> 17.10.4', feature_category: :deployment_management
gem 'gitlab-kas-grpc', '~> 17.11.0', feature_category: :deployment_management
# Lock the version before issues below are resolved:
# https://gitlab.com/gitlab-org/gitlab/-/issues/473169#note_2028352939

2
Gemfile.checksum
View File

@ -230,7 +230,7 @@
{"name":"gitlab-glfm-markdown","version":"0.0.30","platform":"x86_64-darwin","checksum":"583057ac3223900759a2bbc0f7dfe11befba74127217f883cd8ed2de2b7e2251"},
{"name":"gitlab-glfm-markdown","version":"0.0.30","platform":"x86_64-linux-gnu","checksum":"d43b185f48577fbe850fa49a5ff28ce2d875a4dd23b1c5333fae2ef4c40bea40"},
{"name":"gitlab-glfm-markdown","version":"0.0.30","platform":"x86_64-linux-musl","checksum":"50ac59a1f9545387773fe0daa389ae146c73beb1acff4093011c2a9a1889879a"},
{"name":"gitlab-kas-grpc","version":"17.10.4","platform":"ruby","checksum":"eef69a52b39ea69f7c79ba1785b37f985187a8609744d2b17336b738402858b1"},
{"name":"gitlab-kas-grpc","version":"17.11.2","platform":"ruby","checksum":"f2b9054dcf636346e89549e9478a684a38bf03bf853332e84154ec3a9856ae1c"},
{"name":"gitlab-labkit","version":"0.37.0","platform":"ruby","checksum":"d2dd0a60db2149a9a8eebf2975dc23f54ac3ceb01bdba732eb1b26b86dfffa70"},
{"name":"gitlab-license","version":"2.6.0","platform":"ruby","checksum":"2c1f8ae73835640ec77bf758c1d0c9730635043c01cf77902f7976e826d7d016"},
{"name":"gitlab-mail_room","version":"0.0.27","platform":"ruby","checksum":"05c07db892094cf5747ea00afb0a95c5a5406e05f34ae779f4388f2ddf962316"},

4
Gemfile.lock
View File

@ -773,7 +773,7 @@ GEM
nokogiri (~> 1, >= 1.10.8)
gitlab-glfm-markdown (0.0.30)
rb_sys (~> 0.9.109)
gitlab-kas-grpc (17.10.4)
gitlab-kas-grpc (17.11.2)
grpc (~> 1.0)
gitlab-labkit (0.37.0)
actionpack (>= 5.0.0, < 8.1.0)
@ -2172,7 +2172,7 @@ DEPENDENCIES
gitlab-glfm-markdown (~> 0.0.30)
gitlab-housekeeper!
gitlab-http!
gitlab-kas-grpc (~> 17.10.4)
gitlab-kas-grpc (~> 17.11.0)
gitlab-labkit (~> 0.37.0)
gitlab-license (~> 2.6)
gitlab-mail_room (~> 0.0.24)

2
Gemfile.next.checksum
View File

@ -230,7 +230,7 @@
{"name":"gitlab-glfm-markdown","version":"0.0.30","platform":"x86_64-darwin","checksum":"583057ac3223900759a2bbc0f7dfe11befba74127217f883cd8ed2de2b7e2251"},
{"name":"gitlab-glfm-markdown","version":"0.0.30","platform":"x86_64-linux-gnu","checksum":"d43b185f48577fbe850fa49a5ff28ce2d875a4dd23b1c5333fae2ef4c40bea40"},
{"name":"gitlab-glfm-markdown","version":"0.0.30","platform":"x86_64-linux-musl","checksum":"50ac59a1f9545387773fe0daa389ae146c73beb1acff4093011c2a9a1889879a"},
{"name":"gitlab-kas-grpc","version":"17.10.4","platform":"ruby","checksum":"eef69a52b39ea69f7c79ba1785b37f985187a8609744d2b17336b738402858b1"},
{"name":"gitlab-kas-grpc","version":"17.11.2","platform":"ruby","checksum":"f2b9054dcf636346e89549e9478a684a38bf03bf853332e84154ec3a9856ae1c"},
{"name":"gitlab-labkit","version":"0.37.0","platform":"ruby","checksum":"d2dd0a60db2149a9a8eebf2975dc23f54ac3ceb01bdba732eb1b26b86dfffa70"},
{"name":"gitlab-license","version":"2.6.0","platform":"ruby","checksum":"2c1f8ae73835640ec77bf758c1d0c9730635043c01cf77902f7976e826d7d016"},
{"name":"gitlab-mail_room","version":"0.0.27","platform":"ruby","checksum":"05c07db892094cf5747ea00afb0a95c5a5406e05f34ae779f4388f2ddf962316"},

4
Gemfile.next.lock
View File

@ -773,7 +773,7 @@ GEM
nokogiri (~> 1, >= 1.10.8)
gitlab-glfm-markdown (0.0.30)
rb_sys (~> 0.9.109)
gitlab-kas-grpc (17.10.4)
gitlab-kas-grpc (17.11.2)
grpc (~> 1.0)
gitlab-labkit (0.37.0)
actionpack (>= 5.0.0, < 8.1.0)
@ -2172,7 +2172,7 @@ DEPENDENCIES
gitlab-glfm-markdown (~> 0.0.30)
gitlab-housekeeper!
gitlab-http!
gitlab-kas-grpc (~> 17.10.4)
gitlab-kas-grpc (~> 17.11.0)
gitlab-labkit (~> 0.37.0)
gitlab-license (~> 2.6)
gitlab-mail_room (~> 0.0.24)

2
app/models/packages/conan/package_reference.rb
View File

@ -27,6 +27,8 @@ module Packages
validates :info, json_schema: { filename: 'conan_package_info', detail_errors: true }
validate :ensure_info_size
scope :pluck_reference_and_info, -> { limit(MAX_PLUCK).pluck(:reference, :info) }
def self.for_package_id_and_reference(package_id, reference)
where(package_id: package_id, reference: reference)
end

5
app/services/packages/generic/create_package_file_service.rb
View File

@ -5,8 +5,11 @@ module Packages
class CreatePackageFileService < BaseService
def execute
::Packages::Package.transaction do
create_package_file(find_or_create_package)
package_file = create_package_file(find_or_create_package)
ServiceResponse.success(payload: { package_file: package_file })
end
rescue ::Packages::DuplicatePackageError => e
ServiceResponse.error(message: e.message, reason: :package_file_already_exists)
end
private

10
config/feature_flags/gitlab_com_derisk/duo_code_review_system_note.yml
View File

@ -1,10 +0,0 @@
---
name: duo_code_review_system_note
description:
feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/536723
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/189025
rollout_issue_url:
milestone: '18.0'
group: group::code review
type: gitlab_com_derisk
default_enabled: false

145
doc/api/openapi/openapi_v2.yaml
View File

@ -13849,6 +13849,56 @@ paths:
tags:
- conan_packages
operationId: getApiV4ProjectsIdPackagesConanV1ConansSearch
"/api/v4/projects/{id}/packages/conan/v1/conans/{package_name}/{package_version}/{package_username}/{package_channel}/search":
get:
summary: Get package references metadata
description: This feature was introduced in GitLab 18.0
produces:
- application/json
parameters:
- in: path
name: id
description: The ID or URL-encoded path of the project
type: string
required: true
- in: path
name: package_name
description: Package name
type: string
required: true
example: my-package
- in: path
name: package_version
description: Package version
type: string
required: true
example: '1.0'
- in: path
name: package_username
description: Package username
type: string
required: true
example: my-group+my-project
- in: path
name: package_channel
description: Package channel
type: string
required: true
example: stable
responses:
'200':
description: Get package references metadata
'400':
description: Bad Request
'401':
description: Unauthorized
'403':
description: Forbidden
'404':
description: Not Found
tags:
- conan_packages
operationId: getApiV4ProjectsIdPackagesConanV1ConansPackageNamePackageVersionPackageUsernamePackageChannelSearch
"/api/v4/projects/{id}/packages/conan/v1/ping":
get:
summary: Ping the Conan API
@ -14876,6 +14926,56 @@ paths:
tags:
- conan_packages
operationId: getApiV4ProjectsIdPackagesConanV2ConansSearch
"/api/v4/projects/{id}/packages/conan/v2/conans/{package_name}/{package_version}/{package_username}/{package_channel}/search":
get:
summary: Get package references metadata
description: This feature was introduced in GitLab 18.0
produces:
- application/json
parameters:
- in: path
name: id
description: The ID or URL-encoded path of the project
type: string
required: true
- in: path
name: package_name
description: Package name
type: string
required: true
example: my-package
- in: path
name: package_version
description: Package version
type: string
required: true
example: '1.0'
- in: path
name: package_username
description: Package username
type: string
required: true
example: my-group+my-project
- in: path
name: package_channel
description: Package channel
type: string
required: true
example: stable
responses:
'200':
description: Get package references metadata
'400':
description: Bad Request
'401':
description: Unauthorized
'403':
description: Forbidden
'404':
description: Not Found
tags:
- conan_packages
operationId: getApiV4ProjectsIdPackagesConanV2ConansPackageNamePackageVersionPackageUsernamePackageChannelSearch
"/api/v4/projects/{id}/packages/conan/v2/conans/{package_name}/{package_version}/{package_username}/{package_channel}/latest":
get:
summary: Get the latest recipe revision
@ -38357,6 +38457,51 @@ paths:
tags:
- conan_packages
operationId: getApiV4PackagesConanV1ConansSearch
"/api/v4/packages/conan/v1/conans/{package_name}/{package_version}/{package_username}/{package_channel}/search":
get:
summary: Get package references metadata
description: This feature was introduced in GitLab 18.0
produces:
- application/json
parameters:
- in: path
name: package_name
description: Package name
type: string
required: true
example: my-package
- in: path
name: package_version
description: Package version
type: string
required: true
example: '1.0'
- in: path
name: package_username
description: Package username
type: string
required: true
example: my-group+my-project
- in: path
name: package_channel
description: Package channel
type: string
required: true
example: stable
responses:
'200':
description: Get package references metadata
'400':
description: Bad Request
'401':
description: Unauthorized
'403':
description: Forbidden
'404':
description: Not Found
tags:
- conan_packages
operationId: getApiV4PackagesConanV1ConansPackageNamePackageVersionPackageUsernamePackageChannelSearch
"/api/v4/packages/conan/v1/ping":
get:
summary: Ping the Conan API

93
doc/api/packages/conan_v1.md
View File

@ -240,16 +240,13 @@ Example response:
}
```
The URLs in the response have the same route prefix used to request them. If you request them with
the project-level route, the returned URLs contain `/projects/:id`.
## Get a package manifest
Gets a manifest that includes a list of files and associated download URLs for a specified package.
```plaintext
GET /packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference/digest
GET /projects/:id/packages/conan/v1/conans/:package_version/:package_username/:package_channel/packages/:conan_package_reference/digest
GET /projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference/digest
```
| Attribute | Type | Required | Description |
@ -275,9 +272,6 @@ Example response:
}
```
The URLs in the response have the same route prefix used to request them. If you request them with
the project-level route, the returned URLs contain `/projects/:id`.
## List all recipe download URLs
Lists all files and associated download URLs for a specified recipe.
@ -310,9 +304,6 @@ Example response:
}
```
The URLs in the response have the same route prefix used to request them. If you request them with
the project-level route, the returned URLs contain `/projects/:id`.
## List all package download URLs
Lists all files and associated download URLs for a specified package.
@ -346,9 +337,6 @@ Example response:
}
```
The URLs in the response have the same route prefix used to request them. If you request them with
the project-level route, the returned URLs contain `/projects/:id`.
## List all recipe upload URLs
Lists the upload URLs for a specified collection of recipe files. The request must include a JSON object
@ -395,9 +383,6 @@ Example response:
}
```
The URLs in the response have the same route prefix used to request them. If you request them with
the project-level route, the returned URLs contain `/projects/:id`.
## List all package upload URLs
Lists the upload URLs for a specified collection of package files. The request must include a JSON object
@ -447,17 +432,14 @@ Example response:
}
```
The URLs in the response have the same route prefix used to request them. If you request them with
the project-level route, the returned URLs contain `/projects/:id`.
## Get a recipe file
Gets a recipe file from the package registry. You must use the download URL returned from the
[recipe download URLs](#list-all-recipe-download-urls) endpoint.
```plaintext
GET packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name
GET projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name
GET /packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name
GET /projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name
```
| Attribute | Type | Required | Description |
@ -488,8 +470,8 @@ Uploads a specified recipe file in the package registry. You must use the upload
[recipe upload URLs](#list-all-recipe-upload-urls) endpoint.
```plaintext
PUT packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name
PUT projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name
PUT /packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name
PUT /projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name
```
| Attribute | Type | Required | Description |
@ -517,8 +499,8 @@ Gets a package file from the package registry. You must use the download URL ret
[package download URLs](#list-all-package-download-urls) endpoint.
```plaintext
GET packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name
GET projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name
GET /packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name
GET /projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name
```
| Attribute | Type | Required | Description |
@ -551,8 +533,8 @@ Uploads a specified package file in the package registry. You must use the uploa
[package upload URLs](#list-all-package-upload-urls) endpoint.
```plaintext
PUT packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name
PUT projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name
PUT /packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name
PUT /projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name
```
| Attribute | Type | Required | Description |
@ -581,8 +563,8 @@ curl --request PUT \
Deletes a specified Conan recipe and the associated package files from the package registry.
```plaintext
DELETE packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel
DELETE projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel
DELETE /packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel
DELETE /projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel
```
| Attribute | Type | Required | Description |
@ -612,3 +594,56 @@ Example response:
"status": "default"
}
```
## Get package references metadata
Gets the metadata for all package references of a package.
```plaintext
GET /packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/search
GET /projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/search
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | string | Conditionally | The project ID or full project path. Required only for the project endpoint. |
| `package_name` | string | yes | Name of a package. |
| `package_version` | string | yes | Version of a package. |
| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. |
| `package_channel` | string | yes | Channel of a package. |
```shell
curl --header "Authorization: Bearer <authenticate_token>" \
--url "https://gitlab.example.com/api/v4/packages/conan/v1/conans/my-package/1.0/my-group+my-project/stable/search"
```
Example response:
```json
{
"103f6067a947f366ef91fc1b7da351c588d1827f": {
"settings": {
"arch": "x86_64",
"build_type": "Release",
"compiler": "gcc",
"compiler.libcxx": "libstdc++",
"compiler.version": "9",
"os": "Linux"
},
"options": {
"shared": "False"
},
"requires": {
"zlib/1.2.11": null
},
"recipe_hash": "75151329520e7685dcf5da49ded2fec0"
}
}
```
The response includes the following metadata for each package reference:
- `settings`: The build settings used for the package.
- `options`: The package options.
- `requires`: The dependencies required for the package.
- `recipe_hash`: The hash of the recipe.

52
doc/api/packages/conan_v2.md
View File

@ -466,3 +466,55 @@ Example response:
"new_file_path": null
}
```
## Get package references metadata
Gets the metadata for all package references of a package.
```plaintext
GET /projects/:id/packages/conan/v2/conans/:package_name/:package_version/:package_username/:package_channel/search
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | string | yes | The project ID or full project path. |
| `package_name` | string | yes | Name of a package. |
| `package_version` | string | yes | Version of a package. |
| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. |
| `package_channel` | string | yes | Channel of a package. |
```shell
curl --header "Authorization: Bearer <authenticate_token>" \
--url "https://gitlab.example.com/api/v4/projects/9/packages/conan/v2/conans/my-package/1.0/my-group+my-project/stable/search"
```
Example response:
```json
{
"103f6067a947f366ef91fc1b7da351c588d1827f": {
"settings": {
"arch": "x86_64",
"build_type": "Release",
"compiler": "gcc",
"compiler.libcxx": "libstdc++",
"compiler.version": "9",
"os": "Linux"
},
"options": {
"shared": "False"
},
"requires": {
"zlib/1.2.11": null
},
"recipe_hash": "75151329520e7685dcf5da49ded2fec0"
}
}
```
The response includes the following metadata for each package reference:
- `settings`: The build settings used for the package.
- `options`: The package options.
- `requires`: The required dependencies for the package.
- `recipe_hash`: The hash of the recipe.

3
doc/ci/jobs/fine_grained_permissions.md
View File

@ -170,6 +170,7 @@ The following endpoints are available for CI/CD job tokens.
| Packages: Read | `READ_PACKAGES` | `GET /packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference/digest` | Package Digest |
| Packages: Read | `READ_PACKAGES` | `GET /packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference/download_urls` | Package Download Urls |
| Packages: Read | `READ_PACKAGES` | `GET /packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference` | Package Snapshot |
| Packages: Read | `READ_PACKAGES` | `GET /packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/search` | Get package references metadata |
| Packages: Read | `READ_PACKAGES` | `GET /packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel` | Recipe Snapshot |
| Packages: Read | `READ_PACKAGES` | `GET /packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name` | Download recipe files |
| Packages: Read | `READ_PACKAGES` | `GET /packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name` | Download package files |
@ -183,6 +184,7 @@ The following endpoints are available for CI/CD job tokens.
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference/digest` | Package Digest |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference/download_urls` | Package Download Urls |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference` | Package Snapshot |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel/search` | Get package references metadata |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel` | Recipe Snapshot |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name` | Download recipe files |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name` | Download package files |
@ -193,6 +195,7 @@ The following endpoints are available for CI/CD job tokens.
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v2/conans/:package_name/:package_version/:package_username/:package_channel/revisions/:recipe_revision/packages/:conan_package_reference/revisions/:package_revision/files/:file_name` | Download package files |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v2/conans/:package_name/:package_version/:package_username/:package_channel/revisions/:recipe_revision/packages/:conan_package_reference/revisions/:package_revision/files` | List package files |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v2/conans/:package_name/:package_version/:package_username/:package_channel/revisions` | Get the list of revisions |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/conan/v2/conans/:package_name/:package_version/:package_username/:package_channel/search` | Get package references metadata |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/generic/:package_name/*package_version/(*path/):file_name` | Download package file |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/go/*module_name/@v/:module_version.info` | Version metadata |
| Packages: Read | `READ_PACKAGES` | `GET /projects/:id/packages/go/*module_name/@v/:module_version.mod` | Download module file |

2
doc/development/database/_index.md
View File

@ -62,7 +62,7 @@ title: Database development guidelines
## Best practices
- [Adding database indexes](adding_database_indexes.md)
- [Adding a foreign key constraint to an existing column](add_foreign_key_to_existing_column.md)
- [Adding Foreign key constraints without downtime](foreign_keys.md#avoiding-downtime-and-migration-failures)
- [Compatiblity with Cells](../cells/_index.md)
- [Check for background migrations before upgrading](../../update/background_migrations.md)
- [Client-side connection-pool](client_side_connection_pool.md)

267
doc/development/database/add_foreign_key_to_existing_column.md
View File

@ -1,264 +1,13 @@
---
stage: Data Access
group: Database Frameworks
info: Any user with at least the Maintainer role can merge updates to this content. For details, see https://docs.gitlab.com/development/development_processes/#development-guidelines-review.
title: Add a foreign key constraint to an existing column
redirect_to: 'foreign_keys.md'
remove_date: '2025-03-28'
---
Foreign keys ensure consistency between related database tables. The current database review process **always** encourages you to add [foreign keys](foreign_keys.md) when creating tables that reference records from other tables.
<!-- markdownlint-disable -->
Starting with Rails version 4, Rails includes migration helpers to add foreign key constraints
to database tables. Before Rails 4, the only way for ensuring some level of consistency was the
[`dependent`](https://guides.rubyonrails.org/association_basics.html#options-for-belongs-to-dependent)
option in the association definition. Ensuring data consistency on the application level could fail
in some unfortunate cases, so we might end up with inconsistent data in the table. This mostly affects
older tables, where we didn't have the framework support to ensure consistency on the database level.
These data inconsistencies can cause unexpected application behavior or bugs.
This document was moved to [another location](foreign_keys.md).
Adding a foreign key to an existing database column requires database structure changes and potential data changes. In case the table is in use, we should always assume that there is inconsistent data.
To add a foreign key constraint to an existing column:
1. GitLab version `N.M`: Add a `NOT VALID` foreign key constraint to the column to ensure GitLab doesn't create inconsistent records.
1. GitLab version `N.M`: Add a data migration, to fix or clean up existing records.
1. GitLab version `N.M+1`: Validate the whole table by making the foreign key `VALID`.
## Example
Consider the following table structures:
`users` table:
- `id` (integer, primary key)
- `name` (string)
`emails` table:
- `id` (integer, primary key)
- `user_id` (integer)
- `email` (string)
Express the relationship in `ActiveRecord`:
```ruby
class User < ActiveRecord::Base
has_many :emails
end
class Email < ActiveRecord::Base
belongs_to :user
end
```
Problem: when the user is removed, the email records related to the removed user stays in the `emails` table:
```ruby
user = User.find(1)
user.destroy
emails = Email.where(user_id: 1) # returns emails for the deleted user
```
### Prevent invalid records
Add a `NOT VALID` foreign key constraint to the table, which enforces consistency on the record changes.
In the example above, you'd be still able to update records in the `emails` table. However, when you'd try to update the `user_id` with non-existent value, the constraint causes a database error.
Migration file for adding `NOT VALID` foreign key:
```ruby
class AddNotValidForeignKeyToEmailsUser < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
def up
add_concurrent_foreign_key :emails, :users, column: :user_id, on_delete: :cascade, validate: false
end
def down
remove_foreign_key_if_exists :emails, column: :user_id
end
end
```
Adding a foreign key without validating it is a fast operation. It only requires a
short lock on the table before being able to enforce the constraint on new data.
We do still want to enable lock retries for high traffic and large tables.
`add_concurrent_foreign_key` does this for us, and also checks if the foreign key already exists.
#### Data migration to fix existing records
The approach here depends on the data volume and the cleanup strategy. If we can find "invalid"
records by doing a database query and the record count is not high, then the data migration can
be executed in a Rails migration.
In case the data volume is higher (>1000 records), it's better to create a background migration. If unsure, contact the database team for advice.
Example for cleaning up records in the `emails` table in a database migration:
```ruby
class RemoveRecordsWithoutUserFromEmailsTable < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
class Email < ActiveRecord::Base
include EachBatch
end
def up
Email.where('user_id NOT IN (SELECT id FROM users)').each_batch do |relation|
relation.delete_all
end
end
def down
# Can be a no-op when data inconsistency is not affecting the pre and post deployment version of the application.
# In this case we might have records in the `emails` table where the associated record in the `users` table is not there anymore.
end
end
```
### Validate the foreign key
Validating the foreign key scans the whole table and makes sure that each relation is correct.
Fortunately, this does not lock the source table (`users`) while running.
{{< alert type="note" >}}
When using [batched background migrations](batched_background_migrations.md), foreign key validation should happen in the next GitLab release.
{{< /alert >}}
Migration file for validating the foreign key:
```ruby
# frozen_string_literal: true
class ValidateForeignKeyOnEmailUsers < Gitlab::Database::Migration[2.1]
def up
validate_foreign_key :emails, :user_id
end
def down
# Can be safely a no-op if we don't roll back the inconsistent data.
end
end
```
### Validate the foreign key asynchronously
For very large tables, foreign key validation can be a challenge to manage when
it runs for many hours. Necessary database operations like `autovacuum` cannot
run, and on GitLab.com, the deployment process is blocked waiting for the
migrations to finish.
To limit impact on GitLab.com, a process exists to validate them asynchronously
during weekend hours. Due to generally lower traffic and fewer deployments,
FK validation can proceed at a lower level of risk.
#### Schedule foreign key validation for a low-impact time
1. [Schedule the FK to be validated](#schedule-the-fk-to-be-validated).
1. [Verify the MR was deployed and the FK is valid in production](#verify-the-mr-was-deployed-and-the-fk-is-valid-in-production).
1. [Add a migration to validate the FK synchronously](#add-a-migration-to-validate-the-fk-synchronously).
#### Schedule the FK to be validated
1. Create a merge request containing a post-deployment migration, which prepares
the foreign key for asynchronous validation.
1. Create a follow-up issue to add a migration that validates the foreign key
synchronously.
1. In the merge request that prepares the asynchronous foreign key, add a
comment mentioning the follow-up issue.
An example of validating the foreign key using the asynchronous helpers can be
seen in the block below. This migration enters the foreign key name into the
`postgres_async_foreign_key_validations` table. The process that runs on
weekends pulls foreign keys from this table and attempts to validate them.
```ruby
# in db/post_migrate/
FK_NAME = :fk_be5624bf37
# TODO: FK to be validated synchronously in issue or merge request
def up
# `some_column` can be an array of columns, and is not mandatory if `name` is supplied.
# `name` takes precedence over other arguments.
prepare_async_foreign_key_validation :ci_builds, :some_column, name: FK_NAME
# Or in case of partitioned tables, use:
prepare_partitioned_async_foreign_key_validation :p_ci_builds, :some_column, name: FK_NAME
end
def down
unprepare_async_foreign_key_validation :ci_builds, :some_column, name: FK_NAME
# Or in case of partitioned tables, use:
unprepare_partitioned_async_foreign_key_validation :p_ci_builds, :some_column, name: FK_NAME
end
```
#### Verify the MR was deployed and the FK is valid in production
1. Verify that the post-deploy migration was executed on GitLab.com using ChatOps with
`/chatops run auto_deploy status <merge_sha>`. If the output returns `db/gprd`,
the post-deploy migration has been executed in the production database. For more information, see
[How to determine if a post-deploy migration has been executed on GitLab.com](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom).
1. Wait until the next week so that the FK can be validated over a weekend.
1. Use [Database Lab](database_lab.md) to check if validation was successful.
Ensure the output does not indicate the foreign key is `NOT VALID`.
#### Add a migration to validate the FK synchronously
After the foreign key is valid on the production database, create a second
merge request that validates the foreign key synchronously. The schema changes
must be updated and committed to `structure.sql` in this second merge request.
The synchronous migration results in a no-op on GitLab.com, but you should still
add the migration as expected for other installations. The below block
demonstrates how to create the second migration for the previous
asynchronous example.
{{< alert type="warning" >}}
Verify that the foreign key is valid in production before merging a second
migration with `validate_foreign_key`. If the second migration is deployed
before the validation has been executed, the foreign key is validated
synchronously when the second migration executes.
{{< /alert >}}
```ruby
# in db/post_migrate/
FK_NAME = :fk_be5624bf37
def up
validate_foreign_key :ci_builds, :some_column, name: FK_NAME
end
def down
# Can be safely a no-op if we don't roll back the inconsistent data.
end
end
```
### Test database FK changes locally
You must test the database foreign key changes locally before creating a merge request.
#### Verify the foreign keys validated asynchronously
Use the asynchronous helpers on your local environment to test changes for
validating a foreign key:
1. Enable the feature flag by running `Feature.enable(:database_async_foreign_key_validation)`
in the Rails console.
1. Run `bundle exec rails db:migrate` so that it creates an entry in the async validation table.
1. Run `bundle exec rails gitlab:db:validate_async_constraints:all` so that the FK is validated
asynchronously on all databases.
1. To verify the foreign key, open the PostgreSQL console using the
[GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md)
command `gdk psql` and run the command `\d+ table_name` to check that your
foreign key is valid. A successful validation removes `NOT VALID` from
the foreign key definition.
<!-- This redirect file can be deleted after <2025-06-29>. -->
<!-- Redirects that point to other docs in the same project expire in three months. -->
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
<!-- Before deletion, see: https://docs.gitlab.com/development/documentation/redirects -->

19
doc/development/database/avoiding_downtime_in_migrations.md
View File

@ -352,7 +352,7 @@ the whole column type.
You can check the following guides for each specific use case:
- [Adding foreign-key constraints](../migration_style_guide.md#adding-foreign-key-constraints)
- [Adding foreign-key constraints](foreign_keys.md)
- [Adding `NOT NULL` constraints](not_null_constraints.md)
- [Adding limits to text columns](strings_and_the_text_data_type.md)
@ -579,22 +579,7 @@ Renaming a table is possible without downtime by following our multi-release
## Adding foreign keys
Adding foreign keys usually works in 3 steps:
1. Start a transaction
1. Run `ALTER TABLE` to add the constraints
1. Check all existing data
Because `ALTER TABLE` typically acquires an exclusive lock until the end of a
transaction this means this approach would require downtime.
GitLab allows you to work around this by using
`Gitlab::Database::MigrationHelpers#add_concurrent_foreign_key`. This method
ensures that no downtime is needed.
## Removing foreign keys
This operation does not require downtime.
Adding foreign keys can potentially cause downtime, please refer [FK: Avoiding downtime and migration failures](foreign_keys.md#avoiding-downtime-and-migration-failures) docs for details.
## Migrating `integer` primary keys to `bigint`

356
doc/development/database/foreign_keys.md
View File

@ -5,13 +5,19 @@ info: Any user with at least the Maintainer role can merge updates to this conte
title: Foreign keys and associations
---
When adding an association to a model you must also add a foreign key. When
adding a foreign key you must always add an [index](#indexes) first.
Foreign keys ensure consistency between related database tables. Starting with Rails version 4, Rails includes migration helpers to add foreign key constraints
to database tables. Before Rails 4, the only way for ensuring some level of consistency was the
[`dependent`](https://guides.rubyonrails.org/association_basics.html#options-for-belongs-to-dependent)
option in the association definition.
If the [index must be created async](adding_database_indexes.md#create-indexes-asynchronously)
due to duration reasons, you must avoid adding the foreign key until the index
gets created. To ensure data consistency, you must avoid using the new
association in rails until the foreign key and index both get created.
Ensuring data consistency on the application level could fail
in some unfortunate cases, so we might end up with inconsistent data in the table. This mostly affects
older tables, where we didn't have the framework support to ensure consistency on the database level.
These data inconsistencies can cause unexpected application behavior or bugs.
When creating tables that reference records from other tables, a FK should be added to maintain data integrity.
And when adding an association to a model you must also add a foreign key. Also on
adding a foreign key you must always add an [index](#indexes) first.
For example, say you have the following model:
@ -23,19 +29,339 @@ end
Add a foreign key here on column `posts.user_id`. This ensures
that data consistency is enforced on database level. Foreign keys also mean that
the database can very quickly remove associated data (for example, when removing a
the database can remove associated data (for example, when removing a
user), instead of Rails having to do this.
## Adding foreign keys in migrations
## Avoiding downtime and migration failures
Foreign keys can be added concurrently using `add_concurrent_foreign_key` as
defined in `Gitlab::Database::MigrationHelpers`. See the
[Migration Style Guide](../migration_style_guide.md) for more information.
Adding a foreign key has two parts to it
Keep in mind that you can only safely add foreign keys to existing tables after
you have removed any orphaned rows. The method `add_concurrent_foreign_key`
does not take care of this so you must do so manually. See
[adding foreign key constraint to an existing column](add_foreign_key_to_existing_column.md).
1. Adding the FK column and the constraint.
1. Validating the added constraint to maintain data integrity.
(1) uses ALTER TABLE statements which takes the most strict lock (ACCESS EXCLUSIVE) and validating the constraint has to
traverse the entire table which will be time consuming for large/high-traffic tables.
So in almost all cases we have to run them in separate transactions to avoid holding the
stricter lock and blocking other operations on the tables for a longer time.
### On a new column
If the FK is added while creating the table, it is straight forward and
`create_table (t.references, ..., foreign_key: true)` can be used.
If you have a new (without much records) or empty table that doesn't reference a
[high-traffic table](../migration_style_guide.md#high-traffic-tables), either of below approaches can be used.
1. add_reference(... foreign_key: true)
1. add_column(...) and add_foreign_key(...) in the same transaction.
For all other cases, adding the column, adding FK constraint and validating the constraint should be done in
separate transactions.
### On an existing column
Adding a foreign key to an existing database column requires database structure changes and potential data changes.
{{< alert type="note" >}}
In case the table is in use, we should always assume that there is inconsistent data.
{{< /alert >}}
Adding a FK constraint to an existing column is a multi-milestone process:
1. `N.M`: Add a `NOT VALID` FK constraint to the column, it will also ensure there are no inconsistent records created or updated.
1. `N.M`: Add a data migration, to fix or clean up existing records.
2. This can be a regular or post deployment migration if the migration queries lie within the [timing guidelines](query_performance.md).
3. If not, this has to be done in a [batched background migration](batched_background_migrations.md).
1. Validate the FK constraint
2. If the data migration was a regular or a post deployment migration, the constraint can be validated in the same milestone.
3. If it was a background migration, then the FK can be validated only after the BBM is finalized.
This is required so that the FK validation won't happen while the data migration is still running in background.
{{< alert type="note" >}}
Adding a foreign-key constraint to either an existing or a new column
needs an index on the column.
If the index was added [asynchronously](adding_database_indexes.md#create-indexes-asynchronously), we should wait till
the index gets added in the `structure.sql`.
{{< /alert >}}
This is **required** for all foreign-keys, for example, to support efficient cascading
deleting: when a lot of rows in a table get deleted, the referenced records need
to be deleted too. The database has to look for corresponding records in the
referenced table. Without an index, this results in a sequential scan on the
table, which can take a long time.
#### Example
Consider the following table structures:
`users` table:
- `id` (integer, primary key)
- `name` (string)
`emails` table:
- `id` (integer, primary key)
- `user_id` (integer)
- `email` (string)
Express the relationship in `ActiveRecord`:
```ruby
class User < ActiveRecord::Base
has_many :emails
end
class Email < ActiveRecord::Base
belongs_to :user
end
```
Problem: when the user is removed, the email records related to the removed user stays in the `emails` table:
```ruby
user = User.find(1)
user.destroy
emails = Email.where(user_id: 1) # returns emails for the deleted user
```
#### Adding the FK constraint (NOT VALID)
Add a `NOT VALID` foreign key constraint to the table, which enforces consistency on adding or updating records.
In the example above, you will still be able to update records in the `emails` table. However, when you try to update `user_id` with non-existent value, the constraint will throw an error.
Migration file for adding `NOT VALID` foreign key:
```ruby
class AddNotValidForeignKeyToEmailsUser < Gitlab::Database::Migration[2.1]
milestone '17.10'
disable_ddl_transaction!
def up
add_concurrent_foreign_key(
:emails,
:users,
column: :user_id,
on_delete: :cascade,
validate: false
)
end
def down
remove_foreign_key_if_exists :emails, column: :user_id
end
end
```
INFO:
By default _add_concurrent_foreign_key_ method validates the foreign key, so explicitly pass `validate: false`.
Adding a foreign key without validating it is a fast operation. It only requires a
short lock on the table before being able to enforce the constraint on new data.
Also `add_concurrent_foreign_key` will add the constraint only if it's not existing.
{{< alert type="warning" >}}
Avoid using `add_foreign_key` or `add_concurrent_foreign_key` constraints more than
once per migration file, unless the source and target tables are identical.
{{< /alert >}}
#### Data migration to fix existing records
The approach here depends on the data volume and the cleanup strategy. If we can find "invalid"
records by doing a database query and the record count is not high, then the data migration can
be executed in regular or post deployment rails migration.
In case the data volume is higher (>1000 records), it's better to create a background migration. If unsure, refer to our [query guidelines](query_performance.md) or contact the database frameworks team for advice.
Example for cleaning up records in the `emails` table in a database migration:
```ruby
class RemoveRecordsWithoutUserFromEmailsTable < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
class Email < ActiveRecord::Base
include EachBatch
end
def up
Email.each_batch do |batch|
batch.joins('LEFT JOIN users ON emails.user_id = users.id')
.where('users.id IS NULL')
.delete_all
end
end
def down
# Can be a no-op when data inconsistency is not affecting the pre and post deployment version of the application.
# In this case we might have records in the `emails` table where the associated record in the `users` table is not there anymore.
end
end
```
{{< alert type="note" >}}
The MR that adds this data migration should have ~data-deletion label applied.
Refer [preparation-when-adding-data-migrations](../database_review.md#preparation-when-adding-data-migrations) for more information.
{{< /alert >}}
#### Validate the foreign key
Validating the foreign key scans the whole table and makes sure that each relation is correct.
Fortunately, this does not lock the source table (`users`) while running.
As aforementioned when using [batched background migrations](batched_background_migrations.md), foreign key validation should happen only after the BBM is finalized.
Migration file for validating the foreign key:
```ruby
# frozen_string_literal: true
class ValidateForeignKeyOnEmailUsers < Gitlab::Database::Migration[2.1]
def up
validate_foreign_key :emails, :user_id
end
def down
# Can be safely a no-op if we don't roll back the inconsistent data.
end
end
```
#### Validate the foreign key asynchronously
For very large tables, foreign key validation can be a challenge to manage when
it runs for many hours. Necessary database operations like `autovacuum` cannot
run, and on GitLab.com, the deployment process is blocked waiting for the
migrations to finish.
To limit impact on GitLab.com, a process exists to validate them asynchronously
during weekend hours. Due to generally lower traffic and fewer deployments,
FK validation can proceed at a lower level of risk.
##### Schedule foreign key validation for a low-impact time
1. [Schedule the FK to be validated](#schedule-the-fk-to-be-validated).
1. [Verify the MR was deployed and the FK is valid in production](#verify-the-mr-was-deployed-and-the-fk-is-valid-in-production).
1. [Add a migration to validate the FK synchronously](#add-a-migration-to-validate-the-fk-synchronously).
##### Schedule the FK to be validated
1. Create a merge request containing a post-deployment migration, which prepares
the foreign key for asynchronous validation.
1. Create a follow-up issue to add a migration that validates the foreign key
synchronously.
1. In the merge request that prepares the asynchronous foreign key, add a
comment mentioning the follow-up issue.
An example of validating the foreign key using the asynchronous helpers can be
seen in the block below. This migration enters the foreign key name into the
`postgres_async_foreign_key_validations` table. The process that runs on
weekends pulls foreign keys from this table and attempts to validate them.
```ruby
# in db/post_migrate/
FK_NAME = :fk_be5624bf37
# TODO: FK to be validated synchronously in issue or merge request
def up
# `some_column` can be an array of columns, and is not mandatory if `name` is supplied.
# `name` takes precedence over other arguments.
prepare_async_foreign_key_validation :ci_builds, :some_column, name: FK_NAME
# Or in case of partitioned tables, use:
prepare_partitioned_async_foreign_key_validation :p_ci_builds, :some_column, name: FK_NAME
end
def down
unprepare_async_foreign_key_validation :ci_builds, :some_column, name: FK_NAME
# Or in case of partitioned tables, use:
unprepare_partitioned_async_foreign_key_validation :p_ci_builds, :some_column, name: FK_NAME
end
```
##### Verify the MR was deployed and the FK is valid in production
1. Verify that the post-deploy migration was executed on GitLab.com using ChatOps with
`/chatops run auto_deploy status <merge_sha>`. If the output returns `db/gprd`,
the post-deploy migration has been executed in the production database. For more information, see
[How to determine if a post-deploy migration has been executed on GitLab.com](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom).
1. Wait until the next week so that the FK can be validated over a weekend.
1. Use [Database Lab](database_lab.md) to check if validation was successful.
Ensure the output does not indicate the foreign key is `NOT VALID`.
##### Add a migration to validate the FK synchronously
After the foreign key is valid on the production database, create a second
merge request that validates the foreign key synchronously. The schema changes
must be updated and committed to `structure.sql` in this second merge request.
The synchronous migration results in a no-op on GitLab.com, but you should still
add the migration as expected for other installations. The below block
demonstrates how to create the second migration for the previous
asynchronous example.
{{< alert type="warning" >}}
Verify that the foreign key is valid in production before merging a second
migration with `validate_foreign_key`. If the second migration is deployed
before the validation has been executed, the foreign key is validated
synchronously when the second migration executes.
{{< /alert >}}
```ruby
# in db/post_migrate/
FK_NAME = :fk_be5624bf37
def up
validate_foreign_key :ci_builds, :some_column, name: FK_NAME
end
def down
# Can be safely a no-op if we don't roll back the inconsistent data.
end
end
```
#### Test database FK changes locally
You must test the database foreign key changes locally before creating a merge request.
##### Verify the foreign keys validated asynchronously
Use the asynchronous helpers on your local environment to test changes for
validating a foreign key:
1. Enable the feature flag by running `Feature.enable(:database_async_foreign_key_validation)`
in the Rails console.
1. Run `bundle exec rails db:migrate` so that it creates an entry in the async validation table.
1. Run `bundle exec rails gitlab:db:validate_async_constraints:all` so that the FK is validated
asynchronously on all databases.
1. To verify the foreign key, open the PostgreSQL console using the
[GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md)
command `gdk psql` and run the command `\d+ table_name` to check that your
foreign key is valid. A successful validation removes `NOT VALID` from
the foreign key definition.
### Removing foreign keys
This operation does not require downtime.
## Use `bigint` for foreign keys

4
doc/development/database_review.md
View File

@ -230,7 +230,7 @@ Include in the MR description:
#### Preparation when adding tables
- Order columns based on the [Ordering Table Columns](database/ordering_table_columns.md) guidelines.
- Add foreign keys to any columns pointing to data in other tables, including [an index](migration_style_guide.md#adding-foreign-key-constraints).
- Add foreign keys to any columns pointing to data in other tables, including [an index](database/foreign_keys.md).
- Add indexes for fields that are used in statements such as `WHERE`, `ORDER BY`, `GROUP BY`, and `JOIN`s.
- New tables must be seeded by a file in `db/fixtures/development/`. These fixtures are also used
to ensure that [upgrades complete successfully](database/dbmigrate_multi_version_upgrade_job.md),
@ -271,7 +271,7 @@ to add the raw SQL query and query plan to the merge request description, and re
- Review migrations follow [database migration style guide](migration_style_guide.md),
for example
- [Check ordering of columns](database/ordering_table_columns.md)
- [Check indexes are present for foreign keys](migration_style_guide.md#adding-foreign-key-constraints)
- [Check indexes are present for foreign keys](database/foreign_keys.md)
- Ensure that migrations execute in a transaction or only contain
concurrent index/foreign key helpers (with transactions disabled)
- If an index to a large table is added and its execution time was elevated (more than 1h) on [Database Lab](database/database_lab.md):

36
doc/development/migration_style_guide.md
View File

@ -417,7 +417,7 @@ because they require a precise control on when and how to open transactions.
- A foreign key _can_ be added inside a transaction, unlike `CREATE INDEX CONCURRENTLY`.
However, PostgreSQL does not provide an option similar to `CREATE INDEX CONCURRENTLY`.
The helper [`add_concurrent_foreign_key`](database/foreign_keys.md#adding-foreign-keys-in-migrations)
The helper [`add_concurrent_foreign_key`](database/foreign_keys.md#adding-the-fk-constraint-not-valid)
instead opens its own transactions to lock the source and target table
in a manner that minimizes locking while adding and validating the foreign key.
- As advised earlier, skip `disable_ddl_transaction!` if you are unsure
@ -871,40 +871,6 @@ If a migration requires conditional logic based on the absence or presence of an
For more details, review the [Adding Database Indexes](database/adding_database_indexes.md#testing-for-existence-of-indexes)
guide.
## Adding foreign-key constraints
When adding a foreign-key constraint to either an existing or a new column also
remember to add an index on the column.
This is **required** for all foreign-keys, for example, to support efficient cascading
deleting: when a lot of rows in a table get deleted, the referenced records need
to be deleted too. The database has to look for corresponding records in the
referenced table. Without an index, this results in a sequential scan on the
table, which can take a long time.
Here's an example where we add a new column with a foreign key
constraint. Note it includes `index: true` to create an index for it.
```ruby
class Migration < Gitlab::Database::Migration[2.1]
def change
add_reference :model, :other_model, index: true, foreign_key: { on_delete: :cascade }
end
end
```
When adding a foreign-key constraint to an existing column in a non-empty table,
we have to employ `add_concurrent_foreign_key` and `add_concurrent_index`
instead of `add_reference`.
If you have a new or empty table that doesn't reference a
[high-traffic table](#high-traffic-tables),
we recommend that you use `add_reference` in a single-transaction migration. You can
combine it with other operations that don't require `disable_ddl_transaction!`.
You can read more about adding [foreign key constraints to an existing column](database/add_foreign_key_to_existing_column.md).
## `NOT NULL` constraints
See the style guide on [`NOT NULL` constraints](database/not_null_constraints.md) for more information.

67
doc/user/packages/conan_repository/_index.md
View File

@ -358,6 +358,51 @@ The `conan info` command returns information about a package:
conan info Hello/0.1@mycompany/beta
```
## Download a Conan package
{{< alert type="flag" >}}
Packages uploaded before [Conan info metadata extraction](#extract-conan-metadata) was enabled cannot be downloaded with the `conan download` CLI command.
{{< /alert >}}
You can download a Conan package's recipe and binaries to your local cache without using settings that use the `conan download` command.
Prerequisites:
- The Conan remote [must be configured](#add-the-package-registry-as-a-conan-remote).
- For private and internal projects, you must configure [authentication](#authenticate-to-the-package-registry) with the package registry.
### Download all binary packages
You can download all binary packages associated with a recipe from the package registry.
To download all binary packages, run the following command:
```shell
conan download Hello/0.1@foo+bar/stable --remote=gitlab
```
### Download recipe files
You can download only the recipe files without any binary packages.
To download recipe files, run the following command:
```shell
conan download Hello/0.1@foo+bar/stable --remote=gitlab --recipe
```
### Download a specific binary package
You can download a single binary package by referencing its package reference (known as the `package_id` in Conan documentation).
To download a specific binary package, run the following command:
```shell
conan download Hello/0.1@foo+bar/stable:<package_reference> --remote=gitlab
```
## Supported CLI commands
The GitLab Conan repository supports the following Conan CLI commands:
@ -365,11 +410,33 @@ The GitLab Conan repository supports the following Conan CLI commands:
- `conan upload`: Upload your recipe and package files to the package registry.
- `conan install`: Install a Conan package from the package registry, which
includes using the `conanfile.txt` file.
- `conan download`: Download package recipes and binaries to your local cache without using settings.
- `conan search`: Search the package registry for public packages, and private
packages you have permission to view.
- `conan info`: View the information on a given package from the package registry.
- `conan remove`: Delete the package from the package registry.
## Extract Conan metadata
{{< history >}}
- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/178728) in GitLab 17.10 [with a flag](../../../administration/feature_flags.md) named `parse_conan_metadata_on_upload`. Disabled by default.
- [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/186292) in GitLab 17.11. Feature flag `parse_conan_metadata_on_upload` removed.
{{< /history >}}
When you upload a Conan package, GitLab automatically extracts metadata from the `conaninfo.txt` file. This metadata includes:
- Package settings (like `os`, `arch`, `compiler` and `build_type`)
- Package options
- Package requirements and dependencies
{{< alert type="note" >}}
Packages uploaded before this feature was enabled (GitLab 17.10) do not have their metadata extracted. For these packages, some search and download functionalities are limited.
{{< /alert >}}
## Troubleshooting
### Make output verbose

1
doc/user/project/settings/import_export.md
View File

@ -235,6 +235,7 @@ Items that are **not** exported include:
- Build traces and artifacts
- Package and container registry images
- CI/CD variables
- CI/CD job token allowlist
- Webhooks
- Any encrypted tokens
- [Number of required approvals](https://gitlab.com/gitlab-org/gitlab/-/issues/221087)

80
lib/api/concerns/packages/conan/shared_endpoints.rb
View File

@ -94,28 +94,72 @@ module API
end
end
desc 'Search for packages' do
detail 'This feature was introduced in GitLab 12.4'
success code: 200
failure [
{ code: 400, message: 'Bad Request' },
{ code: 404, message: 'Not Found' }
]
tags %w[conan_packages]
end
namespace 'conans' do
desc 'Search for packages' do
detail 'This feature was introduced in GitLab 12.4'
success code: 200
failure [
{ code: 400, message: 'Bad Request' },
{ code: 404, message: 'Not Found' }
]
tags %w[conan_packages]
end
params do
requires :q, type: String, desc: 'Search query', documentation: { example: 'Hello*' }
end
params do
requires :q, type: String, desc: 'Search query', documentation: { example: 'Hello*' }
end
route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
route_setting :authorization, skip_job_token_policies: true
route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
route_setting :authorization, skip_job_token_policies: true
get 'conans/search', urgency: :low do
response = ::Packages::Conan::SearchService.new(search_project, current_user, query: params[:q]).execute
bad_request!(response.message) if response.error?
get 'search', urgency: :low do
response = ::Packages::Conan::SearchService.new(search_project, current_user, query: params[:q]).execute
bad_request!(response.message) if response.error?
response.payload
response.payload
end
params do
with(type: String) do
with(regexp: PACKAGE_COMPONENT_REGEX) do
requires :package_name, desc: 'Package name', documentation: { example: 'my-package' }
requires :package_version, desc: 'Package version', documentation: { example: '1.0' }
end
with(regexp: CONAN_REVISION_USER_CHANNEL_REGEX) do
requires :package_username, desc: 'Package username',
documentation: { example: 'my-group+my-project' }
requires :package_channel, desc: 'Package channel', documentation: { example: 'stable' }
end
end
end
namespace ':package_name/:package_version/:package_username/:package_channel/search',
requirements: PACKAGE_REQUIREMENTS do
desc 'Get package references metadata' do
detail 'This feature was introduced in GitLab 18.0'
success code: 200
failure [
{ code: 400, message: 'Bad Request' },
{ code: 401, message: 'Unauthorized' },
{ code: 403, message: 'Forbidden' },
{ code: 404, message: 'Not Found' }
]
tags %w[conan_packages]
end
route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
route_setting :authorization, job_token_policies: :read_packages,
allow_public_access_for_enabled_project_features: :package_registry
get urgency: :low do
check_username_channel
authorize_read_package!(project)
not_found!('Package') unless package
package.conan_package_references.pluck_reference_and_info.to_h
end
end
end
end
end

10
lib/api/generic_packages.rb
View File

@ -92,12 +92,16 @@ module API
file_name: encoded_file_name,
build: current_authenticated_job
)
package_file = ::Packages::Generic::CreatePackageFileService
response = ::Packages::Generic::CreatePackageFileService
.new(project, current_user, create_package_file_params)
.execute
if response.error? && response.cause.package_file_already_exists?
bad_request!('Duplicate package is not allowed')
end
if params[:select] == 'package_file'
present package_file
present response[:package_file]
else
created!
end
@ -105,8 +109,6 @@ module API
Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id })
forbidden!
rescue ::Packages::DuplicatePackageError
bad_request!('Duplicate package is not allowed')
end
desc 'Download package file' do

3
locale/gitlab.pot
View File

@ -23161,9 +23161,6 @@ msgstr ""
msgid "DuoCodeReview|I have encountered some problems while I was reviewing. Please try again later."
msgstr ""
msgid "DuoCodeReview|I've received your Duo Code Review request, and will review your code shortly."
msgstr ""
msgid "DuoCodeReview|Your account doesn't have GitLab Duo access. Please contact your system administrator for access."
msgstr ""

9
spec/factories/packages/conan/packages.rb
View File

@ -23,7 +23,14 @@ FactoryBot.define do
conan_package_references do
next [] if without_package_references
[association(:conan_package_reference, package: instance, recipe_revision: instance.conan_recipe_revisions.first)]
[association(
:conan_package_reference,
package: instance,
recipe_revision: instance.conan_recipe_revisions.first,
info: Gitlab::Json.parse(
File.read(Rails.root.join('spec/fixtures/packages/conan/parsed_conaninfo/conaninfo.json'))
)
)]
end
conan_package_revisions do

2
spec/lib/gitlab/ci/build/prerequisite/managed_resource_spec.rb
View File

@ -308,7 +308,7 @@ RSpec.describe Gitlab::Ci::Build::Prerequisite::ManagedResource, feature_categor
end
it 'tracks the error and creates the managed resource record with the failed status' do
error_message = 'Failed to ensure the environment. {"object":{"group":"group","apiVersion":"version",' \
error_message = 'Failed to ensure the environment. {"object":{"group":"group","version":"version",' \
'"kind":"kind","namespace":"namespace","name":"name"},"error":"error message"}'
expect { execute_complete }.to raise_error(
Gitlab::Ci::Build::Prerequisite::ManagedResource::ManagedResourceError, error_message)

16
spec/models/packages/conan/package_reference_spec.rb
View File

@ -166,4 +166,20 @@ RSpec.describe Packages::Conan::PackageReference, type: :model, feature_category
it { is_expected.to be_empty }
end
end
describe '.pluck_reference_and_info' do
let_it_be(:package_references) { create_list(:conan_package_reference, 2) }
subject { described_class.id_in(package_references.map(&:id)).pluck_reference_and_info }
it { is_expected.to match_array(package_references.map { |pr| [pr.reference, pr.info] }) }
context 'when there are more records than MAX_PLUCK' do
before do
stub_const('ApplicationRecord::MAX_PLUCK', 1)
end
it { is_expected.to have_attributes(size: 1) }
end
end
end

10
spec/requests/api/conan/v1/instance_packages_spec.rb
View File

@ -45,9 +45,9 @@ RSpec.describe API::Conan::V1::InstancePackages, feature_category: :package_regi
context 'with recipe endpoints' do
let(:project_id) { 9999 }
let(:url_prefix) { "#{Settings.gitlab.base_url}/api/v4" }
let(:recipe_path) { package.conan_recipe_path }
describe 'GET /api/v4/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel' do
let(:recipe_path) { package.conan_recipe_path }
let(:url) { "/packages/conan/v1/conans/#{recipe_path}" }
it_behaves_like 'recipe snapshot endpoint'
@ -55,7 +55,6 @@ RSpec.describe API::Conan::V1::InstancePackages, feature_category: :package_regi
describe 'GET /api/v4/packages/conan/v1/conans/:package_name/:package_version/:package_username/:package_channel' \
'/packages/:conan_package_reference' do
let(:recipe_path) { package.conan_recipe_path }
let(:url) { "/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}" }
it_behaves_like 'package snapshot endpoint'
@ -121,6 +120,13 @@ RSpec.describe API::Conan::V1::InstancePackages, feature_category: :package_regi
it_behaves_like 'delete package endpoint'
end
describe 'GET /api/v4/packages/conan/v1/conans/:package_name/:package_version/:package_username' \
'/:package_channel/search' do
let(:url) { "/packages/conan/v1/conans/#{recipe_path}/search" }
it_behaves_like 'GET package references metadata endpoint'
end
end
context 'with file download endpoints' do

9
spec/requests/api/conan/v1/project_packages_spec.rb
View File

@ -145,6 +145,15 @@ RSpec.describe API::Conan::V1::ProjectPackages, feature_category: :package_regis
it_behaves_like 'delete package endpoint'
end
describe 'GET /api/v4/projects/:id/packages/conan/v1/conans/:package_name/:package_version/:package_username' \
'/:package_channel/search' do
let(:url) { "/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}/search" }
it_behaves_like 'GET package references metadata endpoint'
it_behaves_like 'accept get request on private project with access to package registry for everyone'
it_behaves_like 'project not found by project id'
end
end
context 'with file download endpoints' do

39
spec/requests/api/conan/v2/project_packages_spec.rb
View File

@ -18,14 +18,6 @@ RSpec.describe API::Conan::V2::ProjectPackages, feature_category: :package_regis
message: "404 'conan_package_revisions_support' feature flag is disabled Not Found"
end
shared_examples 'packages feature check' do
before do
stub_packages_setting(enabled: false)
end
it_behaves_like 'returning response status', :not_found
end
shared_examples 'get file list' do |expected_file_list, not_found_err:|
subject(:api_request) { get api(url), headers: headers }
@ -84,20 +76,21 @@ RSpec.describe API::Conan::V2::ProjectPackages, feature_category: :package_regis
let(:url) { "/projects/#{project.id}/packages/conan/v2/users/check_credentials" }
it_behaves_like 'conan check_credentials endpoint'
it_behaves_like 'conan package revisions feature flag check' do
subject { get api(url), headers: headers }
end
end
describe 'GET /api/v4/projects/:id/packages/conan/v2/conans/search' do
let(:url_suffix) { 'search' }
let(:url_suffix) { "search" }
let(:params) { { q: package.conan_recipe } }
subject { get api(url), params: params }
it_behaves_like 'conan search endpoint'
it_behaves_like 'conan FIPS mode' do
let(:params) { { q: package.conan_recipe } }
subject { get api(url), params: params }
end
it_behaves_like 'conan FIPS mode'
it_behaves_like 'conan search endpoint with access to package registry for everyone'
it_behaves_like 'conan package revisions feature flag check'
end
describe 'GET /api/v4/projects/:id/packages/conan/v2/conans/:package_name/:package_version/:package_username/' \
@ -307,6 +300,7 @@ RSpec.describe API::Conan::V2::ProjectPackages, feature_category: :package_regis
end
end
it_behaves_like 'conan package revisions feature flag check'
it_behaves_like 'enforcing read_packages job token policy'
it_behaves_like 'accept get request on private project with access to package registry for everyone'
it_behaves_like 'conan FIPS mode'
@ -428,4 +422,17 @@ RSpec.describe API::Conan::V2::ProjectPackages, feature_category: :package_regis
{ 'files' => { 'conan_package.tgz' => {}, 'conaninfo.txt' => {}, 'conanmanifest.txt' => {} } },
not_found_err: '404 Package files Not Found'
end
describe 'GET /api/v4/projects/:id/packages/conan/v2/conans/:package_name/:package_version/:package_username' \
'/:package_channel/search' do
let(:recipe_path) { package.conan_recipe_path }
let(:url_suffix) { "#{recipe_path}/search" }
subject(:request) { get api(url), headers: headers }
it_behaves_like 'GET package references metadata endpoint'
it_behaves_like 'accept get request on private project with access to package registry for everyone'
it_behaves_like 'project not found by project id'
it_behaves_like 'conan package revisions feature flag check'
end
end

40
spec/services/packages/generic/create_package_file_service_spec.rb
View File

@ -12,6 +12,8 @@ RSpec.describe Packages::Generic::CreatePackageFileService, feature_category: :p
let(:build) { double('build', pipeline: pipeline) }
let(:service) { described_class.new(project, user, params) }
describe '#execute' do
let_it_be(:package) { create(:generic_package, project: project) }
@ -39,7 +41,7 @@ RSpec.describe Packages::Generic::CreatePackageFileService, feature_category: :p
}
end
subject(:execute_service) { described_class.new(project, user, params).execute }
subject(:response) { service.execute }
before do
FileUtils.touch(temp_file)
@ -52,18 +54,26 @@ RSpec.describe Packages::Generic::CreatePackageFileService, feature_category: :p
end
shared_examples 'allows creating the file' do
it { expect { execute_service }.to change { project.package_files.count }.by(1) }
it_behaves_like 'returning a success service response'
it { expect { response }.to change { project.package_files.count }.by(1) }
end
shared_examples 'does not allow duplicates' do
it 'raises a duplicate package error' do
expect { execute_service }.to raise_error(::Packages::DuplicatePackageError)
.and change { project.package_files.count }.by(0)
it_behaves_like 'returning an error service response', message: 'Packages::DuplicatePackageError'
it { is_expected.to have_attributes reason: :package_file_already_exists }
it 'does not add new package file' do
expect { response }.to not_change { project.package_files.count }
.and not_change { project.packages.count }
end
end
it_behaves_like 'returning a success service response'
it 'creates package file', :aggregate_failures do
expect { execute_service }.to change { package.package_files.count }.by(1)
expect { response }.to change { package.package_files.count }.by(1)
.and change { Packages::PackageFileBuildInfo.count }.by(1)
package_file = package.package_files.last
@ -81,7 +91,7 @@ RSpec.describe Packages::Generic::CreatePackageFileService, feature_category: :p
let(:package_params) { super().merge(status: 'hidden') }
it 'updates an existing packages status' do
expect { execute_service }.to change { package.package_files.count }.by(1)
expect { response }.to change { package.package_files.count }.by(1)
.and change { Packages::PackageFileBuildInfo.count }.by(1)
package_file = package.package_files.last
@ -91,12 +101,14 @@ RSpec.describe Packages::Generic::CreatePackageFileService, feature_category: :p
end
end
it_behaves_like 'assigns build to package file'
it_behaves_like 'assigns build to package file' do
subject { super().payload.fetch(:package_file) }
end
context 'with existing package' do
let_it_be(:duplicate_file) { create(:package_file, package: package, file_name: file_name) }
it { expect { execute_service }.to change { project.package_files.count }.by(1) }
it { expect { response }.to change { project.package_files.count }.by(1) }
context 'when duplicates are not allowed' do
before do
@ -169,5 +181,15 @@ RSpec.describe Packages::Generic::CreatePackageFileService, feature_category: :p
end
end
end
context 'when unexpected error is raised' do
before do
allow(service).to receive(:create_package_file).and_raise(StandardError, 'Custom error')
end
it 'returns error instead of service response' do
expect { response }.to raise_error(StandardError, 'Custom error')
end
end
end
end

73
spec/support/shared_examples/requests/api/conan_packages_shared_examples.rb
View File

@ -1225,3 +1225,76 @@ RSpec.shared_examples 'package not found' do
end
end
end
RSpec.shared_examples 'packages feature check' do
before do
stub_packages_setting(enabled: false)
end
it_behaves_like 'returning response status', :not_found
end
RSpec.shared_examples 'GET package references metadata endpoint' do
subject(:request) { get api(url), headers: headers }
let_it_be(:reference1) { package.conan_package_references.first }
let_it_be(:reference2) do
create(:conan_package_reference, package: package, info: { 'settings' => { 'os' => 'Linux' } })
end
it_behaves_like 'conan FIPS mode'
it_behaves_like 'packages feature check'
it_behaves_like 'handling empty values for username and channel'
it_behaves_like 'package not found'
it_behaves_like 'enforcing read_packages job token policy'
context 'When the project is public' do
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel::PUBLIC)
end
it_behaves_like 'allows download with no token'
end
context 'When the project is internal' do
before do
project.team.truncate
project.update_column(:visibility_level, Gitlab::VisibilityLevel::INTERNAL)
end
it_behaves_like 'denies download with no token'
end
context 'When the project is private' do
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel::PRIVATE)
end
it 'returns success with package references metadata', :aggregate_failures do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to include(
reference1.reference => reference1.info,
reference2.reference => reference2.info
)
end
it_behaves_like 'denies download with no token'
context 'when allow_guest_plus_roles_to_pull_packages is disabled' do
before do
stub_feature_flags(allow_guest_plus_roles_to_pull_packages: false)
end
it 'denies download when not enough permissions' do
project.add_guest(user)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
end