From 2d239d2421eea466ff5c51aebdbf05a18ded20c7 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 29 Nov 2023 18:12:41 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/ci/rules.gitlab-ci.yml | 4 +- app/models/packages/nuget/symbol.rb | 7 ++ .../ml/create_model_version_service.rb | 25 ++++++ app/services/ml/increment_version_service.rb | 35 ++++++++ app/views/import/github/new.html.haml | 16 ++-- app/views/import/github/status.html.haml | 9 +- app/views/projects/blob/_editor.html.haml | 2 +- app/workers/all_queues.yml | 9 ++ .../cleanup_package_registry_worker.rb | 5 ++ .../nuget/cleanup_stale_symbols_worker.rb | 46 ++++++++++ ...endency_scanning_on_advisory_ingestion.yml | 2 +- ...ge-container-registry-api-group-update.yml | 4 +- ...te-permissions-change-package-settings.yml | 4 +- ...word-authentication-container-registry.yml | 4 +- ...-pull-through-cache-container-registry.yml | 4 +- .../15-8-third-party-registries.yml | 6 +- .../16-4_support_for_delete_tags_endpoint.yml | 6 +- ...stry-support-storage-drivers-swift-oss.yml | 2 +- db/docs/users.yml | 4 + ...42_add_status_to_packages_nuget_symbols.rb | 14 +++ ...id_default_value_for_job_artifact_state.rb | 14 +++ ...index_users_for_auditors_for_gitlab_com.rb | 26 ++++++ db/schema_migrations/20231113124342 | 1 + db/schema_migrations/20231122084642 | 1 + db/schema_migrations/20231128201403 | 1 + db/structure.sql | 3 +- .../backup_restore/backup_gitlab.md | 4 +- .../backup_large_reference_architectures.md | 2 +- .../backup_restore/restore_gitlab.md | 2 +- .../geo/disaster_recovery/planned_failover.md | 2 +- .../geo/replication/container_registry.md | 18 ++-- doc/administration/geo/replication/faq.md | 2 +- .../geo/secondary_proxy/index.md | 2 +- doc/administration/logs/index.md | 2 +- doc/administration/object_storage.md | 2 +- .../packages/container_registry.md | 52 +++++------ doc/administration/sidekiq/index.md | 2 +- doc/api/container_registry.md | 4 +- doc/api/packages/terraform-modules.md | 36 ++++++++ doc/api/runners.md | 8 +- doc/api/settings.md | 2 +- doc/api/users.md | 2 +- .../cells/deployment-architecture.md | 4 +- .../impacted_features/container-registry.md | 22 ++--- .../index.md | 2 +- .../index.md | 26 +++--- .../index.md | 2 +- .../index.md | 6 +- .../ui_ux.md | 4 +- .../blueprints/rate_limiting/index.md | 2 +- .../blueprints/runner_tokens/index.md | 4 +- .../cloud_deployment/ecs/deploy_to_aws_ecs.md | 6 +- doc/ci/cloud_deployment/index.md | 4 +- doc/ci/docker/buildah_rootless_tutorial.md | 2 +- doc/ci/docker/index.md | 2 +- doc/ci/docker/using_docker_build.md | 10 +-- doc/ci/docker/using_docker_images.md | 6 +- doc/ci/docker/using_kaniko.md | 6 +- .../laravel_with_gitlab_and_envoy/index.md | 8 +- doc/ci/runners/new_creation_workflow.md | 2 +- doc/ci/runners/runners_scope.md | 6 +- doc/ci/services/index.md | 2 +- doc/ci/testing/code_quality.md | 2 +- doc/ci/variables/predefined_variables.md | 2 +- doc/ci/yaml/index.md | 2 +- doc/development/architecture.md | 2 +- doc/development/code_review.md | 5 +- .../fe_guide/registry_architecture.md | 2 +- doc/development/fips_compliance.md | 2 +- doc/development/geo.md | 2 +- doc/development/go_guide/go_upgrade.md | 2 +- doc/development/i18n/proofreader.md | 2 +- .../service_ping/troubleshooting.md | 2 +- .../mergeability_framework.md | 5 +- .../testing_guide/end_to_end/index.md | 2 +- doc/install/google_cloud_platform/index.md | 2 +- doc/install/next_steps.md | 2 +- doc/topics/autodevops/index.md | 2 +- doc/topics/offline/quick_start_guide.md | 2 +- .../automate_runner_creation/index.md | 2 +- doc/update/deprecations.md | 54 ++++++------ doc/update/versions/gitlab_14_changes.md | 6 +- .../application_security/api_fuzzing/index.md | 2 +- .../container_scanning/index.md | 2 +- .../dast/run_dast_offline.md | 2 +- .../application_security/dast_api/index.md | 2 +- .../dependency_scanning/index.md | 2 +- doc/user/application_security/sast/index.md | 2 +- .../agent/gitops/flux_oci_tutorial.md | 4 +- .../agent/gitops/secrets_management.md | 4 +- doc/user/custom_roles.md | 70 ++++++++++++++- .../iac/terraform_template_recipes.md | 2 +- .../build_and_push_images.md | 4 +- .../delete_container_registry_images.md | 2 +- doc/user/packages/container_registry/index.md | 22 ++--- ...reduce_container_registry_data_transfer.md | 2 +- .../reduce_container_registry_storage.md | 6 +- .../troubleshoot_container_registry.md | 14 +-- doc/user/packages/pypi_repository/index.md | 1 + .../account/two_factor_authentication.md | 2 +- doc/user/read_only_namespaces.md | 2 +- doc/user/storage_management_automation.md | 2 +- doc/user/workspace/create_image.md | 14 +-- lib/api/entities/ml/mlflow/model_version.rb | 86 +++++++++++++++++++ .../ml/mlflow/model_versions/responses/get.rb | 17 ---- .../mlflow/model_versions/responses/update.rb | 17 ---- .../model_versions/types/model_version.rb | 85 ------------------ lib/api/ml/mlflow/api_helpers.rb | 8 ++ lib/api/ml/mlflow/model_versions.rb | 43 +++++++--- lib/api/ml/mlflow/registered_models.rb | 4 +- spec/factories/packages/nuget/symbol.rb | 6 ++ .../api/schemas/ml/get_model_version.json | 75 ++++++++++++++++ .../list/ci_resources_list_item_spec.js | 84 ++++++++---------- spec/models/packages/nuget/symbol_spec.rb | 21 +++++ .../api/ml/mlflow/model_versions_spec.rb | 42 ++++++++- .../ml/create_model_version_service_spec.rb | 28 ++++++ .../ml/increment_version_service_spec.rb | 56 ++++++++++++ spec/workers/every_sidekiq_worker_spec.rb | 1 + .../cleanup_package_registry_worker_spec.rb | 22 +++++ .../cleanup_stale_symbols_worker_spec.rb | 77 +++++++++++++++++ 120 files changed, 1027 insertions(+), 432 deletions(-) create mode 100644 app/services/ml/create_model_version_service.rb create mode 100644 app/services/ml/increment_version_service.rb create mode 100644 app/workers/packages/nuget/cleanup_stale_symbols_worker.rb create mode 100644 db/migrate/20231113124342_add_status_to_packages_nuget_symbols.rb create mode 100644 db/post_migrate/20231122084642_remove_partition_id_default_value_for_job_artifact_state.rb create mode 100644 db/post_migrate/20231128201403_drop_index_users_for_auditors_for_gitlab_com.rb create mode 100644 db/schema_migrations/20231113124342 create mode 100644 db/schema_migrations/20231122084642 create mode 100644 db/schema_migrations/20231128201403 create mode 100644 lib/api/entities/ml/mlflow/model_version.rb delete mode 100644 lib/api/entities/ml/mlflow/model_versions/responses/get.rb delete mode 100644 lib/api/entities/ml/mlflow/model_versions/responses/update.rb delete mode 100644 lib/api/entities/ml/mlflow/model_versions/types/model_version.rb create mode 100644 spec/fixtures/api/schemas/ml/get_model_version.json create mode 100644 spec/services/ml/create_model_version_service_spec.rb create mode 100644 spec/services/ml/increment_version_service_spec.rb create mode 100644 spec/workers/packages/nuget/cleanup_stale_symbols_worker_spec.rb diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index fad05f8eb50..8b36d7937f6 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -45,10 +45,10 @@ if: '($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_LABELS =~ /pipeline:mr-approved/' .if-merge-request-approved-and-specific-devops-stage: &if-merge-request-approved-and-specific-devops-stage - if: '($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && ($CI_MERGE_REQUEST_LABELS =~ /pipeline:mr-approved/ && $CI_MERGE_REQUEST_LABELS =~ /devops::(create|govern|manage)/)' + if: '($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && ($CI_MERGE_REQUEST_LABELS =~ /pipeline:mr-approved/ && $CI_MERGE_REQUEST_LABELS =~ /devops::(create|govern|manage|plan|verify|package)/)' .if-merge-request-and-specific-devops-stage: &if-merge-request-and-specific-devops-stage - if: '($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_LABELS =~ /devops::(create|govern|manage)/' + if: '($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_LABELS =~ /devops::(create|govern|manage|plan|verify|package)/' .if-merge-request-not-approved: &if-merge-request-not-approved if: '($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_LABELS !~ /pipeline:mr-approved/' diff --git a/app/models/packages/nuget/symbol.rb b/app/models/packages/nuget/symbol.rb index 3315f11b974..adcfaef4f4c 100644 --- a/app/models/packages/nuget/symbol.rb +++ b/app/models/packages/nuget/symbol.rb @@ -5,6 +5,10 @@ module Packages class Symbol < ApplicationRecord include FileStoreMounter include ShaAttribute + include Packages::Destructible + + # Used in destroying stale symbols in worker + enum :status, default: 0, processing: 1, error: 3 belongs_to :package, -> { where(package_type: :nuget) }, inverse_of: :nuget_symbols @@ -20,6 +24,9 @@ module Packages before_validation :set_object_storage_key, on: :create + scope :stale, -> { where(package_id: nil) } + scope :pending_destruction, -> { stale.default } + private def set_object_storage_key diff --git a/app/services/ml/create_model_version_service.rb b/app/services/ml/create_model_version_service.rb new file mode 100644 index 00000000000..ff27e78221f --- /dev/null +++ b/app/services/ml/create_model_version_service.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Ml + class CreateModelVersionService + def initialize(model, params = {}) + @model = model + @version = params[:version] + @package = params[:package] + @description = params[:description] + end + + def execute + @version ||= Ml::IncrementVersionService.new(@model.latest_version.try(:version)).execute + + model_version = Ml::ModelVersion.find_or_create!(@model, @version, @package, @description) + + model_version.candidate = ::Ml::CreateCandidateService.new( + @model.default_experiment, + { model_version: model_version } + ).execute + + model_version + end + end +end diff --git a/app/services/ml/increment_version_service.rb b/app/services/ml/increment_version_service.rb new file mode 100644 index 00000000000..ef3cbf5269b --- /dev/null +++ b/app/services/ml/increment_version_service.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Ml + INITIAL_VERSION = '1.0.0' + ALLOWED_INCREMENT_TYPES = [:patch, :minor, :major].freeze + + class IncrementVersionService + def initialize(version, increment_type = nil) + @version = version + @increment_type = increment_type || :major + @parsed_version = Packages::SemVer.parse(@version.to_s) + + raise "Version must be in a valid SemVer format" unless @parsed_version || @version.nil? + + return if ALLOWED_INCREMENT_TYPES.include?(@increment_type) + + raise "Increment type must be one of :patch, :minor, or :major" + end + + def execute + return INITIAL_VERSION if @version.nil? + + case @increment_type + when :patch + @parsed_version.patch += 1 + when :minor + @parsed_version.minor += 1 + when :major + @parsed_version.major += 1 + end + + @parsed_version.to_s + end + end +end diff --git a/app/views/import/github/new.html.haml b/app/views/import/github/new.html.haml index 24369ff3d39..5893687605c 100644 --- a/app/views/import/github/new.html.haml +++ b/app/views/import/github/new.html.haml @@ -3,13 +3,12 @@ - header_title _("New project"), new_project_path - add_to_breadcrumbs s_('ProjectsNew|Import project'), new_project_path(anchor: 'import_project') -%h1.page-title.gl-font-size-h-display.d-flex.gl-align-items-center - .gl-display-flex.gl-align-items-center.gl-justify-content-center - = sprite_icon('github', css_class: 'gl-mr-3', size: 48) - = title -%hr +.gl-border-solid.gl-border-gray-100.gl-border-0.gl-border-b-1 + %h1.gl-font-size-h1.gl-my-0.gl-py-4.gl-display-flex.gl-align-items-center.gl-gap-3 + = sprite_icon('github', size: 24) + %span= title -%p +%p.gl-mt-5 = import_github_authorize_message - if github_import_configured? && !has_ci_cd_only_params? @@ -21,8 +20,9 @@ %hr - unless github_import_configured? || has_ci_cd_only_params? - .bs-callout.bs-callout-info - = import_configure_github_admin_message + = render Pajamas::AlertComponent.new(variant: :info, dismissible: false) do |c| + - c.with_body do + = import_configure_github_admin_message = form_tag personal_access_token_import_github_path, method: :post do .form-group diff --git a/app/views/import/github/status.html.haml b/app/views/import/github/status.html.haml index f1a61d72771..97acafe24d0 100644 --- a/app/views/import/github/status.html.haml +++ b/app/views/import/github/status.html.haml @@ -1,9 +1,10 @@ - title = has_ci_cd_only_params? ? _('Connect repositories from GitHub') : _('GitHub import') - page_title title -%h1.page-title.gl-font-size-h-display.d-flex.gl-align-items-center - .gl-display-flex.gl-align-items-center.gl-justify-content-center - = sprite_icon('github', css_class: 'gl-mr-3', size: 48) - = _('Import repositories from GitHub') + +.gl-border-solid.gl-border-gray-100.gl-border-0.gl-border-b-1 + %h1.gl-font-size-h1.gl-my-0.gl-py-4.gl-display-flex.gl-align-items-center.gl-gap-3 + = sprite_icon('github', size: 24) + %span= _('Import repositories from GitHub') = render 'import/githubish_status', provider: 'github', paginatable: true, diff --git a/app/views/projects/blob/_editor.html.haml b/app/views/projects/blob/_editor.html.haml index bb29ea92704..a08d17a3efa 100644 --- a/app/views/projects/blob/_editor.html.haml +++ b/app/views/projects/blob/_editor.html.haml @@ -4,7 +4,7 @@ .file-holder.file.gl-mb-3 .js-file-title.file-title.gl-display-flex.gl-align-items-center.gl-rounded-top-base{ data: { current_action: action } } - .editor-ref.block-truncated.has-tooltip{ title: ref } + .editor-ref.block-truncated.has-tooltip{ title: current_action?(:edit, :update) ? ref : params[:id] } = sprite_icon('branch', size: 12) - if current_action?(:edit, :update) %span#editor_ref diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 5c6567becea..ed8a03ce998 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1749,6 +1749,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: package_cleanup:packages_nuget_cleanup_stale_symbols + :worker_name: Packages::Nuget::CleanupStaleSymbolsWorker + :feature_category: :package_registry + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: package_repositories:packages_debian_generate_distribution :worker_name: Packages::Debian::GenerateDistributionWorker :feature_category: :package_registry diff --git a/app/workers/packages/cleanup_package_registry_worker.rb b/app/workers/packages/cleanup_package_registry_worker.rb index 5b2d8bacd62..50036923e94 100644 --- a/app/workers/packages/cleanup_package_registry_worker.rb +++ b/app/workers/packages/cleanup_package_registry_worker.rb @@ -14,6 +14,7 @@ module Packages enqueue_package_file_cleanup_job if Packages::PackageFile.pending_destruction.exists? enqueue_cleanup_policy_jobs if Packages::Cleanup::Policy.runnable.exists? enqueue_cleanup_stale_npm_metadata_cache_job if Packages::Npm::MetadataCache.pending_destruction.exists? + enqueue_cleanup_stale_nuget_symbols_job if Packages::Nuget::Symbol.pending_destruction.exists? log_counts end @@ -32,6 +33,10 @@ module Packages Packages::Npm::CleanupStaleMetadataCacheWorker.perform_with_capacity end + def enqueue_cleanup_stale_nuget_symbols_job + Packages::Nuget::CleanupStaleSymbolsWorker.perform_with_capacity + end + def log_counts use_replica_if_available do pending_destruction_package_files_count = Packages::PackageFile.pending_destruction.count diff --git a/app/workers/packages/nuget/cleanup_stale_symbols_worker.rb b/app/workers/packages/nuget/cleanup_stale_symbols_worker.rb new file mode 100644 index 00000000000..be90b86604c --- /dev/null +++ b/app/workers/packages/nuget/cleanup_stale_symbols_worker.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class CleanupStaleSymbolsWorker + include ApplicationWorker + include ::Packages::CleanupArtifactWorker + + MAX_CAPACITY = 2 + + data_consistency :sticky + + queue_namespace :package_cleanup + feature_category :package_registry + + deduplicate :until_executed + idempotent! + + def max_running_jobs + MAX_CAPACITY + end + + private + + def model + Packages::Nuget::Symbol + end + + def next_item + model.next_pending_destruction(order_by: nil) + end + + def log_metadata(nuget_symbol) + log_extra_metadata_on_done(:nuget_symbol_id, nuget_symbol.id) + end + + def log_cleanup_item(nuget_symbol) + logger.info( + structured_payload( + nuget_symbol_id: nuget_symbol.id + ) + ) + end + end + end +end diff --git a/config/feature_flags/development/global_dependency_scanning_on_advisory_ingestion.yml b/config/feature_flags/development/global_dependency_scanning_on_advisory_ingestion.yml index ca33869fd2a..fbede45e665 100644 --- a/config/feature_flags/development/global_dependency_scanning_on_advisory_ingestion.yml +++ b/config/feature_flags/development/global_dependency_scanning_on_advisory_ingestion.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/427424 milestone: '16.6' type: development group: group::composition analysis -default_enabled: false +default_enabled: true diff --git a/data/deprecations/14-5-package-container-registry-api-group-update.yml b/data/deprecations/14-5-package-container-registry-api-group-update.yml index d7dbf841c72..9f467709d53 100644 --- a/data/deprecations/14-5-package-container-registry-api-group-update.yml +++ b/data/deprecations/14-5-package-container-registry-api-group-update.yml @@ -1,9 +1,9 @@ -- title: "Update to the Container Registry group-level API" +- title: "Update to the container registry group-level API" announcement_milestone: "14.5" # The milestone when this feature was first announced as deprecated. removal_milestone: "15.0" # the milestone when this feature is planned to be removed breaking_change: true body: | # Do not modify this line, instead modify the lines below. - In milestone 15.0, support for the `tags` and `tags_count` parameters will be removed from the Container Registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). + In milestone 15.0, support for the `tags` and `tags_count` parameters will be removed from the container registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). The `GET /groups/:id/registry/repositories` endpoint will remain, but won't return any info about tags. To get the info about tags, you can use the existing `GET /registry/repositories/:id` endpoint, which will continue to support the `tags` and `tag_count` options as it does today. The latter must be called once per image repository. stage: Package diff --git a/data/deprecations/14-9-deprecate-permissions-change-package-settings.yml b/data/deprecations/14-9-deprecate-permissions-change-package-settings.yml index 8f2aaf3b743..cc16a1155a4 100644 --- a/data/deprecations/14-9-deprecate-permissions-change-package-settings.yml +++ b/data/deprecations/14-9-deprecate-permissions-change-package-settings.yml @@ -4,12 +4,12 @@ breaking_change: true reporter: trizzi body: | # Do not modify this line, instead modify the lines below. - The GitLab Package stage offers a Package Registry, Container Registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. + The GitLab Package stage offers a Package Registry, container registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. The permissions model for GraphQL is being updated. After 15.0, users with the Guest, Reporter, and Developer role can no longer update these settings: - [Package Registry settings](https://docs.gitlab.com/ee/api/graphql/reference/#packagesettings) - - [Container Registry cleanup policy](https://docs.gitlab.com/ee/api/graphql/reference/#containerexpirationpolicy) + - [Container registry cleanup policy](https://docs.gitlab.com/ee/api/graphql/reference/#containerexpirationpolicy) - [Dependency Proxy time-to-live policy](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxyimagettlgrouppolicy) - [Enabling the Dependency Proxy for your group](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxysetting) stage: package diff --git a/data/deprecations/14-9-deprecation-htpassword-authentication-container-registry.yml b/data/deprecations/14-9-deprecation-htpassword-authentication-container-registry.yml index 9dbe6c24968..0e626887edc 100644 --- a/data/deprecations/14-9-deprecation-htpassword-authentication-container-registry.yml +++ b/data/deprecations/14-9-deprecation-htpassword-authentication-container-registry.yml @@ -1,10 +1,10 @@ -- title: "htpasswd Authentication for the Container Registry" +- title: "htpasswd Authentication for the container registry" announcement_milestone: "14.9" removal_milestone: "15.0" breaking_change: true reporter: trizzi body: | # Do not modify this line, instead modify the lines below. - The Container Registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. + The container registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. Since it isn't used in the context of GitLab (the product), `htpasswd` authentication will be deprecated in GitLab 14.9 and removed in GitLab 15.0. stage: package diff --git a/data/deprecations/15-8-pull-through-cache-container-registry.yml b/data/deprecations/15-8-pull-through-cache-container-registry.yml index 6a59dd606e2..18c17f321ff 100644 --- a/data/deprecations/15-8-pull-through-cache-container-registry.yml +++ b/data/deprecations/15-8-pull-through-cache-container-registry.yml @@ -1,4 +1,4 @@ -- title: "Container Registry pull-through cache" # (required) Actionable title. e.g., The `confidential` field for a `Note` is deprecated. Use `internal` instead. +- title: "Container registry pull-through cache" # (required) Actionable title. e.g., The `confidential` field for a `Note` is deprecated. Use `internal` instead. announcement_milestone: "15.8" # (required) The milestone when this feature was first announced as deprecated. removal_milestone: "16.0" # (required) The milestone when this feature is planned to be removed breaking_change: true # (required) If this deprecation is a breaking change, set this value to true @@ -6,4 +6,4 @@ stage: Package # (required) String value of the stage that the feature was created in. e.g., Growth issue_url: https://gitlab.com/gitlab-org/container-registry/-/issues/842 # (required) Link to the deprecation issue in GitLab body: | # (required) Do not modify this line, instead modify the lines below. - The Container Registry [pull-through cache](https://docs.docker.com/registry/recipes/mirror/) is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. The pull-through cache is part of the upstream [Docker Distribution project](https://github.com/distribution/distribution). However, we are removing the pull-through cache in favor of the GitLab Dependency Proxy, which allows you to proxy and cache container images from Docker Hub. Removing the pull-through cache allows us also to remove the upstream client code without sacrificing functionality. + The container registry [pull-through cache](https://docs.docker.com/registry/recipes/mirror/) is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. The pull-through cache is part of the upstream [Docker Distribution project](https://github.com/distribution/distribution). However, we are removing the pull-through cache in favor of the GitLab Dependency Proxy, which allows you to proxy and cache container images from Docker Hub. Removing the pull-through cache allows us also to remove the upstream client code without sacrificing functionality. diff --git a/data/deprecations/15-8-third-party-registries.yml b/data/deprecations/15-8-third-party-registries.yml index c4eddf2e64f..6607835c675 100644 --- a/data/deprecations/15-8-third-party-registries.yml +++ b/data/deprecations/15-8-third-party-registries.yml @@ -8,13 +8,13 @@ body: | # (required) Do not modify this line, instead modify the lines below. Using third-party container registries with GitLab as an auth endpoint is deprecated in GitLab 15.8 and the [end of support](https://docs.gitlab.com/ee/development/deprecation_guidelines/#terminology) is scheduled for GitLab 16.0. This impacts self-managed customers that have connected their external registry to the GitLab user interface to find, view, and delete container images. - Supporting both GitLab's Container Registry as well as third-party container registries is challenging for maintenance, code quality, and backward compatibility. This hinders our ability to stay [efficient](https://about.gitlab.com/handbook/values/#efficiency). As a result we will not support this functionality moving forward. + Supporting both GitLab's container registry as well as third-party container registries is challenging for maintenance, code quality, and backward compatibility. This hinders our ability to stay [efficient](https://about.gitlab.com/handbook/values/#efficiency). As a result we will not support this functionality moving forward. This change will not impact your ability to pull and push container images to external registries using pipelines. - Since we released the new [GitLab Container Registry](https://gitlab.com/groups/gitlab-org/-/epics/5523) version for GitLab.com, we've started to implement additional features that are not available in third-party container registries. These new features have allowed us to achieve significant performance improvements, such as [cleanup policies](https://gitlab.com/groups/gitlab-org/-/epics/8379). We are focusing on delivering [new features](https://gitlab.com/groups/gitlab-org/-/epics/5136), most of which will require functionalities only available on the GitLab Container Registry. This deprecation allows us to reduce fragmentation and user frustration in the long term by focusing on delivering a more robust integrated registry experience and feature set. + Since we released the new [GitLab container registry](https://gitlab.com/groups/gitlab-org/-/epics/5523) version for GitLab.com, we've started to implement additional features that are not available in third-party container registries. These new features have allowed us to achieve significant performance improvements, such as [cleanup policies](https://gitlab.com/groups/gitlab-org/-/epics/8379). We are focusing on delivering [new features](https://gitlab.com/groups/gitlab-org/-/epics/5136), most of which will require functionalities only available on the GitLab container registry. This deprecation allows us to reduce fragmentation and user frustration in the long term by focusing on delivering a more robust integrated registry experience and feature set. - Moving forward, we'll continue to invest in developing and releasing new features that will only be available in the GitLab Container Registry. + Moving forward, we'll continue to invest in developing and releasing new features that will only be available in the GitLab container registry. # # OPTIONAL END OF SUPPORT FIELDS # diff --git a/data/deprecations/16-4_support_for_delete_tags_endpoint.yml b/data/deprecations/16-4_support_for_delete_tags_endpoint.yml index 5805653c027..ccaf26e4a4f 100644 --- a/data/deprecations/16-4_support_for_delete_tags_endpoint.yml +++ b/data/deprecations/16-4_support_for_delete_tags_endpoint.yml @@ -1,4 +1,4 @@ -- title: "Internal Container Registry API tag deletion endpoint" # (required) Actionable title. e.g., The `confidential` field for a `Note` is deprecated. Use `internal` instead. +- title: "Internal container registry API tag deletion endpoint" # (required) Actionable title. e.g., The `confidential` field for a `Note` is deprecated. Use `internal` instead. announcement_milestone: "16.4" # (required) The milestone when this feature was first announced as deprecated. removal_milestone: "17.0" # (required) The milestone when this feature is planned to be removed breaking_change: true # (required) If this deprecation is a breaking change, set this value to true @@ -8,7 +8,7 @@ body: | # (required) Do not modify this line, instead modify the lines below. The [Docker Registry HTTP API V2 Spec](https://docs.docker.com/registry/spec/api/), later replaced by the [OCI Distribution Spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md) did not include a tag delete operation, and an unsafe and slow workaround (involving deleting manifests, not tags) had to be used to achieve the same end. - Tag deletion is an important function, so we added a tag deletion operation to the GitLab Container Registry, extending the V2 API beyond the scope of the Docker and OCI distribution spec. + Tag deletion is an important function, so we added a tag deletion operation to the GitLab container registry, extending the V2 API beyond the scope of the Docker and OCI distribution spec. Since then, the OCI Distribution Spec has had some updates and it now has a tag delete operation, using the [`DELETE /v2//manifests/` endpoint](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-tags). @@ -16,6 +16,6 @@ Support for the custom GitLab tag delete endpoint is deprecated in GitLab 16.4, and it will be removed in GitLab 17.0. - This endpoint is used by the **internal** Container Registry application API, not the public [GitLab Container Registry API](https://docs.gitlab.com/ee/api/container_registry.html). No action should be required by the majority of container registry users. All the GitLab UI and API functionality related to tag deletions will remain intact as we transition to the new OCI-compliant endpoint. + This endpoint is used by the **internal** container registry application API, not the public [GitLab container registry API](https://docs.gitlab.com/ee/api/container_registry.html). No action should be required by the majority of container registry users. All the GitLab UI and API functionality related to tag deletions will remain intact as we transition to the new OCI-compliant endpoint. If you do access the internal container registry API and use the original tag deletion endpoint, you must update to the new endpoint. diff --git a/data/deprecations/16-5-container-registry-support-storage-drivers-swift-oss.yml b/data/deprecations/16-5-container-registry-support-storage-drivers-swift-oss.yml index 0d7364d05ff..3190e02b7f0 100644 --- a/data/deprecations/16-5-container-registry-support-storage-drivers-swift-oss.yml +++ b/data/deprecations/16-5-container-registry-support-storage-drivers-swift-oss.yml @@ -1,4 +1,4 @@ -- title: "Container Registry support for the Swift and OSS storage drivers" +- title: "Container registry support for the Swift and OSS storage drivers" announcement_milestone: "16.6" removal_milestone: "17.0" breaking_change: true diff --git a/db/docs/users.yml b/db/docs/users.yml index 119c89d48cc..43bec30b9dd 100644 --- a/db/docs/users.yml +++ b/db/docs/users.yml @@ -9,3 +9,7 @@ description: TODO introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/9ba1224867665844b117fa037e1465bb706b3685 milestone: "<6.0" gitlab_schema: gitlab_main_clusterwide +schema_inconsistencies: +- type: missing_indexes + object_name: index_users_for_auditors + introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138110 diff --git a/db/migrate/20231113124342_add_status_to_packages_nuget_symbols.rb b/db/migrate/20231113124342_add_status_to_packages_nuget_symbols.rb new file mode 100644 index 00000000000..9b759743029 --- /dev/null +++ b/db/migrate/20231113124342_add_status_to_packages_nuget_symbols.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class AddStatusToPackagesNugetSymbols < Gitlab::Database::Migration[2.2] + enable_lock_retries! + milestone '16.7' + + def up + add_column :packages_nuget_symbols, :status, :smallint, null: false, default: 0 + end + + def down + remove_column :packages_nuget_symbols, :status + end +end diff --git a/db/post_migrate/20231122084642_remove_partition_id_default_value_for_job_artifact_state.rb b/db/post_migrate/20231122084642_remove_partition_id_default_value_for_job_artifact_state.rb new file mode 100644 index 00000000000..3a1e9c2e6a9 --- /dev/null +++ b/db/post_migrate/20231122084642_remove_partition_id_default_value_for_job_artifact_state.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class RemovePartitionIdDefaultValueForJobArtifactState < Gitlab::Database::Migration[2.2] + milestone '16.7' + enable_lock_retries! + + def up + change_column_default :ci_job_artifact_states, :partition_id, from: 100, to: nil + end + + def down + change_column_default :ci_job_artifact_states, :partition_id, from: nil, to: 100 + end +end diff --git a/db/post_migrate/20231128201403_drop_index_users_for_auditors_for_gitlab_com.rb b/db/post_migrate/20231128201403_drop_index_users_for_auditors_for_gitlab_com.rb new file mode 100644 index 00000000000..fc452f7bfc0 --- /dev/null +++ b/db/post_migrate/20231128201403_drop_index_users_for_auditors_for_gitlab_com.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class DropIndexUsersForAuditorsForGitlabCom < Gitlab::Database::Migration[2.2] + milestone '16.7' + + disable_ddl_transaction! + + TABLE_NAME = :users + INDEX_NAME = :index_users_for_auditors + + def up + return unless should_run? + + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end + + def down + return unless should_run? + + add_concurrent_index TABLE_NAME, :created_at, name: INDEX_NAME + end + + def should_run? + Gitlab.com_except_jh? + end +end diff --git a/db/schema_migrations/20231113124342 b/db/schema_migrations/20231113124342 new file mode 100644 index 00000000000..2960c86f6b6 --- /dev/null +++ b/db/schema_migrations/20231113124342 @@ -0,0 +1 @@ +2d87a6e54aa2a5f80f2d36b2718aab081223b25b0d199a843558668e98b3a319 \ No newline at end of file diff --git a/db/schema_migrations/20231122084642 b/db/schema_migrations/20231122084642 new file mode 100644 index 00000000000..ac0abb62234 --- /dev/null +++ b/db/schema_migrations/20231122084642 @@ -0,0 +1 @@ +36492b8e28dea9d2d650981b492a9956e95100edd5db1b3fb25c70f2f974e8c2 \ No newline at end of file diff --git a/db/schema_migrations/20231128201403 b/db/schema_migrations/20231128201403 new file mode 100644 index 00000000000..c69eee93101 --- /dev/null +++ b/db/schema_migrations/20231128201403 @@ -0,0 +1 @@ +cfb411996e7ae1fbdb47886ac5c1fd1c264d324ffa54da5edd4d73ff53cd2d28 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 5de517329ba..f08c94377a3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -14046,7 +14046,7 @@ CREATE TABLE ci_job_artifact_states ( verification_retry_count smallint, verification_checksum bytea, verification_failure text, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, CONSTRAINT check_df832b66ea CHECK ((char_length(verification_failure) <= 255)) ); @@ -20583,6 +20583,7 @@ CREATE TABLE packages_nuget_symbols ( signature text NOT NULL, object_storage_key text NOT NULL, file_sha256 bytea, + status smallint DEFAULT 0 NOT NULL, CONSTRAINT check_0e93ca58b7 CHECK ((char_length(file) <= 255)), CONSTRAINT check_28b82b08fa CHECK ((char_length(object_storage_key) <= 255)), CONSTRAINT check_30b0ef2ca2 CHECK ((char_length(file_path) <= 255)), diff --git a/doc/administration/backup_restore/backup_gitlab.md b/doc/administration/backup_restore/backup_gitlab.md index 4fbbb38bb55..49ae2311ab6 100644 --- a/doc/administration/backup_restore/backup_gitlab.md +++ b/doc/administration/backup_restore/backup_gitlab.md @@ -172,7 +172,7 @@ including: - CI/CD job artifacts - LFS objects - Terraform states ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331806) in GitLab 14.7) -- Container Registry images +- Container registry images - GitLab Pages content - Packages ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332006) in GitLab 14.7) - Snippets @@ -460,7 +460,7 @@ You can exclude specific directories from the backup by adding the environment v - `artifacts` (CI job artifacts) - `lfs` (LFS objects) - `terraform_state` (Terraform states) -- `registry` (Container Registry images) +- `registry` (Container registry images) - `pages` (Pages content) - `repositories` (Git repositories data) - `packages` (Packages) diff --git a/doc/administration/backup_restore/backup_large_reference_architectures.md b/doc/administration/backup_restore/backup_large_reference_architectures.md index 8e03ae7aded..8552024e5ac 100644 --- a/doc/administration/backup_restore/backup_large_reference_architectures.md +++ b/doc/administration/backup_restore/backup_large_reference_architectures.md @@ -17,7 +17,7 @@ This document is intended for environments using: - [Linux package (Omnibus) and cloud-native hybrid reference architectures 3,000 users and up](../reference_architectures/index.md) - [Amazon RDS](https://aws.amazon.com/rds/) for PostgreSQL data - [Amazon S3](https://aws.amazon.com/s3/) for object storage -- [Object storage](../object_storage.md) to store everything possible, including [blobs](backup_gitlab.md#blobs) and [Container Registry](backup_gitlab.md#container-registry) +- [Object storage](../object_storage.md) to store everything possible, including [blobs](backup_gitlab.md#blobs) and [container registry](backup_gitlab.md#container-registry) ## Configure daily backups diff --git a/doc/administration/backup_restore/restore_gitlab.md b/doc/administration/backup_restore/restore_gitlab.md index 9b6c6eb557a..6fa96fb9ca4 100644 --- a/doc/administration/backup_restore/restore_gitlab.md +++ b/doc/administration/backup_restore/restore_gitlab.md @@ -340,7 +340,7 @@ You can exclude specific tasks on restore by adding the environment variable `SK - `artifacts` (CI job artifacts) - `lfs` (LFS objects) - `terraform_state` (Terraform states) -- `registry` (Container Registry images) +- `registry` (Container registry images) - `pages` (Pages content) - `repositories` (Git repositories data) - `packages` (Packages) diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index cdc52b55c3e..58c9f3413a7 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -45,7 +45,7 @@ be adapted for use with any other file-based data. ### Container registry By default, the container registry is not automatically replicated to secondary -sites and this needs to be manually configured, see [Container Registry for a secondary site](../replication/container_registry.md). +sites and this needs to be manually configured, see [container registry for a secondary site](../replication/container_registry.md). If you are using local storage on your current primary site for the container registry, you can `rsync` the container registry objects to the secondary diff --git a/doc/administration/geo/replication/container_registry.md b/doc/administration/geo/replication/container_registry.md index 904e461f844..51cdf0e05e1 100644 --- a/doc/administration/geo/replication/container_registry.md +++ b/doc/administration/geo/replication/container_registry.md @@ -40,7 +40,7 @@ For more information on supported registry storage drivers see Read the [Load balancing considerations](https://docs.docker.com/registry/deploying/#load-balancing-considerations) when deploying the Registry, and how to set up the storage driver for the GitLab integrated -[Container Registry](../../packages/container_registry.md#use-object-storage). +[container registry](../../packages/container_registry.md#use-object-storage). ### Registries that support OCI artifacts @@ -55,22 +55,22 @@ The following registries support OCI artifacts: For more information, see the [OCI Distribution Specification](https://github.com/opencontainers/distribution-spec). -## Configure Container Registry replication +## Configure container registry replication You can enable a storage-agnostic replication so it can be used for cloud or local storage. Whenever a new image is pushed to the **primary** site, each **secondary** site pulls it to its own container repository. -To configure Container Registry replication: +To configure container registry replication: 1. Configure the [**primary** site](#configure-primary-site). 1. Configure the [**secondary** site](#configure-secondary-site). -1. Verify Container Registry [replication](#verify-replication). +1. Verify container registry [replication](#verify-replication). ### Configure **primary** site -Make sure that you have Container Registry set up and working on +Make sure that you have container registry set up and working on the **primary** site before following the next steps. To be able to replicate new container images, the container registry must send notification events to the @@ -124,17 +124,17 @@ To be able to replicate new container images, the container registry must send n ### Configure **secondary** site -Make sure you have Container Registry set up and working on +Make sure you have container registry set up and working on the **secondary** site before following the next steps. The following steps should be done on each **secondary** site you're expecting to see the container images replicated. Because we need to allow the **secondary** site to communicate securely with -the **primary** site Container Registry, we need to have a single key +the **primary** site container registry, we need to have a single key pair for all the sites. The **secondary** site uses this key to generate a short-lived JWT that is pull-only-capable to access the -**primary** site Container Registry. +**primary** site container registry. For each application and Sidekiq node on the **secondary** site: @@ -164,7 +164,7 @@ For each application and Sidekiq node on the **secondary** site: ### Verify replication -To verify Container Registry replication is working, on the **secondary** site: +To verify container registry replication is working, on the **secondary** site: 1. On the left sidebar, select **Search or go to**. 1. Select **Admin Area**. diff --git a/doc/administration/geo/replication/faq.md b/doc/administration/geo/replication/faq.md index d5fb80bf47e..9ef2e04eca9 100644 --- a/doc/administration/geo/replication/faq.md +++ b/doc/administration/geo/replication/faq.md @@ -67,7 +67,7 @@ That's totally fine. We use HTTP(s) to fetch repository changes from the **prima ## Is this possible to set up a container registry for a **secondary** site that mirrors the one on the **primary** site? -Yes. See [Container Registry for a **secondary** site](container_registry.md). +Yes. See [container registry for a **secondary** site](container_registry.md). ## Can you sign in to a secondary site? diff --git a/doc/administration/geo/secondary_proxy/index.md b/doc/administration/geo/secondary_proxy/index.md index f6572ded4a9..bae6ad1371a 100644 --- a/doc/administration/geo/secondary_proxy/index.md +++ b/doc/administration/geo/secondary_proxy/index.md @@ -115,7 +115,7 @@ for details. - Non-Rails requests are not proxied, so other services may need to use a separate, non-unified URL to ensure requests are always sent to the primary. These services include: - - GitLab Container Registry - [can be configured to use a separate domain](../../packages/container_registry.md#configure-container-registry-under-its-own-domain). + - GitLab container registry - [can be configured to use a separate domain](../../packages/container_registry.md#configure-container-registry-under-its-own-domain). - GitLab Pages - should always use a separate domain, as part of [the prerequisites for running GitLab Pages](../../pages/index.md#prerequisites). - With a unified URL, Let's Encrypt can't generate certificates unless it can reach both IPs through the same domain. diff --git a/doc/administration/logs/index.md b/doc/administration/logs/index.md index b61970b6fa5..01a45782d22 100644 --- a/doc/administration/logs/index.md +++ b/doc/administration/logs/index.md @@ -1044,7 +1044,7 @@ This file is located at: ## Registry logs -For Linux package installations, Container Registry logs are in `/var/log/gitlab/registry/current`. +For Linux package installations, container registry logs are in `/var/log/gitlab/registry/current`. ## NGINX logs diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index da0aa426f5e..e1221343de2 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -163,7 +163,7 @@ supported by the consolidated form, refer to the following guides: |---------------------|------------------------------------------| | [Secure Files](secure_files.md#using-object-storage) | **{dotted-circle}** No | | [Backups](../administration/backup_restore/backup_gitlab.md#upload-backups-to-a-remote-cloud-storage) | **{dotted-circle}** No | -| [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | **{dotted-circle}** No | +| [Container registry](packages/container_registry.md#use-object-storage) (optional feature) | **{dotted-circle}** No | | [Mattermost](https://docs.mattermost.com/configure/file-storage-configuration-settings.html)| **{dotted-circle}** No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | **{dotted-circle}** No | | [Job artifacts](job_artifacts.md#using-object-storage) including archived job logs | **{check-circle}** Yes | diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index ef45212f8a1..86b5b403b73 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -4,9 +4,9 @@ group: Container Registry info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# GitLab Container Registry administration **(FREE SELF)** +# GitLab container registry administration **(FREE SELF)** -With the GitLab Container Registry, every project can have its +With the GitLab container registry, every project can have its own space to store Docker images. For more details about the Distribution Registry: @@ -110,7 +110,7 @@ Because the container registry requires a TLS certificate, cost may be a factor. Take this into consideration before configuring the container registry for the first time. -### Configure Container Registry under an existing GitLab domain +### Configure container registry under an existing GitLab domain If the container registry is configured to use the existing GitLab domain, you can expose the container registry on a port. This way you can reuse the existing GitLab TLS @@ -201,7 +201,7 @@ credentials using: docker login gitlab.example.com:5050 ``` -### Configure Container Registry under its own domain +### Configure container registry under its own domain When the Registry is configured to use its own domain, you need a TLS certificate for that specific domain (for example, `registry.example.com`). You might need @@ -270,7 +270,7 @@ credentials: docker login registry.gitlab.example.com ``` -## Disable Container Registry site-wide +## Disable container registry site-wide When you disable the Registry by following these steps, you do not remove any existing Docker images. Docker image removal is handled by the @@ -302,7 +302,7 @@ Registry application itself. ::EndTabs -## Disable Container Registry for new projects site-wide +## Disable container registry for new projects site-wide If the container registry is enabled, then it should be available on all new projects. To disable this function and let the owners of a project to enable @@ -368,7 +368,7 @@ WARNING: Do not directly modify the files or objects stored by the container registry. Anything other than the registry writing or deleting these entries can lead to instance-wide data consistency and instability issues from which recovery may not be possible. You can configure the container registry to use various storage backends by -configuring a storage driver. By default the GitLab Container Registry +configuring a storage driver. By default the GitLab container registry is configured to use the [file system driver](#use-file-system) configuration. @@ -628,7 +628,7 @@ you can pull from the container registry, but you cannot push. flag deletes files that exist in the destination but not in the source. If you swap the source and destination, all data in the Registry is deleted. -1. Verify all Container Registry files have been uploaded to object storage +1. Verify all container registry files have been uploaded to object storage by looking at the file count returned by these two commands: ```shell @@ -832,7 +832,7 @@ In the examples below we set the Registry's port to `5010`. ::EndTabs -## Disable Container Registry per project +## Disable container registry per project If Registry is enabled in your GitLab instance, but you don't need it for your project, you can [disable it from your project's settings](../../user/project/settings/project_features_permissions.md#configure-project-features-and-permissions). @@ -842,7 +842,7 @@ project, you can [disable it from your project's settings](../../user/project/se WARNING: Using external container registries in GitLab is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/376217) in GitLab 15.8 and the end of support is scheduled for GitLab 16.0. -If you need to use external container registries instead of the GitLab Container Registry, +If you need to use external container registries instead of the GitLab container registry, tell us about your use cases in [feedback issue 958](https://gitlab.com/gitlab-org/container-registry/-/issues/958). If you use an external container registry, some features associated with the @@ -883,8 +883,8 @@ You can use GitLab as an auth endpoint with an external container registry. ``` - `gitlab_rails['registry_enabled'] = true` is needed to enable GitLab - Container Registry features and authentication endpoint. The GitLab bundled - Container Registry service does not start, even with this enabled. + container registry features and authentication endpoint. The GitLab bundled + container registry service does not start, even with this enabled. - `gitlab_rails['registry_api_url'] = "http://:5000"` must be changed to match the host where Registry is installed. It must also specify `https` if the external registry is @@ -945,7 +945,7 @@ You can use GitLab as an auth endpoint with an external container registry. 1. Save the file and [restart GitLab](../restart_gitlab.md#self-compiled-installations) for the changes to take effect. -## Configure Container Registry notifications +## Configure container registry notifications You can configure the container registry to send webhook notifications in response to events happening in the registry. @@ -1311,7 +1311,7 @@ for GitLab to run separately from Registry: Registry side. - `gitlab_rails['internal_key']`, contents of the key that GitLab uses to sign the tokens. -## Architecture of GitLab Container Registry +## Architecture of GitLab container registry The GitLab registry is what users use to store their own Docker images. Because of that the Registry is client facing, meaning that we expose it directly @@ -1355,10 +1355,10 @@ in GitLab 15.8 and the end of support occurred in GitLab 16.0. See the [deprecat The integration is not disabled in GitLab 16.0, but support for debugging and fixing issues is no longer provided. Additionally, the integration is no longer being developed or enhanced with new features. Third-party registry functionality might be completely removed -after the new GitLab Container Registry version is available for self-managed (see epic [5521](https://gitlab.com/groups/gitlab-org/-/epics/5521)). Only the GitLab Container Registry is planned to be supported. +after the new GitLab container registry version is available for self-managed (see epic [5521](https://gitlab.com/groups/gitlab-org/-/epics/5521)). Only the GitLab container registry is planned to be supported. This section has guidance for administrators migrating from third-party registries -to the GitLab Container Registry. If the third-party container registry you are using is not listed here, +to the GitLab container registry. If the third-party container registry you are using is not listed here, you can describe your use cases in [the feedback issue](https://gitlab.com/gitlab-org/container-registry/-/issues/958). For all of the instructions provided below, you should try them first on a test environment. @@ -1368,11 +1368,11 @@ Make sure everything continues to work as expected before replicating it in prod The [Docker Distribution Registry](https://docs.docker.com/registry/) was donated to the CNCF and is now known as the [Distribution Registry](https://github.com/distribution/distribution). -This registry is the open source implementation that the GitLab Container Registry is based on. -The GitLab Container Registry is compatible with the basic functionality provided by the Distribution Registry, -including all the supported storage backends. To migrate to the GitLab Container Registry +This registry is the open source implementation that the GitLab container registry is based on. +The GitLab container registry is compatible with the basic functionality provided by the Distribution Registry, +including all the supported storage backends. To migrate to the GitLab container registry you can follow the instructions on this page, and use the same storage backend as the Distribution Registry. -The GitLab Container Registry should accept the same configuration that you are using for the Distribution Registry. +The GitLab container registry should accept the same configuration that you are using for the Distribution Registry. ## Use a PostgreSQL database for metadata **(FREE SELF BETA)** @@ -1426,9 +1426,9 @@ Before diving in to the following sections, here's some basic troubleshooting: for errors (for example `/var/log/gitlab/gitlab-rails/production.log`). You may be able to find clues there. -### Using self-signed certificates with Container Registry +### Using self-signed certificates with container registry -If you're using a self-signed certificate with your Container Registry, you +If you're using a self-signed certificate with your container registry, you might encounter issues during the CI jobs like the following: ```plaintext @@ -1687,9 +1687,9 @@ was: - [Deprecated in GitLab 13.7](https://about.gitlab.com/releases/2020/12/22/gitlab-13-7-released/#deprecate-pulls-that-use-v1-of-the-docker-registry-api) - [Removed in GitLab 13.9](https://about.gitlab.com/releases/2021/02/22/gitlab-13-9-released/#deprecate-pulls-that-use-v1-of-the-docker-registry-api) -It's no longer possible to push or pull v1 images from the GitLab Container Registry. +It's no longer possible to push or pull v1 images from the GitLab container registry. -If you had v1 images in the GitLab Container Registry, but you did not upgrade them (following the +If you had v1 images in the GitLab container registry, but you did not upgrade them (following the [steps Docker recommends](https://docs.docker.com/registry/spec/deprecated-schema-v1/)) ahead of the GitLab 13.9 upgrade, these images are no longer accessible. If you try to pull them, this error appears: @@ -1697,7 +1697,7 @@ this error appears: - `Error response from daemon: manifest invalid: Schema 1 manifest not supported` For self-managed GitLab instances, you can regain access to these images by temporarily downgrading -the GitLab Container Registry to a version lower than `v3.0.0-gitlab`. Follow these steps to regain +the GitLab container registry to a version lower than `v3.0.0-gitlab`. Follow these steps to regain access to these images: 1. Downgrade the container registry to [`v2.13.1-gitlab`](https://gitlab.com/gitlab-org/container-registry/-/releases/v2.13.1-gitlab). @@ -1949,7 +1949,7 @@ Once the right permissions were set, the error goes away. ### Missing `gitlab-registry.key` prevents container repository deletion -If you disable your GitLab instance's Container Registry and try to remove a project that has +If you disable your GitLab instance's container registry and try to remove a project that has container repositories, the following error occurs: ```plaintext diff --git a/doc/administration/sidekiq/index.md b/doc/administration/sidekiq/index.md index 8377db277b3..b92ad13df07 100644 --- a/doc/administration/sidekiq/index.md +++ b/doc/administration/sidekiq/index.md @@ -217,7 +217,7 @@ node than Sidekiq, follow the steps below. sudo gitlab-ctl reconfigure ``` -1. In the instance where Container Registry is hosted, copy the `registry.key` +1. In the instance where container registry is hosted, copy the `registry.key` file to the Sidekiq node. ## Configure the Sidekiq metrics server diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index 6cd29d37dc4..3a03085c6da 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > The use of `CI_JOB_TOKEN` scoped to the current project was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49750) in GitLab 13.12. -This API documentation is about the [GitLab Container Registry](../user/packages/container_registry/index.md). +This API documentation is about the [GitLab container registry](../user/packages/container_registry/index.md). When the `ci_job_token_scope` feature flag is enabled (it is **disabled by default**), you can use the below endpoints from a CI/CD job, by passing the `$CI_JOB_TOKEN` variable as the `JOB-TOKEN` header. @@ -51,7 +51,7 @@ If the project is public, the container registry is also public. If the project private, the container registry is also internal or private. - **private**: The container registry is visible only to project members with Reporter role or -higher. This behavior is similar to that of a private project with Container Registry visibility set +higher. This behavior is similar to that of a private project with container registry visibility set to **enabled**. - **disabled**: The container registry is disabled. diff --git a/doc/api/packages/terraform-modules.md b/doc/api/packages/terraform-modules.md index 30775be5d28..70183bf261a 100644 --- a/doc/api/packages/terraform-modules.md +++ b/doc/api/packages/terraform-modules.md @@ -228,3 +228,39 @@ To write the output to file: ```shell curl --header "Authorization: Bearer " "https://gitlab.example.com/api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0/file" --output hello-world-local.tgz ``` + +## Upload module + +```plaintext +PUT /projects/:id/packages/terraform/modules/:module-name/:module-system/:module-version/file +``` + +| Attribute | Type | Required | Description | +|------------------|-------------------|----------|-------------| +| `id` | integer or string | yes | The ID or URL-encoded path of the project. | +| `module-name` | string | yes | The module name. | +| `module-system` | string | yes | The name of the module system or [provider](https://www.terraform.io/registry/providers). | +| `module-version` | string | yes | Specific module version to upload. | + +```shell +curl --fail-with-body \ + --header "PRIVATE-TOKEN: " \ + --upload-file path/to/file.tgz \ + --url "https://gitlab.example.com/api/v4/projects//packages/terraform/modules/my-module/my-system/0.0.1/file" +``` + +Tokens that can be used to authenticate: + +| Header | Value | +|-----------------|-------| +| `PRIVATE-TOKEN` | A [personal access token](../../user/profile/personal_access_tokens.md) with `api` scope. | +| `DEPLOY-TOKEN` | A [deploy token](../../user/project/deploy_tokens/index.md) with `write_package_registry` scope. | +| `JOB-TOKEN` | A [job token](../../ci/jobs/ci_job_token.md). | + +Example response: + +```json +{ + "message": "201 Created" +} +``` diff --git a/doc/api/runners.md b/doc/api/runners.md index bbb67918afc..61fb71f67da 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Runners API **(FREE ALL)** -This page describes endpoints for runners registered to an instance. To create a runner linked to the current user, see [Create a runner](users.md#create-a-runner). +This page describes endpoints for runners registered to an instance. To create a runner linked to the current user, see [Create a runner](users.md#create-a-runner-linked-to-a-user). [Pagination](rest/index.md#pagination) is available on the following API endpoints (they return 20 items by default): @@ -25,7 +25,7 @@ There are two tokens to take into account when connecting a runner with GitLab. | Token | Description | | ----- | ----------- | | Registration token | Token used to [register the runner](https://docs.gitlab.com/runner/register/). It can be [obtained through GitLab](../ci/runners/index.md). | -| Authentication token | Token used to authenticate the runner with the GitLab instance. It is obtained automatically when you [register a runner](https://docs.gitlab.com/runner/register/) or by the Runner API when you manually [register a runner](#register-a-new-runner) or [reset the authentication token](#reset-runners-authentication-token-by-using-the-runner-id). You can also obtain the authentication token using [Create a runner](users.md#create-a-runner) API method. | +| Authentication token | Token used to authenticate the runner with the GitLab instance. It is obtained automatically when you [register a runner](https://docs.gitlab.com/runner/register/) or by the Runner API when you manually [register a runner](#create-an-instance-runner) or [reset the authentication token](#reset-runners-authentication-token-by-using-the-runner-id). You can also obtain the authentication token using [Create a runner](users.md#create-a-runner-linked-to-a-user) API method. | Here's an example of how the two tokens are used in runner registration: @@ -654,9 +654,9 @@ Example response: ] ``` -## Register a new runner +## Create an instance runner -Register a new runner for the instance. +Create a runner for the instance. ```plaintext POST /runners diff --git a/doc/api/settings.md b/doc/api/settings.md index f65ee907b51..d0f2b44750f 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -345,7 +345,7 @@ listed in the descriptions of the relevant settings. | `container_registry_delete_tags_service_timeout` | integer | no | The maximum time, in seconds, that the cleanup process can take to delete a batch of tags for [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). | | `container_registry_expiration_policies_caching` | boolean | no | Caching during the execution of [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). | | `container_registry_expiration_policies_worker_capacity` | integer | no | Number of workers for [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). | -| `container_registry_token_expire_delay` | integer | no | Container Registry token duration in minutes. | +| `container_registry_token_expire_delay` | integer | no | Container registry token duration in minutes. | | `package_registry_cleanup_policies_worker_capacity` | integer | no | Number of workers assigned to the packages cleanup policies. | | `updating_name_disabled_for_users` | boolean | no | [Disable user profile name changes](../administration/settings/account_and_limit_settings.md#disable-user-profile-name-changes). | | `allow_account_deletion` | boolean | no | Enable [users to delete their accounts](../administration/settings/account_and_limit_settings.md#prevent-users-from-deleting-their-accounts). | diff --git a/doc/api/users.md b/doc/api/users.md index 53f99bb59f4..e82c4b1858d 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -2339,7 +2339,7 @@ Returns: - `403 Forbidden` if not authenticated as an administrator. - `404 User Not Found` if user cannot be found. -## Create a runner **(FREE ALL)** +## Create a runner linked to a user **(FREE ALL)** Creates a runner linked to the current user. diff --git a/doc/architecture/blueprints/cells/deployment-architecture.md b/doc/architecture/blueprints/cells/deployment-architecture.md index 57dabd447b4..1ec8461b138 100644 --- a/doc/architecture/blueprints/cells/deployment-architecture.md +++ b/doc/architecture/blueprints/cells/deployment-architecture.md @@ -96,7 +96,7 @@ The differences compared to [Initial Cells deployment](#3-initial-cells-deployme The differences compared to [Hybrid Cells deployment](#4-hybrid-cells-deployment---initial-complete-cells-architecture) are: -- The Routing Service is expanded to support [GitLab Pages](../../../user/project/pages/index.md) and [GitLab Container Registry](../../../user/packages/container_registry/index.md). +- The Routing Service is expanded to support [GitLab Pages](../../../user/project/pages/index.md) and [GitLab container registry](../../../user/packages/container_registry/index.md). - Each Cell has all services isolated. - It is allowed that some Cells will follow a [hybrid architecture](#4-hybrid-cells-deployment---initial-complete-cells-architecture). @@ -134,7 +134,7 @@ As per the architecture, the above services are required to be run Cell-local: | Service | | Uses | Migrate from cluster-wide to Cell | Description | | ------------------- | --------------- | ------------------------------- | ----------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | | **GitLab Pages** | GitLab-built | Routing Service, Rails API | No problem | Serving CI generated pages under `.gitlab.io` or custom domains | -| **GitLab Registry** | GitLab-built | Object Storage, PostgreSQL | Non-trivial data migration in case of split | Service to provide GitLab Container Registry | +| **GitLab Registry** | GitLab-built | Object Storage, PostgreSQL | Non-trivial data migration in case of split | Service to provide GitLab container registry | | **Gitaly Cluster** | GitLab-built | Disk storage, PostgreSQL | No problem: Built-in migration routines to balance Gitaly nodes | Gitaly holds Git repository data. Many Gitaly clusters can be configured in application. | | **Elasticsearch** | Managed service | Many nodes required by sharding | Time consuming: Rebuild cluster from scratch | Search across all projects | | **Object Storage** | Managed service | | Not straightforward: Rather hard to selectively migrate between buckets | Holds all user and CI uploaded files that is served by GitLab | diff --git a/doc/architecture/blueprints/cells/impacted_features/container-registry.md b/doc/architecture/blueprints/cells/impacted_features/container-registry.md index b2379a9c0bc..4c6ac36a2af 100644 --- a/doc/architecture/blueprints/cells/impacted_features/container-registry.md +++ b/doc/architecture/blueprints/cells/impacted_features/container-registry.md @@ -19,22 +19,22 @@ GitLab [Container Registry](../../../../user/packages/container_registry/index.m ## 1. Definition -GitLab Container Registry is a complex service requiring usage of PostgreSQL, Redis and Object Storage dependencies. -Right now there's undergoing work to introduce [Container Registry Metadata](../../container_registry_metadata_database/index.md) to optimize data storage and image retention policies of Container Registry. +GitLab container registry is a complex service requiring usage of PostgreSQL, Redis and Object Storage dependencies. +Right now there's undergoing work to introduce [Container Registry Metadata](../../container_registry_metadata_database/index.md) to optimize data storage and image retention policies of container registry. -GitLab Container Registry is serving as a container for stored data, but on its own does not authenticate `docker login`. +GitLab container registry is serving as a container for stored data, but on its own does not authenticate `docker login`. The `docker login` is executed with user credentials (can be `personal access token`) or CI build credentials (ephemeral `ci_builds.token`). Container Registry uses data deduplication. It means that the same blob (image layer) that is shared between many Projects is stored only once. Each layer is hashed by `sha256`. -The `docker login` does request a JWT time-limited authentication token that is signed by GitLab, but validated by Container Registry service. +The `docker login` does request a JWT time-limited authentication token that is signed by GitLab, but validated by container registry service. The JWT token does store all authorized scopes (`container repository images`) and operation types (`push` or `pull`). A single JWT authentication token can have many authorized scopes. -This allows Container Registry and client to mount existing blobs from other scopes. +This allows container registry and client to mount existing blobs from other scopes. GitLab responds only with authorized scopes. -Then it is up to GitLab Container Registry to validate if the given operation can be performed. +Then it is up to GitLab container registry to validate if the given operation can be performed. The GitLab.com pages are always scoped to a Project. Each Project can have many container registry images attached. @@ -88,24 +88,24 @@ curl \ ## 3. Proposal -### 3.1. Shard Container Registry separately to Cells architecture +### 3.1. Shard container registry separately to Cells architecture -Due to its extensive and in general highly scalable horizontal architecture it should be evaluated if the GitLab Container Registry should be run not in Cell, but in a Cluster and be scaled independently. +Due to its extensive and in general highly scalable horizontal architecture it should be evaluated if the GitLab container registry should be run not in Cell, but in a Cluster and be scaled independently. This might be easier, but would definitely not offer the same amount of data isolation. -### 3.2. Run Container Registry within a Cell +### 3.2. Run container registry within a Cell It appears that except `/jwt/auth` which would likely have to be processed by Router (to decode `scope`) the container registry could be run as a local service of a Cell. The actual data at least in case of GitLab.com is not forwarded via registry, but rather served directly from Object Storage / CDN. Its design encodes container repository image in a URL that is easily routable. -It appears that we could re-use the same stateless Router service in front of Container Registry to serve manifests and blobs redirect. +It appears that we could re-use the same stateless Router service in front of container registry to serve manifests and blobs redirect. The only downside is increased complexity of managing standalone registry for each Cell, but this might be desired approach. ## 4. Evaluation -There do not seem to be any theoretical problems with running GitLab Container Registry in a Cell. +There do not seem to be any theoretical problems with running GitLab container registry in a Cell. It seems that the service can be easily made routable to work well. The practical complexities are around managing a complex service from an infrastructure side. diff --git a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md index 5b82716cb21..fc35cc0dd50 100644 --- a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md +++ b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md @@ -124,7 +124,7 @@ application layers. This list is not exhaustive, but shows a general list of the - Web GraphQL: provide a flexible API interface, allowing the Web frontend to fetch only the data needed thereby reducing the amount of compute and data transfer - Web ActionCable: provide bi-directional connection to enable real-time features for Users visiting web interface - Web Feature Flags Unleash Backend: provide an Unleash-compatible Server that uses GitLab API -- Web Packages API: provide a REST API compatible with the packaging tools: Debian, Maven, Container Registry Proxy, etc. +- Web Packages API: provide a REST API compatible with the packaging tools: Debian, Maven, container registry proxy, etc. - Git nodes: all code required to authorize `git pull/push` over `SSH` or `HTTPS` - Sidekiq: run background jobs - Services/Models/DB: all code required to maintain our database structure, data validation, business logic, and policies models that needs to be shared with other components diff --git a/doc/architecture/blueprints/container_registry_metadata_database/index.md b/doc/architecture/blueprints/container_registry_metadata_database/index.md index 74d7c947d2f..b62dd660643 100644 --- a/doc/architecture/blueprints/container_registry_metadata_database/index.md +++ b/doc/architecture/blueprints/container_registry_metadata_database/index.md @@ -16,7 +16,7 @@ participating-stages: [] With the [container registry](https://gitlab.com/gitlab-org/container-registry) integrated into GitLab, every GitLab project can have its own space to store its Docker images. You can use the registry to build, push and share images using the Docker client, CI/CD or the GitLab API. -Each day on GitLab.com, between [150k and 200k images are pushed to the registry](https://app.periscopedata.com/app/gitlab/527857/Package-GitLab.com-Stage-Activity-Dashboard?widget=9620193&udv=0), generating about [700k API events](https://app.periscopedata.com/app/gitlab/527857/Package-GitLab.com-Stage-Activity-Dashboard?widget=7601761&udv=0). It's also worth noting that although some customers use other registry vendors, [more than 96% of instances](https://app.periscopedata.com/app/gitlab/527857/Package-GitLab.com-Stage-Activity-Dashboard?widget=9832282&udv=0) are using the GitLab Container Registry. +Each day on GitLab.com, between [150k and 200k images are pushed to the registry](https://app.periscopedata.com/app/gitlab/527857/Package-GitLab.com-Stage-Activity-Dashboard?widget=9620193&udv=0), generating about [700k API events](https://app.periscopedata.com/app/gitlab/527857/Package-GitLab.com-Stage-Activity-Dashboard?widget=7601761&udv=0). It's also worth noting that although some customers use other registry vendors, [more than 96% of instances](https://app.periscopedata.com/app/gitlab/527857/Package-GitLab.com-Stage-Activity-Dashboard?widget=9832282&udv=0) are using the GitLab container registry. For GitLab.com and for GitLab customers, the container registry is a critical component to building and deploying software. @@ -42,18 +42,18 @@ The container registry has two main clients: the GitLab Rails application and th #### Docker -The Docker client (`docker` CLI) interacts with the GitLab Container Registry mainly using the [login](https://docs.docker.com/engine/reference/commandline/login/), [push](https://docs.docker.com/engine/reference/commandline/push/) and [pull](https://docs.docker.com/engine/reference/commandline/pull/) commands. +The Docker client (`docker` CLI) interacts with the GitLab container registry mainly using the [login](https://docs.docker.com/engine/reference/commandline/login/), [push](https://docs.docker.com/engine/reference/commandline/push/) and [pull](https://docs.docker.com/engine/reference/commandline/pull/) commands. ##### Login and Authentication -GitLab Rails is the default token-based authentication provider for the GitLab Container Registry. +GitLab Rails is the default token-based authentication provider for the GitLab container registry. Once the registry receives a request sent by an unauthenticated Docker client, it will reply with `401 Unauthorized` and instruct the client to obtain a token from the GitLab Rails API. The Docker client will then request a Bearer token and embed it in the `Authorization` header of all requests. The registry is responsible for determining if the user is authentication/authorized to perform those requests based on the provided token. ```mermaid sequenceDiagram participant C as Docker client - participant R as GitLab Container Registry + participant R as GitLab container registry participant G as GitLab Rails C->>R: docker login gitlab.example.com @@ -81,7 +81,7 @@ The single entrypoint for the registry is the [HTTP API](https://gitlab.com/gitl | Operation | UI | Background | Observations | | ------------------------------------------------------------ | ------------------ | ------------------------ | ------------------------------------------------------------ | -| [Check API version](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#api-version-check) | **{check-circle}** Yes | **{check-circle}** Yes | Used globally to ensure that the registry supports the Docker Distribution V2 API, as well as for identifying whether GitLab Rails is talking to the GitLab Container Registry or a third-party one (used to toggle features only available in the former). | +| [Check API version](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#api-version-check) | **{check-circle}** Yes | **{check-circle}** Yes | Used globally to ensure that the registry supports the Docker Distribution V2 API, as well as for identifying whether GitLab Rails is talking to the GitLab container registry or a third-party one (used to toggle features only available in the former). | | [List repository tags](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#listing-image-tags) | **{check-circle}** Yes | **{check-circle}** Yes | Used to list and show tags in the UI. Used to list tags in the background for [cleanup policies](../../../user/packages/container_registry/reduce_container_registry_storage.md#cleanup-policy) and [Geo replication](../../../administration/geo/replication/container_registry.md). | | [Check if manifest exists](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#existing-manifests) | **{check-circle}** Yes | **{dotted-circle}** No | Used to get the digest of a manifest by tag. This is then used to pull the manifest and show the tag details in the UI. | | [Pull manifest](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#pulling-an-image-manifest) | **{check-circle}** Yes | **{dotted-circle}** No | Used to show the image size and the manifest digest in the tag details UI. | @@ -208,7 +208,7 @@ for self-managed instances. Customers not able to upgrade to PostgreSQL 12 have with security backports and bug fixes. Apart from online garbage collection, the metadata database's availability unblocks the -implementation of many requested features for the GitLab Container Registry. These features are only +implementation of many requested features for the GitLab container registry. These features are only available for instances using the new version backed by the metadata database. ### Availability @@ -294,29 +294,29 @@ Together, these resources should provide an adequate level of insight into the r #### Third-Party Container Registries -GitLab ships with the GitLab Container Registry by default, but it's also compatible with third-party registries, as long as they comply with the [Docker Distribution V2 Specification](https://docs.docker.com/registry/spec/api/), now superseded by the [Open Container Initiative (OCI) Image Specification](https://github.com/opencontainers/image-spec/blob/master/spec.md). +GitLab ships with the GitLab container registry by default, but it's also compatible with third-party registries, as long as they comply with the [Docker Distribution V2 Specification](https://docs.docker.com/registry/spec/api/), now superseded by the [Open Container Initiative (OCI) Image Specification](https://github.com/opencontainers/image-spec/blob/master/spec.md). So far, we have tried to maintain full compatibility with third-party registries when adding new features. For example, in 12.8, we introduced a new [tag delete feature](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23325) to delete a single tag without deleting the underlying manifest. Because this feature is not part of the Docker or OCI specifications, we have kept the previous behavior as a fallback option to maintain compatibility with third-party registries. -However, this will likely change in the future. Apart from online garbage collection, and as described in [challenges](#challenges), the metadata database will unblock the implementation of many requested features for the GitLab Container Registry in the mid/long term. Most of these features will only be available for instances using the GitLab Container Registry. They are not part of the Docker Distribution or OCI specifications, neither we will be able to provide a compatible fallback option. +However, this will likely change in the future. Apart from online garbage collection, and as described in [challenges](#challenges), the metadata database will unblock the implementation of many requested features for the GitLab container registry in the mid/long term. Most of these features will only be available for instances using the GitLab container registry. They are not part of the Docker Distribution or OCI specifications, neither we will be able to provide a compatible fallback option. -For this reason, any features that require the use of the GitLab Container Registry will be disabled if using a third-party registry, for as long as third-party registries continue to be supported. +For this reason, any features that require the use of the GitLab container registry will be disabled if using a third-party registry, for as long as third-party registries continue to be supported. #### Synchronizing Changes With GitLab Rails -Currently, the GitLab Rails and GitLab Container Registry releases and deployments have been fully independent, as we have not introduced any new API features or breaking changes, apart from the described tag delete feature. +Currently, the GitLab Rails and GitLab container registry releases and deployments have been fully independent, as we have not introduced any new API features or breaking changes, apart from the described tag delete feature. The registry will remain independent from GitLab Rails changes, but in the mid/long term, the implementation of new features or breaking changes will imply a corresponding change in GitLab Rails, so the latter will depend on a specific minimum version of the registry. For example, to track the size of each repository, we may extend the metadata database to store that information and then propagate it to GitLab Rails by extending the HTTP API that it consumes. In GitLab Rails, this new information would likely be stored in its database and processed to offer a new feature at the UI/API level. -This kind of changes will require a synchronization between the GitLab Rails and the GitLab Container Registry releases and deployments, as the former will depend on a specific version of the latter. +This kind of changes will require a synchronization between the GitLab Rails and the GitLab container registry releases and deployments, as the former will depend on a specific version of the latter. ##### Feature Toggling All GitLab Rails features dependent on a specific version of the registry should be guarded by validating the registry vendor and version. -This is already done to determine whether a tag should be deleted using the new tag delete feature (only available in the GitLab Container Registry v2.8.1+) or the old method. In this case, GitLab Rails sends an `OPTIONS` request to the registry tag route to determine whether the `DELETE` method is supported or not. +This is already done to determine whether a tag should be deleted using the new tag delete feature (only available in the GitLab container registry v2.8.1+) or the old method. In this case, GitLab Rails sends an `OPTIONS` request to the registry tag route to determine whether the `DELETE` method is supported or not. Alternatively, and as the universal long-term solution, we need to determine the registry vendor, version, and supported features (the last two are only applicable if the vendor is GitLab) and persist it in the GitLab Rails database. This information can then be used in real time to toggle features or fallback to alternative methods, if possible. The initial implementation of this approach was introduced as part of [#204839](https://gitlab.com/gitlab-org/gitlab/-/issues/204839). Currently, it's only used for metrics purposes. Further improvements are required to guarantee that the version information is kept up to date in self-managed instances, where the registry may be hot swapped. @@ -324,7 +324,7 @@ Alternatively, and as the universal long-term solution, we need to determine the As described above, feature toggling offers a last line of defense against desynchronized releases and deployments, ensuring that GitLab Rails remains functional in case the registry version that supports new features is not yet available. -However, the release and deployment of GitLab Rails and the GitLab Container Registry should be synchronized to avoid any delays. Contrary to GitLab Rails, the registry release and deployment are manual processes, so special attention must be paid by maintainers to ensure that the GitLab Rails changes are only released and deployed after the corresponding registry changes. +However, the release and deployment of GitLab Rails and the GitLab container registry should be synchronized to avoid any delays. Contrary to GitLab Rails, the registry release and deployment are manual processes, so special attention must be paid by maintainers to ensure that the GitLab Rails changes are only released and deployed after the corresponding registry changes. As a solution to strengthen this process, a file can be added to the GitLab Rails codebase, containing the minimum required version of the registry. This file should be updated with every change that depends on a specific version of the registry. It should also be considered when releasing and deploying GitLab Rails, ensuring that the pipeline only goes through once the specified minimum required registry version is deployed. diff --git a/doc/architecture/blueprints/container_registry_metadata_database_self_managed_rollout/index.md b/doc/architecture/blueprints/container_registry_metadata_database_self_managed_rollout/index.md index b2f4eca5f71..4e81761839c 100644 --- a/doc/architecture/blueprints/container_registry_metadata_database_self_managed_rollout/index.md +++ b/doc/architecture/blueprints/container_registry_metadata_database_self_managed_rollout/index.md @@ -50,7 +50,7 @@ of the container registry for both GitLab.com and for self-managed users. ### Non-Goals -- Developing new Container Registry features outside the scope of enabling admins to migrate to the metadata database. +- Developing new container registry features outside the scope of enabling admins to migrate to the metadata database. - Determining lifecycle support decisions, such as when to default to the database, and when to end support for non-database registries. ## Proposal diff --git a/doc/architecture/blueprints/google_artifact_registry_integration/index.md b/doc/architecture/blueprints/google_artifact_registry_integration/index.md index ef66ae33b2a..78a158f7baa 100644 --- a/doc/architecture/blueprints/google_artifact_registry_integration/index.md +++ b/doc/architecture/blueprints/google_artifact_registry_integration/index.md @@ -20,7 +20,7 @@ As highlighted in the announcement, one key goal is the ability to "_use Google' Please refer to the [announcement](https://about.gitlab.com/blog/2023/08/29/gitlab-google-partnership-s3c/) blog post for more details about the motivation and long-term goals of the GitLab and Google Cloud partnership. -Regarding the scope of this design document, our primary focus is to fulfill the Product requirement of providing users with visibility over their container images in GAR. The motivation for this specific goal is rooted in foundational research on the use of external registries as a complement to the GitLab Container Registry ([internal](https://gitlab.com/gitlab-org/ux-research/-/issues/2602)). +Regarding the scope of this design document, our primary focus is to fulfill the Product requirement of providing users with visibility over their container images in GAR. The motivation for this specific goal is rooted in foundational research on the use of external registries as a complement to the GitLab container registry ([internal](https://gitlab.com/gitlab-org/ux-research/-/issues/2602)). Since this marks the first step in the GAR integration, our aim is to achieve this goal in a way that establishes a foundation to facilitate reusability in the future. This groundwork could benefit potential future expansions, such as support for additional artifact formats (npm, Maven, etc.), and features beyond the Package stage (e.g., vulnerability scanning, deployments, etc.). @@ -74,7 +74,7 @@ As previously highlighted, access to the GAR integration features is restricted #### Resource Mapping -For the [GitLab Container Registry](../../../user/packages/container_registry/index.md), repositories within a specific project must have a path that matches the project full path. This is essentially how we establish a resource mapping between GitLab Rails and the registry, which serves multiple purposes, including granular authorization, scoping storage usage to a given project/group/namespace, and more. +For the [GitLab container registry](../../../user/packages/container_registry/index.md), repositories within a specific project must have a path that matches the project full path. This is essentially how we establish a resource mapping between GitLab Rails and the registry, which serves multiple purposes, including granular authorization, scoping storage usage to a given project/group/namespace, and more. Regarding the GAR integration, since there is no equivalent entities for GitLab project/group/namespace resources on the GAR side, we aim to simplify matters by allowing users to attach any [GAR repository](https://cloud.google.com/artifact-registry/docs/repositories) to any GitLab project, regardless of their respective paths. Similarly, we do not plan to restrict the attachment of a particular GAR repository to a single GitLab project. Ultimately, it is up to users to determine how to organize both datasets in the way that best suits their needs. @@ -116,6 +116,6 @@ One alternative solution considered was to use the Docker/OCI API provided by GA - **Multiple Requests**: To retrieve all the required information about each image, multiple requests to different endpoints (listing tags, obtaining image manifests, and image configuration blobs) would have been necessary, leading to a `1+N` performance issue. -GitLab had previously faced significant challenges with the last two limitations, prompting the development of a custom [GitLab Container Registry API](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/gitlab/api.md) to address them. Additionally, GitLab decided to [deprecate support](../../../update/deprecations.md#use-of-third-party-container-registries-is-deprecated) for connecting to third-party container registries using the Docker/OCI API due to these same limitations and the increased cost of maintaining two solutions in parallel. As a result, there is an ongoing effort to replace the use of the Docker/OCI API endpoints with custom API endpoints for all container registry functionalities in GitLab. +GitLab had previously faced significant challenges with the last two limitations, prompting the development of a custom [GitLab container registry API](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/gitlab/api.md) to address them. Additionally, GitLab decided to [deprecate support](../../../update/deprecations.md#use-of-third-party-container-registries-is-deprecated) for connecting to third-party container registries using the Docker/OCI API due to these same limitations and the increased cost of maintaining two solutions in parallel. As a result, there is an ongoing effort to replace the use of the Docker/OCI API endpoints with custom API endpoints for all container registry functionalities in GitLab. Considering these factors, the decision was made to build the GAR integration from scratch using the proprietary GAR API. This approach provides more flexibility and control over the integration and can serve as a foundation for future expansions, such as support for other GAR artifact formats. diff --git a/doc/architecture/blueprints/google_artifact_registry_integration/ui_ux.md b/doc/architecture/blueprints/google_artifact_registry_integration/ui_ux.md index 5cb862d50e7..f82889b9ccf 100644 --- a/doc/architecture/blueprints/google_artifact_registry_integration/ui_ux.md +++ b/doc/architecture/blueprints/google_artifact_registry_integration/ui_ux.md @@ -8,9 +8,9 @@ description: 'UI/UX for Google Artifact Registry Integration' ## Structure and Organization -Unlike the GitLab Container Registry (and therefore the Docker Registry and OCI Distribution), GAR does not treat tags as the primary "artifacts" in a repository. Instead, the primary "artifacts" are the image manifests. For each manifest object (represented by [`DockerImage`](https://cloud.google.com/artifact-registry/docs/reference/rpc/google.devtools.artifactregistry.v1#google.devtools.artifactregistry.v1.DockerImage)), there is a list of assigned tags (if any). Consequently, when listing the contents of a repository through the GAR API, the response comprises a collection of manifest objects (along with their associated tags as properties), rather than a collection of tag objects. Additionally, due to this design choice, untagged manifests are also present in the response. +Unlike the GitLab container registry (and therefore the Docker Registry and OCI Distribution), GAR does not treat tags as the primary "artifacts" in a repository. Instead, the primary "artifacts" are the image manifests. For each manifest object (represented by [`DockerImage`](https://cloud.google.com/artifact-registry/docs/reference/rpc/google.devtools.artifactregistry.v1#google.devtools.artifactregistry.v1.DockerImage)), there is a list of assigned tags (if any). Consequently, when listing the contents of a repository through the GAR API, the response comprises a collection of manifest objects (along with their associated tags as properties), rather than a collection of tag objects. Additionally, due to this design choice, untagged manifests are also present in the response. -To maximize flexibility, extensibility, and maintain familiarity for GAR users, we plan to fully embrace the GAR API data structures while surfacing data in the GitLab UI. We won't attempt to emulate a "list of tags" response to match the UI/UX that we already have for the GitLab Container Registry. +To maximize flexibility, extensibility, and maintain familiarity for GAR users, we plan to fully embrace the GAR API data structures while surfacing data in the GitLab UI. We won't attempt to emulate a "list of tags" response to match the UI/UX that we already have for the GitLab container registry. Considering the above, there will be a view that provides a pageable and sortable list of all images in the configured GAR repository. Additionally, there will be a detail view to display more information about a single image. You can find a list of available image attributes documented [here](https://cloud.google.com/artifact-registry/docs/reference/rpc/google.devtools.artifactregistry.v1#google.devtools.artifactregistry.v1.DockerImage). diff --git a/doc/architecture/blueprints/rate_limiting/index.md b/doc/architecture/blueprints/rate_limiting/index.md index 7af50097e97..94a36d721e9 100644 --- a/doc/architecture/blueprints/rate_limiting/index.md +++ b/doc/architecture/blueprints/rate_limiting/index.md @@ -218,7 +218,7 @@ communicate how the responsible usage is defined at a given moment. Because of how GitLab architecture has been built, GitLab Rails application, in most cases, behaves as a central enterprise service bus (ESB) and there are a -few satellite services communicating with it. Services like Container Registry, +few satellite services communicating with it. Services like container registry, GitLab Runners, Gitaly, Workhorse, KAS could use the API to receive a set of application limits those are supposed to enforce. This will still allow us to define all of them in a single place. diff --git a/doc/architecture/blueprints/runner_tokens/index.md b/doc/architecture/blueprints/runner_tokens/index.md index b4ac818875e..5e1300f9a3b 100644 --- a/doc/architecture/blueprints/runner_tokens/index.md +++ b/doc/architecture/blueprints/runner_tokens/index.md @@ -329,7 +329,7 @@ enum column created in the `ci_runners` table. ### Runner creation through API Automated runner creation is possible through a new GraphQL mutation and the existing -[`POST /user/runners` REST API endpoint](../../../api/users.md#create-a-runner). +[`POST /user/runners` REST API endpoint](../../../api/users.md#create-a-runner-linked-to-a-user). These endpoints are only available to users that are [allowed](../../../user/permissions.md#gitlab-cicd-permissions) to create runners at the specified scope. @@ -413,7 +413,7 @@ scope. | Component | Milestone | Changes | |------------------|----------:|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | GitLab Rails app | `%16.0` | Adapt `register_{group|project}_runner` permissions to take [application setting](https://gitlab.com/gitlab-org/gitlab/-/issues/386712) in consideration. | -| GitLab Rails app | `%16.1` | Make [`POST /api/v4/runners` endpoint](../../../api/runners.md#register-a-new-runner) permanently return `HTTP 410 Gone` if either `allow_runner_registration_token` setting disables registration tokens.
A future v5 version of the API should return `HTTP 404 Not Found`. | +| GitLab Rails app | `%16.1` | Make [`POST /api/v4/runners` endpoint](../../../api/runners.md#create-an-instance-runner) permanently return `HTTP 410 Gone` if either `allow_runner_registration_token` setting disables registration tokens.
A future v5 version of the API should return `HTTP 404 Not Found`. | | GitLab Rails app | `%16.1` | Add runner group metadata to the runner list. | | GitLab Rails app | | Add UI to allow disabling use of registration tokens in top-level group settings. | | GitLab Rails app | | Hide legacy UI showing registration with a registration token, if it disabled on in top-level group settings or by admins. | diff --git a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md index 2a73184eb7f..4ccf8e5d276 100644 --- a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md +++ b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md @@ -56,7 +56,7 @@ bare-bones application built on some well-known frameworks. Now that you created a demo project, you must containerize the application and push it to the container registry. -### Push a containerized application image to GitLab Container Registry +### Push a containerized application image to GitLab container registry [ECS](https://aws.amazon.com/ecs/) is a container orchestration service, meaning that you must provide a containerized application image during the infrastructure build. To do so, you can use @@ -75,7 +75,7 @@ and [Container Registry](../../../user/packages/container_registry/index.md). ``` 1. Select **Commit Changes**. It automatically triggers a new pipeline. In this pipeline, the `build` - job containerizes the application and pushes the image to [GitLab Container Registry](../../../user/packages/container_registry/index.md). + job containerizes the application and pushes the image to [GitLab container registry](../../../user/packages/container_registry/index.md). ![Create project](img/initial-pipeline.png) @@ -106,7 +106,7 @@ is a specification about how the application image is started by an [ECS service 1. Select **Container Definitions > Add container**. This opens a container registration form. 1. Set `web` to **Container name**. 1. Set `registry.gitlab.com//ecs-demo/master:latest` to **Image**. - Alternatively, you can copy and paste the image path from the [GitLab Container Registry page](#push-a-containerized-application-image-to-gitlab-container-registry). + Alternatively, you can copy and paste the image path from the [GitLab container registry page](#push-a-containerized-application-image-to-gitlab-container-registry). ![Create project](img/container-name.png) diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index 16bfced9c34..0d601e2a4a8 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -61,7 +61,7 @@ deploy: GitLab provides a Docker image that includes the AWS CLI: -- Images are hosted in the GitLab Container Registry. The latest image is +- Images are hosted in the GitLab container registry. The latest image is `registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest`. - [Images are stored in a GitLab repository](https://gitlab.com/gitlab-org/cloud-deploy/-/tree/master/aws). @@ -120,7 +120,7 @@ To deploy to your ECS cluster: 1. Commit and push your updated `.gitlab-ci.yml` to your project's repository. -Your application Docker image is rebuilt and pushed to the GitLab Container Registry. +Your application Docker image is rebuilt and pushed to the GitLab container registry. If your image is located in a private registry, make sure your task definition is [configured with a `repositoryCredentials` attribute](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/private-auth.html). diff --git a/doc/ci/docker/buildah_rootless_tutorial.md b/doc/ci/docker/buildah_rootless_tutorial.md index 17482fbc0c3..9597d22f92a 100644 --- a/doc/ci/docker/buildah_rootless_tutorial.md +++ b/doc/ci/docker/buildah_rootless_tutorial.md @@ -53,7 +53,7 @@ We start by preparing a custom image based on the `quay.io/buildah/stable:v1.23. ``` 1. Build and push the Buildah image to a container registry. Let's push to the - [GitLab Container Registry](../../user/packages/container_registry/index.md): + [GitLab container registry](../../user/packages/container_registry/index.md): ```shell docker build -f Containerfile-buildah -t registry.example.com/group/project/buildah:1.23.1 . diff --git a/doc/ci/docker/index.md b/doc/ci/docker/index.md index 49a9e1f8eea..d1dff953a5c 100644 --- a/doc/ci/docker/index.md +++ b/doc/ci/docker/index.md @@ -15,7 +15,7 @@ There are two primary ways to incorporate [Docker](https://www.docker.com) into an application. These jobs can run in Docker containers. For example, you can tell GitLab CI/CD to use a Node image that's hosted on Docker Hub - or in the GitLab Container Registry. Your job then runs in a container that's based on the image. + or in the GitLab container registry. Your job then runs in a container that's based on the image. The container has all the Node dependencies you need to build your app. - **Use [Docker](using_docker_build.md) or [kaniko](using_kaniko.md) to build Docker images.** diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 855d2731ec1..a2ef28bdaa0 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -708,11 +708,11 @@ of the following executors: In this example, you use Buildah to: 1. Build a Docker image. -1. Push it to [GitLab Container Registry](../../user/packages/container_registry/index.md). +1. Push it to [GitLab container registry](../../user/packages/container_registry/index.md). In the last step, Buildah uses the `Dockerfile` under the root directory of the project to build the Docker image. Finally, it pushes the image to the -project's Container Registry: +project's container registry: ```yaml build: @@ -728,7 +728,7 @@ build: BUILDAH_FORMAT: docker FQ_IMAGE_NAME: "$CI_REGISTRY_IMAGE/test" before_script: - # GitLab Container Registry credentials taken from the + # GitLab container registry credentials taken from the # [predefined CI/CD variables](../variables/index.md#predefined-cicd-variables) # to authenticate to the registry. - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY @@ -742,10 +742,10 @@ build: If you are using GitLab Runner Operator deployed to an OpenShift cluster, try the [tutorial for using Buildah to build images in rootless container](buildah_rootless_tutorial.md). -## Use the GitLab Container Registry +## Use the GitLab container registry After you've built a Docker image, you can push it to the -[GitLab Container Registry](../../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd). +[GitLab container registry](../../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd). ## Troubleshooting diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index dd6cd2099a9..06baab38e41 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -203,7 +203,7 @@ Look for the `[runners.docker]` section: The image and services defined this way are added to all jobs run by that runner. -## Access an image from a private Container Registry +## Access an image from a private container registry To access private container registries, the GitLab Runner process can use: @@ -534,11 +534,11 @@ and update Docker images on Amazon ECR, without using manual credential manageme build-image: stage: build script: - - echo "Logging into GitLab Container Registry..." + - echo "Logging into GitLab container registry..." - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - echo "Building Docker image..." - docker build --build-arg GITLAB_RUNNER_VERSION=${GITLAB_RUNNER_VERSION} --build-arg AWS_CLI_VERSION=${AWS_CLI_VERSION} -t ${IMAGE_NAME} . - - echo "Pushing Docker image to GitLab Container Registry..." + - echo "Pushing Docker image to GitLab container registry..." - docker push ${IMAGE_NAME} rules: - changes: diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index 8ab13c7154d..d0a437b6f62 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -43,16 +43,16 @@ few important details: In the following example, kaniko is used to: 1. Build a Docker image. -1. Then push it to [GitLab Container Registry](../../user/packages/container_registry/index.md). +1. Then push it to [GitLab container registry](../../user/packages/container_registry/index.md). The job runs only when a tag is pushed. A `config.json` file is created under -`/kaniko/.docker` with the needed GitLab Container Registry credentials taken from the +`/kaniko/.docker` with the needed GitLab container registry credentials taken from the [predefined CI/CD variables](../variables/index.md#predefined-cicd-variables) GitLab CI/CD provides. These are automatically read by the Kaniko tool. In the last step, kaniko uses the `Dockerfile` under the root directory of the project, builds the Docker image and pushes it to the -project's Container Registry while tagging it with the Git tag: +project's container registry while tagging it with the Git tag: ```yaml build: diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index f4c1ab4402c..88adfe873ca 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -430,11 +430,11 @@ We added the [official PHP 7.4 Docker image](https://hub.docker.com/_/php), whic We used `docker-php-ext-install` (provided by the official PHP Docker image) to install the PHP extensions we need. -#### Setting Up GitLab Container Registry +#### Setting Up GitLab container registry -Now that we have our `Dockerfile` let's build and push it to our [GitLab Container Registry](../../../user/packages/container_registry/index.md). +Now that we have our `Dockerfile` let's build and push it to our [GitLab container registry](../../../user/packages/container_registry/index.md). -> The registry is the place to store and tag images for later use. Developers may want to maintain their own registry for private, company images, or for throw-away images used only in testing. Using GitLab Container Registry means you don't need to set up and administer yet another service or use a public registry. +> The registry is the place to store and tag images for later use. Developers may want to maintain their own registry for private, company images, or for throw-away images used only in testing. Using GitLab container registry means you don't need to set up and administer yet another service or use a public registry. On your GitLab project repository navigate to the **Registry** tab. @@ -442,7 +442,7 @@ On your GitLab project repository navigate to the **Registry** tab. You may need to enable the container registry for your project to see this tab. You'll find it under your project's **Settings > General > Visibility, project features, permissions**. -To start using Container Registry on our machine, we first need to sign in to the GitLab registry using our GitLab username and password. +To start using container registry on our machine, we first need to sign in to the GitLab registry using our GitLab username and password. Make sure you have [Docker](https://docs.docker.com/engine/install/) installed on our machine, then run the following commands: diff --git a/doc/ci/runners/new_creation_workflow.md b/doc/ci/runners/new_creation_workflow.md index 022f7af11ec..026e4ec6470 100644 --- a/doc/ci/runners/new_creation_workflow.md +++ b/doc/ci/runners/new_creation_workflow.md @@ -141,7 +141,7 @@ process is started. ## Creating runners programmatically -In GitLab 15.11 and later, you can use the [POST /user/runners REST API](../../api/users.md#create-a-runner) +In GitLab 15.11 and later, you can use the [POST /user/runners REST API](../../api/users.md#create-a-runner-linked-to-a-user) to create a runner as an authenticated user. This should only be used if the runner configuration is dynamic or not reusable. If the runner configuration is static, you should reuse the runner authentication token of an existing runner. diff --git a/doc/ci/runners/runners_scope.md b/doc/ci/runners/runners_scope.md index 52ce46682d6..46108931757 100644 --- a/doc/ci/runners/runners_scope.md +++ b/doc/ci/runners/runners_scope.md @@ -65,7 +65,7 @@ To create a shared runner: - For the `executor`, enter the type of [executor](https://docs.gitlab.com/runner/executors/). The executor is the environment where the runner executes the job. -You can also [use the API](../../api/users.md#create-a-runner) to create a runner. +You can also [use the API](../../api/users.md#create-a-runner-linked-to-a-user) to create a runner. NOTE: The runner authentication token displays in the UI for a limited period of time during registration. After you register the runner, @@ -263,7 +263,7 @@ To create a group runner: - For the `executor`, enter the type of [executor](https://docs.gitlab.com/runner/executors/). The executor is the environment where the runner executes the job. -You can also [use the API](../../api/users.md#create-a-runner) to create a runner. +You can also [use the API](../../api/users.md#create-a-runner-linked-to-a-user) to create a runner. NOTE: The runner authentication token displays in the UI for only a short period of time during registration. @@ -457,7 +457,7 @@ To create a project runner: - For the `executor`, enter the type of [executor](https://docs.gitlab.com/runner/executors/). The executor is the environment where the runner executes the job. -You can also [use the API](../../api/users.md#create-a-runner) to create a runner. +You can also [use the API](../../api/users.md#create-a-runner-linked-to-a-user) to create a runner. NOTE: The runner authentication token displays in the UI for only a short period of time during registration. diff --git a/doc/ci/services/index.md b/doc/ci/services/index.md index 03acb6e3bf3..d782f233780 100644 --- a/doc/ci/services/index.md +++ b/doc/ci/services/index.md @@ -27,7 +27,7 @@ than to install `mysql`, for example, every time the project is built. You're not limited to only database services. You can add as many services you need to `.gitlab-ci.yml` or manually modify the [`config.toml`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). -Any image found at [Docker Hub](https://hub.docker.com/) or your private Container Registry can be +Any image found at [Docker Hub](https://hub.docker.com/) or your private container registry can be used as a service. Services inherit the same DNS servers, search domains, and additional hosts as diff --git a/doc/ci/testing/code_quality.md b/doc/ci/testing/code_quality.md index b191ce606e9..94968d5dfe2 100644 --- a/doc/ci/testing/code_quality.md +++ b/doc/ci/testing/code_quality.md @@ -352,7 +352,7 @@ the nested method of container execution. The following variables can address all of the required image pulls: - `CODE_QUALITY_IMAGE`: A fully prefixed image name that can be located anywhere - accessible from your job environment. GitLab Container Registry can be used here + accessible from your job environment. GitLab container registry can be used here to host your own copy. - `CODECLIMATE_PREFIX`: The domain of your intended container image registry. This is a configuration option supported by [CodeClimate CLI](https://github.com/codeclimate/codeclimate/pull/948). diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index bc2443fc7b0..93b08cb2968 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -110,7 +110,7 @@ as it can cause the pipeline to behave unexpectedly. | `CI_REGISTRY` | 8.10 | 0.5 | Address of the [container registry](../../user/packages/container_registry/index.md) server, formatted as `[:]`. For example: `registry.gitlab.example.com`. Only available if the container registry is enabled for the GitLab instance. | | `CI_REGISTRY_IMAGE` | 8.10 | 0.5 | Base address for the container registry to push, pull, or tag project's images, formatted as `[:]/`. For example: `registry.gitlab.example.com/my_group/my_project`. Image names must follow the [container registry naming convention](../../user/packages/container_registry/index.md#naming-convention-for-your-container-images). Only available if the container registry is enabled for the project. | | `CI_REGISTRY_PASSWORD` | 9.0 | all | The password to push containers to the GitLab project's container registry. Only available if the container registry is enabled for the project. This password value is the same as the `CI_JOB_TOKEN` and is valid only as long as the job is running. Use the `CI_DEPLOY_PASSWORD` for long-lived access to the registry | -| `CI_REGISTRY_USER` | 9.0 | all | The username to push containers to the project's GitLab Container Registry. Only available if the container registry is enabled for the project. | +| `CI_REGISTRY_USER` | 9.0 | all | The username to push containers to the project's GitLab container registry. Only available if the container registry is enabled for the project. | | `CI_REPOSITORY_URL` | 9.0 | all | The full path to Git clone (HTTP) the repository with a [CI/CD job token](../jobs/ci_job_token.md), in the format `https://gitlab-ci-token:$CI_JOB_TOKEN@gitlab.example.com/my-group/my-project.git`. | | `CI_RUNNER_DESCRIPTION` | 8.10 | 0.5 | The description of the runner. | | `CI_RUNNER_EXECUTABLE_ARCH` | all | 10.6 | The OS/architecture of the GitLab Runner executable. Might not be the same as the environment of the executor. | diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index 7e300f507cf..048a0017a0e 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -3299,7 +3299,7 @@ The release job must have access to the [`release-cli`](https://gitlab.com/gitla which must be in the `$PATH`. If you use the [Docker executor](https://docs.gitlab.com/runner/executors/docker.html), -you can use this image from the GitLab Container Registry: `registry.gitlab.com/gitlab-org/release-cli:latest` +you can use this image from the GitLab container registry: `registry.gitlab.com/gitlab-org/release-cli:latest` If you use the [Shell executor](https://docs.gitlab.com/runner/executors/shell.html) or similar, [install `release-cli`](../../user/project/releases/release_cli.md) on the server where the runner is registered. diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 9fd94f4c75a..dd5d83212d4 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -763,7 +763,7 @@ See our [Redis guidelines](redis.md) for more information about how GitLab uses - [Source](../administration/packages/container_registry.md#enable-the-container-registry) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/registry.md) - Layer: Core Service (Processor) -- GitLab.com: [GitLab Container Registry](../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd) +- GitLab.com: [GitLab container registry](../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd) The registry is what users use to store their own Docker images. The bundled registry uses NGINX as a load balancer and GitLab as an authentication manager. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 663b64c5dfe..f55a7bf44c5 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -620,9 +620,8 @@ WARNING: [very specific cases](https://about.gitlab.com/handbook/engineering/workflow/#criteria-for-merging-during-broken-master). For other cases, follow these [handbook instructions](https://about.gitlab.com/handbook/engineering/workflow/#merging-during-broken-master). - If the latest pipeline was created before the merge request was approved, start a new pipeline to ensure that full RSpec suite has been run. You may skip this step only if the merge request does not contain any backend change. - - If the **latest [merged results pipeline](../ci/pipelines/merged_results_pipelines.md)** was **created less than 6 hours ago**, and **finished less than 2 hours ago**, you - may merge without starting a new pipeline as the merge request is close - enough to `main`. + - If the **latest [merged results pipeline](../ci/pipelines/merged_results_pipelines.md)** was **created less than 4 hours ago**, you + may merge without starting a new pipeline as the merge request is close enough to the target branch. - When you set the MR to auto-merge, you should take over subsequent revisions for anything that would be spotted after that. - For merge requests that have had [Squash and merge](../user/project/merge_requests/squash_and_merge.md) set, diff --git a/doc/development/fe_guide/registry_architecture.md b/doc/development/fe_guide/registry_architecture.md index be35a681e0c..bc46c2643de 100644 --- a/doc/development/fe_guide/registry_architecture.md +++ b/doc/development/fe_guide/registry_architecture.md @@ -13,7 +13,7 @@ already exists to unify the user and developer experiences. Existing registries: - Package registry -- Container Registry +- Container registry - Terraform Module Registry - Dependency Proxy diff --git a/doc/development/fips_compliance.md b/doc/development/fips_compliance.md index 73f2a4829cb..3563f7fc2d5 100644 --- a/doc/development/fips_compliance.md +++ b/doc/development/fips_compliance.md @@ -40,7 +40,7 @@ when FIPS mode is enabled. | Ubuntu 20.04 Libgcrypt Cryptographic Module | [3902](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3902) | EC2 instances | `gpg`, `sshd` | | Amazon Linux 2 Kernel Crypto API Cryptographic Module | [3709](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3709) | EKS nodes | Linux kernel | | Amazon Linux 2 OpenSSL Cryptographic Module | [3553](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3553) | EKS nodes | NGINX | -| RedHat Enterprise Linux 8 OpenSSL Cryptographic Module | [4271](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4271) | EKS nodes | UBI containers: Workhorse, Pages, Container Registry, Rails (Puma/Sidekiq), Security Analyzers | +| RedHat Enterprise Linux 8 OpenSSL Cryptographic Module | [4271](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4271) | EKS nodes | UBI containers: Workhorse, Pages, container registry, Rails (Puma/Sidekiq), Security Analyzers | | RedHat Enterprise Linux 8 Libgcrypt Cryptographic Module | [3784](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3784) | EKS nodes | UBI containers: GitLab Shell, `gpg` | ### Supported Operating Systems diff --git a/doc/development/geo.md b/doc/development/geo.md index 8eb5482f9b8..d8b000c090a 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -628,7 +628,7 @@ As another example, [Dependency Proxy](../administration/packages/dependency_pro If a new feature introduces a new kind of data which is not a Git repository, or a blob, or a combination of the two, then contact the Geo team to discuss how to handle it. -As an example, Container Registry data does not easily fit into the above categories. It is backed by a registry service which owns the data, and GitLab interacts with the registry service's API. So a one off approach is required for Geo support of Container Registry. Still, we are able to reuse much of the glue code of [the Geo self-service framework](geo/framework.md#repository-replicator-strategy). +As an example, container registry data does not easily fit into the above categories. It is backed by a registry service which owns the data, and GitLab interacts with the registry service's API. So a one off approach is required for Geo support of container registry. Still, we are able to reuse much of the glue code of [the Geo self-service framework](geo/framework.md#repository-replicator-strategy). ## History of communication channel diff --git a/doc/development/go_guide/go_upgrade.md b/doc/development/go_guide/go_upgrade.md index 7aecd20b869..5386221ea7e 100644 --- a/doc/development/go_guide/go_upgrade.md +++ b/doc/development/go_guide/go_upgrade.md @@ -153,7 +153,7 @@ if you need help finding the correct person or labels: | Gitaly | [Issue Tracker](https://gitlab.com/gitlab-org/gitaly/-/issues) | | GitLab CLI (`glab`). | [Issue Tracker](https://gitlab.com/gitlab-org/cli/-/issues) | GitLab Compose Kit | [Issuer Tracker](https://gitlab.com/gitlab-org/gitlab-compose-kit/-/issues) | -| GitLab Container Registry | [Issue Tracker](https://gitlab.com/gitlab-org/container-registry) | +| GitLab container registry | [Issue Tracker](https://gitlab.com/gitlab-org/container-registry) | | GitLab Elasticsearch Indexer | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer/-/issues) | | GitLab agent server for Kubernetes (KAS) | [Issue Tracker](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/issues) | | GitLab Pages | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab-pages/-/issues) | diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index a37f682ffe2..35e423b28e9 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -33,7 +33,7 @@ are very appreciative of the work done by translators and proofreaders! - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve) - Victor Wu - [GitLab](https://gitlab.com/_victorwu_), [Crowdin](https://crowdin.com/profile/victorwu) - Xiaogang Wen - [GitLab](https://gitlab.com/xiaogang_cn), [Crowdin](https://crowdin.com/profile/xiaogang_gitlab) - - Qi Zhao - [GitLab](https://gitlab.com/zhaoqi01), [Crowdin](https://crowdin.com/profile/zhaoqi01) + - Zhiyuan Lu - [GitLab](https://gitlab.com/luzhiyuan.deer), [Crowdin](https://crowdin.com/profile/luzhiyuan.deer) - Chinese Traditional 繁體中文 - Weizhe Ding - [GitLab](https://gitlab.com/d.weizhe), [Crowdin](https://crowdin.com/profile/d.weizhe) - Yi-Jyun Pan - [GitLab](https://gitlab.com/pan93412), [Crowdin](https://crowdin.com/profile/pan93412) diff --git a/doc/development/internal_analytics/service_ping/troubleshooting.md b/doc/development/internal_analytics/service_ping/troubleshooting.md index 66ac037e87b..9fdf615863c 100644 --- a/doc/development/internal_analytics/service_ping/troubleshooting.md +++ b/doc/development/internal_analytics/service_ping/troubleshooting.md @@ -69,7 +69,7 @@ and run a local container instance: 1. In the downstream pipeline, wait for the `gitlab-docker` job to finish. 1. Open the job logs and locate the full container name including the version. It takes the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:`. 1. On your local machine, make sure you are signed in to the GitLab Docker registry. You can find the instructions for this in - [Authenticate to the GitLab Container Registry](../../../user/packages/container_registry/authenticate_with_container_registry.md). + [Authenticate to the GitLab container registry](../../../user/packages/container_registry/authenticate_with_container_registry.md). 1. Once signed in, download the new image by using `docker pull registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:` 1. For more information about working with and running Omnibus GitLab containers in Docker, refer to [GitLab Docker images](../../../install/docker.md) documentation. diff --git a/doc/development/merge_request_concepts/mergeability_framework.md b/doc/development/merge_request_concepts/mergeability_framework.md index 332a7bb4b38..a755615c885 100644 --- a/doc/development/merge_request_concepts/mergeability_framework.md +++ b/doc/development/merge_request_concepts/mergeability_framework.md @@ -23,9 +23,8 @@ The mergeability checks live under `app/services/merge_requests/mergeability/`. module MergeRequests module Mergeability class CheckCiStatusService < CheckBaseService - def self.failure_reason - :ci_must_pass - end + identifier :ci_must_pass + description 'Checks whether CI has passed' def execute # If the merge check is behind a setting, we return inactive if the setting is false diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index 4b7e4f06477..7ab1e588a40 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -78,7 +78,7 @@ subgraph " `gitlab-org/gitlab-qa-mirror` pipeline" and polls for the resulting status. We call this a _status attribution_. 1. In the [`gitlab-org/build/omnibus-gitlab-mirror` pipeline](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror): - 1. Docker image is being built and pushed to its Container Registry. + 1. Docker image is being built and pushed to its container registry. 1. Once Docker images are built and pushed jobs in `test` stage are started 1. In the `Test` stage: diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index 7b90d4ad19c..191234ebffe 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -133,7 +133,7 @@ SAML, and Kerberos. Here are some documents you might be interested in reading: - [Linux package documentation](https://docs.gitlab.com/omnibus/) - [Integration documentation](../../integration/index.md) - [GitLab Pages configuration](../../administration/pages/index.md) -- [GitLab Container Registry configuration](../../administration/packages/container_registry.md) +- [GitLab container registry configuration](../../administration/packages/container_registry.md)