From 3a4143ad22b535d83c80d043bdab3a51ede1e7c5 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 12 Sep 2023 03:09:02 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- app/models/packages/protection.rb | 9 + app/models/packages/protection/rule.rb | 21 + app/models/project.rb | 3 + db/docs/packages_protection_rules.yml | 10 + ...170000_create_packages_protection_rules.rb | 18 + db/schema_migrations/20230903170000 | 1 + db/structure.sql | 34 +- doc/api/job_artifacts.md | 14 +- doc/update/versions/gitlab_15_changes.md | 4 +- doc/update/versions/gitlab_16_changes.md | 692 ++++++++++-------- doc/user/clusters/agent/work_with_agent.md | 4 +- doc/user/permissions.md | 2 +- locale/gitlab.pot | 3 + .../packages/package_protection_rules.rb | 10 + spec/lib/gitlab/import_export/all_models.yml | 1 + spec/models/packages/protection/rule_spec.rb | 40 + spec/models/project_spec.rb | 1 + .../packages/policies/project_policy_spec.rb | 2 +- 18 files changed, 540 insertions(+), 329 deletions(-) create mode 100644 app/models/packages/protection.rb create mode 100644 app/models/packages/protection/rule.rb create mode 100644 db/docs/packages_protection_rules.yml create mode 100644 db/migrate/20230903170000_create_packages_protection_rules.rb create mode 100644 db/schema_migrations/20230903170000 create mode 100644 spec/factories/packages/package_protection_rules.rb create mode 100644 spec/models/packages/protection/rule_spec.rb diff --git a/app/models/packages/protection.rb b/app/models/packages/protection.rb new file mode 100644 index 00000000000..ebaecf89992 --- /dev/null +++ b/app/models/packages/protection.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Packages + module Protection + def self.table_name_prefix + 'packages_protection_' + end + end +end diff --git a/app/models/packages/protection/rule.rb b/app/models/packages/protection/rule.rb new file mode 100644 index 00000000000..bb65be92b90 --- /dev/null +++ b/app/models/packages/protection/rule.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Packages + module Protection + class Rule < ApplicationRecord + enum package_type: Packages::Package.package_types.slice(:npm) + + belongs_to :project, inverse_of: :package_protection_rules + + validates :package_name_pattern, presence: true, uniqueness: { scope: [:project_id, :package_type] }, + length: { maximum: 255 } + validates :package_type, presence: true + validates :push_protected_up_to_access_level, presence: true, + inclusion: { in: [ + Gitlab::Access::DEVELOPER, + Gitlab::Access::MAINTAINER, + Gitlab::Access::OWNER + ] } + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 25991108801..ad34948786b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -268,6 +268,9 @@ class Project < ApplicationRecord dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :npm_metadata_caches, class_name: 'Packages::Npm::MetadataCache' has_one :packages_cleanup_policy, class_name: 'Packages::Cleanup::Policy', inverse_of: :project + has_many :package_protection_rules, + class_name: 'Packages::Protection::Rule', + inverse_of: :project has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent diff --git a/db/docs/packages_protection_rules.yml b/db/docs/packages_protection_rules.yml new file mode 100644 index 00000000000..3007c956e26 --- /dev/null +++ b/db/docs/packages_protection_rules.yml @@ -0,0 +1,10 @@ +--- +table_name: packages_protection_rules +classes: +- Packages::Protection::Rule +feature_categories: +- package_registry +description: Represents package protection rules for package registry. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124776 +milestone: '16.4' +gitlab_schema: gitlab_main diff --git a/db/migrate/20230903170000_create_packages_protection_rules.rb b/db/migrate/20230903170000_create_packages_protection_rules.rb new file mode 100644 index 00000000000..d5e8a4af558 --- /dev/null +++ b/db/migrate/20230903170000_create_packages_protection_rules.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class CreatePackagesProtectionRules < Gitlab::Database::Migration[2.1] + enable_lock_retries! + + def change + create_table :packages_protection_rules do |t| + t.references :project, null: false, index: false, foreign_key: { on_delete: :cascade } + t.timestamps_with_timezone null: false + t.integer :push_protected_up_to_access_level, null: false + t.integer :package_type, limit: 2, null: false + t.text :package_name_pattern, limit: 255, null: false + + t.index [:project_id, :package_type, :package_name_pattern], unique: true, + name: :i_packages_unique_project_id_package_type_package_name_pattern + end + end +end diff --git a/db/schema_migrations/20230903170000 b/db/schema_migrations/20230903170000 new file mode 100644 index 00000000000..5189c60657a --- /dev/null +++ b/db/schema_migrations/20230903170000 @@ -0,0 +1 @@ +07fdd8579009aa1d68106cc9919d82cfca5702373d8b69c54ea7fa552209d540 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 9a75a1b72a0..9b696570ea3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11970,11 +11970,11 @@ CREATE TABLE application_settings ( package_registry_allow_anyone_to_pull_option boolean DEFAULT true NOT NULL, bulk_import_max_download_file_size bigint DEFAULT 5120 NOT NULL, max_import_remote_file_size bigint DEFAULT 10240 NOT NULL, - sentry_clientside_traces_sample_rate double precision DEFAULT 0.0 NOT NULL, protected_paths_for_get_request text[] DEFAULT '{}'::text[] NOT NULL, max_decompressed_archive_size integer DEFAULT 25600 NOT NULL, - ci_max_total_yaml_size_bytes integer DEFAULT 157286400 NOT NULL, + sentry_clientside_traces_sample_rate double precision DEFAULT 0.0 NOT NULL, prometheus_alert_db_indicators_settings jsonb, + ci_max_total_yaml_size_bytes integer DEFAULT 157286400 NOT NULL, decompress_archive_file_timeout integer DEFAULT 210 NOT NULL, search_rate_limit_allowlist text[] DEFAULT '{}'::text[] NOT NULL, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), @@ -20296,6 +20296,26 @@ CREATE SEQUENCE packages_packages_id_seq ALTER SEQUENCE packages_packages_id_seq OWNED BY packages_packages.id; +CREATE TABLE packages_protection_rules ( + id bigint NOT NULL, + project_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + push_protected_up_to_access_level integer NOT NULL, + package_type smallint NOT NULL, + package_name_pattern text NOT NULL, + CONSTRAINT check_d2d75d206d CHECK ((char_length(package_name_pattern) <= 255)) +); + +CREATE SEQUENCE packages_protection_rules_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE packages_protection_rules_id_seq OWNED BY packages_protection_rules.id; + CREATE TABLE packages_pypi_metadata ( package_id bigint NOT NULL, required_python text DEFAULT ''::text, @@ -26200,6 +26220,8 @@ ALTER TABLE ONLY packages_package_files ALTER COLUMN id SET DEFAULT nextval('pac ALTER TABLE ONLY packages_packages ALTER COLUMN id SET DEFAULT nextval('packages_packages_id_seq'::regclass); +ALTER TABLE ONLY packages_protection_rules ALTER COLUMN id SET DEFAULT nextval('packages_protection_rules_id_seq'::regclass); + ALTER TABLE ONLY packages_rpm_repository_files ALTER COLUMN id SET DEFAULT nextval('packages_rpm_repository_files_id_seq'::regclass); ALTER TABLE ONLY packages_tags ALTER COLUMN id SET DEFAULT nextval('packages_tags_id_seq'::regclass); @@ -28548,6 +28570,9 @@ ALTER TABLE ONLY packages_package_files ALTER TABLE ONLY packages_packages ADD CONSTRAINT packages_packages_pkey PRIMARY KEY (id); +ALTER TABLE ONLY packages_protection_rules + ADD CONSTRAINT packages_protection_rules_pkey PRIMARY KEY (id); + ALTER TABLE ONLY packages_pypi_metadata ADD CONSTRAINT packages_pypi_metadata_pkey PRIMARY KEY (package_id); @@ -30514,6 +30539,8 @@ CREATE INDEX i_dast_profiles_tags_on_scanner_profiles_id ON dast_profiles_tags U CREATE INDEX i_dast_scanner_profiles_tags_on_scanner_profiles_id ON dast_scanner_profiles_tags USING btree (dast_scanner_profile_id); +CREATE UNIQUE INDEX i_packages_unique_project_id_package_type_package_name_pattern ON packages_protection_rules USING btree (project_id, package_type, package_name_pattern); + CREATE INDEX i_pkgs_deb_file_meta_on_updated_at_package_file_id_when_unknown ON packages_debian_file_metadata USING btree (updated_at, package_file_id) WHERE (file_type = 1); CREATE UNIQUE INDEX i_pm_licenses_on_spdx_identifier ON pm_licenses USING btree (spdx_identifier); @@ -38875,6 +38902,9 @@ ALTER TABLE ONLY clusters_integration_prometheus ALTER TABLE ONLY vulnerability_occurrence_identifiers ADD CONSTRAINT fk_rails_e4ef6d027c FOREIGN KEY (occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE; +ALTER TABLE ONLY packages_protection_rules + ADD CONSTRAINT fk_rails_e52adb5267 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY vulnerability_flags ADD CONSTRAINT fk_rails_e59393b48b FOREIGN KEY (vulnerability_occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE; diff --git a/doc/api/job_artifacts.md b/doc/api/job_artifacts.md index 47e838543b2..a65de457581 100644 --- a/doc/api/job_artifacts.md +++ b/doc/api/job_artifacts.md @@ -12,6 +12,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w Get the job's artifacts zipped archive of a project. +If you use cURL to download artifacts from GitLab.com, use the `--location` parameter +as the request might redirect through a CND. + ```plaintext GET /projects/:id/jobs/:job_id/artifacts ``` @@ -25,7 +28,7 @@ GET /projects/:id/jobs/:job_id/artifacts Example request using the `PRIVATE-TOKEN` header: ```shell -curl --location --output artifacts.zip --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/jobs/42/artifacts" +curl --location --output artifacts.zip --location --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/jobs/42/artifacts" ``` To use this in a [`script` definition](../ci/yaml/index.md#script) inside @@ -69,6 +72,9 @@ the given reference name and job, provided the job finished successfully. This is the same as [getting the job's artifacts](#get-job-artifacts), but by defining the job's name instead of its ID. +If you use cURL to download artifacts from GitLab.com, use the `--location` parameter +as the request might redirect through a CND. + NOTE: If a pipeline is [parent of other child pipelines](../ci/pipelines/downstream_pipelines.md#parent-child-pipelines), artifacts are searched in hierarchical order from parent to child. For example, if both parent and @@ -132,6 +138,9 @@ Download a single artifact file from a job with a specified ID from inside the job's artifacts zipped archive. The file is extracted from the archive and streamed to the client. +If you use cURL to download artifacts from GitLab.com, use the `--location` parameter +as the request might redirect through a CND. + ```plaintext GET /projects/:id/jobs/:job_id/artifacts/*artifact_path ``` @@ -172,6 +181,9 @@ Artifacts for [parent and child pipelines](../ci/pipelines/downstream_pipelines. are searched in hierarchical order from parent to child. For example, if both parent and child pipelines have a job with the same name, the artifact from the parent pipeline is returned. +If you use cURL to download artifacts from GitLab.com, use the `--location` parameter +as the request might redirect through a CND. + ```plaintext GET /projects/:id/jobs/artifacts/:ref_name/raw/*artifact_path?job=name ``` diff --git a/doc/update/versions/gitlab_15_changes.md b/doc/update/versions/gitlab_15_changes.md index 1fcb3007424..a2e7b204a82 100644 --- a/doc/update/versions/gitlab_15_changes.md +++ b/doc/update/versions/gitlab_15_changes.md @@ -64,7 +64,7 @@ see [Packaged PostgreSQL deployed in an HA/Geo Cluster](https://docs.gitlab.com/ - To resolve this issue, upgrade to 15.11 or use the workaround in the issue. - A [bug with zero-downtime reindexing](https://gitlab.com/gitlab-org/gitlab/-/issues/422938) can cause a `Couldn't load task status` error when you reindex. You might also get a `sliceId must be greater than 0 but was [-1]` error on the Elasticsearch host. As a workaround, consider [reindexing from scratch](../../integration/advanced_search/elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index) or upgrading to GitLab 16.3. - Gitaly configuration changes significantly in Omnibus GitLab 16.0. You can begin migrating to the new structure in Omnibus GitLab 15.10 while backwards compatibility is - maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](gitlab_16_changes.md#1600). + maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](gitlab_16_changes.md#gitaly-configuration-structure-change). - You might encounter the following error while upgrading to GitLab 15.10 or later: ```shell @@ -143,7 +143,7 @@ see [Packaged PostgreSQL deployed in an HA/Geo Cluster](https://docs.gitlab.com/ You can find repositories with invalid metadata records prior in GitLab 15.0 and later by searching for the log records outputted by the verifier. [Read more about repository verification, and to see an example log entry](../../administration/gitaly/praefect.md#repository-verification). - Praefect configuration changes significantly in Omnibus GitLab 16.0. You can begin migrating to the new structure in Omnibus GitLab 15.9 while backwards compatibility is - maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](gitlab_16_changes.md#1600). + maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](gitlab_16_changes.md#praefect-configuration-structure-change). ### Self-compiled installations diff --git a/doc/update/versions/gitlab_16_changes.md b/doc/update/versions/gitlab_16_changes.md index c57a0072ebd..1ce58c3f358 100644 --- a/doc/update/versions/gitlab_16_changes.md +++ b/doc/update/versions/gitlab_16_changes.md @@ -23,6 +23,12 @@ For more information about upgrading GitLab Helm Chart, see [the release notes f - If your GitLab instance upgraded first to 15.11.0, 15.11.1, or 15.11.2 the database schema is incorrect. Recommended: perform the workaround before upgrading to 16.x. See [the details and workaround](#undefined-column-error-upgrading-to-162-or-later). +- Linux package installations must change Gitaly and Praefect configuration structure before upgrading to GitLab 16. + **To avoid data loss** reconfigure Praefect first, and as part of the new configuration, disable metadata verification. + Read more: + + - [Praefect configuration structure change](#praefect-configuration-structure-change). + - [Gitaly configuration structure change](#gitaly-configuration-structure-change). ## 16.4.0 @@ -30,12 +36,6 @@ For more information about upgrading GitLab Helm Chart, see [the release notes f If you upgrade to 16.4 from a version lower than 16.3, you must execute `ANALYZE packages_packages;` in the database before you use it. -- A new method of configuring paths for the GitLab secret and custom hooks is preferred in GitLab 16.4 and later: - 1. Update your configuration `[gitlab] secret_file` to [configure the path](../../administration/gitaly/reference.md#gitlab) to the GitLab secret token. - 1. If you have custom hooks, update your configuration `[hooks] custom_hooks_dir` to [configure the path](../../administration/gitaly/reference.md#custom-hooks) to - server-side custom hooks. - 1. Remove the `[gitlab-shell] dir` configuration. - - You might encounter the following error while upgrading to GitLab 16.4 or later: ```plaintext @@ -74,6 +74,14 @@ For more information about upgrading GitLab Helm Chart, see [the release notes f Reduce the value length of the regex field for affected push rules records, then retry the migration. +### Self-compiled installations + +- A new method of configuring paths for the GitLab secret and custom hooks is preferred in GitLab 16.4 and later: + 1. Update your configuration `[gitlab] secret_file` to [configure the path](../../administration/gitaly/reference.md#gitlab) to the GitLab secret token. + 1. If you have custom hooks, update your configuration `[hooks] custom_hooks_dir` to [configure the path](../../administration/gitaly/reference.md#custom-hooks) to + server-side custom hooks. + 1. Remove the `[gitlab-shell] dir` configuration. + ## 16.3.0 - For Go applications, [`crypto/tls`: verifying certificate chains containing large RSA keys is slow (CVE-2023-29409)](https://github.com/golang/go/issues/61460) @@ -241,320 +249,11 @@ Specific information applies to Linux package installations: Workaround is to make use of a different key type, or upgrade the client OpenSSH to a version >= 8.7. -- The Gitaly configuration structure in the Linux package - [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 - to be consistent with the Gitaly configuration structure used in - self-compiled installations. +- [Migrate your Praefect configuration to the new structure](#praefect-configuration-structure-change) + to ensure all your `praefect['..']` settings continue to work in GitLab 16.0 and later. - As a result of this change, a single hash under `gitaly['configuration']` holds most Gitaly - configuration. Some `gitaly['..']` configuration options will continue to be used by GitLab 16.0 and later: - - - `enable` - - `dir` - - `bin_path` - - `env_directory` - - `env` - - `open_files_ulimit` - - `consul_service_name` - - `consul_service_meta` - - Migrate by moving your existing configuration under the new structure. The new structure is supported from GitLab 15.10. - - The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration: - - 1. Replace the `...` with the value from the old key. - 1. Skip any keys you haven't configured a value for previously. - 1. Remove the old keys from the configuration once migrated. - 1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added. - 1. When configuring `storage` to replace `git_data_dirs`, you must append `repositories` to the path as documented below. If you omit this step, your Git repositories are - inaccessible until the configuration is fixed. - - ```ruby - gitaly['configuration'] = { - # gitaly['socket_path'] - socket_path: ..., - # gitaly['runtime_dir'] - runtime_dir: ..., - # gitaly['listen_addr'] - listen_addr: ..., - # gitaly['prometheus_listen_addr'] - prometheus_listen_addr: ..., - # gitaly['tls_listen_addr'] - tls_listen_addr: ..., - tls: { - # gitaly['certificate_path'] - certificate_path: ..., - # gitaly['key_path'] - key_path: ..., - }, - # gitaly['graceful_restart_timeout'] - graceful_restart_timeout: ..., - logging: { - # gitaly['logging_level'] - level: ..., - # gitaly['logging_format'] - format: ..., - # gitaly['logging_sentry_dsn'] - sentry_dsn: ..., - # gitaly['logging_ruby_sentry_dsn'] - ruby_sentry_dsn: ..., - # gitaly['logging_sentry_environment'] - sentry_environment: ..., - # gitaly['log_directory'] - dir: ..., - }, - prometheus: { - # gitaly['prometheus_grpc_latency_buckets']. The old value was configured as a string - # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. - grpc_latency_buckets: ..., - }, - auth: { - # gitaly['auth_token'] - token: ..., - # gitaly['auth_transitioning'] - transitioning: ..., - }, - git: { - # gitaly['git_catfile_cache_size'] - catfile_cache_size: ..., - # gitaly['git_bin_path'] - bin_path: ..., - # gitaly['use_bundled_git'] - use_bundled_binaries: ..., - # gitaly['gpg_signing_key_path'] - signing_key: ..., - # gitaly['gitconfig']. This is still an array but the type of the elements have changed. - config: [ - { - # Previously the elements contained 'section', and 'subsection' in addition to 'key'. Now - # these all should be concatenated into just 'key', separated by dots. For example, - # {section: 'first', subsection: 'middle', key: 'last', value: 'value'}, should become - # {key: 'first.middle.last', value: 'value'}. - key: ..., - value: ..., - }, - ], - }, - # Storage could previously be configured through either gitaly['storage'] or 'git_data_dirs'. Migrate - # the relevant configuration according to the instructions below. - # For 'git_data_dirs', migrate only the 'path' to the gitaly['configuration'] and leave the rest of it untouched. - storage: [ - { - # gitaly['storage'][]['name'] - # - # git_data_dirs[]. The storage name was configured as a key in the map. - name: ..., - # gitaly['storage'][]['path'] - # - # git_data_dirs[]['path']. Use the value from git_data_dirs[]['path'] and append '/repositories' to it. - # - # For example, if the path in 'git_data_dirs' was '/var/opt/gitlab/git-data', use - # '/var/opt/gitlab/git-data/repositories'. The '/repositories' extension was automatically - # appended to the path configured in `git_data_dirs`. - path: ..., - }, - ], - hooks: { - # gitaly['custom_hooks_dir'] - custom_hooks_dir: ..., - }, - daily_maintenance: { - # gitaly['daily_maintenance_disabled'] - disabled: ..., - # gitaly['daily_maintenance_start_hour'] - start_hour: ..., - # gitaly['daily_maintenance_start_minute'] - start_minute: ..., - # gitaly['daily_maintenance_duration'] - duration: ..., - # gitaly['daily_maintenance_storages'] - storages: ..., - }, - cgroups: { - # gitaly['cgroups_mountpoint'] - mountpoint: ..., - # gitaly['cgroups_hierarchy_root'] - hierarchy_root: ..., - # gitaly['cgroups_memory_bytes'] - memory_bytes: ..., - # gitaly['cgroups_cpu_shares'] - cpu_shares: ..., - repositories: { - # gitaly['cgroups_repositories_count'] - count: ..., - # gitaly['cgroups_repositories_memory_bytes'] - memory_bytes: ..., - # gitaly['cgroups_repositories_cpu_shares'] - cpu_shares: ..., - } - }, - # gitaly['concurrency']. While the structure is the same, the string keys in the array elements - # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. - concurrency: ..., - # gitaly['rate_limiting']. While the structure is the same, the string keys in the array elements - # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. - rate_limiting: ..., - pack_objects_cache: { - # gitaly['pack_objects_cache_enabled'] - enabled: ..., - # gitaly['pack_objects_cache_dir'] - dir: ..., - # gitaly['pack_objects_cache_max_age'] - max_age: ..., - } - } - ``` - -- The Praefect configuration structure in the Linux package - [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 - to be consistent with the Praefect configuration structure used in - self-compiled installations. - - As a result of this change, a single hash under `praefect['configuration']` holds most Praefect - configuration. Some `praefect['..']` configuration options will continue to be used by GitLab 16.0 and later: - - - `enable` - - `dir` - - `log_directory` - - `env_directory` - - `env` - - `wrapper_path` - - `auto_migrate` - - `consul_service_name` - - Migrate by moving your existing configuration under the new structure. The new structure is supported from GitLab 15.9. - - The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration: - - 1. Replace the `...` with the value from the old key. - 1. Skip any keys you haven't configured a value for previously. - 1. Remove the old keys from the configuration once migrated. - 1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added. - - ```ruby - praefect['configuration'] = { - # praefect['listen_addr'] - listen_addr: ..., - # praefect['socket_path'] - socket_path: ..., - # praefect['prometheus_listen_addr'] - prometheus_listen_addr: ..., - # praefect['tls_listen_addr'] - tls_listen_addr: ..., - # praefect['separate_database_metrics'] - prometheus_exclude_database_from_default_metrics: ..., - auth: { - # praefect['auth_token'] - token: ..., - # praefect['auth_transitioning'] - transitioning: ..., - }, - logging: { - # praefect['logging_format'] - format: ..., - # praefect['logging_level'] - level: ..., - }, - failover: { - # praefect['failover_enabled'] - enabled: ..., - }, - background_verification: { - # praefect['background_verification_delete_invalid_records'] - delete_invalid_records: ..., - # praefect['background_verification_verification_interval'] - verification_interval: ..., - }, - reconciliation: { - # praefect['reconciliation_scheduling_interval'] - scheduling_interval: ..., - # praefect['reconciliation_histogram_buckets']. The old value was configured as a string - # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. - histogram_buckets: ..., - }, - tls: { - # praefect['certificate_path'] - certificate_path: ..., - # praefect['key_path'] - key_path: ..., - }, - database: { - # praefect['database_host'] - host: ..., - # praefect['database_port'] - port: ..., - # praefect['database_user'] - user: ..., - # praefect['database_password'] - password: ..., - # praefect['database_dbname'] - dbname: ..., - # praefect['database_sslmode'] - sslmode: ..., - # praefect['database_sslcert'] - sslcert: ..., - # praefect['database_sslkey'] - sslkey: ..., - # praefect['database_sslrootcert'] - sslrootcert: ..., - session_pooled: { - # praefect['database_direct_host'] - host: ..., - # praefect['database_direct_port'] - port: ..., - # praefect['database_direct_user'] - user: ..., - # praefect['database_direct_password'] - password: ..., - # praefect['database_direct_dbname'] - dbname: ..., - # praefect['database_direct_sslmode'] - sslmode: ..., - # praefect['database_direct_sslcert'] - sslcert: ..., - # praefect['database_direct_sslkey'] - sslkey: ..., - # praefect['database_direct_sslrootcert'] - sslrootcert: ..., - } - }, - sentry: { - # praefect['sentry_dsn'] - sentry_dsn: ..., - # praefect['sentry_environment'] - sentry_environment: ..., - }, - prometheus: { - # praefect['prometheus_grpc_latency_buckets']. The old value was configured as a string - # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. - grpc_latency_buckets: ..., - }, - # praefect['graceful_stop_timeout'] - graceful_stop_timeout: ..., - - # praefect['virtual_storages']. The old value was a hash map but the new value is an array. - virtual_storage: [ - { - # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]. The name was previously the key in - # the 'virtual_storages' hash. - name: ..., - # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. The old value was a hash map - # but the new value is an array. - node: [ - { - # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. Use NODE_NAME key as the - # storage. - storage: ..., - # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['address']. - address: ..., - # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['token']. - token: ..., - }, - ], - } - ] - } - ``` +- [Migrate your Gitaly configuration to the new structure](#gitaly-configuration-structure-change) + to ensure all your `gitaly['..']` settings continue to work in GitLab 16.0 and later. ### Geo installations **(PREMIUM SELF)** @@ -569,6 +268,359 @@ Specific information applies to installations using Geo: - Impacted versions: GitLab versions 15.11.x, 16.0.x, and 16.1.0 - 16.1.2. - Versions containing fix: GitLab 16.1.3 and later. +### Gitaly configuration structure change + +The Gitaly configuration structure in the Linux package +[changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 +to be consistent with the Gitaly configuration structure used in +self-compiled installations. + +As a result of this change, a single hash under `gitaly['configuration']` holds most Gitaly +configuration. Some `gitaly['..']` configuration options continue to be used by GitLab 16.0 and later: + +- `enable` +- `dir` +- `bin_path` +- `env_directory` +- `env` +- `open_files_ulimit` +- `consul_service_name` +- `consul_service_meta` + +Migrate by moving your existing configuration under the new structure. The new structure is supported from GitLab 15.10. + +**Migrate to the new structure** + +WARNING: +If you are running Gitaly cluster, [migrate Praefect to the new configuration structure **first**](#praefect-configuration-structure-change). +Once this change is tested, proceed with your Gitaly nodes. +If Gitaly is misconfigured as part of the configuration structure change, [repository verification](../../administration/gitaly/praefect.md#repository-verification) +will [delete metadata required for Gitaly cluster to work](https://gitlab.com/gitlab-org/gitaly/-/issues/5529). +To protect against configuration mistakes, temporarily disable repository verification in Praefect. + +1. If you're running Gitaly Cluster, ensure repository verification is disabled on all Praefect nodes. + Configure `verification_interval: 0`, and apply with `gitlab-ctl reconfigure`. +1. When applying the new structure to your configuration + - Replace the `...` with the value from the old key. + - When configuring `storage` to replace `git_data_dirs`, **you must append `repositories` to the path** as documented below. + If you miss this out your Git repositories are inaccessible until the configuration is fixed. + This misconfiguration can cause metadata deletion, and is the reason for disabling repository verification. + - Skip any keys you haven't configured a value for previously. + - Recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added. +1. Apply the change with `gitlab-ctl reconfigure`. +1. Test Git repository functionality in GitLab. +1. Remove the old keys from the configuration once migrated, and then re-run `gitlab-ctl reconfigure`. +1. Recommended, if you're running Gitaly Cluster. Reinstate Praefect [repository verification](../../administration/gitaly/praefect.md#repository-verification) + by removing `verification_interval: 0`. + +The new structure is documented below with the old keys described in a comment above the new keys. + +```ruby +gitaly['configuration'] = { + # gitaly['socket_path'] + socket_path: ..., + # gitaly['runtime_dir'] + runtime_dir: ..., + # gitaly['listen_addr'] + listen_addr: ..., + # gitaly['prometheus_listen_addr'] + prometheus_listen_addr: ..., + # gitaly['tls_listen_addr'] + tls_listen_addr: ..., + tls: { + # gitaly['certificate_path'] + certificate_path: ..., + # gitaly['key_path'] + key_path: ..., + }, + # gitaly['graceful_restart_timeout'] + graceful_restart_timeout: ..., + logging: { + # gitaly['logging_level'] + level: ..., + # gitaly['logging_format'] + format: ..., + # gitaly['logging_sentry_dsn'] + sentry_dsn: ..., + # gitaly['logging_ruby_sentry_dsn'] + ruby_sentry_dsn: ..., + # gitaly['logging_sentry_environment'] + sentry_environment: ..., + # gitaly['log_directory'] + dir: ..., + }, + prometheus: { + # gitaly['prometheus_grpc_latency_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + grpc_latency_buckets: ..., + }, + auth: { + # gitaly['auth_token'] + token: ..., + # gitaly['auth_transitioning'] + transitioning: ..., + }, + git: { + # gitaly['git_catfile_cache_size'] + catfile_cache_size: ..., + # gitaly['git_bin_path'] + bin_path: ..., + # gitaly['use_bundled_git'] + use_bundled_binaries: ..., + # gitaly['gpg_signing_key_path'] + signing_key: ..., + # gitaly['gitconfig']. This is still an array but the type of the elements have changed. + config: [ + { + # Previously the elements contained 'section', and 'subsection' in addition to 'key'. Now + # these all should be concatenated into just 'key', separated by dots. For example, + # {section: 'first', subsection: 'middle', key: 'last', value: 'value'}, should become + # {key: 'first.middle.last', value: 'value'}. + key: ..., + value: ..., + }, + ], + }, + # Storage could previously be configured through either gitaly['storage'] or 'git_data_dirs'. Migrate + # the relevant configuration according to the instructions below. + # For 'git_data_dirs', migrate only the 'path' to the gitaly['configuration'] and leave the rest of it untouched. + storage: [ + { + # gitaly['storage'][]['name'] + # + # git_data_dirs[]. The storage name was configured as a key in the map. + name: ..., + # gitaly['storage'][]['path'] + # + # git_data_dirs[]['path']. Use the value from git_data_dirs[]['path'] and append '/repositories' to it. + # + # For example, if the path in 'git_data_dirs' was '/var/opt/gitlab/git-data', use + # '/var/opt/gitlab/git-data/repositories'. The '/repositories' extension was automatically + # appended to the path configured in `git_data_dirs`. + path: ..., + }, + ], + hooks: { + # gitaly['custom_hooks_dir'] + custom_hooks_dir: ..., + }, + daily_maintenance: { + # gitaly['daily_maintenance_disabled'] + disabled: ..., + # gitaly['daily_maintenance_start_hour'] + start_hour: ..., + # gitaly['daily_maintenance_start_minute'] + start_minute: ..., + # gitaly['daily_maintenance_duration'] + duration: ..., + # gitaly['daily_maintenance_storages'] + storages: ..., + }, + cgroups: { + # gitaly['cgroups_mountpoint'] + mountpoint: ..., + # gitaly['cgroups_hierarchy_root'] + hierarchy_root: ..., + # gitaly['cgroups_memory_bytes'] + memory_bytes: ..., + # gitaly['cgroups_cpu_shares'] + cpu_shares: ..., + repositories: { + # gitaly['cgroups_repositories_count'] + count: ..., + # gitaly['cgroups_repositories_memory_bytes'] + memory_bytes: ..., + # gitaly['cgroups_repositories_cpu_shares'] + cpu_shares: ..., + } + }, + # gitaly['concurrency']. While the structure is the same, the string keys in the array elements + # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. + concurrency: ..., + # gitaly['rate_limiting']. While the structure is the same, the string keys in the array elements + # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. + rate_limiting: ..., + pack_objects_cache: { + # gitaly['pack_objects_cache_enabled'] + enabled: ..., + # gitaly['pack_objects_cache_dir'] + dir: ..., + # gitaly['pack_objects_cache_max_age'] + max_age: ..., + } +} +``` + +### Praefect configuration structure change + +The Praefect configuration structure in the Linux package +[changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 +to be consistent with the Praefect configuration structure used in +self-compiled installations. + +As a result of this change, a single hash under `praefect['configuration']` holds most Praefect +configuration. Some `praefect['..']` configuration options continue to be used by GitLab 16.0 and later: + +- `enable` +- `dir` +- `log_directory` +- `env_directory` +- `env` +- `wrapper_path` +- `auto_migrate` +- `consul_service_name` + +Migrate by moving your existing configuration under the new structure. The new structure is supported from GitLab 15.9. + +**Migrate to the new structure** + +WARNING: +Migrate Praefect to the new configuration structure **first**. +Once this change is tested, [proceed with your Gitaly nodes](#gitaly-configuration-structure-change). +If Gitaly is misconfigured as part of the configuration structure change, [repository verification](../../administration/gitaly/praefect.md#repository-verification) +will [delete metadata required for Gitaly cluster to work](https://gitlab.com/gitlab-org/gitaly/-/issues/5529). +To protect against configuration mistakes, temporarily disable repository verification in Praefect. + +1. When applying the new structure to your configuration: + - Replace the `...` with the value from the old key. + - Disable repository verification using `verification_interval: 0`, as shown below. + - Skip any keys you haven't configured a value for previously. + - Recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added. +1. Apply the change with `gitlab-ctl reconfigure`. +1. Test Git repository functionality in GitLab. +1. Remove the old keys from the configuration once migrated, and then re-run `gitlab-ctl reconfigure`. + +The new structure is documented below with the old keys described in a comment above the new keys. + +```ruby +praefect['configuration'] = { + # praefect['listen_addr'] + listen_addr: ..., + # praefect['socket_path'] + socket_path: ..., + # praefect['prometheus_listen_addr'] + prometheus_listen_addr: ..., + # praefect['tls_listen_addr'] + tls_listen_addr: ..., + # praefect['separate_database_metrics'] + prometheus_exclude_database_from_default_metrics: ..., + auth: { + # praefect['auth_token'] + token: ..., + # praefect['auth_transitioning'] + transitioning: ..., + }, + logging: { + # praefect['logging_format'] + format: ..., + # praefect['logging_level'] + level: ..., + }, + failover: { + # praefect['failover_enabled'] + enabled: ..., + }, + background_verification: { + # praefect['background_verification_delete_invalid_records'] + delete_invalid_records: ..., + # praefect['background_verification_verification_interval'] + # + # IMPORTANT: + # As part of reconfiguring Praefect, disable this feature. + # Read about this above. + # + verification_interval: 0, + }, + reconciliation: { + # praefect['reconciliation_scheduling_interval'] + scheduling_interval: ..., + # praefect['reconciliation_histogram_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + histogram_buckets: ..., + }, + tls: { + # praefect['certificate_path'] + certificate_path: ..., + # praefect['key_path'] + key_path: ..., + }, + database: { + # praefect['database_host'] + host: ..., + # praefect['database_port'] + port: ..., + # praefect['database_user'] + user: ..., + # praefect['database_password'] + password: ..., + # praefect['database_dbname'] + dbname: ..., + # praefect['database_sslmode'] + sslmode: ..., + # praefect['database_sslcert'] + sslcert: ..., + # praefect['database_sslkey'] + sslkey: ..., + # praefect['database_sslrootcert'] + sslrootcert: ..., + session_pooled: { + # praefect['database_direct_host'] + host: ..., + # praefect['database_direct_port'] + port: ..., + # praefect['database_direct_user'] + user: ..., + # praefect['database_direct_password'] + password: ..., + # praefect['database_direct_dbname'] + dbname: ..., + # praefect['database_direct_sslmode'] + sslmode: ..., + # praefect['database_direct_sslcert'] + sslcert: ..., + # praefect['database_direct_sslkey'] + sslkey: ..., + # praefect['database_direct_sslrootcert'] + sslrootcert: ..., + } + }, + sentry: { + # praefect['sentry_dsn'] + sentry_dsn: ..., + # praefect['sentry_environment'] + sentry_environment: ..., + }, + prometheus: { + # praefect['prometheus_grpc_latency_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + grpc_latency_buckets: ..., + }, + # praefect['graceful_stop_timeout'] + graceful_stop_timeout: ..., + # praefect['virtual_storages']. The old value was a hash map but the new value is an array. + virtual_storage: [ + { + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]. The name was previously the key in + # the 'virtual_storages' hash. + name: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. The old value was a hash map + # but the new value is an array. + node: [ + { + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. Use NODE_NAME key as the + # storage. + storage: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['address']. + address: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['token']. + token: ..., + }, + ], + } + ] +} +``` + ## Long-running user type data change GitLab 16.0 is a required stop for large GitLab instances with a lot of records in the `users` table. @@ -584,7 +636,7 @@ migration might take multiple days to complete on larger GitLab instances. Make has completed successfully before upgrading to 16.1.0 or later. GitLab 16.1 introduces the `FinalizeUserTypeMigration` migration which ensures the -16.0 `MigrateHumanUserType` background migration is completed, making the 16.0 changes synchronously +16.0 `MigrateHumanUserType` background migration is completed, executing the 16.0 change synchronously during the upgrade if it's not completed. GitLab 16.2 [implements a `NOT NULL` database constraint](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/122454) diff --git a/doc/user/clusters/agent/work_with_agent.md b/doc/user/clusters/agent/work_with_agent.md index cf388763771..70abbebaaad 100644 --- a/doc/user/clusters/agent/work_with_agent.md +++ b/doc/user/clusters/agent/work_with_agent.md @@ -4,9 +4,9 @@ group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Working with the agent for Kubernetes **(FREE ALL)** +# Managing the agent for Kubernetes instances **(FREE ALL)** -Use the following tasks when working with the agent for Kubernetes. +Use the following tasks when you work with the agent for Kubernetes. ## View your agents diff --git a/doc/user/permissions.md b/doc/user/permissions.md index d7705132d07..ac86ea36b5c 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -66,10 +66,10 @@ The following table lists project permissions available for each role: | [Analytics](analytics/index.md):
View [repository analytics](analytics/repository_analytics.md) | | ✓ | ✓ | ✓ | ✓ | | [Application security](application_security/index.md):
View licenses in [dependency list](application_security/dependency_list/index.md) | | | ✓ | ✓ | ✓ | | [Application security](application_security/index.md):
Create and run [on-demand DAST scans](application_security/dast/proxy-based.md#on-demand-scans) | | | ✓ | ✓ | ✓ | -| [Application security](application_security/index.md):
Manage [security policy](application_security/policies/index.md) | | | ✓ | ✓ | ✓ | | [Application security](application_security/index.md):
View [dependency list](application_security/dependency_list/index.md) | | | ✓ | ✓ | ✓ | | [Application security](application_security/index.md):
Create a [CVE ID Request](application_security/cve_id_request.md) | | | | ✓ | ✓ | | [Application security](application_security/index.md):
Create or assign [security policy project](application_security/policies/index.md) | | | | | ✓ | +| [Application security](application_security/index.md):
Create, edit, delete [individual security policies](application_security/policies/index.md) | | | ✓ | ✓ | ✓ | | [GitLab Agent for Kubernetes](clusters/agent/index.md):
View agents | | | ✓ | ✓ | ✓ | | [GitLab Agent for Kubernetes](clusters/agent/index.md):
Manage agents | | | | ✓ | ✓ | | [Container Registry](packages/container_registry/index.md):
Create, edit, delete [cleanup policies](packages/container_registry/delete_container_registry_images.md#use-a-cleanup-policy) | | | | ✓ | ✓ | diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 152b74c3db8..d99192e7c0d 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -28958,6 +28958,9 @@ msgstr "" msgid "MemberRoles|Select a standard role to add permissions." msgstr "" +msgid "MemberRoles|To add a new role select 'Add new role'." +msgstr "" + msgid "MemberRoles|To add a new role select a group and then 'Add new role'." msgstr "" diff --git a/spec/factories/packages/package_protection_rules.rb b/spec/factories/packages/package_protection_rules.rb new file mode 100644 index 00000000000..3038fb847e7 --- /dev/null +++ b/spec/factories/packages/package_protection_rules.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :package_protection_rule, class: 'Packages::Protection::Rule' do + project + package_name_pattern { '@my_scope/my_package' } + package_type { :npm } + push_protected_up_to_access_level { Gitlab::Access::DEVELOPER } + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index de3f28a5b90..5ca5270a851 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -738,6 +738,7 @@ project: - project_registry - packages - package_files +- package_protection_rules - rpm_repository_files - npm_metadata_caches - packages_cleanup_policy diff --git a/spec/models/packages/protection/rule_spec.rb b/spec/models/packages/protection/rule_spec.rb new file mode 100644 index 00000000000..b368687e6d8 --- /dev/null +++ b/spec/models/packages/protection/rule_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Protection::Rule, type: :model, feature_category: :package_registry do + it_behaves_like 'having unique enum values' + + describe 'relationships' do + it { is_expected.to belong_to(:project).inverse_of(:package_protection_rules) } + end + + describe 'enums' do + describe '#package_type' do + it { is_expected.to define_enum_for(:package_type).with_values(npm: Packages::Package.package_types[:npm]) } + end + end + + describe 'validations' do + subject { build(:package_protection_rule) } + + describe '#package_name_pattern' do + it { is_expected.to validate_presence_of(:package_name_pattern) } + it { is_expected.to validate_uniqueness_of(:package_name_pattern).scoped_to(:project_id, :package_type) } + it { is_expected.to validate_length_of(:package_name_pattern).is_at_most(255) } + end + + describe '#package_type' do + it { is_expected.to validate_presence_of(:package_type) } + end + + describe '#push_protected_up_to_access_level' do + it { is_expected.to validate_presence_of(:push_protected_up_to_access_level) } + + it { + is_expected.to validate_inclusion_of(:push_protected_up_to_access_level).in_array([Gitlab::Access::DEVELOPER, + Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER]) + } + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index db5b5c914c9..6193a565d76 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -152,6 +152,7 @@ RSpec.describe Project, factory_default: :keep, feature_category: :groups_and_pr it { is_expected.to have_many(:debian_distributions).class_name('Packages::Debian::ProjectDistribution').dependent(:destroy) } it { is_expected.to have_many(:npm_metadata_caches).class_name('Packages::Npm::MetadataCache') } it { is_expected.to have_one(:packages_cleanup_policy).class_name('Packages::Cleanup::Policy').inverse_of(:project) } + it { is_expected.to have_many(:package_protection_rules).class_name('Packages::Protection::Rule').inverse_of(:project) } it { is_expected.to have_many(:pipeline_artifacts).dependent(:restrict_with_error) } it { is_expected.to have_many(:terraform_states).class_name('Terraform::State').inverse_of(:project) } it { is_expected.to have_many(:timelogs) } diff --git a/spec/policies/packages/policies/project_policy_spec.rb b/spec/policies/packages/policies/project_policy_spec.rb index fde10f64be8..1f8a653f984 100644 --- a/spec/policies/packages/policies/project_policy_spec.rb +++ b/spec/policies/packages/policies/project_policy_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Packages::Policies::ProjectPolicy do +RSpec.describe Packages::Policies::ProjectPolicy, feature_category: :package_registry do include_context 'ProjectPolicy context' let(:project) { public_project }