diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 4082ce2ee42..11f66fcc87a 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -e2d9b43be9a9c5fcbc1f1ae1660520ba12e55224 +0947ab5a4574d24dd4af6d2de9d47b86835da4e7 diff --git a/app/finders/packages/nuget/package_finder.rb b/app/finders/packages/nuget/package_finder.rb index 23345f29198..064698d3c37 100644 --- a/app/finders/packages/nuget/package_finder.rb +++ b/app/finders/packages/nuget/package_finder.rb @@ -4,19 +4,43 @@ module Packages module Nuget class PackageFinder < ::Packages::GroupOrProjectPackageFinder MAX_PACKAGES_COUNT = 300 + FORCE_NORMALIZATION_CLIENT_VERSION = '>= 3' def execute + return ::Packages::Package.none unless @params[:package_name].present? + packages.limit_recent(@params[:limit] || MAX_PACKAGES_COUNT) end private def packages - result = base.nuget - .has_version - .with_name_like(@params[:package_name]) - result = result.with_case_insensitive_version(@params[:package_version]) if @params[:package_version].present? + result = find_by_name + find_by_version(result) + end + + def find_by_name + base + .nuget + .has_version + .with_case_insensitive_name(@params[:package_name]) + end + + def find_by_version(result) + return result if @params[:package_version].blank? + result + .with_nuget_version_or_normalized_version( + @params[:package_version], + with_normalized: Feature.enabled?(:nuget_normalized_version, @project_or_group) && + client_forces_normalized_version? + ) + end + + def client_forces_normalized_version? + return true if @params[:client_version].blank? + + VersionSorter.compare(FORCE_NORMALIZATION_CLIENT_VERSION, @params[:client_version]) <= 0 end end end diff --git a/app/finders/snippets_finder.rb b/app/finders/snippets_finder.rb index 9dd7e508c22..ca9468ca634 100644 --- a/app/finders/snippets_finder.rb +++ b/app/finders/snippets_finder.rb @@ -67,17 +67,16 @@ class SnippetsFinder < UnionFinder return Snippet.none if project.nil? && params[:project].present? return Snippet.none if project && !project.feature_available?(:snippets, current_user) - items = init_collection - items = by_ids(items) - items = items.with_optional_visibility(visibility_from_scope) - items = by_created_at(items) - - items.order_by(sort_param) + snippets = all_snippets + snippets = by_ids(snippets) + snippets = snippets.with_optional_visibility(visibility_from_scope) + snippets = by_created_at(snippets) + snippets.order_by(sort_param) end private - def init_collection + def all_snippets if explore? snippets_for_explore elsif only_personal? @@ -182,10 +181,10 @@ class SnippetsFinder < UnionFinder end end - def by_ids(items) - return items unless params[:ids].present? + def by_ids(snippets) + return snippets unless params[:ids].present? - items.id_in(params[:ids]) + snippets.id_in(params[:ids]) end def author diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 74072c362bd..d0ccf5c543a 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -71,7 +71,7 @@ module Ci def self.clone_accessors %i[pipeline project ref tag options name allow_failure stage stage_idx - yaml_variables when description needs_attributes + yaml_variables when environment description needs_attributes scheduling_type ci_stage partition_id].freeze end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 371b34a8749..3a5db04a687 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -964,11 +964,15 @@ module Ci Ci::Bridge.latest.where(pipeline: self_and_project_descendants) end + def jobs_in_self_and_project_descendants + Ci::Processable.latest.where(pipeline: self_and_project_descendants) + end + def environments_in_self_and_project_descendants(deployment_status: nil) # We limit to 100 unique environments for application safety. # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 expanded_environment_names = - builds_in_self_and_project_descendants.joins(:metadata) + jobs_in_self_and_project_descendants.joins(:metadata) .where.not(Ci::BuildMetadata.table_name => { expanded_environment_name: nil }) .distinct("#{Ci::BuildMetadata.quoted_table_name}.expanded_environment_name") .limit(100) diff --git a/app/models/concerns/packages/nuget/version_normalizable.rb b/app/models/concerns/packages/nuget/version_normalizable.rb new file mode 100644 index 00000000000..473e5f07811 --- /dev/null +++ b/app/models/concerns/packages/nuget/version_normalizable.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Packages + module Nuget + module VersionNormalizable + extend ActiveSupport::Concern + + LEADING_ZEROES_REGEX = /^(?!0$)0+(?=\d)/ + + included do + before_validation :set_normalized_version, on: %i[create update] + + private + + def set_normalized_version + return unless package && Feature.enabled?(:nuget_normalized_version, package.project) + + self.normalized_version = normalize + end + + def normalize + version = remove_leading_zeroes + version = remove_build_metadata(version) + version = omit_zero_in_fourth_part(version) + append_suffix(version) + end + + def remove_leading_zeroes + package_version.split('.').map { |part| part.sub(LEADING_ZEROES_REGEX, '') }.join('.') + end + + def remove_build_metadata(version) + version.split('+').first.downcase + end + + def omit_zero_in_fourth_part(version) + parts = version.split('.') + parts[3] = nil if parts.fourth == '0' && parts.third.exclude?('-') + parts.compact.join('.') + end + + def append_suffix(version) + version << '.0.0' if version.count('.') == 0 + version << '.0' if version.count('.') == 1 + version + end + end + end + end +end diff --git a/app/models/packages/nuget/metadatum.rb b/app/models/packages/nuget/metadatum.rb index fae7728cccb..e7cf4528f16 100644 --- a/app/models/packages/nuget/metadatum.rb +++ b/app/models/packages/nuget/metadatum.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Packages::Nuget::Metadatum < ApplicationRecord + include Packages::Nuget::VersionNormalizable + MAX_AUTHORS_LENGTH = 255 MAX_DESCRIPTION_LENGTH = 4000 MAX_URL_LENGTH = 255 @@ -13,9 +15,15 @@ class Packages::Nuget::Metadatum < ApplicationRecord validates :icon_url, public_url: { allow_blank: true }, length: { maximum: MAX_URL_LENGTH } validates :authors, presence: true, length: { maximum: MAX_AUTHORS_LENGTH } validates :description, presence: true, length: { maximum: MAX_DESCRIPTION_LENGTH } + validates :normalized_version, presence: true, + if: -> { Feature.enabled?(:nuget_normalized_version, package&.project) } validate :ensure_nuget_package_type + delegate :version, to: :package, prefix: true + + scope :normalized_version_in, ->(version) { where(normalized_version: version.downcase) } + private def ensure_nuget_package_type diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 7dba8b42146..849ca8976a0 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -124,6 +124,22 @@ class Packages::Package < ApplicationRecord where('LOWER(version) = ?', version.downcase) end + scope :with_case_insensitive_name, ->(name) do + where(arel_table[:name].lower.eq(name.downcase)) + end + + scope :with_nuget_version_or_normalized_version, ->(version, with_normalized: true) do + relation = with_case_insensitive_version(version) + + return relation unless with_normalized + + relation + .left_joins(:nuget_metadatum) + .or( + merge(Packages::Nuget::Metadatum.normalized_version_in(version)) + ) + end + scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } scope :with_version, ->(version) { where(version: version) } scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } @@ -368,6 +384,12 @@ class Packages::Package < ApplicationRecord name.gsub(/#{Gitlab::Regex::Packages::PYPI_NORMALIZED_NAME_REGEX_STRING}/o, '-').downcase end + def normalized_nuget_version + return unless nuget? + + nuget_metadatum&.normalized_version + end + def publish_creation_event ::Gitlab::EventStore.publish( ::Packages::PackageCreatedEvent.new(data: { diff --git a/app/services/metrics/dashboard/base_service.rb b/app/services/metrics/dashboard/base_service.rb index 06cd0c46560..1b9f79b420f 100644 --- a/app/services/metrics/dashboard/base_service.rb +++ b/app/services/metrics/dashboard/base_service.rb @@ -9,7 +9,6 @@ module Metrics STAGES = ::Gitlab::Metrics::Dashboard::Stages SEQUENCE = [ - STAGES::CommonMetricsInserter, STAGES::PanelIdsInserter, STAGES::TrackPanelType, STAGES::UrlValidator @@ -128,8 +127,6 @@ module Metrics } end - # If @sequence is [STAGES::CommonMetricsInserter, STAGES::CustomMetricsInserter], - # this function will output `CommonMetricsInserter-CustomMetricsInserter`. def sequence_string sequence.map { |stage_class| stage_class.to_s.split('::').last }.join('-') end diff --git a/app/services/metrics/dashboard/system_dashboard_service.rb b/app/services/metrics/dashboard/system_dashboard_service.rb index 2bc19fcb9e2..6086539e25f 100644 --- a/app/services/metrics/dashboard/system_dashboard_service.rb +++ b/app/services/metrics/dashboard/system_dashboard_service.rb @@ -12,9 +12,6 @@ module Metrics DASHBOARD_VERSION = 'ce9ae27d2913f637de851d61099bc4151583eae68b1386a2176339ef6e653223' SEQUENCE = [ - STAGES::CommonMetricsInserter, - STAGES::CustomMetricsInserter, - STAGES::CustomMetricsDetailsInserter, STAGES::PanelIdsInserter ].freeze diff --git a/config/feature_flags/development/nuget_normalized_version.yml b/config/feature_flags/development/nuget_normalized_version.yml new file mode 100644 index 00000000000..a99a8dbc752 --- /dev/null +++ b/config/feature_flags/development/nuget_normalized_version.yml @@ -0,0 +1,8 @@ +--- +name: nuget_normalized_version +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121260 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/420290 +milestone: '16.3' +type: development +group: group::package registry +default_enabled: false diff --git a/db/migrate/20230718124213_add_normalized_version_to_packages_nuget_metadatum.rb b/db/migrate/20230718124213_add_normalized_version_to_packages_nuget_metadatum.rb new file mode 100644 index 00000000000..ed86be99e0d --- /dev/null +++ b/db/migrate/20230718124213_add_normalized_version_to_packages_nuget_metadatum.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddNormalizedVersionToPackagesNugetMetadatum < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + with_lock_retries do + add_column :packages_nuget_metadata, :normalized_version, :text, if_not_exists: true + end + + add_text_limit :packages_nuget_metadata, :normalized_version, 255 + end + + def down + with_lock_retries do + remove_column :packages_nuget_metadata, :normalized_version, if_exists: true + end + end +end diff --git a/db/migrate/20230718160522_add_index_packages_nuget_metadatum_on_package_id_and_normalized_version.rb b/db/migrate/20230718160522_add_index_packages_nuget_metadatum_on_package_id_and_normalized_version.rb new file mode 100644 index 00000000000..cba9944b8e1 --- /dev/null +++ b/db/migrate/20230718160522_add_index_packages_nuget_metadatum_on_package_id_and_normalized_version.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddIndexPackagesNugetMetadatumOnPackageIdAndNormalizedVersion < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + INDEX_NAME = 'idx_packages_nuget_metadata_on_pkg_id_and_normalized_version' + + def up + add_concurrent_index( + :packages_nuget_metadata, + 'package_id, normalized_version', + name: INDEX_NAME + ) + end + + def down + remove_concurrent_index_by_name(:packages_nuget_metadata, INDEX_NAME) + end +end diff --git a/db/migrate/20230718160749_add_index_packages_packages_on_project_id_and_lower_name_to_packages.rb b/db/migrate/20230718160749_add_index_packages_packages_on_project_id_and_lower_name_to_packages.rb new file mode 100644 index 00000000000..1f21d9fadf3 --- /dev/null +++ b/db/migrate/20230718160749_add_index_packages_packages_on_project_id_and_lower_name_to_packages.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddIndexPackagesPackagesOnProjectIdAndLowerNameToPackages < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + INDEX_NAME = 'index_packages_packages_on_project_id_and_lower_name' + NUGET_PACKAGE_TYPE = 4 + + def up + add_concurrent_index( + :packages_packages, + 'project_id, LOWER(name)', + name: INDEX_NAME, + where: "package_type = #{NUGET_PACKAGE_TYPE}" + ) + end + + def down + remove_concurrent_index_by_name(:packages_packages, INDEX_NAME) + end +end diff --git a/db/schema_migrations/20230718124213 b/db/schema_migrations/20230718124213 new file mode 100644 index 00000000000..e58e81caa08 --- /dev/null +++ b/db/schema_migrations/20230718124213 @@ -0,0 +1 @@ +39f3109f0568f565ca001c52a80c097493a9deaada5409ae0e9bb0e996ef4fb1 \ No newline at end of file diff --git a/db/schema_migrations/20230718160522 b/db/schema_migrations/20230718160522 new file mode 100644 index 00000000000..172123b9764 --- /dev/null +++ b/db/schema_migrations/20230718160522 @@ -0,0 +1 @@ +dfef3fe7df55dccdb97ce1ac5b0f49f8ab2fb7560e1cb6948d170103e87ba3ea \ No newline at end of file diff --git a/db/schema_migrations/20230718160749 b/db/schema_migrations/20230718160749 new file mode 100644 index 00000000000..eb171286502 --- /dev/null +++ b/db/schema_migrations/20230718160749 @@ -0,0 +1 @@ +2cde85daf368c2a7f0a19f9a66710149fcd05da09ca555ddbfadb43849011977 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 8c0a03f0d66..3d58a54933b 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -19988,6 +19988,8 @@ CREATE TABLE packages_nuget_metadata ( icon_url text, authors text, description text, + normalized_version text, + CONSTRAINT check_9973c0cc33 CHECK ((char_length(normalized_version) <= 255)), CONSTRAINT check_d39a5fe9ee CHECK ((char_length(description) <= 4000)), CONSTRAINT check_e2fc129ebd CHECK ((char_length(authors) <= 255)), CONSTRAINT packages_nuget_metadata_icon_url_constraint CHECK ((char_length(icon_url) <= 255)), @@ -30284,6 +30286,8 @@ CREATE INDEX idx_packages_debian_group_component_files_on_architecture_id ON pac CREATE INDEX idx_packages_debian_project_component_files_on_architecture_id ON packages_debian_project_component_files USING btree (architecture_id); +CREATE INDEX idx_packages_nuget_metadata_on_pkg_id_and_normalized_version ON packages_nuget_metadata USING btree (package_id, normalized_version); + CREATE INDEX idx_packages_on_project_id_name_id_version_when_installable_npm ON packages_packages USING btree (project_id, name, id, version) WHERE ((package_type = 2) AND (status = ANY (ARRAY[0, 1]))); CREATE UNIQUE INDEX idx_packages_on_project_id_name_version_unique_when_generic ON packages_packages USING btree (project_id, name, version) WHERE ((package_type = 7) AND (status <> 4)); @@ -32614,6 +32618,8 @@ CREATE INDEX index_packages_packages_on_name_trigram ON packages_packages USING CREATE INDEX index_packages_packages_on_project_id_and_created_at ON packages_packages USING btree (project_id, created_at); +CREATE INDEX index_packages_packages_on_project_id_and_lower_name ON packages_packages USING btree (project_id, lower((name)::text)) WHERE (package_type = 4); + CREATE INDEX index_packages_packages_on_project_id_and_lower_version ON packages_packages USING btree (project_id, lower((version)::text)) WHERE (package_type = 4); CREATE INDEX index_packages_packages_on_project_id_and_package_type ON packages_packages USING btree (project_id, package_type); diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index c89adf9fa10..ed11307f57b 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -163,7 +163,7 @@ http://secondary.example.com/ Sync Settings: Full Database replication lag: 0 seconds Last event ID seen from primary: 12345 (about 2 minutes ago) - Last event ID processed by cursor: 12345 (about 2 minutes ago) + Last event ID processed: 12345 (about 2 minutes ago) Last status report was: 1 minute ago ``` diff --git a/doc/user/version.md b/doc/user/version.md new file mode 100644 index 00000000000..b7846988e06 --- /dev/null +++ b/doc/user/version.md @@ -0,0 +1,23 @@ +--- +stage: none +group: none +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Find the GitLab version + +Find the version of GitLab you're running. + +## For self-managed GitLab + +- On the left sidebar, at the bottom, select **Help**. + +The version is displayed at the top of the dialog. + +## For GitLab.com + +- Go to . + +The version is displayed at the top of the page. For example, +`GitLab Enterprise Edition 16.3.0-pre 1e04d6b7fa9` indicates a pre-release +version of GitLab 16.3. diff --git a/lib/api/concerns/packages/nuget/private_endpoints.rb b/lib/api/concerns/packages/nuget/private_endpoints.rb index 20c02f0a285..a166a7294f4 100644 --- a/lib/api/concerns/packages/nuget/private_endpoints.rb +++ b/lib/api/concerns/packages/nuget/private_endpoints.rb @@ -43,7 +43,8 @@ module API current_user, project_or_group, package_name: package_name, - package_version: package_version + package_version: package_version, + client_version: headers['X-Nuget-Client-Version'] ) end diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 1631f7b2a9b..bff645700f5 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -117,6 +117,11 @@ module API def required_permission :read_package end + + def format_filename(package) + return "#{params[:package_filename]}.#{params[:format]}" if Feature.disabled?(:nuget_normalized_version, project_or_group) || package.version == params[:package_version] + return "#{params[:package_filename].sub(params[:package_version], package.version)}.#{params[:format]}" if package.normalized_nuget_version == params[:package_version] + end end params do @@ -175,8 +180,9 @@ module API requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'mynugetpkg.1.3.0.17.nupkg' } end get '*package_version/*package_filename', format: [:nupkg, :snupkg], urgency: :low do - filename = "#{params[:package_filename]}.#{params[:format]}" - package_file = ::Packages::PackageFileFinder.new(find_package(params[:package_name], params[:package_version]), filename, with_file_name_like: true) + package = find_package(params[:package_name], params[:package_version]) + filename = format_filename(package) + package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: true) .execute not_found!('Package') unless package_file diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 77872e7d13c..c603e2d100d 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -44,6 +44,7 @@ module API authenticate! filter_params = declared_params(include_missing: false).merge(author: current_user) + present paginate(SnippetsFinder.new(current_user, filter_params).execute), with: Entities::Snippet, current_user: current_user end @@ -66,6 +67,7 @@ module API authenticate! filter_params = declared_params(include_missing: false).merge(only_personal: true) + present paginate(SnippetsFinder.new(nil, filter_params).execute), with: Entities::PersonalSnippet, current_user: current_user end diff --git a/lib/gitlab/metrics/dashboard/importer.rb b/lib/gitlab/metrics/dashboard/importer.rb deleted file mode 100644 index ca835650648..00000000000 --- a/lib/gitlab/metrics/dashboard/importer.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - class Importer - def initialize(dashboard_path, project) - @dashboard_path = dashboard_path.to_s - @project = project - end - - def execute - return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path) - - Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute - rescue Gitlab::Config::Loader::FormatError - false - end - - def execute! - Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path) - - Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute! - end - - private - - attr_accessor :dashboard_path, :project - - def dashboard_hash - @dashboard_hash ||= begin - raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path) - return unless raw_dashboard.present? - - ::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw! - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb deleted file mode 100644 index 531e4079632..00000000000 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Importers - class PrometheusMetrics - ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze - - # Takes a JSON schema validated dashboard hash and - # imports metrics to database - def initialize(dashboard_hash, project:, dashboard_path:) - @dashboard_hash = dashboard_hash - @project = project - @dashboard_path = dashboard_path - @affected_environment_ids = [] - end - - def execute - import - rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError - false - end - - def execute! - import - end - - private - - attr_reader :dashboard_hash, :project, :dashboard_path - - def import - delete_stale_metrics - create_or_update_metrics - end - - # rubocop: disable CodeReuse/ActiveRecord - def create_or_update_metrics - # TODO: use upsert and worker for callbacks? - - affected_metric_ids = [] - prometheus_metrics_attributes.each do |attributes| - prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project)) - prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) - - affected_metric_ids << prometheus_metric.id - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def delete_stale_metrics - identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } - - stale_metrics = PrometheusMetric.for_project(project) - .for_dashboard_path(dashboard_path) - .for_group(Enums::PrometheusMetric.groups[:custom]) - .not_identifier(identifiers_from_yml) - - return unless stale_metrics.exists? - - stale_metrics.each_batch { |batch| batch.delete_all } - end - - def prometheus_metrics_attributes - @prometheus_metrics_attributes ||= Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( - dashboard_hash, - project: project, - dashboard_path: dashboard_path - ).execute - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb deleted file mode 100644 index 62479ed6de4..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CommonMetricsInserter < BaseStage - # For each metric in the dashboard config, attempts to - # find a corresponding database record. If found, - # includes the record's id in the dashboard config. - def transform! - common_metrics = ::PrometheusMetricsFinder.new(common: true).execute - - for_metrics do |metric| - metric_record = common_metrics.find { |m| m.identifier == metric[:id] } - metric[:metric_id] = metric_record.id if metric_record - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb deleted file mode 100644 index 06cfa5cc58e..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CustomMetricsDetailsInserter < BaseStage - def transform! - dashboard[:panel_groups].each do |panel_group| - next unless panel_group - - has_custom_metrics = custom_group_titles.include?(panel_group[:group]) - panel_group[:has_custom_metrics] = has_custom_metrics - - panel_group[:panels].each do |panel| - next unless panel - - panel[:metrics].each do |metric| - next unless metric - - metric[:edit_path] = has_custom_metrics ? edit_path(metric) : nil - end - end - end - end - - private - - def custom_group_titles - @custom_group_titles ||= Enums::PrometheusMetric.custom_group_details.values.map { |group_details| group_details[:group_title] } - end - - def edit_path(metric) - Gitlab::Routing.url_helpers.edit_project_prometheus_metric_path(project, metric[:metric_id]) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb deleted file mode 100644 index 3b49eb1c837..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb +++ /dev/null @@ -1,109 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CustomMetricsInserter < BaseStage - # Inserts project-specific metrics into the dashboard - # config. If there are no project-specific metrics, - # this will have no effect. - def transform! - custom_metrics = PrometheusMetricsFinder.new(project: project, ordered: true).execute - custom_metrics = Gitlab::Utils.stable_sort_by(custom_metrics) { |metric| -metric.priority } - - custom_metrics.each do |project_metric| - group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) - panel = find_or_create_panel(group[:panels], project_metric) - find_or_create_metric(panel[:metrics], project_metric) - end - end - - private - - # Looks for a panel_group corresponding to the - # provided metric object. If unavailable, inserts one. - # @param panel_groups [Array] - # @param metric [PrometheusMetric] - def find_or_create_panel_group(panel_groups, metric) - panel_group = find_panel_group(panel_groups, metric) - return panel_group if panel_group - - panel_group = new_panel_group(metric) - panel_groups << panel_group - - panel_group - end - - # Looks for a panel corresponding to the provided - # metric object. If unavailable, inserts one. - # @param panels [Array] - # @param metric [PrometheusMetric] - def find_or_create_panel(panels, metric) - panel = find_panel(panels, metric) - return panel if panel - - panel = new_panel(metric) - panels << panel - - panel - end - - # Looks for a metric corresponding to the provided - # metric object. If unavailable, inserts one. - # @param metrics [Array] - # @param metric [PrometheusMetric] - def find_or_create_metric(metrics, metric) - target_metric = find_metric(metrics, metric) - return target_metric if target_metric - - target_metric = new_metric(metric) - metrics << target_metric - - target_metric - end - - def find_panel_group(panel_groups, metric) - return unless panel_groups - - panel_groups.find { |group| group[:group] == metric.group_title } - end - - def find_panel(panels, metric) - return unless panels - - panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label] - panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers } - end - - def find_metric(metrics, metric) - return unless metrics - return unless metric.identifier - - metrics.find { |m| m[:id] == metric.identifier } - end - - def new_panel_group(metric) - { - group: metric.group_title, - panels: [] - } - end - - def new_panel(metric) - { - type: DEFAULT_PANEL_TYPE, - title: metric.title, - y_label: metric.y_label, - metrics: [] - } - end - - def new_metric(metric) - metric.to_metric_hash - end - end - end - end - end -end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 3e99ae3fdcc..590f05a0ace 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -20726,7 +20726,7 @@ msgstr "" msgid "Geo|Last event ID from primary" msgstr "" -msgid "Geo|Last event ID processed by cursor" +msgid "Geo|Last event ID processed" msgstr "" msgid "Geo|Last repository check run" diff --git a/package.json b/package.json index 9c80b30bb3d..a768f677799 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "@gitlab/svgs": "3.59.0", "@gitlab/ui": "64.20.1", "@gitlab/visual-review-tools": "1.7.3", - "@gitlab/web-ide": "0.0.1-dev-20230802205337", + "@gitlab/web-ide": "0.0.1-dev-20230807045127", "@mattiasbuelens/web-streams-adapter": "^0.1.0", "@popperjs/core": "^2.11.2", "@rails/actioncable": "7.0.6", diff --git a/spec/finders/packages/nuget/package_finder_spec.rb b/spec/finders/packages/nuget/package_finder_spec.rb index 6a6eebca778..792e543e424 100644 --- a/spec/finders/packages/nuget/package_finder_spec.rb +++ b/spec/finders/packages/nuget/package_finder_spec.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true require 'spec_helper' -RSpec.describe Packages::Nuget::PackageFinder do +RSpec.describe Packages::Nuget::PackageFinder, feature_category: :package_registry do let_it_be(:user) { create(:user) } let_it_be(:group) { create(:group) } let_it_be(:subgroup) { create(:group, parent: group) } let_it_be(:project) { create(:project, namespace: subgroup) } let_it_be_with_refind(:package1) { create(:nuget_package, project: project) } - let_it_be(:package2) { create(:nuget_package, name: package1.name, version: '2.0.0-ABC', project: project) } + let_it_be(:package2) { create(:nuget_package, :with_metadatum, name: package1.name, version: '2.0.0+ABC', project: project) } let_it_be(:package3) { create(:nuget_package, name: 'Another.Dummy.Package', project: project) } let_it_be(:other_package_1) { create(:nuget_package, name: package1.name, version: package1.version) } let_it_be(:other_package_2) { create(:nuget_package, name: package1.name, version: package2.version) } @@ -15,9 +15,18 @@ RSpec.describe Packages::Nuget::PackageFinder do let(:package_name) { package1.name } let(:package_version) { nil } let(:limit) { 50 } + let(:client_version) { nil } describe '#execute!' do - subject { described_class.new(user, target, package_name: package_name, package_version: package_version, limit: limit).execute } + subject { described_class.new(user, target, package_name: package_name, package_version: package_version, limit: limit, client_version: client_version).execute } + + shared_examples 'calling with_nuget_version_or_normalized_version scope' do |with_normalized:| + it 'calls with_nuget_version_or_normalized_version scope with the correct arguments' do + expect(::Packages::Package).to receive(:with_nuget_version_or_normalized_version).with(package_version, with_normalized: with_normalized).and_call_original + + subject + end + end shared_examples 'handling all the conditions' do it { is_expected.to match_array([package1, package2]) } @@ -43,13 +52,13 @@ RSpec.describe Packages::Nuget::PackageFinder do end context 'with valid version' do - let(:package_version) { '2.0.0-ABC' } + let(:package_version) { '2.0.0+ABC' } it { is_expected.to match_array([package2]) } end context 'with varying case version' do - let(:package_version) { '2.0.0-abC' } + let(:package_version) { '2.0.0+abC' } it { is_expected.to match_array([package2]) } end @@ -60,6 +69,16 @@ RSpec.describe Packages::Nuget::PackageFinder do it { is_expected.to be_empty } end + context 'with normalized version' do + let(:package_version) { '2.0.0' } + + before do + package2.nuget_metadatum.update_column(:normalized_version, package_version) + end + + it { is_expected.to match_array([package2]) } + end + context 'with limit hit' do let_it_be(:package4) { create(:nuget_package, name: package1.name, project: project) } let_it_be(:package5) { create(:nuget_package, name: package1.name, project: project) } @@ -76,22 +95,34 @@ RSpec.describe Packages::Nuget::PackageFinder do it { is_expected.to match_array([package1, package2]) } end - context 'with prefix wildcard' do - let(:package_name) { "%#{package1.name[3..]}" } + context 'with client version less than 3' do + let(:package_version) { '2.0.0+abc' } + let(:client_version) { '2.8.6' } - it { is_expected.to match_array([package1, package2]) } + it_behaves_like 'calling with_nuget_version_or_normalized_version scope', with_normalized: false end - context 'with suffix wildcard' do - let(:package_name) { "#{package1.name[0..-3]}%" } + context 'with client version greater than or equal to 3' do + let(:package_version) { '2.0.0+abc' } + let(:client_version) { '3.5' } - it { is_expected.to match_array([package1, package2]) } + it_behaves_like 'calling with_nuget_version_or_normalized_version scope', with_normalized: true end - context 'with surrounding wildcards' do - let(:package_name) { "%#{package1.name[3..-3]}%" } + context 'with no client version' do + let(:package_version) { '2.0.0+abc' } - it { is_expected.to match_array([package1, package2]) } + it_behaves_like 'calling with_nuget_version_or_normalized_version scope', with_normalized: true + end + + context 'when nuget_normalized_version feature flag is disabled' do + let(:package_version) { '2.0.0+abc' } + + before do + stub_feature_flags(nuget_normalized_version: false) + end + + it_behaves_like 'calling with_nuget_version_or_normalized_version scope', with_normalized: false end end @@ -130,5 +161,12 @@ RSpec.describe Packages::Nuget::PackageFinder do it { is_expected.to be_empty } end + + context 'when package name is blank' do + let(:target) { project } + let(:package_name) { nil } + + it { is_expected.to be_empty } + end end end diff --git a/spec/finders/snippets_finder_spec.rb b/spec/finders/snippets_finder_spec.rb index 9f4b7612be5..e81475a5c0b 100644 --- a/spec/finders/snippets_finder_spec.rb +++ b/spec/finders/snippets_finder_spec.rb @@ -106,7 +106,7 @@ RSpec.describe SnippetsFinder do expect(snippets).to contain_exactly(public_personal_snippet) end - it 'returns all snippets for an admin in admin mode', :enable_admin_mode do + it 'returns all personal snippets for an admin in admin mode', :enable_admin_mode do snippets = described_class.new(admin, author: user).execute expect(snippets).to contain_exactly(private_personal_snippet, internal_personal_snippet, public_personal_snippet) diff --git a/spec/lib/gitlab/metrics/dashboard/importer_spec.rb b/spec/lib/gitlab/metrics/dashboard/importer_spec.rb deleted file mode 100644 index 8b705395a2c..00000000000 --- a/spec/lib/gitlab/metrics/dashboard/importer_spec.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Metrics::Dashboard::Importer do - include MetricsDashboardHelpers - - let_it_be(:dashboard_path) { '.gitlab/dashboards/sample_dashboard.yml' } - let_it_be(:project) { create(:project) } - - before do - allow(subject).to receive(:dashboard_hash).and_return(dashboard_hash) - end - - subject { described_class.new(dashboard_path, project) } - - describe '.execute' do - context 'valid dashboard hash' do - let(:dashboard_hash) { load_sample_dashboard } - - it 'imports metrics to database' do - expect { subject.execute } - .to change { PrometheusMetric.count }.from(0).to(3) - end - end - - context 'invalid dashboard hash' do - let(:dashboard_hash) { {} } - - it 'returns false' do - expect(subject.execute).to be(false) - end - end - end - - describe '.execute!' do - context 'valid dashboard hash' do - let(:dashboard_hash) { load_sample_dashboard } - - it 'imports metrics to database' do - expect { subject.execute } - .to change { PrometheusMetric.count }.from(0).to(3) - end - end - - context 'invalid dashboard hash' do - let(:dashboard_hash) { {} } - - it 'raises error' do - expect { subject.execute! }.to raise_error(Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError, - 'root is missing required keys: dashboard, panel_groups') - end - end - end -end diff --git a/spec/lib/gitlab/metrics/dashboard/importers/prometheus_metrics_spec.rb b/spec/lib/gitlab/metrics/dashboard/importers/prometheus_metrics_spec.rb deleted file mode 100644 index bc6cd383758..00000000000 --- a/spec/lib/gitlab/metrics/dashboard/importers/prometheus_metrics_spec.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Metrics::Dashboard::Importers::PrometheusMetrics do - include MetricsDashboardHelpers - - describe '#execute' do - let(:project) { create(:project) } - let(:dashboard_path) { 'path/to/dashboard.yml' } - let(:prometheus_adapter) { double('adapter', clear_prometheus_reactive_cache!: nil) } - - subject { described_class.new(dashboard_hash, project: project, dashboard_path: dashboard_path) } - - context 'valid dashboard' do - let(:dashboard_hash) { load_sample_dashboard } - - context 'with all new metrics' do - it 'creates PrometheusMetrics' do - expect { subject.execute }.to change { PrometheusMetric.count }.by(3) - end - end - - context 'with existing metrics' do - let(:existing_metric_attributes) do - { - project: project, - identifier: 'metric_b', - title: 'overwrite', - y_label: 'overwrite', - query: 'overwrite', - unit: 'overwrite', - legend: 'overwrite', - dashboard_path: dashboard_path - } - end - - let!(:existing_metric) do - create(:prometheus_metric, existing_metric_attributes) - end - - it 'updates existing PrometheusMetrics' do - subject.execute - - expect(existing_metric.reload.attributes.with_indifferent_access).to include({ - title: 'Super Chart B', - y_label: 'y_label', - query: 'query', - unit: 'unit', - legend: 'Legend Label' - }) - end - - it 'creates new PrometheusMetrics' do - expect { subject.execute }.to change { PrometheusMetric.count }.by(2) - end - - context 'with stale metrics' do - let!(:stale_metric) do - create(:prometheus_metric, - project: project, - identifier: 'stale_metric', - dashboard_path: dashboard_path, - group: 3 - ) - end - - it 'updates existing PrometheusMetrics' do - subject.execute - - expect(existing_metric.reload.attributes.with_indifferent_access).to include({ - title: 'Super Chart B', - y_label: 'y_label', - query: 'query', - unit: 'unit', - legend: 'Legend Label' - }) - end - - it 'deletes stale metrics' do - subject.execute - - expect { stale_metric.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - end - end - - context 'invalid dashboard' do - let(:dashboard_hash) { {} } - - it 'returns false' do - expect(subject.execute).to eq(false) - end - end - end -end diff --git a/spec/lib/gitlab/metrics/dashboard/processor_spec.rb b/spec/lib/gitlab/metrics/dashboard/processor_spec.rb index 9bf4a7f761a..ae884c58f86 100644 --- a/spec/lib/gitlab/metrics/dashboard/processor_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/processor_spec.rb @@ -12,9 +12,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do describe 'process' do let(:sequence) do [ - Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, - Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter, - Gitlab::Metrics::Dashboard::Stages::CustomMetricsDetailsInserter, Gitlab::Metrics::Dashboard::Stages::PanelIdsInserter, Gitlab::Metrics::Dashboard::Stages::UrlValidator ] @@ -29,10 +26,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do end end - it 'includes boolean to indicate if panel group has custom metrics' do - expect(dashboard[:panel_groups]).to all(include( { has_custom_metrics: boolean } )) - end - context 'when the dashboard is not present' do let(:dashboard_yml) { nil } @@ -41,81 +34,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do end end - context 'when dashboard config corresponds to common metrics' do - let!(:common_metric) { create(:prometheus_metric, :common, identifier: 'metric_a1') } - - it 'inserts metric ids into the config' do - target_metric = all_metrics.find { |metric| metric[:id] == 'metric_a1' } - - expect(target_metric).to include(:metric_id) - expect(target_metric[:metric_id]).to eq(common_metric.id) - end - end - - context 'when the project has associated metrics' do - let!(:project_response_metric) { create(:prometheus_metric, project: project, group: :response) } - let!(:project_system_metric) { create(:prometheus_metric, project: project, group: :system) } - let!(:project_business_metric) { create(:prometheus_metric, project: project, group: :business) } - - it 'includes project-specific metrics' do - expect(all_metrics).to include get_metric_details(project_system_metric) - expect(all_metrics).to include get_metric_details(project_response_metric) - expect(all_metrics).to include get_metric_details(project_business_metric) - end - - it 'display groups and panels in the order they are defined' do - expected_metrics_order = [ - 'metric_b', - 'metric_a2', - 'metric_a1', - project_business_metric.id, - project_response_metric.id, - project_system_metric.id - ] - actual_metrics_order = all_metrics.map { |m| m[:id] || m[:metric_id] } - - expect(actual_metrics_order).to eq expected_metrics_order - end - - context 'when the project has multiple metrics in the same group' do - let!(:project_response_metric) { create(:prometheus_metric, project: project, group: :response) } - let!(:project_response_metric_2) { create(:prometheus_metric, project: project, group: :response) } - - it 'includes multiple metrics' do - expect(all_metrics).to include get_metric_details(project_response_metric) - expect(all_metrics).to include get_metric_details(project_response_metric_2) - end - end - - context 'when the dashboard should not include project metrics' do - let(:sequence) do - [ - Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter - ] - end - - let(:dashboard) { described_class.new(*process_params).process } - - it 'includes only dashboard metrics' do - metrics = all_metrics.map { |m| m[:id] } - - expect(metrics.length).to be(3) - expect(metrics).to eq %w(metric_b metric_a2 metric_a1) - end - end - end - - context 'when there are no alerts' do - let!(:persisted_metric) { create(:prometheus_metric, :common, identifier: 'metric_a1') } - - it 'does not insert an alert_path' do - target_metric = all_metrics.find { |metric| metric[:metric_id] == persisted_metric.id } - - expect(target_metric).to be_a Hash - expect(target_metric).not_to include(:alert_path) - end - end - shared_examples_for 'errors with message' do |expected_message| it 'raises a DashboardLayoutError' do error_class = Gitlab::Metrics::Dashboard::Errors::DashboardProcessingError @@ -135,12 +53,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do it_behaves_like 'errors with message', 'Each "panel_group" must define an array :panels' end - - context 'when the dashboard contains a panel which is missing metrics' do - let(:dashboard_yml) { { panel_groups: [{ panels: [{}] }] } } - - it_behaves_like 'errors with message', 'Each "panel" must define an array :metrics' - end end private diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index c4b8f4f0145..4011040a534 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -3486,99 +3486,109 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep, feature_category: describe '#environments_in_self_and_project_descendants' do subject { pipeline.environments_in_self_and_project_descendants } - context 'when pipeline is not child nor parent' do - let_it_be(:pipeline) { create(:ci_pipeline, :created) } - let_it_be(:build, refind: true) { create(:ci_build, :with_deployment, :deploy_to_production, pipeline: pipeline) } + shared_examples_for 'fetches environments in self and project descendant pipelines' do |factory_type| + context 'when pipeline is not child nor parent' do + let_it_be(:pipeline) { create(:ci_pipeline, :created) } + let_it_be(:job, refind: true) { create(factory_type, :with_deployment, :deploy_to_production, pipeline: pipeline) } - it 'returns just the pipeline environment' do - expect(subject).to contain_exactly(build.deployment.environment) + it 'returns just the pipeline environment' do + expect(subject).to contain_exactly(job.deployment.environment) + end + + context 'when deployment SHA is not matched' do + before do + job.deployment.update!(sha: 'old-sha') + end + + it 'does not return environments' do + expect(subject).to be_empty + end + end end - context 'when deployment SHA is not matched' do - before do - build.deployment.update!(sha: 'old-sha') + context 'when an associated environment does not have deployments' do + let_it_be(:pipeline) { create(:ci_pipeline, :created) } + let_it_be(:job) { create(factory_type, :stop_review_app, pipeline: pipeline) } + let_it_be(:environment) { create(:environment, project: pipeline.project) } + + before_all do + job.metadata.update!(expanded_environment_name: environment.name) end it 'does not return environments' do expect(subject).to be_empty end end - end - context 'when an associated environment does not have deployments' do - let_it_be(:pipeline) { create(:ci_pipeline, :created) } - let_it_be(:build) { create(:ci_build, :stop_review_app, pipeline: pipeline) } - let_it_be(:environment) { create(:environment, project: pipeline.project) } + context 'when pipeline is in extended family' do + let_it_be(:parent) { create(:ci_pipeline) } + let_it_be(:parent_job) { create(factory_type, :with_deployment, environment: 'staging', pipeline: parent) } - before_all do - build.metadata.update!(expanded_environment_name: environment.name) + let_it_be(:pipeline) { create(:ci_pipeline, child_of: parent) } + let_it_be(:job) { create(factory_type, :with_deployment, :deploy_to_production, pipeline: pipeline) } + + let_it_be(:child) { create(:ci_pipeline, child_of: pipeline) } + let_it_be(:child_job) { create(factory_type, :with_deployment, environment: 'canary', pipeline: child) } + + let_it_be(:grandchild) { create(:ci_pipeline, child_of: child) } + let_it_be(:grandchild_job) { create(factory_type, :with_deployment, environment: 'test', pipeline: grandchild) } + + let_it_be(:sibling) { create(:ci_pipeline, child_of: parent) } + let_it_be(:sibling_job) { create(factory_type, :with_deployment, environment: 'review', pipeline: sibling) } + + it 'returns its own environment and from all descendants' do + expected_environments = [ + job.deployment.environment, + child_job.deployment.environment, + grandchild_job.deployment.environment + ] + expect(subject).to match_array(expected_environments) + end + + it 'does not return parent environment' do + expect(subject).not_to include(parent_job.deployment.environment) + end + + it 'does not return sibling environment' do + expect(subject).not_to include(sibling_job.deployment.environment) + end end - it 'does not return environments' do - expect(subject).to be_empty + context 'when each pipeline has multiple environments' do + let_it_be(:pipeline) { create(:ci_pipeline, :created) } + let_it_be(:job1) { create(factory_type, :with_deployment, :deploy_to_production, pipeline: pipeline) } + let_it_be(:job2) { create(factory_type, :with_deployment, environment: 'staging', pipeline: pipeline) } + + let_it_be(:child) { create(:ci_pipeline, child_of: pipeline) } + let_it_be(:child_job1) { create(factory_type, :with_deployment, environment: 'canary', pipeline: child) } + let_it_be(:child_job2) { create(factory_type, :with_deployment, environment: 'test', pipeline: child) } + + it 'returns all related environments' do + expected_environments = [ + job1.deployment.environment, + job2.deployment.environment, + child_job1.deployment.environment, + child_job2.deployment.environment + ] + expect(subject).to match_array(expected_environments) + end + end + + context 'when pipeline has no environment' do + let_it_be(:pipeline) { create(:ci_pipeline, :created) } + + it 'returns empty' do + expect(subject).to be_empty + end end end - context 'when pipeline is in extended family' do - let_it_be(:parent) { create(:ci_pipeline) } - let_it_be(:parent_build) { create(:ci_build, :with_deployment, environment: 'staging', pipeline: parent) } - - let_it_be(:pipeline) { create(:ci_pipeline, child_of: parent) } - let_it_be(:build) { create(:ci_build, :with_deployment, :deploy_to_production, pipeline: pipeline) } - - let_it_be(:child) { create(:ci_pipeline, child_of: pipeline) } - let_it_be(:child_build) { create(:ci_build, :with_deployment, environment: 'canary', pipeline: child) } - - let_it_be(:grandchild) { create(:ci_pipeline, child_of: child) } - let_it_be(:grandchild_build) { create(:ci_build, :with_deployment, environment: 'test', pipeline: grandchild) } - - let_it_be(:sibling) { create(:ci_pipeline, child_of: parent) } - let_it_be(:sibling_build) { create(:ci_build, :with_deployment, environment: 'review', pipeline: sibling) } - - it 'returns its own environment and from all descendants' do - expected_environments = [ - build.deployment.environment, - child_build.deployment.environment, - grandchild_build.deployment.environment - ] - expect(subject).to match_array(expected_environments) - end - - it 'does not return parent environment' do - expect(subject).not_to include(parent_build.deployment.environment) - end - - it 'does not return sibling environment' do - expect(subject).not_to include(sibling_build.deployment.environment) - end + context 'when job is build' do + it_behaves_like 'fetches environments in self and project descendant pipelines', :ci_build end - context 'when each pipeline has multiple environments' do - let_it_be(:pipeline) { create(:ci_pipeline, :created) } - let_it_be(:build1) { create(:ci_build, :with_deployment, :deploy_to_production, pipeline: pipeline) } - let_it_be(:build2) { create(:ci_build, :with_deployment, environment: 'staging', pipeline: pipeline) } - - let_it_be(:child) { create(:ci_pipeline, child_of: pipeline) } - let_it_be(:child_build1) { create(:ci_build, :with_deployment, environment: 'canary', pipeline: child) } - let_it_be(:child_build2) { create(:ci_build, :with_deployment, environment: 'test', pipeline: child) } - - it 'returns all related environments' do - expected_environments = [ - build1.deployment.environment, - build2.deployment.environment, - child_build1.deployment.environment, - child_build2.deployment.environment - ] - expect(subject).to match_array(expected_environments) - end - end - - context 'when pipeline has no environment' do - let_it_be(:pipeline) { create(:ci_pipeline, :created) } - - it 'returns empty' do - expect(subject).to be_empty - end + context 'when job is bridge' do + it_behaves_like 'fetches environments in self and project descendant pipelines', :ci_bridge end end @@ -3963,6 +3973,53 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep, feature_category: end end + describe '#jobs_in_self_and_project_descendants' do + subject(:jobs) { pipeline.jobs_in_self_and_project_descendants } + + let(:pipeline) { create(:ci_pipeline) } + + shared_examples_for 'fetches jobs in self and project descendant pipelines' do |factory_type| + let!(:job) { create(factory_type, pipeline: pipeline) } + + context 'when pipeline is standalone' do + it 'returns the list of jobs' do + expect(jobs).to contain_exactly(job) + end + end + + context 'when pipeline is parent of another pipeline' do + let(:child_pipeline) { create(:ci_pipeline, child_of: pipeline) } + let(:child_source_bridge) { child_pipeline.source_pipeline.source_job } + let!(:child_job) { create(factory_type, pipeline: child_pipeline) } + + it 'returns the list of jobs' do + expect(jobs).to contain_exactly(job, child_job, child_source_bridge) + end + end + + context 'when pipeline is parent of another parent pipeline' do + let(:child_pipeline) { create(:ci_pipeline, child_of: pipeline) } + let(:child_source_bridge) { child_pipeline.source_pipeline.source_job } + let!(:child_job) { create(factory_type, pipeline: child_pipeline) } + let(:child_of_child_pipeline) { create(:ci_pipeline, child_of: child_pipeline) } + let(:child_of_child_source_bridge) { child_of_child_pipeline.source_pipeline.source_job } + let!(:child_of_child_job) { create(factory_type, pipeline: child_of_child_pipeline) } + + it 'returns the list of jobs' do + expect(jobs).to contain_exactly(job, child_job, child_of_child_job, child_source_bridge, child_of_child_source_bridge) + end + end + end + + context 'when job is build' do + it_behaves_like 'fetches jobs in self and project descendant pipelines', :ci_build + end + + context 'when job is bridge' do + it_behaves_like 'fetches jobs in self and project descendant pipelines', :ci_bridge + end + end + describe '#find_job_with_archive_artifacts' do let(:pipeline) { create(:ci_pipeline) } let!(:old_job) { create(:ci_build, name: 'rspec', retried: true, pipeline: pipeline) } diff --git a/spec/models/ci/processable_spec.rb b/spec/models/ci/processable_spec.rb index 0c7a13e415a..c9b2e3e6b23 100644 --- a/spec/models/ci/processable_spec.rb +++ b/spec/models/ci/processable_spec.rb @@ -31,7 +31,8 @@ RSpec.describe Ci::Processable, feature_category: :continuous_integration do let_it_be_with_refind(:processable) do create(:ci_bridge, :success, - pipeline: pipeline, downstream: downstream_project, description: 'a trigger job', stage_id: stage.id) + pipeline: pipeline, downstream: downstream_project, description: 'a trigger job', stage_id: stage.id, + environment: 'production') end let(:clone_accessors) { ::Ci::Bridge.clone_accessors } diff --git a/spec/models/packages/nuget/metadatum_spec.rb b/spec/models/packages/nuget/metadatum_spec.rb index 4b02353d6e8..e1520c0782f 100644 --- a/spec/models/packages/nuget/metadatum_spec.rb +++ b/spec/models/packages/nuget/metadatum_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Packages::Nuget::Metadatum, type: :model, feature_category: :package_registry do + it { is_expected.to be_a Packages::Nuget::VersionNormalizable } + describe 'relationships' do it { is_expected.to belong_to(:package).inverse_of(:nuget_metadatum) } end @@ -15,6 +17,18 @@ RSpec.describe Packages::Nuget::Metadatum, type: :model, feature_category: :pack it { is_expected.to validate_presence_of(:description) } it { is_expected.to validate_length_of(:description).is_at_most(described_class::MAX_DESCRIPTION_LENGTH) } + context 'for normalized_version presence' do + it { is_expected.to validate_presence_of(:normalized_version) } + + context 'when nuget_normalized_version feature flag is disabled' do + before do + stub_feature_flags(nuget_normalized_version: false) + end + + it { is_expected.not_to validate_presence_of(:normalized_version) } + end + end + %i[license_url project_url icon_url].each do |url| describe "##{url}" do it { is_expected.to allow_value('http://sandbox.com').for(url) } @@ -36,4 +50,54 @@ RSpec.describe Packages::Nuget::Metadatum, type: :model, feature_category: :pack end end end + + it { is_expected.to delegate_method(:version).to(:package).with_prefix } + + describe '.normalized_version_in' do + let_it_be(:nuget_metadatums) { create_list(:nuget_metadatum, 2) } + + subject { described_class.normalized_version_in(nuget_metadatums.first.normalized_version) } + + it { is_expected.to contain_exactly(nuget_metadatums.first) } + end + + describe 'callbacks' do + describe '#set_normalized_version' do + using RSpec::Parameterized::TableSyntax + + let_it_be_with_reload(:nuget_metadatum) { create(:nuget_metadatum) } + + where(:version, :normalized_version) do + '1.0' | '1.0.0' + '1.0.0.0' | '1.0.0' + '0.1' | '0.1.0' + '1.0.7+r3456' | '1.0.7' + '8.0.0.00+RC.54' | '8.0.0' + '1.0.0-Alpha' | '1.0.0-alpha' + '1.0.00-RC-02' | '1.0.0-rc-02' + '8.0.000-preview.0.546.0' | '8.0.0-preview.0.546.0' + '0.1.0-dev.37+0999370' | '0.1.0-dev.37' + '1.2.3' | '1.2.3' + end + + with_them do + it 'saves the normalized version' do + nuget_metadatum.package.update_column(:version, version) + nuget_metadatum.save! + + expect(nuget_metadatum.normalized_version).to eq(normalized_version) + end + + context 'when the nuget_normalized_version feature flag is disabled' do + before do + stub_feature_flags(nuget_normalized_version: false) + end + + it 'does not save the normalized version' do + expect(nuget_metadatum.normalized_version).not_to eq(normalized_version) + end + end + end + end + end end diff --git a/spec/models/packages/package_spec.rb b/spec/models/packages/package_spec.rb index 56cb3ef298d..381b5af117e 100644 --- a/spec/models/packages/package_spec.rb +++ b/spec/models/packages/package_spec.rb @@ -976,6 +976,35 @@ RSpec.describe Packages::Package, type: :model, feature_category: :package_regis it { is_expected.to match_array([nuget_package]) } end + describe '.with_case_insensitive_name' do + let_it_be(:nuget_package) { create(:nuget_package, name: 'TestPackage') } + + subject { described_class.with_case_insensitive_name('testpackage') } + + it { is_expected.to match_array([nuget_package]) } + end + + describe '.with_nuget_version_or_normalized_version' do + let_it_be(:nuget_package) { create(:nuget_package, :with_metadatum, version: '1.0.7+r3456') } + + before do + nuget_package.nuget_metadatum.update_column(:normalized_version, '1.0.7') + end + + subject { described_class.with_nuget_version_or_normalized_version(version, with_normalized: with_normalized) } + + where(:version, :with_normalized, :expected) do + '1.0.7' | true | [ref(:nuget_package)] + '1.0.7' | false | [] + '1.0.7+r3456' | true | [ref(:nuget_package)] + '1.0.7+r3456' | false | [ref(:nuget_package)] + end + + with_them do + it { is_expected.to match_array(expected) } + end + end + context 'status scopes' do let_it_be(:default_package) { create(:maven_package, :default) } let_it_be(:hidden_package) { create(:maven_package, :hidden) } @@ -1432,6 +1461,19 @@ RSpec.describe Packages::Package, type: :model, feature_category: :package_regis end end + describe '#normalized_nuget_version' do + let_it_be(:package) { create(:nuget_package, :with_metadatum, version: '1.0') } + let(:normalized_version) { '1.0.0' } + + subject { package.normalized_nuget_version } + + before do + package.nuget_metadatum.update_column(:normalized_version, normalized_version) + end + + it { is_expected.to eq(normalized_version) } + end + describe "#publish_creation_event" do let_it_be(:project) { create(:project) } diff --git a/spec/requests/api/nuget_project_packages_spec.rb b/spec/requests/api/nuget_project_packages_spec.rb index 40068f101f7..2d3781da42b 100644 --- a/spec/requests/api/nuget_project_packages_spec.rb +++ b/spec/requests/api/nuget_project_packages_spec.rb @@ -133,7 +133,7 @@ RSpec.describe API::NugetProjectPackages, feature_category: :package_registry do end describe 'GET /api/v4/projects/:id/packages/nuget/download/*package_name/*package_version/*package_filename' do - let_it_be(:package) { create(:nuget_package, :with_symbol_package, project: project, name: package_name) } + let_it_be(:package) { create(:nuget_package, :with_symbol_package, :with_metadatum, project: project, name: package_name, version: '0.1') } let(:format) { 'nupkg' } let(:url) { "/projects/#{target.id}/packages/nuget/download/#{package.name}/#{package.version}/#{package.name}.#{package.version}.#{format}" } diff --git a/spec/requests/api/snippets_spec.rb b/spec/requests/api/snippets_spec.rb index 4ba2a768e01..eb31f57a0a0 100644 --- a/spec/requests/api/snippets_spec.rb +++ b/spec/requests/api/snippets_spec.rb @@ -13,7 +13,7 @@ RSpec.describe API::Snippets, :aggregate_failures, factory_default: :keep, featu let_it_be_with_refind(:private_snippet) { create(:personal_snippet, :repository, :private, author: user) } let_it_be(:internal_snippet) { create(:personal_snippet, :repository, :internal, author: user) } - let_it_be(:user_token) { create(:personal_access_token, user: user) } + let_it_be(:user_token) { create(:personal_access_token, user: user) } let_it_be(:other_user_token) { create(:personal_access_token, user: other_user) } let_it_be(:project) do create_default(:project, :public).tap do |p| @@ -21,9 +21,17 @@ RSpec.describe API::Snippets, :aggregate_failures, factory_default: :keep, featu end end - describe 'GET /snippets/' do + shared_examples "returns unauthorized when not authenticated" do + it 'returns 401 for non-authenticated' do + get api(path) + + expect(response).to have_gitlab_http_status(:unauthorized) + end + end + + shared_examples "returns filtered snippets for user" do it 'returns snippets available for user' do - get api("/snippets/", personal_access_token: user_token) + get api(path, personal_access_token: user_token) expect(response).to have_gitlab_http_status(:ok) expect(response).to include_pagination_headers @@ -38,32 +46,6 @@ RSpec.describe API::Snippets, :aggregate_failures, factory_default: :keep, featu expect(json_response.last).to have_key('visibility') end - it 'hides private snippets from regular user' do - get api("/snippets/", personal_access_token: other_user_token) - - expect(response).to have_gitlab_http_status(:ok) - expect(response).to include_pagination_headers - expect(json_response).to be_an Array - expect(json_response.size).to eq(0) - end - - it 'returns 401 for non-authenticated' do - get api("/snippets/") - - expect(response).to have_gitlab_http_status(:unauthorized) - end - - it 'does not return snippets related to a project with disable feature visibility' do - public_snippet = create(:project_snippet, :public, author: user, project: project) - project.project_feature.update_attribute(:snippets_access_level, 0) - - get api("/snippets/", personal_access_token: user_token) - - json_response.each do |snippet| - expect(snippet["id"]).not_to eq(public_snippet.id) - end - end - context 'filtering snippets by created_after/created_before' do let_it_be(:private_snippet_before_time_range) { create(:personal_snippet, :repository, :private, author: user, created_at: Time.parse("2021-08-20T00:00:00Z")) } let_it_be(:private_snippet_in_time_range1) { create(:personal_snippet, :repository, :private, author: user, created_at: Time.parse("2021-08-22T00:00:00Z")) } @@ -82,6 +64,33 @@ RSpec.describe API::Snippets, :aggregate_failures, factory_default: :keep, featu end end + describe 'GET /snippets/' do + let(:path) { "/snippets" } + + it_behaves_like "returns unauthorized when not authenticated" + it_behaves_like "returns filtered snippets for user" + + it 'hides private snippets from regular user' do + get api(path, personal_access_token: other_user_token) + + expect(response).to have_gitlab_http_status(:ok) + expect(response).to include_pagination_headers + expect(json_response).to be_an Array + expect(json_response.size).to eq(0) + end + + it 'does not return snippets related to a project with disable feature visibility' do + public_snippet = create(:project_snippet, :public, author: user, project: project) + project.project_feature.update_attribute(:snippets_access_level, 0) + + get api(path, personal_access_token: user_token) + + json_response.each do |snippet| + expect(snippet["id"]).not_to eq(public_snippet.id) + end + end + end + describe 'GET /snippets/public' do let_it_be(:public_snippet_other) { create(:personal_snippet, :repository, :public, author: other_user) } let_it_be(:private_snippet_other) { create(:personal_snippet, :repository, :private, author: other_user) } @@ -92,6 +101,8 @@ RSpec.describe API::Snippets, :aggregate_failures, factory_default: :keep, featu let(:path) { "/snippets/public" } + it_behaves_like "returns unauthorized when not authenticated" + it 'returns only public snippets from all users when authenticated' do get api(path, personal_access_token: user_token) @@ -110,12 +121,6 @@ RSpec.describe API::Snippets, :aggregate_failures, factory_default: :keep, featu end end - it 'requires authentication' do - get api(path, nil) - - expect(response).to have_gitlab_http_status(:unauthorized) - end - context 'filtering public snippets by created_after/created_before' do let_it_be(:public_snippet_before_time_range) { create(:personal_snippet, :repository, :public, author: other_user, created_at: Time.parse("2021-08-20T00:00:00Z")) } let_it_be(:public_snippet_in_time_range) { create(:personal_snippet, :repository, :public, author: other_user, created_at: Time.parse("2021-08-22T00:00:00Z")) } diff --git a/spec/services/ci/retry_job_service_spec.rb b/spec/services/ci/retry_job_service_spec.rb index f15f4a16d4f..caed9815fb3 100644 --- a/spec/services/ci/retry_job_service_spec.rb +++ b/spec/services/ci/retry_job_service_spec.rb @@ -208,6 +208,45 @@ RSpec.describe Ci::RetryJobService, feature_category: :continuous_integration do end end + shared_examples_for 'creates associations for a deployable job' do |factory_type| + context 'when a job with a deployment is retried' do + let!(:job) do + create(factory_type, :with_deployment, :deploy_to_production, pipeline: pipeline, ci_stage: stage) + end + + it 'creates a new deployment' do + expect { new_job }.to change { Deployment.count }.by(1) + end + + it 'does not create a new environment' do + expect { new_job }.not_to change { Environment.count } + end + end + + context 'when a job with a dynamic environment is retried' do + let_it_be(:other_developer) { create(:user).tap { |u| project.add_developer(u) } } + + let(:environment_name) { 'review/$CI_COMMIT_REF_SLUG-$GITLAB_USER_ID' } + + let!(:job) do + create(factory_type, :with_deployment, + environment: environment_name, + options: { environment: { name: environment_name } }, + pipeline: pipeline, + ci_stage: stage, + user: other_developer) + end + + it 'creates a new deployment' do + expect { new_job }.to change { Deployment.count }.by(1) + end + + it 'does not create a new environment' do + expect { new_job }.not_to change { Environment.count } + end + end + end + describe '#clone!' do let(:new_job) { service.clone!(job) } @@ -219,6 +258,7 @@ RSpec.describe Ci::RetryJobService, feature_category: :continuous_integration do include_context 'retryable bridge' it_behaves_like 'clones the job' + it_behaves_like 'creates associations for a deployable job', :ci_bridge context 'when given variables' do let(:new_job) { service.clone!(job, variables: job_variables_attributes) } @@ -235,43 +275,7 @@ RSpec.describe Ci::RetryJobService, feature_category: :continuous_integration do let(:job) { job_to_clone } it_behaves_like 'clones the job' - - context 'when a build with a deployment is retried' do - let!(:job) do - create(:ci_build, :with_deployment, :deploy_to_production, pipeline: pipeline, ci_stage: stage) - end - - it 'creates a new deployment' do - expect { new_job }.to change { Deployment.count }.by(1) - end - - it 'does not create a new environment' do - expect { new_job }.not_to change { Environment.count } - end - end - - context 'when a build with a dynamic environment is retried' do - let_it_be(:other_developer) { create(:user).tap { |u| project.add_developer(u) } } - - let(:environment_name) { 'review/$CI_COMMIT_REF_SLUG-$GITLAB_USER_ID' } - - let!(:job) do - create(:ci_build, :with_deployment, - environment: environment_name, - options: { environment: { name: environment_name } }, - pipeline: pipeline, - ci_stage: stage, - user: other_developer) - end - - it 'creates a new deployment' do - expect { new_job }.to change { Deployment.count }.by(1) - end - - it 'does not create a new environment' do - expect { new_job }.not_to change { Environment.count } - end - end + it_behaves_like 'creates associations for a deployable job', :ci_build context 'when given variables' do let(:new_job) { service.clone!(job, variables: job_variables_attributes) } diff --git a/spec/support/shared_examples/requests/api/nuget_packages_shared_examples.rb b/spec/support/shared_examples/requests/api/nuget_packages_shared_examples.rb index 0f1ec14a0b6..5854958a06e 100644 --- a/spec/support/shared_examples/requests/api/nuget_packages_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/nuget_packages_shared_examples.rb @@ -327,6 +327,33 @@ RSpec.shared_examples 'process nuget download content request' do |user_type, st expect(response.media_type).to eq('application/octet-stream') end end + + context 'with normalized package version' do + let(:normalized_version) { '0.1.0' } + let(:url) { "/projects/#{target.id}/packages/nuget/download/#{package.name}/#{normalized_version}/#{package.name}.#{package.version}.#{format}" } + + before do + package.nuget_metadatum.update_column(:normalized_version, normalized_version) + end + + it_behaves_like 'returning response status', status + + it 'returns a valid package archive' do + subject + + expect(response.media_type).to eq('application/octet-stream') + end + + it_behaves_like 'bumping the package last downloaded at field' + + context 'when nuget_normalized_version feature flag is disabled' do + before do + stub_feature_flags(nuget_normalized_version: false) + end + + it_behaves_like 'returning response status', :not_found + end + end end end diff --git a/yarn.lock b/yarn.lock index e4d728799fe..2beff3c1b74 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1151,10 +1151,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/visual-review-tools/-/visual-review-tools-1.7.3.tgz#9ea641146436da388ffbad25d7f2abe0df52c235" integrity sha512-NMV++7Ew1FSBDN1xiZaauU9tfeSfgDHcOLpn+8bGpP+O5orUPm2Eu66R5eC5gkjBPaXosNAxNWtriee+aFk4+g== -"@gitlab/web-ide@0.0.1-dev-20230802205337": - version "0.0.1-dev-20230802205337" - resolved "https://registry.yarnpkg.com/@gitlab/web-ide/-/web-ide-0.0.1-dev-20230802205337.tgz#bd1954486e9d615d65864cfa9ce4876ebc2a29a4" - integrity sha512-cek6IixB+oW39iYwyce+x9yiHWdZn4EwDeXCLgdzWpYl74tccKFfkjt2ZTtWsfZUKGE+xJkoeTE+ZKnoeaUSPA== +"@gitlab/web-ide@0.0.1-dev-20230807045127": + version "0.0.1-dev-20230807045127" + resolved "https://registry.yarnpkg.com/@gitlab/web-ide/-/web-ide-0.0.1-dev-20230807045127.tgz#9b901a33368fa05c8abe0ef61a1035c0e77e31b7" + integrity sha512-DpwLvqigsNmYQvdxBrYgWMf0cBcDGTIMXooQH0XnsJWImbaYtLJBoTU0TsLIIWlcbpNqFVmf/BWfObfwM0Nfsg== "@graphql-eslint/eslint-plugin@3.20.1": version "3.20.1"