From 5ec2d1e9474e86064d5764bc991252dd1a370895 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 15 Oct 2020 03:08:35 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../bw-delete-jira-tracker-data-jobs.yml | 5 + .../generic-packages-enable-by-default.yml | 5 + .../development/generic_packages.yml | 2 +- ...kfill_jira_tracker_deployment_type_jobs.rb | 25 +++++ db/schema_migrations/20201014205300 | 1 + doc/api/lint.md | 92 ++++++++++++++++++- doc/user/packages/generic_packages/index.md | 14 +-- lib/api/generic_packages.rb | 2 +- .../authorize/authorize_field_service_spec.rb | 49 ++++++---- ..._jira_tracker_deployment_type_jobs_spec.rb | 58 ++++++++++++ 10 files changed, 224 insertions(+), 29 deletions(-) create mode 100644 changelogs/unreleased/bw-delete-jira-tracker-data-jobs.yml create mode 100644 changelogs/unreleased/generic-packages-enable-by-default.yml create mode 100644 db/migrate/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs.rb create mode 100644 db/schema_migrations/20201014205300 create mode 100644 spec/migrations/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs_spec.rb diff --git a/changelogs/unreleased/bw-delete-jira-tracker-data-jobs.yml b/changelogs/unreleased/bw-delete-jira-tracker-data-jobs.yml new file mode 100644 index 00000000000..a7ce3533df2 --- /dev/null +++ b/changelogs/unreleased/bw-delete-jira-tracker-data-jobs.yml @@ -0,0 +1,5 @@ +--- +title: Delete any outstanding BackfillJiraTrackerDeploymentType +merge_request: 45219 +author: +type: fixed diff --git a/changelogs/unreleased/generic-packages-enable-by-default.yml b/changelogs/unreleased/generic-packages-enable-by-default.yml new file mode 100644 index 00000000000..f3c7bd5a67d --- /dev/null +++ b/changelogs/unreleased/generic-packages-enable-by-default.yml @@ -0,0 +1,5 @@ +--- +title: Add support for Generic packages +merge_request: 45102 +author: +type: added diff --git a/config/feature_flags/development/generic_packages.yml b/config/feature_flags/development/generic_packages.yml index 99b89b196ea..7b80e50d372 100644 --- a/config/feature_flags/development/generic_packages.yml +++ b/config/feature_flags/development/generic_packages.yml @@ -4,4 +4,4 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40045 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/239133 group: group::release management type: development -default_enabled: false +default_enabled: true diff --git a/db/migrate/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs.rb b/db/migrate/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs.rb new file mode 100644 index 00000000000..ea45e82dcc4 --- /dev/null +++ b/db/migrate/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class DropBackfillJiraTrackerDeploymentTypeJobs < ActiveRecord::Migration[6.0] + DOWNTIME = false + DROPPED_JOB_CLASS = 'BackfillJiraTrackerDeploymentType'.freeze + QUEUE = 'background_migration'.freeze + + def up + sidekiq_queues.each do |queue| + queue.each do |job| + next unless job.args.first == DROPPED_JOB_CLASS + + job.delete + end + end + end + + def down + # no-op + end + + def sidekiq_queues + [Sidekiq::ScheduledSet.new, Sidekiq::RetrySet.new, Sidekiq::Queue.new(QUEUE)] + end +end diff --git a/db/schema_migrations/20201014205300 b/db/schema_migrations/20201014205300 new file mode 100644 index 00000000000..7d183f22624 --- /dev/null +++ b/db/schema_migrations/20201014205300 @@ -0,0 +1 @@ +05352623a59d56c1633317ba7dbaba7d75bb8e529a748a90512dcf3641b8d3cb \ No newline at end of file diff --git a/doc/api/lint.md b/doc/api/lint.md index addc8d0f9a3..c82e0845f99 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -64,7 +64,8 @@ Example responses: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29568) in GitLab 13.5. The CI lint returns an expanded version of the configuration. The expansion does not -work for CI configuration added with [`include: local`](../ci/yaml/README.md#includelocal). +work for CI configuration added with [`include: local`](../ci/yaml/README.md#includelocal), +or with [`extends:`](../ci/yaml/README.md#extends). Example contents of a `.gitlab-ci.yml` passed to the CI Lint API with `include_merged_yaml` set as true: @@ -145,3 +146,92 @@ Example responses: "warnings": [] } ``` + +## Use jq to create and process YAML & JSON payloads + +To `POST` a YAML configuration to the CI Lint endpoint, it must be properly escaped and JSON encoded. +You can use `jq` and `curl` to escape and upload YAML to the GitLab API. + +### Escape YAML for JSON encoding + +To escape quotes and encode your YAML in a format suitable for embedding within +a JSON payload, you can use `jq`. For example, create a file named `example-gitlab-ci.yml`: + +```yaml +.api_test: + rules: + - if: '$CI_PIPELINE_SOURCE=="merge_request_event"' + changes: + - src/api/* +deploy: + extends: + - .api_test + rules: + - when: manual + allow_failure: true + script: + - echo "hello world" +``` + +Next, use `jq` to escape and encode the YAML file into JSON: + +```shell +jq --raw-input --slurp < example-gitlab-ci.yml +``` + +To escape and encode an input YAML file (`example-gitlab-ci.yml`), and `POST` it to the +GitLab API using `curl` and `jq` in a one-line command: + +```shell +jq --null-input --arg yaml "$( +``` + +Example input: + +```json +{"status":"valid","errors":[],"merged_yaml":"---\n:.api_test:\n :rules:\n - :if: $CI_PIPELINE_SOURCE==\"merge_request_event\"\n :changes:\n - src/api/*\n:deploy:\n :rules:\n - :when: manual\n :allow_failure: true\n :extends:\n - \".api_test\"\n :script:\n - echo \"hello world\"\n"} +``` + +Becomes: + +```yaml +:.api_test: + :rules: + - :if: $CI_PIPELINE_SOURCE=="merge_request_event" + :changes: + - src/api/* +:deploy: + :rules: + - :when: manual + :allow_failure: true + :extends: + - ".api_test" + :script: + - echo "hello world" +``` + +With a one-line command, you can: + +1. Escape the YAML +1. Encode it in JSON +1. POST it to the API with curl +1. Format the response + +```shell +jq --null-input --arg yaml "$( - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4209) in GitLab 13.5. -> - It's [deployed behind a feature flag](../../../user/feature_flags.md), disabled by default. -> - It's disabled on GitLab.com. +> - It's [deployed behind a feature flag](../../../user/feature_flags.md), enabled by default. +> - It's enabled on GitLab.com. > - It's able to be enabled or disabled per-project. -> - It's not recommended for production use. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-generic-packages-in-the-package-registry). +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-generic-packages-in-the-package-registry). CAUTION: **Warning:** This feature might not be available to you. Check the **version history** note above for details. @@ -115,10 +115,10 @@ download: ### Enable or disable generic packages in the Package Registry -Support for generic packages is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. +Support for generic packages is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it. +can opt to disable it. To enable it: diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index e053319c1c6..a0c33ab65b9 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -101,7 +101,7 @@ module API include ::API::Helpers::Packages::BasicAuthHelpers def require_generic_packages_available! - not_found! unless Feature.enabled?(:generic_packages, project) + not_found! unless Feature.enabled?(:generic_packages, project, default_enabled: true) end def project diff --git a/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb b/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb index efe6c27c463..7576523ce52 100644 --- a/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb +++ b/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb @@ -19,24 +19,29 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do options.reverse_merge!(null: true) field :test_field, field_type, authorize: field_authorizations, - resolve: -> (_, _, _) { resolved_value }, **options + + define_method :test_field do + resolved_value + end end end - let(:current_user) { double(:current_user) } - subject(:service) { described_class.new(field) } describe '#authorized_resolve' do - let(:presented_object) { double('presented object') } - let(:presented_type) { double('parent type', object: presented_object) } - let(:query_type) { GraphQL::ObjectType.new } - let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)} - let(:query_context) { OpenStruct.new(schema: schema) } - let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema, context: query_context), values: { current_user: current_user }, object: nil) } + let_it_be(:current_user) { build(:user) } + let_it_be(:presented_object) { 'presented object' } + let_it_be(:query_type) { GraphQL::ObjectType.new } + let_it_be(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)} + let_it_be(:query) { GraphQL::Query.new(schema, document: nil, context: {}, variables: {}) } + let_it_be(:context) { GraphQL::Query::Context.new(query: query, values: { current_user: current_user }, object: nil) } - subject(:resolved) { service.authorized_resolve.call(presented_type, {}, context) } + let(:type_class) { type_with_field(custom_type, :read_field, presented_object) } + let(:type_instance) { type_class.authorized_new(presented_object, context) } + let(:field) { type_class.fields['testField'].to_graphql } + + subject(:resolved) { service.authorized_resolve.call(type_instance, {}, context) } context 'scalar types' do shared_examples 'checking permissions on the presented object' do @@ -48,7 +53,7 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do expect(resolved).to eq('Resolved value') end - it "returns nil if the value wasn't authorized" do + it 'returns nil if the value was not authorized' do allow(Ability).to receive(:allowed?).and_return false expect(resolved).to be_nil @@ -56,28 +61,28 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do end context 'when the field is a built-in scalar type' do - let(:field) { type_with_field(GraphQL::STRING_TYPE, :read_field).fields['testField'].to_graphql } + let(:type_class) { type_with_field(GraphQL::STRING_TYPE, :read_field) } let(:expected_permissions) { [:read_field] } it_behaves_like 'checking permissions on the presented object' end context 'when the field is a list of scalar types' do - let(:field) { type_with_field([GraphQL::STRING_TYPE], :read_field).fields['testField'].to_graphql } + let(:type_class) { type_with_field([GraphQL::STRING_TYPE], :read_field) } let(:expected_permissions) { [:read_field] } it_behaves_like 'checking permissions on the presented object' end context 'when the field is sub-classed scalar type' do - let(:field) { type_with_field(Types::TimeType, :read_field).fields['testField'].to_graphql } + let(:type_class) { type_with_field(Types::TimeType, :read_field) } let(:expected_permissions) { [:read_field] } it_behaves_like 'checking permissions on the presented object' end context 'when the field is a list of sub-classed scalar types' do - let(:field) { type_with_field([Types::TimeType], :read_field).fields['testField'].to_graphql } + let(:type_class) { type_with_field([Types::TimeType], :read_field) } let(:expected_permissions) { [:read_field] } it_behaves_like 'checking permissions on the presented object' @@ -86,7 +91,7 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do context 'when the field is a connection' do context 'when it resolves to nil' do - let(:field) { type_with_field(Types::QueryType.connection_type, :read_field, nil).fields['testField'].to_graphql } + let(:type_class) { type_with_field(Types::QueryType.connection_type, :read_field, nil) } it 'does not fail when authorizing' do expect(resolved).to be_nil @@ -97,7 +102,11 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do context 'when the field is a specific type' do let(:custom_type) { type(:read_type) } let(:object_in_field) { double('presented in field') } - let(:field) { type_with_field(custom_type, :read_field, object_in_field).fields['testField'].to_graphql } + + let(:type_class) { type_with_field(custom_type, :read_field, object_in_field) } + let(:type_instance) { type_class.authorized_new(object_in_field, context) } + + subject(:resolved) { service.authorized_resolve.call(type_instance, {}, context) } it 'checks both field & type permissions' do spy_ability_check_for(:read_field, object_in_field, passed: true) @@ -114,7 +123,7 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do end context 'when the field is not nullable' do - let(:field) { type_with_field(custom_type, [], object_in_field, null: false).fields['testField'].to_graphql } + let(:type_class) { type_with_field(custom_type, :read_field, object_in_field, null: false) } it 'returns nil when viewing is not allowed' do spy_ability_check_for(:read_type, object_in_field, passed: false) @@ -127,7 +136,9 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do let(:object_1) { double('presented in field 1') } let(:object_2) { double('presented in field 2') } let(:presented_types) { [double(object: object_1), double(object: object_2)] } - let(:field) { type_with_field([custom_type], :read_field, presented_types).fields['testField'].to_graphql } + + let(:type_class) { type_with_field([custom_type], :read_field, presented_types) } + let(:type_instance) { type_class.authorized_new(presented_types, context) } it 'checks all permissions' do allow(Ability).to receive(:allowed?) { true } diff --git a/spec/migrations/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs_spec.rb b/spec/migrations/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs_spec.rb new file mode 100644 index 00000000000..134bea6b666 --- /dev/null +++ b/spec/migrations/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'migrate', '20201014205300_drop_backfill_jira_tracker_deployment_type_jobs.rb') + +RSpec.describe DropBackfillJiraTrackerDeploymentTypeJobs, :sidekiq, :redis, schema: 2020_10_14_205300 do + subject(:migration) { described_class.new } + + describe '#up' do + let(:retry_set) { Sidekiq::RetrySet.new } + let(:scheduled_set) { Sidekiq::ScheduledSet.new } + + context 'there are only affected jobs on the queue' do + let(:payload) { { 'class' => ::BackgroundMigrationWorker, 'args' => [described_class::DROPPED_JOB_CLASS, 1] } } + let(:queue_payload) { payload.merge('queue' => described_class::QUEUE) } + + it 'removes enqueued BackfillJiraTrackerDeploymentType background jobs' do + Sidekiq::Testing.disable! do # https://github.com/mperham/sidekiq/wiki/testing#api Sidekiq's API does not have a testing mode + retry_set.schedule(1.hour.from_now, payload) + scheduled_set.schedule(1.hour.from_now, payload) + Sidekiq::Client.push(queue_payload) + + expect { migration.up }.to change { Sidekiq::Queue.new(described_class::QUEUE).size }.from(1).to(0) + expect(retry_set.size).to eq(0) + expect(scheduled_set.size).to eq(0) + end + end + end + + context 'there are not any affected jobs on the queue' do + let(:payload) { { 'class' => ::BackgroundMigrationWorker, 'args' => ['SomeOtherClass', 1] } } + let(:queue_payload) { payload.merge('queue' => described_class::QUEUE) } + + it 'skips other enqueued jobs' do + Sidekiq::Testing.disable! do + retry_set.schedule(1.hour.from_now, payload) + scheduled_set.schedule(1.hour.from_now, payload) + Sidekiq::Client.push(queue_payload) + + expect { migration.up }.not_to change { Sidekiq::Queue.new(described_class::QUEUE).size } + expect(retry_set.size).to eq(1) + expect(scheduled_set.size).to eq(1) + end + end + end + + context 'other queues' do + it 'does not modify them' do + Sidekiq::Testing.disable! do + Sidekiq::Client.push('queue' => 'other', 'class' => ::BackgroundMigrationWorker, 'args' => ['SomeOtherClass', 1]) + Sidekiq::Client.push('queue' => 'other', 'class' => ::BackgroundMigrationWorker, 'args' => [described_class::DROPPED_JOB_CLASS, 1]) + + expect { migration.up }.not_to change { Sidekiq::Queue.new('other').size } + end + end + end + end +end