diff --git a/config/feature_flags/development/variables_in_include_section_ci.yml b/config/feature_flags/development/variables_in_include_section_ci.yml new file mode 100644 index 00000000000..f6fc810e6f2 --- /dev/null +++ b/config/feature_flags/development/variables_in_include_section_ci.yml @@ -0,0 +1,8 @@ +--- +name: variables_in_include_section_ci +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50188/ +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/294294 +milestone: '13.8' +type: development +group: group::compliance +default_enabled: false diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 071a8ef830f..e1f449d5b0b 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -98,7 +98,8 @@ module Gitlab project: project, sha: sha || project&.repository&.root_ref_sha, user: user, - parent_pipeline: parent_pipeline) + parent_pipeline: parent_pipeline, + variables: project&.predefined_variables&.to_runner_variables) end def track_and_raise_for_dev_exception(error) diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index cf6c2961ee7..e0adb1b19c2 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -7,14 +7,15 @@ module Gitlab class Context TimeoutError = Class.new(StandardError) - attr_reader :project, :sha, :user, :parent_pipeline + attr_reader :project, :sha, :user, :parent_pipeline, :variables attr_reader :expandset, :execution_deadline - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: []) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline + @variables = variables @expandset = Set.new @execution_deadline = 0 diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index e74f5b33de7..fdb3e1b00f9 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -41,7 +41,8 @@ module Gitlab project: context.project, sha: context.sha, user: context.user, - parent_pipeline: context.parent_pipeline + parent_pipeline: context.parent_pipeline, + variables: context.variables } end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index be479741784..114d493381c 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -72,7 +72,8 @@ module Gitlab project: project, sha: sha, user: context.user, - parent_pipeline: context.parent_pipeline + parent_pipeline: context.parent_pipeline, + variables: context.variables } end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 90692eafc3f..909b58ad796 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -34,6 +34,7 @@ module Gitlab .compact .map(&method(:normalize_location)) .flat_map(&method(:expand_project_files)) + .map(&method(:expand_variables)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end @@ -47,7 +48,8 @@ module Gitlab # convert location if String to canonical form def normalize_location(location) if location.is_a?(String) - normalize_location_string(location) + expanded_location = expand_variables(location) + normalize_location_string(expanded_location) else location.deep_symbolize_keys end @@ -96,6 +98,33 @@ module Gitlab matching.first end + + def expand_variables(data) + return data unless ::Feature.enabled?(:variables_in_include_section_ci) + + if data.is_a?(String) + expand(data) + else + transform(data) + end + end + + def transform(data) + data.transform_values do |values| + case values + when Array + values.map { |value| expand(value.to_s) } + when String + expand(values) + else + values + end + end + end + + def expand(data) + ExpandVariables.expand(data, context.variables) + end end end end diff --git a/spec/helpers/blob_helper_spec.rb b/spec/helpers/blob_helper_spec.rb index 764c582e987..3366b7eeadf 100644 --- a/spec/helpers/blob_helper_spec.rb +++ b/spec/helpers/blob_helper_spec.rb @@ -94,7 +94,7 @@ RSpec.describe BlobHelper do context 'viewer related' do include FakeBlobHelpers - let(:project) { build(:project, lfs_enabled: true) } + let_it_be(:project) { create(:project, lfs_enabled: true) } before do allow(Gitlab.config.lfs).to receive(:enabled).and_return(true) diff --git a/spec/lib/gitlab/ci/config/external/file/local_spec.rb b/spec/lib/gitlab/ci/config/external/file/local_spec.rb index fdd29afe2d6..7e39fae7b9b 100644 --- a/spec/lib/gitlab/ci/config/external/file/local_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/local_spec.rb @@ -16,7 +16,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do project: project, sha: sha, user: user, - parent_pipeline: parent_pipeline + parent_pipeline: parent_pipeline, + variables: project.predefined_variables.to_runner_variables } end @@ -131,7 +132,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do user: user, project: project, sha: sha, - parent_pipeline: parent_pipeline) + parent_pipeline: parent_pipeline, + variables: project.predefined_variables.to_runner_variables) end end diff --git a/spec/lib/gitlab/ci/config/external/file/project_spec.rb b/spec/lib/gitlab/ci/config/external/file/project_spec.rb index a5e4e27df6f..0e8851ba915 100644 --- a/spec/lib/gitlab/ci/config/external/file/project_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/project_spec.rb @@ -16,7 +16,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do project: context_project, sha: '12345', user: context_user, - parent_pipeline: parent_pipeline + parent_pipeline: parent_pipeline, + variables: project.predefined_variables.to_runner_variables } end @@ -165,7 +166,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do user: user, project: project, sha: project.commit('master').id, - parent_pipeline: parent_pipeline) + parent_pipeline: parent_pipeline, + variables: project.predefined_variables.to_runner_variables) end end diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index 7ad57827e30..bb6a3cc95e0 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' } let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } let(:template_file) { 'Auto-DevOps.gitlab-ci.yml' } - let(:context_params) { { project: project, sha: '123456', user: user } } + let(:context_params) { { project: project, sha: '123456', user: user, variables: project.predefined_variables.to_runner_variables } } let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } let(:file_content) do @@ -236,5 +236,118 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do end end end + + context "when 'include' section uses project variable" do + let(:full_local_file_path) { '$CI_PROJECT_PATH' + local_file } + + context 'when local file is included as a single string' do + let(:values) do + { include: full_local_file_path } + end + + it 'expands the variable', :aggregate_failures do + expect(subject[0].location).to eq(project.full_path + local_file) + expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Local)) + end + end + + context 'when remote file is included as a single string' do + let(:remote_url) { "#{Gitlab.config.gitlab.url}/radio/.gitlab-ci.yml" } + + let(:values) do + { include: '$CI_SERVER_URL/radio/.gitlab-ci.yml' } + end + + it 'expands the variable', :aggregate_failures do + expect(subject[0].location).to eq(remote_url) + expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote)) + end + end + + context 'defined as an array' do + let(:values) do + { include: [full_local_file_path, remote_url], + image: 'ruby:2.7' } + end + + it 'expands the variable' do + expect(subject[0].location).to eq(project.full_path + local_file) + expect(subject[1].location).to eq(remote_url) + end + end + + context 'defined as an array of hashes' do + let(:values) do + { include: [{ local: full_local_file_path }, { remote: remote_url }], + image: 'ruby:2.7' } + end + + it 'expands the variable' do + expect(subject[0].location).to eq(project.full_path + local_file) + expect(subject[1].location).to eq(remote_url) + end + end + + context 'local file hash' do + let(:values) do + { include: { 'local' => full_local_file_path } } + end + + it 'expands the variable' do + expect(subject[0].location).to eq(project.full_path + local_file) + end + end + + context 'project name' do + let(:values) do + { include: { project: '$CI_PROJECT_PATH', file: local_file }, + image: 'ruby:2.7' } + end + + it 'expands the variable', :aggregate_failures do + expect(subject[0].project_name).to eq(project.full_path) + expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Project)) + end + end + + context 'with multiple files' do + let(:values) do + { include: { project: project.full_path, file: [full_local_file_path, 'another_file_path.yml'] }, + image: 'ruby:2.7' } + end + + it 'expands the variable' do + expect(subject[0].location).to eq(project.full_path + local_file) + expect(subject[1].location).to eq('another_file_path.yml') + end + end + + context 'when include variable has an unsupported type for variable expansion' do + let(:values) do + { include: { project: project.id, file: local_file }, + image: 'ruby:2.7' } + end + + it 'does not invoke expansion for the variable', :aggregate_failures do + expect(ExpandVariables).not_to receive(:expand).with(project.id, context_params[:variables]) + + expect { subject }.to raise_error(described_class::AmbigiousSpecificationError) + end + end + + context 'when feature flag is turned off' do + let(:values) do + { include: full_local_file_path } + end + + before do + stub_feature_flags(variables_in_include_section_ci: false) + end + + it 'does not expand the variables' do + expect(subject[0].location).to eq('$CI_PROJECT_PATH' + local_file) + end + end + end end end