Add latest changes from gitlab-org/gitlab@master

app/helpers/application_settings_helper.rb

@@ -450,6 +450,7 @@ module ApplicationSettingsHelper
       :group_export_limit,
       :group_download_export_limit,
       :wiki_page_max_content_bytes,
+      :wiki_asciidoc_allow_uri_includes,
       :container_registry_delete_tags_service_timeout,
       :rate_limiting_response_text,
       :package_registry_cleanup_policies_worker_capacity,

app/models/application_setting.rb

@@ -379,6 +379,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord
 
   validates :snippet_size_limit, numericality: { only_integer: true, greater_than: 0 }
   validates :wiki_page_max_content_bytes, numericality: { only_integer: true, greater_than_or_equal_to: 1.kilobytes }
+  validates :wiki_asciidoc_allow_uri_includes, inclusion: { in: [true, false], message: N_('must be a boolean value') }
   validates :max_yaml_size_bytes, numericality: { only_integer: true, greater_than: 0 }, presence: true
   validates :max_yaml_depth, numericality: { only_integer: true, greater_than: 0 }, presence: true
 

app/models/application_setting_implementation.rb

@@ -223,6 +223,7 @@ module ApplicationSettingImplementation
         user_show_add_ssh_key_message: true,
         valid_runner_registrars: VALID_RUNNER_REGISTRAR_TYPES,
         wiki_page_max_content_bytes: 50.megabytes,
+        wiki_asciidoc_allow_uri_includes: false,
         package_registry_cleanup_policies_worker_capacity: 2,
         container_registry_delete_tags_service_timeout: 250,
         container_registry_expiration_policies_worker_capacity: 4,

db/migrate/20230222131512_add_wiki_asciidoc_allow_uri_includes.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddWikiAsciidocAllowUriIncludes < Gitlab::Database::Migration[2.1]
+  enable_lock_retries!
+  def change
+    add_column :application_settings, :wiki_asciidoc_allow_uri_includes, :boolean, default: false, null: false
+  end
+end

db/schema_migrations/20230222131512

@@ -0,0 +1 @@
+676433c9330c304524c444c3d630558c849654173cd78f7e499087569203b7eb

db/structure.sql

@@ -11834,6 +11834,7 @@ CREATE TABLE application_settings (
     encrypted_anthropic_api_key_iv bytea,
     allow_account_deletion boolean DEFAULT true NOT NULL,
     vertex_project text,
+    wiki_asciidoc_allow_uri_includes boolean DEFAULT false NOT NULL,
     CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
     CONSTRAINT app_settings_container_registry_pre_import_tags_rate_positive CHECK ((container_registry_pre_import_tags_rate >= (0)::numeric)),
     CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)),

doc/administration/wikis/index.md

@@ -82,6 +82,51 @@ so you should keep your wiki repositories as compact as possible.
 For more information about tools to compact repositories,
 read the documentation on [reducing repository size](../../user/project/repository/reducing_the_repo_size_using_git.md).
 
+## Allow URI includes for AsciiDoc
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/348687) in GitLab 16.X (TBD)
+
+Include directives import content from separate pages or external URLs,
+and display them as part of the content of the current document. To enable
+AsciiDoc includes, enable the feature through the Rails console or the API.
+
+### Through the Rails console
+
+To configure this setting through the Rails console:
+
+1. Start the Rails console:
+
+   ```shell
+   # For Omnibus installations
+   sudo gitlab-rails console
+
+   # For installations from source
+   sudo -u git -H bundle exec rails console -e production
+   ```
+
+1. Update the wiki to allow URI includes for AsciiDoc:
+
+   ```ruby
+   ApplicationSetting.first.update!(wiki_asciidoc_allow_uri_includes: true)
+   ```
+
+To check if includes are enabled, start the Rails console and run:
+
+  ```ruby
+  Gitlab::CurrentSettings.wiki_asciidoc_allow_uri_includes
+  ```
+
+### Through the API
+
+To set the wiki to allow URI includes for AsciiDoc through the
+[Application Settings API](../../api/settings.md#change-application-settings),
+use a `curl` command:
+
+```shell
+curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" \
+  "https://gitlab.example.com/api/v4/application/settings?wiki_asciidoc_allow_uri_includes=true"
+```
+
 ## Related topics
 
 - [User documentation for wikis](../../user/project/wiki/index.md)

lib/api/settings.rb

@@ -182,6 +182,7 @@ module API
       optional :issues_create_limit, type: Integer, desc: "Maximum number of issue creation requests allowed per minute per user. Set to 0 for unlimited requests per minute."
       optional :raw_blob_request_limit, type: Integer, desc: "Maximum number of requests per minute for each raw path. Set to 0 for unlimited requests per minute."
       optional :wiki_page_max_content_bytes, type: Integer, desc: "Maximum wiki page content size in bytes"
+      optional :wiki_asciidoc_allow_uri_includes, type: Boolean, desc: "Allow URI includes for AsciiDoc wiki pages"
       optional :require_admin_approval_after_user_signup, type: Boolean, desc: 'Require explicit admin approval for new signups'
       optional :whats_new_variant, type: String, values: ApplicationSetting.whats_new_variants.keys, desc: "What's new variant, possible values: `all_tiers`, `current_tier`, and `disabled`."
       optional :floc_enabled, type: Grape::API::Boolean, desc: 'Enable FloC (Federated Learning of Cohorts)'

lib/gitlab/asciidoc.rb

@@ -70,7 +70,8 @@ module Gitlab
                                         .merge({
                                                    # Define the Kroki server URL from the settings.
                                                    # This attribute cannot be overridden from the AsciiDoc document.
-                                                   'kroki-server-url' => Gitlab::CurrentSettings.kroki_url
+                                                   'kroki-server-url' => Gitlab::CurrentSettings.kroki_url,
+                                                   'allow-uri-read' => Gitlab::CurrentSettings.wiki_asciidoc_allow_uri_includes
                                                }),
                         extensions: extensions }
 

lib/gitlab/asciidoc/include_processor.rb

@@ -9,6 +9,8 @@ module Gitlab
     class IncludeProcessor < Asciidoctor::IncludeExt::IncludeProcessor
       extend ::Gitlab::Utils::Override
 
+      NoData = Class.new(StandardError)
+
       def initialize(context)
         super(logger: Gitlab::AppLogger)
 
@@ -16,6 +18,7 @@ module Gitlab
         @repository = context[:repository] || context[:project].try(:repository)
         @max_includes = context[:max_includes].to_i
         @included = []
+        @included_content = {}
 
         # Note: Asciidoctor calls #freeze on extensions, so we can't set new
         # instance variables after initialization.
@@ -31,9 +34,10 @@ module Gitlab
         doc = reader.document
 
         max_include_depth = doc.attributes.fetch('max-include-depth').to_i
+        allow_uri_read = doc.attributes.fetch('allow-uri-read', false)
 
         return false if max_include_depth < 1
-        return false if target_http?(target)
+        return false if target_http?(target) && !allow_uri_read
         return false if included.size >= max_includes
 
         true
@@ -42,6 +46,7 @@ module Gitlab
       override :resolve_target_path
       def resolve_target_path(target, reader)
         return unless repository.try(:exists?)
+        return target if target_http?(target)
 
         base_path = reader.include_stack.empty? ? requested_path : reader.file
         path = resolve_relative_path(target, base_path)
@@ -51,12 +56,15 @@ module Gitlab
 
       override :read_lines
       def read_lines(filename, selector)
-        blob = read_blob(ref, filename)
+        content = read_content(filename)
+        raise NoData, filename if content.nil?
+
+        included << filename
 
         if selector
-          blob.data.each_line.select.with_index(1, &selector)
+          content.each_line.select.with_index(1, &selector)
         else
-          blob.data
+          content.lines
         end
       end
 
@@ -67,7 +75,17 @@ module Gitlab
 
       private
 
-      attr_reader :context, :repository, :cache, :max_includes, :included
+      attr_reader :context, :repository, :cache, :max_includes, :included, :included_content
+
+      def read_content(filename)
+        return included_content[filename] if included_content.key?(filename)
+
+        included_content[filename] = if target_http?(filename)
+                                       read_uri(filename)
+                                     else
+                                       read_blob(ref, filename)
+                                     end
+      end
 
       # Gets a Blob at a path for a specific revision.
       # This method will check that the Blob exists and contains readable text.
@@ -75,16 +93,22 @@ module Gitlab
       # revision - The String SHA1.
       # path     - The String file path.
       #
-      # Returns a Blob
+      # Returns a string containing the blob content
       def read_blob(ref, filename)
         blob = repository&.blob_at(ref, filename)
 
-        raise 'Blob not found' unless blob
+        raise NoData, 'Blob not found' unless blob
-        raise 'File is not readable' unless blob.readable_text?
+        raise NoData, 'File is not readable' unless blob.readable_text?
 
-        included << filename
+        blob.data
+      end
 
-        blob
+      def read_uri(uri)
+        r = Gitlab::HTTP.get(uri)
+
+        raise NoData, uri unless r.success?
+
+        r.body
       end
 
       # Resolves the given relative path of file in repository into canonical

spec/lib/gitlab/asciidoc/include_processor_spec.rb

@@ -18,32 +18,174 @@ RSpec.describe Gitlab::Asciidoc::IncludeProcessor do
   let(:max_includes) { 10 }
 
   let(:reader) { Asciidoctor::PreprocessorReader.new(document, lines, 'file.adoc') }
+
   let(:document) { Asciidoctor::Document.new(lines) }
 
   subject(:processor) { described_class.new(processor_context) }
 
   let(:a_blob) { double(:Blob, readable_text?: true, data: a_data) }
-  let(:a_data) { StringIO.new('include::b.adoc[]') }
+  let(:a_data) { 'include::b.adoc[]' }
 
-  let(:lines) { [':max-include-depth: 1000'] + Array.new(10, 'include::a.adoc[]') }
+  let(:directives) { [':max-include-depth: 1000'] }
+  let(:lines) { directives + Array.new(10, 'include::a.adoc[]') }
 
   before do
+    allow(project.repository).to receive(:blob_at).with(ref, anything).and_return(nil)
     allow(project.repository).to receive(:blob_at).with(ref, 'a.adoc').and_return(a_blob)
   end
 
+  describe 'read_lines' do
+    let(:result) { processor.send(:read_lines, filename, selector) }
+    let(:selector) { nil }
+
+    context 'when reading a file in the repository' do
+      let(:filename) { 'a.adoc' }
+
+      it 'returns the blob contents' do
+        expect(result).to match_array([a_data])
+      end
+
+      context 'when the blob does not exist' do
+        let(:filename) { 'this-file-does-not-exist' }
+
+        it 'raises NoData' do
+          expect { result }.to raise_error(described_class::NoData)
+        end
+      end
+
+      context 'when there is a selector' do
+        let(:a_data) { %w[a b c d].join("\n") }
+        let(:selector) { ->(_, lineno) { lineno.odd? } }
+
+        it 'selects the lines' do
+          expect(result).to eq %W[a\n c\n]
+        end
+      end
+
+      it 'allows at most N blob includes' do
+        max_includes.times do
+          processor.send(:read_lines, filename, selector)
+        end
+
+        expect(processor.send(:include_allowed?, 'anything', reader)).to be_falsey
+      end
+    end
+
+    context 'when reading content from a URL' do
+      let(:filename) { 'http://example.org/file' }
+
+      it 'fetches the data using a GET request' do
+        stub_request(:get, filename).to_return(status: 200, body: 'something')
+
+        expect(result).to match_array(['something'])
+      end
+
+      context 'when the URI returns 404' do
+        before do
+          stub_request(:get, filename).to_return(status: 404, body: 'not found')
+        end
+
+        it 'raises NoData' do
+          expect { result }.to raise_error(described_class::NoData)
+        end
+      end
+
+      it 'allows at most N HTTP includes' do
+        stub_request(:get, filename).to_return(status: 200, body: 'something')
+
+        max_includes.times do
+          processor.send(:read_lines, filename, selector)
+        end
+
+        expect(processor.send(:include_allowed?, 'anything', reader)).to be_falsey
+      end
+
+      context 'when there is a selector' do
+        let(:http_body) { %w[x y z].join("\n") }
+        let(:selector) { ->(_, lineno) { lineno.odd? } }
+
+        it 'selects the lines' do
+          stub_request(:get, filename).to_return(status: 200, body: http_body)
+
+          expect(result).to eq %W[x\n z]
+        end
+      end
+    end
+  end
+
   describe '#include_allowed?' do
+    context 'when allow-uri-read is nil' do
+      before do
+        allow(document).to receive(:attributes).and_return({ 'max-include-depth' => 100, 'allow-uri-read' => nil })
+      end
+
+      it 'allows http includes' do
+        expect(processor.send(:include_allowed?, 'http://example.com', reader)).to be_falsey
+        expect(processor.send(:include_allowed?, 'https://example.com', reader)).to be_falsey
+      end
+
+      it 'allows blob includes' do
+        expect(processor.send(:include_allowed?, 'a.blob', reader)).to be_truthy
+      end
+    end
+
+    context 'when allow-uri-read is false' do
+      before do
+        allow(document).to receive(:attributes).and_return({ 'max-include-depth' => 100, 'allow-uri-read' => false })
+      end
+
+      it 'allows http includes' do
+        expect(processor.send(:include_allowed?, 'http://example.com', reader)).to be_falsey
+        expect(processor.send(:include_allowed?, 'https://example.com', reader)).to be_falsey
+      end
+
+      it 'allows blob includes' do
+        expect(processor.send(:include_allowed?, 'a.blob', reader)).to be_truthy
+      end
+    end
+
+    context 'when allow-uri-read is true' do
+      before do
+        allow(document).to receive(:attributes).and_return({ 'max-include-depth' => 100, 'allow-uri-read' => true })
+      end
+
+      it 'allows http includes' do
+        expect(processor.send(:include_allowed?, 'http://example.com', reader)).to be_truthy
+        expect(processor.send(:include_allowed?, 'https://example.com', reader)).to be_truthy
+      end
+
+      it 'allows blob includes' do
+        expect(processor.send(:include_allowed?, 'a.blob', reader)).to be_truthy
+      end
+    end
+
+    context 'without allow-uri-read' do
+      before do
+        allow(document).to receive(:attributes).and_return({ 'max-include-depth' => 100 })
+      end
+
+      it 'forbids http includes' do
+        expect(processor.send(:include_allowed?, 'http://example.com', reader)).to be_falsey
+        expect(processor.send(:include_allowed?, 'https://example.com', reader)).to be_falsey
+      end
+
+      it 'allows blob includes' do
+        expect(processor.send(:include_allowed?, 'a.blob', reader)).to be_truthy
+      end
+    end
+
     it 'allows the first include' do
       expect(processor.send(:include_allowed?, 'foo.adoc', reader)).to be_truthy
     end
 
     it 'allows the Nth include' do
-      (max_includes - 1).times { processor.send(:read_blob, ref, 'a.adoc') }
+      (max_includes - 1).times { processor.send(:read_lines, 'a.adoc', nil) }
 
       expect(processor.send(:include_allowed?, 'foo.adoc', reader)).to be_truthy
     end
 
     it 'disallows the Nth + 1 include' do
-      max_includes.times { processor.send(:read_blob, ref, 'a.adoc') }
+      max_includes.times { processor.send(:read_lines, 'a.adoc', nil) }
 
       expect(processor.send(:include_allowed?, 'foo.adoc', reader)).to be_falsey
     end

spec/lib/gitlab/asciidoc_spec.rb

@@ -20,7 +20,7 @@ module Gitlab
         expected_asciidoc_opts = {
             safe: :secure,
             backend: :gitlab_html5,
-            attributes: described_class::DEFAULT_ADOC_ATTRS.merge({ "kroki-server-url" => nil }),
+            attributes: described_class::DEFAULT_ADOC_ATTRS.merge({ "kroki-server-url" => nil, "allow-uri-read" => false }),
             extensions: be_a(Proc)
         }
 
@@ -35,7 +35,7 @@ module Gitlab
           expected_asciidoc_opts = {
               safe: :secure,
               backend: :gitlab_html5,
-              attributes: described_class::DEFAULT_ADOC_ATTRS.merge({ "kroki-server-url" => nil }),
+              attributes: described_class::DEFAULT_ADOC_ATTRS.merge({ "kroki-server-url" => nil, "allow-uri-read" => false }),
               extensions: be_a(Proc)
           }
 
@@ -730,6 +730,19 @@ module Gitlab
           include_examples 'invalid include'
         end
 
+        context 'with a URI that returns 404' do
+          let(:include_path) { 'https://example.com/some_file.adoc' }
+
+          before do
+            stub_request(:get, include_path).to_return(status: 404, body: 'not found')
+            allow_any_instance_of(ApplicationSetting).to receive(:wiki_asciidoc_allow_uri_includes).and_return(true)
+          end
+
+          it 'renders Unresolved directive placeholder' do
+            is_expected.to include("<strong>[ERROR: include::#{include_path}[] - unresolved directive]</strong>")
+          end
+        end
+
         context 'with path to a textual file' do
           let(:include_path) { 'sample.adoc' }
 
@@ -804,6 +817,59 @@ module Gitlab
           end
         end
 
+        describe 'the effect of max-includes' do
+          before do
+            create_file 'doc/preface.adoc', 'source: preface'
+            create_file 'doc/chapter-1.adoc', 'source: chapter-1'
+            create_file 'license.adoc', 'source: license'
+            stub_request(:get, 'https://example.com/some_file.adoc')
+              .to_return(status: 200, body: 'source: interwebs')
+            stub_request(:get, 'https://example.com/other_file.adoc')
+              .to_return(status: 200, body: 'source: intertubes')
+            allow_any_instance_of(ApplicationSetting).to receive(:wiki_asciidoc_allow_uri_includes).and_return(true)
+          end
+
+          let(:input) do
+            <<~ADOC
+              Source: requested file
+
+              include::doc/preface.adoc[]
+              include::https://example.com/some_file.adoc[]
+              include::doc/chapter-1.adoc[]
+              include::https://example.com/other_file.adoc[]
+              include::license.adoc[]
+            ADOC
+          end
+
+          it 'includes the content of all sources' do
+            expect(output.gsub(/<[^>]+>/, '').gsub(/\n\s*/, "\n").strip).to eq <<~ADOC.strip
+              Source: requested file
+              source: preface
+              source: interwebs
+              source: chapter-1
+              source: intertubes
+              source: license
+            ADOC
+          end
+
+          context 'when the document includes more than MAX_INCLUDES' do
+            before do
+              stub_const("#{described_class}::MAX_INCLUDES", 2)
+            end
+
+            it 'includes only the content of the first 2 sources' do
+              expect(output.gsub(/<[^>]+>/, '').gsub(/\n\s*/, "\n").strip).to eq <<~ADOC.strip
+                Source: requested file
+                source: preface
+                source: interwebs
+                doc/chapter-1.adoc
+                https://example.com/other_file.adoc
+                license.adoc
+              ADOC
+            end
+          end
+        end
+
         context 'recursive includes with relative paths' do
           let(:input) do
             <<~ADOC
@@ -811,29 +877,53 @@ module Gitlab
 
               include::doc/README.adoc[]
 
-              include::license.adoc[]
+              include::https://example.com/some_file.adoc[]
+
+              include::license.adoc[lines=1]
             ADOC
           end
 
           before do
+            stub_request(:get, 'https://example.com/some_file.adoc')
+              .to_return(status: 200, body: <<~ADOC)
+                Source: some file from Example.com
+
+                include::https://example.com/other_file[lines=1..2]
+
+                End some file from Example.com
+              ADOC
+
+            stub_request(:get, 'https://example.com/other_file')
+              .to_return(status: 200, body: <<~ADOC)
+                Source: other file from Example.com
+                Other file line 2
+                Other file line 3
+              ADOC
+
             create_file 'doc/README.adoc', <<~ADOC
               Source: doc/README.adoc
 
-              include::../license.adoc[]
+              include::../license.adoc[lines=1;3]
 
               include::api/hello.adoc[]
             ADOC
             create_file 'license.adoc', <<~ADOC
               Source: license.adoc
+              License content
+              License end
             ADOC
             create_file 'doc/api/hello.adoc', <<~ADOC
               Source: doc/api/hello.adoc
 
-              include::./common.adoc[]
+              include::./common.adoc[lines=2..3]
             ADOC
             create_file 'doc/api/common.adoc', <<~ADOC
+              Common start
               Source: doc/api/common.adoc
+              Common end
             ADOC
+
+            allow_any_instance_of(ApplicationSetting).to receive(:wiki_asciidoc_allow_uri_includes).and_return(true)
           end
 
           it 'includes content of the included files recursively' do
@@ -841,8 +931,14 @@ module Gitlab
               Source: requested file
               Source: doc/README.adoc
               Source: license.adoc
+              License end
               Source: doc/api/hello.adoc
               Source: doc/api/common.adoc
+              Common end
+              Source: some file from Example.com
+              Source: other file from Example.com
+              Other file line 2
+              End some file from Example.com
               Source: license.adoc
             ADOC
           end

spec/models/application_setting_spec.rb

@@ -131,6 +131,9 @@ RSpec.describe ApplicationSetting, feature_category: :shared, type: :model do
 
     it { is_expected.to validate_numericality_of(:snippet_size_limit).only_integer.is_greater_than(0) }
     it { is_expected.to validate_numericality_of(:wiki_page_max_content_bytes).only_integer.is_greater_than_or_equal_to(1024) }
+    it { is_expected.to allow_value(true).for(:wiki_asciidoc_allow_uri_includes) }
+    it { is_expected.to allow_value(false).for(:wiki_asciidoc_allow_uri_includes) }
+    it { is_expected.not_to allow_value(nil).for(:wiki_asciidoc_allow_uri_includes) }
     it { is_expected.to validate_presence_of(:max_artifacts_size) }
     it { is_expected.to validate_numericality_of(:max_artifacts_size).only_integer.is_greater_than(0) }
     it { is_expected.to validate_presence_of(:max_yaml_size_bytes) }

spec/requests/api/settings_spec.rb

@@ -46,6 +46,7 @@ RSpec.describe API::Settings, 'Settings', :do_not_mock_admin_mode_setting, featu
       expect(json_response['spam_check_endpoint_url']).to be_nil
       expect(json_response['spam_check_api_key']).to be_nil
       expect(json_response['wiki_page_max_content_bytes']).to be_a(Integer)
+      expect(json_response['wiki_asciidoc_allow_uri_includes']).to be_falsey
       expect(json_response['require_admin_approval_after_user_signup']).to eq(true)
       expect(json_response['personal_access_token_prefix']).to eq('glpat-')
       expect(json_response['admin_mode']).to be(false)
@@ -166,6 +167,7 @@ RSpec.describe API::Settings, 'Settings', :do_not_mock_admin_mode_setting, featu
             disabled_oauth_sign_in_sources: 'unknown',
             import_sources: 'github,bitbucket',
             wiki_page_max_content_bytes: 12345,
+            wiki_asciidoc_allow_uri_includes: true,
             personal_access_token_prefix: "GL-",
             user_deactivation_emails_enabled: false,
             admin_mode: true,
@@ -243,6 +245,7 @@ RSpec.describe API::Settings, 'Settings', :do_not_mock_admin_mode_setting, featu
         expect(json_response['disabled_oauth_sign_in_sources']).to eq([])
         expect(json_response['import_sources']).to match_array(%w(github bitbucket))
         expect(json_response['wiki_page_max_content_bytes']).to eq(12345)
+        expect(json_response['wiki_asciidoc_allow_uri_includes']).to be(true)
         expect(json_response['personal_access_token_prefix']).to eq("GL-")
         expect(json_response['admin_mode']).to be(true)
         expect(json_response['user_deactivation_emails_enabled']).to be(false)