Add latest changes from gitlab-org/gitlab@master

app/models/ci/job_artifact.rb

@@ -235,6 +235,17 @@ module Ci
       hashed_path: 2
     }
 
+    # `locked` will be populated from the source of truth on Ci::Pipeline
+    # in order to clean up expired job artifacts in a performant way.
+    # The values should be the same as `Ci::Pipeline.lockeds` with the
+    # additional value of `unknown` to indicate rows that have not
+    # yet been populated from the parent Ci::Pipeline
+    enum locked: {
+      unlocked: 0,
+      artifacts_locked: 1,
+      unknown: 2
+    }, _prefix: :artifact
+
     def validate_file_format!
       unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
         errors.add(:base, _('Invalid file format with specified file type'))

db/migrate/20210826120834_add_locked_to_ci_job_artifacts.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class AddLockedToCiJobArtifacts < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  TABLE_NAME = 'ci_job_artifacts'
+  COLUMN_NAME = 'locked'
+
+  def up
+    with_lock_retries do
+      add_column TABLE_NAME, COLUMN_NAME, :smallint, default: 2
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_column TABLE_NAME, COLUMN_NAME
+    end
+  end
+end

db/post_migrate/20210826110839_prepare_indexes_for_ci_job_artifacts_expire_at_unlocked.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class PrepareIndexesForCiJobArtifactsExpireAtUnlocked < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  TABLE_NAME = 'ci_job_artifacts'
+  INDEX_NAME = 'ci_job_artifacts_expire_at_unlocked_idx'
+
+  def up
+    prepare_async_index TABLE_NAME, [:expire_at], where: 'locked = 0', name: INDEX_NAME
+  end
+
+  def down
+    unprepare_async_index_by_name TABLE_NAME, INDEX_NAME
+  end
+end

db/schema_migrations/20210826110839

@@ -0,0 +1 @@
+72b64ddbaf86eb296fe49fd38bea759d5247414142fe1cd11aee7e1d6171e142

db/schema_migrations/20210826120834

@@ -0,0 +1 @@
+a8cd5165815a2f1e6b825ea3ee2a9bde88c1790f6ebe92296bee6a9a892b83f2

db/structure.sql

@@ -11567,6 +11567,7 @@ CREATE TABLE ci_job_artifacts (
     file_location smallint,
     id bigint NOT NULL,
     job_id bigint NOT NULL,
+    locked smallint DEFAULT 2,
     CONSTRAINT check_27f0f6dbab CHECK ((file_store IS NOT NULL))
 );
 

doc/user/application_security/dependency_scanning/index.md

@@ -910,3 +910,19 @@ with a dependency on this version of Python should use `retire.js` version 2.10.
 ### Error: `dependency_scanning is used for configuration only, and its script should not be executed`
 
 For information on this, see the [GitLab Secure troubleshooting section](../index.md#error-job-is-used-for-configuration-only-and-its-script-should-not-be-executed).
+
+### Import multiple certificates for Java-based projects
+
+The `gemnasium-maven` analyzer reads the contents of the `ADDITIONAL_CA_CERT_BUNDLE` variable using `keytool`, which imports either a single certificate or a certificate chain. Multiple unrelated certificates are ignored and only the first one is imported by `keytool`.
+
+To add multiple unrelated certificates to the analyzer, you can declare a `before_script` such as this in the definition of the `gemnasium-maven-dependency_scanning` job:
+
+```yaml
+gemnasium-maven-dependency_scanning:
+  before_script:
+    - . $HOME/.bashrc # make the java tools available to the script
+    - OIFS="$IFS"; IFS=""; echo $ADDITIONAL_CA_CERT_BUNDLE > multi.pem; IFS="$OIFS" # write ADDITIONAL_CA_CERT_BUNDLE variable to a PEM file
+    - csplit -z --digits=2 --prefix=cert multi.pem "/-----END CERTIFICATE-----/+1" "{*}" # split the file into individual certificates
+    - for i in `ls cert*`; do keytool -v -importcert -alias "custom-cert-$i" -file $i -trustcacerts -noprompt -storepass changeit -keystore /opt/asdf/installs/java/adoptopenjdk-11.0.7+10.1/lib/security/cacerts 1>/dev/null 2>&1 || true; done # import each certificate using keytool (note the keystore location is related to the Java version being used and should be changed accordingly for other versions)
+    - unset ADDITIONAL_CA_CERT_BUNDLE # unset the variable so that the analyzer doesn't duplicate the import
+```