diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index b22b2d3e264..ad6a7c1c558 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -235,6 +235,17 @@ module Ci hashed_path: 2 } + # `locked` will be populated from the source of truth on Ci::Pipeline + # in order to clean up expired job artifacts in a performant way. + # The values should be the same as `Ci::Pipeline.lockeds` with the + # additional value of `unknown` to indicate rows that have not + # yet been populated from the parent Ci::Pipeline + enum locked: { + unlocked: 0, + artifacts_locked: 1, + unknown: 2 + }, _prefix: :artifact + def validate_file_format! unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym errors.add(:base, _('Invalid file format with specified file type')) diff --git a/db/migrate/20210826120834_add_locked_to_ci_job_artifacts.rb b/db/migrate/20210826120834_add_locked_to_ci_job_artifacts.rb new file mode 100644 index 00000000000..2149265b4e7 --- /dev/null +++ b/db/migrate/20210826120834_add_locked_to_ci_job_artifacts.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AddLockedToCiJobArtifacts < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'ci_job_artifacts' + COLUMN_NAME = 'locked' + + def up + with_lock_retries do + add_column TABLE_NAME, COLUMN_NAME, :smallint, default: 2 + end + end + + def down + with_lock_retries do + remove_column TABLE_NAME, COLUMN_NAME + end + end +end diff --git a/db/post_migrate/20210826110839_prepare_indexes_for_ci_job_artifacts_expire_at_unlocked.rb b/db/post_migrate/20210826110839_prepare_indexes_for_ci_job_artifacts_expire_at_unlocked.rb new file mode 100644 index 00000000000..e11bb25d83c --- /dev/null +++ b/db/post_migrate/20210826110839_prepare_indexes_for_ci_job_artifacts_expire_at_unlocked.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PrepareIndexesForCiJobArtifactsExpireAtUnlocked < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + TABLE_NAME = 'ci_job_artifacts' + INDEX_NAME = 'ci_job_artifacts_expire_at_unlocked_idx' + + def up + prepare_async_index TABLE_NAME, [:expire_at], where: 'locked = 0', name: INDEX_NAME + end + + def down + unprepare_async_index_by_name TABLE_NAME, INDEX_NAME + end +end diff --git a/db/schema_migrations/20210826110839 b/db/schema_migrations/20210826110839 new file mode 100644 index 00000000000..165141ef852 --- /dev/null +++ b/db/schema_migrations/20210826110839 @@ -0,0 +1 @@ +72b64ddbaf86eb296fe49fd38bea759d5247414142fe1cd11aee7e1d6171e142 \ No newline at end of file diff --git a/db/schema_migrations/20210826120834 b/db/schema_migrations/20210826120834 new file mode 100644 index 00000000000..ebbdb86049c --- /dev/null +++ b/db/schema_migrations/20210826120834 @@ -0,0 +1 @@ +a8cd5165815a2f1e6b825ea3ee2a9bde88c1790f6ebe92296bee6a9a892b83f2 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 60630193d43..47916f2ac78 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11567,6 +11567,7 @@ CREATE TABLE ci_job_artifacts ( file_location smallint, id bigint NOT NULL, job_id bigint NOT NULL, + locked smallint DEFAULT 2, CONSTRAINT check_27f0f6dbab CHECK ((file_store IS NOT NULL)) ); diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 29d3d68a8f0..e6372d56cdf 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -910,3 +910,19 @@ with a dependency on this version of Python should use `retire.js` version 2.10. ### Error: `dependency_scanning is used for configuration only, and its script should not be executed` For information on this, see the [GitLab Secure troubleshooting section](../index.md#error-job-is-used-for-configuration-only-and-its-script-should-not-be-executed). + +### Import multiple certificates for Java-based projects + +The `gemnasium-maven` analyzer reads the contents of the `ADDITIONAL_CA_CERT_BUNDLE` variable using `keytool`, which imports either a single certificate or a certificate chain. Multiple unrelated certificates are ignored and only the first one is imported by `keytool`. + +To add multiple unrelated certificates to the analyzer, you can declare a `before_script` such as this in the definition of the `gemnasium-maven-dependency_scanning` job: + +```yaml +gemnasium-maven-dependency_scanning: + before_script: + - . $HOME/.bashrc # make the java tools available to the script + - OIFS="$IFS"; IFS=""; echo $ADDITIONAL_CA_CERT_BUNDLE > multi.pem; IFS="$OIFS" # write ADDITIONAL_CA_CERT_BUNDLE variable to a PEM file + - csplit -z --digits=2 --prefix=cert multi.pem "/-----END CERTIFICATE-----/+1" "{*}" # split the file into individual certificates + - for i in `ls cert*`; do keytool -v -importcert -alias "custom-cert-$i" -file $i -trustcacerts -noprompt -storepass changeit -keystore /opt/asdf/installs/java/adoptopenjdk-11.0.7+10.1/lib/security/cacerts 1>/dev/null 2>&1 || true; done # import each certificate using keytool (note the keystore location is related to the Java version being used and should be changed accordingly for other versions) + - unset ADDITIONAL_CA_CERT_BUNDLE # unset the variable so that the analyzer doesn't duplicate the import +```