Add latest changes from gitlab-org/gitlab@master

db/docs/sbom_source_packages.yml

@@ -0,0 +1,10 @@
+---
+table_name: sbom_source_packages
+classes:
+- Sbom::SourcePackage
+feature_categories:
+- dependency_management
+description: Tracks Source Package of an SBOM Occurrence
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/140539
+milestone: '16.8'
+gitlab_schema: gitlab_main

db/migrate/20231221033539_create_sbom_source_packages_table.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# See https://docs.gitlab.com/ee/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateSbomSourcePackagesTable < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone '16.8'
+
+  SBOM_SOURCE_PACKAGES_INDEX_NAME = 'idx_sbom_source_packages_on_name_and_purl_type'
+  SBOM_OCCURRENCES_SOURCE_PACKAGE_ID_AND_ID_INDEX_NAME = 'index_sbom_source_packages_on_source_package_id_and_id'
+
+  def up
+    with_lock_retries do
+      add_column :sbom_occurrences, :source_package_id, :bigint, if_not_exists: true
+    end
+
+    create_table :sbom_source_packages, if_not_exists: true do |t|
+      t.text :name, null: false, limit: 255
+      t.integer :purl_type, limit: 2, null: false
+      t.index [:name, :purl_type], unique: true, name: SBOM_SOURCE_PACKAGES_INDEX_NAME
+    end
+
+    add_concurrent_index :sbom_occurrences, [:source_package_id, :id],
+      name: SBOM_OCCURRENCES_SOURCE_PACKAGE_ID_AND_ID_INDEX_NAME
+
+    add_concurrent_foreign_key :sbom_occurrences, :sbom_source_packages,
+      column: :source_package_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key_if_exists(
+        :sbom_occurrences,
+        column: :source_package_id,
+        on_delete: :cascade
+      )
+      remove_column :sbom_occurrences, :source_package_id, if_exists: true
+      drop_table :sbom_source_packages, if_exists: true
+    end
+  end
+end

db/schema_migrations/20231221033539

@@ -0,0 +1 @@
+4493149bdc2db628180768717ebea209665e2311c6ce08269a6985ac643017c2

db/structure.sql

@@ -23495,6 +23495,7 @@ CREATE TABLE sbom_occurrences (
     vulnerabilities jsonb DEFAULT '[]'::jsonb,
     highest_severity smallint,
     vulnerability_count integer DEFAULT 0 NOT NULL,
+    source_package_id bigint,
     CONSTRAINT check_3f2d2c7ffc CHECK ((char_length(package_manager) <= 255)),
     CONSTRAINT check_9b29021fa8 CHECK ((char_length(component_name) <= 255)),
     CONSTRAINT check_bd1367d4c1 CHECK ((char_length(input_file_path) <= 255))
@@ -23526,6 +23527,22 @@ CREATE SEQUENCE sbom_occurrences_vulnerabilities_id_seq
 
 ALTER SEQUENCE sbom_occurrences_vulnerabilities_id_seq OWNED BY sbom_occurrences_vulnerabilities.id;
 
+CREATE TABLE sbom_source_packages (
+    id bigint NOT NULL,
+    name text NOT NULL,
+    purl_type smallint NOT NULL,
+    CONSTRAINT check_8fba79abed CHECK ((char_length(name) <= 255))
+);
+
+CREATE SEQUENCE sbom_source_packages_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE sbom_source_packages_id_seq OWNED BY sbom_source_packages.id;
+
 CREATE TABLE sbom_sources (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -27687,6 +27704,8 @@ ALTER TABLE ONLY sbom_occurrences ALTER COLUMN id SET DEFAULT nextval('sbom_occu
 
 ALTER TABLE ONLY sbom_occurrences_vulnerabilities ALTER COLUMN id SET DEFAULT nextval('sbom_occurrences_vulnerabilities_id_seq'::regclass);
 
+ALTER TABLE ONLY sbom_source_packages ALTER COLUMN id SET DEFAULT nextval('sbom_source_packages_id_seq'::regclass);
+
 ALTER TABLE ONLY sbom_sources ALTER COLUMN id SET DEFAULT nextval('sbom_sources_id_seq'::regclass);
 
 ALTER TABLE ONLY scan_result_policies ALTER COLUMN id SET DEFAULT nextval('scan_result_policies_id_seq'::regclass);
@@ -30335,6 +30354,9 @@ ALTER TABLE ONLY sbom_occurrences
 ALTER TABLE ONLY sbom_occurrences_vulnerabilities
     ADD CONSTRAINT sbom_occurrences_vulnerabilities_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY sbom_source_packages
+    ADD CONSTRAINT sbom_source_packages_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY sbom_sources
     ADD CONSTRAINT sbom_sources_pkey PRIMARY KEY (id);
 
@@ -32306,6 +32328,8 @@ CREATE INDEX idx_repository_states_outdated_checksums ON project_repository_stat
 
 CREATE INDEX idx_sbom_occurrences_on_project_id_and_source_id ON sbom_occurrences USING btree (project_id, source_id);
 
+CREATE UNIQUE INDEX idx_sbom_source_packages_on_name_and_purl_type ON sbom_source_packages USING btree (name, purl_type);
+
 CREATE UNIQUE INDEX idx_security_scans_on_build_and_scan_type ON security_scans USING btree (build_id, scan_type);
 
 CREATE INDEX idx_security_scans_on_scan_type ON security_scans USING btree (scan_type);
@@ -35188,6 +35212,8 @@ CREATE UNIQUE INDEX index_sbom_occurrences_on_uuid ON sbom_occurrences USING btr
 
 CREATE INDEX index_sbom_occurrences_vulnerabilities_on_vulnerability_id ON sbom_occurrences_vulnerabilities USING btree (vulnerability_id);
 
+CREATE INDEX index_sbom_source_packages_on_source_package_id_and_id ON sbom_occurrences USING btree (source_package_id, id);
+
 CREATE UNIQUE INDEX index_sbom_sources_on_source_type_and_source ON sbom_sources USING btree (source_type, source);
 
 CREATE UNIQUE INDEX index_scan_result_policies_on_position_in_configuration ON scan_result_policies USING btree (security_orchestration_policy_configuration_id, project_id, orchestration_policy_idx, rule_idx);
@@ -38734,6 +38760,9 @@ ALTER TABLE ONLY fork_network_members
 ALTER TABLE ONLY work_item_colors
     ADD CONSTRAINT fk_b15b0912d0 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE SET NULL;
 
+ALTER TABLE ONLY sbom_occurrences
+    ADD CONSTRAINT fk_b1b65d8d17 FOREIGN KEY (source_package_id) REFERENCES sbom_source_packages(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY vulnerabilities
     ADD CONSTRAINT fk_b1de915a15 FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL;
 

doc/administration/job_artifacts_troubleshooting.md

@@ -272,8 +272,11 @@ To change the number of job artifacts listed, change the number in `limit(50)`.
 WARNING:
 These commands remove data permanently from database and storage. Before running them, we highly recommend seeking guidance from a Support Engineer, or running them in a test environment with a backup of the instance ready to be restored, just in case.
 
-If you need to manually remove job artifacts associated with multiple jobs while
-**retaining their job logs**, this can be done from the [Rails console](operations/rails_console.md):
+You can manually remove job artifacts associated with multiple completed jobs while
+**retaining their job logs** from the [Rails console](operations/rails_console.md).
+A completed job is any job with the status of success, failed, canceled, or skipped.
+
+To delete jobs completed before a specific date:
 
 1. Select jobs to be deleted:
 
@@ -326,8 +329,11 @@ If you need to manually remove job artifacts associated with multiple jobs while
 WARNING:
 These commands remove data permanently from both the database and from disk. Before running them, we highly recommend seeking guidance from a Support Engineer, or running them in a test environment with a backup of the instance ready to be restored, just in case.
 
-If you need to manually remove **all** job artifacts associated with multiple jobs,
-**including job logs**, this can be done from the [Rails console](operations/rails_console.md):
+You can manually remove job artifacts associated with multiple completed jobs while
+**retaining their job logs** from the [Rails console](operations/rails_console.md).
+A completed job is any job with the status of success, failed, canceled, or skipped.
+
+To delete jobs completed before a specific date:
 
 1. Select the jobs to be deleted:
 

lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml

@@ -1,5 +1,5 @@
 variables:
-  DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.0'
+  DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1'
 
 .dast-auto-deploy:
   image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}"

lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml

@@ -1,5 +1,5 @@
 variables:
-  AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.0'
+  AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1'
 
 .auto-deploy:
   image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"

lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml

@@ -1,5 +1,5 @@
 variables:
-  AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.0'
+  AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1'
 
 .auto-deploy:
   image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"

qa/qa/runtime/browser.rb

@@ -69,17 +69,18 @@ module QA
             chrome_options = { args: %w[no-sandbox] }
 
             # Run headless by default unless WEBDRIVER_HEADLESS is false
-            if QA::Runtime::Env.webdriver_headless?
-              chrome_options[:args] << 'headless'
-
-              # Chrome documentation says this flag is needed for now
-              # https://developers.google.com/web/updates/2017/04/headless-chrome#cli
-              chrome_options[:args] << 'disable-gpu'
-            end
+            chrome_options[:args] << 'headless=new' if QA::Runtime::Env.webdriver_headless?
 
             # Disable /dev/shm use in CI. See https://gitlab.com/gitlab-org/gitlab/issues/4252
             chrome_options[:args] << 'disable-dev-shm-usage' if QA::Runtime::Env.disable_dev_shm?
 
+            # Allows chrome to consider all actions as secure when no ssl is used
+            Runtime::Scenario.attributes[:gitlab_address].tap do |address|
+              next unless address.start_with?('http://')
+
+              chrome_options[:args] << "unsafely-treat-insecure-origin-as-secure=#{address}"
+            end
+
             # Set chrome default download path
             # TODO: Set for remote grid as well once Sauce Labs tests are deprecated and Options.chrome is added
             # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112258

spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb

@@ -12,7 +12,8 @@ RSpec.describe 'new tables with gitlab_main schema', feature_category: :cell do
   # Specific tables can be exempted from this requirement, and such tables must be added to the `exempted_tables` list.
   let!(:exempted_tables) do
     [
-      "audit_events_instance_amazon_s3_configurations" # https://gitlab.com/gitlab-org/gitlab/-/issues/431327
+      "audit_events_instance_amazon_s3_configurations", # https://gitlab.com/gitlab-org/gitlab/-/issues/431327
+      "sbom_source_packages"                            # https://gitlab.com/gitlab-org/gitlab/-/issues/437718
     ]
   end