From ef6e077a330fe237cf0fb41487f1eda631cbc82c Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 8 Jul 2024 18:29:54 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/ci/cng/main.gitlab-ci.yml | 7 +- .gitlab/ci/release-environments.gitlab-ci.yml | 28 ---- .../release-environments/main.gitlab-ci.yml | 5 +- .../security.gitlab-ci.yml | 143 ----------------- .gitlab/ci/rules.gitlab-ci.yml | 21 --- .rubocop_todo/gitlab/bounded_contexts.yml | 1 + .../gitlab/feature_flag_without_actor.yml | 1 - .../javascripts/ci/job_details/index.js | 2 + .../javascripts/diffs/components/app.vue | 14 +- .../diffs/components/hidden_files_warning.vue | 7 +- .../concerns/record_user_last_activity.rb | 13 ++ app/events/users/activity_event.rb | 16 ++ app/helpers/ci/jobs_helper.rb | 3 +- .../update_repository_storage_methods.rb | 4 +- app/services/members/activity_service.rb | 47 ++++++ app/workers/all_queues.yml | 9 ++ .../users/record_last_activity_worker.rb | 22 +++ ...tings_security_and_compliance_pageload.yml | 16 ++ ...ulnerability_report_advanced_filtering.yml | 9 -- .../track_member_activity.yml | 9 ++ ...curity_and_compliance_pageload_monthly.yml | 22 +++ ...curity_and_compliance_pageload_monthly.yml | 21 +++ ...ecurity_and_compliance_pageload_weekly.yml | 22 +++ ...ecurity_and_compliance_pageload_weekly.yml | 21 +++ config/sidekiq_queues.yml | 2 + ..._ci_builds_on_partition_id_and_build_id.rb | 2 +- ...vent_hashes_organization_id_foreign_key.rb | 16 ++ ...fault_value_from_vsa_stage_hashes_table.rb | 13 ++ db/schema_migrations/20240704070311 | 1 + db/schema_migrations/20240704111127 | 1 + db/structure.sql | 5 +- doc/.vale/gitlab/InternalLinkFormat.yml | 6 +- doc/administration/geo/index.md | 2 +- doc/api/api_resources.md | 1 + doc/api/graphql/reference/index.md | 35 ++++- ...ect_container_registry_protection_rules.md | 44 ++++++ doc/architecture/blueprints/_template.md | 8 +- .../blueprints/bundle_uri/index.md | 4 +- .../cells/impacted_features/snippets.md | 2 +- doc/architecture/blueprints/epss/index.md | 2 +- .../blueprints/rapid_diffs/index.md | 2 +- .../blueprints/secret_manager/index.md | 21 ++- .../blueprints/work_items/index.md | 10 +- doc/ci/yaml/index.md | 47 +++++- doc/development/go_guide/go_upgrade.md | 2 +- doc/security/token_overview.md | 41 ++++- .../api_security/api_discovery/index.md | 2 +- .../group/saml_sso/troubleshooting_scim.md | 4 +- doc/user/markdown.md | 4 +- ...ect_container_registry_protection_rules.rb | 68 ++++++-- lib/gitlab/event_store.rb | 6 + locale/gitlab.pot | 6 + qa/gdk/Dockerfile.gdk.dockerignore | 1 + ...construct-release-environments-versions.rb | 34 +--- scripts/trigger-build.rb | 2 +- .../record_user_last_activity_spec.rb | 119 ++++++++++++++ spec/frontend/diffs/components/app_spec.js | 3 + spec/helpers/ci/jobs_helper_spec.rb | 3 +- ...ontainer_registry_protection_rules_spec.rb | 148 +++++++++++++++--- .../release_environments_model_spec.rb | 39 ++--- .../services/members/activity_service_spec.rb | 128 +++++++++++++++ .../current_organization_context.rb | 3 +- .../default_organization_context.rb | 9 ++ .../users/record_last_activity_worker_spec.rb | 53 +++++++ 64 files changed, 997 insertions(+), 365 deletions(-) delete mode 100644 .gitlab/ci/release-environments/security.gitlab-ci.yml create mode 100644 app/events/users/activity_event.rb create mode 100644 app/services/members/activity_service.rb create mode 100644 app/workers/users/record_last_activity_worker.rb create mode 100644 config/events/view_admin_application_settings_security_and_compliance_pageload.yml delete mode 100644 config/feature_flags/beta/vulnerability_report_advanced_filtering.yml create mode 100644 config/feature_flags/gitlab_com_derisk/track_member_activity.yml create mode 100644 config/metrics/counts_28d/count_distinct_user_id_from_view_admin_application_settings_security_and_compliance_pageload_monthly.yml create mode 100644 config/metrics/counts_28d/count_total_view_admin_application_settings_security_and_compliance_pageload_monthly.yml create mode 100644 config/metrics/counts_7d/count_distinct_user_id_from_view_admin_application_settings_security_and_compliance_pageload_weekly.yml create mode 100644 config/metrics/counts_7d/count_total_view_admin_application_settings_security_and_compliance_pageload_weekly.yml create mode 100644 db/post_migrate/20240704070311_add_stage_event_hashes_organization_id_foreign_key.rb create mode 100644 db/post_migrate/20240704111127_remove_default_value_from_vsa_stage_hashes_table.rb create mode 100644 db/schema_migrations/20240704070311 create mode 100644 db/schema_migrations/20240704111127 create mode 100644 spec/controllers/concerns/record_user_last_activity_spec.rb create mode 100644 spec/services/members/activity_service_spec.rb create mode 100644 spec/support/shared_contexts/default_organization_context.rb create mode 100644 spec/workers/users/record_last_activity_worker_spec.rb diff --git a/.gitlab/ci/cng/main.gitlab-ci.yml b/.gitlab/ci/cng/main.gitlab-ci.yml index af33e3db637..214669fa4d9 100644 --- a/.gitlab/ci/cng/main.gitlab-ci.yml +++ b/.gitlab/ci/cng/main.gitlab-ci.yml @@ -1,8 +1,3 @@ -spec: - inputs: - cng_path: - type: string - default: 'build/CNG-mirror' --- default: interruptible: true @@ -66,6 +61,6 @@ include: TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}" TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}" trigger: - project: '${CI_PROJECT_NAMESPACE}/$[[ inputs.cng_path ]]' + project: ${CI_PROJECT_NAMESPACE}/build/CNG-mirror branch: $TRIGGER_BRANCH strategy: depend diff --git a/.gitlab/ci/release-environments.gitlab-ci.yml b/.gitlab/ci/release-environments.gitlab-ci.yml index bcd1a3b047c..a6a0e268451 100644 --- a/.gitlab/ci/release-environments.gitlab-ci.yml +++ b/.gitlab/ci/release-environments.gitlab-ci.yml @@ -26,31 +26,3 @@ start-release-environments-pipeline: - project: 'gitlab-org/gitlab' ref: 'master' file: '.gitlab/ci/release-environments/main.gitlab-ci.yml' - -start-release-environments-security-pipeline: - allow_failure: true - extends: - - .release-environments:rules:start-release-environments-security-pipeline - stage: release-environments - # We do not want to have ALL global variables passed as trigger variables, - # as they cannot be overridden. See this issue for more context: - # - # https://gitlab.com/gitlab-org/gitlab/-/issues/387183 - inherit: - variables: - - RUBY_VERSION_DEFAULT - - RUBY_VERSION_NEXT - - RUBY_VERSION - - # These variables are set in the pipeline schedules. - # They need to be explicitly passed on to the child pipeline. - # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword - variables: - # This is needed by `release-environments-build-cng-env` (`.gitlab/ci/release-environments/security.gitlab-ci.yml`). - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - trigger: - strategy: depend - include: - - project: 'gitlab-org/security/gitlab' - ref: 'master' - file: '.gitlab/ci/release-environments/security.gitlab-ci.yml' diff --git a/.gitlab/ci/release-environments/main.gitlab-ci.yml b/.gitlab/ci/release-environments/main.gitlab-ci.yml index a8a05965132..d1097e8326b 100644 --- a/.gitlab/ci/release-environments/main.gitlab-ci.yml +++ b/.gitlab/ci/release-environments/main.gitlab-ci.yml @@ -1,10 +1,8 @@ --- include: - local: .gitlab/ci/cng/main.gitlab-ci.yml - inputs: - cng_path: 'build/CNG-mirror' - project: 'gitlab-org/quality/pipeline-common' - ref: '8.18.4' + ref: '8.18.3' file: ci/base.gitlab-ci.yml stages: @@ -97,6 +95,7 @@ release-environments-qa: GITLAB_INITIAL_ROOT_PASSWORD: "${RELEASE_ENVIRONMENTS_ROOT_PASSWORD}" QA_PRAEFECT_REPOSITORY_STORAGE: "default" SIGNUP_DISABLED: "true" + resource_group: release-environment-${CI_COMMIT_REF_SLUG} release-environments-notification-failure: stage: finish diff --git a/.gitlab/ci/release-environments/security.gitlab-ci.yml b/.gitlab/ci/release-environments/security.gitlab-ci.yml deleted file mode 100644 index 0b68cab6e4b..00000000000 --- a/.gitlab/ci/release-environments/security.gitlab-ci.yml +++ /dev/null @@ -1,143 +0,0 @@ -# Similar to .gitlab/ci/release-environments/main.gitlab-ci.yml, for release-environment pipelines in the security mirror. -# Referenced in .gitlab/ci/release-environments.gitlab-ci.yml to differentiate from the canonical (main) version. -# This file includes .gitlab/ci/cng/security.gitlab-ci.yml, instead of .gitlab/ci/cng/main.gitlab-ci.yml. ---- -include: - - local: .gitlab/ci/cng/main.gitlab-ci.yml - inputs: - cng_path: 'charts/components/images' - - project: 'gitlab-org/quality/pipeline-common' - ref: '8.18.4' - file: ci/base.gitlab-ci.yml - -stages: - - prepare - - start - - deploy - - qa - - finish - -.inherit_variables: - inherit: - variables: - - GIT_DEPTH - - GIT_STRATEGY - -workflow: - auto_cancel: - on_new_commit: none - -variables: - GIT_DEPTH: 20 - GIT_STRATEGY: fetch - -release-environments-build-cng-env: - extends: .build-cng-env - -release-environments-build-cng: - extends: .build-cng - needs: ["release-environments-build-cng-env"] - variables: - IMAGE_TAG_EXT: "-${CI_COMMIT_SHORT_SHA}" - -release-environments-deploy-env: - stage: prepare - needs: ["release-environments-build-cng"] - variables: - DEPLOY_ENV: deploy.env - script: - - ./scripts/release_environment/construct-release-environments-versions.rb - artifacts: - reports: - dotenv: $DEPLOY_ENV - paths: - - $DEPLOY_ENV - expire_in: 7 days - when: always - -release-environments-update-resource-group: - stage: prepare - script: - # Make sure pipelines run in order - # See https://docs.gitlab.com/ee/ci/resource_groups/index.html#change-the-process-mode - - | - curl --request PUT --data "process_mode=oldest_first" --header "PRIVATE-TOKEN:${ENVIRONMENT_API_TOKEN}" \ - "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/resource_groups/release-environment-${CI_COMMIT_REF_SLUG}" - -release-environments-notification-start: - stage: start - extends: .inherit_variables - variables: - RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy" - script: - - ruby scripts/release_environment/notification.rb - needs: ["release-environments-deploy-env"] - -release-environments-deploy: - stage: deploy - inherit: - variables: false - variables: - VERSIONS: "${VERSIONS}" - ENVIRONMENT: "${ENVIRONMENT}" - trigger: - project: gitlab-com/gl-infra/release-environments - branch: main - strategy: depend - needs: ["release-environments-deploy-env"] - resource_group: release-environment-${CI_COMMIT_REF_SLUG} - -release-environments-qa: - stage: qa - extends: - - .qa-base - timeout: 30m - parallel: 5 - variables: - QA_SCENARIO: "Test::Instance::Smoke" - RELEASE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_SHA}" - GITLAB_QA_OPTS: --address "https://gitlab.${ENVIRONMENT}.release.gke.gitlab.net" - GITLAB_INITIAL_ROOT_PASSWORD: "${RELEASE_ENVIRONMENTS_ROOT_PASSWORD}" - QA_PRAEFECT_REPOSITORY_STORAGE: "default" - SIGNUP_DISABLED: "true" - before_script: - - !reference [.qa-base, before_script] - - echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" --password-stdin - -release-environments-notification-failure: - stage: finish - extends: .inherit_variables - variables: - RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy" - script: - - ruby scripts/release_environment/notification.rb - needs: - - job: release-environments-deploy - artifacts: false - - job: release-environments-deploy-env - when: on_failure - -release-environments-notification-success: - stage: finish - extends: .inherit_variables - variables: - RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy" - script: - - ruby scripts/release_environment/notification.rb - needs: - - job: release-environments-qa - artifacts: false - - job: release-environments-deploy-env - -release-environments-notification-qa-failure: - stage: finish - extends: .inherit_variables - variables: - RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "qa" - script: - - ruby scripts/release_environment/notification.rb - needs: - - job: release-environments-qa - artifacts: false - - job: release-environments-deploy-env - when: on_failure diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 3b632fefe55..20fbbcd9ace 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -176,9 +176,6 @@ .if-dot-com-gitlab-org-ee-tag: &if-dot-com-gitlab-org-ee-tag if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_COMMIT_TAG =~ /^v?[\d]+\.[\d]+\.[\d]+[\d\w-]*-ee$/' -.if-dot-com-gitlab-org-security-ee-tag: &if-dot-com-gitlab-org-security-ee-tag - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_TAG =~ /^v?[\d]+\.[\d]+\.[\d]+[\d\w-]*-ee$/' - .if-ruby-branch: &if-ruby-branch if: '$CI_COMMIT_BRANCH =~ /^ruby\d+(_\d)*$/ || (($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby\d+(_\d)*/)' @@ -954,7 +951,6 @@ ARCH: amd64,arm64 - !reference [".build-images:rules:build-qa-image-merge-requests", rules] - !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only-setup-test-env", rules] - - !reference [".releases:rules:canonical-dot-com-security-gitlab-stable-branch-only-setup-test-env", rules] .build-images:rules:build-qa-image-as-if-foss: rules: @@ -2535,13 +2531,6 @@ when: never - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/' -.releases:rules:canonical-dot-com-security-gitlab-stable-branch-only-setup-test-env: - rules: - - if: '$CI_COMMIT_MESSAGE =~ /\[merge-train skip\]/' - when: never - - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/' - changes: *setup-test-env-patterns - ################# # Reports rules # ################# @@ -3293,16 +3282,6 @@ when: always - !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only", rules] -.release-environments:rules:start-release-environments-security-pipeline: - rules: - - <<: *if-not-ee - when: never - - <<: *if-merge-request-labels-pipeline-expedite - when: never - - <<: *if-dot-com-gitlab-org-security-ee-tag - when: always - - !reference [".releases:rules:canonical-dot-com-security-gitlab-stable-branch-only", rules] - ################### # Benchmark rules # ################### diff --git a/.rubocop_todo/gitlab/bounded_contexts.yml b/.rubocop_todo/gitlab/bounded_contexts.yml index 31893ced36a..5c9ae67889a 100644 --- a/.rubocop_todo/gitlab/bounded_contexts.yml +++ b/.rubocop_todo/gitlab/bounded_contexts.yml @@ -2635,6 +2635,7 @@ Gitlab/BoundedContexts: - 'ee/app/graphql/types/pending_group_member_type.rb' - 'ee/app/graphql/types/permission_types/dast_site_profile.rb' - 'ee/app/graphql/types/permission_types/epic.rb' + - 'ee/app/graphql/types/permission_types/merge_trains/car.rb' - 'ee/app/graphql/types/permission_types/pipeline_security_report_finding.rb' - 'ee/app/graphql/types/permission_types/requirement.rb' - 'ee/app/graphql/types/permission_types/vulnerability.rb' diff --git a/.rubocop_todo/gitlab/feature_flag_without_actor.yml b/.rubocop_todo/gitlab/feature_flag_without_actor.yml index 203e78c2a9d..f0efce75312 100644 --- a/.rubocop_todo/gitlab/feature_flag_without_actor.yml +++ b/.rubocop_todo/gitlab/feature_flag_without_actor.yml @@ -159,7 +159,6 @@ Gitlab/FeatureFlagWithoutActor: - 'ee/lib/ee/gitlab/auth/saml/config.rb' - 'ee/lib/ee/gitlab/geo_git_access.rb' - 'ee/lib/ee/gitlab/git_access.rb' - - 'ee/lib/elastic/latest/merge_request_class_proxy.rb' - 'ee/lib/gitlab/elastic/bulk_indexer.rb' - 'ee/lib/gitlab/elastic/client.rb' - 'ee/lib/gitlab/elastic/document_reference.rb' diff --git a/app/assets/javascripts/ci/job_details/index.js b/app/assets/javascripts/ci/job_details/index.js index 7e14572b717..a8c0803f763 100644 --- a/app/assets/javascripts/ci/job_details/index.js +++ b/app/assets/javascripts/ci/job_details/index.js @@ -34,6 +34,7 @@ export const initJobDetails = () => { pipelineTestReportUrl, logViewerPath, duoFeaturesEnabled, + jobGid, } = el.dataset; const fullScreenAPIAvailable = document.fullscreenEnabled; @@ -58,6 +59,7 @@ export const initJobDetails = () => { aiRootCauseAnalysisAvailable: parseBoolean(aiRootCauseAnalysisAvailable), duoFeaturesEnabled: parseBoolean(duoFeaturesEnabled), pipelineTestReportUrl, + jobGid, }, render(h) { return h(JobApp, { diff --git a/app/assets/javascripts/diffs/components/app.vue b/app/assets/javascripts/diffs/components/app.vue index b856ab236da..6470a07dfc4 100644 --- a/app/assets/javascripts/diffs/components/app.vue +++ b/app/assets/javascripts/diffs/components/app.vue @@ -751,13 +751,6 @@ export default { @@ -784,6 +777,13 @@ export default {
+