From 1fc72cb8765dab466da8555b70eb744a53a74a80 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 14 Jun 2023 09:09:21 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/ci/test-on-gdk/main.gitlab-ci.yml | 45 +++- .rubocop_todo/rspec/context_wording.yml | 2 - .../rspec/missing_feature_category.yml | 1 - .../style/class_and_module_children.yml | 1 - .rubocop_todo/style/guard_clause.yml | 1 - .../runner/components/runner_create_form.vue | 35 ++- .../ci/runner/components/runner_details.vue | 4 +- .../components/runner_managers_detail.vue | 24 +- .../components/runner_managers_table.vue | 11 +- app/assets/javascripts/ci/runner/constants.js | 3 + .../show/runner_managers.query.graphql | 6 + .../details/components/package_files.vue | 31 ++- .../achievements/delete_user_achievement.rb | 33 +++ app/graphql/types/mutation_type.rb | 1 + app/policies/group_policy.rb | 4 + .../destroy_user_achievement_service.rb | 33 +++ .../database/mark_migration_service.rb | 58 +++++ .../last_used_service.rb | 9 +- ...daily_hll_events_to_weekly_aggregation.yml | 8 - ...ken_usage_information_every_10_minutes.yml | 8 + .../15-6-deprecate-post-api-v4-runner.yml | 2 +- ...ner-registration-token-reset-endpoints.yml | 2 +- ...230328111013_re_migrate_redis_slot_keys.rb | 6 +- ...ex_members_on_source_id_and_source_type.rb | 15 ++ ...ct_authorizations_on_project_id_user_id.rb | 15 ++ db/schema_migrations/20230608081257 | 1 + db/schema_migrations/20230608081924 | 1 + db/structure.sql | 4 - doc/api/error_tracking.md | 6 +- doc/api/graphql/reference/index.md | 23 ++ doc/architecture/blueprints/pods/index.md | 11 - .../pods/pods-feature-admin-area.md | 11 - .../pods/pods-feature-agent-for-kubernetes.md | 11 - .../blueprints/pods/pods-feature-backups.md | 11 - .../pods/pods-feature-ci-runners.md | 11 - .../pods/pods-feature-container-registry.md | 11 - .../pods/pods-feature-contributions-forks.md | 11 - .../blueprints/pods/pods-feature-dashboard.md | 11 - .../pods/pods-feature-data-migration.md | 11 - .../pods/pods-feature-database-sequences.md | 11 - .../pods/pods-feature-git-access.md | 11 - .../pods/pods-feature-gitlab-pages.md | 11 - .../pods/pods-feature-global-search.md | 11 - .../blueprints/pods/pods-feature-graphql.md | 11 - .../pods/pods-feature-organizations.md | 11 - .../pods/pods-feature-personal-namespaces.md | 11 - ...feature-router-endpoints-classification.md | 11 - .../pods/pods-feature-schema-changes.md | 11 - .../blueprints/pods/pods-feature-secrets.md | 11 - .../blueprints/pods/pods-feature-snippets.md | 11 - .../blueprints/pods/pods-feature-template.md | 11 - .../blueprints/pods/pods-feature-uploads.md | 11 - ...tateless-router-with-buffering-requests.md | 11 - ...l-stateless-router-with-routes-learning.md | 11 - .../blueprints/remote_development/index.md | 2 +- doc/ci/runners/saas/macos_saas_runner.md | 14 ++ .../documentation/styleguide/index.md | 22 +- .../advanced_search_migration_styleguide.md | 20 +- doc/development/workspace/index.md | 11 - doc/gitlab-basics/add-file.md | 2 +- doc/install/azure/index.md | 3 +- doc/raketasks/backup_restore.md | 6 +- .../hardening_application_recommendations.md | 39 +-- doc/security/password_length_limits.md | 6 +- doc/security/reset_user_password.md | 5 +- doc/security/ssh_keys_restrictions.md | 5 +- doc/security/two_factor_authentication.md | 9 +- doc/security/unlock_user.md | 3 +- doc/security/user_email_confirmation.md | 7 +- doc/security/user_file_uploads.md | 4 +- doc/security/webhooks.md | 12 +- doc/update/deprecations.md | 4 +- doc/user/analytics/analytics_dashboards.md | 174 +++++++++++++ doc/user/analytics/index.md | 2 + .../packages/infrastructure_registry/index.md | 11 - doc/user/product_analytics/index.md | 143 ++--------- doc/user/profile/achievements.md | 30 +++ doc/user/profile/personal_access_tokens.md | 7 +- doc/user/project/badges.md | 21 +- doc/user/project/description_templates.md | 20 +- lib/api/admin/migrations.rb | 62 +++++ lib/api/api.rb | 2 +- lib/api/entities/error_tracking.rb | 2 +- lib/api/error_tracking/collector.rb | 156 ------------ .../analytics/internal_events_generator.rb | 1 - .../instrumentation_class_spec.rb.template | 2 +- lib/gitlab/middleware/compressed_json.rb | 4 +- .../usage_data_counters/hll_redis_counter.rb | 80 ++---- lib/tasks/gitlab/packages/events.rake | 5 +- lib/tasks/gitlab/usage_data.rake | 9 +- locale/gitlab.pot | 3 + .../sample_metric_test.rb | 2 +- .../components/runner_create_form_spec.js | 33 +++ .../components/runner_managers_detail_spec.js | 27 +- .../components/runner_managers_table_spec.js | 32 ++- .../details/components/package_files_spec.js | 6 +- .../delete_user_achievement_spec.rb | 55 +++++ .../internal_events_generator_spec.rb | 2 +- .../gitlab/middleware/compressed_json_spec.rb | 24 +- .../hll_redis_counter_spec.rb | 158 ++---------- ...2811133_re_migrate_redis_slot_keys_spec.rb | 8 +- ...s_hll_events_to_weekly_aggregation_spec.rb | 19 +- spec/policies/group_policy_spec.rb | 8 + spec/requests/api/admin/migrations_spec.rb | 89 +++++++ .../api/error_tracking/collector_spec.rb | 233 ------------------ .../delete_user_achievement_spec.rb | 85 +++++++ .../destroy_user_achievement_service_spec.rb | 40 +++ .../database/mark_migration_service_spec.rb | 71 ++++++ .../last_used_service_spec.rb | 52 +++- spec/support/rspec_order_todo.yml | 1 - workhorse/go.mod | 2 +- workhorse/go.sum | 4 +- 112 files changed, 1275 insertions(+), 1241 deletions(-) create mode 100644 app/graphql/mutations/achievements/delete_user_achievement.rb create mode 100644 app/services/achievements/destroy_user_achievement_service.rb create mode 100644 app/services/database/mark_migration_service.rb delete mode 100644 config/feature_flags/development/revert_daily_hll_events_to_weekly_aggregation.yml create mode 100644 config/feature_flags/development/update_personal_access_token_usage_information_every_10_minutes.yml create mode 100644 db/post_migrate/20230608081257_delete_index_members_on_source_id_and_source_type.rb create mode 100644 db/post_migrate/20230608081924_delete_index_unique_project_authorizations_on_project_id_user_id.rb create mode 100644 db/schema_migrations/20230608081257 create mode 100644 db/schema_migrations/20230608081924 delete mode 100644 doc/architecture/blueprints/pods/index.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-admin-area.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-agent-for-kubernetes.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-backups.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-ci-runners.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-container-registry.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-contributions-forks.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-dashboard.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-data-migration.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-database-sequences.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-git-access.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-gitlab-pages.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-global-search.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-graphql.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-organizations.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-personal-namespaces.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-schema-changes.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-secrets.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-snippets.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-template.md delete mode 100644 doc/architecture/blueprints/pods/pods-feature-uploads.md delete mode 100644 doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md delete mode 100644 doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md delete mode 100644 doc/development/workspace/index.md create mode 100644 doc/user/analytics/analytics_dashboards.md delete mode 100644 doc/user/packages/infrastructure_registry/index.md create mode 100644 lib/api/admin/migrations.rb delete mode 100644 lib/api/error_tracking/collector.rb create mode 100644 spec/graphql/mutations/achievements/delete_user_achievement_spec.rb create mode 100644 spec/requests/api/admin/migrations_spec.rb delete mode 100644 spec/requests/api/error_tracking/collector_spec.rb create mode 100644 spec/requests/api/graphql/mutations/achievements/delete_user_achievement_spec.rb create mode 100644 spec/services/achievements/destroy_user_achievement_service_spec.rb create mode 100644 spec/services/database/mark_migration_service_spec.rb diff --git a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml index 3f177f87a30..c2447519268 100644 --- a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml +++ b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml @@ -51,9 +51,9 @@ variables: - echo "$(docker exec gdk bash -c "getent hosts \$HOSTNAME" | awk '{print $1}') gdk.test" >> /etc/hosts - source scripts/utils.sh - cd qa && bundle install + script: - retry_exponential test_url ${GDK_URL}/users/sign_in - echo -e "\e[0Ksection_end:`date +%s`:launch_gdk\r\e[0K" - script: - echo -e "\e[0Ksection_start:`date +%s`:run_tests\r\e[0KRun E2E tests" - QA_COMMAND="bundle exec bin/qa Test::Instance::All ${GDK_URL} -- ${RSPEC_TAGS} ${RSPEC_REPORT_OPTS}" - echo "Running - '${QA_COMMAND}'" @@ -87,6 +87,21 @@ cache-gems: script: - cd qa && bundle install +# Take the existing GDK docker image and reconfigure it with Postgres load +# balancing. Adding 5s lag to 1 of the replicas to validate robustness of +# the load balancer. +.gdk-with-load-balancer-setup: + before_script: + - !reference [".gdk-qa-base", "before_script"] + - | + docker exec gdk bash -c " + gdk config set postgresql.replica.enabled true &&\ + gdk config set postgresql.replica_2.enabled true &&\ + gdk config set load_balancing.enabled true &&\ + gdk reconfigure &&\ + echo 'recovery_min_apply_delay = 5s' >> postgresql-replica-2/data/postgresql.conf &&\ + gdk restart" + gdk-qa-smoke: extends: - .gdk-qa-base @@ -97,6 +112,19 @@ gdk-qa-smoke: rules: - when: always +gdk-qa-smoke-with-load-balancer: + extends: + - .gdk-qa-base + - .gdk-with-load-balancer-setup + variables: + TEST_GDK_TAGS: "--tag smoke" + QA_RUN_TYPE: gdk-qa-smoke + RSPEC_TAGS: --tag smoke + rules: + - changes: + - ".gitlab/ci/test-on-gdk/**" + - "lib/gitlab/database/load_balancing/**/*" + # TODO: set non manual once smoke tests prove to be stable gdk-qa-reliable: extends: @@ -110,6 +138,21 @@ gdk-qa-reliable: rules: - when: manual +gdk-qa-reliable-with-load-balancer: + extends: + - .gdk-qa-base + - .gdk-with-load-balancer-setup + parallel: 5 + variables: + QA_RUN_TYPE: gdk-qa-blocking + QA_KNAPSACK_REPORT_NAME: ee-instance-parallel + RSPEC_TAGS: --tag smoke --tag reliable + allow_failure: true + rules: + - changes: + - ".gitlab/ci/test-on-gdk/**" + - "lib/gitlab/database/load_balancing/**/*" + gdk-qa-non-blocking: extends: - .gdk-qa-base diff --git a/.rubocop_todo/rspec/context_wording.yml b/.rubocop_todo/rspec/context_wording.yml index a0df088b088..aaef30b1e37 100644 --- a/.rubocop_todo/rspec/context_wording.yml +++ b/.rubocop_todo/rspec/context_wording.yml @@ -1957,7 +1957,6 @@ RSpec/ContextWording: - 'spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb' - 'spec/lib/gitlab/metrics_spec.rb' - 'spec/lib/gitlab/middleware/basic_health_check_spec.rb' - - 'spec/lib/gitlab/middleware/compressed_json_spec.rb' - 'spec/lib/gitlab/middleware/go_spec.rb' - 'spec/lib/gitlab/middleware/multipart_spec.rb' - 'spec/lib/gitlab/middleware/request_context_spec.rb' @@ -2390,7 +2389,6 @@ RSpec/ContextWording: - 'spec/requests/api/deploy_tokens_spec.rb' - 'spec/requests/api/deployments_spec.rb' - 'spec/requests/api/environments_spec.rb' - - 'spec/requests/api/error_tracking/collector_spec.rb' - 'spec/requests/api/files_spec.rb' - 'spec/requests/api/generic_packages_spec.rb' - 'spec/requests/api/graphql/ci/runner_spec.rb' diff --git a/.rubocop_todo/rspec/missing_feature_category.yml b/.rubocop_todo/rspec/missing_feature_category.yml index 0c54d92764c..527d7913311 100644 --- a/.rubocop_todo/rspec/missing_feature_category.yml +++ b/.rubocop_todo/rspec/missing_feature_category.yml @@ -4043,7 +4043,6 @@ RSpec/MissingFeatureCategory: - 'spec/lib/gitlab/metrics/web_transaction_spec.rb' - 'spec/lib/gitlab/metrics_spec.rb' - 'spec/lib/gitlab/middleware/basic_health_check_spec.rb' - - 'spec/lib/gitlab/middleware/compressed_json_spec.rb' - 'spec/lib/gitlab/middleware/handle_ip_spoof_attack_error_spec.rb' - 'spec/lib/gitlab/middleware/handle_malformed_strings_spec.rb' - 'spec/lib/gitlab/middleware/memory_report_spec.rb' diff --git a/.rubocop_todo/style/class_and_module_children.yml b/.rubocop_todo/style/class_and_module_children.yml index 81b154f906b..55df73d3633 100644 --- a/.rubocop_todo/style/class_and_module_children.yml +++ b/.rubocop_todo/style/class_and_module_children.yml @@ -501,7 +501,6 @@ Style/ClassAndModuleChildren: - 'ee/lib/ee/gitlab/throttle.rb' - 'ee/lib/gitlab/path_locks_finder.rb' - 'lib/api/error_tracking/client_keys.rb' - - 'lib/api/error_tracking/collector.rb' - 'lib/api/error_tracking/project_settings.rb' - 'lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb' - 'lib/gitlab/ci/badge/base.rb' diff --git a/.rubocop_todo/style/guard_clause.yml b/.rubocop_todo/style/guard_clause.yml index 4fcefa6ef85..8613e240d9c 100644 --- a/.rubocop_todo/style/guard_clause.yml +++ b/.rubocop_todo/style/guard_clause.yml @@ -428,7 +428,6 @@ Style/GuardClause: - 'ee/spec/support/ci/minutes_helpers.rb' - 'haml_lint/linter/documentation_links.rb' - 'lib/api/commits.rb' - - 'lib/api/error_tracking/collector.rb' - 'lib/api/feature_flags.rb' - 'lib/api/helpers.rb' - 'lib/api/helpers/packages/conan/api_helpers.rb' diff --git a/app/assets/javascripts/ci/runner/components/runner_create_form.vue b/app/assets/javascripts/ci/runner/components/runner_create_form.vue index 040e42fa938..1b363174d28 100644 --- a/app/assets/javascripts/ci/runner/components/runner_create_form.vue +++ b/app/assets/javascripts/ci/runner/components/runner_create_form.vue @@ -4,7 +4,13 @@ import RunnerFormFields from '~/ci/runner/components/runner_form_fields.vue'; import runnerCreateMutation from '~/ci/runner/graphql/new/runner_create.mutation.graphql'; import { modelToUpdateMutationVariables } from 'ee_else_ce/ci/runner/runner_update_form_utils'; import { captureException } from '../sentry_utils'; -import { RUNNER_TYPES, DEFAULT_ACCESS_LEVEL, PROJECT_TYPE, GROUP_TYPE } from '../constants'; +import { + RUNNER_TYPES, + DEFAULT_ACCESS_LEVEL, + PROJECT_TYPE, + GROUP_TYPE, + I18N_CREATE_ERROR, +} from '../constants'; export default { name: 'RunnerCreateForm', @@ -82,17 +88,30 @@ export default { }); if (errors?.length) { - this.$emit('error', new Error(errors.join(' '))); - this.saving = false; - } else { - this.onSuccess(runner); + this.onError(new Error(errors.join(' ')), true); + return; } + + if (!runner?.ephemeralRegisterUrl) { + // runner is missing information, report issue and + // fail naviation to register page. + this.onError(new Error(I18N_CREATE_ERROR)); + return; + } + + this.onSuccess(runner); } catch (error) { - captureException({ error, component: this.$options.name }); - this.$emit('error', error); - this.saving = false; + this.onError(error); } }, + onError(error, isValidationError = false) { + if (!isValidationError) { + captureException({ error, component: this.$options.name }); + } + + this.$emit('error', error); + this.saving = false; + }, onSuccess(runner) { this.$emit('saved', runner); }, diff --git a/app/assets/javascripts/ci/runner/components/runner_details.vue b/app/assets/javascripts/ci/runner/components/runner_details.vue index 51c752f0dee..8c1280cffb9 100644 --- a/app/assets/javascripts/ci/runner/components/runner_details.vue +++ b/app/assets/javascripts/ci/runner/components/runner_details.vue @@ -10,6 +10,7 @@ import { GROUP_TYPE, PROJECT_TYPE, RUNNER_MANAGERS_HELP_URL, + I18N_STATUS_NEVER_CONTACTED, } from '../constants'; import RunnerDetail from './runner_detail.vue'; import RunnerGroups from './runner_groups.vue'; @@ -85,6 +86,7 @@ export default { }, ACCESS_LEVEL_REF_PROTECTED, RUNNER_MANAGERS_HELP_URL, + I18N_STATUS_NEVER_CONTACTED, }; @@ -99,7 +101,7 @@ export default { diff --git a/app/assets/javascripts/ci/runner/constants.js b/app/assets/javascripts/ci/runner/constants.js index 395d9ac0d8e..40841696ead 100644 --- a/app/assets/javascripts/ci/runner/constants.js +++ b/app/assets/javascripts/ci/runner/constants.js @@ -9,6 +9,9 @@ export const RUNNER_DETAILS_PROJECTS_PAGE_SIZE = 5; export const RUNNER_DETAILS_JOBS_PAGE_SIZE = 30; export const I18N_FETCH_ERROR = s__('Runners|Something went wrong while fetching runner data.'); +export const I18N_CREATE_ERROR = s__( + 'Runners|An error occurred while creating the runner. Please try again.', +); export const FILTER_CSS_CLASSES = 'gl-bg-gray-10 gl-p-5 gl-border-solid gl-border-gray-100 gl-border-0 gl-border-t-1 gl-border-b-1'; diff --git a/app/assets/javascripts/ci/runner/graphql/show/runner_managers.query.graphql b/app/assets/javascripts/ci/runner/graphql/show/runner_managers.query.graphql index 65fb5f91b60..c4fb63f2b0d 100644 --- a/app/assets/javascripts/ci/runner/graphql/show/runner_managers.query.graphql +++ b/app/assets/javascripts/ci/runner/graphql/show/runner_managers.query.graphql @@ -6,6 +6,12 @@ query getRunnerManagers($runnerId: CiRunnerID!) { nodes { id systemId + version + revision + executorName + architectureName + platformName + ipAddress contactedAt } } diff --git a/app/assets/javascripts/packages_and_registries/infrastructure_registry/details/components/package_files.vue b/app/assets/javascripts/packages_and_registries/infrastructure_registry/details/components/package_files.vue index e45b88bc6d5..ecd1bfb8ebe 100644 --- a/app/assets/javascripts/packages_and_registries/infrastructure_registry/details/components/package_files.vue +++ b/app/assets/javascripts/packages_and_registries/infrastructure_registry/details/components/package_files.vue @@ -1,5 +1,11 @@