From 68d5cc2d9d162def465657d4696eb58e9b3906a7 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 1 Jun 2022 18:09:44 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../Service Ping reporting and monitoring.md | 122 ++++++++++++++ Gemfile | 2 +- Gemfile.lock | 4 +- .../components/error_details.vue | 2 +- .../components/error_tracking_actions.vue | 2 +- .../issuable/components/csv_export_modal.vue | 34 ++-- .../issuable/components/csv_import_modal.vue | 4 + .../components/promote_milestone_modal.vue | 2 +- .../components/popovers/file_tree_popover.vue | 7 +- .../upload_dropzone/upload_dropzone.vue | 4 +- .../upload_dropzone/upload_dropzone.scss | 33 +++- .../stylesheets/framework/typography.scss | 13 +- app/assets/stylesheets/utilities.scss | 35 ++++ app/graphql/gitlab_schema.rb | 28 ++-- app/graphql/mutations/base_mutation.rb | 18 ++- app/graphql/types/base_field.rb | 24 +++ app/graphql/types/ci/detailed_status_type.rb | 2 +- app/graphql/types/ci/runner_web_url_edge.rb | 11 +- app/graphql/types/ci/status_action_type.rb | 3 +- app/graphql/types/concerns/find_closest.rb | 15 -- app/graphql/types/global_id_type.rb | 3 +- .../interacts_with_merge_request.rb | 10 +- app/graphql/types/query_complexity_type.rb | 4 +- app/graphql/types/time_type.rb | 3 + app/helpers/issues_helper.rb | 2 +- db/docs/geo_lfs_object_deleted_events.yml | 8 - ...geo_lfs_object_deleted_event_references.rb | 29 ++++ ...ove_geo_lfs_object_deleted_events_table.rb | 17 ++ db/schema_migrations/20220526042017 | 1 + db/schema_migrations/20220526044516 | 1 + db/structure.sql | 29 ---- .../monitoring/prometheus/puma_exporter.md | 56 +------ .../monitoring/prometheus/web_exporter.md | 53 +++++++ lib/gitlab/application_rate_limiter.rb | 2 + lib/gitlab/database/gitlab_schemas.yml | 1 - lib/gitlab/graphql/generic_tracing.rb | 8 + lib/gitlab/graphql/present/field_extension.rb | 1 + .../query_analyzers/ast/logger_analyzer.rb | 88 ++++++++++ .../query_analyzers/ast/recursion_analyzer.rb | 78 +++++++++ .../query_analyzers/logger_analyzer.rb | 84 ---------- .../query_analyzers/recursion_analyzer.rb | 62 -------- qa/qa/page/project/pipeline_editor/show.rb | 1 - .../1_manage/group/transfer_project_spec.rb | 5 +- qa/qa/specs/spec_helper.rb | 3 + spec/controllers/graphql_controller_spec.rb | 11 +- spec/db/schema_spec.rb | 1 - .../components/csv_export_modal_spec.js | 32 ++-- .../components/csv_import_modal_spec.js | 4 + .../upload_dropzone_spec.js.snap | 30 ++-- spec/graphql/gitlab_schema_spec.rb | 11 +- .../boards/issues/issue_move_list_spec.rb | 2 +- .../graphql/mutations/branches/create_spec.rb | 4 +- .../clusters/agent_tokens/create_spec.rb | 4 +- .../mutations/clusters/agents/create_spec.rb | 4 +- .../mutations/clusters/agents/delete_spec.rb | 4 +- spec/graphql/mutations/commits/create_spec.rb | 4 +- .../timeline_events_resolver_spec.rb | 2 +- .../subscriptions/issuable_updated_spec.rb | 3 +- spec/graphql/types/base_edge_spec.rb | 1 - spec/graphql/types/base_object_spec.rb | 1 - .../types/ci/detailed_status_type_spec.rb | 4 +- .../types/ci/status_action_type_spec.rb | 8 +- spec/graphql/types/time_type_spec.rb | 5 +- .../types/work_items/widget_interface_spec.rb | 5 +- spec/helpers/issues_helper_spec.rb | 12 +- .../lib/gitlab/graphql/markdown_field_spec.rb | 7 - .../graphql/negatable_arguments_spec.rb | 4 +- .../graphql/present/field_extension_spec.rb | 17 -- .../ast/logger_analyzer_spec.rb | 51 ++++++ .../ast/recursion_analyzer_spec.rb | 72 +++++++++ .../query_analyzers/logger_analyzer_spec.rb | 49 ------ .../graphql/tracers/logger_tracer_spec.rb | 6 +- spec/requests/api/graphql/ci/runners_spec.rb | 44 +++++ .../api/graphql/gitlab_schema_spec.rb | 2 +- .../container_repository/destroy_tags_spec.rb | 2 +- .../design_management/delete_spec.rb | 2 +- .../dashboard/annotations/create_spec.rb | 2 - .../notes/reposition_image_diff_note_spec.rb | 10 +- spec/requests/api/graphql_spec.rb | 8 +- spec/support/graphql/field_inspection.rb | 2 +- spec/support/graphql/field_selection.rb | 8 +- spec/support/helpers/graphql_helpers.rb | 150 +++++++++++++----- 82 files changed, 957 insertions(+), 545 deletions(-) create mode 100644 .gitlab/issue_templates/Service Ping reporting and monitoring.md delete mode 100644 app/graphql/types/concerns/find_closest.rb delete mode 100644 db/docs/geo_lfs_object_deleted_events.yml create mode 100644 db/post_migrate/20220526042017_remove_geo_lfs_object_deleted_event_references.rb create mode 100644 db/post_migrate/20220526044516_remove_geo_lfs_object_deleted_events_table.rb create mode 100644 db/schema_migrations/20220526042017 create mode 100644 db/schema_migrations/20220526044516 create mode 100644 doc/administration/monitoring/prometheus/web_exporter.md create mode 100644 lib/gitlab/graphql/query_analyzers/ast/logger_analyzer.rb create mode 100644 lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer.rb delete mode 100644 lib/gitlab/graphql/query_analyzers/logger_analyzer.rb delete mode 100644 lib/gitlab/graphql/query_analyzers/recursion_analyzer.rb create mode 100644 spec/lib/gitlab/graphql/query_analyzers/ast/logger_analyzer_spec.rb create mode 100644 spec/lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer_spec.rb delete mode 100644 spec/lib/gitlab/graphql/query_analyzers/logger_analyzer_spec.rb diff --git a/.gitlab/issue_templates/Service Ping reporting and monitoring.md b/.gitlab/issue_templates/Service Ping reporting and monitoring.md new file mode 100644 index 00000000000..045f13c1c40 --- /dev/null +++ b/.gitlab/issue_templates/Service Ping reporting and monitoring.md @@ -0,0 +1,122 @@ + + +The [Product Intelligence group](https://about.gitlab.com/handbook/engineering/development/growth/product-intelligence/) runs manual reporting of ServicePing for GitLab.com on a weekly basis. This issue captures: + +- Captures the work required to complete the reporting process,. +- Captures the follow-up tasks that are focused on metrics performance verification. +- Identifies any potential issues. + +# New metrics to be verified + + + +# Failed metrics + +Broken metrics issues are marked with the ~"broken metric" label. + +# Use a detached screen session to generate Service Ping for GitLab.com + +## Prerequisites + +1. Make sure the SSH key is added to the local SSH agent: `ssh-add`. + +## Triggering + +1. Add the SSH key to the local SSH agent: `ssh-add`. +1. Connect to the bastion with SSH agent forwarding: `ssh -A lb-bastion.gprd.gitlab.com`. +1. Note which bastion host machine was assigned. For example: `@bastion-01-inf-gprd.c.gitlab-production.internal:~$` shows that you are connected to `bastion-01-inf-gprd.c.gitlab-production.internal`. +1. Create a named screen: `screen -S $USER-service-ping-$(date +%F)`. +1. Connect to the console host: `ssh $USER-rails@console-01-sv-gprd.c.gitlab-production.internal`. +1. Run: `ServicePing::SubmitService.new.execute`. +1. Press Control+a followed by Control+d to detach from the screen session. +1. Exit from the bastion: `exit`. + +## Verification (After approximately 30 hours) + +1. Reconnect to the bastion: `ssh -A lb-bastion.gprd.gitlab.com`. Make sure that you are connected to the same host machine that ServicePing was started on. For example, to connect directly to the host machine, use `ssh bastion-01-inf-gprd.c.gitlab-production.internal`. +1. Find your screen session: `screen -ls`. +1. Attach to your screen session: `screen -x 14226.mwawrzyniak_service_ping_2021_01_22`. +1. Check the last payload in the `raw_usage_data` table: `RawUsageData.last.payload`. +1. Check the when the payload was sent: `RawUsageData.last.sent_at`. + +## Stop the Service Ping process + +Use either of these processes: + +1. Reconnect to the bastion host machine. For example, use: `ssh bastion-01-inf-gprd.c.gitlab-production.internal`. +1. Find your screen session: `$ screen -ls`. +1. Attach to your screen session: `$ sudo -u screen -r`. +1. Press Control+c to stop the Service Ping process. + +OR + +1. Reconnect to the bastion host machine. For example, type: `ssh bastion-01-inf-gprd.c.gitlab-production.internal`. +1. List all process started by your username: `ps faux | grep `. +1. Locate the username that owns ServicePing reporting. +1. Send the kill signal for the ServicePing PID: `kill -9 `. + +## Service Ping process triggering (through a long-running SSH session) + +1. Connect to the `gprd` Rails console. +1. Run `SubmitUsagePingService.new.execute`. This process requires more than 30 hours to complete. +1. Find the last payload in the `raw_usage_data` table: `RawUsageData.last.payload`. +1. Check the when the payload was sent: `RawUsageData.last.sent_at`. + +```plaintext +ServicePing::SubmitService.new.execute + +# Get the payload +RawUsageData.last.payload + +# Time when payload was sent to VersionsAppp +RawUsageData.last.sent_at +``` + +# Verify Service Ping in VersionsApp + +To verify that the ServicePing was received in the VersionsApp do the following steps: + +1. Go to the VersionsApp console and locate: `RawUsageData.find(uuid: '')`. +1. Check the object. Either: + - Go to the Rails console and check the related `RawUsageData` object. + - Go to the VersionsApp UI . + +```ruby +/bin/herokuish procfile exec rails console + +puts UsageData.select(:recorded_at, :app_server_type).where(hostname: 'gitlab.com', uuid: 'ea8bf810-1d6f-4a6a-b4fd-93e8cbd8b57f').order('id desc').limit(5).to_json + +puts UsageData.find(21635202).raw_usage_data.payload.to_json +``` + +# Monitoring events tracked using Redis HLL + +Trigger some events from the User Interface. + +```ruby +Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'event_name', start_date: 28.days.ago, end_date: Date.current) +``` + +# What to do if you get mentioned + +In this issue, we keep the track of new metrics added to the Service Ping, and the metrics that are timing out. + +If you get mentioned, check the failing metric and open an optimization issue. + +# Service Ping manual generation for GitLab.com schedule + +| Generation start date | GitLab developer handle | Link to comment with payload | +| --------------------- | ----------------------- | ---------------------------- | +| 2022-04-18 | | | +| 2022-04-25 | | | +| 2022-05-02 | | | +| 2022-05-09 | | | +| 2022-05-16 | | | + + + +/confidential +/label ~"group::product intelligence" ~"devops::growth" ~backend ~"section::growth" ~"Category:Service Ping" +/epic https://gitlab.com/groups/gitlab-org/-/epics/6000 +/weight 5 +/title Monitor and Generate GitLab.com Service Ping diff --git a/Gemfile b/Gemfile index cc0277a547d..50ef5283522 100644 --- a/Gemfile +++ b/Gemfile @@ -100,7 +100,7 @@ gem 'grape-entity', '~> 0.10.0' gem 'rack-cors', '~> 1.0.6', require: 'rack/cors' # GraphQL API -gem 'graphql', '~> 1.11.10' +gem 'graphql', '~> 1.13.12' gem 'graphiql-rails', '~> 1.8' gem 'apollo_upload_server', '~> 2.1.0' gem 'graphql-docs', '~> 1.6.0', group: [:development, :test] diff --git a/Gemfile.lock b/Gemfile.lock index d95cf0d9d0b..41794b552ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -581,7 +581,7 @@ GEM faraday (>= 1.0) faraday_middleware graphql-client - graphql (1.11.10) + graphql (1.13.12) graphql-client (0.17.0) activesupport (>= 3.0) graphql (~> 1.10) @@ -1526,7 +1526,7 @@ DEPENDENCIES grape_logging (~> 1.8) graphiql-rails (~> 1.8) graphlient (~> 0.5.0) - graphql (~> 1.11.10) + graphql (~> 1.13.12) graphql-docs (~> 1.6.0) grpc (~> 1.42.0) gssapi diff --git a/app/assets/javascripts/error_tracking/components/error_details.vue b/app/assets/javascripts/error_tracking/components/error_details.vue index c369d879739..3fdf70e1621 100644 --- a/app/assets/javascripts/error_tracking/components/error_details.vue +++ b/app/assets/javascripts/error_tracking/components/error_details.vue @@ -301,7 +301,7 @@ export default { {{ __('More details') }} diff --git a/app/assets/javascripts/issuable/components/csv_export_modal.vue b/app/assets/javascripts/issuable/components/csv_export_modal.vue index b0af3612e05..736da92fa9f 100644 --- a/app/assets/javascripts/issuable/components/csv_export_modal.vue +++ b/app/assets/javascripts/issuable/components/csv_export_modal.vue @@ -1,16 +1,18 @@