diff --git a/Gemfile.checksum b/Gemfile.checksum index 0e1da495dfb..12e03df4d27 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -230,7 +230,7 @@ {"name":"gitlab-glfm-markdown","version":"0.0.31","platform":"x86_64-darwin","checksum":"3d9df3278add66356ce07c983b74f6f0bdefcdfb4110f43dd291c43fb69eb6a6"}, {"name":"gitlab-glfm-markdown","version":"0.0.31","platform":"x86_64-linux-gnu","checksum":"c2c4c8f77d078f2afb1420289e32f444e91ece48d8f76a78e43f5b69f5d2248c"}, {"name":"gitlab-glfm-markdown","version":"0.0.31","platform":"x86_64-linux-musl","checksum":"cc3e8529f8eaf6be9bfa71509a58396540bb7c2b025e5dd9615a73ad0cf2a6b3"}, -{"name":"gitlab-kas-grpc","version":"17.11.2","platform":"ruby","checksum":"f2b9054dcf636346e89549e9478a684a38bf03bf853332e84154ec3a9856ae1c"}, +{"name":"gitlab-kas-grpc","version":"17.11.3","platform":"ruby","checksum":"f8b4917c23cae65b4faeb9c2bad6b7aa388317766bf76ca2386dc49359305f53"}, {"name":"gitlab-labkit","version":"0.37.0","platform":"ruby","checksum":"d2dd0a60db2149a9a8eebf2975dc23f54ac3ceb01bdba732eb1b26b86dfffa70"}, {"name":"gitlab-license","version":"2.6.0","platform":"ruby","checksum":"2c1f8ae73835640ec77bf758c1d0c9730635043c01cf77902f7976e826d7d016"}, {"name":"gitlab-mail_room","version":"0.0.27","platform":"ruby","checksum":"05c07db892094cf5747ea00afb0a95c5a5406e05f34ae779f4388f2ddf962316"}, diff --git a/Gemfile.lock b/Gemfile.lock index 043e345b64f..9cbead4ec87 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -768,7 +768,7 @@ GEM nokogiri (~> 1, >= 1.10.8) gitlab-glfm-markdown (0.0.31) rb_sys (~> 0.9.109) - gitlab-kas-grpc (17.11.2) + gitlab-kas-grpc (17.11.3) grpc (~> 1.0) gitlab-labkit (0.37.0) actionpack (>= 5.0.0, < 8.1.0) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index 0e1da495dfb..12e03df4d27 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -230,7 +230,7 @@ {"name":"gitlab-glfm-markdown","version":"0.0.31","platform":"x86_64-darwin","checksum":"3d9df3278add66356ce07c983b74f6f0bdefcdfb4110f43dd291c43fb69eb6a6"}, {"name":"gitlab-glfm-markdown","version":"0.0.31","platform":"x86_64-linux-gnu","checksum":"c2c4c8f77d078f2afb1420289e32f444e91ece48d8f76a78e43f5b69f5d2248c"}, {"name":"gitlab-glfm-markdown","version":"0.0.31","platform":"x86_64-linux-musl","checksum":"cc3e8529f8eaf6be9bfa71509a58396540bb7c2b025e5dd9615a73ad0cf2a6b3"}, -{"name":"gitlab-kas-grpc","version":"17.11.2","platform":"ruby","checksum":"f2b9054dcf636346e89549e9478a684a38bf03bf853332e84154ec3a9856ae1c"}, +{"name":"gitlab-kas-grpc","version":"17.11.3","platform":"ruby","checksum":"f8b4917c23cae65b4faeb9c2bad6b7aa388317766bf76ca2386dc49359305f53"}, {"name":"gitlab-labkit","version":"0.37.0","platform":"ruby","checksum":"d2dd0a60db2149a9a8eebf2975dc23f54ac3ceb01bdba732eb1b26b86dfffa70"}, {"name":"gitlab-license","version":"2.6.0","platform":"ruby","checksum":"2c1f8ae73835640ec77bf758c1d0c9730635043c01cf77902f7976e826d7d016"}, {"name":"gitlab-mail_room","version":"0.0.27","platform":"ruby","checksum":"05c07db892094cf5747ea00afb0a95c5a5406e05f34ae779f4388f2ddf962316"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index 043e345b64f..9cbead4ec87 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -768,7 +768,7 @@ GEM nokogiri (~> 1, >= 1.10.8) gitlab-glfm-markdown (0.0.31) rb_sys (~> 0.9.109) - gitlab-kas-grpc (17.11.2) + gitlab-kas-grpc (17.11.3) grpc (~> 1.0) gitlab-labkit (0.37.0) actionpack (>= 5.0.0, < 8.1.0) diff --git a/doc/administration/logs/log_parsing.md b/doc/administration/logs/log_parsing.md index edf47a9b9d5..75e221722d9 100644 --- a/doc/administration/logs/log_parsing.md +++ b/doc/administration/logs/log_parsing.md @@ -182,22 +182,31 @@ CT: 190 ROUTE: /api/:version/projects/:id/repository/commits DURS: 1079.02, #### Print top API user agents ```shell -jq --raw-output 'select(.remote_ip != "127.0.0.1") | [.remote_ip, .username, .route, .ua] | @tsv' api_json.log | - sort | uniq -c | sort -n | tail +jq --raw-output ' + select(.remote_ip != "127.0.0.1") | [ + (.time | split(".")[0] | strptime("%Y-%m-%dT%H:%M:%S") | strftime("…%m-%dT%H…")), + ."meta.caller_id", .username, .ua + ] | @tsv' api_json.log | sort | uniq -c \ + | grep --invert-match --extended-regexp '^\s+\d{1,3}\b' ``` **Example output**: ```plaintext - 89 1.2.3.4, 127.0.0.1 some_user /api/:version/projects/:id/pipelines # plus browser details; OK - 567 5.6.7.8, 127.0.0.1 /api/:version/jobs/:id/trace gitlab-runner # plus version details; OK -1234 98.76.54.31, 127.0.0.1 some_bot /api/:version/projects/:id/repository/files/:file_path/raw + 1234 …01-12T01… GET /api/:version/projects/:id/pipelines some_user # plus browser details; OK +54321 …01-12T01… POST /api/:version/projects/:id/repository/files/:file_path/raw some_bot + 5678 …01-12T01… PATCH /api/:version/jobs/:id/trace gitlab-runner # plus version details; OK ``` -This example shows a custom tool or script causing an unexpectedly high number of requests. +This example shows a custom tool or script causing an unexpectedly high [request rate (>15 RPS)](../reference_architectures/_index.md#available-reference-architectures). User agents in this situation can be specialized [third-party clients](../../api/rest/third_party_clients.md), or general tools like `curl`. +The hourly aggregation helps to: + +- Correlate spikes of bot or user activity to data from monitoring tools such as [Prometheus](../monitoring/prometheus/_index.md). +- Evaluate [rate limit settings](../settings/user_and_ip_rate_limits.md). + You can also use `fast-stats top` (see top of page) to extract performance statistics for those users or bots. ### Parsing `gitlab-rails/importer.log` @@ -216,8 +225,12 @@ For common issues, see [troubleshooting](../raketasks/import_export_rake_tasks_t #### Print top Workhorse user agents ```shell -jq --raw-output 'select(.remote_ip != "127.0.0.1") | [.remote_ip, .uri, .user_agent] | @tsv' current | - sort | uniq -c | sort -n | tail +jq --raw-output ' + select(.remote_ip != "127.0.0.1") | [ + (.time | split(".")[0] | strptime("%Y-%m-%dT%H:%M:%S") | strftime("…%m-%dT%H…")), + .remote_ip, .uri, .user_agent + ] | @tsv' current | + sort | uniq -c ``` Similar to the [API `ua` example](#print-top-api-user-agents), @@ -237,7 +250,11 @@ repeatedly reports that some items never reach 100%, the following command helps to focus on the most common errors. ```shell -jq --raw-output 'select(.severity == "ERROR") | [.class, .id, .message, .error] | @tsv' geo.log | sort | uniq -c | sort -n | tail +jq --raw-output 'select(.severity == "ERROR") | [ + (.time | split(".")[0] | strptime("%Y-%m-%dT%H:%M:%S") | strftime("…%m-%dT%H:%M…")), + .class, .id, .message, .error + ] | @tsv' geo.log \ + | sort | uniq -c ``` Refer to our [Geo troubleshooting page](../geo/replication/troubleshooting/_index.md) @@ -305,9 +322,29 @@ jq --raw-output --slurp ' #### Types of user and project activity overview ```shell -jq --raw-output '[.username, ."grpc.method", ."grpc.request.glProjectPath"] | @tsv' current | sort | uniq -c | sort -n +jq --raw-output '[ + (.time | split(".")[0] | strptime("%Y-%m-%dT%H:%M:%S") | strftime("…%m-%dT%H…")), + .username, ."grpc.method", ."grpc.request.glProjectPath" + ] | @tsv' current | sort | uniq -c \ + | grep --invert-match --extended-regexp '^\s+\d{1,3}\b' ``` +**Example output**: + +```plaintext + 5678 …01-12T01… ReferenceTransactionHook # Praefect operation; OK +54321 …01-12T01… some_bot GetBlobs namespace/subgroup/project + 1234 …01-12T01… some_user FindCommit namespace/subgroup/project +``` + +This example shows a custom tool or script causing unexpectedly high of [request rate (>15 RPS)](../reference_architectures/_index.md#available-reference-architectures) on Gitaly. +The hourly aggregation helps to: + +- Correlate spikes of bot or user activity to data from monitoring tools such as [Prometheus](../monitoring/prometheus/_index.md). +- Evaluate [rate limit settings](../settings/user_and_ip_rate_limits.md). + +You can also use `fast-stats top` (see top of page) to extract performance statistics for those users or bots. + #### Find all projects affected by a fatal Git problem ```shell diff --git a/doc/ci/cloud_deployment/_index.md b/doc/ci/cloud_deployment/_index.md index f7647ff9554..97cc8cc3849 100644 --- a/doc/ci/cloud_deployment/_index.md +++ b/doc/ci/cloud_deployment/_index.md @@ -209,7 +209,7 @@ To deploy to EC2, complete the following steps. - If you do not want these JSON objects saved in your repository, add each object as a separate [file type CI/CD variable](../variables/_index.md#use-file-type-cicd-variables) - in the project settings. Use the same variable names as above. + in the project settings. Use the same previous variable names. 1. In your `.gitlab-ci.yml` file, create a CI/CD variable for the name of the stack. For example: diff --git a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md index 6901e110994..745143e2103 100644 --- a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md +++ b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md @@ -153,7 +153,7 @@ is a daemon to create an application container based on the [ECS task definition 1. Go to **ECS > Clusters > ecs-demo > Services** on the [AWS console](https://aws.amazon.com/) 1. Select **Deploy**. This opens a service creation form. 1. Select `EC2` in **Launch Type**. -1. Set `ecs_demo` to **Task definition**. This corresponds to [the task definition you created above](#create-an-ecs-task-definition). +1. Set `ecs_demo` to **Task definition**. This corresponds to [the task definition you created previously](#create-an-ecs-task-definition). 1. Set `ecs_demo` to **Service name**. 1. Set `1` to **Desired tasks**. @@ -188,7 +188,7 @@ Now that you have an application running on ECS, you can set up continuous deplo ### Create a new IAM user as a deployer -For GitLab to access the ECS cluster, service, and task definition that you created above, You must +For GitLab to access the ECS cluster, service, and task definition that you previously created, you must create a deployer user on AWS: 1. Go to **IAM > Users** on [AWS console](https://aws.amazon.com/). diff --git a/doc/install/relative_url.md b/doc/install/relative_url.md index 51385968eb0..b38c8fd9d44 100644 --- a/doc/install/relative_url.md +++ b/doc/install/relative_url.md @@ -135,5 +135,5 @@ To disable the relative URL: 1. Remove `/home/git/gitlab/config/initializers/relative_url.rb` -1. Follow the same as above starting from 2. and set up the +1. Follow the previous steps starting from 2. and set up the GitLab URL to one that doesn't contain a relative path. diff --git a/doc/security/crime_vulnerability.md b/doc/security/crime_vulnerability.md index 3ff9b231458..65c85dcb10c 100644 --- a/doc/security/crime_vulnerability.md +++ b/doc/security/crime_vulnerability.md @@ -55,7 +55,7 @@ The following configuration indicates that the remote service may be vulnerable SPDY support earlier than version 4 is advertised. ``` -The report above indicates that Nessus is only checking if +The previous report indicates that Nessus is only checking if TLS advertises the SPDY protocol earlier than version 4. It does not perform an attack nor does it check if compression is enabled. The Nessus scanner alone cannot tell that SPDY compression is disabled and not subject to the CRIME diff --git a/doc/update/_index.md b/doc/update/_index.md index 820a085d4c9..a15f3117865 100644 --- a/doc/update/_index.md +++ b/doc/update/_index.md @@ -166,7 +166,7 @@ If you upgrade your GitLab instance while the GitLab Runner is processing jobs, As for the artifacts, the GitLab Runner attempts to upload them three times, after which the job eventually fails. -To address the above two scenarios, it is advised to do the following prior to upgrading: +To address the two previous scenarios, it is advised to do the following prior to upgrading: 1. Plan your maintenance. 1. Pause your runners, or block new jobs from starting by adding the following to your `/etc/gitlab/gitlab.rb`: diff --git a/doc/update/terminology.md b/doc/update/terminology.md index 36fb5928b85..1af64df26c5 100644 --- a/doc/update/terminology.md +++ b/doc/update/terminology.md @@ -47,7 +47,7 @@ A breaking change can be considered major if it affects many users, or represent ## Third-party dependencies -This section applies to all above terms. +This section applies to all previous terms. Changes (deprecation, end of support, removal, or breaking change) in third-party dependencies are handled separately from changes to features in GitLab itself: diff --git a/doc/update/versions/gitlab_16_changes.md b/doc/update/versions/gitlab_16_changes.md index 65c3ea60b0f..09192323117 100644 --- a/doc/update/versions/gitlab_16_changes.md +++ b/doc/update/versions/gitlab_16_changes.md @@ -875,7 +875,7 @@ Specific information applies to installations using Geo: - Workhorse You should check the size of your RSA keys (`openssl rsa -in -text -noout | grep "Key:"`) - for any of the applications above before + for any of the previous applications before upgrading. - A `BackfillCiPipelineVariablesForPipelineIdBigintConversion` background migration is finalized with @@ -1557,7 +1557,7 @@ praefect['configuration'] = { # # IMPORTANT: # As part of reconfiguring Praefect, disable this feature. - # Read about this above. + # Read about this as described previously. # verification_interval: 0, }, diff --git a/doc/update/zero_downtime.md b/doc/update/zero_downtime.md index 91d76d2f929..9e8607a26f4 100644 --- a/doc/update/zero_downtime.md +++ b/doc/update/zero_downtime.md @@ -48,7 +48,7 @@ The zero-downtime upgrade process has the following requirements: - You have to use post-deployment migrations. - [Zero-downtime upgrades are not available with the GitLab Charts](https://docs.gitlab.com/charts/installation/upgrade.html). Support is available with the [GitLab Operator](https://docs.gitlab.com/operator/gitlab_upgrades.html) but there are [known limitations](https://docs.gitlab.com/operator/#known-issues) with this deployment method and as such it's not covered in this guide at this time. -In addition to the above, be aware of the following considerations: +In addition to the previous, be aware of the following considerations: - Most of the time, you can safely upgrade from a patch release to the next minor release if the patch release is not the latest. For example, upgrading from `16.3.2` to `16.4.1` should be safe even if `16.3.3` has been released. You should verify the @@ -57,7 +57,7 @@ In addition to the above, be aware of the following considerations: - [GitLab 16 changes](versions/gitlab_16_changes.md) - [GitLab 15 changes](versions/gitlab_15_changes.md) - Some releases may include [background migrations](background_migrations.md). These migrations are performed in the background by Sidekiq and are often used for migrating data. Background migrations are only added in the monthly releases. - - Certain major or minor releases may require a set of background migrations to be finished. While this doesn't require downtime (if the above conditions are met), it's required that you [wait for background migrations to complete](background_migrations.md) between each major or minor release upgrade. + - Certain major or minor releases may require a set of background migrations to be finished. While this doesn't require downtime (if the previous conditions are met), it's required that you [wait for background migrations to complete](background_migrations.md) between each major or minor release upgrade. - The time necessary to complete these migrations can be reduced by increasing the number of Sidekiq workers that can process jobs in the `background_migration` queue. To see the size of this queue, [check for background migrations before upgrading](background_migrations.md). - Zero downtime upgrades can be performed for [Gitaly](#gitaly) when it's set up in its Cluster or Sharded setups due to a graceful reload mechanism. For the [Praefect (Gitaly Cluster)](#praefect-gitaly-cluster) component it can also be directly upgraded without downtime, however the GitLab Linux package does not offer HA and subsequently Zero Downtime support for it's database - A third party database solution is required to avoid downtime. @@ -236,7 +236,7 @@ for a graceful reload, Workhorse doesn't. The best approach is to drain the node such as by using your load balancer. You can also do this by using NGINX on the node through its graceful shutdown functionality. This section explains the NGINX approach. -In addition to the above, Rails is where the main database migrations need to be executed. Like Praefect, the best approach is by using the deploy node. If PgBouncer is currently being used, it also needs to be bypassed as Rails uses an advisory lock when attempting to run a migration to prevent concurrent migrations from running on the same database. These locks are not shared across transactions, resulting in `ActiveRecord::ConcurrentMigrationError` and other issues when running database migrations using PgBouncer in transaction pooling mode. +In addition to the previous, Rails is where the main database migrations need to be executed. Like Praefect, the best approach is by using the deploy node. If PgBouncer is currently being used, it also needs to be bypassed as Rails uses an advisory lock when attempting to run a migration to prevent concurrent migrations from running on the same database. These locks are not shared across transactions, resulting in `ActiveRecord::ConcurrentMigrationError` and other issues when running database migrations using PgBouncer in transaction pooling mode. 1. On the **Rails deploy node**: diff --git a/qa/Gemfile b/qa/Gemfile index 848705e9818..e771dc34f04 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' gem 'gitlab-qa', '~> 15', '>= 15.5.0', require: 'gitlab/qa' -gem 'gitlab_quality-test_tooling', '~> 2.11.0', require: false +gem 'gitlab_quality-test_tooling', '~> 2.13.0', require: false gem 'gitlab-utils', path: '../gems/gitlab-utils' gem 'activesupport', '~> 7.1.5.1' # This should stay in sync with the root's Gemfile gem 'allure-rspec', '~> 2.27.0' @@ -11,7 +11,7 @@ gem 'capybara', '~> 3.40.0' gem 'capybara-screenshot', '~> 1.0.26' gem 'rake', '~> 13', '>= 13.2.1' gem 'rspec', '~> 3.13' -gem 'selenium-webdriver', '= 4.32.0' +gem 'selenium-webdriver', '= 4.33.0' gem 'rest-client', '~> 2.1.0' gem 'rspec_junit_formatter', '~> 0.6.0' gem 'faker', '~> 3.5', '>= 3.5.1' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 00f3b6d99e4..8438b00eabc 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -139,7 +139,7 @@ GEM rainbow (>= 3, < 4) table_print (= 1.5.7) zeitwerk (>= 2, < 3) - gitlab_quality-test_tooling (2.11.0) + gitlab_quality-test_tooling (2.13.0) activesupport (>= 7.0, < 7.3) amatch (~> 0.4.1) fog-google (~> 1.24, >= 1.24.1) @@ -149,7 +149,7 @@ GEM nokogiri (~> 1.10) parallel (>= 1, < 2) rainbow (>= 3, < 4) - rspec-parameterized (~> 1.0.0) + rspec-parameterized (>= 1.0, < 3.0) table_print (= 1.5.7) zeitwerk (>= 2, < 3) google-apis-compute_v1 (0.101.0) @@ -316,7 +316,7 @@ GEM addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) securerandom (0.4.1) - selenium-webdriver (4.32.0) + selenium-webdriver (4.33.0) base64 (~> 0.2) logger (~> 1.4) rexml (~> 3.2, >= 3.2.5) @@ -382,7 +382,7 @@ DEPENDENCIES gitlab-orchestrator! gitlab-qa (~> 15, >= 15.5.0) gitlab-utils! - gitlab_quality-test_tooling (~> 2.11.0) + gitlab_quality-test_tooling (~> 2.13.0) googleauth (~> 1.9.0) influxdb-client (~> 3.2) junit_merge (~> 0.1.2) @@ -400,7 +400,7 @@ DEPENDENCIES rspec-parameterized (~> 1.0.2) rspec_junit_formatter (~> 0.6.0) ruby-debug-ide (~> 0.7.5) - selenium-webdriver (= 4.32.0) + selenium-webdriver (= 4.33.0) slack-notifier (~> 2.4) terminal-table (~> 4.0.0) warning (~> 1.5)