root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-11-24 15:07:34 +00:00
parent 496739b38c
commit e778dcbcee
33 changed files with 409 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.rubocop_todo/style/lambda.yml
View File

@ -49,7 +49,6 @@ Style/Lambda:
- 'lib/gitlab/action_cable/request_store_callbacks.rb'
- 'lib/gitlab/checks/diff_check.rb'
- 'lib/gitlab/database/load_balancing/action_cable_callbacks.rb'
- 'lib/gitlab/memory/watchdog/configurator.rb'
- 'lib/gitlab/middleware/rack_multipart_tempfile_factory.rb'
- 'lib/gitlab/omniauth_initializer.rb'
- 'lib/gitlab/prometheus/queries/query_additional_metrics.rb'

8
app/assets/javascripts/flash.js
View File

@ -114,6 +114,7 @@ const addDismissFlashClickListener = (flashEl, fadeTransition) => {
* @param {object} [options.parent] - Reference to parent element under which alert needs to appear. Defaults to `document`.
* @param {Function} [options.onDismiss] - Handler to call when this alert is dismissed.
* @param {string} [options.containerSelector] - Selector for the container of the alert
* @param {boolean} [options.preservePrevious] - Set to `true` to preserve previous alerts. Defaults to `false`.
* @param {object} [options.primaryButton] - Object describing primary button of alert
* @param {string} [options.primaryButton.link] - Href of primary button
* @param {string} [options.primaryButton.text] - Text of primary button
@ -131,6 +132,7 @@ const createAlert = function createAlert({
variant = VARIANT_DANGER,
parent = document,
containerSelector = '.flash-container',
preservePrevious = false,
primaryButton = null,
secondaryButton = null,
onDismiss = null,
@ -143,7 +145,11 @@ const createAlert = function createAlert({
if (!alertContainer) return null;
const el = document.createElement('div');
alertContainer.appendChild(el);
if (preservePrevious) {
alertContainer.appendChild(el);
} else {
alertContainer.replaceChildren(el);
}
return new Vue({
el,

4
app/assets/javascripts/projects/default_project_templates.js
View File

@ -113,4 +113,8 @@ export default {
text: s__('ProjectTemplates|Jsonnet for Dynamic Child Pipelines'),
icon: '.template-option .icon-gitlab_logo',
},
bridgetown: {
text: s__('ProjectTemplates|Pages/Bridgetown'),
icon: '.template-option .icon-gitlab_logo',
},
};

2
config/feature_flags/development/schema_linting.yml
View File

@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/255919
milestone: '13.2'
type: development
group: group::editor
default_enabled: false
default_enabled: true

5
db/docs/ci_job_token_project_scope_links.yml
View File

@ -4,7 +4,10 @@ classes:
- Ci::JobToken::ProjectScopeLink
feature_categories:
- continuous_integration
description: The connection between a source project, which defines the job token scope, and a target project, which is the one allowed to be accessed by the job token.
description: |
Links a source project and target project, allowing a project's job token to give access to another project.
Using the outbound direction, the source project's job token can access target projects.
Using the inbound direction, the source project can be accessed by the target project's job token.
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62733
milestone: '14.0'
gitlab_schema: gitlab_ci

10
db/migrate/20221110183103_add_dashboard_fields_to_namespace_details.rb Normal file
View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
class AddDashboardFieldsToNamespaceDetails < Gitlab::Database::Migration[2.0]
enable_lock_retries!
def change
add_column :namespace_details, :dashboard_notification_at, :datetime_with_timezone
add_column :namespace_details, :dashboard_enforcement_at, :datetime_with_timezone
end
end

1
db/schema_migrations/20221110183103 Normal file
View File

@ -0,0 +1 @@
3a8b69f61d48ed02d1015cf63b1dd89fb7206a3d5ce9668126cfdc52048f1e61

4
db/structure.sql
View File

@ -18034,7 +18034,9 @@ CREATE TABLE namespace_details (
description text,
description_html text,
free_user_cap_over_limt_notified_at timestamp with time zone,
free_user_cap_over_limit_notified_at timestamp with time zone
free_user_cap_over_limit_notified_at timestamp with time zone,
dashboard_notification_at timestamp with time zone,
dashboard_enforcement_at timestamp with time zone
);
CREATE TABLE namespace_limits (

5
doc/operations/incident_management/incidents.md
View File

@ -66,8 +66,9 @@ You can set up a webhook with PagerDuty to automatically create a GitLab inciden
for each PagerDuty incident. This configuration requires you to make changes
in both PagerDuty and GitLab:
1. Sign in as a user with the Maintainer role.
1. Navigate to **Settings > Monitor > Incidents** and expand **Incidents**.
1. On the top bar, select **Main menu > Projects** and find your project.
1. On the left sidebar, select **Settings > Monitor**
1. Expand **Incidents**.
1. Select the **PagerDuty integration** tab:
![PagerDuty incidents integration](img/pagerduty_incidents_integration_v13_3.png)

31
doc/user/product_analytics/index.md
View File

@ -4,9 +4,9 @@ group: Product Analytics
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
# Product analytics **(ULTIMATE)** **Alpha**
# Product analytics **(ULTIMATE)**
> Introduced in GitLab 15.4 [with a flag](../../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default.
> Introduced in GitLab 15.4 as an [Alpha](../../policy/alpha-beta-support.md#alpha-features) feature [with a flag](../../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default.
FLAG:
On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `cube_api_proxy`.
@ -17,6 +17,33 @@ This feature is not ready for production use.
You can view the [product category](https://about.gitlab.com/direction/analytics/product-analytics/) page for more information about our direction. This page is a work in progress and will be updated as we add more features.
## Enable product analytics
You can enable and configure product analytics to track events
within your project applications on a self-managed instance.
Prerequisite:
- You must be an administrator of a self-managed GitLab instance.
1. On the top bar, select **Main menu > Admin**.
1. On the left sidebar, select **Settings > General**.
1. Expand the **Product analytics** section.
1. Select **Enable product analytics** and enter the configuration values.
The following table shows the required configuration parameters and example values:
| Name | Value |
|------------------------------|----------------------------|
| Jitsu host | `https://jitsu.gitlab.com` |
| Jitsu project ID | `g0maofw84gx5sjxgse2k` |
| Jitsu administrator email | `jitsu.admin@gitlab.com` |
| Jitsu administrator password | `<your_password>` |
| Clickhouse URL | `https://<username>:<password>@clickhouse.gitlab.com:8123` |
| Cube API URL | `https://cube.gitlab.com` |
| Cube API key | `25718201b3e9...ae6bbdc62dbb` |
1. Select **Save changes**.
## Product analytics dashboards
Each project can define an unlimited number of dashboards. These dashboards are defined using our YAML schema and stored

1
doc/user/shortcuts.md
View File

@ -37,6 +37,7 @@ These shortcuts are available in most areas of GitLab:
| <kbd>f</kbd> | Put cursor in the filter bar. |
| <kbd>Shift</kbd> + <kbd>i</kbd> | Go to your Issues page. |
| <kbd>Shift</kbd> + <kbd>m</kbd> | Go to your [Merge requests](project/merge_requests/index.md) page. |
| <kbd>Shift</kbd> + <kbd>r</kbd> | Go to your Review requests page. |
| <kbd>Shift</kbd> + <kbd>t</kbd> | Go to your To-Do List page. |
| <kbd>p</kbd>, then <kbd>b</kbd> | Show or hide the Performance Bar. |
| <kbd>Escape</kbd> | Hide tooltips or popovers. |

8
lib/gitlab/ci/config/external/mapper.rb vendored
View File

@ -52,14 +52,8 @@ module Gitlab
.each(&method(:verify!))
end
def normalize_location(location)
logger.instrument(:config_mapper_normalize) do
normalize_location_without_instrumentation(location)
end
end
# convert location if String to canonical form
def normalize_location_without_instrumentation(location)
def normalize_location(location)
if location.is_a?(String)
expanded_location = expand_variables(location)
normalize_location_string(expanded_location)

4
lib/gitlab/ci/config/external/processor.rb vendored
View File

@ -32,9 +32,7 @@ module Gitlab
def validate_external_files!
@external_files.each do |file|
logger.instrument(:config_external_verify) do
raise IncludeError, file.error_message unless file.valid?
end
raise IncludeError, file.error_message unless file.valid?
end
end

10
lib/gitlab/ci/pipeline/chain/seed.rb
View File

@ -62,12 +62,10 @@ module Gitlab
def root_variables
strong_memoize(:root_variables) do
logger.instrument(:pipeline_seed_merge_variables, once: true) do
::Gitlab::Ci::Variables::Helpers.merge_variables(
@command.yaml_processor_result.root_variables,
@command.workflow_rules_result.variables
)
end
::Gitlab::Ci::Variables::Helpers.merge_variables(
@command.yaml_processor_result.root_variables,
@command.workflow_rules_result.variables
)
end
end
end

10
lib/gitlab/memory/watchdog/configuration.rb
View File

@ -10,7 +10,6 @@ module Gitlab
end
def push(monitor_class, *args, **kwargs, &block)
remove(monitor_class)
@monitors.push(build_monitor_state(monitor_class, *args, **kwargs, &block))
end
@ -22,14 +21,11 @@ module Gitlab
private
def remove(monitor_class)
@monitors.delete_if { |monitor| monitor.monitor_class == monitor_class }
end
def build_monitor_state(monitor_class, *args, max_strikes:, **kwargs, &block)
def build_monitor_state(monitor_class, *args, max_strikes:, monitor_name: nil, **kwargs, &block)
monitor = build_monitor(monitor_class, *args, **kwargs, &block)
monitor_name ||= monitor_class.name.demodulize.underscore
Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes)
Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes, monitor_name: monitor_name)
end
def build_monitor(monitor_class, *args, **kwargs, &block)

61
lib/gitlab/memory/watchdog/configurator.rb
View File

@ -4,25 +4,33 @@ module Gitlab
module Memory
class Watchdog
class Configurator
DEFAULT_PUMA_WORKER_RSS_LIMIT_MB = 1200
DEFAULT_SLEEP_INTERVAL_S = 60
DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S = 3
MIN_SIDEKIQ_SLEEP_INTERVAL_S = 2
DEFAULT_MAX_STRIKES = 5
DEFAULT_MAX_HEAP_FRAG = 0.5
DEFAULT_MAX_MEM_GROWTH = 3.0
# grace_time / sleep_interval = max_strikes allowed for Sidekiq process to violate defined limits.
DEFAULT_SIDEKIQ_GRACE_TIME_S = 300
class << self
def configure_for_puma
lambda do |config|
->(config) do
config.logger = Gitlab::AppLogger
config.handler = Gitlab::Memory::Watchdog::PumaHandler.new
config.write_heap_dumps = write_heap_dumps?
config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', 60).to_i
config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', DEFAULT_SLEEP_INTERVAL_S).to_i
config.monitors(&configure_monitors_for_puma)
end
end
def configure_for_sidekiq
lambda do |config|
->(config) do
config.logger = Sidekiq.logger
config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new
config.write_heap_dumps = write_heap_dumps?
config.sleep_time_seconds = [
ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2
].max
config.sleep_time_seconds = sidekiq_sleep_time
config.monitors(&configure_monitors_for_sidekiq)
end
end
@ -34,12 +42,12 @@ module Gitlab
end
def configure_monitors_for_puma
lambda do |stack|
max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', 5).to_i
->(stack) do
max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', DEFAULT_MAX_STRIKES).to_i
if Gitlab::Utils.to_boolean(ENV['DISABLE_PUMA_WORKER_KILLER'])
max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', 0.5).to_f
max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', 3.0).to_f
max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', DEFAULT_MAX_HEAP_FRAG).to_f
max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', DEFAULT_MAX_MEM_GROWTH).to_f
# stack.push MonitorClass, args*, max_strikes:, kwargs**, &block
stack.push Gitlab::Memory::Watchdog::Monitor::HeapFragmentation,
@ -50,17 +58,44 @@ module Gitlab
max_mem_growth: max_mem_growth,
max_strikes: max_strikes
else
memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', 1200).to_i
memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', DEFAULT_PUMA_WORKER_RSS_LIMIT_MB).to_i
stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
memory_limit: memory_limit.megabytes,
memory_limit_bytes: memory_limit.megabytes,
max_strikes: max_strikes
end
end
end
def sidekiq_sleep_time
[
ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S).to_i,
MIN_SIDEKIQ_SLEEP_INTERVAL_S
].max
end
def configure_monitors_for_sidekiq
# NOP - At the moment we don't run watchdog for Sidekiq
->(stack) do
if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero?
soft_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.kilobytes
grace_time = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', DEFAULT_SIDEKIQ_GRACE_TIME_S).to_i
max_strikes = grace_time / sidekiq_sleep_time
stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
memory_limit_bytes: soft_limit_bytes,
max_strikes: max_strikes.to_i,
monitor_name: :rss_memory_soft_limit
end
if ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.nonzero?
hard_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.kilobytes
stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
memory_limit_bytes: hard_limit_bytes,
max_strikes: 0,
monitor_name: :rss_memory_hard_limit
end
end
end
end
end

5
lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb
View File

@ -4,10 +4,7 @@ module Gitlab
module Memory
class Watchdog
module Monitor
# A monitor that observes Ruby heap fragmentation and calls
# memory_violation_callback when the Ruby heap has been fragmented for an extended
# period of time.
#
# A monitor that observes Ruby heap fragmentation.
# See Gitlab::Metrics::Memory for how heap fragmentation is defined.
class HeapFragmentation
attr_reader :max_heap_fragmentation

18
lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb
View File

@ -5,27 +5,27 @@ module Gitlab
class Watchdog
module Monitor
class RssMemoryLimit
attr_reader :memory_limit
attr_reader :memory_limit_bytes
def initialize(memory_limit:)
@memory_limit = memory_limit
def initialize(memory_limit_bytes:)
@memory_limit_bytes = memory_limit_bytes
end
def call
worker_rss = Gitlab::Metrics::System.memory_usage_rss[:total]
worker_rss_bytes = Gitlab::Metrics::System.memory_usage_rss[:total]
return { threshold_violated: false, payload: {} } if worker_rss <= memory_limit
return { threshold_violated: false, payload: {} } if worker_rss_bytes <= memory_limit_bytes
{ threshold_violated: true, payload: payload(worker_rss, memory_limit) }
{ threshold_violated: true, payload: payload(worker_rss_bytes, memory_limit_bytes) }
end
private
def payload(worker_rss, memory_limit)
def payload(worker_rss_bytes, memory_limit_bytes)
{
message: 'rss memory limit exceeded',
memwd_rss_bytes: worker_rss,
memwd_max_rss_bytes: memory_limit
memwd_rss_bytes: worker_rss_bytes,
memwd_max_rss_bytes: memory_limit_bytes
}
end
end

19
lib/gitlab/memory/watchdog/monitor_state.rb
View File

@ -5,12 +5,12 @@ module Gitlab
class Watchdog
class MonitorState
class Result
attr_reader :payload
attr_reader :payload, :monitor_name
def initialize(strikes_exceeded:, threshold_violated:, monitor_class:, payload: )
def initialize(strikes_exceeded:, threshold_violated:, monitor_name:, payload: )
@strikes_exceeded = strikes_exceeded
@threshold_violated = threshold_violated
@monitor_class = monitor_class
@monitor_name = monitor_name.to_s.to_sym
@payload = payload
end
@ -21,15 +21,12 @@ module Gitlab
def threshold_violated?
@threshold_violated
end
def monitor_name
@monitor_class.name.demodulize.underscore.to_sym
end
end
def initialize(monitor, max_strikes:)
def initialize(monitor, max_strikes:, monitor_name:)
@monitor = monitor
@max_strikes = max_strikes
@monitor_name = monitor_name
@strikes = 0
end
@ -47,16 +44,12 @@ module Gitlab
build_result(monitor_result)
end
def monitor_class
@monitor.class
end
private
def build_result(monitor_result)
Result.new(
strikes_exceeded: strikes_exceeded?,
monitor_class: monitor_class,
monitor_name: @monitor_name,
threshold_violated: monitor_result[:threshold_violated],
payload: payload.merge(monitor_result[:payload]))
end

1
lib/gitlab/project_template.rb
View File

@ -60,6 +60,7 @@ module Gitlab
ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/third-party-logos/dotnet.svg'),
ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'),
ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'),
ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/gitlab-org/project-templates/bridgetown'),
ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby'), 'https://gitlab.com/pages/gatsby', 'illustrations/third-party-logos/gatsby.svg'),
ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'),
ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'),

6
locale/gitlab.pot
View File

@ -16167,6 +16167,9 @@ msgstr ""
msgid "Everything on your to-do list is marked as done."
msgstr ""
msgid "Everything you need to create a GitLab Pages site using Bridgetown"
msgstr ""
msgid "Everything you need to create a GitLab Pages site using Gatsby"
msgstr ""
@ -32557,6 +32560,9 @@ msgstr ""
msgid "ProjectTemplates|NodeJS Express"
msgstr ""
msgid "ProjectTemplates|Pages/Bridgetown"
msgstr ""
msgid "ProjectTemplates|Pages/Gatsby"
msgstr ""

2
qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb
View File

@ -4,7 +4,7 @@
# rubocop:disable Rails/Pluck, Layout/LineLength, RSpec/MultipleMemoizedHelpers
module QA
RSpec.describe "Manage", only: { job: "large-gitlab-import" } do
RSpec.describe "Manage", :skip_live_env, only: { job: "large-gitlab-import" } do
describe "Gitlab migration", orchestrated: false, product_group: :import do
include_context "with gitlab group migration"

2
spec/controllers/groups/labels_controller_spec.rb
View File

@ -54,7 +54,7 @@ RSpec.describe Groups::LabelsController do
get :index, params: { group_id: group.to_param }
end
it 'avoids N+1 queries' do
it 'avoids N+1 queries', :use_clean_rails_redis_caching do
control = ActiveRecord::QueryRecorder.new(skip_cached: false) { get :index, params: { group_id: group.to_param } }
create_list(:group_label, 3, group: group)

2
spec/controllers/projects/labels_controller_spec.rb
View File

@ -100,7 +100,7 @@ RSpec.describe Projects::LabelsController do
list_labels
end
it 'avoids N+1 queries' do
it 'avoids N+1 queries', :use_clean_rails_redis_caching do
control = ActiveRecord::QueryRecorder.new(skip_cached: false) { list_labels }
create_list(:label, 3, project: project)

24
spec/frontend/flash_spec.js
View File

@ -12,6 +12,9 @@ import createFlash, {
jest.mock('@sentry/browser');
describe('Flash', () => {
const findTextContent = (containerSelector = '.flash-container') =>
document.querySelector(containerSelector).textContent.replace(/\s+/g, ' ').trim();
describe('hideFlash', () => {
let el;
@ -99,7 +102,7 @@ describe('Flash', () => {
it('adds alert element into the document by default', () => {
alert = createAlert({ message: mockMessage });
expect(document.querySelector('.flash-container').textContent.trim()).toBe(mockMessage);
expect(findTextContent()).toBe(mockMessage);
expect(document.querySelector('.flash-container .gl-alert')).not.toBeNull();
});
@ -202,8 +205,7 @@ describe('Flash', () => {
message: mockMessage,
});
const text = document.querySelector('.flash-container').textContent.trim();
expect(text).toBe(`${mockTitle} ${mockMessage}`);
expect(findTextContent()).toBe(`${mockTitle} ${mockMessage}`);
});
});
@ -319,6 +321,22 @@ describe('Flash', () => {
});
});
});
describe('when called multiple times', () => {
it('clears previous alerts', () => {
createAlert({ message: 'message 1' });
createAlert({ message: 'message 2' });
expect(findTextContent()).toBe('message 2');
});
it('preserves alerts when `preservePrevious` is true', () => {
createAlert({ message: 'message 1' });
createAlert({ message: 'message 2', preservePrevious: true });
expect(findTextContent()).toBe('message 1 message 2');
});
});
});
});

25
spec/lib/gitlab/memory/watchdog/configuration_spec.rb
View File

@ -38,6 +38,8 @@ RSpec.describe Gitlab::Memory::Watchdog::Configuration do
describe '#monitors' do
context 'when monitors are configured to be used' do
let(:monitor_name1) { :monitor1 }
let(:monitor_name2) { :monitor2 }
let(:payload1) do
{
message: 'monitor_1_text',
@ -96,7 +98,7 @@ RSpec.describe Gitlab::Memory::Watchdog::Configuration do
expect(payloads).to eq([payload1, payload2])
expect(thresholds).to eq([false, true])
expect(strikes).to eq([false, true])
expect(monitor_names).to eq([:monitor1, :monitor2])
expect(monitor_names).to eq([monitor_name1, monitor_name2])
end
end
@ -119,18 +121,19 @@ RSpec.describe Gitlab::Memory::Watchdog::Configuration do
include_examples 'executes monitors and returns correct results'
end
end
context 'when same monitor class is configured twice' do
before do
configuration.monitors.push monitor_class_1, max_strikes: 1
configuration.monitors.push monitor_class_1, max_strikes: 1
end
context 'when monitors are configured with monitor name' do
let(:monitor_name1) { :mon_one }
let(:monitor_name2) { :mon_two }
it 'calls same monitor only once' do
expect do |b|
configuration.monitors.call_each(&b)
end.to yield_control.once
before do
configuration.monitors do |stack|
stack.push monitor_class_1, false, { message: 'monitor_1_text' }, max_strikes: 5, monitor_name: :mon_one
stack.push monitor_class_2, true, { message: 'monitor_2_text' }, max_strikes: 0, monitor_name: :mon_two
end
end
include_examples 'executes monitors and returns correct results'
end
end
end

146
spec/lib/gitlab/memory/watchdog/configurator_spec.rb
View File

@ -130,11 +130,11 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do
it_behaves_like 'as configurator',
Gitlab::Memory::Watchdog::PumaHandler,
'GITLAB_MEMWD_SLEEP_TIME_SEC',
60
described_class::DEFAULT_SLEEP_INTERVAL_S
context 'with DISABLE_PUMA_WORKER_KILLER set to true' do
let(:primary_memory) { 2048 }
let(:worker_memory) { max_mem_growth * primary_memory + 1 }
let(:primary_memory_bytes) { 2_097_152_000 }
let(:worker_memory_bytes) { max_mem_growth * primary_memory_bytes + 1 }
let(:expected_payloads) do
{
heap_fragmentation: {
@ -147,9 +147,9 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do
},
unique_memory_growth: {
message: 'memory limit exceeded',
memwd_uss_bytes: worker_memory,
memwd_ref_uss_bytes: primary_memory,
memwd_max_uss_bytes: max_mem_growth * primary_memory,
memwd_uss_bytes: worker_memory_bytes,
memwd_ref_uss_bytes: primary_memory_bytes,
memwd_max_uss_bytes: max_mem_growth * primary_memory_bytes,
memwd_max_strikes: max_strikes,
memwd_cur_strikes: 1
}
@ -159,10 +159,10 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do
before do
stub_env('DISABLE_PUMA_WORKER_KILLER', true)
allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return(max_heap_fragmentation + 0.1)
allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return({ uss: worker_memory })
allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return({ uss: worker_memory_bytes })
allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(
pid: Gitlab::Cluster::PRIMARY_PID
).and_return({ uss: primary_memory })
).and_return({ uss: primary_memory_bytes })
end
context 'when settings are set via environment variables' do
@ -180,21 +180,22 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do
end
context 'when settings are not set via environment variables' do
let(:max_heap_fragmentation) { 0.5 }
let(:max_mem_growth) { 3.0 }
let(:max_strikes) { 5 }
let(:max_heap_fragmentation) { described_class::DEFAULT_MAX_HEAP_FRAG }
let(:max_mem_growth) { described_class::DEFAULT_MAX_MEM_GROWTH }
let(:max_strikes) { described_class::DEFAULT_MAX_STRIKES }
it_behaves_like 'as monitor configurator'
end
end
context 'with DISABLE_PUMA_WORKER_KILLER set to false' do
let(:memory_limit_bytes) { memory_limit_mb.megabytes }
let(:expected_payloads) do
{
rss_memory_limit: {
message: 'rss memory limit exceeded',
memwd_rss_bytes: memory_limit + 1,
memwd_max_rss_bytes: memory_limit,
memwd_rss_bytes: memory_limit_bytes + 1,
memwd_max_rss_bytes: memory_limit_bytes,
memwd_max_strikes: max_strikes,
memwd_cur_strikes: 1
}
@ -203,15 +204,15 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do
before do
stub_env('DISABLE_PUMA_WORKER_KILLER', false)
allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: memory_limit + 1 })
allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: memory_limit_bytes + 1 })
end
context 'when settings are set via environment variables' do
let(:memory_limit) { 1300.megabytes }
let(:memory_limit_mb) { 1300 }
let(:max_strikes) { 4 }
before do
stub_env('PUMA_WORKER_MAX_MEMORY', 1300)
stub_env('PUMA_WORKER_MAX_MEMORY', memory_limit_mb)
stub_env('GITLAB_MEMWD_MAX_STRIKES', 4)
end
@ -219,8 +220,8 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do
end
context 'when settings are not set via environment variables' do
let(:memory_limit) { 1200.megabytes }
let(:max_strikes) { 5 }
let(:memory_limit_mb) { described_class::DEFAULT_PUMA_WORKER_RSS_LIMIT_MB }
let(:max_strikes) { described_class::DEFAULT_MAX_STRIKES }
it_behaves_like 'as monitor configurator'
end
@ -236,6 +237,113 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do
it_behaves_like 'as configurator',
Gitlab::Memory::Watchdog::TermProcessHandler,
'SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL',
3
described_class::DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S
context 'when sleep_time_seconds is less than MIN_SIDEKIQ_SLEEP_INTERVAL_S seconds' do
before do
stub_env('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 0)
end
it 'configures the correct sleep time' do
configurator.call(configuration)
expect(configuration.sleep_time_seconds).to eq(described_class::MIN_SIDEKIQ_SLEEP_INTERVAL_S)
end
end
context 'with monitors' do
let(:soft_limit_bytes) { soft_limit_kb.kilobytes }
let(:hard_limit_bytes) { hard_limit_kb.kilobytes }
context 'when settings are set via environment variables' do
let(:soft_limit_kb) { 2000001 }
let(:hard_limit_kb) { 300000 }
let(:max_strikes) { 150 }
let(:grace_time) { 300 }
let(:expected_payloads) do
{
rss_memory_soft_limit: {
message: 'rss memory limit exceeded',
memwd_rss_bytes: soft_limit_bytes + 1,
memwd_max_rss_bytes: soft_limit_bytes,
memwd_max_strikes: max_strikes,
memwd_cur_strikes: 1
},
rss_memory_hard_limit: {
message: 'rss memory limit exceeded',
memwd_rss_bytes: hard_limit_bytes + 1,
memwd_max_rss_bytes: hard_limit_bytes,
memwd_max_strikes: 0,
memwd_cur_strikes: 1
}
}
end
before do
stub_env('SIDEKIQ_MEMORY_KILLER_MAX_RSS', soft_limit_kb)
stub_env('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', hard_limit_kb)
stub_env('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', grace_time)
stub_env('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 2)
allow(Gitlab::Metrics::System).to receive(:memory_usage_rss)
.and_return({ total: soft_limit_bytes + 1 }, { total: hard_limit_bytes + 1 })
end
it_behaves_like 'as monitor configurator'
end
context 'when only SIDEKIQ_MEMORY_KILLER_MAX_RSS is set via environment variable' do
let(:soft_limit_kb) { 2000000 }
let(:max_strikes) do
described_class::DEFAULT_SIDEKIQ_GRACE_TIME_S / described_class::DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S
end
let(:expected_payloads) do
{
rss_memory_soft_limit: {
message: 'rss memory limit exceeded',
memwd_rss_bytes: soft_limit_bytes + 1,
memwd_max_rss_bytes: soft_limit_bytes,
memwd_max_strikes: max_strikes,
memwd_cur_strikes: 1
}
}
end
before do
allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: soft_limit_bytes + 1 })
stub_env('SIDEKIQ_MEMORY_KILLER_MAX_RSS', soft_limit_kb)
end
it_behaves_like 'as monitor configurator'
end
context 'when only SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS is set via environment variable' do
let(:hard_limit_kb) { 2000000 }
let(:expected_payloads) do
{
rss_memory_hard_limit: {
message: 'rss memory limit exceeded',
memwd_rss_bytes: hard_limit_bytes + 1,
memwd_max_rss_bytes: hard_limit_bytes,
memwd_max_strikes: 0,
memwd_cur_strikes: 1
}
}
end
before do
allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: hard_limit_bytes + 1 })
stub_env('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', hard_limit_kb)
end
it_behaves_like 'as monitor configurator'
end
context 'when both SIDEKIQ_MEMORY_KILLER_MAX_RSS and SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS are not set' do
let(:expected_payloads) { {} }
it_behaves_like 'as monitor configurator'
end
end
end
end

16
spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb
View File

@ -4,25 +4,25 @@ require 'fast_spec_helper'
require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples'
RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit do
let(:memory_limit) { 2048 }
let(:worker_memory) { 1024 }
let(:memory_limit_bytes) { 2_097_152_000 }
let(:worker_memory_bytes) { 1_048_576_000 }
subject(:monitor) do
described_class.new(memory_limit: memory_limit)
described_class.new(memory_limit_bytes: memory_limit_bytes)
end
before do
allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: worker_memory })
allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: worker_memory_bytes })
end
describe '#call' do
context 'when process exceeds threshold' do
let(:worker_memory) { memory_limit + 1 }
let(:worker_memory_bytes) { memory_limit_bytes + 1 }
let(:payload) do
{
message: 'rss memory limit exceeded',
memwd_rss_bytes: worker_memory,
memwd_max_rss_bytes: memory_limit
memwd_rss_bytes: worker_memory_bytes,
memwd_max_rss_bytes: memory_limit_bytes
}
end
@ -30,7 +30,7 @@ RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit do
end
context 'when process does not exceed threshold' do
let(:worker_memory) { memory_limit - 1 }
let(:worker_memory_bytes) { memory_limit_bytes - 1 }
let(:payload) { {} }
include_examples 'returns Watchdog Monitor result', threshold_violated: false

11
spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb
View File

@ -7,6 +7,7 @@ RSpec.describe Gitlab::Memory::Watchdog::MonitorState do
let(:payload) { { message: 'DummyMessage' } }
let(:threshold_violated) { true }
let(:monitor) { monitor_class.new(threshold_violated, payload) }
let(:monitor_name) { :dummy_monitor_name }
let(:monitor_class) do
Struct.new(:threshold_violated, :payload) do
def call
@ -19,7 +20,7 @@ RSpec.describe Gitlab::Memory::Watchdog::MonitorState do
end
end
subject(:monitor_state) { described_class.new(monitor, max_strikes: max_strikes) }
subject(:monitor_state) { described_class.new(monitor, max_strikes: max_strikes, monitor_name: monitor_name) }
shared_examples 'returns correct result' do
it 'returns correct result', :aggregate_failures do
@ -29,7 +30,7 @@ RSpec.describe Gitlab::Memory::Watchdog::MonitorState do
expect(result.strikes_exceeded?).to eq(strikes_exceeded)
expect(result.threshold_violated?).to eq(threshold_violated)
expect(result.payload).to eq(expected_payload)
expect(result.monitor_name).to eq(:monitor_name)
expect(result.monitor_name).to eq(monitor_name)
end
end
@ -63,10 +64,4 @@ RSpec.describe Gitlab::Memory::Watchdog::MonitorState do
end
end
end
describe '#monitor_class' do
subject { monitor_state.monitor_class }
it { is_expected.to eq(monitor_class) }
end
end

128
spec/models/ci/build_metadata_spec.rb
View File

@ -6,7 +6,6 @@ RSpec.describe Ci::BuildMetadata do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, :repository, group: group, build_timeout: 2000) }
let_it_be(:pipeline) do
create(:ci_pipeline, project: project,
sha: project.commit.id,
@ -14,7 +13,9 @@ RSpec.describe Ci::BuildMetadata do
status: 'success')
end
let(:job) { create(:ci_build, pipeline: pipeline) }
let_it_be_with_reload(:runner) { create(:ci_runner) }
let(:job) { create(:ci_build, pipeline: pipeline, runner: runner) }
let(:metadata) { job.metadata }
it_behaves_like 'having unique enum values'
@ -32,63 +33,110 @@ RSpec.describe Ci::BuildMetadata do
end
end
context 'when project timeout is set' do
context 'when runner is assigned to the job' do
context 'when job, project and runner timeouts are set' do
context 'when job timeout is lower then runner timeout' do
before do
job.update!(runner: runner)
runner.update!(maximum_timeout: 4000)
job.update!(options: { job_timeout: 3000 })
end
context 'when runner timeout is not set' do
let(:runner) { create(:ci_runner, maximum_timeout: nil) }
it_behaves_like 'sets timeout', 'project_timeout_source', 2000
end
context 'when runner timeout is lower than project timeout' do
let(:runner) { create(:ci_runner, maximum_timeout: 1900) }
it_behaves_like 'sets timeout', 'runner_timeout_source', 1900
end
context 'when runner timeout is higher than project timeout' do
let(:runner) { create(:ci_runner, maximum_timeout: 2100) }
it_behaves_like 'sets timeout', 'project_timeout_source', 2000
end
it_behaves_like 'sets timeout', 'job_timeout_source', 3000
end
context 'when job timeout is set' do
context 'when job timeout is higher than project timeout' do
let(:job) { create(:ci_build, pipeline: pipeline, options: { job_timeout: 3000 }) }
it_behaves_like 'sets timeout', 'job_timeout_source', 3000
context 'when runner timeout is lower then job timeout' do
before do
runner.update!(maximum_timeout: 2000)
job.update!(options: { job_timeout: 3000 })
end
context 'when job timeout is lower than project timeout' do
let(:job) { create(:ci_build, pipeline: pipeline, options: { job_timeout: 1000 }) }
it_behaves_like 'sets timeout', 'runner_timeout_source', 2000
end
end
it_behaves_like 'sets timeout', 'job_timeout_source', 1000
context 'when job, project timeout values are set and runner is assigned' do
context 'when runner has no timeout set' do
before do
runner.update!(maximum_timeout: nil)
job.update!(options: { job_timeout: 3000 })
end
it_behaves_like 'sets timeout', 'job_timeout_source', 3000
end
end
context 'when only job and project timeouts are defined' do
context 'when job timeout is lower then project timeout' do
before do
job.update!(options: { job_timeout: 1000 })
end
it_behaves_like 'sets timeout', 'job_timeout_source', 1000
end
context 'when both runner and job timeouts are set' do
context 'when project timeout is lower then job timeout' do
before do
job.update!(runner: runner)
job.update!(options: { job_timeout: 3000 })
end
context 'when job timeout is higher than runner timeout' do
let(:job) { create(:ci_build, pipeline: pipeline, options: { job_timeout: 3000 }) }
let(:runner) { create(:ci_runner, maximum_timeout: 2100) }
it_behaves_like 'sets timeout', 'job_timeout_source', 3000
end
end
it_behaves_like 'sets timeout', 'runner_timeout_source', 2100
context 'when only project and runner timeouts are defined' do
before do
runner.update!(maximum_timeout: 1900)
end
context 'when runner timeout is lower then project timeout' do
it_behaves_like 'sets timeout', 'runner_timeout_source', 1900
end
context 'when project timeout is lower then runner timeout' do
before do
runner.update!(maximum_timeout: 2100)
end
context 'when job timeout is lower than runner timeout' do
let(:job) { create(:ci_build, pipeline: pipeline, options: { job_timeout: 1900 }) }
let(:runner) { create(:ci_runner, maximum_timeout: 2100) }
it_behaves_like 'sets timeout', 'project_timeout_source', 2000
end
end
it_behaves_like 'sets timeout', 'job_timeout_source', 1900
context 'when only job and runner timeouts are defined' do
context 'when runner timeout is lower them job timeout' do
before do
job.update!(options: { job_timeout: 2000 })
runner.update!(maximum_timeout: 1900)
end
it_behaves_like 'sets timeout', 'runner_timeout_source', 1900
end
context 'when job timeout is lower them runner timeout' do
before do
job.update!(options: { job_timeout: 1000 })
runner.update!(maximum_timeout: 1900)
end
it_behaves_like 'sets timeout', 'job_timeout_source', 1000
end
end
context 'when only job timeout is defined and runner is assigned, but has no timeout set' do
before do
job.update!(options: { job_timeout: 1000 })
runner.update!(maximum_timeout: nil)
end
it_behaves_like 'sets timeout', 'job_timeout_source', 1000
end
context 'when only one timeout value is defined' do
context 'when only project timeout value is defined' do
before do
job.update!(options: { job_timeout: nil })
runner.update!(maximum_timeout: nil)
end
it_behaves_like 'sets timeout', 'project_timeout_source', 2000
end
end
end

2
spec/requests/api/project_import_spec.rb
View File

@ -44,7 +44,7 @@ RSpec.describe API::ProjectImport, :aggregate_failures do
it_behaves_like 'requires authentication'
it 'executes a limited number of queries' do
it 'executes a limited number of queries', :use_clean_rails_redis_caching do
control_count = ActiveRecord::QueryRecorder.new { subject }.count
expect(control_count).to be <= 111

2
spec/support/helpers/project_template_test_helper.rb
View File

@ -9,7 +9,7 @@ module ProjectTemplateTestHelper
nfjekyll nfplainhtml nfgitbook nfhexo salesforcedx
serverless_framework tencent_serverless_framework
jsonnet cluster_management kotlin_native_linux
pelican
pelican bridgetown
]
end
end

BIN
vendor/project_templates/bridgetown.tar.gz vendored Normal file
View File

Binary file not shown.