Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2020-06-25 00:09:26 +00:00
parent d627a16f3d
commit 5d6119a1a4
31 changed files with 385 additions and 172 deletions

View File

@ -354,6 +354,10 @@ Please view this file on the master branch, on stable branches it's out of date.
- Translate unauthenticated user string for Audit Event. !31856 (Sashi Kumar)
## 12.10.12 (2020-06-24)
- No changes.
## 12.10.11 (2020-06-10)
### Security (1 change)

View File

@ -0,0 +1,148 @@
# frozen_string_literal: true
##
# DEPRECATED
#
# These helpers are deprecated in favor of detailed CI/CD statuses.
#
# See 'detailed_status?` method and `Gitlab::Ci::Status` module.
#
module Ci
module StatusHelper
def ci_label_for_status(status)
if detailed_status?(status)
return status.label
end
label = case status
when 'success'
'passed'
when 'success-with-warnings'
'passed with warnings'
when 'manual'
'waiting for manual action'
when 'scheduled'
'waiting for delayed job'
else
status
end
translation = "CiStatusLabel|#{label}"
s_(translation)
end
def ci_text_for_status(status)
if detailed_status?(status)
return status.text
end
case status
when 'success'
s_('CiStatusText|passed')
when 'success-with-warnings'
s_('CiStatusText|passed')
when 'manual'
s_('CiStatusText|blocked')
when 'scheduled'
s_('CiStatusText|delayed')
else
# All states are already being translated inside the detailed statuses:
# :running => Gitlab::Ci::Status::Running
# :skipped => Gitlab::Ci::Status::Skipped
# :failed => Gitlab::Ci::Status::Failed
# :success => Gitlab::Ci::Status::Success
# :canceled => Gitlab::Ci::Status::Canceled
# The following states are customized above:
# :manual => Gitlab::Ci::Status::Manual
status_translation = "CiStatusText|#{status}"
s_(status_translation)
end
end
def ci_status_for_statuseable(subject)
status = subject.try(:status) || 'not found'
status.humanize
end
# rubocop:disable Metrics/CyclomaticComplexity
def ci_icon_for_status(status, size: 16)
if detailed_status?(status)
return sprite_icon(status.icon, size: size)
end
icon_name =
case status
when 'success'
'status_success'
when 'success-with-warnings'
'status_warning'
when 'failed'
'status_failed'
when 'pending'
'status_pending'
when 'waiting_for_resource'
'status_pending'
when 'preparing'
'status_preparing'
when 'running'
'status_running'
when 'play'
'play'
when 'created'
'status_created'
when 'skipped'
'status_skipped'
when 'manual'
'status_manual'
when 'scheduled'
'status_scheduled'
else
'status_canceled'
end
sprite_icon(icon_name, size: size)
end
# rubocop:enable Metrics/CyclomaticComplexity
def ci_icon_class_for_status(status)
group = detailed_status?(status) ? status.group : status.dasherize
"ci-status-icon-#{group}"
end
def pipeline_status_cache_key(pipeline_status)
"pipeline-status/#{pipeline_status.sha}-#{pipeline_status.status}"
end
def render_commit_status(commit, status, ref: nil, tooltip_placement: 'left')
project = commit.project
path = pipelines_project_commit_path(project, commit, ref: ref)
render_status_with_link(
status,
path,
tooltip_placement: tooltip_placement,
icon_size: 24)
end
def render_status_with_link(status, path = nil, type: _('pipeline'), tooltip_placement: 'left', cssclass: '', container: 'body', icon_size: 16)
klass = "ci-status-link #{ci_icon_class_for_status(status)} d-inline-flex #{cssclass}"
title = "#{type.titleize}: #{ci_label_for_status(status)}"
data = { toggle: 'tooltip', placement: tooltip_placement, container: container }
if path
link_to ci_icon_for_status(status, size: icon_size), path,
class: klass, title: title, data: data
else
content_tag :span, ci_icon_for_status(status, size: icon_size),
class: klass, title: title, data: data
end
end
def detailed_status?(status)
status.respond_to?(:text) &&
status.respond_to?(:group) &&
status.respond_to?(:label) &&
status.respond_to?(:icon)
end
end
end

View File

@ -1,146 +0,0 @@
# frozen_string_literal: true
##
# DEPRECATED
#
# These helpers are deprecated in favor of detailed CI/CD statuses.
#
# See 'detailed_status?` method and `Gitlab::Ci::Status` module.
#
module CiStatusHelper
def ci_label_for_status(status)
if detailed_status?(status)
return status.label
end
label = case status
when 'success'
'passed'
when 'success-with-warnings'
'passed with warnings'
when 'manual'
'waiting for manual action'
when 'scheduled'
'waiting for delayed job'
else
status
end
translation = "CiStatusLabel|#{label}"
s_(translation)
end
def ci_text_for_status(status)
if detailed_status?(status)
return status.text
end
case status
when 'success'
s_('CiStatusText|passed')
when 'success-with-warnings'
s_('CiStatusText|passed')
when 'manual'
s_('CiStatusText|blocked')
when 'scheduled'
s_('CiStatusText|delayed')
else
# All states are already being translated inside the detailed statuses:
# :running => Gitlab::Ci::Status::Running
# :skipped => Gitlab::Ci::Status::Skipped
# :failed => Gitlab::Ci::Status::Failed
# :success => Gitlab::Ci::Status::Success
# :canceled => Gitlab::Ci::Status::Canceled
# The following states are customized above:
# :manual => Gitlab::Ci::Status::Manual
status_translation = "CiStatusText|#{status}"
s_(status_translation)
end
end
def ci_status_for_statuseable(subject)
status = subject.try(:status) || 'not found'
status.humanize
end
# rubocop:disable Metrics/CyclomaticComplexity
def ci_icon_for_status(status, size: 16)
if detailed_status?(status)
return sprite_icon(status.icon, size: size)
end
icon_name =
case status
when 'success'
'status_success'
when 'success-with-warnings'
'status_warning'
when 'failed'
'status_failed'
when 'pending'
'status_pending'
when 'waiting_for_resource'
'status_pending'
when 'preparing'
'status_preparing'
when 'running'
'status_running'
when 'play'
'play'
when 'created'
'status_created'
when 'skipped'
'status_skipped'
when 'manual'
'status_manual'
when 'scheduled'
'status_scheduled'
else
'status_canceled'
end
sprite_icon(icon_name, size: size)
end
# rubocop:enable Metrics/CyclomaticComplexity
def ci_icon_class_for_status(status)
group = detailed_status?(status) ? status.group : status.dasherize
"ci-status-icon-#{group}"
end
def pipeline_status_cache_key(pipeline_status)
"pipeline-status/#{pipeline_status.sha}-#{pipeline_status.status}"
end
def render_commit_status(commit, status, ref: nil, tooltip_placement: 'left')
project = commit.project
path = pipelines_project_commit_path(project, commit, ref: ref)
render_status_with_link(
status,
path,
tooltip_placement: tooltip_placement,
icon_size: 24)
end
def render_status_with_link(status, path = nil, type: _('pipeline'), tooltip_placement: 'left', cssclass: '', container: 'body', icon_size: 16)
klass = "ci-status-link #{ci_icon_class_for_status(status)} d-inline-flex #{cssclass}"
title = "#{type.titleize}: #{ci_label_for_status(status)}"
data = { toggle: 'tooltip', placement: tooltip_placement, container: container }
if path
link_to ci_icon_for_status(status, size: icon_size), path,
class: klass, title: title, data: data
else
content_tag :span, ci_icon_for_status(status, size: icon_size),
class: klass, title: title, data: data
end
end
def detailed_status?(status)
status.respond_to?(:text) &&
status.respond_to?(:group) &&
status.respond_to?(:label) &&
status.respond_to?(:icon)
end
end

View File

@ -7,14 +7,8 @@ module DeployTokensHelper
Rails.env.test?
end
def container_registry_enabled?(subject)
def container_registry_enabled?(project)
Gitlab.config.registry.enabled &&
can?(current_user, :read_container_image, subject)
end
def packages_registry_enabled?(subject)
Gitlab.config.packages.enabled &&
subject.feature_available?(:packages) &&
can?(current_user, :read_package, subject)
can?(current_user, :read_container_image, project)
end
end

View File

@ -35,7 +35,6 @@
= label_tag ("deploy_token_write_registry"), 'write_registry', class: 'label-bold form-check-label'
.text-secondary= s_('DeployTokens|Allows write access to the registry images')
- if packages_registry_enabled?(group_or_project)
%fieldset.form-group.form-check
= f.check_box :read_package_registry, class: 'form-check-input'
= label_tag ("deploy_token_read_package_registry"), 'read_package_registry', class: 'label-bold form-check-label'

View File

@ -0,0 +1,5 @@
---
title: Create time-space partitions in separate schema gitlab_partitions_dynamic
merge_request: 35137
author:
type: other

View File

@ -1,2 +1,5 @@
# Ignore table used temporarily in background migration
ActiveRecord::SchemaDumper.ignore_tables = ["untracked_files_for_uploads"]
# Ignore dynamically managed partitions in static application schema
ActiveRecord::SchemaDumper.ignore_tables += ["#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.*"]

View File

@ -0,0 +1,19 @@
# frozen_string_literal: true
class CreateDynamicPartitionsSchema < ActiveRecord::Migration[6.0]
include Gitlab::Database::SchemaHelpers
DOWNTIME = false
def up
execute 'CREATE SCHEMA gitlab_partitions_dynamic'
create_comment(:schema, :gitlab_partitions_dynamic, <<~EOS.strip)
Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.
EOS
end
def down
execute 'DROP SCHEMA gitlab_partitions_dynamic'
end
end

View File

@ -1,5 +1,9 @@
SET search_path=public;
CREATE SCHEMA gitlab_partitions_dynamic;
COMMENT ON SCHEMA gitlab_partitions_dynamic IS 'Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.';
CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
CREATE TABLE public.abuse_reports (
@ -14152,5 +14156,6 @@ COPY "schema_migrations" (version) FROM STDIN;
20200622235737
20200623000148
20200623000320
20200623121135
\.

View File

@ -31,7 +31,8 @@ From left to right, it displays:
![Redis profiling using the Performance Bar](img/performance_bar_redis_calls.png)
- **Elasticsearch calls**: the time taken (in milliseconds) and the total number of
Elasticsearch calls. Click to display a modal window with more details.
- **Load timings** of the page: several values in milliseconds, separated by slashes.
- **Load timings** of the page: if your browser supports load timings (Chromium
and Chrome) several values in milliseconds, separated by slashes.
Click to display a modal window with more details. The values, from left to right:
- **Backend**: time needed for the base page to load.
- [**First Contentful Paint**](https://web.dev/first-contentful-paint/):

View File

@ -232,6 +232,9 @@ To see the full list of API routes, you can run:
bundle exec rake grape:path_helpers
```
The generated list includes a full list of API endpoints and functional
RESTful API verbs.
For the Rails controllers, run:
```shell

View File

@ -27,12 +27,18 @@ module Backup
progress.print "Dumping PostgreSQL database #{config['database']} ... "
pg_env
pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
if Gitlab.config.backup.pg_schema
pgsql_args << "-n"
pgsql_args << '-n'
pgsql_args << Gitlab.config.backup.pg_schema
Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
pgsql_args << '-n'
pgsql_args << schema.to_s
end
end
spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
Process.spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
end
compress_wr.close

View File

@ -90,9 +90,7 @@ module Gitlab
end
def ordered_and_limited_query
query
.reorder(stage.end_event.timestamp_projection.desc)
.limit(MAX_RECORDS)
order_by_end_event(query).limit(MAX_RECORDS)
end
def records

View File

@ -18,10 +18,14 @@ module Gitlab
end
def timestamp_projection
Arel::Nodes::NamedFunction.new('COALESCE', [
Arel::Nodes::NamedFunction.new('COALESCE', column_list)
end
def column_list
[
issue_metrics_table[:first_associated_with_milestone_at],
issue_metrics_table[:first_added_to_board_at]
])
]
end
# rubocop: disable CodeReuse/ActiveRecord

View File

@ -10,6 +10,10 @@ module Gitlab
query.joins(:metrics)
end
# rubocop: enable CodeReuse/ActiveRecord
def column_list
[timestamp_projection]
end
end
end
end

View File

@ -18,10 +18,14 @@ module Gitlab
end
def timestamp_projection
Arel::Nodes::NamedFunction.new('COALESCE', [
Arel::Nodes::NamedFunction.new('COALESCE', column_list)
end
def column_list
[
issue_metrics_table[:first_associated_with_milestone_at],
issue_metrics_table[:first_added_to_board_at]
])
]
end
# rubocop: disable CodeReuse/ActiveRecord

View File

@ -21,6 +21,10 @@ module Gitlab
mr_metrics_table[:first_deployed_to_production_at]
end
def column_list
[timestamp_projection]
end
# rubocop: disable CodeReuse/ActiveRecord
def apply_query_customization(query)
query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at]))

View File

@ -32,6 +32,13 @@ module Gitlab
raise NotImplementedError
end
# List of columns that are referenced in the `timestamp_projection` expression
# Example timestamp projection: COALESCE(issue_metrics.created_at, issue_metrics.updated_at)
# Expected column list: issue_metrics.created_at, issue_metrics.updated_at
def column_list
[]
end
# Optionally a StageEvent may apply additional filtering or join other tables on the base query.
def apply_query_customization(query)
query

View File

@ -22,6 +22,29 @@ module Gitlab
stage.start_event.timestamp_projection
)
end
# rubocop: disable CodeReuse/ActiveRecord
def order_by_end_event(query)
ordered_query = query.reorder(stage.end_event.timestamp_projection.desc)
# When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause
if requires_grouping?
column_list = [
ordered_query.arel_table[:id],
*stage.end_event.column_list,
*stage.start_event.column_list
]
ordered_query = ordered_query.group(column_list)
end
ordered_query
end
# rubocop: enable CodeReuse/ActiveRecord
def requires_grouping?
Array(params[:label_name]).size > 1
end
end
end
end

View File

@ -22,6 +22,13 @@ module Gitlab
MIN_SCHEMA_VERSION = 20190506135400
MIN_SCHEMA_GITLAB_VERSION = '11.11.0'
# Schema we store dynamically managed partitions in
DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic
# This is an extensive list of postgres schemas owned by GitLab
# It does not include the default public schema
EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA].freeze
define_histogram :gitlab_database_transaction_seconds do
docstring "Time spent in database transactions, in seconds"
end

View File

@ -152,7 +152,7 @@ module Gitlab
end
def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound)
if table_exists?(partition_name)
if table_exists?(table_for_range_partition(partition_name))
# rubocop:disable Gitlab/RailsLogger
Rails.logger.warn "Partition not created because it already exists" \
" (this may be due to an aborted migration or similar): partition_name: #{partition_name}"

View File

@ -84,9 +84,13 @@ module Gitlab
private
def table_for_range_partition(partition_name)
"#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partition_name}"
end
def create_range_partition(partition_name, table_name, lower_bound, upper_bound)
execute(<<~SQL)
CREATE TABLE #{partition_name} PARTITION OF #{table_name}
CREATE TABLE #{table_for_range_partition(partition_name)} PARTITION OF #{table_name}
FOR VALUES FROM (#{lower_bound}) TO (#{upper_bound})
SQL
end

View File

@ -39,6 +39,11 @@ namespace :gitlab do
# PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
# Add `IF EXISTS` because cascade could have already deleted a table.
tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
# Drop all extra schema objects GitLab owns
Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)}")
end
end
desc 'GitLab | DB | Configures the database by running migrate, or by loading the schema and seeding if needed'

View File

@ -19057,6 +19057,9 @@ msgstr ""
msgid "Reporting"
msgstr ""
msgid "Reports"
msgstr ""
msgid "Reports|%{combinedString} and %{resolvedString}"
msgstr ""

View File

@ -202,6 +202,36 @@ RSpec.describe 'Database schema' do
end
end
context 'existence of Postgres schemas' do
def get_schemas
sql = <<~SQL
SELECT schema_name FROM
information_schema.schemata
WHERE
NOT schema_name ~* '^pg_' AND NOT schema_name = 'information_schema'
AND catalog_name = current_database()
SQL
ApplicationRecord.connection.select_all(sql).map do |row|
row['schema_name']
end
end
it 'we have a public schema' do
expect(get_schemas).to include('public')
end
Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
it "we have a '#{schema}' schema'" do
expect(get_schemas).to include(schema.to_s)
end
end
it 'we do not have unexpected schemas' do
expect(get_schemas.size).to eq(Gitlab::Database::EXTRA_SCHEMAS.size + 1)
end
end
private
def retrieve_columns_name_with_jsonb

View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe CiStatusHelper do
RSpec.describe Ci::StatusHelper do
include IconsHelper
let(:success_commit) { double("Ci::Pipeline", status: 'success') }

View File

@ -0,0 +1,67 @@
# frozen_string_literal: true
require 'spec_helper'
describe Backup::Database do
let(:progress) { double('progress', print: nil, puts: nil) }
describe '#dump' do
subject { described_class.new(progress).dump }
let(:pg_schema) { nil }
let(:backup_config) { double('config', pg_schema: pg_schema, path: File.join(Rails.root, 'tmp')) }
before do
allow(Settings).to receive(:backup).and_return(backup_config)
allow(Process).to receive(:waitpid)
end
it 'does not limit pg_dump to any specific schema' do
expect(Process).to receive(:spawn) do |*cmd, _|
expect(cmd.join(' ')).not_to include('-n')
end
subject
end
it 'includes option to drop objects before restoration' do
expect(Process).to receive(:spawn) do |*cmd, _|
expect(cmd.join(' ')).to include('--clean')
end
subject
end
context 'with pg_schema configured explicitly' do
let(:pg_schema) { 'some_schema' }
it 'calls pg_dump' do
expect(Process).to receive(:spawn) do |*cmd, _|
expect(cmd.join(' ')).to start_with('pg_dump')
end
subject
end
it 'limits the psql dump to the specified schema' do
expect(Process).to receive(:spawn) do |*cmd, _|
expect(cmd.join(' ')).to include("-n #{pg_schema}")
end
subject
end
context 'extra schemas' do
Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
it "includes the extra schema #{schema}" do
expect(Process).to receive(:spawn) do |*cmd, _|
expect(cmd.join(' ')).to include("-n #{schema}")
end
subject
end
end
end
end
end
end

View File

@ -275,7 +275,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
describe '#drop_partitioned_table_for' do
let(:expected_tables) do
%w[000000 201912 202001 202002].map { |suffix| "#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
%w[000000 201912 202001 202002].map { |suffix| "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
end
context 'when the table is not allowed' do

View File

@ -7,6 +7,14 @@ RSpec.describe Gitlab::Database do
stub_const('MigrationTest', Class.new { include Gitlab::Database })
end
describe 'EXTRA_SCHEMAS' do
it 'contains only schemas starting with gitlab_ prefix' do
described_class::EXTRA_SCHEMAS.each do |schema|
expect(schema.to_s).to start_with('gitlab_')
end
end
end
describe '.config' do
it 'returns a Hash' do
expect(described_class.config).to be_an_instance_of(Hash)

View File

@ -9,7 +9,7 @@ module PartitioningHelpers
end
def expect_range_partition_of(partition_name, table_name, min_value, max_value)
definition = find_partition_definition(partition_name)
definition = find_partition_definition(partition_name, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)
expect(definition).not_to be_nil
expect(definition['base_table']).to eq(table_name.to_s)
@ -40,7 +40,7 @@ module PartitioningHelpers
SQL
end
def find_partition_definition(partition)
def find_partition_definition(partition, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)
connection.select_one(<<~SQL)
select
parent_class.relname as base_table,
@ -48,7 +48,10 @@ module PartitioningHelpers
from pg_class
inner join pg_inherits i on pg_class.oid = inhrelid
inner join pg_class parent_class on parent_class.oid = inhparent
where pg_class.relname = '#{partition}' and pg_class.relispartition;
inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace
where pg_namespace.nspname = '#{schema}'
and pg_class.relname = '#{partition}'
and pg_class.relispartition
SQL
end
end

View File

@ -8,6 +8,7 @@ RSpec.shared_examples_for 'cycle analytics event' do
it { expect(described_class.identifier).to be_a_kind_of(Symbol) }
it { expect(instance.object_type.ancestors).to include(ApplicationRecord) }
it { expect(instance).to respond_to(:timestamp_projection) }
it { expect(instance.column_list).to be_a_kind_of(Array) }
describe '#apply_query_customization' do
it 'expects an ActiveRecord::Relation object as argument and returns a modified version of it' do