Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2025-05-09 21:10:50 +00:00
parent 0815945e44
commit 94356b7dcd
60 changed files with 1057 additions and 436 deletions

View File

@ -1260,7 +1260,6 @@ Gitlab/BoundedContexts:
- 'app/policies/commit_signatures/x509_commit_signature_policy.rb'
- 'app/policies/commit_status_policy.rb'
- 'app/policies/concerns/archived_abilities.rb'
- 'app/policies/concerns/crud_policy_helpers.rb'
- 'app/policies/concerns/find_group_projects.rb'
- 'app/policies/concerns/member_policy_helpers.rb'
- 'app/policies/concerns/policy_actor.rb'

View File

@ -3346,7 +3346,6 @@ RSpec/FeatureCategory:
- 'spec/policies/clusters/cluster_policy_spec.rb'
- 'spec/policies/clusters/instance_policy_spec.rb'
- 'spec/policies/commit_policy_spec.rb'
- 'spec/policies/concerns/crud_policy_helpers_spec.rb'
- 'spec/policies/container_expiration_policy_policy_spec.rb'
- 'spec/policies/custom_emoji_policy_spec.rb'
- 'spec/policies/deploy_keys_project_policy_spec.rb'

View File

@ -644,7 +644,7 @@
{"name":"rubocop-rspec","version":"3.0.5","platform":"ruby","checksum":"c6a8e29fb1b00d227c32df159e92f5ebb9e0ff734e52955fb13aff5c74977e0f"},
{"name":"rubocop-rspec_rails","version":"2.30.0","platform":"ruby","checksum":"888112e83f9d7ef7ad2397e9d69a0b9614a4bae24f072c399804a180f80c4c46"},
{"name":"ruby-fogbugz","version":"0.3.0","platform":"ruby","checksum":"5e04cde474648f498a71cf1e1a7ab42c66b953862fbe224f793ec0a7a1d5f657"},
{"name":"ruby-lsp","version":"0.23.13","platform":"ruby","checksum":"a1875a9905a79a41c63d8df52bd016f238d635b64c8f0aac3639336bcf659f48"},
{"name":"ruby-lsp","version":"0.23.15","platform":"ruby","checksum":"5e3dd3e775ba477854e577dc4aa5f0d3d59f32d90f8622787f01080d4e84e09f"},
{"name":"ruby-lsp-rails","version":"0.3.31","platform":"ruby","checksum":"670aed466e54b5632e4907b8dedb91d8b144917c42513e013d656af175bf8c76"},
{"name":"ruby-lsp-rspec","version":"0.1.22","platform":"ruby","checksum":"e982edf5cd6ec1530c3f5fa7e423624ad00532ebeff7fc94e02c7516a9b759c0"},
{"name":"ruby-magic","version":"0.6.0","platform":"ruby","checksum":"7b2138877b7d23aff812c95564eba6473b74b815ef85beb0eb792e729a2b6101"},

View File

@ -42,6 +42,7 @@ PATH
activerecord (>= 7)
activesupport (>= 7)
addressable (~> 2.8)
bigdecimal (~> 3.1)
concurrent-ruby (~> 1.1)
faraday (~> 2)
google-cloud-storage_transfer (~> 1.2.0)
@ -52,6 +53,7 @@ PATH
jwt (~> 2.5)
logger (~> 1.5)
minitest (~> 5.11.0)
mutex_m (~> 0.3)
parallel (~> 1.19)
pg (~> 1.5.6)
rack (~> 2.2.9)
@ -1741,7 +1743,7 @@ GEM
ruby-fogbugz (0.3.0)
crack (~> 0.4)
multipart-post (~> 2.0)
ruby-lsp (0.23.13)
ruby-lsp (0.23.15)
language_server-protocol (~> 3.17.0)
prism (>= 1.2, < 2.0)
rbs (>= 3, < 4)

View File

@ -644,7 +644,7 @@
{"name":"rubocop-rspec","version":"3.0.5","platform":"ruby","checksum":"c6a8e29fb1b00d227c32df159e92f5ebb9e0ff734e52955fb13aff5c74977e0f"},
{"name":"rubocop-rspec_rails","version":"2.30.0","platform":"ruby","checksum":"888112e83f9d7ef7ad2397e9d69a0b9614a4bae24f072c399804a180f80c4c46"},
{"name":"ruby-fogbugz","version":"0.3.0","platform":"ruby","checksum":"5e04cde474648f498a71cf1e1a7ab42c66b953862fbe224f793ec0a7a1d5f657"},
{"name":"ruby-lsp","version":"0.23.13","platform":"ruby","checksum":"a1875a9905a79a41c63d8df52bd016f238d635b64c8f0aac3639336bcf659f48"},
{"name":"ruby-lsp","version":"0.23.15","platform":"ruby","checksum":"5e3dd3e775ba477854e577dc4aa5f0d3d59f32d90f8622787f01080d4e84e09f"},
{"name":"ruby-lsp-rails","version":"0.3.31","platform":"ruby","checksum":"670aed466e54b5632e4907b8dedb91d8b144917c42513e013d656af175bf8c76"},
{"name":"ruby-lsp-rspec","version":"0.1.22","platform":"ruby","checksum":"e982edf5cd6ec1530c3f5fa7e423624ad00532ebeff7fc94e02c7516a9b759c0"},
{"name":"ruby-magic","version":"0.6.0","platform":"ruby","checksum":"7b2138877b7d23aff812c95564eba6473b74b815ef85beb0eb792e729a2b6101"},

View File

@ -42,6 +42,7 @@ PATH
activerecord (>= 7)
activesupport (>= 7)
addressable (~> 2.8)
bigdecimal (~> 3.1)
concurrent-ruby (~> 1.1)
faraday (~> 2)
google-cloud-storage_transfer (~> 1.2.0)
@ -52,6 +53,7 @@ PATH
jwt (~> 2.5)
logger (~> 1.5)
minitest (~> 5.11.0)
mutex_m (~> 0.3)
parallel (~> 1.19)
pg (~> 1.5.6)
rack (~> 2.2.9)
@ -1741,7 +1743,7 @@ GEM
ruby-fogbugz (0.3.0)
crack (~> 0.4)
multipart-post (~> 2.0)
ruby-lsp (0.23.13)
ruby-lsp (0.23.15)
language_server-protocol (~> 3.17.0)
prism (>= 1.2, < 2.0)
rbs (>= 3, < 4)

View File

@ -1,52 +1,93 @@
<script>
import { GlTooltipDirective, GlButton, GlAvatarLink, GlAvatarLabeled, GlTooltip } from '@gitlab/ui';
import {
GlTooltipDirective,
GlButton,
GlAvatarLink,
GlAvatarLabeled,
GlTooltip,
GlLoadingIcon,
} from '@gitlab/ui';
import { createAlert } from '~/alert';
import SafeHtml from '~/vue_shared/directives/safe_html';
import { isGid, getIdFromGraphQLId } from '~/graphql_shared/utils';
import { TYPENAME_CI_BUILD } from '~/graphql_shared/constants';
import { isGid, getIdFromGraphQLId, convertToGraphQLId } from '~/graphql_shared/utils';
import { glEmojiTag } from '~/emoji';
import { __, sprintf } from '~/locale';
import { __, s__, sprintf } from '~/locale';
import PageHeading from '~/vue_shared/components/page_heading.vue';
import CiIcon from '~/vue_shared/components/ci_icon/ci_icon.vue';
import TimeagoTooltip from '~/vue_shared/components/time_ago_tooltip.vue';
import getJobQuery from '../graphql/queries/get_job.query.graphql';
export default {
components: {
CiIcon,
TimeagoTooltip,
GlButton,
GlAvatarLink,
GlAvatarLabeled,
GlAvatarLink,
GlButton,
GlLoadingIcon,
GlTooltip,
PageHeading,
TimeagoTooltip,
},
directives: {
GlTooltip: GlTooltipDirective,
SafeHtml,
},
EMOJI_REF: 'EMOJI_REF',
inject: {
projectPath: {
default: '',
},
},
props: {
status: {
type: Object,
required: true,
},
name: {
type: String,
required: true,
},
time: {
type: String,
jobId: {
type: Number,
required: true,
},
user: {
type: Object,
required: true,
},
shouldRenderTriggeredLabel: {
type: Boolean,
required: true,
},
apollo: {
job: {
query: getJobQuery,
pollInterval: 10000,
variables() {
return {
fullPath: this.projectPath,
id: convertToGraphQLId(TYPENAME_CI_BUILD, this.jobId),
};
},
update({ project }) {
return project.job;
},
error(error) {
createAlert({
message: s__('Job|An error occurred while fetching the job header data.'),
captureError: true,
error,
});
},
},
},
data() {
return {
job: null,
};
},
computed: {
loading() {
return this.$apollo.queries.job.loading;
},
detailedStatus() {
return this.job?.detailedStatus || {};
},
shouldRenderTriggeredLabel() {
return Boolean(this.job.startedAt);
},
time() {
return this.job.startedAt || this.job.createdAt;
},
userAvatarAltText() {
return sprintf(__(`%{username}'s avatar`), { username: this.user.name });
},
@ -79,69 +120,72 @@ export default {
return isGid(this.user?.id) ? getIdFromGraphQLId(this.user?.id) : this.user?.id;
},
},
methods: {
onClickSidebarButton() {
this.$emit('clickedSidebarButton');
},
},
safeHtmlConfig: { ADD_TAGS: ['gl-emoji'] },
EMOJI_REF: 'EMOJI_REF',
};
</script>
<template>
<page-heading v-if="name" data-testid="job-header-content">
<template #heading>
<span data-testid="job-name">{{ name }}</span>
</template>
<template #actions>
<gl-button
:aria-label="__('Toggle sidebar')"
category="secondary"
class="gl-ml-2 lg:gl-hidden"
icon="chevron-double-lg-left"
@click="onClickSidebarButton"
/>
</template>
<template #description>
<ci-icon class="gl-mr-1" :status="status" show-status-text />
<template v-if="shouldRenderTriggeredLabel">{{ __('Started') }}</template>
<template v-else>{{ __('Created') }}</template>
<timeago-tooltip :time="time" />
{{ __('by') }}
<template v-if="user">
<gl-avatar-link
:data-user-id="userId"
:data-username="user.username"
:data-name="user.name"
:href="webUrl"
target="_blank"
class="js-user-link gl-mx-2 gl-items-center gl-align-middle"
>
<gl-avatar-labeled
:size="24"
:src="avatarUrl"
:label="user.name"
class="gl-hidden sm:gl-inline-flex"
/>
<strong class="author gl-inline sm:gl-hidden">@{{ user.username }}</strong>
<gl-tooltip v-if="message" :target="() => $refs[$options.EMOJI_REF]">
{{ message }}
</gl-tooltip>
<span
v-if="statusTooltipHTML"
:ref="$options.EMOJI_REF"
v-safe-html:[$options.safeHtmlConfig]="statusTooltipHTML"
class="gl-ml-2"
:data-testid="message"
></span>
</gl-avatar-link>
<gl-loading-icon v-if="loading" class="my-5" size="lg" />
<div v-else>
<page-heading v-if="job" data-testid="job-header-content">
<template #heading>
<span data-testid="job-name">{{ job.name }}</span>
</template>
</template>
</page-heading>
<template #actions>
<gl-button
:aria-label="__('Toggle sidebar')"
category="secondary"
class="gl-ml-2 lg:gl-hidden"
icon="chevron-double-lg-left"
@click="onClickSidebarButton"
/>
</template>
<template #description>
<ci-icon class="gl-mr-1" :status="detailedStatus" show-status-text />
<template v-if="shouldRenderTriggeredLabel">{{ __('Started') }}</template>
<template v-else>{{ __('Created') }}</template>
<timeago-tooltip :time="time" />
{{ __('by') }}
<template v-if="user">
<gl-avatar-link
:data-user-id="userId"
:data-username="user.username"
:data-name="user.name"
:href="webUrl"
target="_blank"
class="js-user-link gl-mx-2 gl-items-center gl-align-middle"
>
<gl-avatar-labeled
:size="24"
:src="avatarUrl"
:label="user.name"
class="gl-hidden sm:gl-inline-flex"
/>
<strong class="author gl-inline sm:gl-hidden">@{{ user.username }}</strong>
<gl-tooltip v-if="message" :target="() => $refs[$options.EMOJI_REF]">
{{ message }}
</gl-tooltip>
<span
v-if="statusTooltipHTML"
:ref="$options.EMOJI_REF"
v-safe-html:[$options.safeHtmlConfig]="statusTooltipHTML"
class="gl-ml-2"
:data-testid="message"
></span>
</gl-avatar-link>
</template>
</template>
</page-heading>
</div>
</template>

View File

@ -1,4 +1,5 @@
#import "~/ci/job_details/graphql/fragments/ci_job.fragment.graphql"
#import "~/graphql_shared/fragments/ci_icon.fragment.graphql"
query getJob($fullPath: ID!, $id: JobID!) {
project(fullPath: $fullPath) {
@ -7,6 +8,11 @@ query getJob($fullPath: ID!, $id: JobID!) {
...BaseCiJob
manualJob
name
detailedStatus {
...CiIcon
}
startedAt
createdAt
}
}
}

View File

@ -92,10 +92,8 @@ export default {
'fullScreenEnabled',
]),
...mapGetters([
'headerTime',
'hasUnmetPrerequisitesFailure',
'shouldRenderCalloutMessage',
'shouldRenderTriggeredLabel',
'hasEnvironment',
'shouldRenderSharedRunnerLimitWarning',
'hasJobLog',
@ -237,14 +235,7 @@ export default {
<div class="build-page" data-testid="job-content">
<!-- Header Section -->
<header>
<job-header
:status="job.status"
:time="headerTime"
:user="job.user"
:should-render-triggered-label="shouldRenderTriggeredLabel"
:name="jobName"
@clickedSidebarButton="toggleSidebar"
/>
<job-header :job-id="job.id" :user="job.user" @clickedSidebarButton="toggleSidebar" />
<gl-alert
v-if="shouldRenderHeaderCallout"
variant="danger"

View File

@ -2,8 +2,6 @@ import { isEmpty } from 'lodash';
import { isScrolledToBottom } from '~/lib/utils/scroll_utils';
import { checkJobHasLog } from './utils';
export const headerTime = (state) => state.job.started_at || state.job.created_at;
export const hasForwardDeploymentFailure = (state) =>
state?.job?.failure_reason === 'forward_deployment_failure';
@ -13,12 +11,6 @@ export const hasUnmetPrerequisitesFailure = (state) =>
export const shouldRenderCalloutMessage = (state) =>
!isEmpty(state.job.status) && !isEmpty(state.job.callout_message);
/**
* When the job has not started the value of job.started_at will be null
* When job has started the value of job.started_at will be a string with a date.
*/
export const shouldRenderTriggeredLabel = (state) => Boolean(state.job.started_at);
export const hasEnvironment = (state) => !isEmpty(state.job.deployment_status);
/**

View File

@ -6,5 +6,3 @@ module ReadmeHelper
{}
end
end
ReadmeHelper.prepend_mod_with("ReadmeHelper")

View File

@ -1,22 +0,0 @@
# frozen_string_literal: true
module CrudPolicyHelpers
extend ActiveSupport::Concern
class_methods do
def create_update_admin_destroy(name)
[
*create_update_admin(name),
:"destroy_#{name}"
]
end
def create_update_admin(name)
[
:"create_#{name}",
:"update_#{name}",
:"admin_#{name}"
]
end
end
end

View File

@ -1,7 +1,6 @@
# frozen_string_literal: true
class ProjectPolicy < BasePolicy
include CrudPolicyHelpers
include ArchivedAbilities
desc "Project has public builds enabled"
@ -739,7 +738,11 @@ class ProjectPolicy < BasePolicy
prevent(*archived_abilities)
archived_features.each do |feature|
prevent(*create_update_admin(feature))
prevent(
:"create_#{feature}",
:"update_#{feature}",
:"admin_#{feature}"
)
end
end

View File

@ -11,15 +11,25 @@ rspec:
BUNDLE_FROZEN: "false"
POSTGRES_USER: gitlab
POSTGRES_PASSWORD: password
POSTGRES_VERSION: 14
POSTGRES_VERSION: 16
PGPASSWORD: "${POSTGRES_PASSWORD}"
services:
- name: postgres:${POSTGRES_VERSION}
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
needs:
- project: $CI_PROJECT_PATH
job: "db:setup pg16"
ref: "master"
artifacts: true
before_script:
- apt update && apt install -y postgresql-client
- echo -e "\e[0Ksection_start:`date +%s`:postgresql16\r\e[0KInstalling PostgreSQL 16"
- curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc|gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg
- echo "deb https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" > /etc/apt/sources.list.d/pgdg.list
- apt update && apt install -y postgresql-client-16
- echo -e "\e[0Ksection_end:`date +%s`:postgresql16\r\e[0K"
- psql -h postgres -U $POSTGRES_USER -c 'create database gitlabhq_test;'
- psql -h postgres -U $POSTGRES_USER -c 'create database gitlabhq_ci_test;'
- psql -h postgres -U $POSTGRES_USER -q < pg_dumpall.sql > /dev/null
- cp gems/gitlab-backup-cli/spec/fixtures/config/database.yml config/
- "sed -i \"s/username: postgres$/username: $POSTGRES_USER/g\" config/database.yml"
- "sed -i \"s/password:\\s*$/password: $POSTGRES_PASSWORD/g\" config/database.yml"

View File

@ -13,6 +13,7 @@ PATH
activerecord (>= 7)
activesupport (>= 7)
addressable (~> 2.8)
bigdecimal (~> 3.1)
concurrent-ruby (~> 1.1)
faraday (~> 2)
google-cloud-storage_transfer (~> 1.2.0)
@ -23,6 +24,7 @@ PATH
jwt (~> 2.5)
logger (~> 1.5)
minitest (~> 5.11.0)
mutex_m (~> 0.3)
parallel (~> 1.19)
pg (~> 1.5.6)
rack (~> 2.2.9)
@ -47,6 +49,7 @@ GEM
public_suffix (>= 2.0.2, < 7.0)
ast (2.4.2)
base64 (0.2.0)
bigdecimal (3.1.9)
coderay (1.1.3)
concurrent-ruby (1.3.4)
diff-lcs (1.5.1)
@ -131,6 +134,7 @@ GEM
method_source (1.1.0)
minitest (5.11.3)
multi_json (1.15.0)
mutex_m (0.3.0)
net-http (0.4.1)
uri
os (1.1.4)
@ -232,6 +236,7 @@ CHECKSUMS
addressable (2.8.7) sha256=462986537cf3735ab5f3c0f557f14155d778f4b43ea4f485a9deb9c8f7c58232
ast (2.4.2) sha256=1e280232e6a33754cde542bc5ef85520b74db2aac73ec14acef453784447cc12
base64 (0.2.0) sha256=0f25e9b21a02a0cc0cea8ef92b2041035d39350946e8789c562b2d1a3da01507
bigdecimal (3.1.9)
coderay (1.1.3) sha256=dc530018a4684512f8f38143cd2a096c9f02a1fc2459edcfe534787a7fc77d4b
concurrent-ruby (1.3.4) sha256=d4aa926339b0a86b5b5054a0a8c580163e6f5dcbdfd0f4bb916b1a2570731c32
diff-lcs (1.5.1) sha256=273223dfb40685548436d32b4733aa67351769c7dea621da7d9dd4813e63ddfe
@ -271,6 +276,7 @@ CHECKSUMS
method_source (1.1.0) sha256=181301c9c45b731b4769bc81e8860e72f9161ad7d66dd99103c9ab84f560f5c5
minitest (5.11.3) sha256=78e18aa2c49c58e9bc53c54a0b900e87ad0a96394e92fbbfa58d3ff860a68f45
multi_json (1.15.0) sha256=1fd04138b6e4a90017e8d1b804c039031399866ff3fbabb7822aea367c78615d
mutex_m (0.3.0)
net-http (0.4.1) sha256=a96efc5ea18bcb9715e24dda4159d10f67ff0345c8a980d04630028055b2c282
os (1.1.4) sha256=57816d6a334e7bd6aed048f4b0308226c5fb027433b67d90a9ab435f35108d3f
parallel (1.26.3) sha256=d86babb7a2b814be9f4b81587bf0b6ce2da7d45969fab24d8ae4bf2bb4d4c7ef

View File

@ -0,0 +1,14 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
$LOAD_PATH.unshift File.expand_path('lib', __dir__)
# GITLAB_PATH points to the Rails.root, which the tool can use to load
# the Rails environment when necessary or to help find configuration files
# when used with GDK
GITLAB_PATH = File.expand_path('../../../', __dir__)
require 'bundler/setup'
require 'gitlab/backup/cli'
Gitlab::Backup::Cli.start(ARGV)

View File

@ -26,8 +26,10 @@ Gem::Specification.new do |spec|
spec.add_dependency "activerecord", ">= 7"
spec.add_dependency "activesupport", ">= 7"
spec.add_dependency "bigdecimal", "~> 3.1"
spec.add_dependency "googleauth", "~> 1.8.1" # https://gitlab.com/gitlab-org/gitlab/-/issues/449019
spec.add_dependency "google-cloud-storage_transfer", "~> 1.2.0"
spec.add_dependency "mutex_m", "~> 0.3"
spec.add_dependency "pg", "~> 1.5.6"
spec.add_dependency "rainbow", "~> 3.0"
spec.add_dependency "thor", "~> 1.3"

View File

@ -5,8 +5,10 @@
# want to use the Rainbow refinement in the gem code going forward, but
# while we have this dependency, we need this external require
require "rainbow/ext/string"
require 'active_support/all'
require 'active_record'
require 'active_support/all' # Used to provide timezone support on timestamp among other things
require 'active_record' # Used to connect to database views to help run gitaly backups
require 'tmpdir' # Used to create temporary folders during backup
require 'base64' # Used by gitaly backup client
module Gitlab
module Backup
@ -21,8 +23,8 @@ module Gitlab
autoload :Errors, 'gitlab/backup/cli/errors'
autoload :GitlabConfig, 'gitlab/backup/cli/gitlab_config'
autoload :Metadata, 'gitlab/backup/cli/metadata'
autoload :Models, 'gitlab/backup/cli/models'
autoload :Output, 'gitlab/backup/cli/output'
autoload :RepoType, 'gitlab/backup/cli/repo_type'
autoload :RestoreExecutor, 'gitlab/backup/cli/restore_executor'
autoload :Runner, 'gitlab/backup/cli/runner'
autoload :Shell, 'gitlab/backup/cli/shell'
@ -51,14 +53,6 @@ module Gitlab
def self.root
Pathname.new(File.expand_path(File.join(__dir__, '../../../')))
end
def self.rails_environment!
require File.join(GITLAB_PATH, 'config/application')
Rails.application.require_environment!
Rails.application.autoloaders
Rails.application.load_tasks
end
end
end
end

View File

@ -33,6 +33,7 @@ module Gitlab
end
def execute
initialize_database_connection!
execute_all_tasks
write_metadata!
@ -46,6 +47,10 @@ module Gitlab
private
def initialize_database_connection!
Models::Base.initialize_connection!(context: context)
end
def build_metadata
@metadata = Gitlab::Backup::Cli::Metadata::BackupMetadata.build(gitlab_version: context.gitlab_version)
end

View File

@ -11,12 +11,6 @@ module Gitlab
def all
Gitlab::Backup::Cli.update_process_title!('backup all')
duration = measure_duration do
Gitlab::Backup::Cli::Output.info("Initializing environment...")
Gitlab::Backup::Cli.rails_environment!
end
Gitlab::Backup::Cli::Output.success("Environment loaded. (#{duration.in_seconds}s)")
backup_executor = Gitlab::Backup::Cli::BackupExecutor.new(
context: build_context,
backup_bucket: options["backup_bucket"],

View File

@ -11,12 +11,6 @@ module Gitlab
def all(backup_id)
Gitlab::Backup::Cli.update_process_title!("restore all from #{backup_id}")
duration = measure_duration do
Gitlab::Backup::Cli::Output.info("Initializing environment...")
Gitlab::Backup::Cli.rails_environment!
end
Gitlab::Backup::Cli::Output.success("Environment loaded. (#{duration.in_seconds}s)")
restore_executor =
Gitlab::Backup::Cli::RestoreExecutor.new(
context: build_context,

View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
autoload :Base, 'gitlab/backup/cli/models/base'
autoload :GroupWiki, 'gitlab/backup/cli/models/group_wiki'
autoload :PersonalSnippet, 'gitlab/backup/cli/models/personal_snippet'
autoload :ProjectDesignManagement, 'gitlab/backup/cli/models/project_design_management'
autoload :ProjectSnippet, 'gitlab/backup/cli/models/project_snippet'
autoload :Project, 'gitlab/backup/cli/models/project'
autoload :ProjectWiki, 'gitlab/backup/cli/models/project_wiki'
autoload :RepositoryStorage, 'gitlab/backup/cli/models/repository_storage'
end
end
end
end

View File

@ -0,0 +1,19 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
class Base < ActiveRecord::Base
self.abstract_class = true
def self.initialize_connection!(context:)
connection_params = Gitlab::Backup::Cli::Services::Postgres.new(context).main_database.connection_params
establish_connection(connection_params)
end
end
end
end
end
end

View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
class GroupWiki < Base
self.table_name = 'group_wikis_routes_view'
self.primary_key = :group_id
scope :default, -> { readonly }
def storage
@storage ||= RepositoryStorage.new(self, prefix: RepositoryStorage::GROUP_REPOSITORY_PATH_PREFIX)
end
def disk_path
"#{storage.disk_path}.wiki.git"
end
def path_with_namespace
"#{read_attribute(:path_with_namespace)}.wiki"
end
end
end
end
end
end

View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
class PersonalSnippet < Base
self.table_name = 'personal_snippets_view'
self.primary_key = :id
scope :default, -> { readonly }
def storage
@storage ||= RepositoryStorage.new(self, prefix: RepositoryStorage::SNIPPET_REPOSITORY_PATH_PREFIX)
end
def disk_path
"#{storage.disk_path}.git"
end
def path_with_namespace
"snippets/#{id}"
end
end
end
end
end
end

View File

@ -0,0 +1,24 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
class Project < Base
self.table_name = 'project_routes_view'
self.primary_key = :id
scope :default, -> { readonly }
def storage
@storage ||= RepositoryStorage.new(self, prefix: RepositoryStorage::REPOSITORY_PATH_PREFIX)
end
def disk_path
"#{storage.disk_path}.git"
end
end
end
end
end
end

View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
class ProjectDesignManagement < Base
self.table_name = 'project_design_management_routes_view'
self.primary_key = :id
scope :default, -> { readonly }
def storage
@storage ||= RepositoryStorage.new(self, prefix: RepositoryStorage::REPOSITORY_PATH_PREFIX)
end
def disk_path
"#{storage.disk_path}.design.git"
end
def path_with_namespace
"#{read_attribute(:path_with_namespace)}.design"
end
end
end
end
end
end

View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
class ProjectSnippet < Base
self.table_name = 'project_snippets_routes_view'
self.primary_key = :id
scope :default, -> { readonly }
def storage
@storage ||= RepositoryStorage.new(self, prefix: RepositoryStorage::SNIPPET_REPOSITORY_PATH_PREFIX)
end
def disk_path
"#{storage.disk_path}.git"
end
def path_with_namespace
"#{read_attribute(:path_with_namespace)}/snippets/#{id}"
end
end
end
end
end
end

View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
class ProjectWiki < Base
self.table_name = 'project_routes_view'
self.primary_key = :id
scope :default, -> { readonly }
def storage
@storage ||= RepositoryStorage.new(self, prefix: RepositoryStorage::REPOSITORY_PATH_PREFIX)
end
def disk_path
"#{storage.disk_path}.wiki.git"
end
def path_with_namespace
"#{read_attribute(:path_with_namespace)}.wiki"
end
end
end
end
end
end

View File

@ -0,0 +1,46 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Models
class RepositoryStorage
attr_accessor :container
REPOSITORY_PATH_PREFIX = '@hashed'
GROUP_REPOSITORY_PATH_PREFIX = '@groups'
SNIPPET_REPOSITORY_PATH_PREFIX = '@snippets'
POOL_PATH_PREFIX = '@pools'
def initialize(container, prefix:)
@container = container
@prefix = prefix
end
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
"#{@prefix}/#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash
end
# Disk path is used to build repository path on disk
#
# @return [String] combination of base_dir and the repository own name
# without `.git`, `.wiki.git`, or any other extension
def disk_path
"#{base_dir}/#{disk_hash}" if disk_hash
end
private
# Generates the hash for the repository path and name on disk
# If you need to refer to the repository on disk, use the `#disk_path`
def disk_hash
@disk_hash ||= OpenSSL::Digest::SHA256.hexdigest(container.id.to_s) if container.id
end
end
end
end
end
end

View File

@ -10,11 +10,11 @@ module Gitlab
using Rainbow
ICONS = {
info: "\u2139\ufe0f ", # requires an extra space
success: "\u2705\ufe0f",
warning: "\u26A0\ufe0f ", # requires an extra space
error: "\u274C\ufe0f",
debug: "\u26CF\ufe0f " # requires an extra space
info: "\u2139\ufe0f ",
success: "\u2705\ufe0f ",
warning: "\u26A0\ufe0f ",
error: "\u274C\ufe0f ",
debug: "\u26CF\ufe0f "
}.freeze
STATES = [

View File

@ -1,14 +0,0 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
class RepoType
PROJECT = :project
WIKI = :wiki
SNIPPET = :snippet
DESIGN = :design
end
end
end
end

View File

@ -36,6 +36,7 @@ module Gitlab
end
def execute
initialize_database_connection!
read_metadata!
execute_all_tasks
@ -52,6 +53,10 @@ module Gitlab
private
def initialize_database_connection!
Models::Base.initialize_connection!(context: context)
end
def execute_all_tasks
tasks = []
Gitlab::Backup::Cli::Tasks.build_each(context: context) do |task|

View File

@ -46,53 +46,23 @@ module Gitlab
"gitaly-backup exit status #{status.exitstatus}"
end
def enqueue(container, repo_type)
def enqueue(container, always_create: false)
raise Gitlab::Backup::Cli::Errors::GitalyBackupError, 'not started' unless started?
raise Gitlab::Backup::Cli::Errors::GitalyBackupError, 'no container for repo type' unless container
storage, relative_path, gl_project_path, always_create = repository_info_for(container, repo_type)
container_methods = [:disk_path, :storage, :path_with_namespace]
unless container_methods.all? { |method| container.respond_to?(method) }
raise Gitlab::Backup::Cli::Errors::GitalyBackupError, 'not a valid container'
end
storage = container.repository_storage
relative_path = container.disk_path
gl_project_path = container.path_with_namespace
schedule_backup_job(storage, relative_path, gl_project_path, always_create)
end
private
def repository_info_for(container, repo_type)
case repo_type
when RepoType::PROJECT
[container.repository_storage,
container.disk_path || container.full_path,
container.full_path,
true]
when RepoType::WIKI
wiki_repo_info(container)
when RepoType::SNIPPET
[container.repository_storage,
container.disk_path || container.full_path,
container.full_path,
false]
when RepoType::DESIGN
[design_repo_storage(container),
container.project.disk_path,
container.project.full_path,
false]
end
end
def design_repo_storage(container)
return container.repository.repository_storage if container.repository.respond_to?(:repository_storage)
container.repository_storage
end
def wiki_repo_info(container)
wiki = container.respond_to?(:wiki) ? container.wiki : container
[wiki.repository_storage,
wiki.disk_path || wiki.full_path,
wiki.full_path,
false]
end
def gitaly_backup_args(type, backup_repos_path, backup_id, remove_all_repositories)
command = case type
when :create

View File

@ -66,6 +66,47 @@ module Gitlab
Result.new(stdout: stdout, stderr: stderr, status: status, duration: duration)
end
# Execute a process and intercept its captured output line by line
#
# @example Usage
# Shell::Command.new('echo', 'Some amazing output').capture_each { |stream, output| puts output }
# @yieldparam [Symbol] stream type (either :stdout or :stderr)
# @yieldparam [String] output
# @return [Command::Result] -- Captured output from executing a process
def capture_each
start = Time.now
stdout = +''
stderr = +''
status = Open3.popen3(env, *cmd_args, chdir: chdir) do |i, o, e, wait_thread|
i.close_write
io_threads = []
io_threads << Thread.new do
until o.eof?
o.gets.tap do |output|
stdout << output
yield :stdout, output.chomp
end
end
end
io_threads << Thread.new do
until e.eof?
e.gets.tap do |error|
stderr << error
yield :stderr, error.chomp
end
end
end
io_threads.each(&:join)
wait_thread.value
end
duration = Time.now - start
Result.new(stdout: stdout, stderr: stderr, status: status, duration: duration)
end
# Run single command in pipeline mode with optional input or output redirection
#
# @param [IO|String|Array] input stdin redirection

View File

@ -8,9 +8,11 @@ module Gitlab
module Targets
# Backup and restores repositories by querying the database
class Repositories < Target
BATCH_SIZE = 1000
def dump(destination)
gitaly_backup.start(:create, destination)
enqueue_consecutive
enqueue_repositories
ensure
gitaly_backup.finish!
@ -18,7 +20,7 @@ module Gitlab
def restore(source)
gitaly_backup.start(:restore, source, remove_all_repositories: remove_all_repositories)
enqueue_consecutive
enqueue_repositories
ensure
gitaly_backup.finish!
@ -36,56 +38,54 @@ module Gitlab
context.config_repositories_storages.keys
end
def enqueue_consecutive
enqueue_consecutive_projects
enqueue_consecutive_snippets
def enqueue_repositories
enqueue_project_source_code
enqueue_project_wiki
enqueue_group_wiki
enqueue_project_design_management
enqueue_project_snippets
enqueue_personal_snippets
end
def enqueue_consecutive_projects
project_relation.find_each(batch_size: 1000) do |project|
enqueue_project(project)
def enqueue_project_source_code
Models::Project.find_each(batch_size: BATCH_SIZE) do |project|
gitaly_backup.enqueue(project, always_create: true)
end
end
def enqueue_consecutive_snippets
snippet_relation.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) }
def enqueue_project_wiki
Models::ProjectWiki.find_each(batch_size: BATCH_SIZE) do |project_wiki|
gitaly_backup.enqueue(project_wiki)
end
end
def enqueue_project(project)
gitaly_backup.enqueue(project, Gitlab::Backup::Cli::RepoType::PROJECT)
gitaly_backup.enqueue(project, Gitlab::Backup::Cli::RepoType::WIKI)
return unless project.design_management_repository
gitaly_backup.enqueue(project.design_management_repository, Gitlab::Backup::Cli::RepoType::DESIGN)
def enqueue_group_wiki
Models::GroupWiki.find_each(batch_size: BATCH_SIZE) do |group_wiki|
gitaly_backup.enqueue(group_wiki)
end
end
def enqueue_snippet(snippet)
gitaly_backup.enqueue(snippet, Gitlab::Backup::Cli::RepoType::SNIPPET)
def enqueue_project_design_management
Models::ProjectDesignManagement.find_each(batch_size: BATCH_SIZE) do |project_design_management|
gitaly_backup.enqueue(project_design_management)
end
end
def project_relation
Project.includes(:route, :group, :namespace)
def enqueue_project_snippets
Models::ProjectSnippet.find_each(batch_size: BATCH_SIZE) do |snippet|
gitaly_backup.enqueue(snippet)
end
end
def snippet_relation
Snippet.all
def enqueue_personal_snippets
Models::PersonalSnippet.find_each(batch_size: BATCH_SIZE) do |snippet|
gitaly_backup.enqueue(snippet)
end
end
def restore_object_pools
PoolRepository.includes(:source_project).find_each do |pool|
Output.info " - Object pool #{pool.disk_path}..."
unless pool.source_project
Output.info " - Object pool #{pool.disk_path}... [SKIPPED]"
next
end
pool.state = 'none'
pool.save
pool.schedule
end
pool = Gitlab::Backup::Cli::Utils::PoolRepositories.new(gitlab_basepath: context.gitlab_basepath)
pool.reinitialize!
end
end
end

View File

@ -6,6 +6,7 @@ module Gitlab
module Utils
autoload :Compression, 'gitlab/backup/cli/utils/compression'
autoload :PgDump, 'gitlab/backup/cli/utils/pg_dump'
autoload :PoolRepositories, 'gitlab/backup/cli/utils/pool_repositories'
autoload :Rake, 'gitlab/backup/cli/utils/rake'
autoload :Tar, 'gitlab/backup/cli/utils/tar'
end

View File

@ -0,0 +1,58 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Utils
class PoolRepositories
PoolReinitializationResult = Struct.new(:disk_path, :status, :error_message, keyword_init: true)
attr_reader :gitlab_basepath
def initialize(gitlab_basepath:)
@gitlab_basepath = gitlab_basepath
end
def reinitialize!
Gitlab::Backup::Cli::Output.info "Reinitializing object pools..."
rake = build_reset_task
rake.capture_each do |stream, output|
next Gitlab::Backup::Cli::Output.warning output if stream == :stderr
pool = parse_pool_results(output)
next Gitlab::Backup::Cli::Output.warning "Failed to parse: #{output}" unless pool
case pool.status.to_sym
when :scheduled
Gitlab::Backup::Cli::Output.success "Object pool #{pool.disk_path}..."
when :skipped
Gitlab::Backup::Cli::Output.info "Object pool #{pool.disk_path}... [SKIPPED]"
when :failed
Gitlab::Backup::Cli::Output.info "Object pool #{pool.disk_path}... [FAILED]"
Gitlab::Backup::Cli::Output.error(
"Object pool #{pool.disk_path} failed to reset (#{pool.error_message})")
end
end
end
private
def build_reset_task
Gitlab::Backup::Cli::Utils::Rake.new(
'gitlab:backup:repo:reset_pool_repositories',
chdir: gitlab_basepath)
end
def parse_pool_results(line)
return unless line.start_with?('{') && line.end_with?('}')
JSON.parse(line, object_class: PoolReinitializationResult)
rescue JSON::ParserError
nil
end
end
end
end
end
end

View File

@ -18,6 +18,8 @@ module Gitlab
@chdir = chdir
end
# Execute the rake task and return its execution result status
#
# @return [self]
def execute
Bundler.with_original_env do
@ -27,6 +29,22 @@ module Gitlab
self
end
# Execute the rake task and intercept its output line by line including a final result status
#
# @example Usage
# Rake.new('some:task').capture_each { |stream, output| puts output if stream == :stdout }
# @yield |stream, output| Return output from :stdout or :stderr stream line by line
# @yieldparam [Symbol] stream type (either :stdout or :stderr)
# @yieldparam [String] output content
# @return [Gitlab::Backup::Cli::Command::Result] -- Captured output from executing a process
def capture_each(&block)
Bundler.with_original_env do
@result = Shell::Command.new(*rake_command, chdir: chdir).capture_each(&block)
end
self
end
# Return whether the execution was a success or not
#
# @return [Boolean] whether the execution was a success

View File

@ -0,0 +1,8 @@
# frozen_string_literal: true
FactoryBot.define do
factory :personal_snippet, class: 'Gitlab::Backup::Cli::Models::PersonalSnippet' do
sequence(:id)
repository_storage { 'snippets_storage' }
end
end

View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
FactoryBot.define do
factory :project, class: 'Gitlab::Backup::Cli::Models::Project' do
sequence(:id)
repository_storage { 'storage' }
path_with_namespace { 'group/project' }
name_with_namespace { 'My Group / My Project' }
end
end

View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
FactoryBot.define do
factory :project_design_management, class: 'Gitlab::Backup::Cli::Models::ProjectDesignManagement' do
sequence(:id)
repository_storage { 'design_storage' }
path_with_namespace { 'group/project' }
name_with_namespace { 'My Group / My Project' }
end
end

View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
FactoryBot.define do
factory :project_snippet, class: 'Gitlab::Backup::Cli::Models::ProjectSnippet' do
sequence(:id)
repository_storage { 'snippets_storage' }
path_with_namespace { 'group/myproject' }
end
end

View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
FactoryBot.define do
factory :project_wiki, class: 'Gitlab::Backup::Cli::Models::ProjectWiki' do
sequence(:id)
repository_storage { 'wiki_storage' }
path_with_namespace { 'group/project' }
name_with_namespace { 'My Group / My Project' }
end
end

View File

@ -1,12 +1,15 @@
# frozen_string_literal: true
require 'spec_helper'
require 'open3'
RSpec.describe Gitlab::Backup::Cli::Services::GitalyBackup do
let(:context) { Gitlab::Backup::Cli::Context.build }
let(:context) { build_test_context }
let(:gitaly_backup) { described_class.new(context) }
before do
Gitlab::Backup::Cli::Models::Base.initialize_connection!(context: context)
end
describe '#start' do
context 'when creating a backup' do
it 'starts the gitaly-backup process with the correct arguments' do
@ -92,7 +95,7 @@ RSpec.describe Gitlab::Backup::Cli::Services::GitalyBackup do
context 'when not started' do
it 'raises an error' do
expect do
gitaly_backup.enqueue(double, :project)
gitaly_backup.enqueue(double)
end.to raise_error(Gitlab::Backup::Cli::Errors::GitalyBackupError, /not started/)
end
end
@ -106,92 +109,95 @@ RSpec.describe Gitlab::Backup::Cli::Services::GitalyBackup do
end
context 'with a project repository' do
let(:container) do
instance_double('Project', repository_storage: 'storage', disk_path: 'disk/path', full_path: 'group/project')
end
let(:container) { build(:project) }
it 'schedules a backup job with the correct parameters' do
expected_json = {
storage_name: 'storage',
relative_path: 'disk/path',
relative_path: container.disk_path,
gl_project_path: 'group/project',
always_create: true
}.to_json
expect(input_stream).to receive(:puts).with(expected_json)
gitaly_backup.enqueue(container, :project)
gitaly_backup.enqueue(container, always_create: true)
end
end
context 'with a wiki repository' do
let(:wiki) do
instance_double('Wiki', repository_storage: 'wiki_storage', disk_path: 'wiki/disk/path',
full_path: 'group/project.wiki')
end
let(:container) { instance_double('Project', wiki: wiki) }
let(:container) { build(:project_wiki) }
it 'schedules a backup job with the correct parameters' do
expected_json = {
storage_name: 'wiki_storage',
relative_path: 'wiki/disk/path',
relative_path: container.disk_path,
gl_project_path: 'group/project.wiki',
always_create: false
}.to_json
expect(input_stream).to receive(:puts).with(expected_json)
gitaly_backup.enqueue(container, :wiki)
gitaly_backup.enqueue(container)
end
end
context 'with a snippet repository' do
let(:container) do
instance_double('Snippet', repository_storage: 'storage', disk_path: 'disk/path', full_path: 'snippets/1')
end
context 'with a personal snippet repository' do
let(:container) { build(:personal_snippet, id: 1) }
it 'schedules a backup job with the correct parameters' do
expected_json = {
storage_name: 'storage',
relative_path: 'disk/path',
storage_name: 'snippets_storage',
relative_path: container.disk_path,
gl_project_path: 'snippets/1',
always_create: false
}.to_json
expect(input_stream).to receive(:puts).with(expected_json)
gitaly_backup.enqueue(container, :snippet)
gitaly_backup.enqueue(container)
end
end
context 'with a design repository' do
let(:project) { instance_double('Project', disk_path: 'disk/path', full_path: 'group/project') }
let(:container) do
instance_double('DesignRepository', project: project,
repository: instance_double('Repository', repository_storage: 'storage'))
end
context 'with a project snippet repository' do
let(:container) { build(:project_snippet, id: 1) }
it 'schedules a backup job with the correct parameters' do
expected_json = {
storage_name: 'storage',
relative_path: 'disk/path',
gl_project_path: 'group/project',
storage_name: 'snippets_storage',
relative_path: container.disk_path,
gl_project_path: 'group/myproject/snippets/1',
always_create: false
}.to_json
expect(input_stream).to receive(:puts).with(expected_json)
gitaly_backup.enqueue(container, :design)
gitaly_backup.enqueue(container)
end
end
context 'with an invalid repository type' do
context 'with a design repository' do
let(:container) { build(:project_design_management) }
it 'schedules a backup job with the correct parameters' do
expected_json = {
storage_name: 'design_storage',
relative_path: container.disk_path,
gl_project_path: 'group/project.design',
always_create: false
}.to_json
expect(input_stream).to receive(:puts).with(expected_json)
gitaly_backup.enqueue(container)
end
end
context 'with an invalid container class' do
it 'raises an error' do
expect do
gitaly_backup.enqueue(nil,
:invalid)
end.to raise_error(Gitlab::Backup::Cli::Errors::GitalyBackupError, /no container for repo type/)
gitaly_backup.enqueue(nil)
end.to raise_error(Gitlab::Backup::Cli::Errors::GitalyBackupError, /not a valid container/)
end
end
end

View File

@ -94,6 +94,61 @@ RSpec.describe Gitlab::Backup::Cli::Shell::Command do
end
end
describe '#capture_each' do
it 'streams each stdout line from executed command' do
content = "first line\nsecond line"
expected_output = [
[:stdout, 'first line'], [:stdout, 'second line']
]
output_streams = []
result = command.new('echo', content).capture_each do |stream, line|
output_streams << [stream, line]
end
expect(output_streams).to eq(expected_output)
expect(result.stdout.chomp).to eq(content)
end
it 'streams each stderr line from executed command' do
content = "first line\nsecond line"
input_file = tmpdir.join('input.txt')
File.open(input_file, 'w+') do |file|
file.write(content)
end
expected_output = [
[:stderr, 'first line'], [:stderr, 'second line']
]
output_streams = []
result = command.new("cat #{input_file} > /dev/stderr").capture_each do |stream, line|
output_streams << [stream, line]
end
expect(output_streams).to eq(expected_output)
expect(result.stderr.chomp).to eq(content)
end
it 'returns a Process::Status from the executed command' do
result = command.new('pwd').capture_each do |_, _|
# no-op
end
expect(result.status).to be_a(Process::Status)
expect(result.status).to respond_to(:exited?, :termsig, :stopsig, :exitstatus, :success?, :pid)
end
it 'returns the execution duration' do
result = command.new('sleep 0.1').capture_each do |_, _|
# no-op
end
expect(result.duration).to be > 0.1
end
end
describe '#run_single_pipeline!' do
it 'runs without any exceptions' do
expect { command.new('true').run_single_pipeline! }.not_to raise_exception

View File

@ -3,15 +3,19 @@
require 'spec_helper'
RSpec.describe Gitlab::Backup::Cli::Targets::Repositories do
let(:context) { Gitlab::Backup::Cli::Context.build }
let(:context) { build_test_context }
let(:gitaly_backup) { repo_target.gitaly_backup }
subject(:repo_target) { described_class.new(context) }
before do
Gitlab::Backup::Cli::Models::Base.initialize_connection!(context: context)
end
describe '#dump' do
it 'starts and finishes the gitaly_backup' do
expect(gitaly_backup).to receive(:start).with(:create, '/path/to/destination')
expect(repo_target).to receive(:enqueue_consecutive)
expect(repo_target).to receive(:enqueue_repositories)
expect(gitaly_backup).to receive(:finish!)
repo_target.dump('/path/to/destination')
@ -22,7 +26,7 @@ RSpec.describe Gitlab::Backup::Cli::Targets::Repositories do
it 'starts and finishes the gitaly_backup' do
expect(gitaly_backup).to receive(:start)
.with(:restore, '/path/to/destination', remove_all_repositories: ["default"])
expect(repo_target).to receive(:enqueue_consecutive)
expect(repo_target).to receive(:enqueue_repositories)
expect(gitaly_backup).to receive(:finish!)
expect(repo_target).to receive(:restore_object_pools)
@ -30,46 +34,94 @@ RSpec.describe Gitlab::Backup::Cli::Targets::Repositories do
end
end
describe '#enqueue_consecutive' do
it 'calls enqueue_consecutive_projects and enqueue_consecutive_snippets' do
expect(repo_target).to receive(:enqueue_consecutive_projects)
expect(repo_target).to receive(:enqueue_consecutive_snippets)
describe '#enqueue_repositories' do
it 'calls each resource respective enqueue methods', :aggregate_failures do
expect(repo_target).to receive(:enqueue_project_source_code)
expect(repo_target).to receive(:enqueue_project_wiki)
expect(repo_target).to receive(:enqueue_group_wiki)
expect(repo_target).to receive(:enqueue_project_design_management)
expect(repo_target).to receive(:enqueue_project_snippets)
expect(repo_target).to receive(:enqueue_personal_snippets)
repo_target.send(:enqueue_consecutive)
repo_target.send(:enqueue_repositories)
end
end
describe '#enqueue_project' do
let(:project) { instance_double('Project', design_management_repository: nil) }
describe '#enqueue_project_source_code' do
let(:resource) { Gitlab::Backup::Cli::Models::Project }
let(:repository) { object_double(resource.new) }
it 'enqueues project and wiki' do
expect(gitaly_backup).to receive(:enqueue).with(project, Gitlab::Backup::Cli::RepoType::PROJECT)
expect(gitaly_backup).to receive(:enqueue).with(project, Gitlab::Backup::Cli::RepoType::WIKI)
it 'enqueues project repository' do
allow(resource).to receive(:find_each).and_yield(repository)
repo_target.send(:enqueue_project, project)
end
expect(gitaly_backup).to receive(:enqueue).with(repository, always_create: true)
context 'when project has design management repository' do
let(:design_repo) { instance_double('DesignRepository') }
let(:project) { instance_double('Project', design_management_repository: design_repo) }
it 'enqueues project, wiki, and design' do
expect(gitaly_backup).to receive(:enqueue).with(project, Gitlab::Backup::Cli::RepoType::PROJECT)
expect(gitaly_backup).to receive(:enqueue).with(project, Gitlab::Backup::Cli::RepoType::WIKI)
expect(gitaly_backup).to receive(:enqueue).with(design_repo, Gitlab::Backup::Cli::RepoType::DESIGN)
repo_target.send(:enqueue_project, project)
end
repo_target.send(:enqueue_project_source_code)
end
end
describe '#enqueue_snippet' do
let(:snippet) { instance_double('Snippet') }
describe '#enqueue_project_wiki' do
let(:resource) { Gitlab::Backup::Cli::Models::ProjectWiki }
let(:repository) { object_double(resource.new) }
it 'enqueues wiki repository' do
allow(resource).to receive(:find_each).and_yield(repository)
expect(gitaly_backup).to receive(:enqueue).with(repository)
repo_target.send(:enqueue_project_wiki)
end
end
describe '#enqueue_group_wiki' do
let(:resource) { Gitlab::Backup::Cli::Models::GroupWiki }
let(:repository) { object_double(resource.new) }
it 'enqueues wiki repository' do
allow(resource).to receive(:find_each).and_yield(repository)
expect(gitaly_backup).to receive(:enqueue).with(repository)
repo_target.send(:enqueue_group_wiki)
end
end
describe '#enqueue_project_design_management' do
let(:resource) { Gitlab::Backup::Cli::Models::ProjectDesignManagement }
let(:repository) { object_double(resource.new) }
it 'enqueues design management repository' do
allow(resource).to receive(:find_each).and_yield(repository)
expect(gitaly_backup).to receive(:enqueue).with(repository)
repo_target.send(:enqueue_project_design_management)
end
end
describe '#enqueue_project_snippets' do
let(:resource) { Gitlab::Backup::Cli::Models::ProjectSnippet }
let(:repository) { object_double(resource.new) }
it 'enqueues the snippet' do
expect(gitaly_backup).to receive(:enqueue).with(snippet, Gitlab::Backup::Cli::RepoType::SNIPPET)
allow(resource).to receive(:find_each).and_yield(repository)
repo_target.send(:enqueue_snippet, snippet)
expect(gitaly_backup).to receive(:enqueue).with(repository)
repo_target.send(:enqueue_project_snippets)
end
end
describe '#enqueue_personal_snippets' do
let(:resource) { Gitlab::Backup::Cli::Models::PersonalSnippet }
let(:repository) { object_double(resource.new) }
it 'enqueues the snippet' do
allow(resource).to receive(:find_each).and_yield(repository)
expect(gitaly_backup).to receive(:enqueue).with(repository)
repo_target.send(:enqueue_personal_snippets)
end
end
end

View File

@ -0,0 +1,67 @@
# frozen_string_literal: true
RSpec.describe Gitlab::Backup::Cli::Utils::PoolRepositories do
let(:context) { build_test_context }
subject(:pool) { described_class.new(gitlab_basepath: context.gitlab_basepath) }
describe '#gitlab_basepath' do
it 'returns a path' do
expect(pool.gitlab_basepath).to be_a(Pathname)
expect(pool.gitlab_basepath).to eq(context.gitlab_basepath)
end
end
describe '#reinitialize!' do
let(:json_output) do
[
%q({"disk_path":"aa/bb/repo1.git","status":"scheduled","error_message":null}),
%q({"disk_path":"cc/dd/repo2.git","status":"skipped","error_message":null}),
%q({"disk_path":"ee/ff/repo3.git","status":"failed","error_message":"Error message"})
]
end
it 'output parsed content and output to terminal' do
fake_output = json_output.map { |json| { stream: :stdout, output: json } }
fake_rake = FakeRake.new(fake_output: fake_output)
expect(pool).to receive(:build_reset_task).and_return(fake_rake)
timestamp_pattern = /\[[\d\- :]+UTC\]/
expected_stdout = /
#{timestamp_pattern}#{Regexp.escape(' Reinitializing object pools...')}\n
#{timestamp_pattern}#{Regexp.escape(' ✅️ Object pool aa/bb/repo1.git...')}\n
#{timestamp_pattern}#{Regexp.escape(' Object pool cc/dd/repo2.git... [SKIPPED]')}\n
#{timestamp_pattern}#{Regexp.escape(' Object pool ee/ff/repo3.git... [FAILED]')}\n
/mx
expected_stderr = /
#{timestamp_pattern}#{Regexp.escape(' ❌️ Object pool ee/ff/repo3.git failed to reset (Error message)')}\n
/mx
expect { pool.reinitialize! }.to output(expected_stdout).to_stdout.and output(expected_stderr).to_stderr
end
it 'handles content not in json format and output to terminal' do
fake_output = [
{ stream: :stdout, output: 'stdout content not in JSON format' },
{ stream: :stderr, output: 'stderr content not in JSON format' }
]
fake_rake = FakeRake.new(fake_output: fake_output)
expect(pool).to receive(:build_reset_task).and_return(fake_rake)
timestamp_pattern = /\[[\d\- :]+UTC\]/
expected_stdout = /
#{timestamp_pattern}#{Regexp.escape(' Reinitializing object pools...')}\n
/mx
expected_stderr = /
#{timestamp_pattern}#{Regexp.escape(' ⚠️ Failed to parse: stdout content not in JSON format')}\n
#{timestamp_pattern}#{Regexp.escape(' ⚠️ stderr content not in JSON format')}\n
/mx
expect { pool.reinitialize! }.to output(expected_stdout).to_stdout.and output(expected_stderr).to_stderr
end
end
end

View File

@ -52,6 +52,24 @@ RSpec.describe Gitlab::Backup::Cli::Utils::Rake do
end
end
describe '#capture_each' do
it 'allows processing captured streams using blocks' do
expect_next_instance_of(Gitlab::Backup::Cli::Shell::Command) do |shell|
expect(shell.cmd_args).to end_with(%w[version])
end
output_streams = []
rake.capture_each do |stream, output|
output_streams << [stream, output]
end
expect(output_streams).to eq([[:stdout, Gitlab::Backup::Cli::VERSION]])
expect(rake.success?).to eq(true)
end
end
describe '#success?' do
subject(:rake) { described_class.new('--version') } # valid command that has no side-effect

View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
FakeRake = Struct.new(:fake_output, keyword_init: true) do
def capture_each
fake_output.each do |line|
yield line[:stream], line[:output]
end
end
end

View File

@ -40,6 +40,10 @@ module GitlabBackupHelpers
test_db = context.gitlab_basepath.join('config/database.yml')
FileUtils.mkdir_p(File.dirname(test_db))
FileUtils.copy(db, test_db)
# config/gitlab.yml
gitlab_cfg = fixtures_path.join('config/gitlab.yml')
gitlab_cfg_path = context.gitlab_basepath.join('config/gitlab.yml')
FileUtils.copy(gitlab_cfg, gitlab_cfg_path)
# Mocked Rakefile and Gemfile
FileUtils.cp_r(fixtures_path.join('gitlab_fake').glob('*'), context.gitlab_basepath)

View File

@ -41,8 +41,6 @@ RSpec.describe 'gitlab-backup-cli backup subcommand', type: :thor do
let(:executor) { Gitlab::Backup::Cli::BackupExecutor }
before do
allow(Gitlab::Backup::Cli).to receive(:rails_environment!)
expect_next_instance_of(backup_subcommand) do |instance|
allow(instance).to receive(:build_context).and_return(context)
end

View File

@ -42,8 +42,6 @@ RSpec.describe 'gitlab-backup-cli restore subcommand', type: :thor do
let(:backup_id) { "1715018771_2024_05_06_17.0.0-pre" }
before do
allow(Gitlab::Backup::Cli).to receive(:rails_environment!)
expect_next_instance_of(restore_subcommand) do |instance|
allow(instance).to receive(:build_context).and_return(context)
end

View File

@ -34811,6 +34811,9 @@ msgid_plural "Job|%{searchLength} results found for %{searchTerm}"
msgstr[0] ""
msgstr[1] ""
msgid "Job|An error occurred while fetching the job header data."
msgstr ""
msgid "Job|Are you sure you want to erase this job log and artifacts?"
msgstr ""

View File

@ -1,23 +1,26 @@
import { GlButton, GlAvatarLink, GlTooltip } from '@gitlab/ui';
import { shallowMount } from '@vue/test-utils';
import { extendedWrapper } from 'helpers/vue_test_utils_helper';
import { GlButton, GlAvatarLink, GlTooltip, GlLoadingIcon } from '@gitlab/ui';
import Vue from 'vue';
import VueApollo from 'vue-apollo';
import { shallowMountExtended } from 'helpers/vue_test_utils_helper';
import createMockApollo from 'helpers/mock_apollo_helper';
import waitForPromises from 'helpers/wait_for_promises';
import { createAlert } from '~/alert';
import CiIcon from '~/vue_shared/components/ci_icon/ci_icon.vue';
import JobHeader from '~/ci/job_details/components/job_header.vue';
import TimeagoTooltip from '~/vue_shared/components/time_ago_tooltip.vue';
import getJobQuery from '~/ci/job_details/graphql/queries/get_job.query.graphql';
import { mockJobResponse } from '../mock_data';
Vue.use(VueApollo);
jest.mock('~/alert');
describe('Header CI Component', () => {
let wrapper;
let apolloProvider;
const defaultProps = {
status: {
group: 'failed',
icon: 'status_failed',
label: 'failed',
text: 'failed',
details_path: 'path',
},
name: 'build_job',
time: '2017-05-08T14:57:39.781Z',
jobId: 13051,
user: {
id: 1234,
web_url: 'path',
@ -26,34 +29,61 @@ describe('Header CI Component', () => {
email: 'foo@bar.com',
avatar_url: 'link',
},
shouldRenderTriggeredLabel: true,
};
const successHandler = jest.fn().mockResolvedValue(mockJobResponse);
const failedHandler = jest.fn().mockRejectedValue(new Error('GraphQL Error'));
const findCiIcon = () => wrapper.findComponent(CiIcon);
const findTimeAgo = () => wrapper.findComponent(TimeagoTooltip);
const findUserLink = () => wrapper.findComponent(GlAvatarLink);
const findSidebarToggleBtn = () => wrapper.findComponent(GlButton);
const findStatusTooltip = () => wrapper.findComponent(GlTooltip);
const findLoadingIcon = () => wrapper.findComponent(GlLoadingIcon);
const findJobName = () => wrapper.findByTestId('job-name');
const createComponent = (props) => {
wrapper = extendedWrapper(
shallowMount(JobHeader, {
propsData: {
...defaultProps,
...props,
},
}),
);
const defaultHandlers = [[getJobQuery, successHandler]];
const createMockApolloProvider = (handlers) => {
return createMockApollo(handlers);
};
describe('render', () => {
beforeEach(() => {
const createComponent = (props, handlers = defaultHandlers) => {
apolloProvider = createMockApolloProvider(handlers);
wrapper = shallowMountExtended(JobHeader, {
propsData: {
...defaultProps,
...props,
},
provide: {
projectPath: 'gitlab-org/gitlab',
},
apolloProvider,
});
};
describe('loading', () => {
it('should display a loading icon', () => {
createComponent();
expect(findLoadingIcon().exists()).toBe(true);
});
});
describe('render', () => {
beforeEach(async () => {
createComponent();
await waitForPromises();
});
it('should not display a loading icon', () => {
expect(findLoadingIcon().exists()).toBe(false);
});
it('renders the correct job name', () => {
expect(findJobName().text()).toBe(defaultProps.name);
expect(findJobName().text()).toBe('artifact_job');
});
it('should render status badge', () => {
@ -67,11 +97,28 @@ describe('Header CI Component', () => {
it('should render sidebar toggle button', () => {
expect(findSidebarToggleBtn().exists()).toBe(true);
});
it('calls query with correct variables', () => {
expect(successHandler).toHaveBeenCalledWith({
fullPath: 'gitlab-org/gitlab',
id: 'gid://gitlab/Ci::Build/13051',
});
});
it('polls query to receive status updates', () => {
expect(successHandler).toHaveBeenCalledTimes(1);
jest.advanceTimersByTime(10000);
expect(successHandler).toHaveBeenCalledTimes(2);
});
});
describe('user avatar', () => {
beforeEach(() => {
beforeEach(async () => {
createComponent();
await waitForPromises();
});
it('contains the username', () => {
@ -90,10 +137,12 @@ describe('Header CI Component', () => {
describe('when the user has a status', () => {
const STATUS_MESSAGE = 'Working on exciting features...';
beforeEach(() => {
beforeEach(async () => {
createComponent({
user: { ...defaultProps.user, status: { message: STATUS_MESSAGE } },
});
await waitForPromises();
});
it('renders a tooltip', () => {
@ -104,10 +153,12 @@ describe('Header CI Component', () => {
describe('with data from GraphQL', () => {
const userId = 1;
beforeEach(() => {
beforeEach(async () => {
createComponent({
user: { ...defaultProps.user, id: `gid://gitlab/User/${1}` },
});
await waitForPromises();
});
it('has the correct user id', () => {
@ -122,12 +173,28 @@ describe('Header CI Component', () => {
});
});
describe('shouldRenderTriggeredLabel', () => {
it('should render created keyword when the shouldRenderTriggeredLabel is false', () => {
createComponent({ shouldRenderTriggeredLabel: false });
describe('triggered label', () => {
it('should render created keyword', async () => {
createComponent();
await waitForPromises();
expect(wrapper.text()).toContain('Created');
expect(wrapper.text()).not.toContain('Started');
});
});
describe('error', () => {
it('shows error alert on GraphQL error', async () => {
createComponent(defaultProps, [[getJobQuery, failedHandler]]);
await waitForPromises();
expect(createAlert).toHaveBeenCalledWith({
captureError: true,
error: expect.any(Error),
message: 'An error occurred while fetching the job header data.',
});
});
});
});

View File

@ -4,18 +4,25 @@ export const mockId = 401;
export const mockJobResponse = {
data: {
project: {
id: 'gid://gitlab/Project/4',
id: 'gid://gitlab/Project/20',
job: {
id: 'gid://gitlab/Ci::Build/401',
manualJob: true,
id: 'gid://gitlab/Ci::Build/13051',
manualVariables: {
nodes: [],
__typename: 'CiManualVariableConnection',
},
name: 'manual_job',
retryable: true,
status: 'SUCCESS',
__typename: 'CiJob',
manualJob: false,
name: 'artifact_job',
detailedStatus: {
id: 'success-13051-13051',
icon: 'status_success',
text: 'Passed',
detailsPath: '/root/ci-project/-/jobs/13051',
__typename: 'DetailedStatus',
},
startedAt: '',
createdAt: '2025-04-21T16:19:15Z',
},
__typename: 'Project',
},
@ -41,8 +48,15 @@ export const mockJobWithVariablesResponse = {
__typename: 'CiManualVariableConnection',
},
name: 'manual_job',
retryable: true,
status: 'SUCCESS',
detailedStatus: {
id: 'manual-13046-13046',
icon: 'status_manual',
text: 'Manual',
detailsPath: '/root/ci-project/-/jobs/13046',
__typename: 'DetailedStatus',
},
startedAt: null,
createdAt: '2025-04-21T16:19:15Z',
__typename: 'CiJob',
},
__typename: 'Project',

View File

@ -8,28 +8,6 @@ describe('Job Store Getters', () => {
localState = state();
});
describe('headerTime', () => {
describe('when the job has started key', () => {
it('returns started_at value', () => {
const started = '2018-08-31T16:20:49.023Z';
const startedAt = '2018-08-31T16:20:49.023Z';
localState.job.started_at = startedAt;
localState.job.started = started;
expect(getters.headerTime(localState)).toEqual(startedAt);
});
});
describe('when the job does not have started key', () => {
it('returns created_at value', () => {
const created = '2018-08-31T16:20:49.023Z';
localState.job.created_at = created;
expect(getters.headerTime(localState)).toEqual(created);
});
});
});
describe('shouldRenderCalloutMessage', () => {
describe('with status and callout message', () => {
it('returns true', () => {
@ -57,24 +35,6 @@ describe('Job Store Getters', () => {
});
});
describe('shouldRenderTriggeredLabel', () => {
describe('when started equals null', () => {
it('returns false', () => {
localState.job.started_at = null;
expect(getters.shouldRenderTriggeredLabel(localState)).toEqual(false);
});
});
describe('when started equals string', () => {
it('returns true', () => {
localState.job.started_at = '2018-08-31T16:20:49.023Z';
expect(getters.shouldRenderTriggeredLabel(localState)).toEqual(true);
});
});
});
describe('hasEnvironment', () => {
describe('without `deployment_status`', () => {
it('returns false', () => {

View File

@ -1,40 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe CrudPolicyHelpers do
let(:policy_test_class) do
Class.new do
include CrudPolicyHelpers
end
end
let(:feature_name) { :foo }
before do
stub_const('PolicyTestClass', policy_test_class)
end
describe '.create_update_admin_destroy' do
it 'returns an array of the appropriate abilites given a feature name' do
expect(PolicyTestClass.create_update_admin_destroy(feature_name)).to eq(
[
:create_foo,
:update_foo,
:admin_foo,
:destroy_foo
])
end
end
describe '.create_update_admin' do
it 'returns an array of the appropriate abilites given a feature name' do
expect(PolicyTestClass.create_update_admin(feature_name)).to eq(
[
:create_foo,
:update_foo,
:admin_foo
])
end
end
end

View File

@ -6339,7 +6339,6 @@
- './spec/policies/clusters/cluster_policy_spec.rb'
- './spec/policies/clusters/instance_policy_spec.rb'
- './spec/policies/commit_policy_spec.rb'
- './spec/policies/concerns/crud_policy_helpers_spec.rb'
- './spec/policies/concerns/policy_actor_spec.rb'
- './spec/policies/container_expiration_policy_policy_spec.rb'
- './spec/policies/custom_emoji_policy_spec.rb'

View File

@ -3,7 +3,12 @@
RSpec.shared_examples 'archived project policies' do
let(:feature_write_abilities) do
described_class.archived_features.flat_map do |feature|
described_class.create_update_admin_destroy(feature)
[
:"create_#{feature}",
:"update_#{feature}",
:"admin_#{feature}",
:"destroy_#{feature}"
]
end + additional_maintainer_permissions
end