root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-12-24 00:07:14 +00:00
parent 08a284ca5e
commit c4e4f0d167
29 changed files with 433 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/assets/javascripts/pages/projects/blob/show/index.js
View File

@ -27,7 +27,8 @@ const router = new VueRouter({ mode: 'history' });
const viewBlobEl = document.querySelector('#js-view-blob-app');
if (viewBlobEl) {
const { blobPath, projectPath, targetBranch, originalBranch, refType } = viewBlobEl.dataset;
const { blobPath, projectPath, targetBranch, originalBranch } = viewBlobEl.dataset;
// eslint-disable-next-line no-new
new Vue({
el: viewBlobEl,
@ -37,7 +38,6 @@ if (viewBlobEl) {
provide: {
targetBranch,
originalBranch,
refType,
},
render(createElement) {
return createElement(BlobContentViewer, {

4
app/assets/javascripts/repository/components/blob_content_viewer.vue
View File

@ -40,9 +40,6 @@ export default {
originalBranch: {
default: '',
},
refType: {
default: '',
},
},
apollo: {
projectInfo: {
@ -79,7 +76,6 @@ export default {
projectPath: this.projectPath,
filePath: this.path,
ref: this.originalBranch || this.ref,
refType: this.refType,
shouldFetchRawText: Boolean(this.glFeatures.highlightJs),
};
},

3
app/assets/javascripts/search/index.js
View File

@ -1,6 +1,7 @@
import setHighlightClass from 'ee_else_ce/search/highlight_blob_search_result';
import { queryToObject } from '~/lib/utils/url_utility';
import refreshCounts from '~/pages/search/show/refresh_counts';
import syntaxHighlight from '~/syntax_highlight';
import { initSidebar, sidebarInitState } from './sidebar';
import { initSearchSort } from './sort';
import createStore from './store';
@ -8,6 +9,8 @@ import { initTopbar } from './topbar';
import { initBlobRefSwitcher } from './under_topbar';
export const initSearchApp = () => {
syntaxHighlight(document.querySelectorAll('.js-search-results'));
const query = queryToObject(window.location.search);
const navigation = sidebarInitState();

2
app/assets/javascripts/syntax_highlight.js
View File

@ -11,7 +11,7 @@
//
export default function syntaxHighlight($els = null) {
if (!$els) return;
if (!$els || $els.length === 0) return;
const els = $els.get ? $els.get() : $els;
const handler = (el) => {

5
app/controllers/projects/blob_controller.rb
View File

@ -62,7 +62,6 @@ class Projects::BlobController < Projects::ApplicationController
def show
conditionally_expand_blob(@blob)
@ref_type = ref_type
respond_to do |format|
format.html do
show_html
@ -147,8 +146,8 @@ class Projects::BlobController < Projects::ApplicationController
end
def commit
@repo = @repository
super
@commit ||= @repository.commit(@ref)
return render_404 unless @commit
end

6
app/controllers/projects/refs_controller.rb
View File

@ -22,11 +22,7 @@ class Projects::RefsController < Projects::ApplicationController
when "tree"
project_tree_path(@project, @id)
when "blob"
if Feature.enabled?(:use_ref_type_parameter, @project)
project_blob_path(@project, @id, ref_type: ref_type)
else
project_blob_path(@project, @id)
end
project_blob_path(@project, @id)
when "graph"
if Feature.enabled?(:use_ref_type_parameter, @project)
project_network_path(@project, @id, ref_type: ref_type)

3
app/graphql/queries/repository/blob_info.query.graphql
View File

@ -2,7 +2,6 @@ query getBlobInfo(
$projectPath: ID!
$filePath: String!
$ref: String!
$refType: String
$shouldFetchRawText: Boolean!
) {
project(fullPath: $projectPath) {
@ -11,7 +10,7 @@ query getBlobInfo(
repository {
__typename
empty
blobs(paths: [$filePath], ref: $ref, refType: $refType) {
blobs(paths: [$filePath], ref: $ref) {
__typename
nodes {
__typename

21
app/graphql/resolvers/blobs_resolver.rb
View File

@ -17,10 +17,6 @@ module Resolvers
required: false,
default_value: nil,
description: 'Commit ref to get the blobs from. Default value is HEAD.'
argument :ref_type, GraphQL::Types::String,
required: false,
default_value: nil,
description: 'Type of the ref. heads for branches and tags for tags.'
# We fetch blobs from Gitaly efficiently but it still scales O(N) with the
# number of paths being fetched, so apply a scaling limit to that.
@ -28,7 +24,7 @@ module Resolvers
super + (args[:paths] || []).size
end
def resolve(paths:, ref:, ref_type:)
def resolve(paths:, ref:)
authorize!(repository.container)
return [] if repository.empty?
@ -36,24 +32,11 @@ module Resolvers
ref ||= repository.root_ref
validate_ref(ref)
ref = fully_qualifed_ref(ref, ref_type)
repository.blobs_at(paths.map { |path| [ref, path] }).tap do |blobs|
blobs.each do |blob|
blob.ref_type = ref_type
end
end
repository.blobs_at(paths.map { |path| [ref, path] })
end
private
def fully_qualifed_ref(ref, ref_type)
return ref unless ref_type.present? && Feature.enabled?(:use_ref_type_parameter, repository.project)
ref_type = ref_type == 'tags' ? 'tags' : 'heads'
%(refs/#{ref_type}/#{ref})
end
def validate_ref(ref)
unless Gitlab::GitRefValidator.validate(ref)
raise Gitlab::Graphql::Errors::ArgumentError, 'Ref is not valid'

1
app/models/blob.rb
View File

@ -72,7 +72,6 @@ class Blob < SimpleDelegator
].freeze
attr_reader :container
attr_accessor :ref_type
delegate :repository, to: :container, allow_nil: true
delegate :project, to: :repository, allow_nil: true

29
app/presenters/blob_presenter.rb
View File

@ -56,23 +56,23 @@ class BlobPresenter < Gitlab::View::Presenter::Delegated
end
def web_url
url_helpers.project_blob_url(*path_params)
url_helpers.project_blob_url(project, ref_qualified_path)
end
def web_path
url_helpers.project_blob_path(*path_params)
url_helpers.project_blob_path(project, ref_qualified_path)
end
def edit_blob_path
url_helpers.project_edit_blob_path(*path_params)
url_helpers.project_edit_blob_path(project, ref_qualified_path)
end
def raw_path
url_helpers.project_raw_path(*path_params)
url_helpers.project_raw_path(project, ref_qualified_path)
end
def replace_path
url_helpers.project_update_blob_path(*path_params)
url_helpers.project_update_blob_path(project, ref_qualified_path)
end
def pipeline_editor_path
@ -164,18 +164,6 @@ class BlobPresenter < Gitlab::View::Presenter::Delegated
private
def path_params
if Feature.enabled?(:use_ref_type_parameter, project) && ref_type
[project, ref_qualified_path, { ref_type: ref_type }]
else
[project, ref_qualified_path]
end
end
def ref_type
blob.try(:ref_type)
end
def url_helpers
Gitlab::Routing.url_helpers
end
@ -191,12 +179,7 @@ class BlobPresenter < Gitlab::View::Presenter::Delegated
end
def ref_qualified_path
commit_id = blob.commit_id
# A hack to make the commit_id response from gitaly something controllers can handle
commit_id = commit_id.sub(%r{^refs/(heads|tags)/}, '') if ref_type && Feature.enabled?(:use_ref_type_parameter, project)
File.join(commit_id, blob.path)
File.join(blob.commit_id, blob.path)
end
def load_all_blob_data

3
app/presenters/ci/build_runner_presenter.rb
View File

@ -38,8 +38,7 @@ module Ci
variables
.sort_and_expand_all(keep_undefined: true,
expand_file_refs: false,
expand_raw_refs: !stop_expanding_raw_refs,
project: project)
expand_raw_refs: !stop_expanding_raw_refs)
.to_runner_variables
end

1
app/views/projects/blob/_blob.html.haml
View File

@ -22,7 +22,6 @@
#js-view-blob-app{ data: { blob_path: blob.path,
project_path: @project.full_path,
target_branch: project.empty_repo? ? ref : @ref,
ref_type: @ref_type.to_s,
original_branch: @ref } }
= gl_loading_icon(size: 'md')
- else

2
app/views/projects/blob/show.html.haml
View File

@ -4,7 +4,7 @@
- signatures_path = namespace_project_signatures_path(namespace_id: @project.namespace.full_path, project_id: @project.path, id: @last_commit, limit: 1)
- content_for :prefetch_asset_tags do
- webpack_preload_asset_tag('monaco', prefetch: true)
- add_page_startup_graphql_call('repository/blob_info', { projectPath: @project.full_path, ref: current_ref, refType: @ref_type.to_s, filePath: @blob.path, shouldFetchRawText: @blob.rendered_as_text? && !@blob.rich_viewer })
- add_page_startup_graphql_call('repository/blob_info', { projectPath: @project.full_path, ref: current_ref, filePath: @blob.path, shouldFetchRawText: @blob.rendered_as_text? && !@blob.rich_viewer })
.js-signature-container{ data: { 'signatures-path': signatures_path } }

2
app/views/search/_results_list.html.haml
View File

@ -7,7 +7,7 @@
%ul.content-list.commit-list
= render partial: "search/results/commit", collection: @search_objects
- else
.search-results
.search-results.js-search-results
- if @scope == 'projects'
.term
= render 'shared/projects/list', projects: @search_objects, pipeline_status: false

1
doc/api/graphql/reference/index.md
View File

@ -18847,7 +18847,6 @@ four standard [pagination arguments](#connection-pagination-arguments):
| ---- | ---- | ----------- |
| <a id="repositoryblobspaths"></a>`paths` | [`[String!]!`](#string) | Array of desired blob paths. |
| <a id="repositoryblobsref"></a>`ref` | [`String`](#string) | Commit ref to get the blobs from. Default value is HEAD. |
| <a id="repositoryblobsreftype"></a>`refType` | [`String`](#string) | Type of the ref. heads for branches and tags for tags. |
##### `Repository.branchNames`

89
doc/development/gitlab_shell/features.md Normal file
View File

@ -0,0 +1,89 @@
---
stage: Create
group: Source Code
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
# GitLab Shell feature list
## Discover
Allows users to identify themselves on an instance via SSH. The command helps to
confirm quickly whether a user has SSH access to the instance:
```shell
ssh git@<hostname>
PTY allocation request failed on channel 0
Welcome to GitLab, @username!
Connection to staging.gitlab.com closed.
```
When permission is denied, it returns:
```shell
ssh git@<hostname>
git@<hostname>: Permission denied (publickey).
```
## Git operations
GitLab Shell provides support for Git operations over SSH by processing
`git-upload-pack`, `git-receive-pack` and `git-upload-archive` SSH commands.
It limits the set of commands to predefined Git commands:
- `git archive`
- `git clone`
- `git pull`
- `git push`
## Generate new 2FA recovery codes
Enables users to
[generate new 2FA recovery codes](../../user/profile/account/two_factor_authentication.md#generate-new-recovery-codes-using-ssh):
```shell
$ ssh git@<hostname> 2fa_recovery_codes
Are you sure you want to generate new two-factor recovery codes?
Any existing recovery codes you saved will be invalidated. (yes/no)
yes
Your two-factor authentication recovery codes are:
...
```
## Verify 2FA OTP
Allows users to verify their
[2FA one-time password (OTP)](../../security/two_factor_authentication.md#2fa-for-git-over-ssh-operations):
```shell
$ ssh git@<hostname> 2fa_verify
OTP: 347419
OTP validation failed.
```
## LFS authentication
Enables users to generate credentials for LFS authentication:
```shell
$ ssh git@<hostname> git-lfs-authenticate <project-path> <upload/download>
{"header":{"Authorization":"Basic ..."},"href":"https://gitlab.com/user/project.git/info/lfs","expires_in":7200}
```
## Personal access token
Enables users to use personal access tokens via SSH:
```shell
$ ssh git@<hostname> personal_access_token <name> <scope1[,scope2,...]> [ttl_days]
Token: glpat-...
Scopes: api
Expires: 2022-02-05
```

36
doc/development/gitlab_shell/gitlab_sshd.md Normal file
View File

@ -0,0 +1,36 @@
---
stage: Create
group: Source Code
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# `gitlab-sshd` in GitLab Shell
`gitlab-sshd` is a binary in [`gitlab-shell`](https://gitlab.com/gitlab-org/gitlab-shell)
which runs as a persistent SSH daemon. It is intended to replace `OpenSSH` on GitLab SaaS,
and eventually other cloud-native environments. Instead of running an `sshd` process,
we run a `gitlab-sshd` process that does the same job, in a more focused manner:
```mermaid
sequenceDiagram
participant Git on client
participant GitLab SSHD
participant Rails
participant Gitaly
participant Git on server
Note left of Git on client: git fetch
Git on client->>+GitLab SSHD: ssh git fetch-pack request
GitLab SSHD->>+Rails: GET /internal/api/authorized_keys?key=AAAA...
Note right of Rails: Lookup key ID
Rails-->>-GitLab SSHD: 200 OK, command="gitlab-shell upload-pack key_id=1"
GitLab SSHD->>+Rails: GET /internal/api/allowed?action=upload_pack&key_id=1
Note right of Rails: Auth check
Rails-->>-GitLab SSHD: 200 OK, { gitaly: ... }
GitLab SSHD->>+Gitaly: SSHService.SSHUploadPack request
Gitaly->>+Git on server: git upload-pack request
Note over Git on client,Git on server: Bidirectional communication between Git client and server
Git on server-->>-Gitaly: git upload-pack response
Gitaly -->>-GitLab SSHD: SSHService.SSHUploadPack response
GitLab SSHD-->>-Git on client: ssh git fetch-pack response
```

127
doc/development/gitlab_shell/index.md
View File

@ -6,9 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# GitLab Shell
[![pipeline status](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/pipeline.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main)
[![coverage report](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/coverage.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main)
[![Code Climate](https://codeclimate.com/github/gitlabhq/gitlab-shell.svg)](https://codeclimate.com/github/gitlabhq/gitlab-shell)
[![pipeline status](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/pipeline.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main) [![coverage report](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/coverage.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main) [![Code Climate](https://codeclimate.com/github/gitlabhq/gitlab-shell.svg)](https://codeclimate.com/github/gitlabhq/gitlab-shell)
GitLab Shell handles Git SSH sessions for GitLab and modifies the list of authorized keys.
GitLab Shell is not a Unix shell nor a replacement for Bash or Zsh.
@ -38,7 +36,7 @@ When you access the GitLab server over SSH, GitLab Shell then:
1. Calls the GitLab Rails API to check if you are authorized, and what Gitaly server your repository is on.
1. Copies data back and forth between the SSH client and the Gitaly server.
If you access a GitLab server over HTTP(S) you end up in [`gitlab-workhorse`](https://gitlab.com/gitlab-org/gitlab/tree/master/workhorse).
If you access a GitLab server over HTTP(S) you end up in [`gitlab-workhorse`](../workhorse/index.md).
### `git pull` over SSH
@ -65,21 +63,87 @@ subgraph Gitaly
end
```
[Full feature list](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/doc/features.md)
[Full feature list](features.md)
### Modifies `authorized_keys`
GitLab Shell modifies the `authorized_keys` file on the client machine.
## Rate Limiting
## Contribute to GitLab Shell
GitLab Shell performs rate-limiting by user account and project for Git operations. GitLab Shell accepts Git operation requests and then makes a call to the Rails rate-limiter (backed by Redis). If the `user + project` exceeds the rate limit then GitLab Shell then drop further connection requests for that `user + project`.
To contribute to GitLab Shell:
The rate-limiter is applied at the Git command (plumbing) level. Each command has a rate limit of 600 per minute. For example, `git push` has 600 per minute, and `git pull` has another 600 per minute.
1. Check if GitLab API access, and Redis via the internal API, can be reached: `make check`
1. Compile the `gitlab-shell` binaries, placing them into `bin/`: `make compile`
1. Run `make install` to build the `gitlab-shell` binaries and install. them onto the file system.
The default location is `/usr/local`. To change it, set the `PREFIX` and `DESTDIR` environment variables.
1. To install GitLab from source on a single machine, run `make setup`.
It compiles the GitLab Shell binaries, and ensures that various paths on the file system
exist with the correct permissions. Do not run this command unless your installation method
documentation instructs you to.
Because they are using the same plumbing command, `git-upload-pack`, `git pull` and `git clone` are in effect the same command for the purposes of rate-limiting.
For more information, see
[CONTRIBUTING.md](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/CONTRIBUTING.md).
Gitaly also has a rate-limiter in place, but calls are never made to Gitaly if the rate limit is exceeded in GitLab Shell (Rails).
### Run tests
When contributing, run tests:
1. Run tests with `bundle install` and `make test`.
1. Run Gofmt: `make verify`
1. Run both test and verify (the default Makefile target):
```shell
bundle install
make validate
```
1. If needed, configure Gitaly.
### Configure Gitaly for local testing
Some tests need a Gitaly server. The
[`docker-compose.yml`](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/docker-compose.yml) file runs Gitaly on port 8075.
To tell the tests where Gitaly is, set `GITALY_CONNECTION_INFO`:
```plaintext
export GITALY_CONNECTION_INFO='{"address": "tcp://localhost:8075", "storage": "default"}'
make test
```
If no `GITALY_CONNECTION_INFO` is set, the test suite still runs, but any
tests requiring Gitaly are skipped. The tests always run in the CI environment.
## Rate limiting
GitLab Shell performs rate-limiting by user account and project for Git operations.
GitLab Shell accepts Git operation requests and then makes a call to the Rails
rate-limiter, backed by Redis. If the `user + project` exceeds the rate limit,
then GitLab Shell then drop further connection requests for that `user + project`.
The rate-limiter is applied at the Git command (plumbing) level. Each command has
a rate limit of 600 per minute. For example, `git push` has 600 per minute, and
`git pull` has another 600 per minute.
Because they are using the same plumbing command, `git-upload-pack`, `git pull`,
and `git clone` are in effect the same command for the purposes of rate-limiting.
Gitaly also has a rate-limiter in place, but calls are never made to Gitaly if
the rate limit is exceeded in GitLab Shell (Rails).
## Logs in GitLab Shell
In general, you can determine the structure, but not content, of a GitLab Shell
or `gitlab-sshd` session by inspecting the logs. Some guidelines:
- We use [`gitlab.com/gitlab-org/labkit/log`](https://pkg.go.dev/gitlab.com/gitlab-org/labkit/log)
for logging.
- Always include a correlation ID.
- Log messages should be invariant and unique. Include accessory information in
fields, using `log.WithField`, `log.WithFields`, or `log.WithError`.
- Log both success cases and error cases.
- Logging too much is better than not logging enough. If a message seems too
verbose, consider reducing the log level before removing the message.
## GitLab SaaS
@ -119,19 +183,40 @@ graph LR
end
```
## Releasing
## GitLab Shell architecture
See [PROCESS.md](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/PROCESS.md)
```mermaid
sequenceDiagram
participant Git on client
participant SSH server
participant AuthorizedKeysCommand
participant GitLab Shell
participant Rails
participant Gitaly
participant Git on server
## Contributing
- See [CONTRIBUTING.md](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/CONTRIBUTING.md).
- See the [beginner's guide](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/doc/beginners_guide.md).
## License
See [LICENSE](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/LICENSE).
Note left of Git on client: git fetch
Git on client->>+SSH server: ssh git fetch-pack request
SSH server->>+AuthorizedKeysCommand: gitlab-shell-authorized-keys-check git AAAA...
AuthorizedKeysCommand->>+Rails: GET /internal/api/authorized_keys?key=AAAA...
Note right of Rails: Lookup key ID
Rails-->>-AuthorizedKeysCommand: 200 OK, command="gitlab-shell upload-pack key_id=1"
AuthorizedKeysCommand-->>-SSH server: command="gitlab-shell upload-pack key_id=1"
SSH server->>+GitLab Shell: gitlab-shell upload-pack key_id=1
GitLab Shell->>+Rails: GET /internal/api/allowed?action=upload_pack&key_id=1
Note right of Rails: Auth check
Rails-->>-GitLab Shell: 200 OK, { gitaly: ... }
GitLab Shell->>+Gitaly: SSHService.SSHUploadPack request
Gitaly->>+Git on server: git upload-pack request
Note over Git on client,Git on server: Bidirectional communication between Git client and server
Git on server-->>-Gitaly: git upload-pack response
Gitaly -->>-GitLab Shell: SSHService.SSHUploadPack response
GitLab Shell-->>-SSH server: gitlab-shell upload-pack response
SSH server-->>-Git on client: ssh git fetch-pack response
```
## Related topics
- [Using the GitLab Shell chart](https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/doc/charts/gitlab/gitlab-shell/index.md)
- [LICENSE](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/LICENSE).
- [PROCESS.md](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/PROCESS.md)
- [Using the GitLab Shell chart](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/)

2
doc/development/testing_guide/testing_migrations_guide.md
View File

@ -17,6 +17,8 @@ a database schema.
- If your migration is a data migration then it **must** have a migration test.
- Other migrations may have a migration test if necessary.
We don't enforce tests on post migrations that only perform schema changes.
## How does it work?
Adding a `:migration` tag to a test signature enables some custom RSpec

137
doc/user/infrastructure/iac/gitlab_terraform_helpers.md Normal file
View File

@ -0,0 +1,137 @@
---
stage: Configure
group: Configure
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
# GitLab Terraform helpers **(FREE)**
GitLab provides two helpers to ease your integration with the [GitLab-managed Terraform State](terraform_state.md).
- The `gitlab-terraform` script, which is a thin wrapper around the `terraform` command.
- The `terraform-images` container images, which include the `gitlab-terraform` script and `terraform` itself.
Both helpers are maintained in the [Terraform Images](https://gitlab.com/gitlab-org/terraform-images)
project.
## `gitlab-terraform`
The `gitlab-terraform` script is a thin wrapper around the `terraform` command.
Run `gitlab-terraform` in a CI/CD pipeline to set up the necessary environment
variables to connect to the [GitLab-managed Terraform State](terraform_state.md) backend.
### Source (but do not run) the helper script
When the `gitlab-terraform` script is sourced, it
configures the environment for a `terraform` call, but does not
actually run `terraform`. You can source the script when you need to do
extra steps to prepare your environment, or to use alternative
tools like `terragrunt`.
To source the script, execute:
```shell
source $(which gitlab-terraform)
```
Some shells, like BusyBox, do not support the case
of another script sourcing your script. For more information, see [this Stack Overflow thread](https://stackoverflow.com/a/28776166).
To resolve this issue, you should use `bash`, `zsh` or `ksh`, or source `gitlab-terraform` directly
from the shell.
### Commands
You can run `gitlab-terraform` with the following commands.
| Command | Forwards command line? | Implicit init? | Description |
|------------------------------|------------------------|-----------------------|--------------------------------------------------------------------------------------------------------|
| `gitlab-terraform apply` | Yes | Yes | Runs `terraform apply`. |
| `gitlab-terraform destroy` | Yes | Yes | Runs `terraform destroy`. |
| `gitlab-terraform fmt` | Yes | No | Runs `terraform fmt` in check mode. |
| `gitlab-terraform init` | Yes | Not applicable | Runs `terraform init`. |
| `gitlab-terraforn plan` | Yes | Yes | Runs `terraform plan` and produces a `plan.cache` file. |
| `gitlab-terraform plan-json` | No | No | Converts a `plan.cache` file into a GitLab Terraform report for a [MR integration](mr_integration.md). |
| `gitlab-terraform validate` | Yes | Yes (without backend) | Runs `terraform validate`. |
| `gitlab-terraform -- <cmd>` | Yes | No | Runs `terraform <cmd>`, even if it is wrapped. |
| `gitlab-terraform <cmd>` | Yes | No | Runs `terraform <cmd>`, if the command is not wrapped. |
### Generic variables
When you run `gitlab-terraform`, these variables are configured.
| Variable | Default | Description |
|----------------------|--------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `TF_ROOT` | Not set | Root of the Terraform configuration. If set, it is used as the Terraform `-chdir` argument value. All read and written files are relative to the given configuration root. |
| `TF_CLI_CONFIG_FILE` | `$HOME/.terraformrc` | Location of the [Terraform configuration file](https://developer.hashicorp.com/terraform/cli/config/config-file). |
| `TF_IN_AUTOMATION` | `true` | Set to `true` to indicate that Terraform commands are automated. |
| `TF_GITLAB_SOURCED` | `false` | Set to `true` if `gitlab-terraform` [was sourced](#source-but-do-not-run-the-helper-script). |
| `TF_PLAN_CACHE` | `$TF_ROOT/plan.cache` or `$PWD/plan.cache` | Location of the plan cache file. If `TF_ROOT` is not set, then its path is relative to the current working directory (`$PWD`). |
| `TF_PLAN_JSON` | `$TF_ROOT/plan.json` or `$PWD/plan.json` | Location of the plan JSON file for [MR integration](mr_integration.md). If `TF_ROOT` is not set, then its path is relative to the current working directory (`$PWD`). |
| `DEBUG_OUTPUT` | `"false"` | If set to `"true"` every statement is logged with `set -x`. |
### GitLab-managed Terraform state variables
When you run `gitlab-terraform`, these variables are configured.
| Variable | Default | Description |
|--------------------------|-------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `TF_STATE_NAME` | Not set | If `TF_ADDRESS` is not set, and `TF_STATE_NAME` is provided, then the value of `TF_STATE_NAME` is used as [GitLab-managed Terraform State](terraform_state.md) name. |
| `TF_ADDRESS` | Terraform State API URL for `$TF_STATE_NAME` | Used as default for [`TF_HTTP_ADDRESS`](https://developer.hashicorp.com/terraform/language/settings/backends/http#address). Uses `TF_STATE_NAME` as [GitLab-managed Terraform State](terraform_state.md) name by default. |
| `TF_USERNAME` | [`$GITLAB_USER_LOGIN`](../../../ci/variables/predefined_variables.md) or `gitlab-ci-token` if `$TF_PASSWORD` is not set | Used as default for [`TF_HTTP_USERNAME`](https://developer.hashicorp.com/terraform/language/settings/backends/http#username). |
| `TF_PASSWORD` | [`$CI_JOB_TOKEN`](../../../ci/variables/predefined_variables.md) | Used as default for [`TF_HTTP_PASSWORD`](https://developer.hashicorp.com/terraform/language/settings/backends/http#password). |
| `TF_HTTP_ADDRESS` | `$TF_ADDRESS` | [Address to the Terraform backend](https://developer.hashicorp.com/terraform/language/settings/backends/http#address). |
| `TF_HTTP_LOCK_ADDRESS` | `$TF_ADDRESS/lock` | [Address to the Terraform backend lock endpoint](https://developer.hashicorp.com/terraform/language/settings/backends/http#lock_address). |
| `TF_HTTP_LOCK_METHOD` | `POST` | [Method to use for the Terraform backend lock endpoint](https://developer.hashicorp.com/terraform/language/settings/backends/http#lock_method). |
| `TF_HTTP_UNLOCK_ADDRESS` | `$TF_ADDRESS/lock` | [Address to the Terraform backend unlock endpoint](https://developer.hashicorp.com/terraform/language/settings/backends/http#unlock_address). |
| `TF_HTTP_UNLOCK_METHOD` | `DELETE` | [Method to use for the Terraform backend unlock endpoint](https://developer.hashicorp.com/terraform/language/settings/backends/http#unlock_method). |
| `TF_HTTP_USERNAME` | `$TF_USERNAME` | [Username to authenticate with the Terraform backend](https://developer.hashicorp.com/terraform/language/settings/backends/http#username). |
| `TF_HTTP_PASSWORD` | `$TF_PASSWORD` | [Password to authenticate with the Terraform backend](https://developer.hashicorp.com/terraform/language/settings/backends/http#password). |
| `TF_HTTP_RETRY_WAIT_MIN` | `5` | [Minimum time in seconds to wait](https://developer.hashicorp.com/terraform/language/settings/backends/http#retry_wait_min) between HTTP request attempts to the Terraform backend. |
### Command variables
When you run `gitlab-terraform`, these variables are configured.
| Variable | Default | Description |
|--------------------------|----------|-------------------------------------------------------------------------------------------|
| `TF_IMPLICIT_INIT` | `true` | If `true`, an implicit `terraform init` runs before the wrapped commands that require it. |
| `TF_INIT_NO_RECONFIGURE` | `false` | If `true`, the implicit `terraform init` runs without `-reconfigure`. |
| `TF_INIT_FLAGS` | Not set | Additional `terraform init` flags. |
### Terraform input variables
When you run `gitlab-terraform`, these Terraform input variables are set automatically.
For more information about the default values, see [Predefined variables](../../../ci/variables/predefined_variables.md).
| Variable | Default |
|-------------------------------|-------------------------|
| `TF_VAR_CI_JOB_ID` | `$CI_JOB_ID` |
| `TF_VAR_CI_COMMIT_SHA` | `$CI_COMMIT_SHA` |
| `TF_VAR_CI_JOB_STAGE` | `$CI_JOB_STAGE` |
| `TF_VAR_CI_PROJECT_ID` | `$CI_PROJECT_ID` |
| `TF_VAR_CI_PROJECT_NAME` | `$CI_PROJECT_NAME` |
| `TF_VAR_CI_PROJECT_NAMESPACE` | `$CI_PROJECT_NAMESPACE` |
| `TF_VAR_CI_PROJECT_PATH` | `$CI_PROJECT_PATH` |
| `TF_VAR_CI_PROJECT_URL` | `$CI_PROJECT_URL` |
## Terraform images
The `gitlab-terraform` helper script and `terraform` itself are provided in container images
under `registry.gitlab.com/gitlab-org/terraform-images/`. You can use these images to configure
and manage your integration.
The following images are provided:
| Image name | Tag | Description |
|-------------------------------|-----------------------------|--------------------------------------------------------------------------------|
| `stable` | `latest` | Latest `terraform-images` release bundled with the latest Terraform release. |
| `releases/$TERRAFORM_VERSION` | `latest` | Latest `terraform-images` release bundled with a specific Terraform release. |
| `releases/$TERRAFORM_VERSION` | `$TERRAFORM_IMAGES_VERSION` | Specific `terraform-images` release bundled with a specific Terraform release. |
For supported combinations, see [the `terraform-images` container registry](https://gitlab.com/gitlab-org/terraform-images/container_registry).
## Related topics
- [Terraform CI/CD templates](index.md)
- [Terraform template recipes](terraform_template_recipes.md)

4
doc/user/infrastructure/iac/terraform_state.md
View File

@ -82,7 +82,9 @@ To configure GitLab CI/CD as a backend:
The output from the above `terraform` commands should be viewable in the job logs.
The `gitlab-terraform` CLI is a wrapper around the `terraform` CLI. You can [view the source code of `gitlab-terraform`](https://gitlab.com/gitlab-org/terraform-images/-/blob/master/src/bin/gitlab-terraform.sh) if you're interested.
The `gitlab-terraform` CLI is a wrapper around the `terraform` CLI. For more information,
see [GitLab Terraform helpers](gitlab_terraform_helpers.md),
or [view the source code of `gitlab-terraform`](https://gitlab.com/gitlab-org/terraform-images/-/blob/master/src/bin/gitlab-terraform.sh).
If you prefer to call the `terraform` commands explicitly, you can override
the template, and instead, use it as reference for what you can achieve.

6
lib/extracts_ref.rb
View File

@ -68,12 +68,6 @@ module ExtractsRef
return unless @ref.present?
commit
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def commit
@commit = if ref_type && Feature.enabled?(:use_ref_type_parameter, @repo.project)
@fully_qualified_ref = %(refs/#{ref_type}/#{@ref})
@repo.commit(@fully_qualified_ref)

16
lib/gitlab/ci/variables/collection.rb
View File

@ -73,7 +73,7 @@ module Gitlab
end
# `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`.
def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil)
def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true)
value.gsub(Item::VARIABLES_REGEXP) do
match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%)
full_match = match[0]
@ -88,15 +88,6 @@ module Gitlab
if variable # VARIABLE_NAME is an existing variable
if variable.file?
# Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266
if project
# We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter`
# when the variables are sent to Runner.
Gitlab::AppJsonLogger.info(event: 'file_variable_is_referenced_in_another_variable',
project_id: project.id,
variable: variable_name)
end
expand_file_refs ? variable.value : full_match
elsif variable.raw?
# With `full_match`, we defer the expansion of raw variables to the runner. If we expand them here,
@ -116,7 +107,7 @@ module Gitlab
end
# `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`.
def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil)
def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true)
sorted = Sort.new(self)
return self.class.new(self, sorted.errors) unless sorted.valid?
@ -132,8 +123,7 @@ module Gitlab
variable = item.to_runner_variable
variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined,
expand_file_refs: expand_file_refs,
expand_raw_refs: expand_raw_refs,
project: project)
expand_raw_refs: expand_raw_refs)
new_collection.append(variable)
end

2
spec/controllers/projects/refs_controller_spec.rb
View File

@ -60,7 +60,7 @@ RSpec.describe Projects::RefsController, feature_category: :source_code_manageme
'tree' | nil | lazy { project_tree_path(project, id) }
'tree' | 'heads' | lazy { project_tree_path(project, id) }
'blob' | nil | lazy { project_blob_path(project, id) }
'blob' | 'heads' | lazy { project_blob_path(project, id, ref_type: 'heads') }
'blob' | 'heads' | lazy { project_blob_path(project, id) }
'graph' | nil | lazy { project_network_path(project, id) }
'graph' | 'heads' | lazy { project_network_path(project, id, ref_type: 'heads') }
'graphs' | nil | lazy { project_graph_path(project, id) }

59
spec/features/search/user_searches_for_code_spec.rb
View File

@ -17,20 +17,24 @@ RSpec.describe 'User searches for code', :js, :disable_rate_limiter, feature_cat
sign_in(user)
end
it 'finds a file' do
visit(project_path(project))
context 'when on a project page' do
before do
visit(project_path(project))
end
submit_search('application.js')
select_search_scope('Code')
it 'finds a file' do
submit_search('application.js')
select_search_scope('Code')
expect(page).to have_selector('.results', text: 'application.js')
expect(page).to have_selector('.file-content .code')
expect(page).to have_selector("span.line[lang='javascript']")
expect(page).to have_link('application.js', href: %r{master/files/js/application.js})
expect(page).to have_button('Copy file path')
expect(page).to have_selector('.results', text: 'application.js')
expect(page).to have_selector('.file-content .code')
expect(page).to have_selector("span.line[lang='javascript']")
expect(page).to have_link('application.js', href: %r{master/files/js/application.js})
expect(page).to have_button('Copy file path')
end
end
context 'when on a project page' do
context 'when on a project search page' do
before do
visit(search_path)
find('[data-testid="project-filter"]').click
@ -47,28 +51,31 @@ RSpec.describe 'User searches for code', :js, :disable_rate_limiter, feature_cat
let(:additional_params) { { project_id: project.id } }
end
it 'finds code and links to blob' do
expected_result = 'Update capybara, rspec-rails, poltergeist to recent versions'
context 'when searching code' do
let(:expected_result) { 'Update capybara, rspec-rails, poltergeist to recent versions' }
fill_in('dashboard_search', with: 'rspec')
find('.gl-search-box-by-click-search-button').click
before do
fill_in('dashboard_search', with: 'rspec')
find('.gl-search-box-by-click-search-button').click
end
expect(page).to have_selector('.results', text: expected_result)
it 'finds code and links to blob' do
expect(page).to have_selector('.results', text: expected_result)
find("#blob-L3").click
expect(current_url).to match(%r{blob/master/.gitignore#L3})
end
find("#blob-L3").click
expect(current_url).to match(%r{blob/master/.gitignore#L3})
end
it 'finds code and links to blame' do
expected_result = 'Update capybara, rspec-rails, poltergeist to recent versions'
it 'finds code and links to blame' do
expect(page).to have_selector('.results', text: expected_result)
fill_in('dashboard_search', with: 'rspec')
find('.gl-search-box-by-click-search-button').click
find("#blame-L3").click
expect(current_url).to match(%r{blame/master/.gitignore#L3})
end
expect(page).to have_selector('.results', text: expected_result)
find("#blame-L3").click
expect(current_url).to match(%r{blame/master/.gitignore#L3})
it_behaves_like 'code highlight' do
subject { page }
end
end
it 'search multiple words with refs switching' do

40
spec/lib/gitlab/ci/variables/collection_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::Ci::Variables::Collection do
RSpec.describe Gitlab::Ci::Variables::Collection, feature_category: :pipeline_authoring do
describe '.new' do
it 'can be initialized with an array' do
variable = { key: 'VAR', value: 'value', public: true, masked: false }
@ -585,43 +585,5 @@ RSpec.describe Gitlab::Ci::Variables::Collection do
end
end
end
context 'with the file_variable_is_referenced_in_another_variable logging' do
let(:collection) do
Gitlab::Ci::Variables::Collection.new
.append(key: 'VAR1', value: 'test-1')
.append(key: 'VAR2', value: '$VAR1')
.append(key: 'VAR3', value: '$VAR1', raw: true)
.append(key: 'FILEVAR4', value: 'file-test-4', file: true)
.append(key: 'VAR5', value: '$FILEVAR4')
.append(key: 'VAR6', value: '$FILEVAR4', raw: true)
end
subject(:sort_and_expand_all) { collection.sort_and_expand_all(project: project) }
context 'when a project is not passed' do
let(:project) {}
it 'does not log anything' do
expect(Gitlab::AppJsonLogger).not_to receive(:info)
sort_and_expand_all
end
end
context 'when a project is passed' do
let(:project) { create(:project) }
it 'logs file_variable_is_referenced_in_another_variable once for VAR5' do
expect(Gitlab::AppJsonLogger).to receive(:info).with(
event: 'file_variable_is_referenced_in_another_variable',
project_id: project.id,
variable: 'FILEVAR4'
).once
sort_and_expand_all
end
end
end
end
end

26
spec/presenters/blob_presenter_spec.rb
View File

@ -31,32 +31,6 @@ RSpec.describe BlobPresenter do
it { expect(presenter.replace_path).to eq("/#{project.full_path}/-/update/#{blob.commit_id}/#{blob.path}") }
end
context 'when blob has ref_type' do
before do
blob.ref_type = 'heads'
end
describe '#web_url' do
it { expect(presenter.web_url).to eq("http://localhost/#{project.full_path}/-/blob/#{blob.commit_id}/#{blob.path}?ref_type=heads") }
end
describe '#web_path' do
it { expect(presenter.web_path).to eq("/#{project.full_path}/-/blob/#{blob.commit_id}/#{blob.path}?ref_type=heads") }
end
describe '#edit_blob_path' do
it { expect(presenter.edit_blob_path).to eq("/#{project.full_path}/-/edit/#{blob.commit_id}/#{blob.path}?ref_type=heads") }
end
describe '#raw_path' do
it { expect(presenter.raw_path).to eq("/#{project.full_path}/-/raw/#{blob.commit_id}/#{blob.path}?ref_type=heads") }
end
describe '#replace_path' do
it { expect(presenter.replace_path).to eq("/#{project.full_path}/-/update/#{blob.commit_id}/#{blob.path}?ref_type=heads") }
end
end
describe '#can_current_user_push_to_branch' do
let(:branch_exists) { true }

10
spec/presenters/ci/build_runner_presenter_spec.rb
View File

@ -349,16 +349,6 @@ RSpec.describe Ci::BuildRunnerPresenter do
public: false, masked: false }
)
end
it 'logs file_variable_is_referenced_in_another_variable' do
expect(Gitlab::AppJsonLogger).to receive(:info).with(
event: 'file_variable_is_referenced_in_another_variable',
project_id: project.id,
variable: 'file_var'
).once
runner_variables
end
end
context 'when there is a raw variable to expand' do

2
spec/services/merge_requests/update_service_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe MergeRequests::UpdateService, :mailer do
RSpec.describe MergeRequests::UpdateService, :mailer, feature_category: :code_review do
include ProjectForksHelper
let(:group) { create(:group, :public) }