diff --git a/app/assets/javascripts/search/graphql/blob_search_zoekt_count_only.query.graphql b/app/assets/javascripts/search/graphql/blob_search_zoekt_count_only.query.graphql index 3e8fd1d54c7..fd449f67522 100644 --- a/app/assets/javascripts/search/graphql/blob_search_zoekt_count_only.query.graphql +++ b/app/assets/javascripts/search/graphql/blob_search_zoekt_count_only.query.graphql @@ -2,8 +2,8 @@ query getBlobSearchCountQuery( $search: String! $groupId: GroupID $projectId: ProjectID - $regex: Boolean $chunkCount: Int + $regex: Boolean $includeArchived: Boolean $includeForked: Boolean ) { @@ -11,8 +11,8 @@ query getBlobSearchCountQuery( search: $search groupId: $groupId projectId: $projectId - regex: $regex chunkCount: $chunkCount + regex: $regex includeArchived: $includeArchived includeForked: $includeForked ) { diff --git a/app/assets/javascripts/search/sidebar/components/scope_sidebar_navigation.vue b/app/assets/javascripts/search/sidebar/components/scope_sidebar_navigation.vue index 09807822b6b..9a1d8695b20 100644 --- a/app/assets/javascripts/search/sidebar/components/scope_sidebar_navigation.vue +++ b/app/assets/javascripts/search/sidebar/components/scope_sidebar_navigation.vue @@ -12,7 +12,7 @@ import getBlobSearchCountQuery from '~/search/graphql/blob_search_zoekt_count_on import glFeatureFlagsMixin from '~/vue_shared/mixins/gl_feature_flags_mixin'; import { DEFAULT_FETCH_CHUNKS } from '~/search/results/constants'; import { RECEIVE_NAVIGATION_COUNT } from '../../store/mutation_types'; -import { NAV_LINK_DEFAULT_CLASSES, NAV_LINK_COUNT_DEFAULT_CLASSES } from '../constants'; +import { NAV_LINK_DEFAULT_CLASSES, NAV_LINK_COUNT_DEFAULT_CLASSES, SCOPE_BLOB } from '../constants'; export default { name: 'ScopeSidebarNavigation', @@ -40,15 +40,11 @@ export default { }; }, skip() { - return !( - (this.query?.group_id || this.query?.project_id) && - this.glFeatures.zoektMultimatchFrontend && - this.zoektAvailable - ); + return this.legacyBlobsCount; }, update(data) { this.receiveNavigationCount({ - key: 'blobs', + key: SCOPE_BLOB, count: data?.blobSearch?.matchCount.toString(), }); }, @@ -64,11 +60,32 @@ export default { }; }, computed: { - ...mapGetters(['navigationItems']), + ...mapGetters(['navigationItems', 'currentScope']), ...mapState(['zoektAvailable', 'query']), + legacyBlobsCount() { + if (this.currentScope === SCOPE_BLOB) { + // if current scope is blobs skip this no matter what + return true; + } + + if (!this.glFeatures.zoektMultimatchFrontend || !this.zoektAvailable) { + // skip this if no multimatch feature is available + return true; + } + + if ( + !this.glFeatures.zoektCrossNamespaceSearch && + !(this.query?.group_id || this.query?.project_id) + ) { + // skip this if we have no group or project ID or crossNamespaceSearch is enabled + return true; + } + + return false; + }, }, created() { - this.fetchSidebarCount(); + this.fetchSidebarCount(this.legacyBlobsCount); }, methods: { ...mapActions(['fetchSidebarCount']), diff --git a/app/assets/javascripts/search/store/actions.js b/app/assets/javascripts/search/store/actions.js index c7a7e338898..b22b8865124 100644 --- a/app/assets/javascripts/search/store/actions.js +++ b/app/assets/javascripts/search/store/actions.js @@ -23,7 +23,6 @@ import { getAggregationsUrl, prepareSearchAggregations, setDataToLS, - skipBlobESCount, buildDocumentTitle, } from './utils'; @@ -105,13 +104,17 @@ export const setFrequentProject = ({ state, commit }, item) => { commit(types.LOAD_FREQUENT_ITEMS, { key: PROJECTS_LOCAL_STORAGE_KEY, data: frequentItems }); }; -export const fetchSidebarCount = ({ commit, state }) => { +const filterBlobs = (navigationItemScope, skipBlobs) => { + return navigationItemScope !== SCOPE_BLOB ? true : skipBlobs; +}; + +export const fetchSidebarCount = ({ commit, state }, skipBlobs) => { const items = Object.values(state.navigation) .filter( (navigationItem) => !navigationItem.active && navigationItem.count_link && - skipBlobESCount(state, navigationItem.scope), + filterBlobs(navigationItem.scope, skipBlobs), ) .map((navItem) => { const navigationItem = { ...navItem }; diff --git a/app/assets/javascripts/search/store/utils.js b/app/assets/javascripts/search/store/utils.js index 2f535fbb6f2..51cf0a0c3a8 100644 --- a/app/assets/javascripts/search/store/utils.js +++ b/app/assets/javascripts/search/store/utils.js @@ -2,11 +2,7 @@ import { isEqual, orderBy, isEmpty } from 'lodash'; import AccessorUtilities from '~/lib/utils/accessor'; import { formatNumber } from '~/locale'; import { joinPaths, queryToObject, objectToQuery, getBaseURL } from '~/lib/utils/url_utility'; -import { - LABEL_AGREGATION_NAME, - LANGUAGE_FILTER_PARAM, - SCOPE_BLOB, -} from '~/search/sidebar/constants'; +import { LABEL_AGREGATION_NAME, LANGUAGE_FILTER_PARAM } from '~/search/sidebar/constants'; import { SEARCH_SCOPE, USER_HANDLE, @@ -226,14 +222,6 @@ export const scopeCrawler = (navigation, parentScope = null) => { return null; }; -export const skipBlobESCount = (state, itemScope) => - !( - (state.query?.group_id || state.query?.project_id) && - window.gon.features?.zoektMultimatchFrontend && - state.zoektAvailable && - itemScope === SCOPE_BLOB - ); - export const buildDocumentTitle = (title) => { const prevTitle = document.title; diff --git a/data/deprecations/17-2-deprecation-rename-skip-gitguardian-scanning.yml b/data/deprecations/17-2-deprecation-rename-skip-gitguardian-scanning.yml index bc89148878a..9674e8e093c 100644 --- a/data/deprecations/17-2-deprecation-rename-skip-gitguardian-scanning.yml +++ b/data/deprecations/17-2-deprecation-rename-skip-gitguardian-scanning.yml @@ -4,7 +4,9 @@ breaking_change: false body: | # Do not modify this line, instead modify the lines below. The options to skip GitGuardian secret detection, `[skip secret detection]` and `secret_detection.skip_all`, are deprecated. You should use `[skip secret push protection]` and `secret_push_protection.skip_all` instead. - stage: secure + + While we recommend using the new wording, we no longer will remove the old option in GitLab 18.0. + stage: application_security_testing tiers: premium, ultimate issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/470119 documentation_url: https://docs.gitlab.com/user/project/integrations/git_guardian/#skip-secret-detection diff --git a/data/deprecations/17-3-codeclimate.yml b/data/deprecations/17-3-codeclimate.yml index 972ff728538..c207818b512 100644 --- a/data/deprecations/17-3-codeclimate.yml +++ b/data/deprecations/17-3-codeclimate.yml @@ -10,8 +10,10 @@ issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/471677 # (required) Link to the deprecation issue in GitLab body: | # (required) Do not modify this line, instead modify the lines below. In GitLab 19.0, we will remove CodeClimate-based Code Quality scanning. + This change was previously scheduled for GitLab 18.0 and has now been delayed. + In its place, you should use quality tools directly in your CI/CD pipeline and [provide the tool's report as an artifact](https://docs.gitlab.com/ci/testing/code_quality/#import-code-quality-results-from-a-cicd-job). - Many tools already support the required report format, and you can integrate them by following the [documented steps](https://docs.gitlab.com/ci/testing/code_quality/#integrate-common-tools-with-code-quality). + We've already documented how to integrate many tools directly, and you can integrate them by following the [documentation](https://docs.gitlab.com/ci/testing/code_quality/#integrate-common-tools-with-code-quality). We expect to implement this change by: diff --git a/data/deprecations/17-4-secure-container-registries.yml b/data/deprecations/17-4-secure-container-registries.yml index a032a098976..2d419bd9671 100644 --- a/data/deprecations/17-4-secure-container-registries.yml +++ b/data/deprecations/17-4-secure-container-registries.yml @@ -1,5 +1,5 @@ - title: "Public use of Secure container registries is deprecated" - removal_milestone: "Pending" + removal_milestone: "Cancelled" announcement_milestone: "17.4" breaking_change: true window: 1 diff --git a/data/deprecations/17-7-deprecate-SPPgraphqlmutation.yml b/data/deprecations/17-7-deprecate-SPPgraphqlmutation.yml index cb01f4d65e3..6a2f03281f4 100644 --- a/data/deprecations/17-7-deprecate-SPPgraphqlmutation.yml +++ b/data/deprecations/17-7-deprecate-SPPgraphqlmutation.yml @@ -1,5 +1,5 @@ - title: "Rename `setPreReceiveSecretDetection` GraphQL mutation to `setSecretPushProtection`" - removal_milestone: "Pending" + removal_milestone: "18.0" announcement_milestone: "17.7" breaking_change: false window: 3 @@ -12,6 +12,8 @@ manual_task: true body: | The `setPreReceiveSecretDetection` GraphQL mutation has been renamed to `setSecretPushProtection`. We are also renaming some fields in the mutation's response to reflect the name change of the feature `pre_receive_secret_detection` to `secret_push_protection`. - Following [new guidance](https://docs.gitlab.com/development/api_styleguide/#what-to-do-instead-of-a-breaking-change), we're adapting the schema to support both `setPreReceiveSecretDetection` and `setSecretPushProtection`. You can use either parameter because they both point to the `secret_push_protection_enabled` column. In GitLab 18.0, we will [remove](https://gitlab.com/gitlab-org/gitlab/-/issues/514414) the old `setPreReceiveSecretDetection` column. + We have added the new mutation name, but will no longer remove the old mutation name in GitLab 18.0 as originally announced. + + We will still update the database to [remove](https://gitlab.com/gitlab-org/gitlab/-/issues/512996) the old `pre_receive_secret_detection_enabled` database column, but you'll be able to use either mutation name. Both will reflect the value of the new `secret_push_protection_enabled` database column. documentation_url: https://docs.gitlab.com/api/graphql/reference/#mutationsetsecretpushprotection diff --git a/data/deprecations/17-9-ast-da-dast-devtools-api-timeout-env-change.yml b/data/deprecations/17-9-ast-da-dast-devtools-api-timeout-env-change.yml index 3d73377594a..fc8378cd791 100644 --- a/data/deprecations/17-9-ast-da-dast-devtools-api-timeout-env-change.yml +++ b/data/deprecations/17-9-ast-da-dast-devtools-api-timeout-env-change.yml @@ -16,13 +16,13 @@ To lessen potential disruptions, we will incrementally adjust the default timeout value according to this schedule: - | Timeout value | Milestone | - |:--------------|:----------| - | 45 | Current | - | 30 | 18.0 | - | 20 | 18.1 | - | 10 | 18.2 | - | 5 | 18.3 | + | Timeout value | Milestone | + |:--------------|:------------------| + | 45 | 17.11 and earlier | + | 30 | 18.0 | + | 20 | 18.1 | + | 10 | 18.2 | + | 5 | 18.3 | end_of_support_milestone: tiers: [Ultimate] diff --git a/data/deprecations/17-9-enable-gitlab-advanced-sast-by-default.yml b/data/deprecations/17-9-enable-gitlab-advanced-sast-by-default.yml index b94fa0e096c..3f6c0cc4559 100644 --- a/data/deprecations/17-9-enable-gitlab-advanced-sast-by-default.yml +++ b/data/deprecations/17-9-enable-gitlab-advanced-sast-by-default.yml @@ -3,7 +3,7 @@ announcement_milestone: "17.9" breaking_change: true window: 1 - reporter: adamcohen + reporter: connorgilbert stage: application security testing issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/513685 impact: medium @@ -13,7 +13,8 @@ manual_task: true body: | # (required) Don't change this line. In GitLab 19.0, we will update the [SAST CI/CD templates](https://docs.gitlab.com/user/application_security/sast#stable-vs-latest-sast-templates) to enable [GitLab Advanced SAST](https://docs.gitlab.com/user/application_security/sast/gitlab_advanced_sast) by default in projects with GitLab Ultimate. - Before this change, the GitLab Advanced SAST analyzer was enabled only if you set the CI/CD variable `GITLAB_ADVANCED_SAST_ENABLED` to `true`. + Before this change, the GitLab Advanced SAST analyzer is enabled only if you set the CI/CD variable `GITLAB_ADVANCED_SAST_ENABLED` to `true`. + This change was previously scheduled for GitLab 18.0 and has now been delayed. Advanced SAST delivers more accurate results by using cross-file, cross-function scanning and a new ruleset. Advanced SAST takes over coverage for [supported languages](https://docs.gitlab.com/user/application_security/sast/gitlab_advanced_sast#supported-languages) and disables scanning for that language in the previous scanner. @@ -21,4 +22,4 @@ Because it scans your project in more detail, Advanced SAST may take more time to scan your project. If needed, you can [disable GitLab Advanced SAST](https://docs.gitlab.com/user/application_security/sast/gitlab_advanced_sast#disable-gitlab-advanced-sast-scanning) by setting the CI/CD variable `GITLAB_ADVANCED_SAST_ENABLED` to `false`. - You can set this variable in your project, group, or policy now to prevent Advanced SAST from being enabled by default in GitLab 18.0. + You can set this variable in your project, group, or policy now to prevent Advanced SAST from being enabled by default in GitLab 19.0. diff --git a/data/deprecations/17-9-pre_receive_secret_detection_enabled-REST-API.yml b/data/deprecations/17-9-pre_receive_secret_detection_enabled-REST-API.yml index 686dbf4c095..3bd34297dfc 100644 --- a/data/deprecations/17-9-pre_receive_secret_detection_enabled-REST-API.yml +++ b/data/deprecations/17-9-pre_receive_secret_detection_enabled-REST-API.yml @@ -1,6 +1,6 @@ - title: "REST API endpoint `pre_receive_secret_detection_enabled` is deprecated" announcement_milestone: "17.9" - removal_milestone: "Pending" + removal_milestone: "18.0" breaking_change: false reporter: abellucci stage: application_security_testing @@ -11,6 +11,9 @@ manual_task: true body: | # (required) Don't change this line. The REST API endpoint `pre_receive_secret_detection_enabled` is deprecated in favor of `secret_push_protection_enabled`. We are renaming some API fields to reflect the name change of the feature `pre_receive_secret_detection` to `secret_push_protection`. - Following [new guidance](https://docs.gitlab.com/development/api_styleguide/#what-to-do-instead-of-a-breaking-change), we will adapt the schema to support both `pre_receive_secret_detection_enabled` and `secret_push_protection_enabled`. We will still [remove](https://gitlab.com/gitlab-org/gitlab/-/issues/512996) the old `pre_receive_secret_detection_enabled` column, but customers will be able to use either parameter, with both pointing to the new `secret_push_protection_enabled` column. + + We have added the new API field name, but will no longer remove the old field name in GitLab 18.0 as originally announced. + + We will still update the database to [remove](https://gitlab.com/gitlab-org/gitlab/-/issues/512996) the old `pre_receive_secret_detection_enabled` database column, but you'll be able to use either API field name. Both will reflect the value of the new `secret_push_protection_enabled` database column. tiers: ultimate documentation_url: https://docs.gitlab.com/api/projects/#secret-push-protection-status diff --git a/data/deprecations/17-9-remove-cache-for-sast-jobs.yml b/data/deprecations/17-9-remove-cache-for-sast-jobs.yml index cfae5b07386..f467b032f8c 100644 --- a/data/deprecations/17-9-remove-cache-for-sast-jobs.yml +++ b/data/deprecations/17-9-remove-cache-for-sast-jobs.yml @@ -1,5 +1,5 @@ - title: "SAST jobs no longer use global cache settings" - removal_milestone: "Pending" + removal_milestone: "Cancelled" announcement_milestone: "17.9" breaking_change: true window: 1 diff --git a/data/deprecations/17-9-secret-detection-root-user.yml b/data/deprecations/17-9-secret-detection-root-user.yml index 09f71fd8643..26136578e1e 100644 --- a/data/deprecations/17-9-secret-detection-root-user.yml +++ b/data/deprecations/17-9-secret-detection-root-user.yml @@ -1,5 +1,5 @@ - title: "Secret detection analyzer doesn't run as root user by default" - removal_milestone: "19.0" + removal_milestone: "Cancelled" announcement_milestone: "17.9" breaking_change: false window: 3 diff --git a/data/deprecations/17-9-secure-analyzers-bump.yml b/data/deprecations/17-9-secure-analyzers-bump.yml index 8bc8aa826cc..8c303c5e69b 100644 --- a/data/deprecations/17-9-secure-analyzers-bump.yml +++ b/data/deprecations/17-9-secure-analyzers-bump.yml @@ -11,22 +11,21 @@ resolution_role: Developer manual_task: false body: | # Do not modify this line, instead modify the lines below. - The Application Security Testing stage will be bumping the major versions of its analyzers in - tandem with the GitLab 18.0 release. + In GitLab 18.0, we will update the major version of all Application Security Testing analyzer container images. If you are not using the default included templates, or have pinned your analyzer versions, you must update your CI/CD job definition to either remove the pinned version or update the latest major version. - Users of GitLab 17.0 to GitLab 17.11 will continue to experience analyzer updates until the + Users of GitLab 17.0 to GitLab 17.11 will continue to receive analyzer updates until the release of GitLab 18.0, after which all newly fixed bugs and released features will be released only in the new major version of the analyzers. + However, we will not remove any published container images from the container registry. We do not backport bugs and features to deprecated versions as per our maintenance policy. As required, security patches will be backported within the latest 3 minor releases. - Specifically, the following analyzers are being deprecated and will no longer be updated after - the GitLab 18.0 release: + Specifically, the following analyzers will no longer be updated after the GitLab 18.0 release: - GitLab Advanced SAST: version 1 - Container Scanning: version 7 diff --git a/data/deprecations/17-9-spotbugs-builds.yml b/data/deprecations/17-9-spotbugs-builds.yml index b37e0a3e8d2..3b56e859647 100644 --- a/data/deprecations/17-9-spotbugs-builds.yml +++ b/data/deprecations/17-9-spotbugs-builds.yml @@ -1,5 +1,5 @@ - title: "Support for project build as part of SpotBugs scans" - removal_milestone: "18.0" + removal_milestone: "Cancelled" announcement_milestone: "17.9" breaking_change: false window: 1 # Note: a change window is not applicable to a non-breaking change @@ -9,7 +9,7 @@ impact: low scope: project resolution_role: Developer - manual_task: true + manual_task: false body: | # (required) Don't change this line. The SpotBugs [SAST analyzer](https://docs.gitlab.com/user/application_security/sast/#supported-languages-and-frameworks) can perform a build when the artifacts to be scanned aren't present. While this usually works well for simple projects, it can fail on more complex builds. @@ -18,6 +18,8 @@ 1. [Pre-compile](https://docs.gitlab.com/user/application_security/sast/#pre-compilation) the project. 1. Pass the artifacts you want to scan to the analyzer. + + This is not a change in functionality, so we have marked this announcement "Cancelled" for clarity. end_of_support_milestone: 18.0 tiers: [Free, Silver, Gold, Core, Premium, Ultimate] documentation_url: https://docs.gitlab.com/user/application_security/sast/troubleshooting/#project-couldnt-be-built diff --git a/doc/administration/invalidate_markdown_cache.md b/doc/administration/invalidate_markdown_cache.md index 2d3a31c9296..9a2183a3ea8 100644 --- a/doc/administration/invalidate_markdown_cache.md +++ b/doc/administration/invalidate_markdown_cache.md @@ -23,13 +23,109 @@ in the cached text would still refer to the old URL. ## Invalidate the cache -Pre-requisite: +You can invalidate the Markdown cache by using either the API or the Rails console. -- You must be an administrator. +### Use the API -To avoid problems caused by cached HTML versions, invalidate the existing cache by increasing the `local_markdown_version` -setting in application settings [using the API](../api/settings.md#update-application-settings): +Prerequisites: -```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/application/settings?local_markdown_version=" -``` +- You must have administrator access. + +To invalidate the existing cache using the API: + +1. Increase the `local_markdown_version` setting in application settings by sending a PUT request: + + ```shell + curl --request PUT --header "PRIVATE-TOKEN: " \ + --url "https://gitlab.example.com/api/v4/application/settings?local_markdown_version=" + ``` + +For more information about this API endpoint, see [update application settings](../api/settings.md#update-application-settings). + +### Use the Rails console + +Prerequisites: + +- You must have [Rails console](operations/rails_console.md) access. + +#### For a group + +To invalidate the cache for a group: + +1. Start a Rails console: + + ```shell + sudo gitlab-rails console + ``` + +1. Find the group to update: + + ```ruby + group = Group.find() + ``` + +1. Invalidate the cache for all projects in the group: + + ```ruby + group.all_projects.each_slice(10) do |projects| + projects.each do |project| + # Invalidate issues + project.issues.update_all( + description_html: nil, + title_html: nil + ) + + # Invalidate merge requests + project.merge_requests.update_all( + description_html: nil, + title_html: nil + ) + + # Invalidate notes/comments + project.notes.update_all(note_html: nil) + end + + # Pause for one second after updating 10 projects + sleep 1 + end + ``` + +#### For a project + +To invalidate the cache for a single project: + +1. Start a Rails console: + + ```shell + sudo gitlab-rails console + ``` + +1. Find the project to update: + + ```ruby + project = Project.find() + ``` + +1. Invalidate issues: + + ```ruby + project.issues.update_all( + description_html: nil, + title_html: nil + ) + ``` + +1. Invalidate merge requests: + + ```ruby + project.merge_requests.update_all( + description_html: nil, + title_html: nil + ) + ``` + +1. Invalidate notes and comments: + + ```ruby + project.notes.update_all(note_html: nil) + ``` diff --git a/doc/api/packages.md b/doc/api/packages.md index c9b8ed5b722..1f414c5eb3f 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -38,7 +38,7 @@ GET /projects/:id/packages | `id` | integer/string | yes | ID or [URL-encoded path of the project](rest/_index.md#namespaced-paths). | | `order_by` | string | no | The field to use as order. One of `created_at` (default), `name`, `version`, or `type`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | -| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, `helm`, `terraform_module`, or `golang`. | +| `package_type` | string | no | Filter the returned packages by type. One of `composer`, `conan`, `generic`, `golang`, `helm`, `maven`, `npm`, `nuget`, `pypi`, or `terraform_module`. | | `package_name` | string | no | Filter the project packages with a fuzzy search by name. | | `package_version` | string | no | Filter the project packages by version. If used in combination with `include_versionless`, then no versionless packages are returned. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349065) in GitLab 16.6. | | `include_versionless` | boolean | no | When set to true, versionless packages are included in the response. | @@ -108,7 +108,7 @@ GET /groups/:id/packages | `exclude_subgroups` | boolean | no | If the parameter is included as true, packages from projects from subgroups are not listed. Default is `false`. | | `order_by` | string | no | The field to use as order. One of `created_at` (default), `name`, `version`, `type`, or `project_path`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | -| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, `helm`, or `golang`. | +| `package_type` | string | no | Filter the returned packages by type. One of `composer`, `conan`, `generic`, `golang`, `helm`, `maven`, `npm`, `nuget`, `pypi`, or `terraform_module`. | | `package_name` | string | no | Filter the project packages with a fuzzy search by name. | | `package_version` | string | no | Filter the returned packages by version. If used in combination with `include_versionless`, then no versionless packages are returned. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349065) in GitLab 16.6. | | `include_versionless` | boolean | no | When set to true, versionless packages are included in the response. | diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index 31d0cad9b44..d37346335f1 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -32,7 +32,59 @@ of the following executors is required: - [Docker](https://docs.gitlab.com/runner/executors/docker.html). - [Docker Machine](https://docs.gitlab.com/runner/executors/docker_machine.html). -## Building a Docker image with kaniko +## Authentication with registries + +Authentication is required when building and pushing images with kaniko: + +- For the GitLab container registry, authentication happens automatically without any manual configuration. +- For pulling images through the GitLab dependency proxy, additional authentication configuration is required. + +For more information on authentication with other registries, +see [pushing to different registries](https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#pushing-to-different-registries). + +### Authenticate with the container registry + +When pushing to the GitLab container registry, authentication happens automatically without requiring any manual configuration: + +- GitLab CI/CD automatically creates a `config.json` file at `/kaniko/.docker/config.json` before kaniko runs. +- This file contains authentication credentials needed for the container registry. +- These credentials are derived from the `CI_REGISTRY`, `CI_REGISTRY_USER`, and `CI_REGISTRY_PASSWORD` predefined CI/CD variables. +- kaniko automatically detects and uses this file for authentication. + +{{< alert type="note" >}} + +You typically won't see this configuration file when inspecting the container as it's managed internally by GitLab CI/CD. +Manually creating or modifying this file might cause authentication issues. + +{{< /alert >}} + +### Authenticate with the dependency proxy + +If you need to pull images through the dependency proxy, +you must add the corresponding CI/CD variables for authentication to the `config.json` file: + +```yaml +build: + script: + # Create a config.json with authentication for both the GitLab container registry and dependency proxy + - | + echo "{ + \"auths\": { + \"${CI_REGISTRY}\": { + \"auth\": \"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\" + }, + \"$(echo -n $CI_DEPENDENCY_PROXY_SERVER | awk -F[:] '{print $1}')\": { + \"auth\": \"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\" + } + } + }" > /kaniko/.docker/config.json +``` + +This configuration strips the port, for example `:443`, from `CI_DEPENDENCY_PROXY_SERVER` so you don't have to include it when referencing images. + +For more information, see [authenticate within CI/CD](../../user/packages/dependency_proxy/_index.md#authenticate-within-cicd). + +## Build and push an image When building an image with kaniko and GitLab CI/CD, you should be aware of a few important details: @@ -46,12 +98,9 @@ few important details: In the following example, kaniko is used to: 1. Build a Docker image. -1. Then push it to [GitLab container registry](../../user/packages/container_registry/_index.md). +1. Push it to [GitLab container registry](../../user/packages/container_registry/_index.md). -The job runs only when a tag is pushed. A `config.json` file is created under -`/kaniko/.docker` with the needed GitLab container registry credentials taken from the -[predefined CI/CD variables](../variables/_index.md#predefined-cicd-variables) -GitLab CI/CD provides. These are automatically read by the Kaniko tool. +The job runs only when a tag is pushed. In the last step, kaniko uses the `Dockerfile` under the root directory of the project, builds the Docker image and pushes it to the @@ -72,16 +121,7 @@ build: - if: $CI_COMMIT_TAG ``` -If you authenticate against the [Dependency Proxy](../../user/packages/dependency_proxy/_index.md#authenticate-within-cicd), -you must add the corresponding CI/CD variables for authentication to the `config.json` file: - -```yaml -- echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$(echo -n $CI_DEPENDENCY_PROXY_SERVER | awk -F[:] '{print $1}')\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json -``` - -This command strips the port, for example `:443`, from `CI_DEPENDENCY_PROXY_SERVER`, so you don't have to include it when referencing images. - -### Building an image with kaniko behind a proxy +### Build an image behind a proxy If you use a custom GitLab Runner behind an http(s) proxy, kaniko needs to be set up accordingly. This means: @@ -118,9 +158,10 @@ build: You can build [multi-arch images](https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/) inside a container by using [`manifest-tool`](https://github.com/estesp/manifest-tool). -For a detailed guide on how to build a multi-arch image, read [Building a multi-arch container image in unprivileged containers](https://blog.siemens.com/2022/07/building-a-multi-arch-container-image-in-unprivileged-containers/). +For a detailed guide on how to build a multi-arch image, see +[Building a multi-arch container image in unprivileged containers](https://blog.siemens.com/2022/07/building-a-multi-arch-container-image-in-unprivileged-containers/). -## Using a registry with a custom certificate +## Use a registry with a custom certificate When trying to push to a Docker registry that uses a certificate that is signed by a custom CA, you might get the following error: diff --git a/doc/development/advanced_search/tips.md b/doc/development/advanced_search/tips.md index a7e2a709e3a..f98550ae347 100644 --- a/doc/development/advanced_search/tips.md +++ b/doc/development/advanced_search/tips.md @@ -105,7 +105,7 @@ Elastic::ProcessBookkeepingService.new.execute {{< alert type="warning" >}} Elasticsearch tests do not run on every merge request. Add `~pipeline:run-search-tests` or `~group::global search` labels to the merge -request to run tests with the production versions of Elasticsearch and PostgreSQL. +request to run tests with the production versions of Elasticsearch and PostgreSQL. {{< /alert >}} @@ -117,7 +117,9 @@ request to run tests with the production versions of Elasticsearch and PostgreSQ - You can consider disabling the migration worker to have more control: `Feature.disable(:elastic_migration_worker)`. - See if the migration is pending: `::Elastic::DataMigrationService.pending_migrations`. - Check that the migration is not completed: `Elastic::DataMigrationService.pending_migrations.first.completed?`. - - Make sure the mappings aren't already applied by checking in Kibana: `GET gitlab-development-some-index/_mapping`. + - Make sure the mappings aren't already applied + - either by checking in Kibana `GET gitlab-development-some-index/_mapping` + - or sending a curl request `curl "http://localhost:9200/gitlab-development-some-index/_mappings" | jq` 1. Tail the logs to see logged messages: `tail -f log/elasticsearch.log`. 1. Execute the migration in one of the following ways: - Run the migration worker: `Elastic::MigrationWorker.new.perform` (remember the flag should be enabled). diff --git a/doc/development/documentation/styleguide/_index.md b/doc/development/documentation/styleguide/_index.md index e2f25ff264b..c8e6bcda210 100644 --- a/doc/development/documentation/styleguide/_index.md +++ b/doc/development/documentation/styleguide/_index.md @@ -410,7 +410,7 @@ For numbers in text, spell out zero through nine and use numbers for 10 and grea To make the source content easy to read, and to more easily compare diffs, follow these best practices when possible. -- Split long lines at approximately 100 characters. +- Split long lines at approximately 100 characters. (Exception: Do not split links.) - Start each new sentence on a new line. ### Comments @@ -877,6 +877,7 @@ Too many links can hinder readability. - Do not duplicate links on the same page. For example, on **Page A**, do not link to **Page B** multiple times. - Do not use links in headings. Headings that contain links cause errors. +- Do not use a hard line wrap between any words in a link. - Avoid multiple links in a single paragraph. - Avoid multiple links in a single task. - On any one page, try not to use more than 15 links to other pages. diff --git a/doc/development/search/advanced_search_migration_styleguide.md b/doc/development/search/advanced_search_migration_styleguide.md index 2c336d7ab52..6f89c036ed4 100644 --- a/doc/development/search/advanced_search_migration_styleguide.md +++ b/doc/development/search/advanced_search_migration_styleguide.md @@ -745,3 +745,16 @@ The MR assignee must: 1. Backup migrations from the default branch to the [migration graveyard](https://gitlab.com/gitlab-org/search-team/migration-graveyard) 1. Verify that no references to the migration or spec files exist in the `.rubocop_todo/` directory. 1. Push any required changes to the merge request. + +## ChatOps commands for monitoring migrations + +You can check migration status from Slack (or any ChatOps‑enabled channel) at any time: + +```plaintext +/chatops run search_migrations --help +/chatops run search_migrations list +/chatops run search_migrations get MigrationName +/chatops run search_migrations get VersionNumber +``` + +The above uses the [search_migrations](https://gitlab.com/gitlab-com/chatops/-/blob/master/lib/chatops/commands/search_migrations.rb) ChatOps plugin to fetch current migration state. diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index f6fa8e20a26..a817e3f529a 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -359,8 +359,10 @@ If your pipeline relies on forwarding protected variables, update your configura In GitLab 19.0, we will remove CodeClimate-based Code Quality scanning. +This change was previously scheduled for GitLab 18.0 and has now been delayed. + In its place, you should use quality tools directly in your CI/CD pipeline and [provide the tool's report as an artifact](https://docs.gitlab.com/ci/testing/code_quality/#import-code-quality-results-from-a-cicd-job). -Many tools already support the required report format, and you can integrate them by following the [documented steps](https://docs.gitlab.com/ci/testing/code_quality/#integrate-common-tools-with-code-quality). +We've already documented how to integrate many tools directly, and you can integrate them by following the [documentation](https://docs.gitlab.com/ci/testing/code_quality/#integrate-common-tools-with-code-quality). We expect to implement this change by: @@ -524,7 +526,8 @@ we will enforce keyset pagination on these APIs. In GitLab 19.0, we will update the [SAST CI/CD templates](https://docs.gitlab.com/user/application_security/sast#stable-vs-latest-sast-templates) to enable [GitLab Advanced SAST](https://docs.gitlab.com/user/application_security/sast/gitlab_advanced_sast) by default in projects with GitLab Ultimate. -Before this change, the GitLab Advanced SAST analyzer was enabled only if you set the CI/CD variable `GITLAB_ADVANCED_SAST_ENABLED` to `true`. +Before this change, the GitLab Advanced SAST analyzer is enabled only if you set the CI/CD variable `GITLAB_ADVANCED_SAST_ENABLED` to `true`. +This change was previously scheduled for GitLab 18.0 and has now been delayed. Advanced SAST delivers more accurate results by using cross-file, cross-function scanning and a new ruleset. Advanced SAST takes over coverage for [supported languages](https://docs.gitlab.com/user/application_security/sast/gitlab_advanced_sast#supported-languages) and disables scanning for that language in the previous scanner. @@ -532,7 +535,7 @@ An automated process migrates results from previous scanners after the first sca Because it scans your project in more detail, Advanced SAST may take more time to scan your project. If needed, you can [disable GitLab Advanced SAST](https://docs.gitlab.com/user/application_security/sast/gitlab_advanced_sast#disable-gitlab-advanced-sast-scanning) by setting the CI/CD variable `GITLAB_ADVANCED_SAST_ENABLED` to `false`. -You can set this variable in your project, group, or policy now to prevent Advanced SAST from being enabled by default in GitLab 18.0. +You can set this variable in your project, group, or policy now to prevent Advanced SAST from being enabled by default in GitLab 19.0. @@ -783,22 +786,6 @@ For more information about updating your storage driver configuration, see [use -
- -### Secret detection analyzer doesn't run as root user by default - -
- -- Announced in GitLab 17.9 -- Removal in GitLab 19.0 -- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/476160). - -
- -This planned change to the secret detection analyzer is cancelled. You can still use the root user by default. - -
-
### Single database connection is deprecated @@ -1218,22 +1205,21 @@ Starting in GitLab 18.0, we'll align this template's behavior with the behavior
-The Application Security Testing stage will be bumping the major versions of its analyzers in -tandem with the GitLab 18.0 release. +In GitLab 18.0, we will update the major version of all Application Security Testing analyzer container images. If you are not using the default included templates, or have pinned your analyzer versions, you must update your CI/CD job definition to either remove the pinned version or update the latest major version. -Users of GitLab 17.0 to GitLab 17.11 will continue to experience analyzer updates until the +Users of GitLab 17.0 to GitLab 17.11 will continue to receive analyzer updates until the release of GitLab 18.0, after which all newly fixed bugs and released features will be released only in the new major version of the analyzers. +However, we will not remove any published container images from the container registry. We do not backport bugs and features to deprecated versions as per our maintenance policy. As required, security patches will be backported within the latest 3 minor releases. -Specifically, the following analyzers are being deprecated and will no longer be updated after -the GitLab 18.0 release: +Specifically, the following analyzers will no longer be updated after the GitLab 18.0 release: - GitLab Advanced SAST: version 1 - Container Scanning: version 7 @@ -1379,13 +1365,13 @@ In most cases, the 45-second value was higher than the timeout value of many sca To lessen potential disruptions, we will incrementally adjust the default timeout value according to this schedule: -| Timeout value | Milestone | -|:--------------|:----------| -| 45 | Current | -| 30 | 18.0 | -| 20 | 18.1 | -| 10 | 18.2 | -| 5 | 18.3 | +| Timeout value | Milestone | +|:--------------|:------------------| +| 45 | 17.11 and earlier | +| 30 | 18.0 | +| 20 | 18.1 | +| 10 | 18.2 | +| 5 | 18.3 | @@ -1733,6 +1719,26 @@ The project page will be removed entirely from the group settings in 18.0. +
+ +### REST API endpoint `pre_receive_secret_detection_enabled` is deprecated + +
+ +- Announced in GitLab 17.9 +- Removal in GitLab 18.0 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/514413). + +
+ +The REST API endpoint `pre_receive_secret_detection_enabled` is deprecated in favor of `secret_push_protection_enabled`. We are renaming some API fields to reflect the name change of the feature `pre_receive_secret_detection` to `secret_push_protection`. + +We have added the new API field name, but will no longer remove the old field name in GitLab 18.0 as originally announced. + +We will still update the database to [remove](https://gitlab.com/gitlab-org/gitlab/-/issues/512996) the old `pre_receive_secret_detection_enabled` database column, but you'll be able to use either API field name. Both will reflect the value of the new `secret_push_protection_enabled` database column. + +
+
### Raspberry Pi 32-bit packages are deprecated @@ -1795,6 +1801,26 @@ In 18.0 we are removing the `duoProAssignedUsersCount` GraphQL field. Users may
+### Rename `setPreReceiveSecretDetection` GraphQL mutation to `setSecretPushProtection` + +
+ +- Announced in GitLab 17.7 +- Removal in GitLab 18.0 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/514414). + +
+ +The `setPreReceiveSecretDetection` GraphQL mutation has been renamed to `setSecretPushProtection`. We are also renaming some fields in the mutation's response to reflect the name change of the feature `pre_receive_secret_detection` to `secret_push_protection`. + +We have added the new mutation name, but will no longer remove the old mutation name in GitLab 18.0 as originally announced. + +We will still update the database to [remove](https://gitlab.com/gitlab-org/gitlab/-/issues/512996) the old `pre_receive_secret_detection_enabled` database column, but you'll be able to use either mutation name. Both will reflect the value of the new `secret_push_protection_enabled` database column. + +
+ +
+ ### Rename options to skip GitGuardian secret detection
@@ -1807,6 +1833,8 @@ In 18.0 we are removing the `duoProAssignedUsersCount` GraphQL field. Users may The options to skip GitGuardian secret detection, `[skip secret detection]` and `secret_detection.skip_all`, are deprecated. You should use `[skip secret push protection]` and `secret_push_protection.skip_all` instead. +While we recommend using the new wording, we no longer will remove the old option in GitLab 18.0. +
@@ -1860,29 +1888,6 @@ SLES 15 SP6 for continued support.
-
- -### Support for project build as part of SpotBugs scans - -
- -- Announced in GitLab 17.9 -- End of Support in GitLab 18.0 -- Removal in GitLab 18.0 -- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/513409). - -
- -The SpotBugs [SAST analyzer](https://docs.gitlab.com/user/application_security/sast/#supported-languages-and-frameworks) -can perform a build when the artifacts to be scanned aren't present. While this usually works well for simple projects, it can fail on more complex builds. - -From GitLab 18.0, to resolve SpotBugs analyzer build failures, you should: - -1. [Pre-compile](https://docs.gitlab.com/user/application_security/sast/#pre-compilation) the project. -1. Pass the artifacts you want to scan to the analyzer. - -
-
### The `direction` GraphQL argument for `ciJobTokenScopeRemoveProject` is deprecated @@ -7796,60 +7801,6 @@ For information about migrating from the CI/CD template to the component, see th
-### Public use of Secure container registries is deprecated - -
- -- Announced in GitLab 17.4 -- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/470641). - -
-{{< alert type="note" >}} - -This change has been removed from its original milestone and is being reassessed. - -{{< /alert >}} - -Container registries under `registry.gitlab.com/gitlab-org/security-products/` -are no longer accessible in GitLab 18.0. [Since GitLab 14.8](https://docs.gitlab.com/update/deprecations/#secure-and-protect-analyzer-images-published-in-new-location) -the correct location is under `registry.gitlab.com/security-products` (note the absence of -`gitlab-org` in the address). - -This change improves the security of the release process for GitLab [vulnerability scanners](https://docs.gitlab.com/user/application_security/#vulnerability-scanner-maintenance). - -Users are advised to use the equivalent registry under `registry.gitlab.com/security-products/`, -which is the canonical location for GitLab security scanner images. The relevant GitLab CI -templates already use this location, so no changes should be necessary for users that use the -unmodified templates. - -Offline deployments should review the [specific scanner instructions](https://docs.gitlab.com/user/application_security/offline_deployments/#specific-scanner-instructions) -to ensure the correct locations are being used to mirror the required scanner images. - -
- -
- -### REST API endpoint `pre_receive_secret_detection_enabled` is deprecated - -
- -- Announced in GitLab 17.9 -- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/514413). - -
-{{< alert type="note" >}} - -This change has been removed from its original milestone and is being reassessed. - -{{< /alert >}} - -The REST API endpoint `pre_receive_secret_detection_enabled` is deprecated in favor of `secret_push_protection_enabled`. We are renaming some API fields to reflect the name change of the feature `pre_receive_secret_detection` to `secret_push_protection`. -Following [new guidance](https://docs.gitlab.com/development/api_styleguide/#what-to-do-instead-of-a-breaking-change), we will adapt the schema to support both `pre_receive_secret_detection_enabled` and `secret_push_protection_enabled`. We will still [remove](https://gitlab.com/gitlab-org/gitlab/-/issues/512996) the old `pre_receive_secret_detection_enabled` column, but customers will be able to use either parameter, with both pointing to the new `secret_push_protection_enabled` column. - -
- -
- ### Rate limits for common User, Project, and Group API endpoints
@@ -7895,59 +7846,6 @@ The `previousStageJobsOrNeeds` field in GraphQL will be removed as it has been r
-
- -### Rename `setPreReceiveSecretDetection` GraphQL mutation to `setSecretPushProtection` - -
- -- Announced in GitLab 17.7 -- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/514414). - -
-{{< alert type="note" >}} - -This change has been removed from its original milestone and is being reassessed. - -{{< /alert >}} - -The `setPreReceiveSecretDetection` GraphQL mutation has been renamed to `setSecretPushProtection`. We are also renaming some fields in the mutation's response to reflect the name change of the feature `pre_receive_secret_detection` to `secret_push_protection`. -Following [new guidance](https://docs.gitlab.com/development/api_styleguide/#what-to-do-instead-of-a-breaking-change), we're adapting the schema to support both `setPreReceiveSecretDetection` and `setSecretPushProtection`. You can use either parameter because they both point to the `secret_push_protection_enabled` column. In GitLab 18.0, we will [remove](https://gitlab.com/gitlab-org/gitlab/-/issues/514414) the old `setPreReceiveSecretDetection` column. - -
- -
- -### SAST jobs no longer use global cache settings - -
- -- Announced in GitLab 17.9 -- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/512564). - -
-{{< alert type="note" >}} - -This change has been removed from its original milestone and is being reassessed. - -{{< /alert >}} - -In GitLab 18.0, we will update SAST and IaC Scanning to explicitly [disable the use of the CI/CD job cache](https://docs.gitlab.com/ci/caching/#disable-cache-for-specific-jobs) by default. - -This change affects the CI/CD templates for: - -- SAST: `SAST.gitlab-ci.yml`. -- IaC Scanning: `SAST-IaC.gitlab-ci.yml`. - -We already updated the `latest` templates `SAST.latest.gitlab-ci.yml` and `SAST-IaC.latest.gitlab-ci.yml`. See [stable and latest templates](https://docs.gitlab.com/user/application_security/sast/#stable-vs-latest-sast-templates) for more details on these template versions. - -The cache directories are not in scope for scanning in most projects, so fetching the cache can cause timeouts or false-positive results. - -If you need to use the cache when scanning a project, you can restore the previous behavior by [overriding](https://docs.gitlab.com/user/application_security/sast/#overriding-sast-jobs) the -[`cache`](https://docs.gitlab.com/ci/yaml/#cache) property in the project's CI configuration. - -
-
### The `agentk` container registry is moving to Cloud Native GitLab @@ -8119,6 +8017,119 @@ However, you might have customized the template to continue to use these jobs or If you have any customization that depends on the jobs above, perform the [actions required](https://gitlab.com/gitlab-org/gitlab/-/issues/519133#actions-required) before upgrading to 18.0 to avoid disruptions to your CI/CD pipelines. +
+ +
+ +### Public use of Secure container registries is deprecated + +
+ +- Announced in GitLab 17.4 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/470641). + +
+{{< alert type="note" >}} + +This change has been cancelled. + +{{< /alert >}} + +Container registries under `registry.gitlab.com/gitlab-org/security-products/` +are no longer accessible in GitLab 18.0. [Since GitLab 14.8](https://docs.gitlab.com/update/deprecations/#secure-and-protect-analyzer-images-published-in-new-location) +the correct location is under `registry.gitlab.com/security-products` (note the absence of +`gitlab-org` in the address). + +This change improves the security of the release process for GitLab [vulnerability scanners](https://docs.gitlab.com/user/application_security/#vulnerability-scanner-maintenance). + +Users are advised to use the equivalent registry under `registry.gitlab.com/security-products/`, +which is the canonical location for GitLab security scanner images. The relevant GitLab CI +templates already use this location, so no changes should be necessary for users that use the +unmodified templates. + +Offline deployments should review the [specific scanner instructions](https://docs.gitlab.com/user/application_security/offline_deployments/#specific-scanner-instructions) +to ensure the correct locations are being used to mirror the required scanner images. + +
+ +
+ +### SAST jobs no longer use global cache settings + +
+ +- Announced in GitLab 17.9 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/512564). + +
+{{< alert type="note" >}} + +This change has been cancelled. + +{{< /alert >}} + +In GitLab 18.0, we will update SAST and IaC Scanning to explicitly [disable the use of the CI/CD job cache](https://docs.gitlab.com/ci/caching/#disable-cache-for-specific-jobs) by default. + +This change affects the CI/CD templates for: + +- SAST: `SAST.gitlab-ci.yml`. +- IaC Scanning: `SAST-IaC.gitlab-ci.yml`. + +We already updated the `latest` templates `SAST.latest.gitlab-ci.yml` and `SAST-IaC.latest.gitlab-ci.yml`. See [stable and latest templates](https://docs.gitlab.com/user/application_security/sast/#stable-vs-latest-sast-templates) for more details on these template versions. + +The cache directories are not in scope for scanning in most projects, so fetching the cache can cause timeouts or false-positive results. + +If you need to use the cache when scanning a project, you can restore the previous behavior by [overriding](https://docs.gitlab.com/user/application_security/sast/#overriding-sast-jobs) the +[`cache`](https://docs.gitlab.com/ci/yaml/#cache) property in the project's CI configuration. + +
+ +
+ +### Secret detection analyzer doesn't run as root user by default + +
+ +- Announced in GitLab 17.9 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/476160). + +
+{{< alert type="note" >}} + +This change has been cancelled. + +{{< /alert >}} + +This planned change to the secret detection analyzer is cancelled. You can still use the root user by default. + +
+ +
+ +### Support for project build as part of SpotBugs scans + +
+ +- Announced in GitLab 17.9 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/513409). + +
+{{< alert type="note" >}} + +This change has been cancelled. + +{{< /alert >}} + +The SpotBugs [SAST analyzer](https://docs.gitlab.com/user/application_security/sast/#supported-languages-and-frameworks) +can perform a build when the artifacts to be scanned aren't present. While this usually works well for simple projects, it can fail on more complex builds. + +From GitLab 18.0, to resolve SpotBugs analyzer build failures, you should: + +1. [Pre-compile](https://docs.gitlab.com/user/application_security/sast/#pre-compilation) the project. +1. Pass the artifacts you want to scan to the analyzer. + +This is not a change in functionality, so we have marked this announcement "Cancelled" for clarity. +
diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 5d7a8d51284..025e2954b4a 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -36,19 +36,9 @@ module Gitlab end def lock_writes - unless table_exist? - logger&.info "Skipping lock_writes, because #{table_name} does not exist" - return result_hash(action: 'skipped') - end - - if table_locked_for_writes? - logger&.info "Skipping lock_writes, because #{table_name} is already locked for writes" - return result_hash(action: 'skipped') - end - logger&.info Rainbow("Database: '#{database_name}', Table: '#{table_name}': Lock Writes").yellow sql_statement = <<~SQL - CREATE TRIGGER #{write_trigger_name} + CREATE OR REPLACE TRIGGER #{write_trigger_name} BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON #{table_name} FOR EACH STATEMENT EXECUTE FUNCTION #{TRIGGER_FUNCTION_NAME}(); @@ -57,14 +47,14 @@ module Gitlab result = process_query(sql_statement, 'lock') result_hash(action: result) + rescue ActiveRecord::StatementInvalid => e + # Errors like Gitlab::Database::GitlabSchema::UnknownSchemaError, PG::UndefinedTable + # are raised under this error + logger&.warn "Failed lock_writes, because #{table_name} raised an error. Error: #{e}" + result_hash(action: 'skipped') end def unlock_writes - unless table_locked_for_writes? - logger&.info "Skipping unlock_writes, because #{table_name} is already unlocked for writes" - return result_hash(action: 'skipped') - end - logger&.info Rainbow("Database: '#{database_name}', Table: '#{table_name}': Allow Writes").green sql_statement = <<~SQL DROP TRIGGER IF EXISTS #{write_trigger_name} ON #{table_name}; diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2c20107ade2..91668d25fe9 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -66467,6 +66467,12 @@ msgstr "" msgid "VirtualRegistry|New maven virtual registry" msgstr "" +msgid "VirtualRegistry|No upstreams yet" +msgstr "" + +msgid "VirtualRegistry|Upstreams" +msgstr "" + msgid "VirtualRegistry|Virtual registries" msgstr "" diff --git a/spec/frontend/search/sidebar/components/scope_sidebar_navigation_spec.js b/spec/frontend/search/sidebar/components/scope_sidebar_navigation_spec.js index cc3f0aadb2e..c7c890f7cf6 100644 --- a/spec/frontend/search/sidebar/components/scope_sidebar_navigation_spec.js +++ b/spec/frontend/search/sidebar/components/scope_sidebar_navigation_spec.js @@ -31,6 +31,7 @@ describe('ScopeSidebarNavigation', () => { const getterSpies = { navigationItems: jest.fn(() => MOCK_NAVIGATION_ITEMS), + currentScope: jest.fn(), }; const mutationSpies = { @@ -262,5 +263,178 @@ describe('ScopeSidebarNavigation', () => { expect(Sentry.captureException).toHaveBeenCalledWith(mockError); }); }); + + describe('when zoektCrossNamespaceSearch feature is enabled', () => { + beforeEach(() => { + blobCountHandler.mockClear(); + createComponent( + { + zoektAvailable: true, + query: { + search: 'test', + regex: 'false', + }, + }, + { + glFeatures: { + zoektMultimatchFrontend: true, + zoektCrossNamespaceSearch: true, + }, + }, + ); + }); + + it('makes query even without group_id or project_id', () => { + expect(blobCountHandler).toHaveBeenCalled(); + }); + }); + + describe('when current scope is blobs', () => { + beforeEach(() => { + blobCountHandler.mockClear(); + getterSpies.currentScope.mockReturnValue('blobs'); + createComponent( + { + zoektAvailable: true, + query: { + search: 'test search', + group_id: '123', + regex: 'false', + }, + }, + { glFeatures: { zoektMultimatchFrontend: true } }, + ); + }); + + it('does not make query regardless of other conditions', () => { + expect(blobCountHandler).not.toHaveBeenCalled(); + }); + }); + + describe('legacyBlobsCount computed property', () => { + const legacyBlobsCountCases = [ + { + name: 'returns true when currentScope is "blobs"', + initialState: { + zoektAvailable: true, + query: { + search: 'test', + group_id: '123', + }, + }, + currentScope: 'blobs', + features: { + zoektMultimatchFrontend: true, + zoektCrossNamespaceSearch: true, + }, + expected: true, + }, + { + name: 'returns true when zoektMultimatchFrontend feature flag is off', + initialState: { + zoektAvailable: true, + query: { + search: 'test', + group_id: '123', + }, + }, + currentScope: 'notes', + features: { + zoektMultimatchFrontend: false, + zoektCrossNamespaceSearch: true, + }, + expected: true, + }, + { + name: 'returns true when zoektAvailable is false', + initialState: { + zoektAvailable: false, + query: { + search: 'test', + group_id: '123', + }, + }, + currentScope: 'notes', + features: { + zoektMultimatchFrontend: true, + zoektCrossNamespaceSearch: true, + }, + expected: true, + }, + { + name: 'returns true when zoektCrossNamespaceSearch is off and no group_id or project_id', + initialState: { + zoektAvailable: true, + query: { + search: 'test', + }, + }, + currentScope: 'notes', + features: { + zoektMultimatchFrontend: true, + zoektCrossNamespaceSearch: false, + }, + expected: true, + }, + { + name: 'returns false when all conditions allow the query to run with group_id', + initialState: { + zoektAvailable: true, + query: { + search: 'test', + group_id: '123', + }, + }, + currentScope: 'notes', + features: { + zoektMultimatchFrontend: true, + zoektCrossNamespaceSearch: false, + }, + expected: false, + }, + { + name: 'returns false when all conditions allow the query to run with project_id', + initialState: { + zoektAvailable: true, + query: { + search: 'test', + project_id: '456', + }, + }, + currentScope: 'notes', + features: { + zoektMultimatchFrontend: true, + zoektCrossNamespaceSearch: false, + }, + expected: false, + }, + { + name: 'returns false when all conditions allow with zoektCrossNamespaceSearch even without IDs', + initialState: { + zoektAvailable: true, + query: { + search: 'test', + }, + }, + currentScope: 'notes', + features: { + zoektMultimatchFrontend: true, + zoektCrossNamespaceSearch: true, + }, + expected: false, + }, + ]; + + legacyBlobsCountCases.forEach(({ name, initialState, currentScope, features, expected }) => { + it(`test ${name}`, () => { + getterSpies.currentScope.mockReturnValue(currentScope); + createComponent(initialState, { + glFeatures: { ...features, workItemScopeFrontend: true }, + }); + + expect(wrapper.vm.legacyBlobsCount).toBe(expected); + }); + }); + }); }); }); diff --git a/spec/frontend/search/store/utils_spec.js b/spec/frontend/search/store/utils_spec.js index 3ca5d8a009e..72a6df2dfbf 100644 --- a/spec/frontend/search/store/utils_spec.js +++ b/spec/frontend/search/store/utils_spec.js @@ -18,7 +18,6 @@ import { injectRegexSearch, injectUsersScope, scopeCrawler, - skipBlobESCount, buildDocumentTitle, } from '~/search/store/utils'; import { useMockLocationHelper } from 'helpers/mock_window_location_helper'; @@ -407,67 +406,6 @@ describe('Global Search Store Utils', () => { }); }); - describe('skipBlobESCount', () => { - const SCOPE_BLOB = 'blobs'; - - let state; - - beforeEach(() => { - state = { - query: {}, - zoektAvailable: false, - }; - - window.gon = { - features: { - zoektMultimatchFrontend: false, - }, - }; - }); - - it('returns true when no group_id or project_id is present', () => { - expect(skipBlobESCount(state, SCOPE_BLOB)).toBe(true); - }); - - it('returns true when zoektMultimatchFrontend feature flag is off', () => { - state.query.group_id = '1'; - state.zoektAvailable = true; - - expect(skipBlobESCount(state, SCOPE_BLOB)).toBe(true); - }); - - it('returns true when zoekt is not available', () => { - state.query.group_id = '1'; - window.gon.features.zoektMultimatchFrontend = true; - - expect(skipBlobESCount(state, SCOPE_BLOB)).toBe(true); - }); - - it('returns true when scope is not blob', () => { - state.query.group_id = '1'; - state.zoektAvailable = true; - window.gon.features.zoektMultimatchFrontend = true; - - expect(skipBlobESCount(state, 'not_blob')).toBe(true); - }); - - it('returns false when all conditions are met', () => { - state.query.group_id = '1'; - state.zoektAvailable = true; - window.gon.features.zoektMultimatchFrontend = true; - - expect(skipBlobESCount(state, SCOPE_BLOB)).toBe(false); - }); - - it('returns false when using project_id instead of group_id', () => { - state.query.project_id = '1'; - state.zoektAvailable = true; - window.gon.features.zoektMultimatchFrontend = true; - - expect(skipBlobESCount(state, SCOPE_BLOB)).toBe(false); - }); - }); - describe('buildDocumentTitle', () => { const SEARCH_WINDOW_TITLE = `Search`; // Make sure this matches your actual constant let originalTitle; diff --git a/spec/lib/gitlab/database/lock_writes_manager_spec.rb b/spec/lib/gitlab/database/lock_writes_manager_spec.rb index 4485d7b6e4f..e241075fd3d 100644 --- a/spec/lib/gitlab/database/lock_writes_manager_spec.rb +++ b/spec/lib/gitlab/database/lock_writes_manager_spec.rb @@ -95,7 +95,9 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : it 'retries again if it receives a statement_timeout a few number of times' do error_message = "PG::QueryCanceled: ERROR: canceling statement due to statement timeout" call_count = 0 - expect(connection).to receive(:execute).twice.with(/^CREATE TRIGGER gitlab_schema_write_trigger_for_/) do + expect(connection).to receive(:execute).twice.with( + /^CREATE OR REPLACE TRIGGER gitlab_schema_write_trigger_for_/ + ) do call_count += 1 raise(ActiveRecord::QueryCanceled, error_message) if call_count.odd? end @@ -104,22 +106,16 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : expect(call_count).to eq(2) # The first call fails, the 2nd call succeeds end - it 'raises the exception if it happened many times' do + it 'logs retried errors as skipped' do error_message = "PG::QueryCanceled: ERROR: canceling statement due to statement timeout" - allow(connection).to receive(:execute).with(/^CREATE TRIGGER gitlab_schema_write_trigger_for_/) do + allow(connection).to receive(:execute).with(/^CREATE OR REPLACE TRIGGER gitlab_schema_write_trigger_for_/) do raise(ActiveRecord::QueryCanceled, error_message) end - expect do - subject.lock_writes - end.to raise_error(ActiveRecord::QueryCanceled) - end - - it 'skips the operation if the table is already locked for writes' do - subject.lock_writes - - expect(logger).to receive(:info).with("Skipping lock_writes, because #{test_table} is already locked for writes") - expect(connection).not_to receive(:execute).with(/CREATE TRIGGER/) + expect(logger).to receive(:warn).with( + /Failed lock_writes, because #{test_table} raised an error. Error:/ + ) + expect(connection).to receive(:execute).with(/CREATE OR REPLACE TRIGGER/) expect do result = subject.lock_writes @@ -129,13 +125,36 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : } end + it 'replaces the trigger if the table is already locked' do + subject.lock_writes + + expect(connection).to receive(:execute).with(/CREATE OR REPLACE TRIGGER/) + + expect do + result = subject.lock_writes + expect(result).to eq({ action: "locked", database: "main", dry_run: false, table: test_table }) + end.not_to change { + number_of_triggers_on(connection, test_table) + } + end + context 'when table does not exist' do let(:skip_table_creation) { true } let(:test_table) { non_existent_table } - it 'skips locking table' do - expect(logger).to receive(:info).with("Skipping lock_writes, because #{test_table} does not exist") - expect(connection).not_to receive(:execute).with(/CREATE TRIGGER/) + # In tests if we don't add the fake table to the gitlab schema then our test only schema checks will fail. + before do + allow(Gitlab::Database::GitlabSchema).to receive(:table_schema).and_call_original + allow(Gitlab::Database::GitlabSchema).to receive(:table_schema).with( + non_existent_table + ).and_return(:gitlab_main_cell) + end + + it 'tries to lock the table anyways, and logs the failure' do + expect(logger).to receive(:warn).with( + /Failed lock_writes, because #{test_table} raised an error. Error:/ + ) + expect(connection).to receive(:execute).with(/CREATE OR REPLACE TRIGGER/) expect do result = subject.lock_writes @@ -152,7 +171,7 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : it 'prints the sql statement to the logger' do expect(logger).to receive(:info).with("Database: 'main', Table: '#{test_table}': Lock Writes") expected_sql_statement = <<~SQL - CREATE TRIGGER gitlab_schema_write_trigger_for_#{test_table} + CREATE OR REPLACE TRIGGER gitlab_schema_write_trigger_for_#{test_table} BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON #{test_table} FOR EACH STATEMENT EXECUTE FUNCTION gitlab_schema_prevent_write(); @@ -181,6 +200,12 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : describe '#unlock_writes' do before do + allow(logger).to receive(:warn).with(/Failed lock_writes, because #{test_table} raised an error. Error:/) + + # In tests if we don't add the fake table to the gitlab schema then our test only schema checks will fail. + allow(Gitlab::Database::GitlabSchema).to receive(:table_schema).and_call_original + allow(Gitlab::Database::GitlabSchema).to receive(:table_schema).with(test_table).and_return(:gitlab_main_cell) + # Locking the table without the considering the value of dry_run described_class.new( table_name: test_table, @@ -202,12 +227,12 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : end.not_to raise_error end - it 'skips unlocking the table if the table was already unlocked for writes' do + it 'idempotently drops the lock even if it does not exist' do subject.unlock_writes - expect(subject).not_to receive(:execute_sql_statement) + expect(subject).to receive(:execute_sql_statement) expect(subject.unlock_writes).to eq( - { action: "skipped", database: "main", dry_run: dry_run, table: test_table } + { action: "unlocked", database: "main", dry_run: dry_run, table: test_table } ) end @@ -231,13 +256,10 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : let(:skip_table_creation) { true } let(:test_table) { non_existent_table } - it 'skips unlocking table' do - subject.unlock_writes + it 'tries to unlock the table which postgres handles gracefully' do + expect(subject).to receive(:execute_sql_statement).and_call_original - expect(subject).not_to receive(:execute_sql_statement) - expect(subject.unlock_writes).to eq( - { action: "skipped", database: "main", dry_run: dry_run, table: test_table } - ) + expect(subject.unlock_writes).to eq({ action: "unlocked", database: "main", dry_run: false, table: test_table }) end end diff --git a/spec/workers/database/lock_tables_worker_spec.rb b/spec/workers/database/lock_tables_worker_spec.rb index cb720959db0..33531b2d36f 100644 --- a/spec/workers/database/lock_tables_worker_spec.rb +++ b/spec/workers/database/lock_tables_worker_spec.rb @@ -93,13 +93,13 @@ RSpec.describe Database::LockTablesWorker, feature_category: :cell do end with_them do - it 'skips locking the tables on the corresponding database' do + it 'tries to lock the table anyways, and logs the failure' do tables.each do |table_name| lock_table(database_name, table_name) end expected_log_results = tables.map do |table_name| - { action: 'skipped', database: database_name, dry_run: false, table: table_name } + { action: 'locked', database: database_name, dry_run: false, table: table_name } end expect(worker).to receive(:log_extra_metadata_on_done).with(:performed_actions, expected_log_results)