diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml index 3efcbb62c24..f109fe253dd 100644 --- a/.rubocop_todo/layout/line_length.yml +++ b/.rubocop_todo/layout/line_length.yml @@ -4263,7 +4263,6 @@ Layout/LineLength: - 'spec/requests/api/graphql/mutations/snippets/create_spec.rb' - 'spec/requests/api/graphql/mutations/work_items/create_from_task_spec.rb' - 'spec/requests/api/graphql/mutations/work_items/delete_spec.rb' - - 'spec/requests/api/graphql/namespace/root_storage_statistics_spec.rb' - 'spec/requests/api/graphql/namespace_query_spec.rb' - 'spec/requests/api/graphql/packages/package_spec.rb' - 'spec/requests/api/graphql/project/alert_management/alert/notes_spec.rb' diff --git a/app/assets/javascripts/environments/graphql/resolvers/kubernetes.js b/app/assets/javascripts/environments/graphql/resolvers/kubernetes.js index d57f2498fcd..8375b8793d9 100644 --- a/app/assets/javascripts/environments/graphql/resolvers/kubernetes.js +++ b/app/assets/javascripts/environments/graphql/resolvers/kubernetes.js @@ -5,8 +5,6 @@ import { BatchV1Api, WatchApi, EVENT_DATA, - EVENT_TIMEOUT, - EVENT_ERROR, } from '@gitlab/cluster-client'; import { humanizeClusterErrors } from '../../helpers/k8s_integration_helper'; import k8sPodsQuery from '../queries/k8s_pods.query.graphql'; @@ -63,60 +61,57 @@ const handleClusterError = async (err) => { throw errorData; }; +const watchPods = ({ configuration, namespace, client }) => { + const path = namespace ? `/api/v1/namespaces/${namespace}/pods` : '/api/v1/pods'; + const config = new Configuration(configuration); + const watcherApi = new WatchApi(config); + + watcherApi + .subscribeToStream(path, { watch: true }) + .then((watcher) => { + let result = []; + + watcher.on(EVENT_DATA, (data) => { + result = data.map((item) => { + return { status: { phase: item.status.phase } }; + }); + + client.writeQuery({ + query: k8sPodsQuery, + variables: { configuration, namespace }, + data: { k8sPods: result }, + }); + }); + }) + .catch((err) => { + handleClusterError(err); + }); +}; + export default { k8sPods(_, { configuration, namespace }, { client }) { const config = new Configuration(configuration); - if (!gon.features?.k8sWatchApi) { - const coreV1Api = new CoreV1Api(config); - const podsApi = namespace - ? coreV1Api.listCoreV1NamespacedPod({ namespace }) - : coreV1Api.listCoreV1PodForAllNamespaces(); + const coreV1Api = new CoreV1Api(config); + const podsApi = namespace + ? coreV1Api.listCoreV1NamespacedPod({ namespace }) + : coreV1Api.listCoreV1PodForAllNamespaces(); - return podsApi - .then((res) => res?.items || []) - .catch(async (err) => { - try { - await handleClusterError(err); - } catch (error) { - throw new Error(error.message); - } - }); - } + return podsApi + .then((res) => { + if (gon.features?.k8sWatchApi) { + watchPods({ configuration, namespace, client }); + } - const path = namespace ? `/api/v1/namespaces/${namespace}/pods` : '/api/v1/pods'; - const watcherApi = new WatchApi(config); - - return watcherApi.subscribeToStream(path, { watch: true }).then((watcher) => { - let result = []; - - return new Promise((resolve, reject) => { - watcher.on(EVENT_DATA, (data) => { - result = data.map((item) => { - return { status: { phase: item.status.phase } }; - }); - - resolve(result); - - setTimeout(() => { - client.writeQuery({ - query: k8sPodsQuery, - variables: { configuration, namespace }, - data: { k8sPods: result }, - }); - }, 0); - }); - - watcher.on(EVENT_TIMEOUT, () => { - resolve(result); - }); - - watcher.on(EVENT_ERROR, (errorData) => { - const error = errorData?.message ? new Error(errorData.message) : errorData; - reject(error); - }); + return res?.items || []; + }) + .catch(async (err) => { + try { + await handleClusterError(err); + } catch (error) { + throw new Error(error.message); + } }); - }); }, k8sServices(_, { configuration, namespace }) { const coreV1Api = new CoreV1Api(new Configuration(configuration)); diff --git a/app/assets/javascripts/vue_merge_request_widget/components/checks/conflicts.vue b/app/assets/javascripts/vue_merge_request_widget/components/checks/conflicts.vue index 303952c787e..32c3f19014b 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/checks/conflicts.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/checks/conflicts.vue @@ -72,6 +72,8 @@ export default { diff --git a/app/assets/javascripts/vue_merge_request_widget/components/checks/message.vue b/app/assets/javascripts/vue_merge_request_widget/components/checks/message.vue index 37c07183c88..058b9e1fe99 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/checks/message.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/checks/message.vue @@ -57,6 +57,7 @@ export default {
{{ failureReason }}
+ diff --git a/app/assets/javascripts/vue_merge_request_widget/components/checks/rebase.vue b/app/assets/javascripts/vue_merge_request_widget/components/checks/rebase.vue index 823a30c7063..72140c22a89 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/checks/rebase.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/checks/rebase.vue @@ -184,7 +184,9 @@ export default { diff --git a/app/assets/javascripts/vue_merge_request_widget/components/merge_checks.stories.js b/app/assets/javascripts/vue_merge_request_widget/components/merge_checks.stories.js index d8476733656..77dc5b1d0da 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/merge_checks.stories.js +++ b/app/assets/javascripts/vue_merge_request_widget/components/merge_checks.stories.js @@ -17,7 +17,7 @@ const defaultRender = (apolloProvider) => ({ data() { return { service: {}, mr: { conflictResolutionPath: 'https://gitlab.com' } }; }, - template: '', + template: '', }); const Template = ({ canMerge, failed, pushToSourceBranch }) => { diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 437118c36e8..a075c2f7e4f 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -124,6 +124,10 @@ class BulkImports::Entity < ApplicationRecord entity_type.pluralize end + def portable_class + entity_type.classify.constantize + end + def base_resource_url_path "/#{pluralized_name}/#{encoded_source_full_path}" end diff --git a/app/workers/bulk_imports/pipeline_batch_worker.rb b/app/workers/bulk_imports/pipeline_batch_worker.rb index 55936e85d48..1485275e616 100644 --- a/app/workers/bulk_imports/pipeline_batch_worker.rb +++ b/app/workers/bulk_imports/pipeline_batch_worker.rb @@ -5,6 +5,8 @@ module BulkImports include ApplicationWorker include ExclusiveLeaseGuard + DEFER_ON_HEALTH_DELAY = 5.minutes + data_consistency :always # rubocop:disable SidekiqLoadBalancing/WorkerDataConsistency feature_category :importers sidekiq_options dead: false, retry: 3 @@ -16,6 +18,26 @@ module BulkImports new.perform_failure(msg['args'].first, exception) end + defer_on_database_health_signal(:gitlab_main, [], DEFER_ON_HEALTH_DELAY) do |job_args, schema, tables| + batch = ::BulkImports::BatchTracker.find(job_args.first) + pipeline_tracker = batch.tracker + pipeline_schema = ::BulkImports::PipelineSchemaInfo.new( + pipeline_tracker.pipeline_class, + pipeline_tracker.entity.portable_class + ) + + if pipeline_schema.db_schema && pipeline_schema.db_table + schema = pipeline_schema.db_schema + tables = [pipeline_schema.db_table] + end + + [schema, tables] + end + + def self.defer_on_database_health_signal? + Feature.enabled?(:bulk_import_deferred_workers) + end + def perform(batch_id) @batch = ::BulkImports::BatchTracker.find(batch_id) diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb index 4b1df9c85a6..2c1d28b33c5 100644 --- a/app/workers/bulk_imports/pipeline_worker.rb +++ b/app/workers/bulk_imports/pipeline_worker.rb @@ -7,6 +7,8 @@ module BulkImports FILE_EXTRACTION_PIPELINE_PERFORM_DELAY = 10.seconds + DEFER_ON_HEALTH_DELAY = 5.minutes + data_consistency :always feature_category :importers sidekiq_options dead: false, retry: 3 @@ -21,6 +23,25 @@ module BulkImports new.perform_failure(msg['args'][0], msg['args'][2], exception) end + defer_on_database_health_signal(:gitlab_main, [], DEFER_ON_HEALTH_DELAY) do |job_args, schema, tables| + pipeline_tracker = ::BulkImports::Tracker.find(job_args.first) + pipeline_schema = ::BulkImports::PipelineSchemaInfo.new( + pipeline_tracker.pipeline_class, + pipeline_tracker.entity.portable_class + ) + + if pipeline_schema.db_schema && pipeline_schema.db_table + schema = pipeline_schema.db_schema + tables = [pipeline_schema.db_table] + end + + [schema, tables] + end + + def self.defer_on_database_health_signal? + Feature.enabled?(:bulk_import_deferred_workers) + end + # Keep _stage parameter for backwards compatibility. def perform(pipeline_tracker_id, _stage, entity_id) @entity = ::BulkImports::Entity.find(entity_id) diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index cb09aaf1a6a..28c82a5a38e 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -201,10 +201,10 @@ module WorkerAttributes !!get_class_attribute(:big_payload) end - def defer_on_database_health_signal(gitlab_schema, tables = [], delay_by = DEFAULT_DEFER_DELAY) + def defer_on_database_health_signal(gitlab_schema, tables = [], delay_by = DEFAULT_DEFER_DELAY, &block) set_class_attribute( :database_health_check_attrs, - { gitlab_schema: gitlab_schema, tables: tables, delay_by: delay_by } + { gitlab_schema: gitlab_schema, tables: tables, delay_by: delay_by, block: block } ) end diff --git a/config/feature_flags/development/bulk_import_deferred_workers.yml b/config/feature_flags/development/bulk_import_deferred_workers.yml new file mode 100644 index 00000000000..1b6a022099c --- /dev/null +++ b/config/feature_flags/development/bulk_import_deferred_workers.yml @@ -0,0 +1,8 @@ +--- +name: bulk_import_deferred_workers +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136137 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/431032 +milestone: '16.6' +type: development +group: group::import and integrate +default_enabled: false diff --git a/config/metrics/counts_28d/20231102160653_i_quickactions_request_changes_monthly.yml b/config/metrics/counts_28d/20231102160653_i_quickactions_request_changes_monthly.yml new file mode 100644 index 00000000000..970c49e3962 --- /dev/null +++ b/config/metrics/counts_28d/20231102160653_i_quickactions_request_changes_monthly.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.quickactions.i_quickactions_request_changes_monthly +description: Count using the `/request_changes` quick action on Merge Requests +product_section: dev +product_stage: create +product_group: code_review +value_type: number +status: active +milestone: '16.6' +time_frame: 28d +data_source: redis_hll +data_category: optional +instrumentation_class: RedisHLLMetric +options: + events: + - i_quickactions_request_changes +distribution: + - ce + - ee +tier: + - free + - premium + - ultimate diff --git a/config/metrics/counts_7d/20231102160653_i_quickactions_request_changes_weekly.yml b/config/metrics/counts_7d/20231102160653_i_quickactions_request_changes_weekly.yml new file mode 100644 index 00000000000..e529dfd4e51 --- /dev/null +++ b/config/metrics/counts_7d/20231102160653_i_quickactions_request_changes_weekly.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.quickactions.i_quickactions_request_changes_weekly +description: Count using the `/request_changes` quick action on Merge Requests +product_section: dev +product_stage: create +product_group: code_review +value_type: number +status: active +milestone: '16.6' +time_frame: 7d +data_source: redis_hll +data_category: optional +instrumentation_class: RedisHLLMetric +options: + events: + - i_quickactions_request_changes +distribution: + - ce + - ee +tier: + - free + - premium + - ultimate diff --git a/data/deprecations/16-6-lfs-integrity-check-feature-flag-deprecation.yml b/data/deprecations/16-6-lfs-integrity-check-feature-flag-deprecation.yml new file mode 100644 index 00000000000..8e6088dc4e4 --- /dev/null +++ b/data/deprecations/16-6-lfs-integrity-check-feature-flag-deprecation.yml @@ -0,0 +1,13 @@ +- title: "Deprecation of `lfs_check` feature flag" # (required) Clearly explain the change, or planned change. For example, "The `confidential` field for a `Note` is deprecated" or "CI/CD job names will be limited to 250 characters." + removal_milestone: "16.9" # (required) The milestone when this feature is planned to be removed + announcement_milestone: "16.6" # (required) The milestone when this feature was first announced as deprecated. + breaking_change: false # (required) Change to false if this is not a breaking change. + reporter: derekferguson # (required) GitLab username of the person reporting the change + stage: Create # (required) String value of the stage that the feature was created in. e.g., Growth + issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/233550 # (required) Link to the deprecation issue in GitLab + body: | # (required) Do not modify this line, instead modify the lines below. + In GitLab 16.9, we will remove the `lfs_check` feature flag. This feature flag was [introduced 4 years ago](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/60588) and controls whether the LFS integrity check is enabled. The feature flag is enabled by default, but some customers experienced performance issues with the LFS integrity check and explicitly disabled it. + + After [dramatically improving the performance](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61355) of the LFS integrity check, we are ready to remove the feature flag. After the flag is removed, the feature will automatically be turned on for any environment in which it is currently disabled. + + If this feature flag is disabled for your environment, and you are concerned about performance issues, please enable it and monitor the performance before it is removed in 16.9. If you see any performance issues after enabling it, please let us know in [this feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/233550). diff --git a/doc/architecture/blueprints/new_diffs.md b/doc/architecture/blueprints/new_diffs.md index b5aeb9b8aa8..cb9715cddda 100644 --- a/doc/architecture/blueprints/new_diffs.md +++ b/doc/architecture/blueprints/new_diffs.md @@ -68,6 +68,25 @@ compared with the pros and cons of alternatives. ## Design and implementation details +### Workspace & Artifacts + +- We will store implementation details like metrics, budgets, and development & architectural patterns here in the docs +- We will store large bodies of research, the results of audits, etc. in the [wiki](https://gitlab.com/gitlab-com/create-stage/new-diffs/-/wikis/home) of the [New Diffs project](https://gitlab.com/gitlab-com/create-stage/new-diffs) +- We will store audio & video recordings on the public Youtube channel in the Code Review / New Diffs playlist +- We will store drafts, meeting notes, and other temporary documents in public Google docs + +### Definitions + +#### Maintainability + +Maintainable projects are _simple_ projects. + +Simplicity is the opposite of complexity. This uses a definition of simple and complex [described by Rich Hickey in "Simple Made Easy"](https://www.infoq.com/presentations/Simple-Made-Easy/) (Strange Loop, 2011). + +- Maintainable code is simple (single task, single concept, separate from other things). +- Maintainable projects expand on simple code by having simple structure (folders define classes of behaviors, e.g. you can be assured that a component directory will never initiate a network call, because that would be complecting visual display with data access) +- Maintainable applications flow out of simple organization and simple code. The old saying is a cluttered desk is representative of a cluttered mind. Rigorous discipline on simplicity will be represented in our output (the product). By being strict about working simply, we will naturally produce applications where our users can more easily reason about their behavior. +