From 63fd08e6b429cd3af81cf63dfc0e5fc853d04086 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 11 Aug 2021 21:10:33 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../components/graph/graph_component.vue | 15 +- .../graph/graph_component_wrapper.vue | 18 +- .../graph/linked_pipelines_column.vue | 10 +- .../pipelines/components/graph/perf_utils.js | 50 ++++ .../pipelines/components/graph/utils.js | 26 ++ .../components/graph_shared/drawing_utils.js | 2 +- .../components/graph_shared/links_inner.vue | 10 +- .../components/graph_shared/links_layer.vue | 117 ++------- .../pipelines/components/parsing_utils.js | 8 +- .../vue_shared/components/markdown/header.vue | 237 +++++++++--------- .../components/markdown/toolbar_button.vue | 17 +- .../new_namespace/components/welcome.vue | 12 +- .../stylesheets/framework/animations.scss | 1 - .../stylesheets/framework/markdown_area.scss | 30 --- app/finders/error_tracking/errors_finder.rb | 37 +++ app/helpers/markup_helper.rb | 2 +- app/models/error_tracking/error.rb | 69 +++++ app/models/error_tracking/error_event.rb | 65 +++++ .../project_error_tracking_setting.rb | 21 +- app/policies/group_policy.rb | 8 +- .../error_tracking/issue_details_service.rb | 26 +- .../issue_latest_event_service.rb | 27 +- .../error_tracking/issue_update_service.rb | 32 ++- .../error_tracking/list_issues_service.rb | 37 ++- .../shared/blob/_markdown_buttons.html.haml | 2 +- .../development/dast_meta_tag_validation.yml | 8 + db/fixtures/development/31_error_tracking.rb | 49 ++++ ...dd_integrated_to_error_tracking_setting.rb | 11 + ...0654_add_status_to_error_tracking_error.rb | 11 + ...nversion_to_bigint_for_ci_job_artifacts.rb | 79 ++++++ db/schema_migrations/20210726134950 | 1 + db/schema_migrations/20210728110654 | 1 + db/schema_migrations/20210809143931 | 1 + db/structure.sql | 8 +- doc/development/elasticsearch.md | 4 +- doc/development/geo.md | 2 +- doc/development/snowplow/index.md | 4 +- doc/install/aws/index.md | 2 +- doc/user/group/saml_sso/index.md | 1 + doc/user/packages/dependency_proxy/index.md | 11 +- doc/user/project/import/clearcase.md | 2 +- doc/user/report_abuse.md | 2 +- doc/user/search/index.md | 4 +- lib/gitlab/ci/config.rb | 12 +- .../ci/pipeline/chain/config/process.rb | 2 +- .../import_export/project/import_export.yml | 1 + lib/gitlab/project_search_results.rb | 3 +- lib/gitlab/search_results.rb | 2 +- locale/gitlab.pot | 20 +- ...cy_proxy_for_containers_controller_spec.rb | 34 --- spec/factories/error_tracking/error.rb | 5 + .../projects/tags/user_edits_tags_spec.rb | 2 +- .../error_tracking/errors_finder_spec.rb | 28 +++ .../pipelines/graph/graph_component_spec.js | 4 +- .../graph/graph_component_wrapper_spec.js | 118 +++++++++ .../graph_shared/links_inner_spec.js | 2 +- .../graph_shared/links_layer_spec.js | 145 ----------- spec/frontend/pipelines/parsing_utils_spec.js | 6 +- .../markdown/toolbar_button_spec.js | 3 +- .../gitlab/ci/pipeline/chain/populate_spec.rb | 1 - .../models/error_tracking/error_event_spec.rb | 31 +++ spec/models/error_tracking/error_spec.rb | 14 ++ .../project_error_tracking_setting_spec.rb | 41 ++- .../issue_details_service_spec.rb | 15 ++ .../issue_latest_event_service_spec.rb | 20 +- .../issue_update_service_spec.rb | 15 ++ .../list_issues_service_spec.rb | 14 ++ .../dependency_proxy_shared_examples.rb | 35 +++ ...nches_access_control_ce_shared_examples.rb | 1 + 69 files changed, 1116 insertions(+), 538 deletions(-) create mode 100644 app/assets/javascripts/pipelines/components/graph/perf_utils.js create mode 100644 app/finders/error_tracking/errors_finder.rb create mode 100644 config/feature_flags/development/dast_meta_tag_validation.yml create mode 100644 db/fixtures/development/31_error_tracking.rb create mode 100644 db/migrate/20210726134950_add_integrated_to_error_tracking_setting.rb create mode 100644 db/migrate/20210728110654_add_status_to_error_tracking_error.rb create mode 100644 db/post_migrate/20210809143931_finalize_job_id_conversion_to_bigint_for_ci_job_artifacts.rb create mode 100644 db/schema_migrations/20210726134950 create mode 100644 db/schema_migrations/20210728110654 create mode 100644 db/schema_migrations/20210809143931 create mode 100644 spec/finders/error_tracking/errors_finder_spec.rb create mode 100644 spec/support/shared_examples/features/dependency_proxy_shared_examples.rb diff --git a/app/assets/javascripts/pipelines/components/graph/graph_component.vue b/app/assets/javascripts/pipelines/components/graph/graph_component.vue index ea45b5e3ec7..015f0519c72 100644 --- a/app/assets/javascripts/pipelines/components/graph/graph_component.vue +++ b/app/assets/javascripts/pipelines/components/graph/graph_component.vue @@ -39,10 +39,10 @@ export default { required: false, default: false, }, - pipelineLayers: { - type: Array, + computedPipelineInfo: { + type: Object, required: false, - default: () => [], + default: () => ({}), }, type: { type: String, @@ -81,7 +81,10 @@ export default { layout() { return this.isStageView ? this.pipeline.stages - : generateColumnsFromLayersListMemoized(this.pipeline, this.pipelineLayers); + : generateColumnsFromLayersListMemoized( + this.pipeline, + this.computedPipelineInfo.pipelineLayers, + ); }, hasDownstreamPipelines() { return Boolean(this.pipeline?.downstream?.length > 0); @@ -92,6 +95,9 @@ export default { isStageView() { return this.viewType === STAGE_VIEW; }, + linksData() { + return this.computedPipelineInfo?.linksData ?? null; + }, metricsConfig() { return { path: this.configPaths.metricsPath, @@ -188,6 +194,7 @@ export default { :container-id="containerId" :container-measurements="measurements" :highlighted-job="hoveredJobName" + :links-data="linksData" :metrics-config="metricsConfig" :show-links="showJobLinks" :view-type="viewType" diff --git a/app/assets/javascripts/pipelines/components/graph/graph_component_wrapper.vue b/app/assets/javascripts/pipelines/components/graph/graph_component_wrapper.vue index 5d51d97eaee..8462fb752b7 100644 --- a/app/assets/javascripts/pipelines/components/graph/graph_component_wrapper.vue +++ b/app/assets/javascripts/pipelines/components/graph/graph_component_wrapper.vue @@ -9,11 +9,11 @@ import { DEFAULT, DRAW_FAILURE, LOAD_FAILURE } from '../../constants'; import DismissPipelineGraphCallout from '../../graphql/mutations/dismiss_pipeline_notification.graphql'; import getPipelineQuery from '../../graphql/queries/get_pipeline_header_data.query.graphql'; import { reportToSentry, reportMessageToSentry } from '../../utils'; -import { listByLayers } from '../parsing_utils'; import { IID_FAILURE, LAYER_VIEW, STAGE_VIEW, VIEW_TYPE_KEY } from './constants'; import PipelineGraph from './graph_component.vue'; import GraphViewSelector from './graph_view_selector.vue'; import { + calculatePipelineLayersInfo, getQueryHeaders, serializeLoadErrors, toggleQueryPollingByVisibility, @@ -51,10 +51,10 @@ export default { return { alertType: null, callouts: [], + computedPipelineInfo: null, currentViewType: STAGE_VIEW, canRefetchHeaderPipeline: false, pipeline: null, - pipelineLayers: null, showAlert: false, showLinks: false, }; @@ -214,12 +214,16 @@ export default { reportToSentry(this.$options.name, `error: ${err}, info: ${info}`); }, methods: { - getPipelineLayers() { - if (this.currentViewType === LAYER_VIEW && !this.pipelineLayers) { - this.pipelineLayers = listByLayers(this.pipeline); + getPipelineInfo() { + if (this.currentViewType === LAYER_VIEW && !this.computedPipelineInfo) { + this.computedPipelineInfo = calculatePipelineLayersInfo( + this.pipeline, + this.$options.name, + this.metricsPath, + ); } - return this.pipelineLayers; + return this.computedPipelineInfo; }, handleTipDismissal() { try { @@ -288,7 +292,7 @@ export default { v-if="pipeline" :config-paths="configPaths" :pipeline="pipeline" - :pipeline-layers="getPipelineLayers()" + :computed-pipeline-info="getPipelineInfo()" :show-links="showLinks" :view-type="graphViewType" @error="reportFailure" diff --git a/app/assets/javascripts/pipelines/components/graph/linked_pipelines_column.vue b/app/assets/javascripts/pipelines/components/graph/linked_pipelines_column.vue index 52ee40bd982..d251e0d8bd8 100644 --- a/app/assets/javascripts/pipelines/components/graph/linked_pipelines_column.vue +++ b/app/assets/javascripts/pipelines/components/graph/linked_pipelines_column.vue @@ -2,10 +2,10 @@ import getPipelineDetails from 'shared_queries/pipelines/get_pipeline_details.query.graphql'; import { LOAD_FAILURE } from '../../constants'; import { reportToSentry } from '../../utils'; -import { listByLayers } from '../parsing_utils'; import { ONE_COL_WIDTH, UPSTREAM, LAYER_VIEW, STAGE_VIEW } from './constants'; import LinkedPipeline from './linked_pipeline.vue'; import { + calculatePipelineLayersInfo, getQueryHeaders, serializeLoadErrors, toggleQueryPollingByVisibility, @@ -138,7 +138,11 @@ export default { }, getPipelineLayers(id) { if (this.viewType === LAYER_VIEW && !this.pipelineLayers[id]) { - this.pipelineLayers[id] = listByLayers(this.currentPipeline); + this.pipelineLayers[id] = calculatePipelineLayersInfo( + this.currentPipeline, + this.$options.name, + this.configPaths.metricsPath, + ); } return this.pipelineLayers[id]; @@ -223,7 +227,7 @@ export default { class="d-inline-block gl-mt-n2" :config-paths="configPaths" :pipeline="currentPipeline" - :pipeline-layers="getPipelineLayers(pipeline.id)" + :computed-pipeline-info="getPipelineLayers(pipeline.id)" :show-links="showLinks" :is-linked-pipeline="true" :view-type="graphViewType" diff --git a/app/assets/javascripts/pipelines/components/graph/perf_utils.js b/app/assets/javascripts/pipelines/components/graph/perf_utils.js new file mode 100644 index 00000000000..3737a209f5c --- /dev/null +++ b/app/assets/javascripts/pipelines/components/graph/perf_utils.js @@ -0,0 +1,50 @@ +import { + PIPELINES_DETAIL_LINKS_MARK_CALCULATE_START, + PIPELINES_DETAIL_LINKS_MARK_CALCULATE_END, + PIPELINES_DETAIL_LINKS_MEASURE_CALCULATION, + PIPELINES_DETAIL_LINK_DURATION, + PIPELINES_DETAIL_LINKS_TOTAL, + PIPELINES_DETAIL_LINKS_JOB_RATIO, +} from '~/performance/constants'; + +import { performanceMarkAndMeasure } from '~/performance/utils'; +import { reportPerformance } from '../graph_shared/api'; + +export const beginPerfMeasure = () => { + performanceMarkAndMeasure({ mark: PIPELINES_DETAIL_LINKS_MARK_CALCULATE_START }); +}; + +export const finishPerfMeasureAndSend = (numLinks, numGroups, metricsPath) => { + performanceMarkAndMeasure({ + mark: PIPELINES_DETAIL_LINKS_MARK_CALCULATE_END, + measures: [ + { + name: PIPELINES_DETAIL_LINKS_MEASURE_CALCULATION, + start: PIPELINES_DETAIL_LINKS_MARK_CALCULATE_START, + }, + ], + }); + + window.requestAnimationFrame(() => { + const duration = window.performance.getEntriesByName( + PIPELINES_DETAIL_LINKS_MEASURE_CALCULATION, + )[0]?.duration; + + if (!duration) { + return; + } + + const data = { + histograms: [ + { name: PIPELINES_DETAIL_LINK_DURATION, value: duration / 1000 }, + { name: PIPELINES_DETAIL_LINKS_TOTAL, value: numLinks }, + { + name: PIPELINES_DETAIL_LINKS_JOB_RATIO, + value: numLinks / numGroups, + }, + ], + }; + + reportPerformance(metricsPath, data); + }); +}; diff --git a/app/assets/javascripts/pipelines/components/graph/utils.js b/app/assets/javascripts/pipelines/components/graph/utils.js index 163b3898c28..3acfc10108b 100644 --- a/app/assets/javascripts/pipelines/components/graph/utils.js +++ b/app/assets/javascripts/pipelines/components/graph/utils.js @@ -1,7 +1,10 @@ import { isEmpty } from 'lodash'; import Visibility from 'visibilityjs'; import { getIdFromGraphQLId } from '~/graphql_shared/utils'; +import { reportToSentry } from '../../utils'; +import { listByLayers } from '../parsing_utils'; import { unwrapStagesWithNeedsAndLookup } from '../unwrapping_utils'; +import { beginPerfMeasure, finishPerfMeasureAndSend } from './perf_utils'; const addMulti = (mainPipelineProjectPath, linkedPipeline) => { return { @@ -10,6 +13,28 @@ const addMulti = (mainPipelineProjectPath, linkedPipeline) => { }; }; +const calculatePipelineLayersInfo = (pipeline, componentName, metricsPath) => { + const shouldCollectMetrics = Boolean(metricsPath.length); + + if (shouldCollectMetrics) { + beginPerfMeasure(); + } + + let layers = null; + + try { + layers = listByLayers(pipeline); + + if (shouldCollectMetrics) { + finishPerfMeasureAndSend(layers.linksData.length, layers.numGroups, metricsPath); + } + } catch (err) { + reportToSentry(componentName, err); + } + + return layers; +}; + /* eslint-disable @gitlab/require-i18n-strings */ const getQueryHeaders = (etagResource) => { return { @@ -106,6 +131,7 @@ const unwrapPipelineData = (mainPipelineProjectPath, data) => { const validateConfigPaths = (value) => value.graphqlResourceEtag?.length > 0; export { + calculatePipelineLayersInfo, getQueryHeaders, serializeGqlErr, serializeLoadErrors, diff --git a/app/assets/javascripts/pipelines/components/graph_shared/drawing_utils.js b/app/assets/javascripts/pipelines/components/graph_shared/drawing_utils.js index 83f2466f0bf..d6d9ea94c13 100644 --- a/app/assets/javascripts/pipelines/components/graph_shared/drawing_utils.js +++ b/app/assets/javascripts/pipelines/components/graph_shared/drawing_utils.js @@ -13,7 +13,7 @@ export const createUniqueLinkId = (stageName, jobName) => `${stageName}-${jobNam * @returns {Array} Links that contain all the information about them */ -export const generateLinksData = ({ links }, containerID, modifier = '') => { +export const generateLinksData = (links, containerID, modifier = '') => { const containerEl = document.getElementById(containerID); return links.map((link) => { diff --git a/app/assets/javascripts/pipelines/components/graph_shared/links_inner.vue b/app/assets/javascripts/pipelines/components/graph_shared/links_inner.vue index 5c775df7b48..1189c2ebad8 100644 --- a/app/assets/javascripts/pipelines/components/graph_shared/links_inner.vue +++ b/app/assets/javascripts/pipelines/components/graph_shared/links_inner.vue @@ -17,8 +17,8 @@ export default { type: Object, required: true, }, - parsedData: { - type: Object, + linksData: { + type: Array, required: true, }, pipelineId: { @@ -95,7 +95,7 @@ export default { highlightedJobs(jobs) { this.$emit('highlightedJobsChange', jobs); }, - parsedData() { + linksData() { this.calculateLinkData(); }, viewType() { @@ -112,7 +112,7 @@ export default { reportToSentry(this.$options.name, `error: ${err}, info: ${info}`); }, mounted() { - if (!isEmpty(this.parsedData)) { + if (!isEmpty(this.linksData)) { this.calculateLinkData(); } }, @@ -122,7 +122,7 @@ export default { }, calculateLinkData() { try { - this.links = generateLinksData(this.parsedData, this.containerId, `-${this.pipelineId}`); + this.links = generateLinksData(this.linksData, this.containerId, `-${this.pipelineId}`); } catch (err) { this.$emit('error', { type: DRAW_FAILURE, reportToSentry: false }); reportToSentry(this.$options.name, err); diff --git a/app/assets/javascripts/pipelines/components/graph_shared/links_layer.vue b/app/assets/javascripts/pipelines/components/graph_shared/links_layer.vue index 81409752621..ef24694e494 100644 --- a/app/assets/javascripts/pipelines/components/graph_shared/links_layer.vue +++ b/app/assets/javascripts/pipelines/components/graph_shared/links_layer.vue @@ -1,20 +1,16 @@