Merge branch 'backport_custom_metrics_ce_components' into 'master'

Backport custom metrics ce components

See merge request gitlab-org/gitlab-ce!17300

app/controllers/projects/prometheus/metrics_controller.rb

@@ -0,0 +1,27 @@
+module Projects
+  module Prometheus
+    class MetricsController < Projects::ApplicationController
+      before_action :authorize_admin_project!
+
+      def active_common
+        respond_to do |format|
+          format.json do
+            matched_metrics = prometheus_service.matched_metrics || {}
+
+            if matched_metrics.any?
+              render json: matched_metrics
+            else
+              head :no_content
+            end
+          end
+        end
+      end
+
+      private
+
+      def prometheus_service
+        @prometheus_service ||= project.find_or_initialize_service('prometheus')
+      end
+    end
+  end
+end

app/controllers/projects/prometheus_controller.rb

@@ -1,24 +0,0 @@
-class Projects::PrometheusController < Projects::ApplicationController
-  before_action :authorize_read_project!
-  before_action :require_prometheus_metrics!
-
-  def active_metrics
-    respond_to do |format|
-      format.json do
-        matched_metrics = project.prometheus_service.matched_metrics || {}
-
-        if matched_metrics.any?
-          render json: matched_metrics
-        else
-          head :no_content
-        end
-      end
-    end
-  end
-
-  private
-
-  def require_prometheus_metrics!
-    render_404 unless project.prometheus_service.present?
-  end
-end

app/models/project_services/prometheus_service.rb

@@ -69,16 +69,16 @@ class PrometheusService < MonitoringService
     client.ping
 
     { success: true, result: 'Checked API endpoint' }
-  rescue Gitlab::PrometheusError => err
+  rescue Gitlab::PrometheusClient::Error => err
     { success: false, result: err }
   end
 
   def environment_metrics(environment)
-    with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics))
+    with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &rename_field(:data, :metrics))
   end
 
   def deployment_metrics(deployment)
-    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &method(:rename_data_to_metrics))
+    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &rename_field(:data, :metrics))
     metrics&.merge(deployment_time: deployment.created_at.to_i) || {}
   end
 
@@ -107,7 +107,7 @@ class PrometheusService < MonitoringService
       data: data,
       last_update: Time.now.utc
     }
-  rescue Gitlab::PrometheusError => err
+  rescue Gitlab::PrometheusClient::Error => err
     { success: false, result: err.message }
   end
 
@@ -116,10 +116,10 @@ class PrometheusService < MonitoringService
       Gitlab::PrometheusClient.new(RestClient::Resource.new(api_url))
     else
       cluster = cluster_with_prometheus(environment_id)
-      raise Gitlab::PrometheusError, "couldn't find cluster with Prometheus installed" unless cluster
+      raise Gitlab::PrometheusClient::Error, "couldn't find cluster with Prometheus installed" unless cluster
 
       rest_client = client_from_cluster(cluster)
-      raise Gitlab::PrometheusError, "couldn't create proxy Prometheus client" unless rest_client
+      raise Gitlab::PrometheusClient::Error, "couldn't create proxy Prometheus client" unless rest_client
 
       Gitlab::PrometheusClient.new(rest_client)
     end
@@ -152,10 +152,12 @@ class PrometheusService < MonitoringService
     cluster.application_prometheus.proxy_client
   end
 
-  def rename_data_to_metrics(metrics)
+  def rename_field(old_field, new_field)
-    metrics[:metrics] = metrics.delete :data
+    -> (metrics) do
+      metrics[new_field] = metrics.delete(old_field)
       metrics
     end
+  end
 
   def synchronize_service_state!
     self.active = prometheus_installed? || manual_configuration?

app/views/projects/services/prometheus/_show.html.haml

@@ -7,7 +7,7 @@
       = link_to s_('PrometheusService|More information'), help_page_path('user/project/integrations/prometheus')
 
   .col-lg-9
-    .panel.panel-default.js-panel-monitored-metrics{ data: { "active-metrics" => "#{project_prometheus_active_metrics_path(@project, :json)}" } }
+    .panel.panel-default.js-panel-monitored-metrics{ data: { active_metrics: active_common_project_prometheus_metrics_path(@project, :json) } }
       .panel-heading
         %h3.panel-title
           = s_('PrometheusService|Monitored')

config/routes/project.rb

@@ -78,7 +78,9 @@ constraints(ProjectUrlConstrainer.new) do
       resource :mattermost, only: [:new, :create]
 
       namespace :prometheus do
-        get :active_metrics
+        resources :metrics, constraints: { id: %r{[^\/]+} }, only: [] do
+          get :active_common, on: :collection
+        end
       end
 
       resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create, :edit, :update] do

lib/gitlab/prometheus/metric_group.rb

@@ -6,9 +6,14 @@ module Gitlab
       attr_accessor :name, :priority, :metrics
       validates :name, :priority, :metrics, presence: true
 
-      def self.all
+      def self.common_metrics
         AdditionalMetricsParser.load_groups_from_yaml
       end
+
+      # EE only
+      def self.for_project(_)
+        common_metrics
+      end
     end
   end
 end

lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb

@@ -7,6 +7,7 @@ module Gitlab
         def query(environment_id, deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             query_metrics(
+              deployment.project,
               common_query_context(
                 deployment.environment,
                 timeframe_start: (deployment.created_at - 30.minutes).to_f,

lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb

@@ -7,6 +7,7 @@ module Gitlab
         def query(environment_id)
           ::Environment.find_by(id: environment_id).try do |environment|
             query_metrics(
+              environment.project,
               common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f)
             )
           end

lib/gitlab/prometheus/queries/matched_metrics_query.rb

@@ -18,7 +18,7 @@ module Gitlab
         private
 
         def groups_data
-          metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all)
+          metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics)
           lookup = active_series_lookup(metrics_groups)
 
           groups = {}

lib/gitlab/prometheus/queries/query_additional_metrics.rb

@@ -2,10 +2,10 @@ module Gitlab
   module Prometheus
     module Queries
       module QueryAdditionalMetrics
-        def query_metrics(query_context)
+        def query_metrics(project, query_context)
           query_processor = method(:process_query).curry[query_context]
 
-          groups = matched_metrics.map do |group|
+          groups = matched_metrics(project).map do |group|
             metrics = group.metrics.map do |metric|
               {
                 title: metric.title,
@@ -60,8 +60,8 @@ module Gitlab
           @available_metrics ||= client_label_values || []
         end
 
-        def matched_metrics
+        def matched_metrics(project)
-          result = Gitlab::Prometheus::MetricGroup.all.map do |group|
+          result = Gitlab::Prometheus::MetricGroup.for_project(project).map do |group|
             group.metrics.select! do |metric|
               metric.required_metrics.all?(&available_metrics.method(:include?))
             end

lib/gitlab/prometheus_client.rb

@@ -1,8 +1,9 @@
 module Gitlab
-  PrometheusError = Class.new(StandardError)
-
   # Helper methods to interact with Prometheus network services & resources
   class PrometheusClient
+    Error = Class.new(StandardError)
+    QueryError = Class.new(Gitlab::PrometheusClient::Error)
+
     attr_reader :rest_client, :headers
 
     def initialize(rest_client)
@@ -43,22 +44,22 @@ module Gitlab
       path = ['api', 'v1', type].join('/')
       get(path, args)
     rescue JSON::ParserError
-      raise PrometheusError, 'Parsing response failed'
+      raise PrometheusClient::Error, 'Parsing response failed'
     rescue Errno::ECONNREFUSED
-      raise PrometheusError, 'Connection refused'
+      raise PrometheusClient::Error, 'Connection refused'
     end
 
     def get(path, args)
       response = rest_client[path].get(params: args)
       handle_response(response)
     rescue SocketError
-      raise PrometheusError, "Can't connect to #{rest_client.url}"
+      raise PrometheusClient::Error, "Can't connect to #{rest_client.url}"
     rescue OpenSSL::SSL::SSLError
-      raise PrometheusError, "#{rest_client.url} contains invalid SSL data"
+      raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data"
     rescue RestClient::ExceptionWithResponse => ex
       handle_exception_response(ex.response)
     rescue RestClient::Exception
-      raise PrometheusError, "Network connection error"
+      raise PrometheusClient::Error, "Network connection error"
     end
 
     def handle_response(response)
@@ -66,16 +67,18 @@ module Gitlab
       if response.code == 200 && json_data['status'] == 'success'
         json_data['data'] || {}
       else
-        raise PrometheusError, "#{response.code} - #{response.body}"
+        raise PrometheusClient::Error, "#{response.code} - #{response.body}"
       end
     end
 
     def handle_exception_response(response)
-      if response.code == 400
+      if response.code == 200 && response['status'] == 'success'
+        response['data'] || {}
+      elsif response.code == 400
         json_data = JSON.parse(response.body)
-        raise PrometheusError, json_data['error'] || 'Bad data received'
+        raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received'
       else
-        raise PrometheusError, "#{response.code} - #{response.body}"
+        raise PrometheusClient::Error, "#{response.code} - #{response.body}"
       end
     end
 

spec/controllers/projects/prometheus/metrics_controller_spec.rb

@@ -1,20 +1,20 @@
-require('spec_helper')
+require 'spec_helper'
 
-describe Projects::PrometheusController do
+describe Projects::Prometheus::MetricsController do
   let(:user) { create(:user) }
-  let!(:project) { create(:project) }
+  let(:project) { create(:project) }
 
   let(:prometheus_service) { double('prometheus_service') }
 
   before do
     allow(controller).to receive(:project).and_return(project)
-    allow(project).to receive(:prometheus_service).and_return(prometheus_service)
+    allow(project).to receive(:find_or_initialize_service).with('prometheus').and_return(prometheus_service)
 
     project.add_master(user)
     sign_in(user)
   end
 
-  describe 'GET #active_metrics' do
+  describe 'GET #active_common' do
     context 'when prometheus metrics are enabled' do
       context 'when data is not present' do
         before do
@@ -22,7 +22,7 @@ describe Projects::PrometheusController do
         end
 
         it 'returns no content response' do
-          get :active_metrics, project_params(format: :json)
+          get :active_common, project_params(format: :json)
 
           expect(response).to have_gitlab_http_status(204)
         end
@@ -36,7 +36,7 @@ describe Projects::PrometheusController do
         end
 
         it 'returns no content response' do
-          get :active_metrics, project_params(format: :json)
+          get :active_common, project_params(format: :json)
 
           expect(response).to have_gitlab_http_status(200)
           expect(json_response).to eq(sample_response.deep_stringify_keys)
@@ -45,7 +45,7 @@ describe Projects::PrometheusController do
 
       context 'when requesting non json response' do
         it 'returns not found response' do
-          get :active_metrics, project_params
+          get :active_common, project_params
 
           expect(response).to have_gitlab_http_status(404)
         end

spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb

@@ -24,7 +24,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do
 
   context 'with one group where two metrics is found' do
     before do
-      allow(metric_group_class).to receive(:all).and_return([simple_metric_group])
+      allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group])
       allow(client).to receive(:label_values).and_return(metric_names)
     end
 
@@ -70,7 +70,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do
 
   context 'with one group where only one metric is found' do
     before do
-      allow(metric_group_class).to receive(:all).and_return([simple_metric_group])
+      allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group])
       allow(client).to receive(:label_values).and_return('metric_a')
     end
 
@@ -99,7 +99,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do
     let(:second_metric_group) { simple_metric_group(name: 'nameb', metrics: simple_metrics(added_metric_name: 'metric_c')) }
 
     before do
-      allow(metric_group_class).to receive(:all).and_return([simple_metric_group, second_metric_group])
+      allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group, second_metric_group])
       allow(client).to receive(:label_values).and_return('metric_c')
     end
 

spec/lib/gitlab/prometheus_client_spec.rb

@@ -19,41 +19,41 @@ describe Gitlab::PrometheusClient do
   # - execute_query: A query call
   shared_examples 'failure response' do
     context 'when request returns 400 with an error message' do
-      it 'raises a Gitlab::PrometheusError error' do
+      it 'raises a Gitlab::PrometheusClient::Error error' do
         req_stub = stub_prometheus_request(query_url, status: 400, body: { error: 'bar!' })
 
         expect { execute_query }
-          .to raise_error(Gitlab::PrometheusError, 'bar!')
+          .to raise_error(Gitlab::PrometheusClient::Error, 'bar!')
         expect(req_stub).to have_been_requested
       end
     end
 
     context 'when request returns 400 without an error message' do
-      it 'raises a Gitlab::PrometheusError error' do
+      it 'raises a Gitlab::PrometheusClient::Error error' do
         req_stub = stub_prometheus_request(query_url, status: 400)
 
         expect { execute_query }
-          .to raise_error(Gitlab::PrometheusError, 'Bad data received')
+          .to raise_error(Gitlab::PrometheusClient::Error, 'Bad data received')
         expect(req_stub).to have_been_requested
       end
     end
 
     context 'when request returns 500' do
-      it 'raises a Gitlab::PrometheusError error' do
+      it 'raises a Gitlab::PrometheusClient::Error error' do
         req_stub = stub_prometheus_request(query_url, status: 500, body: { message: 'FAIL!' })
 
         expect { execute_query }
-          .to raise_error(Gitlab::PrometheusError, '500 - {"message":"FAIL!"}')
+          .to raise_error(Gitlab::PrometheusClient::Error, '500 - {"message":"FAIL!"}')
         expect(req_stub).to have_been_requested
       end
     end
 
     context 'when request returns non json data' do
-      it 'raises a Gitlab::PrometheusError error' do
+      it 'raises a Gitlab::PrometheusClient::Error error' do
         req_stub = stub_prometheus_request(query_url, status: 200, body: 'not json')
 
         expect { execute_query }
-          .to raise_error(Gitlab::PrometheusError, 'Parsing response failed')
+          .to raise_error(Gitlab::PrometheusClient::Error, 'Parsing response failed')
         expect(req_stub).to have_been_requested
       end
     end
@@ -65,27 +65,27 @@ describe Gitlab::PrometheusClient do
     subject { described_class.new(RestClient::Resource.new(prometheus_url)) }
 
     context 'exceptions are raised' do
-      it 'raises a Gitlab::PrometheusError error when a SocketError is rescued' do
+      it 'raises a Gitlab::PrometheusClient::Error error when a SocketError is rescued' do
         req_stub = stub_prometheus_request_with_exception(prometheus_url, SocketError)
 
         expect { subject.send(:get, '/', {}) }
-          .to raise_error(Gitlab::PrometheusError, "Can't connect to #{prometheus_url}")
+          .to raise_error(Gitlab::PrometheusClient::Error, "Can't connect to #{prometheus_url}")
         expect(req_stub).to have_been_requested
       end
 
-      it 'raises a Gitlab::PrometheusError error when a SSLError is rescued' do
+      it 'raises a Gitlab::PrometheusClient::Error error when a SSLError is rescued' do
         req_stub = stub_prometheus_request_with_exception(prometheus_url, OpenSSL::SSL::SSLError)
 
         expect { subject.send(:get, '/', {}) }
-          .to raise_error(Gitlab::PrometheusError, "#{prometheus_url} contains invalid SSL data")
+          .to raise_error(Gitlab::PrometheusClient::Error, "#{prometheus_url} contains invalid SSL data")
         expect(req_stub).to have_been_requested
       end
 
-      it 'raises a Gitlab::PrometheusError error when a RestClient::Exception is rescued' do
+      it 'raises a Gitlab::PrometheusClient::Error error when a RestClient::Exception is rescued' do
         req_stub = stub_prometheus_request_with_exception(prometheus_url, RestClient::Exception)
 
         expect { subject.send(:get, '/', {}) }
-          .to raise_error(Gitlab::PrometheusError, "Network connection error")
+          .to raise_error(Gitlab::PrometheusClient::Error, "Network connection error")
         expect(req_stub).to have_been_requested
       end
     end

spec/models/project_services/prometheus_service_spec.rb

@@ -223,8 +223,8 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
 
       context 'with cluster for all environments without prometheus installed' do
         context 'without environment supplied' do
-          it 'raises PrometheusError because cluster was not found' do
+          it 'raises PrometheusClient::Error because cluster was not found' do
-            expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/)
+            expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/)
           end
         end
 
@@ -242,8 +242,8 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
         context 'with prod environment supplied' do
           let!(:environment) { create(:environment, project: project, name: 'prod') }
 
-          it 'raises PrometheusError because cluster was not found' do
+          it 'raises PrometheusClient::Error because cluster was not found' do
-            expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/)
+            expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/)
           end
         end
       end

spec/support/prometheus/additional_metrics_shared_examples.rb

@@ -12,11 +12,12 @@ RSpec.shared_examples 'additional metrics query' do
 
   let(:client) { double('prometheus_client') }
   let(:query_result) { described_class.new(client).query(*query_params) }
-  let(:environment) { create(:environment, slug: 'environment-slug') }
+  let(:project) { create(:project) }
+  let(:environment) { create(:environment, slug: 'environment-slug', project: project) }
 
   before do
     allow(client).to receive(:label_values).and_return(metric_names)
-    allow(metric_group_class).to receive(:all).and_return([simple_metric_group(metrics: [simple_metric])])
+    allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group(metrics: [simple_metric])])
   end
 
   context 'metrics query context' do
@@ -24,13 +25,14 @@ RSpec.shared_examples 'additional metrics query' do
 
     shared_examples 'query context containing environment slug and filter' do
       it 'contains ci_environment_slug' do
-        expect(subject).to receive(:query_metrics).with(hash_including(ci_environment_slug: environment.slug))
+        expect(subject).to receive(:query_metrics).with(project, hash_including(ci_environment_slug: environment.slug))
 
         subject.query(*query_params)
       end
 
       it 'contains environment filter' do
         expect(subject).to receive(:query_metrics).with(
+          project,
           hash_including(
             environment_filter: "container_name!=\"POD\",environment=\"#{environment.slug}\""
           )
@@ -48,7 +50,7 @@ RSpec.shared_examples 'additional metrics query' do
         it_behaves_like 'query context containing environment slug and filter'
 
         it 'query context contains kube_namespace' do
-          expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: kube_namespace))
+          expect(subject).to receive(:query_metrics).with(project, hash_including(kube_namespace: kube_namespace))
 
           subject.query(*query_params)
         end
@@ -72,7 +74,7 @@ RSpec.shared_examples 'additional metrics query' do
       it_behaves_like 'query context containing environment slug and filter'
 
       it 'query context contains empty kube_namespace' do
-        expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: ''))
+        expect(subject).to receive(:query_metrics).with(project, hash_including(kube_namespace: ''))
 
         subject.query(*query_params)
       end
@@ -81,7 +83,7 @@ RSpec.shared_examples 'additional metrics query' do
 
   context 'with one group where two metrics is found' do
     before do
-      allow(metric_group_class).to receive(:all).and_return([simple_metric_group])
+      allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group])
     end
 
     context 'some queries return results' do
@@ -117,7 +119,7 @@ RSpec.shared_examples 'additional metrics query' do
     let(:metrics) { [simple_metric(queries: [simple_query])] }
 
     before do
-      allow(metric_group_class).to receive(:all).and_return(
+      allow(metric_group_class).to receive(:common_metrics).and_return(
         [
           simple_metric_group(name: 'group_a', metrics: [simple_metric(queries: [simple_query])]),
           simple_metric_group(name: 'group_b', metrics: [simple_metric(title: 'title_b', queries: [simple_query('b')])])