gitlab-ce/qa/spec/tools/reliable_report_spec.rb

# frozen_string_literal: true

describe QA::Tools::ReliableReport do
  include QA::Support::Helpers::StubEnv

  before do
    stub_env("QA_INFLUXDB_URL", "url")
    stub_env("QA_INFLUXDB_TOKEN", "token")
    stub_env("SLACK_WEBHOOK", "slack_url")
    stub_env("CI_API_V4_URL", "gitlab_api_url")
    stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token")

    allow(RestClient::Request).to receive(:execute)
    allow(Slack::Notifier).to receive(:new).and_return(slack_notifier)
    allow(InfluxDB2::Client).to receive(:new).and_return(influx_client)

    allow(query_api).to receive(:query).with(query: flux_query(reliable: false)).and_return(runs)
    allow(query_api).to receive(:query).with(query: flux_query(reliable: true)).and_return(reliable_runs)
  end

  let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) }
  let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) }
  let(:query_api) { instance_double("InfluxDB2::QueryApi") }

  let(:slack_channel) { "#quality-reports" }
  let(:range) { 14 }
  let(:issue_url) { "https://gitlab.com/issue/1" }
  let(:time) { "2021-12-07T04:05:25.000000000+00:00" }
  let(:failure_message) { 'random failure message' }

  let(:run_values) do
    {
      "name" => "stable spec1",
      "status" => "passed",
      "file_path" => "/some/spec.rb",
      "stage" => "create",
      "product_group" => "code_review",
      "testcase" => "https://testcase/url",
      "run_type" => "staging",
      "_time" => time
    }
  end

  let(:run_more_values) do
    {
      "name" => "stable spec2",
      "status" => "passed",
      "file_path" => "/some/spec.rb",
      "stage" => "manage",
      "product_group" => "import_and_integrate",
      "testcase" => "https://testcase/url",
      "run_type" => "staging",
      "_time" => time
    }
  end

  let(:runs) do
    [
      instance_double(
        "InfluxDB2::FluxTable",
        records: [
          instance_double("InfluxDB2::FluxRecord", values: run_values),
          instance_double("InfluxDB2::FluxRecord", values: run_values),
          instance_double("InfluxDB2::FluxRecord", values: run_values.merge({ "_time" => Time.now.to_s }))
        ]
      ),
      instance_double(
        "InfluxDB2::FluxTable",
        records: [
          instance_double("InfluxDB2::FluxRecord", values: run_more_values),
          instance_double("InfluxDB2::FluxRecord", values: run_more_values),
          instance_double("InfluxDB2::FluxRecord", values: run_more_values.merge({ "_time" => Time.now.to_s }))
        ]
      )
    ]
  end

  let(:reliable_run_values) do
    {
      "name" => "unstable spec",
      "status" => "failed",
      "file_path" => "/some/spec.rb",
      "stage" => "create",
      "product_group" => "code_review",
      "failure_exception" => failure_message,
      "job_url" => "https://job/url",
      "testcase" => "https://testcase/url",
      "failure_issue" => "https://issues/url",
      "run_type" => "staging",
      "_time" => time
    }
  end

  let(:reliable_run_more_values) do
    {
      "name" => "unstable spec",
      "status" => "failed",
      "file_path" => "/some/spec.rb",
      "stage" => "manage",
      "product_group" => "import_and_integrate",
      "failure_exception" => failure_message,
      "job_url" => "https://job/url",
      "testcase" => "https://testcase/url",
      "failure_issue" => "https://issues/url",
      "run_type" => "staging",
      "_time" => time
    }
  end

  let(:reliable_runs) do
    [
      instance_double(
        "InfluxDB2::FluxTable",
        records: [
          instance_double("InfluxDB2::FluxRecord", values: { **reliable_run_values, "status" => "passed" }),
          instance_double("InfluxDB2::FluxRecord", values: reliable_run_values),
          instance_double("InfluxDB2::FluxRecord", values: reliable_run_values.merge({ "_time" => Time.now.to_s }))
        ]
      ),
      instance_double(
        "InfluxDB2::FluxTable",
        records: [
          instance_double("InfluxDB2::FluxRecord", values: { **reliable_run_more_values, "status" => "passed" }),
          instance_double("InfluxDB2::FluxRecord", values: reliable_run_more_values),
          instance_double("InfluxDB2::FluxRecord", values: reliable_run_more_values.merge({ "_time" => Time.now.to_s }))
        ]
      )
    ]
  end

  def flux_query(reliable:)
    <<~QUERY
      from(bucket: "e2e-test-stats-main")
        |> range(start: -#{range}d)
        |> filter(fn: (r) => r._measurement == "test-stats")
        |> filter(fn: (r) => r.run_type == "staging-full" or
          r.run_type == "staging-sanity" or
          r.run_type == "production-full" or
          r.run_type == "production-sanity" or
          r.run_type == "package-and-qa" or
          r.run_type == "nightly"
        )
        |> filter(fn: (r) => r.job_name != "airgapped" and
          r.job_name != "instance-image-slow-network" and
          r.job_name != "nplus1-instance-image"
        )
        |> filter(fn: (r) => r.status != "pending" and
          r.merge_request == "false" and
          r.quarantined == "false" and
          r.smoke == "false" and
          r.reliable == "#{reliable}"
        )
        |> filter(fn: (r) => r["_field"] == "job_url" or
          r["_field"] == "failure_exception" or
          r["_field"] == "id" or
          r["_field"] == "failure_issue"
        )
        |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value")
        |> group(columns: ["name"])
    QUERY
  end

  def expected_stage_markdown(result, stage, product_group, type)
    <<~SECTION.strip
      ## #{stage.capitalize} (1)

      <details>
      <summary>Executions table ~\"group::#{product_group}\" (1)</summary>

      #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)}

      </details>
    SECTION
  end

  def expected_summary_table(summary, type, markdown = false)
    table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown)
  end

  def table(rows, headings, title, markdown = false)
    Terminal::Table.new(
      headings: headings,
      title: markdown ? nil : title,
      rows: rows,
      style: markdown ? { border: :markdown } : { all_separators: true }
    )
  end

  def name_column(spec_name, exceptions_and_related_urls = {})
    "**Name**: #{spec_name}<br>**File**: [spec.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb)#{exceptions_markdown(exceptions_and_related_urls)}"
  end

  def exceptions_markdown(exceptions_and_related_urls)
    exceptions_and_related_urls.empty? ? '' : "<br>**Exceptions**:<br>- [`#{failure_message}`](https://issues/url)"
  end

  describe '.run' do
    subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) }

    context "without report creation" do
      let(:create_issue) { "false" }

      it "does not create report issue", :aggregate_failures do
        expect { run }.to output.to_stdout

        expect(RestClient::Request).not_to have_received(:execute)
        expect(slack_notifier).not_to have_received(:post)
      end
    end

    context "with report creation" do
      let(:create_issue) { "true" }
      let(:iid) { 2 }
      let(:old_iid) { 1 }
      let(:issue_endpoint) { "gitlab_api_url/projects/278964/issues" }

      let(:common_api_args) do
        {
          verify_ssl: false,
          headers: { "PRIVATE-TOKEN" => "gitlab_token" },
          cookies: {}
        }
      end

      let(:create_issue_response) do
        instance_double(
          "RestClient::Response",
          code: 200,
          body: { web_url: issue_url, iid: iid }.to_json
        )
      end

      let(:open_issues_response) do
        instance_double(
          "RestClient::Response",
          code: 200,
          body: [{ web_url: issue_url, iid: iid }, { web_url: issue_url, iid: old_iid }].to_json
        )
      end

      let(:success_response) do
        instance_double("RestClient::Response", code: 200, body: {}.to_json)
      end

      before do
        allow(RestClient::Request).to receive(:execute).exactly(4).times.and_return(
          create_issue_response,
          open_issues_response,
          success_response,
          success_response
        )
      end

      shared_examples 'report creation' do
        it "creates report issue" do
          expect { run }.to output.to_stdout

          expect(RestClient::Request).to have_received(:execute).with(
            method: :post,
            url: issue_endpoint,
            payload: {
              title: "Reliable e2e test report",
              description: expected_issue_body,
              labels: "reliable test report,Quality,test,type::maintenance,automation:ml"
            },
            **common_api_args
          )
          expect(RestClient::Request).to have_received(:execute).with(
            method: :get,
            url: "#{issue_endpoint}?labels=reliable test report&state=opened",
            **common_api_args
          )
          expect(RestClient::Request).to have_received(:execute).with(
            method: :put,
            url: "#{issue_endpoint}/#{old_iid}",
            payload: {
              state_event: "close"
            },
            **common_api_args
          )
          expect(RestClient::Request).to have_received(:execute).with(
            method: :post,
            url: "#{issue_endpoint}/#{old_iid}/notes",
            payload: {
              body: "Closed issue in favor of ##{iid}"
            },
            **common_api_args
          )
          expect(slack_notifier).to have_received(:post).with(
            icon_emoji: ":tanuki-protect:",
            text: expected_slack_text
          )
        end
      end

      context "with disallowed exception" do
        let(:failure_message) { 'random failure message' }

        let(:expected_issue_body) do
          <<~TXT.strip
            [[_TOC_]]

            # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})

            Total amount: **2**

            #{expected_summary_table([['create', 1], ['manage', 1]], :stable, true)}

            #{expected_stage_markdown([[name_column('stable spec1'), 3, 0, '0%']], 'create', 'code review', :stable)}

            #{expected_stage_markdown([[name_column('stable spec2'), 3, 0, '0%']], 'manage', 'import and integrate', :stable)}

            # Reliable specs with failures (#{Date.today - range} - #{Date.today})

            Total amount: **2**

            #{expected_summary_table([['create', 1], ['manage', 1]], :unstable, true)}

            #{expected_stage_markdown([[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'create', 'code review', :unstable)}

            #{expected_stage_markdown([[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'manage', 'import and integrate', :unstable)}
          TXT
        end

        let(:expected_slack_text) do
          <<~TEXT
            ```#{expected_summary_table([['create', 1], ['manage', 1]], :stable)}```
            ```#{expected_summary_table([['create', 1], ['manage', 1]], :unstable)}```

            #{issue_url}
          TEXT
        end

        it_behaves_like "report creation"
      end

      context "with allowed exception" do
        let(:failure_message) { 'Ambiguous match' }

        let(:expected_issue_body) do
          <<~TXT.strip
            [[_TOC_]]

            # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})

            Total amount: **2**

            #{expected_summary_table([['create', 1], ['manage', 1]], :stable, true)}

            #{expected_stage_markdown([[name_column('stable spec1'), 3, 0, '0%']], 'create', 'code review', :stable)}

            #{expected_stage_markdown([[name_column('stable spec2'), 3, 0, '0%']], 'manage', 'import and integrate', :stable)}
          TXT
        end

        let(:expected_slack_text) do
          <<~TEXT
            ```#{expected_summary_table([['create', 1], ['manage', 1]], :stable)}```
            ```#{expected_summary_table([], :unstable)}```

            #{issue_url}
          TEXT
        end

        it_behaves_like "report creation"
      end
    end

    context "with failure" do
      let(:create_issue) { "true" }

      before do
        allow(query_api).to receive(:query).and_raise("Connection error!")
      end

      it "notifies failure", :aggregate_failures do
        expect { expect { run }.to raise_error("Connection error!") }.to output.to_stdout

        expect(slack_notifier).to have_received(:post).with(
          icon_emoji: ":sadpanda:",
          text: "Reliable reporter failed to create report. Error: ```Connection error!```"
        )
      end
    end
  end

  describe "#allowed_failure?" do
    subject(:reliable_report) { described_class.new(14) }

    it "returns true for an allowed failure" do
      expect(reliable_report.send(:allowed_failure?, "Couldn't find option named abc")).to be true
    end

    it "returns false for disallowed failure" do
      expect(reliable_report.send(:allowed_failure?,
        %q([Unable to find css "[data-testid=\"user_action_dropdown\"]"]))).to be false
    end
  end

  describe "#exceptions_and_related_urls" do
    subject(:reliable_report) { described_class.new(14) }

    let(:failure_message) { "This is a failure exception" }
    let(:job_url) { "https://example.com/job/url" }
    let(:failure_issue_url) { "https://example.com/failure/issue" }

    let(:records) do
      [instance_double("InfluxDB2::FluxRecord", values: values)]
    end

    context "without failure_exception" do
      let(:values) do
        {
          "failure_exception" => nil,
          "job_url" => job_url,
          "failure_issue" => failure_issue_url
        }
      end

      it "returns an empty hash" do
        expect(reliable_report.send(:exceptions_and_related_urls, records)).to be_empty
      end
    end

    context "with failure_exception" do
      context "without failure_issue" do
        let(:values) do
          {
            "failure_exception" => failure_message,
            "job_url" => job_url
          }
        end

        it "returns job_url as value" do
          expect(reliable_report.send(:exceptions_and_related_urls, records).values).to eq([job_url])
        end
      end

      context "with failure_issue and job_url" do
        let(:values) do
          {
            "failure_exception" => failure_message,
            "failure_issue" => failure_issue_url,
            "job_url" => job_url
          }
        end

        it "returns failure_issue as value" do
          expect(reliable_report.send(:exceptions_and_related_urls, records).values).to eq([failure_issue_url])
        end
      end
    end
  end

  describe "#specs_attributes" do
    subject(:reliable_report) { described_class.new(14) }

    let(:report_web_url) { 'https://report/url' }

    before do
      allow(reliable_report).to receive(:report_web_url).and_return(report_web_url)
    end

    shared_examples "spec attributes" do |stable|
      it "returns #{stable} spec attributes" do
        expect(reliable_report.send(:specs_attributes, stable: stable)).to eq(expected_specs_attributes)
      end
    end

    context "with stable false" do
      let(:expected_specs_attributes) do
        { type: "Unstable Specs",
          report_issue: "https://report/url",
          specs:
            [
              { stage: "create",
                product_group: "code_review",
                name: "unstable spec",
                file: "spec.rb",
                link: "https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb",
                runs: 3,
                failed: 2,
                failure_issue: "https://issues/url",
                failure_rate: 66.67,
                testcase: "https://testcase/url",
                file_path: "/qa/qa/specs/features/some/spec.rb",
                run_type: "staging" },
              { stage: "manage",
                product_group: "import_and_integrate",
                name: "unstable spec",
                file: "spec.rb",
                link: "https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb",
                runs: 3,
                failed: 2,
                failure_issue: "https://issues/url",
                failure_rate: 66.67,
                testcase: "https://testcase/url",
                file_path: "/qa/qa/specs/features/some/spec.rb",
                run_type: "staging" }
            ] }
      end

      it_behaves_like "spec attributes", false
    end

    context "with stable true" do
      let(:expected_specs_attributes) do
        {
          type: "Stable Specs",
          report_issue: "https://report/url",
          specs:
            [
              { stage: "create",
                product_group: "code_review",
                name: "stable spec1",
                file: "spec.rb",
                link: "https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb",
                runs: 3,
                failed: 0,
                failure_issue: "",
                failure_rate: 0,
                testcase: "https://testcase/url",
                file_path: "/qa/qa/specs/features/some/spec.rb",
                run_type: "staging" },
              { stage: "manage",
                product_group: "import_and_integrate",
                name: "stable spec2",
                file: "spec.rb",
                link: "https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb",
                runs: 3,
                failed: 0,
                failure_issue: "",
                failure_rate: 0,
                testcase: "https://testcase/url",
                file_path: "/qa/qa/specs/features/some/spec.rb",
                run_type: "staging" }
            ]
        }
      end

      it_behaves_like "spec attributes", true
    end
  end
end