Elasticsearch: Detect Elasticsearch version (#63341)

* elasticsearch: detect database version

* more test-friendly code

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/MetricEditor.tsx

@@ -1,9 +1,9 @@
 import { cx } from '@emotion/css';
 import React, { useCallback } from 'react';
-import { satisfies } from 'semver';
+import { satisfies, SemVer } from 'semver';
 
 import { SelectableValue } from '@grafana/data';
-import { InlineSegmentGroup, Segment, SegmentAsync, useTheme2 } from '@grafana/ui';
+import { InlineSegmentGroup, SegmentAsync, useTheme2 } from '@grafana/ui';
 
 import { useFields } from '../../../hooks/useFields';
 import { useDispatch } from '../../../hooks/useStatelessReducer';
@@ -41,15 +41,16 @@ const isBasicAggregation = (metric: MetricAggregation) => !metricAggregationConf
 
 const getTypeOptions = (
   previousMetrics: MetricAggregation[],
-  esVersion: string
+  esVersion: SemVer | null
 ): Array<SelectableValue<MetricAggregationType>> => {
   // we'll include Pipeline Aggregations only if at least one previous metric is a "Basic" one
   const includePipelineAggregations = previousMetrics.some(isBasicAggregation);
 
   return (
     Object.entries(metricAggregationConfig)
-      // Only showing metrics type supported by the configured version of ES
+      // Only showing metrics type supported by the version of ES.
-      .filter(([_, { versionRange = '*' }]) => satisfies(esVersion, versionRange))
+      // if we cannot determine the version, we assume it is suitable.
+      .filter(([_, { versionRange = '*' }]) => (esVersion != null ? satisfies(esVersion, versionRange) : true))
       // Filtering out Pipeline Aggregations if there's no basic metric selected before
       .filter(([_, config]) => includePipelineAggregations || !config.isPipelineAgg)
       .map(([key, { label }]) => ({
@@ -66,6 +67,11 @@ export const MetricEditor = ({ value }: Props) => {
   const dispatch = useDispatch();
   const getFields = useFields(value.type);
 
+  const getTypeOptionsAsync = async (previousMetrics: MetricAggregation[]) => {
+    const dbVersion = await datasource.getDatabaseVersion();
+    return getTypeOptions(previousMetrics, dbVersion);
+  };
+
   const loadOptions = useCallback(async () => {
     const remoteFields = await getFields();
 
@@ -85,9 +91,9 @@ export const MetricEditor = ({ value }: Props) => {
   return (
     <>
       <InlineSegmentGroup>
-        <Segment
+        <SegmentAsync
           className={cx(styles.color, segmentStyles)}
-          options={getTypeOptions(previousMetrics, datasource.esVersion)}
+          loadOptions={() => getTypeOptionsAsync(previousMetrics)}
           onChange={(e) => dispatch(changeMetricType({ id: value.id, type: e.value! }))}
           value={toOption(value)}
         />

public/app/plugins/datasource/elasticsearch/components/QueryEditor/index.test.tsx

@@ -9,6 +9,7 @@ import { QueryEditor } from '.';
 const noop = () => void 0;
 const datasourceMock = {
   esVersion: '7.10.0',
+  getDatabaseVersion: () => Promise.resolve(null),
 } as ElasticDatasource;
 
 describe('QueryEditor', () => {

public/app/plugins/datasource/elasticsearch/components/QueryEditor/index.tsx

@@ -1,5 +1,6 @@
 import { css } from '@emotion/css';
-import React from 'react';
+import React, { useEffect, useState } from 'react';
+import { SemVer } from 'semver';
 
 import { getDefaultTimeRange, GrafanaTheme2, QueryEditorProps } from '@grafana/data';
 import { Alert, InlineField, InlineLabel, Input, QueryField, useStyles2 } from '@grafana/ui';
@@ -8,7 +9,7 @@ import { ElasticDatasource } from '../../datasource';
 import { useNextId } from '../../hooks/useNextId';
 import { useDispatch } from '../../hooks/useStatelessReducer';
 import { ElasticsearchOptions, ElasticsearchQuery } from '../../types';
-import { isSupportedVersion } from '../../utils';
+import { isSupportedVersion, unsupportedVersionMessage } from '../../utils';
 
 import { BucketAggregationsEditor } from './BucketAggregationsEditor';
 import { ElasticsearchProvider } from './ElasticsearchQueryContext';
@@ -18,14 +19,35 @@ import { changeAliasPattern, changeQuery } from './state';
 
 export type ElasticQueryEditorProps = QueryEditorProps<ElasticDatasource, ElasticsearchQuery, ElasticsearchOptions>;
 
-export const QueryEditor = ({ query, onChange, onRunQuery, datasource, range }: ElasticQueryEditorProps) => {
+// a react hook that returns the elasticsearch database version,
-  if (!isSupportedVersion(datasource.esVersion)) {
+// or `null`, while loading, or if it is not possible to determine the value.
-    return (
+function useElasticVersion(datasource: ElasticDatasource): SemVer | null {
-      <Alert
+  const [version, setVersion] = useState<SemVer | null>(null);
-        title={`Support for Elasticsearch versions after their end-of-life (currently versions < 7.10) was removed`}
+  useEffect(() => {
-      ></Alert>
+    let canceled = false;
-    );
+    datasource.getDatabaseVersion().then(
+      (version) => {
+        if (!canceled) {
+          setVersion(version);
         }
+      },
+      (error) => {
+        // we do nothing
+        console.log(error);
+      }
+    );
+
+    return () => {
+      canceled = true;
+    };
+  }, [datasource]);
+
+  return version;
+}
+
+export const QueryEditor = ({ query, onChange, onRunQuery, datasource, range }: ElasticQueryEditorProps) => {
+  const elasticVersion = useElasticVersion(datasource);
+  const showUnsupportedMessage = elasticVersion != null && !isSupportedVersion(elasticVersion);
   return (
     <ElasticsearchProvider
       datasource={datasource}
@@ -34,6 +56,7 @@ export const QueryEditor = ({ query, onChange, onRunQuery, datasource, range }:
       query={query}
       range={range || getDefaultTimeRange()}
     >
+      {showUnsupportedMessage && <Alert title={unsupportedVersionMessage} />}
       <QueryEditorForm value={query} />
     </ElasticsearchProvider>
   );

public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx

@@ -6,7 +6,6 @@ import { Alert, DataSourceHttpSettings, SecureSocksProxySettings } from '@grafan
 import { config } from 'app/core/config';
 
 import { ElasticsearchOptions } from '../types';
-import { isSupportedVersion } from '../utils';
 
 import { DataLinks } from './DataLinks';
 import { ElasticDetails } from './ElasticDetails';
@@ -34,8 +33,6 @@ export const ConfigEditor = (props: Props) => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
-  const supportedVersion = isSupportedVersion(options.jsonData.esVersion);
-
   return (
     <>
       {options.access === 'direct' && (
@@ -43,11 +40,6 @@ export const ConfigEditor = (props: Props) => {
           Browser access mode in the Elasticsearch datasource is no longer available. Switch to server access mode.
         </Alert>
       )}
-      {!supportedVersion && (
-        <Alert title="Deprecation notice" severity="error">
-          {`Support for Elasticsearch versions after their end-of-life (currently versions < 7.10) was removed`}
-        </Alert>
-      )}
       <DataSourceHttpSettings
         defaultUrl="http://localhost:9200"
         dataSourceConfig={options}

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -181,14 +181,14 @@ describe('ElasticDatasource', () => {
   });
 
   describe('When testing datasource with index pattern', () => {
-    it('should translate index pattern to current day', () => {
+    it('should translate index pattern to current day', async () => {
       const { ds, fetchMock } = getTestContext({ jsonData: { interval: 'Daily', esVersion: '7.10.0' } });
 
-      ds.testDatasource();
+      await ds.testDatasource();
 
       const today = toUtc().format('YYYY.MM.DD');
-      expect(fetchMock).toHaveBeenCalledTimes(1);
+      const lastCall = fetchMock.mock.calls[fetchMock.mock.calls.length - 1];
-      expect(fetchMock.mock.calls[0][0].url).toBe(`${ELASTICSEARCH_MOCK_URL}/test-${today}/_mapping`);
+      expect(lastCall[0].url).toBe(`${ELASTICSEARCH_MOCK_URL}/test-${today}/_mapping`);
     });
   });
 

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -1,6 +1,7 @@
 import { cloneDeep, find, first as _first, isNumber, isObject, isString, map as _map } from 'lodash';
 import { generate, lastValueFrom, Observable, of, throwError } from 'rxjs';
 import { catchError, first, map, mergeMap, skipWhile, throwIfEmpty, tap } from 'rxjs/operators';
+import { SemVer } from 'semver';
 
 import {
   DataFrame,
@@ -49,7 +50,7 @@ import { metricAggregationConfig } from './components/QueryEditor/MetricAggregat
 import { defaultBucketAgg, hasMetricOfType } from './queryDef';
 import { trackQuery } from './tracking';
 import { Logs, BucketAggregation, DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery, TermsQuery } from './types';
-import { coerceESVersion, getScriptValue, isSupportedVersion } from './utils';
+import { coerceESVersion, getScriptValue, isSupportedVersion, unsupportedVersionMessage } from './utils';
 
 export const REF_ID_STARTER_LOG_VOLUME = 'log-volume-';
 // Those are metadata fields as defined in https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-fields.html#_identity_metadata_fields.
@@ -92,6 +93,7 @@ export class ElasticDatasource
   includeFrozen: boolean;
   isProxyAccess: boolean;
   timeSrv: TimeSrv;
+  databaseVersion: SemVer | null;
 
   constructor(
     instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
@@ -119,6 +121,7 @@ export class ElasticDatasource
     this.logLevelField = settingsData.logLevelField || '';
     this.dataLinks = settingsData.dataLinks || [];
     this.includeFrozen = settingsData.includeFrozen ?? false;
+    this.databaseVersion = null;
     this.annotations = {
       QueryEditor: ElasticsearchAnnotationsQueryEditor,
     };
@@ -147,13 +150,6 @@ export class ElasticDatasource
       return throwError(() => error);
     }
 
-    if (!isSupportedVersion(this.esVersion)) {
-      const error = new Error(
-        'Support for Elasticsearch versions after their end-of-life (currently versions < 7.10) was removed.'
-      );
-      return throwError(() => error);
-    }
-
     const options: BackendSrvRequest = {
       url: this.url + '/' + url,
       method,
@@ -395,16 +391,24 @@ export class ElasticDatasource
     return queries.map((q) => this.applyTemplateVariables(q, scopedVars));
   }
 
-  testDatasource() {
+  async testDatasource() {
+    // we explicitly ask for uncached, "fresh" data here
+    const dbVersion = await this.getDatabaseVersion(false);
+    // if we are not able to determine the elastic-version, we assume it is a good version.
+    const isSupported = dbVersion != null ? isSupportedVersion(dbVersion) : true;
+    const versionMessage = isSupported ? '' : `WARNING: ${unsupportedVersionMessage} `;
     // validate that the index exist and has date field
     return lastValueFrom(
       this.getFields(['date']).pipe(
         mergeMap((dateFields) => {
           const timeField: any = find(dateFields, { text: this.timeField });
           if (!timeField) {
-            return of({ status: 'error', message: 'No date field named ' + this.timeField + ' found' });
+            return of({
+              status: 'error',
+              message: 'No date field named ' + this.timeField + ' found',
+            });
           }
-          return of({ status: 'success', message: 'Index OK. Time field name OK.' });
+          return of({ status: 'success', message: `${versionMessage}Index OK. Time field name OK` });
         }),
         catchError((err) => {
           console.error(err);
@@ -1040,6 +1044,41 @@ export class ElasticDatasource
     const finalQuery = JSON.parse(this.templateSrv.replace(JSON.stringify(expandedQuery), scopedVars));
     return finalQuery;
   }
+
+  private getDatabaseVersionUncached(): Promise<SemVer | null> {
+    // we want this function to never fail
+    return lastValueFrom(this.request('GET', '/')).then(
+      (data) => {
+        const versionNumber = data?.version?.number;
+        if (typeof versionNumber !== 'string') {
+          return null;
+        }
+        try {
+          return new SemVer(versionNumber);
+        } catch (error) {
+          console.error(error);
+          return null;
+        }
+      },
+      (error) => {
+        console.error(error);
+        return null;
+      }
+    );
+  }
+
+  async getDatabaseVersion(useCachedData = true): Promise<SemVer | null> {
+    if (useCachedData) {
+      const cached = this.databaseVersion;
+      if (cached != null) {
+        return cached;
+      }
+    }
+
+    const freshDatabaseVersion = await this.getDatabaseVersionUncached();
+    this.databaseVersion = freshDatabaseVersion;
+    return freshDatabaseVersion;
+  }
 }
 
 /**

public/app/plugins/datasource/elasticsearch/utils.ts

@@ -1,4 +1,4 @@
-import { valid, gte } from 'semver';
+import { valid, gte, SemVer } from 'semver';
 
 import { isMetricAggregationWithField } from './components/QueryEditor/MetricAggregationsEditor/aggregations';
 import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils';
@@ -117,10 +117,13 @@ export const coerceESVersion = (version: string | number | undefined): string =>
   }
 };
 
-export const isSupportedVersion = (version: string): boolean => {
+export const isSupportedVersion = (version: SemVer): boolean => {
   if (gte(version, '7.10.0')) {
     return true;
   }
 
   return false;
 };
+
+export const unsupportedVersionMessage =
+  'Support for Elasticsearch versions after their end-of-life (currently versions < 7.10) was removed. Using unsupported version of Elasticsearch may lead to unexpected and incorrect results.';