grafana/public/app/plugins/datasource/elasticsearch/datasource.ts

///<reference path="../../../headers/common.d.ts" />

import angular from 'angular';
import _ from 'lodash';
import moment from 'moment';
import {ElasticQueryBuilder} from './query_builder';
import {IndexPattern} from './index_pattern';
import {ElasticResponse} from './elastic_response';

export class ElasticDatasource {
  basicAuth: string;
  withCredentials: boolean;
  url: string;
  name: string;
  index: string;
  timeField: string;
  esVersion: number;
  interval: string;
  maxConcurrentShardRequests: number;
  queryBuilder: ElasticQueryBuilder;
  indexPattern: IndexPattern;

  /** @ngInject */
  constructor(instanceSettings, private $q, private backendSrv, private templateSrv, private timeSrv) {
    this.basicAuth = instanceSettings.basicAuth;
    this.withCredentials = instanceSettings.withCredentials;
    this.url = instanceSettings.url;
    this.name = instanceSettings.name;
    this.index = instanceSettings.index;
    this.timeField = instanceSettings.jsonData.timeField;
    this.esVersion = instanceSettings.jsonData.esVersion;
    this.indexPattern = new IndexPattern(instanceSettings.index, instanceSettings.jsonData.interval);
    this.interval = instanceSettings.jsonData.timeInterval;
    this.maxConcurrentShardRequests = instanceSettings.jsonData.maxConcurrentShardRequests;
    this.queryBuilder = new ElasticQueryBuilder({
      timeField: this.timeField,
      esVersion: this.esVersion,
    });
  }

  private request(method, url, data?) {
    var options: any = {
      url: this.url + "/" + url,
      method: method,
      data: data
    };

    if (this.basicAuth || this.withCredentials) {
      options.withCredentials = true;
    }
    if (this.basicAuth) {
      options.headers = {
        "Authorization": this.basicAuth
      };
    }

    return this.backendSrv.datasourceRequest(options);
  }

  private get(url) {
    var range = this.timeSrv.timeRange();
    var index_list = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf());
    if (_.isArray(index_list) && index_list.length) {
      return this.request('GET', index_list[0] + url).then(function(results) {
        results.data.$$config = results.config;
        return results.data;
      });
    } else {
      return this.request('GET', this.indexPattern.getIndexForToday() + url).then(function(results) {
        results.data.$$config = results.config;
        return results.data;
      });
    }
  }

  private post(url, data) {
    return this.request('POST', url, data).then(function(results) {
      results.data.$$config = results.config;
      return results.data;
    }).catch(err => {
      if (err.data && err.data.error) {
        throw {message: 'Elasticsearch error: ' + err.data.error.reason, error: err.data.error};
      }

      throw err;
    });
  }

  annotationQuery(options) {
    var annotation = options.annotation;
    var timeField = annotation.timeField || '@timestamp';
    var queryString = annotation.query || '*';
    var tagsField = annotation.tagsField || 'tags';
    var textField = annotation.textField || null;

    var range = {};
    range[timeField]= {
      from: options.range.from.valueOf(),
      to: options.range.to.valueOf(),
      format: "epoch_millis",
    };

    var queryInterpolated = this.templateSrv.replace(queryString, {}, 'lucene');
    var query = {
      "bool": {
        "filter": [
          { "range": range },
          {
            "query_string": {
              "query": queryInterpolated
            }
          }
        ]
      }
    };

    var data = {
      "query" : query,
      "size": 10000
    };

    // fields field not supported on ES 5.x
    if (this.esVersion < 5) {
      data["fields"] = [timeField, "_source"];
    }

    var header: any = {search_type: "query_then_fetch", "ignore_unavailable": true};

    // old elastic annotations had index specified on them
    if (annotation.index) {
      header.index = annotation.index;
    } else {
      header.index = this.indexPattern.getIndexList(options.range.from, options.range.to);
    }

    var payload = angular.toJson(header) + '\n' + angular.toJson(data) + '\n';

    return this.post('_msearch', payload).then(res => {
      var list = [];
      var hits = res.responses[0].hits.hits;

      var getFieldFromSource = function(source, fieldName) {
        if (!fieldName) { return; }

        var fieldNames = fieldName.split('.');
        var fieldValue = source;

        for (var i = 0; i < fieldNames.length; i++) {
          fieldValue = fieldValue[fieldNames[i]];
          if (!fieldValue) {
            console.log('could not find field in annotation: ', fieldName);
            return '';
          }
        }

        return fieldValue;
      };

      for (var i = 0; i < hits.length; i++) {
        var source = hits[i]._source;
        var time = source[timeField];
        if (typeof hits[i].fields !== 'undefined') {
          var fields = hits[i].fields;
          if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
            time = fields[timeField];
          }
        }

        var event = {
          annotation: annotation,
          time: moment.utc(time).valueOf(),
          text: getFieldFromSource(source, textField),
          tags: getFieldFromSource(source, tagsField),
        };

        // legacy support for title tield
        if (annotation.titleField) {
          const title = getFieldFromSource(source, annotation.titleField);
          if (title) {
            event.text = title + '\n' + event.text;
          }
        }

        if (typeof event.tags === 'string') {
          event.tags = event.tags.split(',');
        }

        list.push(event);
      }
      return list;
    });
  }

  testDatasource() {
    this.timeSrv.setTime({ from: 'now-1m', to: 'now' }, true);
    // validate that the index exist and has date field
    return this.getFields({type: 'date'}).then(function(dateFields) {
      var timeField = _.find(dateFields, {text: this.timeField});
      if (!timeField) {
        return { status: "error", message: "No date field named " + this.timeField + ' found' };
      }
      return { status: "success", message: "Index OK. Time field name OK." };
    }.bind(this), function(err) {
      console.log(err);
      if (err.data && err.data.error) {
        var message = angular.toJson(err.data.error);
        if (err.data.error.reason) {
          message = err.data.error.reason;
        }
        return { status: "error", message: message };
      } else {
        return { status: "error", message: err.status };
      }
    });
  }

  getQueryHeader(searchType, timeFrom, timeTo) {
    var query_header: any = {
            search_type: searchType,
            "ignore_unavailable": true,
            index: this.indexPattern.getIndexList(timeFrom, timeTo),
        };
    if (this.esVersion >= 56) {
        query_header["max_concurrent_shard_requests"] = this.maxConcurrentShardRequests;
    }
    return angular.toJson(query_header);
  }

  query(options) {
    var payload = "";
    var target;
    var sentTargets = [];

    // add global adhoc filters to timeFilter
    var adhocFilters = this.templateSrv.getAdhocFilters(this.name);

    for (var i = 0; i < options.targets.length; i++) {
      target = options.targets[i];
      if (target.hide) {continue;}

      var queryString = this.templateSrv.replace(target.query || '*', options.scopedVars, 'lucene');
      var queryObj = this.queryBuilder.build(target, adhocFilters, queryString);
      var esQuery = angular.toJson(queryObj);

      var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch';
      var header = this.getQueryHeader(searchType, options.range.from, options.range.to);
      payload +=  header + '\n';

      payload += esQuery + '\n';
      sentTargets.push(target);
    }

    if (sentTargets.length === 0) {
      return this.$q.when([]);
    }

    payload = payload.replace(/\$timeFrom/g, options.range.from.valueOf());
    payload = payload.replace(/\$timeTo/g, options.range.to.valueOf());
    payload = this.templateSrv.replace(payload, options.scopedVars);

    return this.post('_msearch', payload).then(function(res) {
      return new ElasticResponse(sentTargets, res).getTimeSeries();
    });
  }

  getFields(query) {
    return this.get('/_mapping').then(function(result) {

      var typeMap = {
        'float': 'number',
        'double': 'number',
        'integer': 'number',
        'long': 'number',
        'date': 'date',
        'string': 'string',
        'text': 'string',
        'scaled_float': 'number',
        'nested': 'nested'
      };

      function shouldAddField(obj, key, query) {
        if (key[0] === '_') {
          return false;
        }

        if (!query.type) {
          return true;
        }

        // equal query type filter, or via typemap translation
        return query.type === obj.type || query.type === typeMap[obj.type];
      }

      // Store subfield names: [system, process, cpu, total] -> system.process.cpu.total
      var fieldNameParts = [];
      var fields = {};

      function getFieldsRecursively(obj) {
        for (var key in obj) {
          var subObj = obj[key];

          // Check mapping field for nested fields
          if (_.isObject(subObj.properties)) {
            fieldNameParts.push(key);
            getFieldsRecursively(subObj.properties);
          }

          if (_.isObject(subObj.fields)) {
            fieldNameParts.push(key);
            getFieldsRecursively(subObj.fields);
          }

          if (_.isString(subObj.type)) {
            var fieldName = fieldNameParts.concat(key).join('.');

            // Hide meta-fields and check field type
            if (shouldAddField(subObj, key, query)) {
              fields[fieldName] = {
                text: fieldName,
                type: subObj.type
              };
            }
          }
        }
        fieldNameParts.pop();
      }

      for (var indexName in result) {
        var index = result[indexName];
        if (index && index.mappings) {
          var mappings = index.mappings;
          for (var typeName in mappings) {
            var properties = mappings[typeName].properties;
            getFieldsRecursively(properties);
          }
        }
      }

      // transform to array
      return _.map(fields, function(value) {
        return value;
      });
    });
  }

  getTerms(queryDef) {
    var range = this.timeSrv.timeRange();
    var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ;
    var header = this.getQueryHeader(searchType, range.from, range.to);
    var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));

    esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf());
    esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
    esQuery = header + '\n' + esQuery + '\n';

    return this.post('_msearch?search_type=' + searchType, esQuery).then(function(res) {
      if (!res.responses[0].aggregations) {
        return [];
      }

      var buckets = res.responses[0].aggregations["1"].buckets;
      return _.map(buckets, function(bucket) {
        return {
          text: bucket.key_as_string || bucket.key,
          value: bucket.key
        };
      });
    });
  }

  metricFindQuery(query) {
    query = angular.fromJson(query);
    if (!query) {
      return this.$q.when([]);
    }

    if (query.find === 'fields') {
      query.field = this.templateSrv.replace(query.field, {}, 'lucene');
      return this.getFields(query);
    }

    if (query.find === 'terms') {
      query.query = this.templateSrv.replace(query.query || '*', {}, 'lucene');
      return this.getTerms(query);
    }
  }

  getTagKeys() {
    return this.getFields({});
  }

  getTagValues(options) {
    return this.getTerms({field: options.key, query: '*'});
  }
}