From 76b517b361235e9478eb84d994be74e0761b5a49 Mon Sep 17 00:00:00 2001 From: William Wei Date: Fri, 17 Apr 2015 15:51:26 +0800 Subject: [PATCH 001/287] allow graphite metrics name contain '~' when a metrics name contains '~', id does not impact graph display. but you can not use grafana UI to edit metrics with realtime graphite query. --- public/app/plugins/datasource/graphite/lexer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/plugins/datasource/graphite/lexer.js b/public/app/plugins/datasource/graphite/lexer.js index 7306737d96e..ffb65121a60 100644 --- a/public/app/plugins/datasource/graphite/lexer.js +++ b/public/app/plugins/datasource/graphite/lexer.js @@ -119,6 +119,7 @@ define([ identifierStartTable[i] = i >= 48 && i <= 57 || // 0-9 i === 36 || // $ + i === 126 || // ~ i >= 65 && i <= 90 || // A-Z i === 95 || // _ i === 45 || // - From ea993b64046ce46198580aba27a313e1f617bc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 15 May 2015 09:37:16 +0200 Subject: [PATCH 002/287] Some inital ldap work --- pkg/api/ldap/ldap.go | 0 pkg/components/ldap/LICENSE | 27 ++ pkg/components/ldap/README | 33 ++ pkg/components/ldap/_examples/enterprise.ldif | 63 ++++ pkg/components/ldap/_examples/modify.go | 89 +++++ pkg/components/ldap/_examples/search.go | 52 +++ pkg/components/ldap/_examples/searchSSL.go | 45 +++ pkg/components/ldap/_examples/searchTLS.go | 45 +++ pkg/components/ldap/_examples/slapd.conf | 67 ++++ pkg/components/ldap/bind.go | 55 +++ pkg/components/ldap/conn.go | 275 ++++++++++++++ pkg/components/ldap/control.go | 157 ++++++++ pkg/components/ldap/debug.go | 24 ++ pkg/components/ldap/filter.go | 248 +++++++++++++ pkg/components/ldap/filter_test.go | 78 ++++ pkg/components/ldap/ldap.go | 302 +++++++++++++++ pkg/components/ldap/ldap_test.go | 123 ++++++ pkg/components/ldap/modify.go | 156 ++++++++ pkg/components/ldap/search.go | 350 ++++++++++++++++++ 19 files changed, 2189 insertions(+) create mode 100644 pkg/api/ldap/ldap.go create mode 100644 pkg/components/ldap/LICENSE create mode 100644 pkg/components/ldap/README create mode 100644 pkg/components/ldap/_examples/enterprise.ldif create mode 100644 pkg/components/ldap/_examples/modify.go create mode 100644 pkg/components/ldap/_examples/search.go create mode 100644 pkg/components/ldap/_examples/searchSSL.go create mode 100644 pkg/components/ldap/_examples/searchTLS.go create mode 100644 pkg/components/ldap/_examples/slapd.conf create mode 100644 pkg/components/ldap/bind.go create mode 100644 pkg/components/ldap/conn.go create mode 100644 pkg/components/ldap/control.go create mode 100644 pkg/components/ldap/debug.go create mode 100644 pkg/components/ldap/filter.go create mode 100644 pkg/components/ldap/filter_test.go create mode 100644 pkg/components/ldap/ldap.go create mode 100644 pkg/components/ldap/ldap_test.go create mode 100644 pkg/components/ldap/modify.go create mode 100644 pkg/components/ldap/search.go diff --git a/pkg/api/ldap/ldap.go b/pkg/api/ldap/ldap.go new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/components/ldap/LICENSE b/pkg/components/ldap/LICENSE new file mode 100644 index 00000000000..74487567632 --- /dev/null +++ b/pkg/components/ldap/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkg/components/ldap/README b/pkg/components/ldap/README new file mode 100644 index 00000000000..edb54de0ac5 --- /dev/null +++ b/pkg/components/ldap/README @@ -0,0 +1,33 @@ +Basic LDAP v3 functionality for the GO programming language. + +Required Librarys: + github.com/johnweldon/asn1-ber + +Working: + Connecting to LDAP server + Binding to LDAP server + Searching for entries + Compiling string filters to LDAP filters + Paging Search Results + Modify Requests / Responses + +Examples: + search + modify + +Tests Implemented: + Filter Compile / Decompile + +TODO: + Add Requests / Responses + Delete Requests / Responses + Modify DN Requests / Responses + Compare Requests / Responses + Implement Tests / Benchmarks + +This feature is disabled at the moment, because in some cases the "Search Request Done" packet will be handled before the last "Search Request Entry": + Mulitple internal goroutines to handle network traffic + Makes library goroutine safe + Can perform multiple search requests at the same time and return + the results to the proper goroutine. All requests are blocking + requests, so the goroutine does not need special handling diff --git a/pkg/components/ldap/_examples/enterprise.ldif b/pkg/components/ldap/_examples/enterprise.ldif new file mode 100644 index 00000000000..f0ec28f16be --- /dev/null +++ b/pkg/components/ldap/_examples/enterprise.ldif @@ -0,0 +1,63 @@ +dn: dc=enterprise,dc=org +objectClass: dcObject +objectClass: organization +o: acme + +dn: cn=admin,dc=enterprise,dc=org +objectClass: person +cn: admin +sn: admin +description: "LDAP Admin" + +dn: ou=crew,dc=enterprise,dc=org +ou: crew +objectClass: organizationalUnit + + +dn: cn=kirkj,ou=crew,dc=enterprise,dc=org +cn: kirkj +sn: Kirk +gn: James Tiberius +mail: james.kirk@enterprise.org +objectClass: inetOrgPerson + +dn: cn=spock,ou=crew,dc=enterprise,dc=org +cn: spock +sn: Spock +mail: spock@enterprise.org +objectClass: inetOrgPerson + +dn: cn=mccoyl,ou=crew,dc=enterprise,dc=org +cn: mccoyl +sn: McCoy +gn: Leonard +mail: leonard.mccoy@enterprise.org +objectClass: inetOrgPerson + +dn: cn=scottm,ou=crew,dc=enterprise,dc=org +cn: scottm +sn: Scott +gn: Montgomery +mail: Montgomery.scott@enterprise.org +objectClass: inetOrgPerson + +dn: cn=uhuran,ou=crew,dc=enterprise,dc=org +cn: uhuran +sn: Uhura +gn: Nyota +mail: nyota.uhura@enterprise.org +objectClass: inetOrgPerson + +dn: cn=suluh,ou=crew,dc=enterprise,dc=org +cn: suluh +sn: Sulu +gn: Hikaru +mail: hikaru.sulu@enterprise.org +objectClass: inetOrgPerson + +dn: cn=chekovp,ou=crew,dc=enterprise,dc=org +cn: chekovp +sn: Chekov +gn: pavel +mail: pavel.chekov@enterprise.org +objectClass: inetOrgPerson diff --git a/pkg/components/ldap/_examples/modify.go b/pkg/components/ldap/_examples/modify.go new file mode 100644 index 00000000000..cd6dfc9eb71 --- /dev/null +++ b/pkg/components/ldap/_examples/modify.go @@ -0,0 +1,89 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "errors" + "fmt" + "log" + + "github.com/gogits/gogs/modules/ldap" +) + +var ( + LdapServer string = "localhost" + LdapPort uint16 = 389 + BaseDN string = "dc=enterprise,dc=org" + BindDN string = "cn=admin,dc=enterprise,dc=org" + BindPW string = "enterprise" + Filter string = "(cn=kirkj)" +) + +func search(l *ldap.Conn, filter string, attributes []string) (*ldap.Entry, *ldap.Error) { + search := ldap.NewSearchRequest( + BaseDN, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + filter, + attributes, + nil) + + sr, err := l.Search(search) + if err != nil { + log.Fatalf("ERROR: %s\n", err) + return nil, err + } + + log.Printf("Search: %s -> num of entries = %d\n", search.Filter, len(sr.Entries)) + if len(sr.Entries) == 0 { + return nil, ldap.NewError(ldap.ErrorDebugging, errors.New(fmt.Sprintf("no entries found for: %s", filter))) + } + return sr.Entries[0], nil +} + +func main() { + l, err := ldap.Dial("tcp", fmt.Sprintf("%s:%d", LdapServer, LdapPort)) + if err != nil { + log.Fatalf("ERROR: %s\n", err.Error()) + } + defer l.Close() + // l.Debug = true + + l.Bind(BindDN, BindPW) + + log.Printf("The Search for Kirk ... %s\n", Filter) + entry, err := search(l, Filter, []string{}) + if err != nil { + log.Fatal("could not get entry") + } + entry.PrettyPrint(0) + + log.Printf("modify the mail address and add a description ... \n") + modify := ldap.NewModifyRequest(entry.DN) + modify.Add("description", []string{"Captain of the USS Enterprise"}) + modify.Replace("mail", []string{"captain@enterprise.org"}) + if err := l.Modify(modify); err != nil { + log.Fatalf("ERROR: %s\n", err.Error()) + } + + entry, err = search(l, Filter, []string{}) + if err != nil { + log.Fatal("could not get entry") + } + entry.PrettyPrint(0) + + log.Printf("reset the entry ... \n") + modify = ldap.NewModifyRequest(entry.DN) + modify.Delete("description", []string{}) + modify.Replace("mail", []string{"james.kirk@enterprise.org"}) + if err := l.Modify(modify); err != nil { + log.Fatalf("ERROR: %s\n", err.Error()) + } + + entry, err = search(l, Filter, []string{}) + if err != nil { + log.Fatal("could not get entry") + } + entry.PrettyPrint(0) +} diff --git a/pkg/components/ldap/_examples/search.go b/pkg/components/ldap/_examples/search.go new file mode 100644 index 00000000000..609256f4d3c --- /dev/null +++ b/pkg/components/ldap/_examples/search.go @@ -0,0 +1,52 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "log" + + "github.com/gogits/gogs/modules/ldap" +) + +var ( + ldapServer string = "adserver" + ldapPort uint16 = 3268 + baseDN string = "dc=*,dc=*" + filter string = "(&(objectClass=user)(sAMAccountName=*)(memberOf=CN=*,OU=*,DC=*,DC=*))" + Attributes []string = []string{"memberof"} + user string = "*" + passwd string = "*" +) + +func main() { + l, err := ldap.Dial("tcp", fmt.Sprintf("%s:%d", ldapServer, ldapPort)) + if err != nil { + log.Fatalf("ERROR: %s\n", err.Error()) + } + defer l.Close() + // l.Debug = true + + err = l.Bind(user, passwd) + if err != nil { + log.Printf("ERROR: Cannot bind: %s\n", err.Error()) + return + } + search := ldap.NewSearchRequest( + baseDN, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + filter, + Attributes, + nil) + + sr, err := l.Search(search) + if err != nil { + log.Fatalf("ERROR: %s\n", err.Error()) + return + } + + log.Printf("Search: %s -> num of entries = %d\n", search.Filter, len(sr.Entries)) + sr.PrettyPrint(0) +} diff --git a/pkg/components/ldap/_examples/searchSSL.go b/pkg/components/ldap/_examples/searchSSL.go new file mode 100644 index 00000000000..aa9cbcc1249 --- /dev/null +++ b/pkg/components/ldap/_examples/searchSSL.go @@ -0,0 +1,45 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "log" + + "github.com/gogits/gogs/modules/ldap" +) + +var ( + LdapServer string = "localhost" + LdapPort uint16 = 636 + BaseDN string = "dc=enterprise,dc=org" + Filter string = "(cn=kirkj)" + Attributes []string = []string{"mail"} +) + +func main() { + l, err := ldap.DialSSL("tcp", fmt.Sprintf("%s:%d", LdapServer, LdapPort), nil) + if err != nil { + log.Fatalf("ERROR: %s\n", err.String()) + } + defer l.Close() + // l.Debug = true + + search := ldap.NewSearchRequest( + BaseDN, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + Filter, + Attributes, + nil) + + sr, err := l.Search(search) + if err != nil { + log.Fatalf("ERROR: %s\n", err.String()) + return + } + + log.Printf("Search: %s -> num of entries = %d\n", search.Filter, len(sr.Entries)) + sr.PrettyPrint(0) +} diff --git a/pkg/components/ldap/_examples/searchTLS.go b/pkg/components/ldap/_examples/searchTLS.go new file mode 100644 index 00000000000..c771a8eda87 --- /dev/null +++ b/pkg/components/ldap/_examples/searchTLS.go @@ -0,0 +1,45 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "log" + + "github.com/gogits/gogs/modules/ldap" +) + +var ( + LdapServer string = "localhost" + LdapPort uint16 = 389 + BaseDN string = "dc=enterprise,dc=org" + Filter string = "(cn=kirkj)" + Attributes []string = []string{"mail"} +) + +func main() { + l, err := ldap.DialTLS("tcp", fmt.Sprintf("%s:%d", LdapServer, LdapPort), nil) + if err != nil { + log.Fatalf("ERROR: %s\n", err.Error()) + } + defer l.Close() + // l.Debug = true + + search := ldap.NewSearchRequest( + BaseDN, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + Filter, + Attributes, + nil) + + sr, err := l.Search(search) + if err != nil { + log.Fatalf("ERROR: %s\n", err.Error()) + return + } + + log.Printf("Search: %s -> num of entries = %d\n", search.Filter, len(sr.Entries)) + sr.PrettyPrint(0) +} diff --git a/pkg/components/ldap/_examples/slapd.conf b/pkg/components/ldap/_examples/slapd.conf new file mode 100644 index 00000000000..5a66be0152d --- /dev/null +++ b/pkg/components/ldap/_examples/slapd.conf @@ -0,0 +1,67 @@ +# +# See slapd.conf(5) for details on configuration options. +# This file should NOT be world readable. +# +include /private/etc/openldap/schema/core.schema +include /private/etc/openldap/schema/cosine.schema +include /private/etc/openldap/schema/inetorgperson.schema + +# Define global ACLs to disable default read access. + +# Do not enable referrals until AFTER you have a working directory +# service AND an understanding of referrals. +#referral ldap://root.openldap.org + +pidfile /private/var/db/openldap/run/slapd.pid +argsfile /private/var/db/openldap/run/slapd.args + +# Load dynamic backend modules: +# modulepath /usr/libexec/openldap +# moduleload back_bdb.la +# moduleload back_hdb.la +# moduleload back_ldap.la + +# Sample security restrictions +# Require integrity protection (prevent hijacking) +# Require 112-bit (3DES or better) encryption for updates +# Require 63-bit encryption for simple bind +# security ssf=1 update_ssf=112 simple_bind=64 + +# Sample access control policy: +# Root DSE: allow anyone to read it +# Subschema (sub)entry DSE: allow anyone to read it +# Other DSEs: +# Allow self write access +# Allow authenticated users read access +# Allow anonymous users to authenticate +# Directives needed to implement policy: +# access to dn.base="" by * read +# access to dn.base="cn=Subschema" by * read +# access to * +# by self write +# by users read +# by anonymous auth +# +# if no access controls are present, the default policy +# allows anyone and everyone to read anything but restricts +# updates to rootdn. (e.g., "access to * by * read") +# +# rootdn can always read and write EVERYTHING! + +####################################################################### +# BDB database definitions +####################################################################### + +database bdb +suffix "dc=enterprise,dc=org" +rootdn "cn=admin,dc=enterprise,dc=org" +# Cleartext passwords, especially for the rootdn, should +# be avoid. See slappasswd(8) and slapd.conf(5) for details. +# Use of strong authentication encouraged. +rootpw {SSHA}laO00HsgszhK1O0Z5qR0/i/US69Osfeu +# The database directory MUST exist prior to running slapd AND +# should only be accessible by the slapd and slap tools. +# Mode 700 recommended. +directory /private/var/db/openldap/openldap-data +# Indices to maintain +index objectClass eq diff --git a/pkg/components/ldap/bind.go b/pkg/components/ldap/bind.go new file mode 100644 index 00000000000..0561e611d1d --- /dev/null +++ b/pkg/components/ldap/bind.go @@ -0,0 +1,55 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ldap + +import ( + "errors" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +func (l *Conn) Bind(username, password string) error { + messageID := l.nextMessageID() + + packet := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "LDAP Request") + packet.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagInteger, messageID, "MessageID")) + bindRequest := ber.Encode(ber.ClassApplication, ber.TypeConstructed, ApplicationBindRequest, nil, "Bind Request") + bindRequest.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagInteger, 3, "Version")) + bindRequest.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, username, "User Name")) + bindRequest.AppendChild(ber.NewString(ber.ClassContext, ber.TypePrimitive, 0, password, "Password")) + packet.AppendChild(bindRequest) + + if l.Debug { + ber.PrintPacket(packet) + } + + channel, err := l.sendMessage(packet) + if err != nil { + return err + } + if channel == nil { + return NewError(ErrorNetwork, errors.New("ldap: could not send message")) + } + defer l.finishMessage(messageID) + + packet = <-channel + if packet == nil { + return NewError(ErrorNetwork, errors.New("ldap: could not retrieve response")) + } + + if l.Debug { + if err := addLDAPDescriptions(packet); err != nil { + return err + } + ber.PrintPacket(packet) + } + + resultCode, resultDescription := getLDAPResultCode(packet) + if resultCode != 0 { + return NewError(resultCode, errors.New(resultDescription)) + } + + return nil +} diff --git a/pkg/components/ldap/conn.go b/pkg/components/ldap/conn.go new file mode 100644 index 00000000000..6a244f1253b --- /dev/null +++ b/pkg/components/ldap/conn.go @@ -0,0 +1,275 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ldap + +import ( + "crypto/tls" + "errors" + "log" + "net" + "sync" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +const ( + MessageQuit = 0 + MessageRequest = 1 + MessageResponse = 2 + MessageFinish = 3 +) + +type messagePacket struct { + Op int + MessageID uint64 + Packet *ber.Packet + Channel chan *ber.Packet +} + +// Conn represents an LDAP Connection +type Conn struct { + conn net.Conn + isTLS bool + isClosing bool + Debug debugging + chanConfirm chan bool + chanResults map[uint64]chan *ber.Packet + chanMessage chan *messagePacket + chanMessageID chan uint64 + wgSender sync.WaitGroup + wgClose sync.WaitGroup + once sync.Once +} + +// Dial connects to the given address on the given network using net.Dial +// and then returns a new Conn for the connection. +func Dial(network, addr string) (*Conn, error) { + c, err := net.Dial(network, addr) + if err != nil { + return nil, NewError(ErrorNetwork, err) + } + conn := NewConn(c) + conn.start() + return conn, nil +} + +// DialTLS connects to the given address on the given network using tls.Dial +// and then returns a new Conn for the connection. +func DialTLS(network, addr string, config *tls.Config) (*Conn, error) { + c, err := tls.Dial(network, addr, config) + if err != nil { + return nil, NewError(ErrorNetwork, err) + } + conn := NewConn(c) + conn.isTLS = true + conn.start() + return conn, nil +} + +// NewConn returns a new Conn using conn for network I/O. +func NewConn(conn net.Conn) *Conn { + return &Conn{ + conn: conn, + chanConfirm: make(chan bool), + chanMessageID: make(chan uint64), + chanMessage: make(chan *messagePacket, 10), + chanResults: map[uint64]chan *ber.Packet{}, + } +} + +func (l *Conn) start() { + go l.reader() + go l.processMessages() + l.wgClose.Add(1) +} + +// Close closes the connection. +func (l *Conn) Close() { + l.once.Do(func() { + l.isClosing = true + l.wgSender.Wait() + + l.Debug.Printf("Sending quit message and waiting for confirmation") + l.chanMessage <- &messagePacket{Op: MessageQuit} + <-l.chanConfirm + close(l.chanMessage) + + l.Debug.Printf("Closing network connection") + if err := l.conn.Close(); err != nil { + log.Print(err) + } + + l.conn = nil + l.wgClose.Done() + }) + l.wgClose.Wait() +} + +// Returns the next available messageID +func (l *Conn) nextMessageID() uint64 { + if l.chanMessageID != nil { + if messageID, ok := <-l.chanMessageID; ok { + return messageID + } + } + return 0 +} + +// StartTLS sends the command to start a TLS session and then creates a new TLS Client +func (l *Conn) StartTLS(config *tls.Config) error { + messageID := l.nextMessageID() + + if l.isTLS { + return NewError(ErrorNetwork, errors.New("ldap: already encrypted")) + } + + packet := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "LDAP Request") + packet.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagInteger, messageID, "MessageID")) + request := ber.Encode(ber.ClassApplication, ber.TypeConstructed, ApplicationExtendedRequest, nil, "Start TLS") + request.AppendChild(ber.NewString(ber.ClassContext, ber.TypePrimitive, 0, "1.3.6.1.4.1.1466.20037", "TLS Extended Command")) + packet.AppendChild(request) + l.Debug.PrintPacket(packet) + + _, err := l.conn.Write(packet.Bytes()) + if err != nil { + return NewError(ErrorNetwork, err) + } + + packet, err = ber.ReadPacket(l.conn) + if err != nil { + return NewError(ErrorNetwork, err) + } + + if l.Debug { + if err := addLDAPDescriptions(packet); err != nil { + return err + } + ber.PrintPacket(packet) + } + + if packet.Children[1].Children[0].Value.(uint64) == 0 { + conn := tls.Client(l.conn, config) + l.isTLS = true + l.conn = conn + } + + return nil +} + +func (l *Conn) sendMessage(packet *ber.Packet) (chan *ber.Packet, error) { + if l.isClosing { + return nil, NewError(ErrorNetwork, errors.New("ldap: connection closed")) + } + out := make(chan *ber.Packet) + message := &messagePacket{ + Op: MessageRequest, + MessageID: packet.Children[0].Value.(uint64), + Packet: packet, + Channel: out, + } + l.sendProcessMessage(message) + return out, nil +} + +func (l *Conn) finishMessage(messageID uint64) { + if l.isClosing { + return + } + message := &messagePacket{ + Op: MessageFinish, + MessageID: messageID, + } + l.sendProcessMessage(message) +} + +func (l *Conn) sendProcessMessage(message *messagePacket) bool { + if l.isClosing { + return false + } + l.wgSender.Add(1) + l.chanMessage <- message + l.wgSender.Done() + return true +} + +func (l *Conn) processMessages() { + defer func() { + for messageID, channel := range l.chanResults { + l.Debug.Printf("Closing channel for MessageID %d", messageID) + close(channel) + delete(l.chanResults, messageID) + } + close(l.chanMessageID) + l.chanConfirm <- true + close(l.chanConfirm) + }() + + var messageID uint64 = 1 + for { + select { + case l.chanMessageID <- messageID: + messageID++ + case messagePacket, ok := <-l.chanMessage: + if !ok { + l.Debug.Printf("Shutting down - message channel is closed") + return + } + switch messagePacket.Op { + case MessageQuit: + l.Debug.Printf("Shutting down - quit message received") + return + case MessageRequest: + // Add to message list and write to network + l.Debug.Printf("Sending message %d", messagePacket.MessageID) + l.chanResults[messagePacket.MessageID] = messagePacket.Channel + // go routine + buf := messagePacket.Packet.Bytes() + + _, err := l.conn.Write(buf) + if err != nil { + l.Debug.Printf("Error Sending Message: %s", err.Error()) + break + } + case MessageResponse: + l.Debug.Printf("Receiving message %d", messagePacket.MessageID) + if chanResult, ok := l.chanResults[messagePacket.MessageID]; ok { + chanResult <- messagePacket.Packet + } else { + log.Printf("Received unexpected message %d", messagePacket.MessageID) + ber.PrintPacket(messagePacket.Packet) + } + case MessageFinish: + // Remove from message list + l.Debug.Printf("Finished message %d", messagePacket.MessageID) + close(l.chanResults[messagePacket.MessageID]) + delete(l.chanResults, messagePacket.MessageID) + } + } + } +} + +func (l *Conn) reader() { + defer func() { + l.Close() + }() + + for { + packet, err := ber.ReadPacket(l.conn) + if err != nil { + l.Debug.Printf("reader: %s", err.Error()) + return + } + addLDAPDescriptions(packet) + message := &messagePacket{ + Op: MessageResponse, + MessageID: packet.Children[0].Value.(uint64), + Packet: packet, + } + if !l.sendProcessMessage(message) { + return + } + + } +} diff --git a/pkg/components/ldap/control.go b/pkg/components/ldap/control.go new file mode 100644 index 00000000000..4b15f1bd4a8 --- /dev/null +++ b/pkg/components/ldap/control.go @@ -0,0 +1,157 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ldap + +import ( + "fmt" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +const ( + ControlTypePaging = "1.2.840.113556.1.4.319" +) + +var ControlTypeMap = map[string]string{ + ControlTypePaging: "Paging", +} + +type Control interface { + GetControlType() string + Encode() *ber.Packet + String() string +} + +type ControlString struct { + ControlType string + Criticality bool + ControlValue string +} + +func (c *ControlString) GetControlType() string { + return c.ControlType +} + +func (c *ControlString) Encode() *ber.Packet { + packet := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Control") + packet.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, c.ControlType, "Control Type ("+ControlTypeMap[c.ControlType]+")")) + if c.Criticality { + packet.AppendChild(ber.NewBoolean(ber.ClassUniversal, ber.TypePrimitive, ber.TagBoolean, c.Criticality, "Criticality")) + } + packet.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, c.ControlValue, "Control Value")) + return packet +} + +func (c *ControlString) String() string { + return fmt.Sprintf("Control Type: %s (%q) Criticality: %t Control Value: %s", ControlTypeMap[c.ControlType], c.ControlType, c.Criticality, c.ControlValue) +} + +type ControlPaging struct { + PagingSize uint32 + Cookie []byte +} + +func (c *ControlPaging) GetControlType() string { + return ControlTypePaging +} + +func (c *ControlPaging) Encode() *ber.Packet { + packet := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Control") + packet.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, ControlTypePaging, "Control Type ("+ControlTypeMap[ControlTypePaging]+")")) + + p2 := ber.Encode(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, nil, "Control Value (Paging)") + seq := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Search Control Value") + seq.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagInteger, uint64(c.PagingSize), "Paging Size")) + cookie := ber.Encode(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, nil, "Cookie") + cookie.Value = c.Cookie + cookie.Data.Write(c.Cookie) + seq.AppendChild(cookie) + p2.AppendChild(seq) + + packet.AppendChild(p2) + return packet +} + +func (c *ControlPaging) String() string { + return fmt.Sprintf( + "Control Type: %s (%q) Criticality: %t PagingSize: %d Cookie: %q", + ControlTypeMap[ControlTypePaging], + ControlTypePaging, + false, + c.PagingSize, + c.Cookie) +} + +func (c *ControlPaging) SetCookie(cookie []byte) { + c.Cookie = cookie +} + +func FindControl(controls []Control, controlType string) Control { + for _, c := range controls { + if c.GetControlType() == controlType { + return c + } + } + return nil +} + +func DecodeControl(packet *ber.Packet) Control { + ControlType := packet.Children[0].Value.(string) + Criticality := false + + packet.Children[0].Description = "Control Type (" + ControlTypeMap[ControlType] + ")" + value := packet.Children[1] + if len(packet.Children) == 3 { + value = packet.Children[2] + packet.Children[1].Description = "Criticality" + Criticality = packet.Children[1].Value.(bool) + } + + value.Description = "Control Value" + switch ControlType { + case ControlTypePaging: + value.Description += " (Paging)" + c := new(ControlPaging) + if value.Value != nil { + valueChildren := ber.DecodePacket(value.Data.Bytes()) + value.Data.Truncate(0) + value.Value = nil + value.AppendChild(valueChildren) + } + value = value.Children[0] + value.Description = "Search Control Value" + value.Children[0].Description = "Paging Size" + value.Children[1].Description = "Cookie" + c.PagingSize = uint32(value.Children[0].Value.(uint64)) + c.Cookie = value.Children[1].Data.Bytes() + value.Children[1].Value = c.Cookie + return c + } + c := new(ControlString) + c.ControlType = ControlType + c.Criticality = Criticality + c.ControlValue = value.Value.(string) + return c +} + +func NewControlString(controlType string, criticality bool, controlValue string) *ControlString { + return &ControlString{ + ControlType: controlType, + Criticality: criticality, + ControlValue: controlValue, + } +} + +func NewControlPaging(pagingSize uint32) *ControlPaging { + return &ControlPaging{PagingSize: pagingSize} +} + +func encodeControls(controls []Control) *ber.Packet { + packet := ber.Encode(ber.ClassContext, ber.TypeConstructed, 0, nil, "Controls") + for _, control := range controls { + packet.AppendChild(control.Encode()) + } + return packet +} diff --git a/pkg/components/ldap/debug.go b/pkg/components/ldap/debug.go new file mode 100644 index 00000000000..67856fe7a60 --- /dev/null +++ b/pkg/components/ldap/debug.go @@ -0,0 +1,24 @@ +package ldap + +import ( + "log" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +// debugging type +// - has a Printf method to write the debug output +type debugging bool + +// write debug output +func (debug debugging) Printf(format string, args ...interface{}) { + if debug { + log.Printf(format, args...) + } +} + +func (debug debugging) PrintPacket(packet *ber.Packet) { + if debug { + ber.PrintPacket(packet) + } +} diff --git a/pkg/components/ldap/filter.go b/pkg/components/ldap/filter.go new file mode 100644 index 00000000000..0ad7a403bca --- /dev/null +++ b/pkg/components/ldap/filter.go @@ -0,0 +1,248 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ldap + +import ( + "errors" + "fmt" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +const ( + FilterAnd = 0 + FilterOr = 1 + FilterNot = 2 + FilterEqualityMatch = 3 + FilterSubstrings = 4 + FilterGreaterOrEqual = 5 + FilterLessOrEqual = 6 + FilterPresent = 7 + FilterApproxMatch = 8 + FilterExtensibleMatch = 9 +) + +var FilterMap = map[uint64]string{ + FilterAnd: "And", + FilterOr: "Or", + FilterNot: "Not", + FilterEqualityMatch: "Equality Match", + FilterSubstrings: "Substrings", + FilterGreaterOrEqual: "Greater Or Equal", + FilterLessOrEqual: "Less Or Equal", + FilterPresent: "Present", + FilterApproxMatch: "Approx Match", + FilterExtensibleMatch: "Extensible Match", +} + +const ( + FilterSubstringsInitial = 0 + FilterSubstringsAny = 1 + FilterSubstringsFinal = 2 +) + +var FilterSubstringsMap = map[uint64]string{ + FilterSubstringsInitial: "Substrings Initial", + FilterSubstringsAny: "Substrings Any", + FilterSubstringsFinal: "Substrings Final", +} + +func CompileFilter(filter string) (*ber.Packet, error) { + if len(filter) == 0 || filter[0] != '(' { + return nil, NewError(ErrorFilterCompile, errors.New("ldap: filter does not start with an '('")) + } + packet, pos, err := compileFilter(filter, 1) + if err != nil { + return nil, err + } + if pos != len(filter) { + return nil, NewError(ErrorFilterCompile, errors.New("ldap: finished compiling filter with extra at end: "+fmt.Sprint(filter[pos:]))) + } + return packet, nil +} + +func DecompileFilter(packet *ber.Packet) (ret string, err error) { + defer func() { + if r := recover(); r != nil { + err = NewError(ErrorFilterDecompile, errors.New("ldap: error decompiling filter")) + } + }() + ret = "(" + err = nil + childStr := "" + + switch packet.Tag { + case FilterAnd: + ret += "&" + for _, child := range packet.Children { + childStr, err = DecompileFilter(child) + if err != nil { + return + } + ret += childStr + } + case FilterOr: + ret += "|" + for _, child := range packet.Children { + childStr, err = DecompileFilter(child) + if err != nil { + return + } + ret += childStr + } + case FilterNot: + ret += "!" + childStr, err = DecompileFilter(packet.Children[0]) + if err != nil { + return + } + ret += childStr + + case FilterSubstrings: + ret += ber.DecodeString(packet.Children[0].Data.Bytes()) + ret += "=" + switch packet.Children[1].Children[0].Tag { + case FilterSubstringsInitial: + ret += ber.DecodeString(packet.Children[1].Children[0].Data.Bytes()) + "*" + case FilterSubstringsAny: + ret += "*" + ber.DecodeString(packet.Children[1].Children[0].Data.Bytes()) + "*" + case FilterSubstringsFinal: + ret += "*" + ber.DecodeString(packet.Children[1].Children[0].Data.Bytes()) + } + case FilterEqualityMatch: + ret += ber.DecodeString(packet.Children[0].Data.Bytes()) + ret += "=" + ret += ber.DecodeString(packet.Children[1].Data.Bytes()) + case FilterGreaterOrEqual: + ret += ber.DecodeString(packet.Children[0].Data.Bytes()) + ret += ">=" + ret += ber.DecodeString(packet.Children[1].Data.Bytes()) + case FilterLessOrEqual: + ret += ber.DecodeString(packet.Children[0].Data.Bytes()) + ret += "<=" + ret += ber.DecodeString(packet.Children[1].Data.Bytes()) + case FilterPresent: + ret += ber.DecodeString(packet.Children[0].Data.Bytes()) + ret += "=*" + case FilterApproxMatch: + ret += ber.DecodeString(packet.Children[0].Data.Bytes()) + ret += "~=" + ret += ber.DecodeString(packet.Children[1].Data.Bytes()) + } + + ret += ")" + return +} + +func compileFilterSet(filter string, pos int, parent *ber.Packet) (int, error) { + for pos < len(filter) && filter[pos] == '(' { + child, newPos, err := compileFilter(filter, pos+1) + if err != nil { + return pos, err + } + pos = newPos + parent.AppendChild(child) + } + if pos == len(filter) { + return pos, NewError(ErrorFilterCompile, errors.New("ldap: unexpected end of filter")) + } + + return pos + 1, nil +} + +func compileFilter(filter string, pos int) (*ber.Packet, int, error) { + var packet *ber.Packet + var err error + + defer func() { + if r := recover(); r != nil { + err = NewError(ErrorFilterCompile, errors.New("ldap: error compiling filter")) + } + }() + + newPos := pos + switch filter[pos] { + case '(': + packet, newPos, err = compileFilter(filter, pos+1) + newPos++ + return packet, newPos, err + case '&': + packet = ber.Encode(ber.ClassContext, ber.TypeConstructed, FilterAnd, nil, FilterMap[FilterAnd]) + newPos, err = compileFilterSet(filter, pos+1, packet) + return packet, newPos, err + case '|': + packet = ber.Encode(ber.ClassContext, ber.TypeConstructed, FilterOr, nil, FilterMap[FilterOr]) + newPos, err = compileFilterSet(filter, pos+1, packet) + return packet, newPos, err + case '!': + packet = ber.Encode(ber.ClassContext, ber.TypeConstructed, FilterNot, nil, FilterMap[FilterNot]) + var child *ber.Packet + child, newPos, err = compileFilter(filter, pos+1) + packet.AppendChild(child) + return packet, newPos, err + default: + attribute := "" + condition := "" + for newPos < len(filter) && filter[newPos] != ')' { + switch { + case packet != nil: + condition += fmt.Sprintf("%c", filter[newPos]) + case filter[newPos] == '=': + packet = ber.Encode(ber.ClassContext, ber.TypeConstructed, FilterEqualityMatch, nil, FilterMap[FilterEqualityMatch]) + case filter[newPos] == '>' && filter[newPos+1] == '=': + packet = ber.Encode(ber.ClassContext, ber.TypeConstructed, FilterGreaterOrEqual, nil, FilterMap[FilterGreaterOrEqual]) + newPos++ + case filter[newPos] == '<' && filter[newPos+1] == '=': + packet = ber.Encode(ber.ClassContext, ber.TypeConstructed, FilterLessOrEqual, nil, FilterMap[FilterLessOrEqual]) + newPos++ + case filter[newPos] == '~' && filter[newPos+1] == '=': + packet = ber.Encode(ber.ClassContext, ber.TypeConstructed, FilterApproxMatch, nil, FilterMap[FilterLessOrEqual]) + newPos++ + case packet == nil: + attribute += fmt.Sprintf("%c", filter[newPos]) + } + newPos++ + } + if newPos == len(filter) { + err = NewError(ErrorFilterCompile, errors.New("ldap: unexpected end of filter")) + return packet, newPos, err + } + if packet == nil { + err = NewError(ErrorFilterCompile, errors.New("ldap: error parsing filter")) + return packet, newPos, err + } + packet.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, attribute, "Attribute")) + switch { + case packet.Tag == FilterEqualityMatch && condition == "*": + packet.Tag = FilterPresent + packet.Description = FilterMap[uint64(packet.Tag)] + case packet.Tag == FilterEqualityMatch && condition[0] == '*' && condition[len(condition)-1] == '*': + // Any + packet.Tag = FilterSubstrings + packet.Description = FilterMap[uint64(packet.Tag)] + seq := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Substrings") + seq.AppendChild(ber.NewString(ber.ClassContext, ber.TypePrimitive, FilterSubstringsAny, condition[1:len(condition)-1], "Any Substring")) + packet.AppendChild(seq) + case packet.Tag == FilterEqualityMatch && condition[0] == '*': + // Final + packet.Tag = FilterSubstrings + packet.Description = FilterMap[uint64(packet.Tag)] + seq := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Substrings") + seq.AppendChild(ber.NewString(ber.ClassContext, ber.TypePrimitive, FilterSubstringsFinal, condition[1:], "Final Substring")) + packet.AppendChild(seq) + case packet.Tag == FilterEqualityMatch && condition[len(condition)-1] == '*': + // Initial + packet.Tag = FilterSubstrings + packet.Description = FilterMap[uint64(packet.Tag)] + seq := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Substrings") + seq.AppendChild(ber.NewString(ber.ClassContext, ber.TypePrimitive, FilterSubstringsInitial, condition[:len(condition)-1], "Initial Substring")) + packet.AppendChild(seq) + default: + packet.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, condition, "Condition")) + } + newPos++ + return packet, newPos, err + } +} diff --git a/pkg/components/ldap/filter_test.go b/pkg/components/ldap/filter_test.go new file mode 100644 index 00000000000..761ff42fd51 --- /dev/null +++ b/pkg/components/ldap/filter_test.go @@ -0,0 +1,78 @@ +package ldap + +import ( + "testing" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +type compileTest struct { + filterStr string + filterType int +} + +var testFilters = []compileTest{ + compileTest{filterStr: "(&(sn=Miller)(givenName=Bob))", filterType: FilterAnd}, + compileTest{filterStr: "(|(sn=Miller)(givenName=Bob))", filterType: FilterOr}, + compileTest{filterStr: "(!(sn=Miller))", filterType: FilterNot}, + compileTest{filterStr: "(sn=Miller)", filterType: FilterEqualityMatch}, + compileTest{filterStr: "(sn=Mill*)", filterType: FilterSubstrings}, + compileTest{filterStr: "(sn=*Mill)", filterType: FilterSubstrings}, + compileTest{filterStr: "(sn=*Mill*)", filterType: FilterSubstrings}, + compileTest{filterStr: "(sn>=Miller)", filterType: FilterGreaterOrEqual}, + compileTest{filterStr: "(sn<=Miller)", filterType: FilterLessOrEqual}, + compileTest{filterStr: "(sn=*)", filterType: FilterPresent}, + compileTest{filterStr: "(sn~=Miller)", filterType: FilterApproxMatch}, + // compileTest{ filterStr: "()", filterType: FilterExtensibleMatch }, +} + +func TestFilter(t *testing.T) { + // Test Compiler and Decompiler + for _, i := range testFilters { + filter, err := CompileFilter(i.filterStr) + if err != nil { + t.Errorf("Problem compiling %s - %s", i.filterStr, err.Error()) + } else if filter.Tag != uint8(i.filterType) { + t.Errorf("%q Expected %q got %q", i.filterStr, FilterMap[uint64(i.filterType)], FilterMap[uint64(filter.Tag)]) + } else { + o, err := DecompileFilter(filter) + if err != nil { + t.Errorf("Problem compiling %s - %s", i.filterStr, err.Error()) + } else if i.filterStr != o { + t.Errorf("%q expected, got %q", i.filterStr, o) + } + } + } +} + +func BenchmarkFilterCompile(b *testing.B) { + b.StopTimer() + filters := make([]string, len(testFilters)) + + // Test Compiler and Decompiler + for idx, i := range testFilters { + filters[idx] = i.filterStr + } + + maxIdx := len(filters) + b.StartTimer() + for i := 0; i < b.N; i++ { + CompileFilter(filters[i%maxIdx]) + } +} + +func BenchmarkFilterDecompile(b *testing.B) { + b.StopTimer() + filters := make([]*ber.Packet, len(testFilters)) + + // Test Compiler and Decompiler + for idx, i := range testFilters { + filters[idx], _ = CompileFilter(i.filterStr) + } + + maxIdx := len(filters) + b.StartTimer() + for i := 0; i < b.N; i++ { + DecompileFilter(filters[i%maxIdx]) + } +} diff --git a/pkg/components/ldap/ldap.go b/pkg/components/ldap/ldap.go new file mode 100644 index 00000000000..e990b36231f --- /dev/null +++ b/pkg/components/ldap/ldap.go @@ -0,0 +1,302 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ldap + +import ( + "errors" + "fmt" + "io/ioutil" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +// LDAP Application Codes +const ( + ApplicationBindRequest = 0 + ApplicationBindResponse = 1 + ApplicationUnbindRequest = 2 + ApplicationSearchRequest = 3 + ApplicationSearchResultEntry = 4 + ApplicationSearchResultDone = 5 + ApplicationModifyRequest = 6 + ApplicationModifyResponse = 7 + ApplicationAddRequest = 8 + ApplicationAddResponse = 9 + ApplicationDelRequest = 10 + ApplicationDelResponse = 11 + ApplicationModifyDNRequest = 12 + ApplicationModifyDNResponse = 13 + ApplicationCompareRequest = 14 + ApplicationCompareResponse = 15 + ApplicationAbandonRequest = 16 + ApplicationSearchResultReference = 19 + ApplicationExtendedRequest = 23 + ApplicationExtendedResponse = 24 +) + +var ApplicationMap = map[uint8]string{ + ApplicationBindRequest: "Bind Request", + ApplicationBindResponse: "Bind Response", + ApplicationUnbindRequest: "Unbind Request", + ApplicationSearchRequest: "Search Request", + ApplicationSearchResultEntry: "Search Result Entry", + ApplicationSearchResultDone: "Search Result Done", + ApplicationModifyRequest: "Modify Request", + ApplicationModifyResponse: "Modify Response", + ApplicationAddRequest: "Add Request", + ApplicationAddResponse: "Add Response", + ApplicationDelRequest: "Del Request", + ApplicationDelResponse: "Del Response", + ApplicationModifyDNRequest: "Modify DN Request", + ApplicationModifyDNResponse: "Modify DN Response", + ApplicationCompareRequest: "Compare Request", + ApplicationCompareResponse: "Compare Response", + ApplicationAbandonRequest: "Abandon Request", + ApplicationSearchResultReference: "Search Result Reference", + ApplicationExtendedRequest: "Extended Request", + ApplicationExtendedResponse: "Extended Response", +} + +// LDAP Result Codes +const ( + LDAPResultSuccess = 0 + LDAPResultOperationsError = 1 + LDAPResultProtocolError = 2 + LDAPResultTimeLimitExceeded = 3 + LDAPResultSizeLimitExceeded = 4 + LDAPResultCompareFalse = 5 + LDAPResultCompareTrue = 6 + LDAPResultAuthMethodNotSupported = 7 + LDAPResultStrongAuthRequired = 8 + LDAPResultReferral = 10 + LDAPResultAdminLimitExceeded = 11 + LDAPResultUnavailableCriticalExtension = 12 + LDAPResultConfidentialityRequired = 13 + LDAPResultSaslBindInProgress = 14 + LDAPResultNoSuchAttribute = 16 + LDAPResultUndefinedAttributeType = 17 + LDAPResultInappropriateMatching = 18 + LDAPResultConstraintViolation = 19 + LDAPResultAttributeOrValueExists = 20 + LDAPResultInvalidAttributeSyntax = 21 + LDAPResultNoSuchObject = 32 + LDAPResultAliasProblem = 33 + LDAPResultInvalidDNSyntax = 34 + LDAPResultAliasDereferencingProblem = 36 + LDAPResultInappropriateAuthentication = 48 + LDAPResultInvalidCredentials = 49 + LDAPResultInsufficientAccessRights = 50 + LDAPResultBusy = 51 + LDAPResultUnavailable = 52 + LDAPResultUnwillingToPerform = 53 + LDAPResultLoopDetect = 54 + LDAPResultNamingViolation = 64 + LDAPResultObjectClassViolation = 65 + LDAPResultNotAllowedOnNonLeaf = 66 + LDAPResultNotAllowedOnRDN = 67 + LDAPResultEntryAlreadyExists = 68 + LDAPResultObjectClassModsProhibited = 69 + LDAPResultAffectsMultipleDSAs = 71 + LDAPResultOther = 80 + + ErrorNetwork = 200 + ErrorFilterCompile = 201 + ErrorFilterDecompile = 202 + ErrorDebugging = 203 +) + +var LDAPResultCodeMap = map[uint8]string{ + LDAPResultSuccess: "Success", + LDAPResultOperationsError: "Operations Error", + LDAPResultProtocolError: "Protocol Error", + LDAPResultTimeLimitExceeded: "Time Limit Exceeded", + LDAPResultSizeLimitExceeded: "Size Limit Exceeded", + LDAPResultCompareFalse: "Compare False", + LDAPResultCompareTrue: "Compare True", + LDAPResultAuthMethodNotSupported: "Auth Method Not Supported", + LDAPResultStrongAuthRequired: "Strong Auth Required", + LDAPResultReferral: "Referral", + LDAPResultAdminLimitExceeded: "Admin Limit Exceeded", + LDAPResultUnavailableCriticalExtension: "Unavailable Critical Extension", + LDAPResultConfidentialityRequired: "Confidentiality Required", + LDAPResultSaslBindInProgress: "Sasl Bind In Progress", + LDAPResultNoSuchAttribute: "No Such Attribute", + LDAPResultUndefinedAttributeType: "Undefined Attribute Type", + LDAPResultInappropriateMatching: "Inappropriate Matching", + LDAPResultConstraintViolation: "Constraint Violation", + LDAPResultAttributeOrValueExists: "Attribute Or Value Exists", + LDAPResultInvalidAttributeSyntax: "Invalid Attribute Syntax", + LDAPResultNoSuchObject: "No Such Object", + LDAPResultAliasProblem: "Alias Problem", + LDAPResultInvalidDNSyntax: "Invalid DN Syntax", + LDAPResultAliasDereferencingProblem: "Alias Dereferencing Problem", + LDAPResultInappropriateAuthentication: "Inappropriate Authentication", + LDAPResultInvalidCredentials: "Invalid Credentials", + LDAPResultInsufficientAccessRights: "Insufficient Access Rights", + LDAPResultBusy: "Busy", + LDAPResultUnavailable: "Unavailable", + LDAPResultUnwillingToPerform: "Unwilling To Perform", + LDAPResultLoopDetect: "Loop Detect", + LDAPResultNamingViolation: "Naming Violation", + LDAPResultObjectClassViolation: "Object Class Violation", + LDAPResultNotAllowedOnNonLeaf: "Not Allowed On Non Leaf", + LDAPResultNotAllowedOnRDN: "Not Allowed On RDN", + LDAPResultEntryAlreadyExists: "Entry Already Exists", + LDAPResultObjectClassModsProhibited: "Object Class Mods Prohibited", + LDAPResultAffectsMultipleDSAs: "Affects Multiple DSAs", + LDAPResultOther: "Other", +} + +// Adds descriptions to an LDAP Response packet for debugging +func addLDAPDescriptions(packet *ber.Packet) (err error) { + defer func() { + if r := recover(); r != nil { + err = NewError(ErrorDebugging, errors.New("ldap: cannot process packet to add descriptions")) + } + }() + packet.Description = "LDAP Response" + packet.Children[0].Description = "Message ID" + + application := packet.Children[1].Tag + packet.Children[1].Description = ApplicationMap[application] + + switch application { + case ApplicationBindRequest: + addRequestDescriptions(packet) + case ApplicationBindResponse: + addDefaultLDAPResponseDescriptions(packet) + case ApplicationUnbindRequest: + addRequestDescriptions(packet) + case ApplicationSearchRequest: + addRequestDescriptions(packet) + case ApplicationSearchResultEntry: + packet.Children[1].Children[0].Description = "Object Name" + packet.Children[1].Children[1].Description = "Attributes" + for _, child := range packet.Children[1].Children[1].Children { + child.Description = "Attribute" + child.Children[0].Description = "Attribute Name" + child.Children[1].Description = "Attribute Values" + for _, grandchild := range child.Children[1].Children { + grandchild.Description = "Attribute Value" + } + } + if len(packet.Children) == 3 { + addControlDescriptions(packet.Children[2]) + } + case ApplicationSearchResultDone: + addDefaultLDAPResponseDescriptions(packet) + case ApplicationModifyRequest: + addRequestDescriptions(packet) + case ApplicationModifyResponse: + case ApplicationAddRequest: + addRequestDescriptions(packet) + case ApplicationAddResponse: + case ApplicationDelRequest: + addRequestDescriptions(packet) + case ApplicationDelResponse: + case ApplicationModifyDNRequest: + addRequestDescriptions(packet) + case ApplicationModifyDNResponse: + case ApplicationCompareRequest: + addRequestDescriptions(packet) + case ApplicationCompareResponse: + case ApplicationAbandonRequest: + addRequestDescriptions(packet) + case ApplicationSearchResultReference: + case ApplicationExtendedRequest: + addRequestDescriptions(packet) + case ApplicationExtendedResponse: + } + + return nil +} + +func addControlDescriptions(packet *ber.Packet) { + packet.Description = "Controls" + for _, child := range packet.Children { + child.Description = "Control" + child.Children[0].Description = "Control Type (" + ControlTypeMap[child.Children[0].Value.(string)] + ")" + value := child.Children[1] + if len(child.Children) == 3 { + child.Children[1].Description = "Criticality" + value = child.Children[2] + } + value.Description = "Control Value" + + switch child.Children[0].Value.(string) { + case ControlTypePaging: + value.Description += " (Paging)" + if value.Value != nil { + valueChildren := ber.DecodePacket(value.Data.Bytes()) + value.Data.Truncate(0) + value.Value = nil + valueChildren.Children[1].Value = valueChildren.Children[1].Data.Bytes() + value.AppendChild(valueChildren) + } + value.Children[0].Description = "Real Search Control Value" + value.Children[0].Children[0].Description = "Paging Size" + value.Children[0].Children[1].Description = "Cookie" + } + } +} + +func addRequestDescriptions(packet *ber.Packet) { + packet.Description = "LDAP Request" + packet.Children[0].Description = "Message ID" + packet.Children[1].Description = ApplicationMap[packet.Children[1].Tag] + if len(packet.Children) == 3 { + addControlDescriptions(packet.Children[2]) + } +} + +func addDefaultLDAPResponseDescriptions(packet *ber.Packet) { + resultCode := packet.Children[1].Children[0].Value.(uint64) + packet.Children[1].Children[0].Description = "Result Code (" + LDAPResultCodeMap[uint8(resultCode)] + ")" + packet.Children[1].Children[1].Description = "Matched DN" + packet.Children[1].Children[2].Description = "Error Message" + if len(packet.Children[1].Children) > 3 { + packet.Children[1].Children[3].Description = "Referral" + } + if len(packet.Children) == 3 { + addControlDescriptions(packet.Children[2]) + } +} + +func DebugBinaryFile(fileName string) error { + file, err := ioutil.ReadFile(fileName) + if err != nil { + return NewError(ErrorDebugging, err) + } + ber.PrintBytes(file, "") + packet := ber.DecodePacket(file) + addLDAPDescriptions(packet) + ber.PrintPacket(packet) + + return nil +} + +type Error struct { + Err error + ResultCode uint8 +} + +func (e *Error) Error() string { + return fmt.Sprintf("LDAP Result Code %d %q: %s", e.ResultCode, LDAPResultCodeMap[e.ResultCode], e.Err.Error()) +} + +func NewError(resultCode uint8, err error) error { + return &Error{ResultCode: resultCode, Err: err} +} + +func getLDAPResultCode(packet *ber.Packet) (code uint8, description string) { + if len(packet.Children) >= 2 { + response := packet.Children[1] + if response.ClassType == ber.ClassApplication && response.TagType == ber.TypeConstructed && len(response.Children) == 3 { + return uint8(response.Children[0].Value.(uint64)), response.Children[2].Value.(string) + } + } + + return ErrorNetwork, "Invalid packet format" +} diff --git a/pkg/components/ldap/ldap_test.go b/pkg/components/ldap/ldap_test.go new file mode 100644 index 00000000000..31cfbf02f1b --- /dev/null +++ b/pkg/components/ldap/ldap_test.go @@ -0,0 +1,123 @@ +package ldap + +import ( + "fmt" + "testing" +) + +var ldapServer = "ldap.itd.umich.edu" +var ldapPort = uint16(389) +var baseDN = "dc=umich,dc=edu" +var filter = []string{ + "(cn=cis-fac)", + "(&(objectclass=rfc822mailgroup)(cn=*Computer*))", + "(&(objectclass=rfc822mailgroup)(cn=*Mathematics*))"} +var attributes = []string{ + "cn", + "description"} + +func TestConnect(t *testing.T) { + fmt.Printf("TestConnect: starting...\n") + l, err := Dial("tcp", fmt.Sprintf("%s:%d", ldapServer, ldapPort)) + if err != nil { + t.Errorf(err.Error()) + return + } + defer l.Close() + fmt.Printf("TestConnect: finished...\n") +} + +func TestSearch(t *testing.T) { + fmt.Printf("TestSearch: starting...\n") + l, err := Dial("tcp", fmt.Sprintf("%s:%d", ldapServer, ldapPort)) + if err != nil { + t.Errorf(err.Error()) + return + } + defer l.Close() + + searchRequest := NewSearchRequest( + baseDN, + ScopeWholeSubtree, DerefAlways, 0, 0, false, + filter[0], + attributes, + nil) + + sr, err := l.Search(searchRequest) + if err != nil { + t.Errorf(err.Error()) + return + } + + fmt.Printf("TestSearch: %s -> num of entries = %d\n", searchRequest.Filter, len(sr.Entries)) +} + +func TestSearchWithPaging(t *testing.T) { + fmt.Printf("TestSearchWithPaging: starting...\n") + l, err := Dial("tcp", fmt.Sprintf("%s:%d", ldapServer, ldapPort)) + if err != nil { + t.Errorf(err.Error()) + return + } + defer l.Close() + + err = l.Bind("", "") + if err != nil { + t.Errorf(err.Error()) + return + } + + searchRequest := NewSearchRequest( + baseDN, + ScopeWholeSubtree, DerefAlways, 0, 0, false, + filter[1], + attributes, + nil) + sr, err := l.SearchWithPaging(searchRequest, 5) + if err != nil { + t.Errorf(err.Error()) + return + } + + fmt.Printf("TestSearchWithPaging: %s -> num of entries = %d\n", searchRequest.Filter, len(sr.Entries)) +} + +func testMultiGoroutineSearch(t *testing.T, l *Conn, results chan *SearchResult, i int) { + searchRequest := NewSearchRequest( + baseDN, + ScopeWholeSubtree, DerefAlways, 0, 0, false, + filter[i], + attributes, + nil) + sr, err := l.Search(searchRequest) + if err != nil { + t.Errorf(err.Error()) + results <- nil + return + } + results <- sr +} + +func TestMultiGoroutineSearch(t *testing.T) { + fmt.Printf("TestMultiGoroutineSearch: starting...\n") + l, err := Dial("tcp", fmt.Sprintf("%s:%d", ldapServer, ldapPort)) + if err != nil { + t.Errorf(err.Error()) + return + } + defer l.Close() + + results := make([]chan *SearchResult, len(filter)) + for i := range filter { + results[i] = make(chan *SearchResult) + go testMultiGoroutineSearch(t, l, results[i], i) + } + for i := range filter { + sr := <-results[i] + if sr == nil { + t.Errorf("Did not receive results from goroutine for %q", filter[i]) + } else { + fmt.Printf("TestMultiGoroutineSearch(%d): %s -> num of entries = %d\n", i, filter[i], len(sr.Entries)) + } + } +} diff --git a/pkg/components/ldap/modify.go b/pkg/components/ldap/modify.go new file mode 100644 index 00000000000..decc1eddca0 --- /dev/null +++ b/pkg/components/ldap/modify.go @@ -0,0 +1,156 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// File contains Modify functionality +// +// https://tools.ietf.org/html/rfc4511 +// +// ModifyRequest ::= [APPLICATION 6] SEQUENCE { +// object LDAPDN, +// changes SEQUENCE OF change SEQUENCE { +// operation ENUMERATED { +// add (0), +// delete (1), +// replace (2), +// ... }, +// modification PartialAttribute } } +// +// PartialAttribute ::= SEQUENCE { +// type AttributeDescription, +// vals SET OF value AttributeValue } +// +// AttributeDescription ::= LDAPString +// -- Constrained to +// -- [RFC4512] +// +// AttributeValue ::= OCTET STRING +// + +package ldap + +import ( + "errors" + "log" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +const ( + AddAttribute = 0 + DeleteAttribute = 1 + ReplaceAttribute = 2 +) + +type PartialAttribute struct { + attrType string + attrVals []string +} + +func (p *PartialAttribute) encode() *ber.Packet { + seq := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "PartialAttribute") + seq.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, p.attrType, "Type")) + set := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSet, nil, "AttributeValue") + for _, value := range p.attrVals { + set.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, value, "Vals")) + } + seq.AppendChild(set) + return seq +} + +type ModifyRequest struct { + dn string + addAttributes []PartialAttribute + deleteAttributes []PartialAttribute + replaceAttributes []PartialAttribute +} + +func (m *ModifyRequest) Add(attrType string, attrVals []string) { + m.addAttributes = append(m.addAttributes, PartialAttribute{attrType: attrType, attrVals: attrVals}) +} + +func (m *ModifyRequest) Delete(attrType string, attrVals []string) { + m.deleteAttributes = append(m.deleteAttributes, PartialAttribute{attrType: attrType, attrVals: attrVals}) +} + +func (m *ModifyRequest) Replace(attrType string, attrVals []string) { + m.replaceAttributes = append(m.replaceAttributes, PartialAttribute{attrType: attrType, attrVals: attrVals}) +} + +func (m ModifyRequest) encode() *ber.Packet { + request := ber.Encode(ber.ClassApplication, ber.TypeConstructed, ApplicationModifyRequest, nil, "Modify Request") + request.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, m.dn, "DN")) + changes := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Changes") + for _, attribute := range m.addAttributes { + change := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Change") + change.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagEnumerated, uint64(AddAttribute), "Operation")) + change.AppendChild(attribute.encode()) + changes.AppendChild(change) + } + for _, attribute := range m.deleteAttributes { + change := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Change") + change.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagEnumerated, uint64(DeleteAttribute), "Operation")) + change.AppendChild(attribute.encode()) + changes.AppendChild(change) + } + for _, attribute := range m.replaceAttributes { + change := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Change") + change.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagEnumerated, uint64(ReplaceAttribute), "Operation")) + change.AppendChild(attribute.encode()) + changes.AppendChild(change) + } + request.AppendChild(changes) + return request +} + +func NewModifyRequest( + dn string, +) *ModifyRequest { + return &ModifyRequest{ + dn: dn, + } +} + +func (l *Conn) Modify(modifyRequest *ModifyRequest) error { + messageID := l.nextMessageID() + packet := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "LDAP Request") + packet.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagInteger, messageID, "MessageID")) + packet.AppendChild(modifyRequest.encode()) + + l.Debug.PrintPacket(packet) + + channel, err := l.sendMessage(packet) + if err != nil { + return err + } + if channel == nil { + return NewError(ErrorNetwork, errors.New("ldap: could not send message")) + } + defer l.finishMessage(messageID) + + l.Debug.Printf("%d: waiting for response", messageID) + packet = <-channel + l.Debug.Printf("%d: got response %p", messageID, packet) + if packet == nil { + return NewError(ErrorNetwork, errors.New("ldap: could not retrieve message")) + } + + if l.Debug { + if err := addLDAPDescriptions(packet); err != nil { + return err + } + ber.PrintPacket(packet) + } + + if packet.Children[1].Tag == ApplicationModifyResponse { + resultCode, resultDescription := getLDAPResultCode(packet) + if resultCode != 0 { + return NewError(resultCode, errors.New(resultDescription)) + } + } else { + log.Printf("Unexpected Response: %d", packet.Children[1].Tag) + } + + l.Debug.Printf("%d: returning", messageID) + return nil +} diff --git a/pkg/components/ldap/search.go b/pkg/components/ldap/search.go new file mode 100644 index 00000000000..e2a62064468 --- /dev/null +++ b/pkg/components/ldap/search.go @@ -0,0 +1,350 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// File contains Search functionality +// +// https://tools.ietf.org/html/rfc4511 +// +// SearchRequest ::= [APPLICATION 3] SEQUENCE { +// baseObject LDAPDN, +// scope ENUMERATED { +// baseObject (0), +// singleLevel (1), +// wholeSubtree (2), +// ... }, +// derefAliases ENUMERATED { +// neverDerefAliases (0), +// derefInSearching (1), +// derefFindingBaseObj (2), +// derefAlways (3) }, +// sizeLimit INTEGER (0 .. maxInt), +// timeLimit INTEGER (0 .. maxInt), +// typesOnly BOOLEAN, +// filter Filter, +// attributes AttributeSelection } +// +// AttributeSelection ::= SEQUENCE OF selector LDAPString +// -- The LDAPString is constrained to +// -- in Section 4.5.1.8 +// +// Filter ::= CHOICE { +// and [0] SET SIZE (1..MAX) OF filter Filter, +// or [1] SET SIZE (1..MAX) OF filter Filter, +// not [2] Filter, +// equalityMatch [3] AttributeValueAssertion, +// substrings [4] SubstringFilter, +// greaterOrEqual [5] AttributeValueAssertion, +// lessOrEqual [6] AttributeValueAssertion, +// present [7] AttributeDescription, +// approxMatch [8] AttributeValueAssertion, +// extensibleMatch [9] MatchingRuleAssertion, +// ... } +// +// SubstringFilter ::= SEQUENCE { +// type AttributeDescription, +// substrings SEQUENCE SIZE (1..MAX) OF substring CHOICE { +// initial [0] AssertionValue, -- can occur at most once +// any [1] AssertionValue, +// final [2] AssertionValue } -- can occur at most once +// } +// +// MatchingRuleAssertion ::= SEQUENCE { +// matchingRule [1] MatchingRuleId OPTIONAL, +// type [2] AttributeDescription OPTIONAL, +// matchValue [3] AssertionValue, +// dnAttributes [4] BOOLEAN DEFAULT FALSE } +// +// + +package ldap + +import ( + "errors" + "fmt" + "strings" + + "github.com/gogits/gogs/modules/asn1-ber" +) + +const ( + ScopeBaseObject = 0 + ScopeSingleLevel = 1 + ScopeWholeSubtree = 2 +) + +var ScopeMap = map[int]string{ + ScopeBaseObject: "Base Object", + ScopeSingleLevel: "Single Level", + ScopeWholeSubtree: "Whole Subtree", +} + +const ( + NeverDerefAliases = 0 + DerefInSearching = 1 + DerefFindingBaseObj = 2 + DerefAlways = 3 +) + +var DerefMap = map[int]string{ + NeverDerefAliases: "NeverDerefAliases", + DerefInSearching: "DerefInSearching", + DerefFindingBaseObj: "DerefFindingBaseObj", + DerefAlways: "DerefAlways", +} + +type Entry struct { + DN string + Attributes []*EntryAttribute +} + +func (e *Entry) GetAttributeValues(attribute string) []string { + for _, attr := range e.Attributes { + if attr.Name == attribute { + return attr.Values + } + } + return []string{} +} + +func (e *Entry) GetAttributeValue(attribute string) string { + values := e.GetAttributeValues(attribute) + if len(values) == 0 { + return "" + } + return values[0] +} + +func (e *Entry) Print() { + fmt.Printf("DN: %s\n", e.DN) + for _, attr := range e.Attributes { + attr.Print() + } +} + +func (e *Entry) PrettyPrint(indent int) { + fmt.Printf("%sDN: %s\n", strings.Repeat(" ", indent), e.DN) + for _, attr := range e.Attributes { + attr.PrettyPrint(indent + 2) + } +} + +type EntryAttribute struct { + Name string + Values []string +} + +func (e *EntryAttribute) Print() { + fmt.Printf("%s: %s\n", e.Name, e.Values) +} + +func (e *EntryAttribute) PrettyPrint(indent int) { + fmt.Printf("%s%s: %s\n", strings.Repeat(" ", indent), e.Name, e.Values) +} + +type SearchResult struct { + Entries []*Entry + Referrals []string + Controls []Control +} + +func (s *SearchResult) Print() { + for _, entry := range s.Entries { + entry.Print() + } +} + +func (s *SearchResult) PrettyPrint(indent int) { + for _, entry := range s.Entries { + entry.PrettyPrint(indent) + } +} + +type SearchRequest struct { + BaseDN string + Scope int + DerefAliases int + SizeLimit int + TimeLimit int + TypesOnly bool + Filter string + Attributes []string + Controls []Control +} + +func (s *SearchRequest) encode() (*ber.Packet, error) { + request := ber.Encode(ber.ClassApplication, ber.TypeConstructed, ApplicationSearchRequest, nil, "Search Request") + request.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, s.BaseDN, "Base DN")) + request.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagEnumerated, uint64(s.Scope), "Scope")) + request.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagEnumerated, uint64(s.DerefAliases), "Deref Aliases")) + request.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagInteger, uint64(s.SizeLimit), "Size Limit")) + request.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagInteger, uint64(s.TimeLimit), "Time Limit")) + request.AppendChild(ber.NewBoolean(ber.ClassUniversal, ber.TypePrimitive, ber.TagBoolean, s.TypesOnly, "Types Only")) + // compile and encode filter + filterPacket, err := CompileFilter(s.Filter) + if err != nil { + return nil, err + } + request.AppendChild(filterPacket) + // encode attributes + attributesPacket := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "Attributes") + for _, attribute := range s.Attributes { + attributesPacket.AppendChild(ber.NewString(ber.ClassUniversal, ber.TypePrimitive, ber.TagOctetString, attribute, "Attribute")) + } + request.AppendChild(attributesPacket) + return request, nil +} + +func NewSearchRequest( + BaseDN string, + Scope, DerefAliases, SizeLimit, TimeLimit int, + TypesOnly bool, + Filter string, + Attributes []string, + Controls []Control, +) *SearchRequest { + return &SearchRequest{ + BaseDN: BaseDN, + Scope: Scope, + DerefAliases: DerefAliases, + SizeLimit: SizeLimit, + TimeLimit: TimeLimit, + TypesOnly: TypesOnly, + Filter: Filter, + Attributes: Attributes, + Controls: Controls, + } +} + +func (l *Conn) SearchWithPaging(searchRequest *SearchRequest, pagingSize uint32) (*SearchResult, error) { + if searchRequest.Controls == nil { + searchRequest.Controls = make([]Control, 0) + } + + pagingControl := NewControlPaging(pagingSize) + searchRequest.Controls = append(searchRequest.Controls, pagingControl) + searchResult := new(SearchResult) + for { + result, err := l.Search(searchRequest) + l.Debug.Printf("Looking for Paging Control...") + if err != nil { + return searchResult, err + } + if result == nil { + return searchResult, NewError(ErrorNetwork, errors.New("ldap: packet not received")) + } + + for _, entry := range result.Entries { + searchResult.Entries = append(searchResult.Entries, entry) + } + for _, referral := range result.Referrals { + searchResult.Referrals = append(searchResult.Referrals, referral) + } + for _, control := range result.Controls { + searchResult.Controls = append(searchResult.Controls, control) + } + + l.Debug.Printf("Looking for Paging Control...") + pagingResult := FindControl(result.Controls, ControlTypePaging) + if pagingResult == nil { + pagingControl = nil + l.Debug.Printf("Could not find paging control. Breaking...") + break + } + + cookie := pagingResult.(*ControlPaging).Cookie + if len(cookie) == 0 { + pagingControl = nil + l.Debug.Printf("Could not find cookie. Breaking...") + break + } + pagingControl.SetCookie(cookie) + } + + if pagingControl != nil { + l.Debug.Printf("Abandoning Paging...") + pagingControl.PagingSize = 0 + l.Search(searchRequest) + } + + return searchResult, nil +} + +func (l *Conn) Search(searchRequest *SearchRequest) (*SearchResult, error) { + messageID := l.nextMessageID() + packet := ber.Encode(ber.ClassUniversal, ber.TypeConstructed, ber.TagSequence, nil, "LDAP Request") + packet.AppendChild(ber.NewInteger(ber.ClassUniversal, ber.TypePrimitive, ber.TagInteger, messageID, "MessageID")) + // encode search request + encodedSearchRequest, err := searchRequest.encode() + if err != nil { + return nil, err + } + packet.AppendChild(encodedSearchRequest) + // encode search controls + if searchRequest.Controls != nil { + packet.AppendChild(encodeControls(searchRequest.Controls)) + } + + l.Debug.PrintPacket(packet) + + channel, err := l.sendMessage(packet) + if err != nil { + return nil, err + } + if channel == nil { + return nil, NewError(ErrorNetwork, errors.New("ldap: could not send message")) + } + defer l.finishMessage(messageID) + + result := &SearchResult{ + Entries: make([]*Entry, 0), + Referrals: make([]string, 0), + Controls: make([]Control, 0)} + + foundSearchResultDone := false + for !foundSearchResultDone { + l.Debug.Printf("%d: waiting for response", messageID) + packet = <-channel + l.Debug.Printf("%d: got response %p", messageID, packet) + if packet == nil { + return nil, NewError(ErrorNetwork, errors.New("ldap: could not retrieve message")) + } + + if l.Debug { + if err := addLDAPDescriptions(packet); err != nil { + return nil, err + } + ber.PrintPacket(packet) + } + + switch packet.Children[1].Tag { + case 4: + entry := new(Entry) + entry.DN = packet.Children[1].Children[0].Value.(string) + for _, child := range packet.Children[1].Children[1].Children { + attr := new(EntryAttribute) + attr.Name = child.Children[0].Value.(string) + for _, value := range child.Children[1].Children { + attr.Values = append(attr.Values, value.Value.(string)) + } + entry.Attributes = append(entry.Attributes, attr) + } + result.Entries = append(result.Entries, entry) + case 5: + resultCode, resultDescription := getLDAPResultCode(packet) + if resultCode != 0 { + return result, NewError(resultCode, errors.New(resultDescription)) + } + if len(packet.Children) == 3 { + for _, child := range packet.Children[2].Children { + result.Controls = append(result.Controls, DecodeControl(child)) + } + } + foundSearchResultDone = true + case 19: + result.Referrals = append(result.Referrals, packet.Children[1].Children[0].Value.(string)) + } + } + l.Debug.Printf("%d: returning", messageID) + return result, nil +} From eb793f7feb38a10ce5e6c50a6e55d35b3afff2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 4 Jun 2015 09:34:42 +0200 Subject: [PATCH 003/287] Initial work on ldap support, #1450 --- conf/defaults.ini | 12 ++++++ pkg/api/api.go | 2 +- pkg/api/ldap/ldap.go | 0 pkg/api/ldapauth/ldapauth.go | 56 +++++++++++++++++++++++++++ pkg/api/login.go | 48 +++++++++++++++++------- pkg/auth/auth.go | 73 ++++++++++++++++++++++++++++++++++++ pkg/setting/setting.go | 8 ++++ pkg/setting/setting_ldap.go | 19 ++++++++++ 8 files changed, 204 insertions(+), 14 deletions(-) delete mode 100644 pkg/api/ldap/ldap.go create mode 100644 pkg/api/ldapauth/ldapauth.go create mode 100644 pkg/auth/auth.go create mode 100644 pkg/setting/setting_ldap.go diff --git a/conf/defaults.ini b/conf/defaults.ini index 258a0198155..e3e5f6eb5f9 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -174,6 +174,18 @@ header_name = X-WEBAUTH-USER header_property = username auto_sign_up = true +#################################### Auth LDAP ########################## +[auth.ldap] +enabled = true +hosts = ldap://localhost.com:389 +use_ssl = false +base_dn = dc=grafana,dc=org +bind_path = cn=%username%,dc=grafana,dc=org +attr_username = cn +attr_name = cn +attr_surname = sn +attr_email = email + #################################### Logging ########################## [log] # Either "console", "file", default is "console" diff --git a/pkg/api/api.go b/pkg/api/api.go index 6ecaa51652e..0d8bceed4f8 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -19,7 +19,7 @@ func Register(r *macaron.Macaron) { // not logged in views r.Get("/", reqSignedIn, Index) r.Get("/logout", Logout) - r.Post("/login", bind(dtos.LoginCommand{}), LoginPost) + r.Post("/login", bind(dtos.LoginCommand{}), wrap(LoginPost)) r.Get("/login/:name", OAuthLogin) r.Get("/login", LoginView) diff --git a/pkg/api/ldap/ldap.go b/pkg/api/ldap/ldap.go deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/pkg/api/ldapauth/ldapauth.go b/pkg/api/ldapauth/ldapauth.go new file mode 100644 index 00000000000..ea6c0421e12 --- /dev/null +++ b/pkg/api/ldapauth/ldapauth.go @@ -0,0 +1,56 @@ +package ldapauth + +import ( + "errors" + "fmt" + "net/url" + + "github.com/gogits/gogs/modules/ldap" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/setting" +) + +var ( + ErrInvalidCredentials = errors.New("Invalid Username or Password") +) + +func Login(username, password string) error { + url, err := url.Parse(setting.LdapUrls[0]) + if err != nil { + return err + } + + log.Info("Host: %v", url.Host) + conn, err := ldap.Dial("tcp", url.Host) + if err != nil { + return err + } + + defer conn.Close() + + bindFormat := "cn=%s,dc=grafana,dc=org" + + nx := fmt.Sprintf(bindFormat, username) + err = conn.Bind(nx, password) + + if err != nil { + if ldapErr, ok := err.(*ldap.Error); ok { + if ldapErr.ResultCode == 49 { + return ErrInvalidCredentials + } + } + return err + } + return nil + + // search := ldap.NewSearchRequest(url.Path, + // ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + // fmt.Sprintf(ls.Filter, name), + // []string{ls.AttributeUsername, ls.AttributeName, ls.AttributeSurname, ls.AttributeMail}, + // nil) + // sr, err := l.Search(search) + // if err != nil { + // log.Debug("LDAP Authen OK but not in filter %s", name) + // return "", "", "", "", false + // } +} diff --git a/pkg/api/login.go b/pkg/api/login.go index 0fc5651d5f9..f6a511bffbe 100644 --- a/pkg/api/login.go +++ b/pkg/api/login.go @@ -4,6 +4,8 @@ import ( "net/url" "github.com/grafana/grafana/pkg/api/dtos" + "github.com/grafana/grafana/pkg/api/ldapauth" + "github.com/grafana/grafana/pkg/auth" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" @@ -86,21 +88,28 @@ func LoginApiPing(c *middleware.Context) { c.JsonOK("Logged in") } -func LoginPost(c *middleware.Context, cmd dtos.LoginCommand) { - userQuery := m.GetUserByLoginQuery{LoginOrEmail: cmd.User} - err := bus.Dispatch(&userQuery) - - if err != nil { - c.JsonApiErr(401, "Invalid username or password", err) - return +func LoginPost(c *middleware.Context, cmd dtos.LoginCommand) Response { + sourcesQuery := auth.GetAuthSourcesQuery{} + if err := bus.Dispatch(&sourcesQuery); err != nil { + return ApiError(500, "Could not get login sources", err) } - user := userQuery.Result + var err error + var user *m.User - passwordHashed := util.EncodePassword(cmd.Password, user.Salt) - if passwordHashed != user.Password { - c.JsonApiErr(401, "Invalid username or password", err) - return + for _, authSource := range sourcesQuery.Sources { + user, err = authSource.AuthenticateUser(cmd.User, cmd.Password) + if err == nil { + break + } + // handle non invalid credentials error, otherwise try next auth source + if err != auth.ErrInvalidCredentials { + return ApiError(500, "Error while trying to authenticate user", err) + } + } + + if err != nil { + return ApiError(401, "Invalid username or password", err) } loginUserWithUser(user, c) @@ -116,7 +125,20 @@ func LoginPost(c *middleware.Context, cmd dtos.LoginCommand) { metrics.M_Api_Login_Post.Inc(1) - c.JSON(200, result) + return Json(200, result) +} + +func LoginUsingLdap(c *middleware.Context, cmd dtos.LoginCommand) Response { + err := ldapauth.Login(cmd.User, cmd.Password) + + if err != nil { + if err == ldapauth.ErrInvalidCredentials { + return ApiError(401, "Invalid username or password", err) + } + return ApiError(500, "Ldap login failed", err) + } + + return Empty(401) } func loginUserWithUser(user *m.User, c *middleware.Context) { diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go new file mode 100644 index 00000000000..a236663ed44 --- /dev/null +++ b/pkg/auth/auth.go @@ -0,0 +1,73 @@ +package auth + +import ( + "errors" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/util" +) + +var ( + ErrInvalidCredentials = errors.New("Invalid Username or Password") +) + +type LoginSettings struct { + LdapEnabled bool +} + +type LdapFilterToOrg struct { + Filter string + OrgId int + OrgRole string +} + +type LdapSettings struct { + Enabled bool + Hosts []string + UseSSL bool + BindDN string + AttrUsername string + AttrName string + AttrSurname string + AttrMail string + Filters []LdapFilterToOrg +} + +type AuthSource interface { + AuthenticateUser(username, password string) (*m.User, error) +} + +type GetAuthSourcesQuery struct { + Sources []AuthSource +} + +func init() { + bus.AddHandler("auth", GetAuthSources) +} + +func GetAuthSources(query *GetAuthSourcesQuery) error { + query.Sources = []AuthSource{&GrafanaDBAuthSource{}} + return nil +} + +type GrafanaDBAuthSource struct { +} + +func (s *GrafanaDBAuthSource) AuthenticateUser(username, password string) (*m.User, error) { + userQuery := m.GetUserByLoginQuery{LoginOrEmail: username} + err := bus.Dispatch(&userQuery) + + if err != nil { + return nil, ErrInvalidCredentials + } + + user := userQuery.Result + + passwordHashed := util.EncodePassword(password, user.Salt) + if passwordHashed != user.Password { + return nil, ErrInvalidCredentials + } + + return user, nil +} diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 6768f9aabd9..3cb4792ca82 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -114,6 +114,10 @@ var ( ReportingEnabled bool GoogleAnalyticsId string + + // LDAP + LdapEnabled bool + LdapUrls []string ) type CommandLineArgs struct { @@ -406,6 +410,10 @@ func NewConfigContext(args *CommandLineArgs) { ReportingEnabled = analytics.Key("reporting_enabled").MustBool(true) GoogleAnalyticsId = analytics.Key("google_analytics_ua_id").String() + ldapSec := Cfg.Section("auth.ldap") + LdapEnabled = ldapSec.Key("enabled").MustBool(false) + LdapUrls = ldapSec.Key("urls").Strings(" ") + readSessionConfig() } diff --git a/pkg/setting/setting_ldap.go b/pkg/setting/setting_ldap.go new file mode 100644 index 00000000000..26592159f1f --- /dev/null +++ b/pkg/setting/setting_ldap.go @@ -0,0 +1,19 @@ +package setting + +type LdapFilterToOrg struct { + Filter string + OrgId int + OrgRole string +} + +type LdapSettings struct { + Enabled bool + Hosts []string + UseSSL bool + BindDN string + AttrUsername string + AttrName string + AttrSurname string + AttrMail string + Filters []LdapFilterToOrg +} From 463a750c0772312b825b6646477acff2d9bcca47 Mon Sep 17 00:00:00 2001 From: Lex Herbert Date: Thu, 4 Jun 2015 17:45:09 -0700 Subject: [PATCH 004/287] OpenTSDB: Restrict typeahead tag keys and values When selecting metric tag keys, we only are interested in keys which are associated with this metric. Likewise, when selecting a value for a certain key, we only want to consider values which apply to the given key and metric. --- .../plugins/datasource/opentsdb/datasource.js | 49 +++++++++++++++++++ .../plugins/datasource/opentsdb/queryCtrl.js | 4 +- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index 1e60bf1e54e..57559cc6687 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -90,6 +90,55 @@ function (angular, _, kbn) { }); }; + OpenTSDBDatasource.prototype.performMetricKeyValueLookup = function(metric, key) { + if(metric === "") { + throw "Metric not set."; + } else if(key === "") { + throw "Key not set."; + } + var m = metric + "{" + key + "=*}"; + var options = { + method: 'GET', + url: this.url + '/api/search/lookup', + params: { + m: m, + } + }; + return backendSrv.datasourceRequest(options).then(function(result) { + result = result.data.results; + var tagvs = []; + _.each(result, function(r) { + tagvs.push(r.tags[key]); + }); + return tagvs; + }); + }; + + OpenTSDBDatasource.prototype.performMetricKeyLookup = function(metric) { + if(metric === "") { + throw "Metric not set."; + } + var options = { + method: 'GET', + url: this.url + '/api/search/lookup', + params: { + m: metric, + } + }; + return backendSrv.datasourceRequest(options).then(function(result) { + result = result.data.results; + var tagks = []; + _.each(result, function(r) { + _.each(r.tags, function(tagv, tagk) { + if(tagks.indexOf(tagk) === -1) { + tagks.push(tagk); + } + }); + }); + return tagks; + }); + }; + OpenTSDBDatasource.prototype.testDatasource = function() { return this.performSuggestQuery('cpu', 'metrics').then(function () { return { status: "success", message: "Data source is working", title: "Success" }; diff --git a/public/app/plugins/datasource/opentsdb/queryCtrl.js b/public/app/plugins/datasource/opentsdb/queryCtrl.js index 576517a50b6..17a2f4bd640 100644 --- a/public/app/plugins/datasource/opentsdb/queryCtrl.js +++ b/public/app/plugins/datasource/opentsdb/queryCtrl.js @@ -50,13 +50,13 @@ function (angular, _, kbn) { $scope.suggestTagKeys = function(query, callback) { $scope.datasource - .performSuggestQuery(query, 'tagk') + .performMetricKeyLookup($scope.target.metric) .then(callback); }; $scope.suggestTagValues = function(query, callback) { $scope.datasource - .performSuggestQuery(query, 'tagv') + .performMetricKeyValueLookup($scope.target.metric, $scope.target.currentTagKey) .then(callback); }; From 5e6d876bd020a64c49e1a6fd6d3fb4e49be118b3 Mon Sep 17 00:00:00 2001 From: "Randy D. Wallace Jr" Date: Wed, 17 Jun 2015 02:40:40 -0400 Subject: [PATCH 005/287] Support Multiple Fields in InfluxDB 0.9+ * Update influxdb 0.9 plugin to iterate thru fields returned by InfluxDB and output ea. field metric separately * Updates default label to include Field Name * Update spec to include updated label * Add spec for multiple fields * Do not print the field in the label when the name is value --- .../datasource/influxdb/influxSeries.js | 36 ++++--- public/test/specs/influxSeries-specs.js | 94 ++++++++++++++++++- 2 files changed, 117 insertions(+), 13 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/influxSeries.js b/public/app/plugins/datasource/influxdb/influxSeries.js index a4a21b1f6f0..874cd4746d2 100644 --- a/public/app/plugins/datasource/influxdb/influxSeries.js +++ b/public/app/plugins/datasource/influxdb/influxSeries.js @@ -20,25 +20,39 @@ function (_) { return output; } + var field_datapoints = function(datapoints, column_index) { + return _.map(datapoints, function(datapoint) { + return [datapoint[column_index - 1], _.last(datapoint)]; + }); + }; + _.each(self.series, function(series) { var datapoints = []; + var columns = series.columns.length; for (var i = 0; i < series.values.length; i++) { - datapoints[i] = [series.values[i][1], new Date(series.values[i][0]).getTime()]; + datapoints[i] = series.values[i].slice(1); + datapoints[i].push(new Date(series.values[i][0]).getTime()); } - var seriesName = series.name; + for (var j = 1; j < columns; j++) { + var seriesName = series.name; + var columnName = series.columns[j]; - if (self.alias) { - seriesName = self._getSeriesName(series); - } else if (series.tags) { - var tags = _.map(series.tags, function(value, key) { - return key + ': ' + value; - }); + if (self.alias) { + seriesName = self._getSeriesName(series); + } else if (series.tags) { + var tags = _.map(series.tags, function(value, key) { + return key + ': ' + value; + }); + if (columnName === 'value') { + seriesName = seriesName + ' {' + tags.join(', ') + '}'; + } else { + seriesName = seriesName + '.' + columnName + ' {' + tags.join(', ') + '}'; + } + } - seriesName = seriesName + ' {' + tags.join(', ') + '}'; + output.push({ target: seriesName, datapoints: field_datapoints(datapoints, j)}); } - - output.push({ target: seriesName, datapoints: datapoints }); }); return output; diff --git a/public/test/specs/influxSeries-specs.js b/public/test/specs/influxSeries-specs.js index 3a767f23988..4cff47d5385 100644 --- a/public/test/specs/influxSeries-specs.js +++ b/public/test/specs/influxSeries-specs.js @@ -5,6 +5,96 @@ define([ describe('when generating timeseries from influxdb response', function() { + describe('given multiple fields for series', function() { + var options = { series: [ + { + name: 'cpu', + tags: {app: 'test', server: 'server1'}, + columns: ['time', 'mean', 'max', 'min'], + values: [["2015-05-18T10:57:05Z", 10, 11, 9], ["2015-05-18T10:57:06Z", 20, 21, 19]] + } + ]}; + describe('and no alias', function() { + it('should generate multiple datapoints for each column', function() { + var series = new InfluxSeries(options); + var result = series.getTimeSeries(); + + expect(result.length).to.be(3); + expect(result[0].target).to.be('cpu.mean {app: test, server: server1}'); + expect(result[0].datapoints[0][0]).to.be(10); + expect(result[0].datapoints[0][1]).to.be(1431946625000); + expect(result[0].datapoints[1][0]).to.be(20); + expect(result[0].datapoints[1][1]).to.be(1431946626000); + + expect(result[1].target).to.be('cpu.max {app: test, server: server1}'); + expect(result[1].datapoints[0][0]).to.be(11); + expect(result[1].datapoints[0][1]).to.be(1431946625000); + expect(result[1].datapoints[1][0]).to.be(21); + expect(result[1].datapoints[1][1]).to.be(1431946626000); + + expect(result[2].target).to.be('cpu.min {app: test, server: server1}'); + expect(result[2].datapoints[0][0]).to.be(9); + expect(result[2].datapoints[0][1]).to.be(1431946625000); + expect(result[2].datapoints[1][0]).to.be(19); + expect(result[2].datapoints[1][1]).to.be(1431946626000); + + }); + }); + + describe('and simple alias', function() { + it('should use alias', function() { + options.alias = 'new series'; + var series = new InfluxSeries(options); + var result = series.getTimeSeries(); + + expect(result[0].target).to.be('new series'); + expect(result[1].target).to.be('new series'); + expect(result[2].target).to.be('new series'); + }); + + }); + + describe('and alias patterns', function() { + it('should replace patterns', function() { + options.alias = 'alias: $m -> $tag_server ([[measurement]])'; + var series = new InfluxSeries(options); + var result = series.getTimeSeries(); + + expect(result[0].target).to.be('alias: cpu -> server1 (cpu)'); + expect(result[1].target).to.be('alias: cpu -> server1 (cpu)'); + expect(result[2].target).to.be('alias: cpu -> server1 (cpu)'); + }); + + }); + }); + describe('given measurement with default fieldname', function() { + var options = { series: [ + { + name: 'cpu', + tags: {app: 'test', server: 'server1'}, + columns: ['time', 'value'], + values: [["2015-05-18T10:57:05Z", 10], ["2015-05-18T10:57:06Z", 12]] + }, + { + name: 'cpu', + tags: {app: 'test2', server: 'server2'}, + columns: ['time', 'value'], + values: [["2015-05-18T10:57:05Z", 15], ["2015-05-18T10:57:06Z", 16]] + } + ]}; + + describe('and no alias', function() { + + it('should generate label with no field', function() { + var series = new InfluxSeries(options); + var result = series.getTimeSeries(); + + expect(result[0].target).to.be('cpu {app: test, server: server1}'); + expect(result[1].target).to.be('cpu {app: test2, server: server2}'); + }); + }); + + }); describe('given two series', function() { var options = { series: [ { @@ -28,13 +118,13 @@ define([ var result = series.getTimeSeries(); expect(result.length).to.be(2); - expect(result[0].target).to.be('cpu {app: test, server: server1}'); + expect(result[0].target).to.be('cpu.mean {app: test, server: server1}'); expect(result[0].datapoints[0][0]).to.be(10); expect(result[0].datapoints[0][1]).to.be(1431946625000); expect(result[0].datapoints[1][0]).to.be(12); expect(result[0].datapoints[1][1]).to.be(1431946626000); - expect(result[1].target).to.be('cpu {app: test2, server: server2}'); + expect(result[1].target).to.be('cpu.mean {app: test2, server: server2}'); expect(result[1].datapoints[0][0]).to.be(15); expect(result[1].datapoints[0][1]).to.be(1431946625000); expect(result[1].datapoints[1][0]).to.be(16); From 999894cccbecee3cad37ccd2ad1b7ff0068d1ece Mon Sep 17 00:00:00 2001 From: Date: Fri, 26 Jun 2015 17:22:41 +0200 Subject: [PATCH 006/287] Added Pressure unit (mbar and hPa) --- public/app/components/kbn.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/app/components/kbn.js b/public/app/components/kbn.js index aa5146c1538..f3262e3add4 100644 --- a/public/app/components/kbn.js +++ b/public/app/components/kbn.js @@ -399,6 +399,8 @@ function($, _, moment) { kbn.valueFormats.celsius = function(value, decimals) { return kbn.toFixed(value, decimals) + ' °C'; }; kbn.valueFormats.farenheit = function(value, decimals) { return kbn.toFixed(value, decimals) + ' °F'; }; kbn.valueFormats.humidity = function(value, decimals) { return kbn.toFixed(value, decimals) + ' %H'; }; + kbn.valueFormats.pressurebar = function(value, decimals) { return kbn.toFixed(value, decimals) + ' mbar'; }; + kbn.valueFormats.pressurehpa = function(value, decimals) { return kbn.toFixed(value, decimals) + ' hPa'; }; kbn.valueFormats.ppm = function(value, decimals) { return kbn.toFixed(value, decimals) + ' ppm'; }; kbn.valueFormats.velocityms = function(value, decimals) { return kbn.toFixed(value, decimals) + ' m/s'; }; kbn.valueFormats.velocitykmh = function(value, decimals) { return kbn.toFixed(value, decimals) + ' km/h'; }; @@ -590,6 +592,8 @@ function($, _, moment) { {text: 'Celcius (°C)', value: 'celsius' }, {text: 'Farenheit (°F)', value: 'farenheit'}, {text: 'Humidity (%H)', value: 'humidity' }, + {text: 'Pressure (mbar)', value: 'pressurembar' }, + {text: 'Pressure (hPa)', value: 'pressurehpa' }, ] }, { From 5f513773c1d8a1c738138e3da16748f41433a7f8 Mon Sep 17 00:00:00 2001 From: Date: Fri, 26 Jun 2015 17:28:53 +0200 Subject: [PATCH 007/287] Added Pressure units (mbar and hPa) --- public/app/components/kbn.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/app/components/kbn.js b/public/app/components/kbn.js index f3262e3add4..ab5a83e6c23 100644 --- a/public/app/components/kbn.js +++ b/public/app/components/kbn.js @@ -592,8 +592,8 @@ function($, _, moment) { {text: 'Celcius (°C)', value: 'celsius' }, {text: 'Farenheit (°F)', value: 'farenheit'}, {text: 'Humidity (%H)', value: 'humidity' }, - {text: 'Pressure (mbar)', value: 'pressurembar' }, - {text: 'Pressure (hPa)', value: 'pressurehpa' }, + {text: 'Pressure (mbar)', value: 'pressurembar' }, + {text: 'Pressure (hPa)', value: 'pressurehpa' }, ] }, { From aba824a3177d355a2aa405842874e740b6164a2c Mon Sep 17 00:00:00 2001 From: Date: Fri, 26 Jun 2015 17:34:21 +0200 Subject: [PATCH 008/287] Added Pressure units (mbar and hPa) --- public/app/components/kbn.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/components/kbn.js b/public/app/components/kbn.js index ab5a83e6c23..d4a1a3c5178 100644 --- a/public/app/components/kbn.js +++ b/public/app/components/kbn.js @@ -399,7 +399,7 @@ function($, _, moment) { kbn.valueFormats.celsius = function(value, decimals) { return kbn.toFixed(value, decimals) + ' °C'; }; kbn.valueFormats.farenheit = function(value, decimals) { return kbn.toFixed(value, decimals) + ' °F'; }; kbn.valueFormats.humidity = function(value, decimals) { return kbn.toFixed(value, decimals) + ' %H'; }; - kbn.valueFormats.pressurebar = function(value, decimals) { return kbn.toFixed(value, decimals) + ' mbar'; }; + kbn.valueFormats.pressurembar = function(value, decimals) { return kbn.toFixed(value, decimals) + ' mbar'; }; kbn.valueFormats.pressurehpa = function(value, decimals) { return kbn.toFixed(value, decimals) + ' hPa'; }; kbn.valueFormats.ppm = function(value, decimals) { return kbn.toFixed(value, decimals) + ' ppm'; }; kbn.valueFormats.velocityms = function(value, decimals) { return kbn.toFixed(value, decimals) + ' m/s'; }; From 693af182c7761678ba3be5aab8c8c8f6b5f5524d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 26 Jun 2015 19:11:52 +0200 Subject: [PATCH 009/287] Fixed broken playlist in master, Fixes #2240 --- public/app/features/dashboard/playlistCtrl.js | 4 ++-- public/app/features/dashboard/playlistSrv.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/public/app/features/dashboard/playlistCtrl.js b/public/app/features/dashboard/playlistCtrl.js index 5320a0b808e..b5d04374e9a 100644 --- a/public/app/features/dashboard/playlistCtrl.js +++ b/public/app/features/dashboard/playlistCtrl.js @@ -25,14 +25,14 @@ function (angular, _, config) { } backendSrv.search(query).then(function(results) { - $scope.searchHits = results.dashboards; + $scope.searchHits = results; $scope.filterHits(); }); }; $scope.filterHits = function() { $scope.filteredHits = _.reject($scope.searchHits, function(dash) { - return _.findWhere($scope.playlist, {slug: dash.slug}); + return _.findWhere($scope.playlist, {uri: dash.uri}); }); }; diff --git a/public/app/features/dashboard/playlistSrv.js b/public/app/features/dashboard/playlistSrv.js index 0711cb7c453..9997581fbc3 100644 --- a/public/app/features/dashboard/playlistSrv.js +++ b/public/app/features/dashboard/playlistSrv.js @@ -18,7 +18,7 @@ function (angular, _, kbn) { angular.element(window).unbind('resize'); var dash = self.dashboards[self.index % self.dashboards.length]; - $location.url('dashboard/db/' + dash.slug); + $location.url('dashboard/' + dash.uri); self.index++; self.cancelPromise = $timeout(self.next, self.interval); From 20d5d0eee6e926779fa5f011e8a9b7acd13278cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 26 Jun 2015 19:42:33 +0200 Subject: [PATCH 010/287] Fixed issue with annotations that only occurs in optimized build, Fixes #2176, Fixes #2239 --- public/app/components/extend-jquery.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/app/components/extend-jquery.js b/public/app/components/extend-jquery.js index 3e1f6b0c054..f44245103b5 100644 --- a/public/app/components/extend-jquery.js +++ b/public/app/components/extend-jquery.js @@ -24,14 +24,14 @@ function ($, angular, _) { $tooltip.appendTo(document.body); if (opts.compile) { - angular.element(document).injector().invoke(function($compile, $rootScope) { + angular.element(document).injector().invoke(["$compile", "$rootScope", function($compile, $rootScope) { var tmpScope = $rootScope.$new(true); _.extend(tmpScope, opts.scopeData); $compile($tooltip)(tmpScope); tmpScope.$digest(); - //tmpScope.$destroy(); - }); + tmpScope.$destroy(); + }]); } width = $tooltip.outerWidth(true); From ca42d976a7f2de7b9033a2cd52e24df5b5fcab1a Mon Sep 17 00:00:00 2001 From: David Raifaizen Date: Fri, 26 Jun 2015 14:05:37 -0400 Subject: [PATCH 011/287] Added fix for template variables when no options are available to select --- public/app/features/templating/templateSrv.js | 2 +- public/app/features/templating/templateValuesSrv.js | 4 ++++ public/test/specs/templateValuesSrv-specs.js | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/public/app/features/templating/templateSrv.js b/public/app/features/templating/templateSrv.js index 028146a1314..4c9d47ae2e2 100644 --- a/public/app/features/templating/templateSrv.js +++ b/public/app/features/templating/templateSrv.js @@ -27,7 +27,7 @@ function (angular, _) { this._texts = {}; _.each(this.variables, function(variable) { - if (!variable.current || !variable.current.value) { return; } + if (!variable.current || !variable.current.isNone && !variable.current.value) { return; } this._values[variable.name] = this.renderVariableValue(variable); this._texts[variable.name] = variable.current.text; diff --git a/public/app/features/templating/templateValuesSrv.js b/public/app/features/templating/templateValuesSrv.js index 4192279d74a..8e91403c1e4 100644 --- a/public/app/features/templating/templateValuesSrv.js +++ b/public/app/features/templating/templateValuesSrv.js @@ -10,6 +10,7 @@ function (angular, _, kbn) { module.service('templateValuesSrv', function($q, $rootScope, datasourceSrv, $location, templateSrv, timeSrv) { var self = this; + function getNoneOption() { return { text: 'None', value: '', isNone: true }; } $rootScope.onAppEvent('time-range-changed', function() { var variable = _.findWhere(self.variables, { type: 'interval' }); @@ -175,6 +176,9 @@ function (angular, _, kbn) { if (variable.includeAll) { self.addAllOption(variable); } + if (!variable.options.length) { + variable.options.push(getNoneOption()); + } return datasource; }); }; diff --git a/public/test/specs/templateValuesSrv-specs.js b/public/test/specs/templateValuesSrv-specs.js index 3c75d91bbb1..6c7a3035ff9 100644 --- a/public/test/specs/templateValuesSrv-specs.js +++ b/public/test/specs/templateValuesSrv-specs.js @@ -224,8 +224,9 @@ define([ scenario.queryResult = [{text: 'apps.backend.backend_01.counters.req'}, {text: 'apps.backend.backend_02.counters.req'}]; }); - it('should not add non matching items', function() { - expect(scenario.variable.options.length).to.be(0); + it('should not add non matching items, None option should be added instead', function() { + expect(scenario.variable.options.length).to.be(1); + expect(scenario.variable.options[0].isNone).to.be(true); }); }); From 194273a643cba50538e292e3b936724f65573cc5 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Fri, 26 Jun 2015 12:32:24 -0700 Subject: [PATCH 012/287] clarify which handler is not found --- pkg/bus/bus.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go index 80b80698a1c..6eb4b741a27 100644 --- a/pkg/bus/bus.go +++ b/pkg/bus/bus.go @@ -1,7 +1,7 @@ package bus import ( - "errors" + "fmt" "reflect" ) @@ -39,7 +39,7 @@ func (b *InProcBus) Dispatch(msg Msg) error { var handler = b.handlers[msgName] if handler == nil { - return errors.New("handler not found") + return fmt.Errorf("handler not found for %s", msgName) } var params = make([]reflect.Value, 1) From 9d4cce74b2bff70272ea01bbc2a7100d34453ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Bouchex=20Bellomi=C3=A9?= Date: Sat, 27 Jun 2015 16:59:34 +0200 Subject: [PATCH 013/287] Added dB unit --- public/app/components/kbn.js | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/components/kbn.js b/public/app/components/kbn.js index d4a1a3c5178..d817ada2ebe 100644 --- a/public/app/components/kbn.js +++ b/public/app/components/kbn.js @@ -543,6 +543,7 @@ function($, _, moment) { {text: 'short', value: 'short'}, {text: 'percent', value: 'percent'}, {text: 'ppm', value: 'ppm'}, + {text: 'dB', value: 'dB'}, ] }, { From f14c6efaf8c31fdca37ed11783034c100ae1f017 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Sun, 28 Jun 2015 14:42:39 -0400 Subject: [PATCH 014/287] Capitalise panel menu items It's a minor thing but other options seem to be generally capitalised. This felt discordant. --- public/app/components/panelmeta.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/app/components/panelmeta.js b/public/app/components/panelmeta.js index 013919c174a..7eee8fa970f 100644 --- a/public/app/components/panelmeta.js +++ b/public/app/components/panelmeta.js @@ -13,12 +13,12 @@ function () { this.extendedMenu = []; if (options.fullscreen) { - this.addMenuItem('view', 'icon-eye-open', 'toggleFullscreen(false); dismiss();'); + this.addMenuItem('View', 'icon-eye-open', 'toggleFullscreen(false); dismiss();'); } - this.addMenuItem('edit', 'icon-cog', 'editPanel(); dismiss();', 'Editor'); - this.addMenuItem('duplicate', 'icon-copy', 'duplicatePanel()', 'Editor'); - this.addMenuItem('share', 'icon-share', 'sharePanel(); dismiss();'); + this.addMenuItem('Edit', 'icon-cog', 'editPanel(); dismiss();', 'Editor'); + this.addMenuItem('Duplicate', 'icon-copy', 'duplicatePanel()', 'Editor'); + this.addMenuItem('Share', 'icon-share', 'sharePanel(); dismiss();'); this.addEditorTab('General', 'app/partials/panelgeneral.html'); From 0838f432ca598394bb648e89a68eaaa6c5a1231d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 29 Jun 2015 09:02:07 +0200 Subject: [PATCH 015/287] refactor(influxdb series handling): performance and refactoring of PR #2179, also switched to InfluxDB epoch json format --- .../plugins/datasource/influxdb/datasource.js | 2 +- .../datasource/influxdb/influxSeries.js | 36 ++++++++----------- public/test/specs/influxSeries-specs.js | 6 ++-- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/datasource.js b/public/app/plugins/datasource/influxdb/datasource.js index dd6dde89e02..cc2e798f70b 100644 --- a/public/app/plugins/datasource/influxdb/datasource.js +++ b/public/app/plugins/datasource/influxdb/datasource.js @@ -130,7 +130,7 @@ function (angular, _, kbn, InfluxSeries, InfluxQueryBuilder) { } InfluxDatasource.prototype._seriesQuery = function(query) { - return this._influxRequest('GET', '/query', {q: query}); + return this._influxRequest('GET', '/query', {q: query, epoch: 'ms'}); }; InfluxDatasource.prototype.testDatasource = function() { diff --git a/public/app/plugins/datasource/influxdb/influxSeries.js b/public/app/plugins/datasource/influxdb/influxSeries.js index 874cd4746d2..f911d758a78 100644 --- a/public/app/plugins/datasource/influxdb/influxSeries.js +++ b/public/app/plugins/datasource/influxdb/influxSeries.js @@ -15,43 +15,37 @@ function (_) { p.getTimeSeries = function() { var output = []; var self = this; + var i, j; if (self.series.length === 0) { return output; } - var field_datapoints = function(datapoints, column_index) { - return _.map(datapoints, function(datapoint) { - return [datapoint[column_index - 1], _.last(datapoint)]; - }); - }; - _.each(self.series, function(series) { - var datapoints = []; var columns = series.columns.length; - for (var i = 0; i < series.values.length; i++) { - datapoints[i] = series.values[i].slice(1); - datapoints[i].push(new Date(series.values[i][0]).getTime()); - } + var tags = _.map(series.tags, function(value, key) { + return key + ': ' + value; + }); - for (var j = 1; j < columns; j++) { + for (j = 1; j < columns; j++) { var seriesName = series.name; var columnName = series.columns[j]; + if (columnName !== 'value') { + seriesName = seriesName + '.' + columnName; + } if (self.alias) { seriesName = self._getSeriesName(series); } else if (series.tags) { - var tags = _.map(series.tags, function(value, key) { - return key + ': ' + value; - }); - if (columnName === 'value') { - seriesName = seriesName + ' {' + tags.join(', ') + '}'; - } else { - seriesName = seriesName + '.' + columnName + ' {' + tags.join(', ') + '}'; - } + seriesName = seriesName + ' {' + tags.join(', ') + '}'; } - output.push({ target: seriesName, datapoints: field_datapoints(datapoints, j)}); + var datapoints = []; + for (i = 0; i < series.values.length; i++) { + datapoints[i] = [series.values[i][j], series.values[i][0]]; + } + + output.push({ target: seriesName, datapoints: datapoints}); } }); diff --git a/public/test/specs/influxSeries-specs.js b/public/test/specs/influxSeries-specs.js index 4cff47d5385..fddb873ea35 100644 --- a/public/test/specs/influxSeries-specs.js +++ b/public/test/specs/influxSeries-specs.js @@ -11,7 +11,7 @@ define([ name: 'cpu', tags: {app: 'test', server: 'server1'}, columns: ['time', 'mean', 'max', 'min'], - values: [["2015-05-18T10:57:05Z", 10, 11, 9], ["2015-05-18T10:57:06Z", 20, 21, 19]] + values: [[1431946625000, 10, 11, 9], [1431946626000, 20, 21, 19]] } ]}; describe('and no alias', function() { @@ -101,13 +101,13 @@ define([ name: 'cpu', tags: {app: 'test', server: 'server1'}, columns: ['time', 'mean'], - values: [["2015-05-18T10:57:05Z", 10], ["2015-05-18T10:57:06Z", 12]] + values: [[1431946625000, 10], [1431946626000, 12]] }, { name: 'cpu', tags: {app: 'test2', server: 'server2'}, columns: ['time', 'mean'], - values: [["2015-05-18T10:57:05Z", 15], ["2015-05-18T10:57:06Z", 16]] + values: [[1431946625000, 15], [1431946626000, 16]] } ]}; From 11170dd34ca2a213b89e6ba0364176e450664f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 29 Jun 2015 09:22:08 +0200 Subject: [PATCH 016/287] feat(singlestat): Added support for string values, Closes #2203 --- CHANGELOG.md | 1 + public/app/panels/singlestat/module.js | 21 ++++++++++++------- .../datasource/influxdb/influxSeries.js | 6 ++++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f9b6d48f0a..06f4712d263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - [Issue #590](https://github.com/grafana/grafana/issues/590). Graph: Define series color using regex rule - [Issue #2162](https://github.com/grafana/grafana/issues/2162). Graph: New series style override, negative-y transform and stack groups - [Issue #2096](https://github.com/grafana/grafana/issues/2096). Dashboard list panel: Now supports search by multiple tags +- [Issue #2203](https://github.com/grafana/grafana/issues/2203). Singlestat: Now support string values **User or Organization admin** - [Issue #1899](https://github.com/grafana/grafana/issues/1899). Organization: You can now update the organization user role directly (without removing and readding the organization user). diff --git a/public/app/panels/singlestat/module.js b/public/app/panels/singlestat/module.js index 24855634ba6..b38912605bb 100644 --- a/public/app/panels/singlestat/module.js +++ b/public/app/panels/singlestat/module.js @@ -186,14 +186,21 @@ function (angular, app, _, TimeSeries, kbn, PanelMeta) { data.flotpairs = []; if ($scope.series && $scope.series.length > 0) { - data.value = $scope.series[0].stats[$scope.panel.valueName]; - data.flotpairs = $scope.series[0].flotpairs; - } + var lastValue = _.last($scope.series[0].datapoints)[0]; + if (_.isString(lastValue)) { + data.value = 0; + data.valueFormated = lastValue; + data.valueRounded = 0; + } else { + data.value = $scope.series[0].stats[$scope.panel.valueName]; + data.flotpairs = $scope.series[0].flotpairs; - var decimalInfo = $scope.getDecimalsForValue(data.value); - var formatFunc = kbn.valueFormats[$scope.panel.format]; - data.valueFormated = formatFunc(data.value, decimalInfo.decimals, decimalInfo.scaledDecimals); - data.valueRounded = kbn.roundValue(data.value, decimalInfo.decimals); + var decimalInfo = $scope.getDecimalsForValue(data.value); + var formatFunc = kbn.valueFormats[$scope.panel.format]; + data.valueFormated = formatFunc(data.value, decimalInfo.decimals, decimalInfo.scaledDecimals); + data.valueRounded = kbn.roundValue(data.value, decimalInfo.decimals); + } + } // check value to text mappings for(var i = 0; i < $scope.panel.valueMaps.length; i++) { diff --git a/public/app/plugins/datasource/influxdb/influxSeries.js b/public/app/plugins/datasource/influxdb/influxSeries.js index f911d758a78..43fb484e9ce 100644 --- a/public/app/plugins/datasource/influxdb/influxSeries.js +++ b/public/app/plugins/datasource/influxdb/influxSeries.js @@ -41,8 +41,10 @@ function (_) { } var datapoints = []; - for (i = 0; i < series.values.length; i++) { - datapoints[i] = [series.values[i][j], series.values[i][0]]; + if (series.values) { + for (i = 0; i < series.values.length; i++) { + datapoints[i] = [series.values[i][j], series.values[i][0]]; + } } output.push({ target: seriesName, datapoints: datapoints}); From 15e6a4266cef8820a5ab1788a93ba78c04f7832d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 29 Jun 2015 16:03:30 +0200 Subject: [PATCH 017/287] Fixed OR statement for influxdb 0.9 editor, #1525 --- .../datasource/influxdb/queryBuilder.js | 25 +++++++++++-------- .../test/specs/influx09-querybuilder-specs.js | 14 +++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/queryBuilder.js b/public/app/plugins/datasource/influxdb/queryBuilder.js index fd99a96e447..ce76e545743 100644 --- a/public/app/plugins/datasource/influxdb/queryBuilder.js +++ b/public/app/plugins/datasource/influxdb/queryBuilder.js @@ -8,11 +8,16 @@ function (_) { this.target = target; } - function renderTagCondition (key, value) { - if (value && value[0] === '/' && value[value.length - 1] === '/') { - return key + ' =~ ' + value; + function renderTagCondition (tag, index) { + var str = ""; + if (index > 0) { + str = (tag.condition || 'AND') + ' '; } - return key + " = '" + value + "'"; + + if (tag.value && tag.value[0] === '/' && tag.value[tag.value.length - 1] === '/') { + return str + tag.key + ' =~ ' + tag.value; + } + return str + tag.key + " = '" + tag.value + "'"; } var p = InfluxQueryBuilder.prototype; @@ -49,12 +54,12 @@ function (_) { if (tag.key === withKey) { return memo; } - memo.push(renderTagCondition(tag.key, tag.value)); + memo.push(renderTagCondition(tag, memo.length)); return memo; }, []); if (whereConditions.length > 0) { - query += ' WHERE ' + whereConditions.join(' AND '); + query += ' WHERE ' + whereConditions.join(' '); } } @@ -78,12 +83,12 @@ function (_) { query += aggregationFunc + '(value)'; query += ' FROM ' + measurement + ' WHERE '; - var conditions = _.map(target.tags, function(tag) { - return renderTagCondition(tag.key, tag.value); + var conditions = _.map(target.tags, function(tag, index) { + return renderTagCondition(tag, index); }); - conditions.push('$timeFilter'); - query += conditions.join(' AND '); + query += conditions.join(' '); + query += (conditions.length > 0 ? ' AND ' : '') + '$timeFilter'; query += ' GROUP BY time($interval)'; if (target.groupByTags && target.groupByTags.length > 0) { diff --git a/public/test/specs/influx09-querybuilder-specs.js b/public/test/specs/influx09-querybuilder-specs.js index c09aa71c5bb..93f1cef155a 100644 --- a/public/test/specs/influx09-querybuilder-specs.js +++ b/public/test/specs/influx09-querybuilder-specs.js @@ -52,6 +52,20 @@ define([ }); }); + describe('series with tags OR condition', function() { + var builder = new InfluxQueryBuilder({ + measurement: 'cpu', + tags: [{key: 'hostname', value: 'server1'}, {key: 'hostname', value: 'server2', condition: "OR"}] + }); + + var query = builder.build(); + + it('should generate correct query', function() { + expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE hostname = \'server1\' OR hostname = \'server2\' AND ' + + '$timeFilter GROUP BY time($interval) ORDER BY asc'); + }); + }); + describe('series with groupByTag', function() { it('should generate correct query', function() { var builder = new InfluxQueryBuilder({ From a38a06a077a36af9b4a67a2b937cf943c37dfa77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 29 Jun 2015 16:07:58 +0200 Subject: [PATCH 018/287] updated changelog with influxdb 0.9 support comment, Closes #1525 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f4712d263..fdaa12d83c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # 2.1.0 (unreleased - master branch) +**Data sources** +- [Issue #1525](https://github.com/grafana/grafana/issues/1525). InfluxDB: Full support for InfluxDB 0.9 with new adapted query editor +- [Issue #2191](https://github.com/grafana/grafana/issues/2191). KariosDB: Grafana now ships with a KariosDB data source plugin, thx @masaori335 + **New dashboard features** - [Issue #1144](https://github.com/grafana/grafana/issues/1144). Templating: You can now select multiple template variables values at the same time. - [Issue #1922](https://github.com/grafana/grafana/issues/1922). Templating: Specify multiple variable values via URL params. From df33cbc8c55ff1f1d52f62309847288234ad3b96 Mon Sep 17 00:00:00 2001 From: Donn Pebe Date: Tue, 30 Jun 2015 12:52:55 +0700 Subject: [PATCH 019/287] Fix wrong metrics counter --- pkg/api/common.go | 6 +++--- pkg/middleware/middleware.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/api/common.go b/pkg/api/common.go index 4d8b3c28032..28e95866402 100644 --- a/pkg/api/common.go +++ b/pkg/api/common.go @@ -87,10 +87,10 @@ func ApiError(status int, message string, err error) *NormalResponse { switch status { case 404: - resp["message"] = "Not Found" - metrics.M_Api_Status_500.Inc(1) - case 500: metrics.M_Api_Status_404.Inc(1) + resp["message"] = "Not Found" + case 500: + metrics.M_Api_Status_500.Inc(1) resp["message"] = "Internal Server Error" } diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 2a6873076c6..e6c2fdbea38 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -197,10 +197,10 @@ func (ctx *Context) JsonApiErr(status int, message string, err error) { switch status { case 404: - resp["message"] = "Not Found" - metrics.M_Api_Status_500.Inc(1) - case 500: metrics.M_Api_Status_404.Inc(1) + resp["message"] = "Not Found" + case 500: + metrics.M_Api_Status_500.Inc(1) resp["message"] = "Internal Server Error" } From 0d856cc135e089110e2606af6f8e9f1735617557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 30 Jun 2015 08:05:05 +0200 Subject: [PATCH 020/287] Error message for missing dashboards.json config section, Closes #2256 --- pkg/services/search/handlers.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/services/search/handlers.go b/pkg/services/search/handlers.go index 326924f05f4..1c480992cbc 100644 --- a/pkg/services/search/handlers.go +++ b/pkg/services/search/handlers.go @@ -1,6 +1,7 @@ package search import ( + "log" "path/filepath" "sort" @@ -15,6 +16,12 @@ func Init() { bus.AddHandler("search", searchHandler) jsonIndexCfg, _ := setting.Cfg.GetSection("dashboards.json") + + if jsonIndexCfg == nil { + log.Fatal("Config section missing: dashboards.json") + return + } + jsonIndexEnabled := jsonIndexCfg.Key("enabled").MustBool(false) if jsonIndexEnabled { From d0e7d53c691e881601cd1437139636ae294ee3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 30 Jun 2015 08:17:06 +0200 Subject: [PATCH 021/287] Fixed case insensitive search for file based dashboards, Fixes #2258 --- pkg/services/search/json_index.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/services/search/json_index.go b/pkg/services/search/json_index.go index a0fc02343e2..e70c662438d 100644 --- a/pkg/services/search/json_index.go +++ b/pkg/services/search/json_index.go @@ -51,13 +51,15 @@ func (index *JsonDashIndex) Search(query *Query) ([]*Hit, error) { return results, nil } + queryStr := strings.ToLower(query.Title) + for _, item := range index.items { if len(results) > query.Limit { break } // add results with matchig title filter - if strings.Contains(item.TitleLower, query.Title) { + if strings.Contains(item.TitleLower, queryStr) { results = append(results, &Hit{ Type: DashHitJson, Title: item.Dashboard.Title, From ae0f8c77d16840bf4526cab68da54460b043db9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 30 Jun 2015 09:37:52 +0200 Subject: [PATCH 022/287] Auth: You can now authenicate against api with username / password using basic auth, Closes #2218 --- CHANGELOG.md | 1 + conf/defaults.ini | 4 +++ conf/sample.ini | 4 +++ pkg/middleware/middleware.go | 43 +++++++++++++++++++++++++++++++ pkg/middleware/middleware_test.go | 36 ++++++++++++++++++++++++++ pkg/setting/setting.go | 6 +++++ pkg/util/encoding.go | 22 ++++++++++++++++ pkg/util/encoding_test.go | 10 +++++++ 8 files changed, 126 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdaa12d83c9..e4902751245 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - [Issue #2088](https://github.com/grafana/grafana/issues/2088). Roles: New user role `Read Only Editor` that replaces the old `Viewer` role behavior **Backend** +- [Issue #2218](https://github.com/grafana/grafana/issues/2218). Auth: You can now authenicate against api with username / password using basic auth - [Issue #2095](https://github.com/grafana/grafana/issues/2095). Search: Search now supports filtering by multiple dashboard tags - [Issue #1905](https://github.com/grafana/grafana/issues/1905). Github OAuth: You can now configure a Github team membership requirement, thx @dewski - [Issue #2052](https://github.com/grafana/grafana/issues/2052). Github OAuth: You can now configure a Github organization requirement, thx @indrekj diff --git a/conf/defaults.ini b/conf/defaults.ini index e9d29bccac9..d6d81c8028c 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -168,6 +168,10 @@ token_url = https://accounts.google.com/o/oauth2/token api_url = https://www.googleapis.com/oauth2/v1/userinfo allowed_domains = +#################################### Basic Auth ########################## +[auth.basic] +enabled = true + #################################### Auth Proxy ########################## [auth.proxy] enabled = false diff --git a/conf/sample.ini b/conf/sample.ini index d3e122fbfd5..ef082ccff98 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -174,6 +174,10 @@ ;header_property = username ;auto_sign_up = true +#################################### Basic Auth ########################## +[auth.basic] +;enabled = true + #################################### SMTP / Emailing ########################## [smtp] ;enabled = false diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index e6c2fdbea38..8704ec5a787 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/grafana/pkg/metrics" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" ) type Context struct { @@ -40,6 +41,7 @@ func GetContextHandler() macaron.Handler { // then look for api key in session (special case for render calls via api) // then test if anonymous access is enabled if initContextWithApiKey(ctx) || + initContextWithBasicAuth(ctx) || initContextWithAuthProxy(ctx) || initContextWithUserSessionCookie(ctx) || initContextWithApiKeyFromSession(ctx) || @@ -128,6 +130,47 @@ func initContextWithApiKey(ctx *Context) bool { } } +func initContextWithBasicAuth(ctx *Context) bool { + if !setting.BasicAuthEnabled { + return false + } + + header := ctx.Req.Header.Get("Authorization") + if header == "" { + return false + } + + username, password, err := util.DecodeBasicAuthHeader(header) + if err != nil { + ctx.JsonApiErr(401, "Invalid Basic Auth Header", err) + return true + } + + loginQuery := m.GetUserByLoginQuery{LoginOrEmail: username} + if err := bus.Dispatch(&loginQuery); err != nil { + ctx.JsonApiErr(401, "Basic auth failed", err) + return true + } + + user := loginQuery.Result + + // validate password + if util.EncodePassword(password, user.Salt) != user.Password { + ctx.JsonApiErr(401, "Invalid username or password", nil) + return true + } + + query := m.GetSignedInUserQuery{UserId: user.Id} + if err := bus.Dispatch(&query); err != nil { + ctx.JsonApiErr(401, "Authentication error", err) + return true + } else { + ctx.SignedInUser = query.Result + ctx.IsSignedIn = true + return true + } +} + // special case for panel render calls with api key func initContextWithApiKeyFromSession(ctx *Context) bool { keyId := ctx.Session.Get(SESS_KEY_APIKEY) diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index 212e250cc1c..97d369d00cf 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -48,6 +48,32 @@ func TestMiddlewareContext(t *testing.T) { }) }) + middlewareScenario("Using basic auth", func(sc *scenarioContext) { + + bus.AddHandler("test", func(query *m.GetUserByLoginQuery) error { + query.Result = &m.User{ + Password: util.EncodePassword("myPass", "salt"), + Salt: "salt", + } + return nil + }) + + bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { + query.Result = &m.SignedInUser{OrgId: 2, UserId: 12} + return nil + }) + + setting.BasicAuthEnabled = true + authHeader := util.GetBasicAuthHeader("myUser", "myPass") + sc.fakeReq("GET", "/").withAuthoriziationHeader(authHeader).exec() + + Convey("Should init middleware context with user", func() { + So(sc.context.IsSignedIn, ShouldEqual, true) + So(sc.context.OrgId, ShouldEqual, 2) + So(sc.context.UserId, ShouldEqual, 12) + }) + }) + middlewareScenario("Valid api key", func(sc *scenarioContext) { keyhash := util.EncodePassword("v5nAwpMafFP6znaS4urhdWDLS5511M42", "asd") @@ -223,6 +249,7 @@ type scenarioContext struct { context *Context resp *httptest.ResponseRecorder apiKey string + authHeader string respJson map[string]interface{} handlerFunc handlerFunc defaultHandler macaron.Handler @@ -240,6 +267,11 @@ func (sc *scenarioContext) withInvalidApiKey() *scenarioContext { return sc } +func (sc *scenarioContext) withAuthoriziationHeader(authHeader string) *scenarioContext { + sc.authHeader = authHeader + return sc +} + func (sc *scenarioContext) fakeReq(method, url string) *scenarioContext { sc.resp = httptest.NewRecorder() req, err := http.NewRequest(method, url, nil) @@ -266,6 +298,10 @@ func (sc *scenarioContext) exec() { sc.req.Header.Add("Authorization", "Bearer "+sc.apiKey) } + if sc.authHeader != "" { + sc.req.Header.Add("Authorization", sc.authHeader) + } + sc.m.ServeHTTP(sc.resp, sc.req) if sc.resp.Header().Get("Content-Type") == "application/json; charset=UTF-8" { diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index b015062fa9b..ddca37c41ff 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -94,6 +94,9 @@ var ( AuthProxyHeaderProperty string AuthProxyAutoSignUp bool + // Basic Auth + BasicAuthEnabled bool + // Session settings. SessionOptions session.Options @@ -398,6 +401,9 @@ func NewConfigContext(args *CommandLineArgs) { AuthProxyHeaderProperty = authProxy.Key("header_property").String() AuthProxyAutoSignUp = authProxy.Key("auto_sign_up").MustBool(true) + authBasic := Cfg.Section("auth.basic") + AuthProxyEnabled = authBasic.Key("enabled").MustBool(true) + // PhantomJS rendering ImagesDir = filepath.Join(DataPath, "png") PhantomDir = filepath.Join(HomePath, "vendor/phantomjs") diff --git a/pkg/util/encoding.go b/pkg/util/encoding.go index 27169133a42..e87da9d3d55 100644 --- a/pkg/util/encoding.go +++ b/pkg/util/encoding.go @@ -7,8 +7,10 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "errors" "fmt" "hash" + "strings" ) // source: https://github.com/gogits/gogs/blob/9ee80e3e5426821f03a4e99fad34418f5c736413/modules/base/tool.go#L58 @@ -80,3 +82,23 @@ func GetBasicAuthHeader(user string, password string) string { var userAndPass = user + ":" + password return "Basic " + base64.StdEncoding.EncodeToString([]byte(userAndPass)) } + +func DecodeBasicAuthHeader(header string) (string, string, error) { + var code string + parts := strings.SplitN(header, " ", 2) + if len(parts) == 2 && parts[0] == "Basic" { + code = parts[1] + } + + decoded, err := base64.StdEncoding.DecodeString(code) + if err != nil { + return "", "", err + } + + userAndPass := strings.SplitN(string(decoded), ":", 2) + if len(userAndPass) != 2 { + return "", "", errors.New("Invalid basic auth header") + } + + return userAndPass[0], userAndPass[1], nil +} diff --git a/pkg/util/encoding_test.go b/pkg/util/encoding_test.go index afe299f9f92..abcf5425826 100644 --- a/pkg/util/encoding_test.go +++ b/pkg/util/encoding_test.go @@ -13,4 +13,14 @@ func TestEncoding(t *testing.T) { So(result, ShouldEqual, "Basic Z3JhZmFuYToxMjM0") }) + + Convey("When decoding basic auth header", t, func() { + header := GetBasicAuthHeader("grafana", "1234") + username, password, err := DecodeBasicAuthHeader(header) + So(err, ShouldBeNil) + + So(username, ShouldEqual, "grafana") + So(password, ShouldEqual, "1234") + }) + } From aedaae852b34da03083c63d9cad97fe5c894f301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 30 Jun 2015 12:14:13 +0200 Subject: [PATCH 023/287] Fixed minor mistake in last commit --- pkg/setting/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index ddca37c41ff..d0f28fb809d 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -402,7 +402,7 @@ func NewConfigContext(args *CommandLineArgs) { AuthProxyAutoSignUp = authProxy.Key("auto_sign_up").MustBool(true) authBasic := Cfg.Section("auth.basic") - AuthProxyEnabled = authBasic.Key("enabled").MustBool(true) + BasicAuthEnabled = authBasic.Key("enabled").MustBool(true) // PhantomJS rendering ImagesDir = filepath.Join(DataPath, "png") From 97f54ac3853b7567da0d0786ff6c70be2a0f5c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 30 Jun 2015 12:27:42 +0200 Subject: [PATCH 024/287] Small fix to PR 2119 --- public/app/plugins/datasource/opentsdb/datasource.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index 57559cc6687..d244b56436e 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -91,11 +91,10 @@ function (angular, _, kbn) { }; OpenTSDBDatasource.prototype.performMetricKeyValueLookup = function(metric, key) { - if(metric === "") { - throw "Metric not set."; - } else if(key === "") { - throw "Key not set."; + if(!metric || !key) { + return $q.when([]); } + var m = metric + "{" + key + "=*}"; var options = { method: 'GET', @@ -104,6 +103,7 @@ function (angular, _, kbn) { m: m, } }; + return backendSrv.datasourceRequest(options).then(function(result) { result = result.data.results; var tagvs = []; From f7b7401a53467f1643eb4619ecc1b5e788145651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 30 Jun 2015 12:34:57 +0200 Subject: [PATCH 025/287] Updated changelog with new OpenTSDB enhancement, #1177 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4902751245..15bfe325b0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ **Data sources** - [Issue #1525](https://github.com/grafana/grafana/issues/1525). InfluxDB: Full support for InfluxDB 0.9 with new adapted query editor - [Issue #2191](https://github.com/grafana/grafana/issues/2191). KariosDB: Grafana now ships with a KariosDB data source plugin, thx @masaori335 +- [Issue #1177](https://github.com/grafana/grafana/issues/1177). OpenTSDB: Limit tags by metric, OpenTSDB config option tsd.core.meta.enable_realtime_ts must enabled for OpenTSDB lookup api **New dashboard features** - [Issue #1144](https://github.com/grafana/grafana/issues/1144). Templating: You can now select multiple template variables values at the same time. From e508db994eec1e2b24493fd0a7d4397d0d2c0439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 30 Jun 2015 15:32:31 +0200 Subject: [PATCH 026/287] OpenTSDB: Support for template variable values lookup queries, Closes #1250 --- CHANGELOG.md | 1 + .../plugins/datasource/opentsdb/datasource.js | 82 ++++++++++++------- .../plugins/datasource/opentsdb/queryCtrl.js | 16 ++-- public/test/specs/opentsdbDatasource-specs.js | 52 ++++++++++++ public/test/test-main.js | 1 + 5 files changed, 115 insertions(+), 37 deletions(-) create mode 100644 public/test/specs/opentsdbDatasource-specs.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 15bfe325b0c..c046d3364cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [Issue #1525](https://github.com/grafana/grafana/issues/1525). InfluxDB: Full support for InfluxDB 0.9 with new adapted query editor - [Issue #2191](https://github.com/grafana/grafana/issues/2191). KariosDB: Grafana now ships with a KariosDB data source plugin, thx @masaori335 - [Issue #1177](https://github.com/grafana/grafana/issues/1177). OpenTSDB: Limit tags by metric, OpenTSDB config option tsd.core.meta.enable_realtime_ts must enabled for OpenTSDB lookup api +- [Issue #1250](https://github.com/grafana/grafana/issues/1250). OpenTSDB: Support for template variable values lookup queries **New dashboard features** - [Issue #1144](https://github.com/grafana/grafana/issues/1144). Templating: You can now select multiple template variables values at the same time. diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index d244b56436e..80d51ed01c2 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -76,35 +76,20 @@ function (angular, _, kbn) { return backendSrv.datasourceRequest(options); }; - OpenTSDBDatasource.prototype.performSuggestQuery = function(query, type) { - var options = { - method: 'GET', - url: this.url + '/api/suggest', - params: { - type: type, - q: query - } - }; - return backendSrv.datasourceRequest(options).then(function(result) { + OpenTSDBDatasource.prototype._performSuggestQuery = function(query) { + return this._get('/api/suggest', {type: 'metrics', q: query}).then(function(result) { return result.data; }); }; - OpenTSDBDatasource.prototype.performMetricKeyValueLookup = function(metric, key) { + OpenTSDBDatasource.prototype._performMetricKeyValueLookup = function(metric, key) { if(!metric || !key) { return $q.when([]); } var m = metric + "{" + key + "=*}"; - var options = { - method: 'GET', - url: this.url + '/api/search/lookup', - params: { - m: m, - } - }; - return backendSrv.datasourceRequest(options).then(function(result) { + return this._get('/api/search/lookup', {m: m}).then(function(result) { result = result.data.results; var tagvs = []; _.each(result, function(r) { @@ -114,18 +99,10 @@ function (angular, _, kbn) { }); }; - OpenTSDBDatasource.prototype.performMetricKeyLookup = function(metric) { - if(metric === "") { - throw "Metric not set."; - } - var options = { - method: 'GET', - url: this.url + '/api/search/lookup', - params: { - m: metric, - } - }; - return backendSrv.datasourceRequest(options).then(function(result) { + OpenTSDBDatasource.prototype._performMetricKeyLookup = function(metric) { + if(!metric) { return $q.when([]); } + + return this._get('/api/search/lookup', {m: metric}).then(function(result) { result = result.data.results; var tagks = []; _.each(result, function(r) { @@ -139,6 +116,49 @@ function (angular, _, kbn) { }); }; + OpenTSDBDatasource.prototype._get = function(relativeUrl, params) { + return backendSrv.datasourceRequest({ + method: 'GET', + url: this.url + relativeUrl, + params: params, + }); + }; + + OpenTSDBDatasource.prototype.metricFindQuery = function(query) { + var interpolated; + try { + interpolated = templateSrv.replace(query); + } + catch (err) { + return $q.reject(err); + } + + var responseTransform = function(result) { + return _.map(result, function(value) { + return {text: value}; + }); + }; + + var metrics_regex = /metrics\((.*)\)/; + var tag_names_regex = /tag_names\((.*)\)/; + var tag_values_regex = /tag_values\((\w+),\s?(\w+)/; + + var metrics_query = interpolated.match(metrics_regex); + if (metrics_query) { + return this._performSuggestQuery(metrics_query[1]).then(responseTransform); + } + var tag_names_query = interpolated.match(tag_names_regex); + + if (tag_names_query) { + return this._performMetricKeyLookup(tag_names_query[1]).then(responseTransform); + } + + var tag_values_query = interpolated.match(tag_values_regex); + if (tag_values_query) { + return this._performMetricKeyValueLookup(tag_values_query[1], tag_values_query[2]).then(responseTransform); + } + }; + OpenTSDBDatasource.prototype.testDatasource = function() { return this.performSuggestQuery('cpu', 'metrics').then(function () { return { status: "success", message: "Data source is working", title: "Success" }; diff --git a/public/app/plugins/datasource/opentsdb/queryCtrl.js b/public/app/plugins/datasource/opentsdb/queryCtrl.js index 17a2f4bd640..0f6ad54eed3 100644 --- a/public/app/plugins/datasource/opentsdb/queryCtrl.js +++ b/public/app/plugins/datasource/opentsdb/queryCtrl.js @@ -42,21 +42,25 @@ function (angular, _, kbn) { $scope.panel.targets.push(clone); }; + $scope.getTextValues = function(metricFindResult) { + return _.map(metricFindResult, function(value) { return value.text; }); + }; + $scope.suggestMetrics = function(query, callback) { - $scope.datasource - .performSuggestQuery(query, 'metrics') + $scope.datasource.metricFindQuery('metrics()') + .then($scope.getTextValues) .then(callback); }; $scope.suggestTagKeys = function(query, callback) { - $scope.datasource - .performMetricKeyLookup($scope.target.metric) + $scope.datasource.metricFindQuery('tag_names(' + $scope.target.metric + ')') + .then($scope.getTextValues) .then(callback); }; $scope.suggestTagValues = function(query, callback) { - $scope.datasource - .performMetricKeyValueLookup($scope.target.metric, $scope.target.currentTagKey) + $scope.datasource.metricFindQuery('tag_names(' + $scope.target.metric + ',' + $scope.target.currentTagKey + ')') + .then($scope.getTextValues) .then(callback); }; diff --git a/public/test/specs/opentsdbDatasource-specs.js b/public/test/specs/opentsdbDatasource-specs.js new file mode 100644 index 00000000000..ace7e21292a --- /dev/null +++ b/public/test/specs/opentsdbDatasource-specs.js @@ -0,0 +1,52 @@ +define([ + 'helpers', + 'plugins/datasource/opentsdb/datasource' +], function(helpers) { + 'use strict'; + + describe('opentsdb', function() { + var ctx = new helpers.ServiceTestContext(); + + beforeEach(module('grafana.services')); + beforeEach(ctx.providePhase(['backendSrv'])); + + beforeEach(ctx.createService('OpenTSDBDatasource')); + beforeEach(function() { + ctx.ds = new ctx.service({ url: [''] }); + }); + + describe('When performing metricFindQuery', function() { + var results; + var requestOptions; + + beforeEach(function() { + ctx.backendSrv.datasourceRequest = function(options) { + requestOptions = options; + return ctx.$q.when({data: [{ target: 'prod1.count', datapoints: [[10, 1], [12,1]] }]}); + }; + }); + + it('metrics() should generate api suggest query', function() { + ctx.ds.metricFindQuery('metrics()').then(function(data) { results = data; }); + ctx.$rootScope.$apply(); + expect(requestOptions.url).to.be('/api/suggest'); + }); + + it('tag_names(cpu) should generate looku query', function() { + ctx.ds.metricFindQuery('tag_names(cpu)').then(function(data) { results = data; }); + ctx.$rootScope.$apply(); + expect(requestOptions.url).to.be('/api/search/lookup'); + expect(requestOptions.params.m).to.be('cpu'); + }); + + it('tag_values(cpu, test) should generate looku query', function() { + ctx.ds.metricFindQuery('tag_values(cpu, hostname)').then(function(data) { results = data; }); + ctx.$rootScope.$apply(); + expect(requestOptions.url).to.be('/api/search/lookup'); + expect(requestOptions.params.m).to.be('cpu{hostname=*}'); + }); + + }); + }); +}); + diff --git a/public/test/test-main.js b/public/test/test-main.js index 5d761b8d3ea..e6e18512216 100644 --- a/public/test/test-main.js +++ b/public/test/test-main.js @@ -146,6 +146,7 @@ require([ 'specs/dynamicDashboardSrv-specs', 'specs/unsavedChangesSrv-specs', 'specs/valueSelectDropdown-specs', + 'specs/opentsdbDatasource-specs', ]; var pluginSpecs = (config.plugins.specs || []).map(function (spec) { From e8c65dc384b0097737681cd85eea31c7b5358eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 30 Jun 2015 15:37:59 +0200 Subject: [PATCH 027/287] Updated OpenTSDB docs --- docs/sources/datasources/opentsdb.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/sources/datasources/opentsdb.md b/docs/sources/datasources/opentsdb.md index 4e85bdc9555..3d91559269d 100644 --- a/docs/sources/datasources/opentsdb.md +++ b/docs/sources/datasources/opentsdb.md @@ -27,6 +27,14 @@ Open a graph in edit mode by click the title. ![](/img/v2/opentsdb_query_editor.png) +## Templating queries + +When using OpenTSDB with a template variable of `query` type you can use following syntax for lookup. + + metrics() // returns metric names + tag_names(cpu) // return tag names (i.e. keys) for a specific cpu metric + tag_values(cpu, hostname) // return tag values for metric cpu and tag key hostname + For details on opentsdb metric queries checkout the official [OpenTSDB documentation](http://opentsdb.net/docs/build/html/index.html) From 8ed0df64c7a4a8feee2482eb9eb0bffd7bc1983d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 1 Jul 2015 08:56:47 +0200 Subject: [PATCH 028/287] Fixes to OpenTSDB datasource, and added support for repeat panels and repeat rows --- .../plugins/datasource/opentsdb/datasource.js | 38 ++++++++++--------- .../plugins/datasource/opentsdb/queryCtrl.js | 2 +- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index 80d51ed01c2..fe6bc5a07db 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -27,7 +27,7 @@ function (angular, _, kbn) { var qs = []; _.each(options.targets, function(target) { - qs.push(convertTargetToQuery(target, options.interval)); + qs.push(convertTargetToQuery(target, options)); }); var queries = _.compact(qs); @@ -47,10 +47,10 @@ function (angular, _, kbn) { }); return this.performTimeSeriesQuery(queries, start, end).then(function(response) { - var metricToTargetMapping = mapMetricsToTargets(response.data, options.targets); + var metricToTargetMapping = mapMetricsToTargets(response.data, options); var result = _.map(response.data, function(metricData, index) { index = metricToTargetMapping[index]; - return transformMetricData(metricData, groupByTags, options.targets[index]); + return transformMetricData(metricData, groupByTags, options.targets[index], options); }); return { data: result }; }); @@ -125,6 +125,8 @@ function (angular, _, kbn) { }; OpenTSDBDatasource.prototype.metricFindQuery = function(query) { + if (!query) { return $q.when([]); } + var interpolated; try { interpolated = templateSrv.replace(query); @@ -147,8 +149,8 @@ function (angular, _, kbn) { if (metrics_query) { return this._performSuggestQuery(metrics_query[1]).then(responseTransform); } - var tag_names_query = interpolated.match(tag_names_regex); + var tag_names_query = interpolated.match(tag_names_regex); if (tag_names_query) { return this._performMetricKeyLookup(tag_names_query[1]).then(responseTransform); } @@ -157,6 +159,8 @@ function (angular, _, kbn) { if (tag_values_query) { return this._performMetricKeyValueLookup(tag_values_query[1], tag_values_query[2]).then(responseTransform); } + + return $q.when([]); }; OpenTSDBDatasource.prototype.testDatasource = function() { @@ -165,8 +169,8 @@ function (angular, _, kbn) { }); }; - function transformMetricData(md, groupByTags, options) { - var metricLabel = createMetricLabel(md, options, groupByTags); + function transformMetricData(md, groupByTags, target, options) { + var metricLabel = createMetricLabel(md, target, groupByTags, options); var dps = []; // TSDB returns datapoints has a hash of ts => value. @@ -178,13 +182,13 @@ function (angular, _, kbn) { return { target: metricLabel, datapoints: dps }; } - function createMetricLabel(md, options, groupByTags) { - if (!_.isUndefined(options) && options.alias) { - var scopedVars = {}; + function createMetricLabel(md, target, groupByTags, options) { + if (target.alias) { + var scopedVars = _.clone(options.scopedVars || {}); _.each(md.tags, function(value, key) { scopedVars['tag_' + key] = {value: value}; }); - return templateSrv.replace(options.alias, scopedVars); + return templateSrv.replace(target.alias, scopedVars); } var label = md.metric; @@ -205,13 +209,13 @@ function (angular, _, kbn) { return label; } - function convertTargetToQuery(target, interval) { + function convertTargetToQuery(target, options) { if (!target.metric || target.hide) { return null; } var query = { - metric: templateSrv.replace(target.metric), + metric: templateSrv.replace(target.metric, options.scopedVars), aggregator: "avg" }; @@ -235,7 +239,7 @@ function (angular, _, kbn) { } if (!target.disableDownsampling) { - interval = templateSrv.replace(target.downsampleInterval || interval); + var interval = templateSrv.replace(target.downsampleInterval || options.interval); if (interval.match(/\.[0-9]+s/)) { interval = parseFloat(interval)*1000 + "ms"; @@ -247,20 +251,20 @@ function (angular, _, kbn) { query.tags = angular.copy(target.tags); if(query.tags){ for(var key in query.tags){ - query.tags[key] = templateSrv.replace(query.tags[key]); + query.tags[key] = templateSrv.replace(query.tags[key], options.scopedVars); } } return query; } - function mapMetricsToTargets(metrics, targets) { + function mapMetricsToTargets(metrics, options) { var interpolatedTagValue; return _.map(metrics, function(metricData) { - return _.findIndex(targets, function(target) { + return _.findIndex(options.targets, function(target) { return target.metric === metricData.metric && _.all(target.tags, function(tagV, tagK) { - interpolatedTagValue = templateSrv.replace(tagV); + interpolatedTagValue = templateSrv.replace(tagV, options.scopedVars); return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*"; }); }); diff --git a/public/app/plugins/datasource/opentsdb/queryCtrl.js b/public/app/plugins/datasource/opentsdb/queryCtrl.js index 0f6ad54eed3..be8ff2aa06c 100644 --- a/public/app/plugins/datasource/opentsdb/queryCtrl.js +++ b/public/app/plugins/datasource/opentsdb/queryCtrl.js @@ -59,7 +59,7 @@ function (angular, _, kbn) { }; $scope.suggestTagValues = function(query, callback) { - $scope.datasource.metricFindQuery('tag_names(' + $scope.target.metric + ',' + $scope.target.currentTagKey + ')') + $scope.datasource.metricFindQuery('tag_values(' + $scope.target.metric + ',' + $scope.target.currentTagKey + ')') .then($scope.getTextValues) .then(callback); }; From 5494be44938fee9338db6a33d8d5518f83a664a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 2 Jul 2015 10:04:15 +0200 Subject: [PATCH 029/287] Some fixes related to OpenTSDB enhancements, #1250 --- public/app/plugins/datasource/opentsdb/datasource.js | 1 + .../app/plugins/datasource/opentsdb/partials/query.editor.html | 2 +- public/app/plugins/datasource/opentsdb/queryCtrl.js | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index fe6bc5a07db..baf05d8d0dc 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -27,6 +27,7 @@ function (angular, _, kbn) { var qs = []; _.each(options.targets, function(target) { + if (!target.metric) { return; } qs.push(convertTargetToQuery(target, options)); }); diff --git a/public/app/plugins/datasource/opentsdb/partials/query.editor.html b/public/app/plugins/datasource/opentsdb/partials/query.editor.html index a5478ff0cc3..0e456a29b8b 100644 --- a/public/app/plugins/datasource/opentsdb/partials/query.editor.html +++ b/public/app/plugins/datasource/opentsdb/partials/query.editor.html @@ -55,7 +55,7 @@ placeholder="metric name" data-min-length=0 data-items=100 ng-model-onblur - ng-blur="targetBlur()" + ng-change="targetBlur()" > Date: Thu, 2 Jul 2015 17:13:12 +0200 Subject: [PATCH 030/287] Updated influxdb docs, adding docs specific to influxdb 0.9, #2270 --- docs/sources/datasources/influxdb.md | 61 ++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/docs/sources/datasources/influxdb.md b/docs/sources/datasources/influxdb.md index 04fbd520a7e..9feca2e4a10 100644 --- a/docs/sources/datasources/influxdb.md +++ b/docs/sources/datasources/influxdb.md @@ -4,10 +4,11 @@ page_description: InfluxDB query guide page_keywords: grafana, influxdb, metrics, query, documentation --- - # InfluxDB -There are currently two separate datasources for InfluxDB in Grafana: InfluxDB 0.8.x and InfluxDB 0.9.x. The API and capabilities of InfluxDB 0.9.x are completely different from InfluxDB 0.8.x. InfluxDB 0.9.x data source support is provided on an experimental basis. +There are currently two separate datasources for InfluxDB in Grafana: InfluxDB 0.8.x and InfluxDB 0.9.x. +The API and capabilities of InfluxDB 0.9.x are completely different from InfluxDB 0.8.x which is why Grafana handles +them as different data sources. ## Adding the data source to Grafana Open the side menu by clicking the the Grafana icon in the top header. In the side menu under the `Dashboards` link you @@ -31,32 +32,66 @@ Password | Database user's password > *Note* When using Proxy access mode the InfluxDB database, user and password will be hidden from the browser/frontend. When > using direct access mode all users will be able to see the database user & password. -## InfluxDB 0.9.x query editor +## InfluxDB 0.9.x -This editor & data source is not compatible with InfluxDB 0.8.x, please use the right data source for you InfluxDB version. -The InfluxDB 0.9.x editor is currently under development and is not yet fully usable. +![](/img/influxdb/InfluxDB_09_editor.png) -## InfluxDB 0.8.x query editor +You find the InfluxDB editor in the metrics tab in Graph or Singlestat panel's edit mode. You enter edit mode by clicking the +panel title, then edit. The editor allows you to select metrics and tags. -![](/img/v1/influxdb_editor.png) +### Editor tag filters +To add a tag filter click the plus icon to the right of the `WHERE` condition. You can remove tag filters by clicking on +the tag key and select `--remove tag filter--`. -When you add an InfluxDB query you can specify series name (can be regex), value column and a function. Group by time can be specified or if left blank will be automatically set depending on how long the current time span is. It will translate to a InfluxDB query that looks like this: +### Editor group by +To group by a tag click the plus icon after the `GROUP BY ($interval)` text. Pick a tag from the dropdown that appears. +You can remove the group by by clicking on the tag and then select `--remove group by--` from the dropdown. +### Editor RAW Query +You can switch to raw query mode by pressing the pen icon. + +> If you use Raw Query be sure your query at minimum have `WHERE $timeFilter` clause and ends with `order by asc`. +> Also please always have a group by time and an aggregation function, otherwise InfluxDB can easily return hundreds of thousands +> of data points that will hang the browser. + +### Alias patterns + +- $m = replaced with measurement name +- $measurement = replaced with measurement name +- $tag_hostname = replaced with the value of the hostname tag +- You can also use [[tag_hostname]] pattern replacement syntax + +### Templating +You can create a template variable in Grafana and have that variable filled with values from any InfluxDB metric exploration query. +You can then use this variable in your InfluxDB metric queries. + +For example you can have a variable that contains all values for tag `hostname` if you specify a query like this +in the templating edit view. ```sql -select [[func]]([[column]]) from [[series]] where [[timeFilter]] group by time([[interval]]) order asc +SHOW TAG VALUES WITH KEY = "hostname" ``` -To write the complete query yourself click the cog wheel icon to the right and select ``Raw query mode``. +You can also create nested variables. For example if you had another variable, for example `region`. Then you could have +the hosts variable only show hosts from the current selected region with a query like this: -## InfluxDB 0.9 Filters & Templates queries +```sql +SHOW TAG VALUES WITH KEY = "hostname" WHERE region =~ /$region/ +``` -The InfluxDB 0.9 data source does not currently support filters or templates. +> Always you `regex values` or `regex wildcard` for All format or multi select format. + +![](/img/influxdb/templating_simple_ex1.png) + +### Annotations + +### InfluxDB 0.8.x + +![](/img/v1/influxdb_editor.png) ## InfluxDB 0.8 Filters & Templated queries ![](/img/animated_gifs/influxdb_templated_query.gif) - Use a distinct influxdb query in the filter query input box: ```sql From 7a030e0c3b56e5428b1ff44a5c6061701bea660e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 3 Jul 2015 15:09:46 +0200 Subject: [PATCH 031/287] Finished improving InfluxDB docs, Closes #2270 --- docs/sources/datasources/influxdb.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/sources/datasources/influxdb.md b/docs/sources/datasources/influxdb.md index 9feca2e4a10..05e627967eb 100644 --- a/docs/sources/datasources/influxdb.md +++ b/docs/sources/datasources/influxdb.md @@ -43,6 +43,10 @@ panel title, then edit. The editor allows you to select metrics and tags. To add a tag filter click the plus icon to the right of the `WHERE` condition. You can remove tag filters by clicking on the tag key and select `--remove tag filter--`. +### Regex matching +You can type in regex patterns for metric names or tag filter values, be sure to wrap the regex pattern in forward slashes (`/`). Grafana +will automaticallay adjust the filter tag condition to use the InfluxDB regex match condition operator (`=~`). + ### Editor group by To group by a tag click the plus icon after the `GROUP BY ($interval)` text. Pick a tag from the dropdown that appears. You can remove the group by by clicking on the tag and then select `--remove group by--` from the dropdown. @@ -83,20 +87,18 @@ SHOW TAG VALUES WITH KEY = "hostname" WHERE region =~ /$region/ ![](/img/influxdb/templating_simple_ex1.png) ### Annotations +Annotations allows you to overlay rich event information on top of graphs. + +An example query: + +```SQL +SELECT title, description from events WHERE $timeFilter order asc +``` ### InfluxDB 0.8.x ![](/img/v1/influxdb_editor.png) -## InfluxDB 0.8 Filters & Templated queries - -![](/img/animated_gifs/influxdb_templated_query.gif) - -Use a distinct influxdb query in the filter query input box: - -```sql -select distinct(host) from app.status -``` From fc81cda971e2e3a930b9b479cba03fd65149cd5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 3 Jul 2015 15:13:06 +0200 Subject: [PATCH 032/287] Updated OpenTSDB docs, Closes #2272 --- docs/sources/datasources/opentsdb.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/sources/datasources/opentsdb.md b/docs/sources/datasources/opentsdb.md index 3d91559269d..e9110418f4c 100644 --- a/docs/sources/datasources/opentsdb.md +++ b/docs/sources/datasources/opentsdb.md @@ -27,6 +27,10 @@ Open a graph in edit mode by click the title. ![](/img/v2/opentsdb_query_editor.png) +### Auto complete suggestions +You should get auto complete suggestions for tags and tag values. If you do not you need to enable `tsd.core.meta.enable_realtime_ts` in +the OpentSDB server settings. This is required for the OpenTSDB `lookup` api to work. + ## Templating queries When using OpenTSDB with a template variable of `query` type you can use following syntax for lookup. From 4a9949e643c84e89ede4cb4da49b17ade367413d Mon Sep 17 00:00:00 2001 From: gcormier9 Date: Fri, 3 Jul 2015 15:57:46 -0400 Subject: [PATCH 033/287] Support dot (.) in metric name I'd like to update the regex in order to support dot in metric name. For example "cpu.usage.average". --- public/app/plugins/datasource/opentsdb/datasource.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index baf05d8d0dc..27d4f001b0b 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -144,7 +144,7 @@ function (angular, _, kbn) { var metrics_regex = /metrics\((.*)\)/; var tag_names_regex = /tag_names\((.*)\)/; - var tag_values_regex = /tag_values\((\w+),\s?(\w+)/; + var tag_values_regex = /tag_values\((.*),\s?(.*)\)/; var metrics_query = interpolated.match(metrics_regex); if (metrics_query) { From 582ebf25f9446177f94919905034fb39dd8fe757 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Fri, 3 Jul 2015 17:27:55 -0700 Subject: [PATCH 034/287] fix config flag --- docs/sources/installation/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index add1999f714..cd4b4d20276 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -18,7 +18,7 @@ specified in a `.ini` configuration file or specified using environment variable > **Note.** If you have installed Grafana using the `deb` or `rpm` > packages, then your configuration file is located at > `/etc/grafana/grafana.ini`. This path is specified in the Grafana -> init.d script using `--config` file parameter. +> init.d script using `-config` file parameter. ## Using environment variables From 7f602feff4965f3a39228372a8905732d0f79b49 Mon Sep 17 00:00:00 2001 From: Andrew Widdersheim Date: Sun, 5 Jul 2015 21:19:12 -0400 Subject: [PATCH 035/287] Allow pipe ('|') in Graphite lexer Allow for Graphite lexer to understand metric names like 'net|usage_average'. --- public/app/plugins/datasource/graphite/lexer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/plugins/datasource/graphite/lexer.js b/public/app/plugins/datasource/graphite/lexer.js index ffb65121a60..457100b163e 100644 --- a/public/app/plugins/datasource/graphite/lexer.js +++ b/public/app/plugins/datasource/graphite/lexer.js @@ -120,6 +120,7 @@ define([ i >= 48 && i <= 57 || // 0-9 i === 36 || // $ i === 126 || // ~ + i === 124 || // | i >= 65 && i <= 90 || // A-Z i === 95 || // _ i === 45 || // - From e20e2117f7d1cc50a679d679484bd9d59a620422 Mon Sep 17 00:00:00 2001 From: Andrew Widdersheim Date: Sun, 5 Jul 2015 21:22:40 -0400 Subject: [PATCH 036/287] Fix 'paranthesis' typo to 'parenthesis' This typo is most noticable in the Graphite lexer error hint message. --- public/app/plugins/datasource/graphite/parser.js | 4 ++-- public/test/specs/parser-specs.js | 4 ++-- public/vendor/bootstrap/bootstrap.js | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/graphite/parser.js b/public/app/plugins/datasource/graphite/parser.js index 242068f9d85..2ff15cda5b0 100644 --- a/public/app/plugins/datasource/graphite/parser.js +++ b/public/app/plugins/datasource/graphite/parser.js @@ -142,13 +142,13 @@ define([ name: this.consumeToken().value, }; - // consume left paranthesis + // consume left parenthesis this.consumeToken(); node.params = this.functionParameters(); if (!this.match(')')) { - this.errorMark('Expected closing paranthesis'); + this.errorMark('Expected closing parenthesis'); } this.consumeToken(); diff --git a/public/test/specs/parser-specs.js b/public/test/specs/parser-specs.js index 9fead11a9c3..8f5dd1b37ef 100644 --- a/public/test/specs/parser-specs.js +++ b/public/test/specs/parser-specs.js @@ -118,11 +118,11 @@ define([ expect(rootNode.pos).to.be(19); }); - it('invalid function expression missing closing paranthesis', function() { + it('invalid function expression missing closing parenthesis', function() { var parser = new Parser('sum(test'); var rootNode = parser.getAst(); - expect(rootNode.message).to.be('Expected closing paranthesis instead found end of string'); + expect(rootNode.message).to.be('Expected closing parenthesis instead found end of string'); expect(rootNode.pos).to.be(9); }); diff --git a/public/vendor/bootstrap/bootstrap.js b/public/vendor/bootstrap/bootstrap.js index bec8dee3c3c..62e23675410 100644 --- a/public/vendor/bootstrap/bootstrap.js +++ b/public/vendor/bootstrap/bootstrap.js @@ -2069,7 +2069,7 @@ , move: function (e) { if (!this.shown) return - // grafana change, shift+left paranthesis + // grafana change, shift+left parenthesis if (e.shiftKey && e.keyCode === 40) { return; } From 19812feb623135fc150da6afff5acc0fa5f43719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Jul 2015 09:22:11 +0200 Subject: [PATCH 037/287] Increased max metric suggest count to 1000 for OpenTSDB, Closes #2281 --- public/app/plugins/datasource/opentsdb/datasource.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index 27d4f001b0b..43b595371e3 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -78,7 +78,7 @@ function (angular, _, kbn) { }; OpenTSDBDatasource.prototype._performSuggestQuery = function(query) { - return this._get('/api/suggest', {type: 'metrics', q: query}).then(function(result) { + return this._get('/api/suggest', {type: 'metrics', q: query, max: 1000}).then(function(result) { return result.data; }); }; From 37d75905ed9e3d1f13f822a1a63653a3276c5851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Jul 2015 09:57:00 +0200 Subject: [PATCH 038/287] Added min span option for panel repeater, #1888 --- public/app/features/dashboard/dynamicDashboardSrv.js | 2 +- public/app/partials/panelgeneral.html | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/public/app/features/dashboard/dynamicDashboardSrv.js b/public/app/features/dashboard/dynamicDashboardSrv.js index fe0ffe47e51..bb58e2dbc06 100644 --- a/public/app/features/dashboard/dynamicDashboardSrv.js +++ b/public/app/features/dashboard/dynamicDashboardSrv.js @@ -164,7 +164,7 @@ function (angular, _) { _.each(selected, function(option, index) { var copy = self.getPanelClone(panel, row, index); - copy.span = 12 / selected.length; + copy.span = Math.max(12 / selected.length, panel.minSpan); copy.scopedVars = copy.scopedVars || {}; copy.scopedVars[variable.name] = option; }); diff --git a/public/app/partials/panelgeneral.html b/public/app/partials/panelgeneral.html index 02db54cd6c6..0d7cbb2c939 100644 --- a/public/app/partials/panelgeneral.html +++ b/public/app/partials/panelgeneral.html @@ -42,6 +42,14 @@ +
  • + Min span +
    • +
  • + +
    • From 3c86c9908cc00e655c0eac0bbee329f0d2976f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Jul 2015 12:24:11 +0200 Subject: [PATCH 039/287] Added docs for dashboard list panel, #2276 --- docs/mkdocs.yml | 2 +- docs/sources/reference/dashlist.md | 18 ++++++++++++++++++ public/app/panels/dashlist/editor.html | 4 ++-- public/app/panels/dashlist/module.js | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 7d84dba5f0d..13eaf757c94 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -45,7 +45,7 @@ pages: - ['reference/graph.md', 'Reference', 'Graph Panel'] - ['reference/singlestat.md', 'Reference', 'Singlestat Panel'] -- ['reference/dashlist.md', 'Reference', 'Dashlist Panel'] +- ['reference/dashlist.md', 'Reference', 'Dashboard list Panel'] - ['reference/sharing.md', 'Reference', 'Sharing'] - ['reference/annotations.md', 'Reference', 'Annotations'] - ['reference/timerange.md', 'Reference', 'Time range controls'] diff --git a/docs/sources/reference/dashlist.md b/docs/sources/reference/dashlist.md index 6f591169f2c..9a54a18e288 100644 --- a/docs/sources/reference/dashlist.md +++ b/docs/sources/reference/dashlist.md @@ -6,4 +6,22 @@ page_keywords: grafana, dashlist, panel, documentation # Dashlist Panel +## Overview +![](/img/v2/dashboard_list_panel.png) + +The dashboard list panel allows you to show a list of links to other dashboards. The list +can be based on a search query or dashboard tag query. You can also configure it to show your starred +dashboards. + +## Options +![](/img/v2/dashboard_list_panel_options.png) + +Name | Description +------------ | ------------- +Mode | Set search or starred mode +Query | If in search mode specify the search query +Tags | if in search mode specify dashboard tags to search for +Limit number to | Specify the maximum number of dashboards + + diff --git a/public/app/panels/dashlist/editor.html b/public/app/panels/dashlist/editor.html index ff2e75fd95c..7b176b74317 100644 --- a/public/app/panels/dashlist/editor.html +++ b/public/app/panels/dashlist/editor.html @@ -7,7 +7,7 @@ Mode
  • - +
    • @@ -47,7 +47,7 @@ Limit number to
  • - +
    • diff --git a/public/app/panels/dashlist/module.js b/public/app/panels/dashlist/module.js index 647199d6538..3e7c8c5587c 100644 --- a/public/app/panels/dashlist/module.js +++ b/public/app/panels/dashlist/module.js @@ -21,7 +21,7 @@ function (angular, app, _, config, PanelMeta) { module.controller('DashListPanelCtrl', function($scope, panelSrv, backendSrv) { $scope.panelMeta = new PanelMeta({ - panelName: 'Dash list', + panelName: 'Dashboard list', editIcon: "fa fa-star", fullscreen: true, }); From dbd46a523fe1023f56614fd665cfbf7b2cfcace0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Jul 2015 13:43:08 +0200 Subject: [PATCH 040/287] Work arround for slower template variable dropdown when variable has many thousand values, now only top 1000 values are rendered to html, you can still search all values, Closes #2246 --- public/app/directives/valueSelectDropdown.js | 8 +++++++- public/app/features/templating/templateValuesSrv.js | 1 + public/app/partials/valueSelectDropdown.html | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/public/app/directives/valueSelectDropdown.js b/public/app/directives/valueSelectDropdown.js index 8c963bcf1eb..65b45135fd5 100644 --- a/public/app/directives/valueSelectDropdown.js +++ b/public/app/directives/valueSelectDropdown.js @@ -29,7 +29,11 @@ function (angular, app, _) { return tag; }); - vm.search = {query: '', options: vm.options}; + vm.search = { + query: '', + options: vm.options.slice(0, Math.min(vm.options.length, 1000)) + }; + vm.dropdownVisible = true; }; @@ -204,6 +208,8 @@ function (angular, app, _) { vm.search.options = _.filter(vm.options, function(option) { return option.text.toLowerCase().indexOf(vm.search.query.toLowerCase()) !== -1; }); + + vm.search.options = vm.search.options.slice(0, Math.min(vm.search.options.length, 1000)); }; vm.init = function() { diff --git a/public/app/features/templating/templateValuesSrv.js b/public/app/features/templating/templateValuesSrv.js index 8e91403c1e4..a5767ffa312 100644 --- a/public/app/features/templating/templateValuesSrv.js +++ b/public/app/features/templating/templateValuesSrv.js @@ -10,6 +10,7 @@ function (angular, _, kbn) { module.service('templateValuesSrv', function($q, $rootScope, datasourceSrv, $location, templateSrv, timeSrv) { var self = this; + function getNoneOption() { return { text: 'None', value: '', isNone: true }; } $rootScope.onAppEvent('time-range-changed', function() { diff --git a/public/app/partials/valueSelectDropdown.html b/public/app/partials/valueSelectDropdown.html index 4ecb46db14b..a75b7e8a6a9 100644 --- a/public/app/partials/valueSelectDropdown.html +++ b/public/app/partials/valueSelectDropdown.html @@ -19,7 +19,7 @@ Selected ({{vm.selectedValues.length}})     - + {{option.text}} From a4b8a88ae59d980e00970a5eec5d4676b25a12cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Jul 2015 15:01:37 +0200 Subject: [PATCH 041/287] Worked on templating docs, Closes #2274 --- docs/sources/reference/templating.md | 41 ++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/docs/sources/reference/templating.md b/docs/sources/reference/templating.md index f66a8838441..397aaddec91 100644 --- a/docs/sources/reference/templating.md +++ b/docs/sources/reference/templating.md @@ -5,14 +5,43 @@ page_keywords: grafana, templating, variables, guide, documentation --- # Templated Dashboards +![](/img/v2/templating_var_list.png) -Templating feature can be enabled under dashboard settings, in the Features tab. The templating feature allows -you to create variables that can be used in your metric queries, series names and panel titles. Use this feature to -create generic dashboards that can quickly be changed to show graphs for different servers or metrics. +## Overview +Templating allows you to create dashboard variables that can be used in your metric queries, series +names and panel titles. Use this feature to create generic dashboards that can quickly be +changed to show graphs for different servers or metrics. + +You find this feature in the dashboard cog dropdown menu. + +## Variable types +There are three different types of template variables. They can all be used in the +same way but they differ in how the list variables values is created. + +### Query +This is the most common type of variable. It allows you to create a variable +with values fetched directly from a data source via a metric exploration query. + +For example a query like `prod.servers.*` will fill the variable with all possible +values that exists in the wildcard position (Graphite example). + +You can also create nested variables that use other variables in their definition. For example +`apps.$app.servers.*` uses the variable `$app` in its query definition. + +> For examples of template queries appropriate for your data source checkout the documentation +> page for your data source. + +### Interval +This variable type is useful for time ranges like `1m`,`1h`, `1d`. There is also an auto +option that will change depending on the current time range, you can specify how many times +the current time range should be divided to calculate the current `auto` range. + +![](/img/v2/templated_variable_parameter.png) + +### Custom +This variable type allow you to manually specify all the different values as a comma seperated +string. ## Screencast - Templated Graphite Queries -
    -## Screencast - Templated InfluxDB Queries -Coming soon From 572d35506b3ca1e02f908aaac0b9483f802acee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Jul 2015 15:58:16 +0200 Subject: [PATCH 042/287] Removed some editor srv properites from data sources, they are not needed there anymore, they are in plugin.json, Closes #2283 --- public/app/plugins/datasource/influxdb/datasource.js | 2 -- public/app/plugins/datasource/kairosdb/datasource.js | 1 - public/app/plugins/datasource/opentsdb/datasource.js | 1 - 3 files changed, 4 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/datasource.js b/public/app/plugins/datasource/influxdb/datasource.js index cc2e798f70b..f5b01fad79f 100644 --- a/public/app/plugins/datasource/influxdb/datasource.js +++ b/public/app/plugins/datasource/influxdb/datasource.js @@ -28,8 +28,6 @@ function (angular, _, kbn, InfluxSeries, InfluxQueryBuilder) { this.supportAnnotations = true; this.supportMetrics = true; - this.editorSrc = 'app/features/influxdb/partials/query.editor.html'; - this.annotationEditorSrc = 'app/features/influxdb/partials/annotations.editor.html'; } InfluxDatasource.prototype.query = function(options) { diff --git a/public/app/plugins/datasource/kairosdb/datasource.js b/public/app/plugins/datasource/kairosdb/datasource.js index 09864e0548a..d0cddca3bec 100644 --- a/public/app/plugins/datasource/kairosdb/datasource.js +++ b/public/app/plugins/datasource/kairosdb/datasource.js @@ -13,7 +13,6 @@ function (angular, _, kbn) { function KairosDBDatasource(datasource) { this.type = datasource.type; - this.editorSrc = 'plugins/datasources/kairosdb/kairosdb.editor.html'; this.url = datasource.url; this.name = datasource.name; this.supportMetrics = true; diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index 43b595371e3..324d42e48a7 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -14,7 +14,6 @@ function (angular, _, kbn) { function OpenTSDBDatasource(datasource) { this.type = 'opentsdb'; - this.editorSrc = 'app/features/opentsdb/partials/query.editor.html'; this.url = datasource.url; this.name = datasource.name; this.supportMetrics = true; From af50188a60ff4921ab3b4acc9e32fac88a30cdbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Jul 2015 17:16:17 +0200 Subject: [PATCH 043/287] Removed alpha notice for influxdb 0.9 data source config --- .../app/plugins/datasource/influxdb/partials/config.html | 8 -------- 1 file changed, 8 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/partials/config.html b/public/app/plugins/datasource/influxdb/partials/config.html index 1f7bf27a2a1..66c39fe7b69 100644 --- a/public/app/plugins/datasource/influxdb/partials/config.html +++ b/public/app/plugins/datasource/influxdb/partials/config.html @@ -1,11 +1,3 @@ -
    -
    Data source implementation: Alpha stage
    -
      -
    • This data source implementation is not complete, a lot is not working and implemented yet
      • -
    • Updates can be tracked, and feedback directed here #1525.
      • -
    -
    -

    From 8d7ac3862c3f12df3ee3abea46bc858fd67a11c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 7 Jul 2015 14:28:49 +0200 Subject: [PATCH 044/287] Fixed issue with dashlinks/panellinks and sending template variable value 'All', Fixes #2297 --- public/app/features/templating/templateSrv.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/public/app/features/templating/templateSrv.js b/public/app/features/templating/templateSrv.js index 4c9d47ae2e2..40b661b0cbc 100644 --- a/public/app/features/templating/templateSrv.js +++ b/public/app/features/templating/templateSrv.js @@ -116,8 +116,16 @@ function (angular, _) { }; this.fillVariableValuesForUrl = function(params) { + var toUrlVal = function(current) { + if (current.text === 'All') { + return 'All'; + } else { + return current.value; + } + }; + _.each(this.variables, function(variable) { - params['var-' + variable.name] = variable.current.value; + params['var-' + variable.name] = toUrlVal(variable.current); }); }; From 24e527ae0264e1f6bee34c749c8644f8bc313450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 7 Jul 2015 14:50:23 +0200 Subject: [PATCH 045/287] Updated Graphite docs, Closes #2271 --- docs/sources/datasources/graphite.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/sources/datasources/graphite.md b/docs/sources/datasources/graphite.md index 368feeda8c3..05a4891c581 100644 --- a/docs/sources/datasources/graphite.md +++ b/docs/sources/datasources/graphite.md @@ -57,3 +57,16 @@ this consolidation is done using `avg` function. You can how graphite consolidat > *Notice* This means that legend summary values (max, min, total) cannot be all correct at the same time. They are calculated > client side by Grafana. And depending on your consolidation function only one or two can be correct at the same time. + +## Templating +You can create a template variable in Grafana and have that variable filled with values from any Graphite metric exploration query. +You can then use this variable in your Graphite queries, either as part of a metric path or as arguments to functions. + +For example a query like `prod.servers.*` will fill the variable with all possible +values that exists in the wildcard position. + +You can also create nested variables that use other variables in their definition. For example +`apps.$app.servers.*` uses the variable `$app` in its query definition. + +![](/img/v2/templated_variable_parameter.png) + From f6ad386ba7de521de0461841c84d20f3a434cf4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 7 Jul 2015 17:14:57 +0200 Subject: [PATCH 046/287] Fixed capital G in Graphite in docs, reviewed annotation docs, Closes #2275 --- docs/sources/datasources/graphite.md | 6 +++--- docs/sources/reference/annotations.md | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/sources/datasources/graphite.md b/docs/sources/datasources/graphite.md index 05a4891c581..d41a987514c 100644 --- a/docs/sources/datasources/graphite.md +++ b/docs/sources/datasources/graphite.md @@ -6,7 +6,7 @@ page_keywords: grafana, graphite, metrics, query, documentation # Graphite -Grafana has an advanced graphite query editor that lets you quickly navigate the metric space, add functions, +Grafana has an advanced Graphite query editor that lets you quickly navigate the metric space, add functions, change function parameters and much more. The editor can handle all types of graphite queries. It can even handle complex nested queries through the use of query references. @@ -52,8 +52,8 @@ Some functions like aliasByNode support an optional second argument. To add this ## Point consolidation -All graphite metrics are consolidated so that graphite doesn't return more data points than there are pixels in the graph. By default -this consolidation is done using `avg` function. You can how graphite consolidates metrics by adding the Graphite consolidateBy function. +All Graphite metrics are consolidated so that Graphite doesn't return more data points than there are pixels in the graph. By default +this consolidation is done using `avg` function. You can how Graphite consolidates metrics by adding the Graphite consolidateBy function. > *Notice* This means that legend summary values (max, min, total) cannot be all correct at the same time. They are calculated > client side by Grafana. And depending on your consolidation function only one or two can be correct at the same time. diff --git a/docs/sources/reference/annotations.md b/docs/sources/reference/annotations.md index 51c9ac1ba32..b0e84ef762b 100644 --- a/docs/sources/reference/annotations.md +++ b/docs/sources/reference/annotations.md @@ -13,7 +13,7 @@ you can get title, tags, and text information for the event. To add an annotation query click dashboard settings icon in top menu and select `Annotations` from the dropdown. This will open the `Annotations` edit view. Click the `Add` tab to add a new annotation query. -### Graphite annotations +## Graphite annotations Graphite supports two ways to query annotations. @@ -36,5 +36,4 @@ as the name for the fields that should be used for the annotation title, tags an For InfluxDB you need to enter a query like in the above screenshot. You need to have the ```where $timeFilter``` part. If you only select one column you will not need to enter anything in the column mapping fields. -If you have multiple columns you need to specify which column should be treated as title, tags and text column. From f237cac074f916557d07cb022995635bcbc7375b Mon Sep 17 00:00:00 2001 From: "Haneysmith, Nathan" Date: Tue, 7 Jul 2015 11:56:36 -0700 Subject: [PATCH 047/287] add double quotes around tags --- public/app/plugins/datasource/influxdb/queryBuilder.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/queryBuilder.js b/public/app/plugins/datasource/influxdb/queryBuilder.js index ce76e545743..a167abb81d6 100644 --- a/public/app/plugins/datasource/influxdb/queryBuilder.js +++ b/public/app/plugins/datasource/influxdb/queryBuilder.js @@ -15,9 +15,9 @@ function (_) { } if (tag.value && tag.value[0] === '/' && tag.value[tag.value.length - 1] === '/') { - return str + tag.key + ' =~ ' + tag.value; + return str + '"' +tag.key + '"' + ' =~ ' + tag.value; } - return str + tag.key + " = '" + tag.value + "'"; + return str + '"' + tag.key + '"' + " = '" + tag.value + "'"; } var p = InfluxQueryBuilder.prototype; @@ -92,7 +92,7 @@ function (_) { query += ' GROUP BY time($interval)'; if (target.groupByTags && target.groupByTags.length > 0) { - query += ', ' + target.groupByTags.join(); + query += ', "' + target.groupByTags.join('", "') + '"'; } if (target.fill) { From cd53c78449938b3ff401c8b09e2f7e181ba41283 Mon Sep 17 00:00:00 2001 From: "Haneysmith, Nathan" Date: Tue, 7 Jul 2015 13:15:51 -0700 Subject: [PATCH 048/287] updating tests for influxdb quoting --- public/test/specs/influx09-querybuilder-specs.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/public/test/specs/influx09-querybuilder-specs.js b/public/test/specs/influx09-querybuilder-specs.js index 93f1cef155a..611b384d24e 100644 --- a/public/test/specs/influx09-querybuilder-specs.js +++ b/public/test/specs/influx09-querybuilder-specs.js @@ -27,14 +27,14 @@ define([ var query = builder.build(); it('should generate correct query', function() { - expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE hostname = \'server1\' AND $timeFilter' + expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE "hostname" = \'server1\' AND $timeFilter' + ' GROUP BY time($interval) ORDER BY asc'); }); it('should switch regex operator with tag value is regex', function() { var builder = new InfluxQueryBuilder({measurement: 'cpu', tags: [{key: 'app', value: '/e.*/'}]}); var query = builder.build(); - expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE app =~ /e.*/ AND $timeFilter GROUP BY time($interval) ORDER BY asc'); + expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE "app" =~ /e.*/ AND $timeFilter GROUP BY time($interval) ORDER BY asc'); }); }); @@ -47,7 +47,7 @@ define([ var query = builder.build(); it('should generate correct query', function() { - expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE hostname = \'server1\' AND app = \'email\' AND ' + + expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE "hostname" = \'server1\' AND "app" = \'email\' AND ' + '$timeFilter GROUP BY time($interval) ORDER BY asc'); }); }); @@ -61,7 +61,7 @@ define([ var query = builder.build(); it('should generate correct query', function() { - expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE hostname = \'server1\' OR hostname = \'server2\' AND ' + + expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE "hostname" = \'server1\' OR "hostname" = \'server2\' AND ' + '$timeFilter GROUP BY time($interval) ORDER BY asc'); }); }); @@ -76,7 +76,7 @@ define([ var query = builder.build(); expect(query).to.be('SELECT mean(value) FROM "cpu" WHERE $timeFilter ' + - 'GROUP BY time($interval), host ORDER BY asc'); + 'GROUP BY time($interval), "host" ORDER BY asc'); }); }); @@ -121,13 +121,13 @@ define([ it('should have measurement tag condition and tag name IN filter in tag values query', function() { var builder = new InfluxQueryBuilder({measurement: 'cpu', tags: [{key: 'app', value: 'email'}, {key: 'host', value: 'server1'}]}); var query = builder.buildExploreQuery('TAG_VALUES', 'app'); - expect(query).to.be('SHOW TAG VALUES FROM "cpu" WITH KEY = "app" WHERE host = \'server1\''); + expect(query).to.be('SHOW TAG VALUES FROM "cpu" WITH KEY = "app" WHERE "host" = \'server1\''); }); it('should switch to regex operator in tag condition', function() { var builder = new InfluxQueryBuilder({measurement: 'cpu', tags: [{key: 'host', value: '/server.*/'}]}); var query = builder.buildExploreQuery('TAG_VALUES', 'app'); - expect(query).to.be('SHOW TAG VALUES FROM "cpu" WITH KEY = "app" WHERE host =~ /server.*/'); + expect(query).to.be('SHOW TAG VALUES FROM "cpu" WITH KEY = "app" WHERE "host" =~ /server.*/'); }); }); From 56c332a3dddde48bde40e873cceffc904d14f095 Mon Sep 17 00:00:00 2001 From: "Haneysmith, Nathan" Date: Tue, 7 Jul 2015 13:19:15 -0700 Subject: [PATCH 049/287] more test updates for influxdb quoting --- public/test/specs/influx09-querybuilder-specs.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/test/specs/influx09-querybuilder-specs.js b/public/test/specs/influx09-querybuilder-specs.js index 611b384d24e..838e1950072 100644 --- a/public/test/specs/influx09-querybuilder-specs.js +++ b/public/test/specs/influx09-querybuilder-specs.js @@ -97,7 +97,7 @@ define([ it('should have where condition in tag keys query with tags', function() { var builder = new InfluxQueryBuilder({ measurement: '', tags: [{key: 'host', value: 'se1'}] }); var query = builder.buildExploreQuery('TAG_KEYS'); - expect(query).to.be("SHOW TAG KEYS WHERE host = 'se1'"); + expect(query).to.be("SHOW TAG KEYS WHERE \"host\" = 'se1'"); }); it('should have no conditions in measurement query for query with no tags', function() { @@ -109,7 +109,7 @@ define([ it('should have where condition in measurement query for query with tags', function() { var builder = new InfluxQueryBuilder({measurement: '', tags: [{key: 'app', value: 'email'}]}); var query = builder.buildExploreQuery('MEASUREMENTS'); - expect(query).to.be("SHOW MEASUREMENTS WHERE app = 'email'"); + expect(query).to.be("SHOW MEASUREMENTS WHERE \"app\" = 'email'"); }); it('should have where tag name IN filter in tag values query for query with one tag', function() { From ac37b54ddb3c2c9d7eb18c572fe033418d9913f4 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Tue, 7 Jul 2015 16:07:19 -0700 Subject: [PATCH 050/287] fix datasource api docs reflects change in e2f6633d57624664654463578e8d2502bfd7ffef --- docs/sources/reference/http_api.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sources/reference/http_api.md b/docs/sources/reference/http_api.md index 9e24ccb39d7..90b7274cce6 100644 --- a/docs/sources/reference/http_api.md +++ b/docs/sources/reference/http_api.md @@ -183,7 +183,7 @@ Status Codes: ### Create data source -`PUT /api/datasources` +`POST /api/datasources` **Example Response**: @@ -192,9 +192,9 @@ Status Codes: {"message":"Datasource added"} -### Edit an existing data source +### Update an existing data source -`POST /api/datasources` +`PUT /api/datasources/:datasourceId` ### Delete an existing data source @@ -269,7 +269,7 @@ Adds a global user to the actual organisation. ### Delete User in Organisation -`DELETE /api/orgs/:orgId/users/:userId` +`DELETE /api/orgs/:orgId/users/:userId` ## Users From 66ba19b7ba3d9a3e27dd76b9a46be41c56e0009d Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Tue, 7 Jul 2015 12:51:38 -0700 Subject: [PATCH 051/287] clearer errors "Not found" should only be for http path/method not found (404) if it's about specific resources, we should be explicit for clarity --- pkg/api/dashboard_snapshot.go | 2 +- pkg/models/dashboards.go | 1 + pkg/models/models.go | 4 ---- pkg/services/sqlstore/dashboard_snapshot.go | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/api/dashboard_snapshot.go b/pkg/api/dashboard_snapshot.go index 0061a2eba19..be044cc25eb 100644 --- a/pkg/api/dashboard_snapshot.go +++ b/pkg/api/dashboard_snapshot.go @@ -59,7 +59,7 @@ func GetDashboardSnapshot(c *middleware.Context) { // expired snapshots should also be removed from db if snapshot.Expires.Before(time.Now()) { - c.JsonApiErr(404, "Snapshot not found", err) + c.JsonApiErr(404, "Dashboard snapshot not found", err) return } diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index cdcc4d52364..7d4a2690556 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -11,6 +11,7 @@ import ( // Typed errors var ( ErrDashboardNotFound = errors.New("Dashboard not found") + ErrDashboardSnapshotNotFound = errors.New("Dashboard snapshot not found") ErrDashboardWithSameNameExists = errors.New("A dashboard with the same name already exists") ErrDashboardVersionMismatch = errors.New("The dashboard has been changed by someone else") ) diff --git a/pkg/models/models.go b/pkg/models/models.go index c38f0c5a391..189e594576b 100644 --- a/pkg/models/models.go +++ b/pkg/models/models.go @@ -1,7 +1,5 @@ package models -import "errors" - type OAuthType int const ( @@ -9,5 +7,3 @@ const ( GOOGLE TWITTER ) - -var ErrNotFound = errors.New("Not found") diff --git a/pkg/services/sqlstore/dashboard_snapshot.go b/pkg/services/sqlstore/dashboard_snapshot.go index 0bbb01ed6bd..f4611050a77 100644 --- a/pkg/services/sqlstore/dashboard_snapshot.go +++ b/pkg/services/sqlstore/dashboard_snapshot.go @@ -57,7 +57,7 @@ func GetDashboardSnapshot(query *m.GetDashboardSnapshotQuery) error { if err != nil { return err } else if has == false { - return m.ErrNotFound + return m.ErrDashboardSnapshotNotFound } query.Result = &snapshot From 521072daea675a90518d9a913f69799437e6694a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Espen=20Fjellv=C3=A6r=20Olsen?= Date: Tue, 7 Jul 2015 19:57:36 +0200 Subject: [PATCH 052/287] KairosDB: Streamline the Templating with the very related OpenTSDB plugin Since KairosDB is a fork of OpenTSDB it makes sense to (At least for the time being) keep their code bases similar-ish. --- .../plugins/datasource/kairosdb/datasource.js | 102 ++++++++++-------- 1 file changed, 60 insertions(+), 42 deletions(-) diff --git a/public/app/plugins/datasource/kairosdb/datasource.js b/public/app/plugins/datasource/kairosdb/datasource.js index d0cddca3bec..a47b6a9e83f 100644 --- a/public/app/plugins/datasource/kairosdb/datasource.js +++ b/public/app/plugins/datasource/kairosdb/datasource.js @@ -76,7 +76,7 @@ function (angular, _, kbn) { * Gets the list of metrics * @returns {*|Promise} */ - KairosDBDatasource.prototype.performMetricSuggestQuery = function() { + KairosDBDatasource.prototype._performMetricSuggestQuery = function(metric) { var options = { url : this.url + '/api/v1/metricnames', method : 'GET' @@ -84,46 +84,74 @@ function (angular, _, kbn) { return $http(options).then(function(response) { if (!response.data) { - return []; + return $q.when([]); } - return response.data.results; + var metrics = []; + _.each(response.data.results, function(r) { + if (r.indexOf(metric) >= 0) { + metrics.push(r); + } + }); + return metrics; }); }; - KairosDBDatasource.prototype.performListTagNames = function() { + KairosDBDatasource.prototype._performMetricKeyLookup = function(metric) { + if(!metric) { return $q.when([]); } + var options = { - url : this.url + '/api/v1/tagnames', - method : 'GET' + method: 'POST', + url: this.url + '/api/v1/datapoints/query/tags', + data: { + metrics : [{ name : metric }], + cache_time : 0, + start_absolute: 0 + } }; - return $http(options).then(function(response) { - if (!response.data) { - return []; + return $http(options).then(function(result) { + if (!result.data) { + return $q.when([]); } - return response.data.results; + var tagks = []; + _.each(result.data.queries[0].results[0].tags, function(tagv, tagk) { + if(tagks.indexOf(tagk) === -1) { + tagks.push(tagk); + } + }); + return tagks; }); }; - KairosDBDatasource.prototype.performListTagValues = function() { + KairosDBDatasource.prototype._performMetricKeyValueLookup = function(metric, key) { + if(!metric || !key) { + return $q.when([]); + } + var options = { - url : this.url + '/api/v1/tagvalues', - method : 'GET' + method: 'POST', + url: this.url + '/api/v1/datapoints/query/tags', + data: { + metrics : [{ name : metric }], + cache_time : 0, + start_absolute: 0 + } }; - return $http(options).then(function(response) { - if (!response.data) { - return []; + return $http(options).then(function(result) { + if (!result.data) { + return $q.when([]); } - return response.data.results; + return result.data.queries[0].results[0].tags[key]; }); }; - KairosDBDatasource.prototype.performTagSuggestQuery = function(metricname) { + KairosDBDatasource.prototype.performTagSuggestQuery = function(metric) { var options = { url : this.url + '/api/v1/datapoints/query/tags', method : 'POST', data : { - metrics : [{ name : metricname }], + metrics : [{ name : metric }], cache_time : 0, start_absolute: 0 } @@ -140,19 +168,7 @@ function (angular, _, kbn) { }; KairosDBDatasource.prototype.metricFindQuery = function(query) { - function format(results, query) { - return _.chain(results) - .filter(function(result) { - return result.indexOf(query) >= 0; - }) - .map(function(result) { - return { - text: result, - expandable: true - }; - }) - .value(); - } + if (!query) { return $q.when([]); } var interpolated; try { @@ -162,30 +178,32 @@ function (angular, _, kbn) { return $q.reject(err); } + var responseTransform = function(result) { + return _.map(result, function(value) { + return {text: value}; + }); + }; + var metrics_regex = /metrics\((.*)\)/; var tag_names_regex = /tag_names\((.*)\)/; - var tag_values_regex = /tag_values\((.*)\)/; + var tag_values_regex = /tag_values\((.*),\s?(.*?)\)/; var metrics_query = interpolated.match(metrics_regex); if (metrics_query) { - return this.performMetricSuggestQuery().then(function(metrics) { - return format(metrics, metrics_query[1]); - }); + return this._performMetricSuggestQuery(metrics_query[1]).then(responseTransform); } var tag_names_query = interpolated.match(tag_names_regex); if (tag_names_query) { - return this.performListTagNames().then(function(tag_names) { - return format(tag_names, tag_names_query[1]); - }); + return this._performMetricKeyLookup(tag_names_query[1]).then(responseTransform); } var tag_values_query = interpolated.match(tag_values_regex); if (tag_values_query) { - return this.performListTagValues().then(function(tag_values) { - return format(tag_values, tag_values_query[1]); - }); + return this._performMetricKeyValueLookup(tag_values_query[1], tag_values_query[2]).then(responseTransform); } + + return $q.when([]); }; ///////////////////////////////////////////////////////////////////////// From 2255cb53f0867087c994c7dd20a5b83adfcee72f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Espen=20Fjellv=C3=A6r=20Olsen?= Date: Wed, 8 Jul 2015 09:18:02 +0200 Subject: [PATCH 053/287] Close the gap between the key and the value in the js objects --- .../plugins/datasource/kairosdb/datasource.js | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/kairosdb/datasource.js b/public/app/plugins/datasource/kairosdb/datasource.js index a47b6a9e83f..ad2dd0f52de 100644 --- a/public/app/plugins/datasource/kairosdb/datasource.js +++ b/public/app/plugins/datasource/kairosdb/datasource.js @@ -78,8 +78,8 @@ function (angular, _, kbn) { */ KairosDBDatasource.prototype._performMetricSuggestQuery = function(metric) { var options = { - url : this.url + '/api/v1/metricnames', - method : 'GET' + url: this.url + '/api/v1/metricnames', + method: 'GET' }; return $http(options).then(function(response) { @@ -103,8 +103,8 @@ function (angular, _, kbn) { method: 'POST', url: this.url + '/api/v1/datapoints/query/tags', data: { - metrics : [{ name : metric }], - cache_time : 0, + metrics: [{ name: metric }], + cache_time: 0, start_absolute: 0 } }; @@ -132,8 +132,8 @@ function (angular, _, kbn) { method: 'POST', url: this.url + '/api/v1/datapoints/query/tags', data: { - metrics : [{ name : metric }], - cache_time : 0, + metrics: [{ name: metric }], + cache_time: 0, start_absolute: 0 } }; @@ -148,11 +148,11 @@ function (angular, _, kbn) { KairosDBDatasource.prototype.performTagSuggestQuery = function(metric) { var options = { - url : this.url + '/api/v1/datapoints/query/tags', - method : 'POST', - data : { - metrics : [{ name : metric }], - cache_time : 0, + url: this.url + '/api/v1/datapoints/query/tags', + method: 'POST', + data: { + metrics: [{ name: metric }], + cache_time: 0, start_absolute: 0 } }; From 3980d25c23e2fd59b3c5eb6ed7a0b32e190e71a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Espen=20Fjellv=C3=A6r=20Olsen?= Date: Wed, 8 Jul 2015 09:18:47 +0200 Subject: [PATCH 054/287] Remove superfluous whitespaces --- .../plugins/datasource/kairosdb/datasource.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/public/app/plugins/datasource/kairosdb/datasource.js b/public/app/plugins/datasource/kairosdb/datasource.js index ad2dd0f52de..a718d4f0e44 100644 --- a/public/app/plugins/datasource/kairosdb/datasource.js +++ b/public/app/plugins/datasource/kairosdb/datasource.js @@ -100,13 +100,13 @@ function (angular, _, kbn) { if(!metric) { return $q.when([]); } var options = { - method: 'POST', - url: this.url + '/api/v1/datapoints/query/tags', - data: { + method: 'POST', + url: this.url + '/api/v1/datapoints/query/tags', + data: { metrics: [{ name: metric }], cache_time: 0, start_absolute: 0 - } + } }; return $http(options).then(function(result) { @@ -129,13 +129,13 @@ function (angular, _, kbn) { } var options = { - method: 'POST', - url: this.url + '/api/v1/datapoints/query/tags', - data: { + method: 'POST', + url: this.url + '/api/v1/datapoints/query/tags', + data: { metrics: [{ name: metric }], cache_time: 0, start_absolute: 0 - } + } }; return $http(options).then(function(result) { From 3e05eb23fd489d5d4b60692e948544144733d922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Espen=20Fjellv=C3=A6r=20Olsen?= Date: Wed, 8 Jul 2015 09:24:05 +0200 Subject: [PATCH 055/287] Update documentation based on templating changes --- docs/sources/datasources/kairosdb.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/sources/datasources/kairosdb.md b/docs/sources/datasources/kairosdb.md index f0d52b91548..f7ead4897ab 100644 --- a/docs/sources/datasources/kairosdb.md +++ b/docs/sources/datasources/kairosdb.md @@ -36,12 +36,13 @@ KairosDB Datasource Plugin provides following functions in `Variables values que Name | Description ---- | ---- -`metrics(query)` | Returns a list of metric names. If nothing is given, returns a list of all metric names. -`tag_names(query)` | Returns a list of tag names. If nothing is given, returns a list of all tag names. -`tag_values(query)` | Returns a list of tag values. If nothing is given, returns a list of all tag values. +`metrics(query)` | Returns a list of metric names matching `query`. If nothing is given, returns a list of all metric names. +`tag_names(query)` | Returns a list of tag names matching `query`. If nothing is given, returns a list of all tag names. +`tag_values(metric, tag)` | Returns a list of values for `tag` from the given `metric`. For details of `metric names`, `tag names`, and `tag values`, please refer to the KairosDB documentations. - [List Metric Names - KairosDB 0.9.4 documentation](http://kairosdb.github.io/kairosdocs/restapi/ListMetricNames.html) - [List Tag Names - KairosDB 0.9.4 documentation](http://kairosdb.github.io/kairosdocs/restapi/ListTagNames.html) - [List Tag Values - KairosDB 0.9.4 documentation](http://kairosdb.github.io/kairosdocs/restapi/ListTagValues.html) +- [Query Metrics - KairosDB 0.9.4 documentation](http://kairosdb.github.io/kairosdocs/restapi/QueryMetrics.html). From 48975e6533e2427603a47ae4119fc5c96bb45cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Espen=20Fjellv=C3=A6r=20Olsen?= Date: Wed, 8 Jul 2015 09:36:25 +0200 Subject: [PATCH 056/287] Keep QueryController up to date as well --- public/app/plugins/datasource/kairosdb/queryCtrl.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/kairosdb/queryCtrl.js b/public/app/plugins/datasource/kairosdb/queryCtrl.js index 9e8c5817dd1..a647b3f84a3 100644 --- a/public/app/plugins/datasource/kairosdb/queryCtrl.js +++ b/public/app/plugins/datasource/kairosdb/queryCtrl.js @@ -53,7 +53,7 @@ function (angular, _) { return metricList; } else { - $scope.datasource.performMetricSuggestQuery().then(function(result) { + $scope.datasource._performMetricSuggestQuery().then(function(result) { metricList = result; callback(metricList); }); @@ -69,7 +69,7 @@ function (angular, _) { } } - $scope.datasource.performTagSuggestQuery($scope.target.metric).then(function(result) { + $scope.datasource._performTagSuggestQuery($scope.target.metric).then(function(result) { if (!_.isEmpty(result)) { tagList.push(result); callback(_.keys(result.tags)); From b746b63036046b95702c5681db35b18f6a895f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 8 Jul 2015 10:52:50 +0200 Subject: [PATCH 057/287] Updated config docs with info about [dashboards.json] section , #2302 --- docs/sources/installation/configuration.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index cd4b4d20276..69142d57eb8 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -32,10 +32,10 @@ should be upper case, `.` should be replaced by `_`. For example, given these co [security] admin_user = admin - + [auth.google] client_secret = 0ldS3cretKey - + Then you can override that using: @@ -367,3 +367,14 @@ enabled. Counters are sent every 24 hours. Default value is `true`. If you want to track Grafana usage via Google analytics specify *your* Universal Analytics ID here. By default this feature is disabled. + +## [dashboards.json] + +If you have a system that automatically builds dashboards as json files you can enable this feature to have the +Grafana backend index those json dashboards which will make them appear in regular dashboard search. + +### enabled +`true` or `false`. Is disabled by default. + +### path +The full path to a directory containing your json dashboards. From 3b39c43193ad56030968ee0d441ad6d8b495660a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 8 Jul 2015 11:06:11 +0200 Subject: [PATCH 058/287] Updated docs version and version fragment --- docs/VERSION | 2 +- docs/sources/versions.html_fragment | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/VERSION b/docs/VERSION index edb49bc725f..7ec1d6db408 100644 --- a/docs/VERSION +++ b/docs/VERSION @@ -1 +1 @@ -2.0.0-beta +2.1.0 diff --git a/docs/sources/versions.html_fragment b/docs/sources/versions.html_fragment index 00ee000a0f0..0ad0144831e 100644 --- a/docs/sources/versions.html_fragment +++ b/docs/sources/versions.html_fragment @@ -1,2 +1,3 @@ +
  • Version v2.1
  • Version v2.0
  • Version v1.9
    • From d97f24cfc3775006162db8c8af967cad8275ee49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Espen=20Fjellv=C3=A6r=20Olsen?= Date: Wed, 8 Jul 2015 13:00:22 +0200 Subject: [PATCH 059/287] Update QueryController to conform with OpenTSDB Controller --- .../plugins/datasource/kairosdb/queryCtrl.js | 52 +++++-------------- 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/public/app/plugins/datasource/kairosdb/queryCtrl.js b/public/app/plugins/datasource/kairosdb/queryCtrl.js index a647b3f84a3..4cf1b8c0f21 100644 --- a/public/app/plugins/datasource/kairosdb/queryCtrl.js +++ b/public/app/plugins/datasource/kairosdb/queryCtrl.js @@ -6,8 +6,6 @@ function (angular, _) { 'use strict'; var module = angular.module('grafana.controllers'); - var metricList = []; - var tagList = []; module.controller('KairosDBQueryCtrl', function($scope) { @@ -48,50 +46,26 @@ function (angular, _) { _.move($scope.panel.targets, fromIndex, toIndex); }; + $scope.getTextValues = function(metricFindResult) { + return _.map(metricFindResult, function(value) { return value.text; }); + }; + $scope.suggestMetrics = function(query, callback) { - if (!_.isEmpty(metricList)) { - return metricList; - } - else { - $scope.datasource._performMetricSuggestQuery().then(function(result) { - metricList = result; - callback(metricList); - }); - } + $scope.datasource.metricFindQuery('metrics(' + query + ')') + .then($scope.getTextValues) + .then(callback); }; $scope.suggestTagKeys = function(query, callback) { - if (!_.isEmpty(tagList)) { - var result = _.find(tagList, { name : $scope.target.metric }); - - if (!_.isEmpty(result)) { - return _.keys(result.tags); - } - } - - $scope.datasource._performTagSuggestQuery($scope.target.metric).then(function(result) { - if (!_.isEmpty(result)) { - tagList.push(result); - callback(_.keys(result.tags)); - } - }); + $scope.datasource.metricFindQuery('tag_names(' + $scope.target.metric + ')') + .then($scope.getTextValues) + .then(callback); }; $scope.suggestTagValues = function(query, callback) { - if (!_.isEmpty(tagList)) { - var result = _.find(tagList, { name : $scope.target.metric }); - - if (!_.isEmpty(result)) { - return result.tags[$scope.target.currentTagKey]; - } - } - - $scope.datasource.performTagSuggestQuery($scope.target.metric).then(function(result) { - if (!_.isEmpty(result)) { - tagList.push(result); - callback(result.tags[$scope.target.currentTagKey]); - } - }); + $scope.datasource.metricFindQuery('tag_values(' + $scope.target.metric + ',' + $scope.target.currentTagKey + ')') + .then($scope.getTextValues) + .then(callback); }; // Filter metric by tag From 5d7c485991f34e90e1c5398785e93d6ac71ee82f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 8 Jul 2015 13:44:36 +0200 Subject: [PATCH 060/287] fix(templating): can now use template vars in collapsed row title, Fixes #2305 --- docs/sources/versions.html_fragment | 2 +- public/app/partials/dashboard.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/versions.html_fragment b/docs/sources/versions.html_fragment index 0ad0144831e..55190af8b37 100644 --- a/docs/sources/versions.html_fragment +++ b/docs/sources/versions.html_fragment @@ -1,3 +1,3 @@ -
  • Version v2.1
    • +
  • Version v2.1
  • Version v2.0
  • Version v1.9
    • diff --git a/public/app/partials/dashboard.html b/public/app/partials/dashboard.html index 3ea34d0db0a..57c9cb74521 100644 --- a/public/app/partials/dashboard.html +++ b/public/app/partials/dashboard.html @@ -21,7 +21,7 @@ -
    +
    From a958965e642a56ad278baf1f5c4270944f2201eb Mon Sep 17 00:00:00 2001 From: David Gillies Date: Wed, 8 Jul 2015 22:13:38 +1000 Subject: [PATCH 061/287] set mode 0640 and group grafana on /etc/grafana/grafana.ini --- packaging/deb/control/postinst | 4 ++-- packaging/rpm/control/postinst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/deb/control/postinst b/packaging/deb/control/postinst index edb163ba7fb..7585d3f879d 100755 --- a/packaging/deb/control/postinst +++ b/packaging/deb/control/postinst @@ -43,9 +43,9 @@ case "$1" in chmod 755 /var/log/grafana /var/lib/grafana # configuration files should not be modifiable by grafana user, as this can be a security issue - chown -Rh root:root /etc/grafana/* + chown -Rh root:$GRAFANA_GROUP /etc/grafana/* chmod 755 /etc/grafana - find /etc/grafana -type f -exec chmod 644 {} ';' + find /etc/grafana -type f -exec chmod 640 {} ';' find /etc/grafana -type d -exec chmod 755 {} ';' # if $2 is set, this is an upgrade diff --git a/packaging/rpm/control/postinst b/packaging/rpm/control/postinst index 9e5e9accf79..fce80719115 100644 --- a/packaging/rpm/control/postinst +++ b/packaging/rpm/control/postinst @@ -43,9 +43,9 @@ if [ $1 -eq 1 ] ; then chmod 755 /var/log/grafana /var/lib/grafana # configuration files should not be modifiable by grafana user, as this can be a security issue - chown -Rh root:root /etc/grafana/* + chown -Rh root:$GRAFANA_GROUP /etc/grafana/* chmod 755 /etc/grafana - find /etc/grafana -type f -exec chmod 644 {} ';' + find /etc/grafana -type f -exec chmod 640 {} ';' find /etc/grafana -type d -exec chmod 755 {} ';' if [ -x /bin/systemctl ] ; then From 63f8800007e5740bb6f2a0db701fc54d96f89379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 8 Jul 2015 14:17:53 +0200 Subject: [PATCH 062/287] fix(docs): Updated getting started docs, Closes #2273, Closes #1978 --- docs/sources/guides/gettingstarted.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/sources/guides/gettingstarted.md b/docs/sources/guides/gettingstarted.md index 641dc9516f5..a2939b3724a 100644 --- a/docs/sources/guides/gettingstarted.md +++ b/docs/sources/guides/gettingstarted.md @@ -40,6 +40,15 @@ in the main Time Picker in the upper right, but they can also have relative time 5. Dashboard panel. You edit panels by clicking the panel title. 6. Graph legend. You can change series colors, y-axis and series visibility directly from the legend. +## Adding & Editing Graphs and Panels + +![](/img/v2/graph_metrics_tab_graphite.png) + +1. You add panels via row menu. The row menu is the green icon to the left of each row. +2. To edit the graph you click on the graph title to open the panel menu, then `Edit`. +3. This should take you to the `Metrics` tab. In this tab you should see the editor for your default data source. + + ## Drag-and-Drop panels You can Drag-and-Drop Panels within and between Rows. Click and hold the Panel title, and drag it to its new location. From 5cd1c5c522f4cc68aeb1b8aab07089231a668b25 Mon Sep 17 00:00:00 2001 From: David Gillies Date: Wed, 8 Jul 2015 22:40:43 +1000 Subject: [PATCH 063/287] Minor update to documentation for MySQL sessions --- docs/sources/installation/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index 69142d57eb8..6310ccb7973 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -327,7 +327,7 @@ This option should be configured differently depending on what type of session provider you have configured. - **file:** session file path, e.g. `data/sessions` -- **mysql:** go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1)/database_name` +- **mysql:** go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` - **postgres:** ex: user=a password=b host=localhost port=5432 dbname=c sslmode=disable If you use MySQL or Postgres as the session store you need to create the From 597c1f6a411f9ae867035aad7d2aa0f400f353e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 8 Jul 2015 17:41:36 +0200 Subject: [PATCH 064/287] feat(influxdb): began work on some influxdb enhancements, #2311 --- docs/sources/guides/gettingstarted.md | 1 - .../plugins/datasource/influxdb/funcEditor.js | 18 +++++++----------- .../influxdb/partials/query.editor.html | 11 +++++------ .../datasource/influxdb/queryBuilder.js | 7 +++++-- .../plugins/datasource/influxdb/queryCtrl.js | 14 +++++++++++++- .../test/specs/influx09-querybuilder-specs.js | 6 ++++++ 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/docs/sources/guides/gettingstarted.md b/docs/sources/guides/gettingstarted.md index a2939b3724a..4bb250ef55c 100644 --- a/docs/sources/guides/gettingstarted.md +++ b/docs/sources/guides/gettingstarted.md @@ -48,7 +48,6 @@ in the main Time Picker in the upper right, but they can also have relative time 2. To edit the graph you click on the graph title to open the panel menu, then `Edit`. 3. This should take you to the `Metrics` tab. In this tab you should see the editor for your default data source. - ## Drag-and-Drop panels You can Drag-and-Drop Panels within and between Rows. Click and hold the Panel title, and drag it to its new location. diff --git a/public/app/plugins/datasource/influxdb/funcEditor.js b/public/app/plugins/datasource/influxdb/funcEditor.js index 12731c0859d..05bcae6835b 100644 --- a/public/app/plugins/datasource/influxdb/funcEditor.js +++ b/public/app/plugins/datasource/influxdb/funcEditor.js @@ -11,30 +11,26 @@ function (angular, _, $) { .directive('influxdbFuncEditor', function($compile) { var funcSpanTemplate = '{{target.function}}('; + 'data-toggle="dropdown">{{field.func}}('; var paramTemplate = ''; return { restrict: 'A', + scope: { + field: "=", + }, link: function postLink($scope, elem) { var $funcLink = $(funcSpanTemplate); - $scope.functionMenu = _.map($scope.functions, function(func) { - return { - text: func, - click: "changeFunction('" + func + "');" - }; - }); - function clickFuncParam() { /*jshint validthis:true */ var $link = $(this); var $input = $link.next(); - $input.val($scope.target.column); + $input.val($scope.field.name); $input.css('width', ($link.width() + 16) + 'px'); $link.hide(); @@ -58,7 +54,7 @@ function (angular, _, $) { if ($input.val() !== '') { $link.text($input.val()); - $scope.target.column = $input.val(); + $scope.field.name = $input.val(); $scope.$apply($scope.get_data); } @@ -84,7 +80,7 @@ function (angular, _, $) { $input.typeahead({ source: function () { - return $scope.listColumns.apply(null, arguments); + return $scope.getFields.apply(null, arguments); }, minLength: 0, items: 20, diff --git a/public/app/plugins/datasource/influxdb/partials/query.editor.html b/public/app/plugins/datasource/influxdb/partials/query.editor.html index 89571c06022..81e0b07cbd9 100644 --- a/public/app/plugins/datasource/influxdb/partials/query.editor.html +++ b/public/app/plugins/datasource/influxdb/partials/query.editor.html @@ -65,10 +65,9 @@
  • SELECT
    • -
  • - - {{target.function}}(value) - +
  • + +
    • @@ -114,8 +113,8 @@ -
  • - GROUP BY +
  • + GROUP BY
  • diff --git a/public/app/plugins/datasource/influxdb/queryBuilder.js b/public/app/plugins/datasource/influxdb/queryBuilder.js index a167abb81d6..ca7bb02496d 100644 --- a/public/app/plugins/datasource/influxdb/queryBuilder.js +++ b/public/app/plugins/datasource/influxdb/queryBuilder.js @@ -32,12 +32,15 @@ function (_) { if (type === 'TAG_KEYS') { query = 'SHOW TAG KEYS'; - measurement= this.target.measurement; + measurement = this.target.measurement; } else if (type === 'TAG_VALUES') { query = 'SHOW TAG VALUES'; - measurement= this.target.measurement; + measurement = this.target.measurement; } else if (type === 'MEASUREMENTS') { query = 'SHOW MEASUREMENTS'; + } else if (type === 'FIELDS') { + query = 'SHOW FIELD KEYS FROM "' + this.target.measurement + '"'; + return query; } if (measurement) { diff --git a/public/app/plugins/datasource/influxdb/queryCtrl.js b/public/app/plugins/datasource/influxdb/queryCtrl.js index d1a524d2277..16e74c8e6bf 100644 --- a/public/app/plugins/datasource/influxdb/queryCtrl.js +++ b/public/app/plugins/datasource/influxdb/queryCtrl.js @@ -21,9 +21,12 @@ function (angular, _, InfluxQueryBuilder) { $scope.init = function() { var target = $scope.target; - target.function = target.function || 'mean'; target.tags = target.tags || []; target.groupByTags = target.groupByTags || []; + target.fields = target.fields || [{ + name: 'value', + func: target.function || 'mean' + }]; $scope.queryBuilder = new InfluxQueryBuilder(target); @@ -94,6 +97,15 @@ function (angular, _, InfluxQueryBuilder) { $scope.$parent.get_data(); }; + $scope.getFields = function(query, callback) { + var fieldsQuery = $scope.queryBuilder.buildExploreQuery('FIELDS'); + return $scope.datasource.metricFindQuery(fieldsQuery) + .then(function(results) { + var fields = _.pluck(results, 'text'); + callback(fields); + }); + }; + $scope.toggleQueryMode = function () { $scope.target.rawQuery = !$scope.target.rawQuery; }; diff --git a/public/test/specs/influx09-querybuilder-specs.js b/public/test/specs/influx09-querybuilder-specs.js index 838e1950072..6a18f0013bd 100644 --- a/public/test/specs/influx09-querybuilder-specs.js +++ b/public/test/specs/influx09-querybuilder-specs.js @@ -130,6 +130,12 @@ define([ expect(query).to.be('SHOW TAG VALUES FROM "cpu" WITH KEY = "app" WHERE "host" =~ /server.*/'); }); + it('should build show field query', function() { + var builder = new InfluxQueryBuilder({measurement: 'cpu', tags: [{key: 'app', value: 'email'}]}); + var query = builder.buildExploreQuery('FIELDS'); + expect(query).to.be('SHOW FIELD KEYS FROM "cpu"'); + }); + }); }); From dca45f177fa50843800dfc40bf89b140e6288acd Mon Sep 17 00:00:00 2001 From: Alex Laverty Date: Thu, 9 Jul 2015 11:50:29 +1000 Subject: [PATCH 065/287] Fixed invalid json example, added missing comma --- docs/sources/reference/http_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/reference/http_api.md b/docs/sources/reference/http_api.md index 90b7274cce6..fb9e7600ecf 100644 --- a/docs/sources/reference/http_api.md +++ b/docs/sources/reference/http_api.md @@ -53,7 +53,7 @@ Creates a new dashboard or updates an existing dashboard. "rows": [ { } - ] + ], "schemaVersion": 6, "version": 0 }, From 3f2c1de28d406f45f387da3f9960967b9a76580e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 9 Jul 2015 08:59:58 +0200 Subject: [PATCH 066/287] fix(OpenTSDB): fixed issue with OpenTSDB and query to result matching, Fixes #2315 --- public/app/plugins/datasource/opentsdb/datasource.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/app/plugins/datasource/opentsdb/datasource.js b/public/app/plugins/datasource/opentsdb/datasource.js index 324d42e48a7..d35c88fc45d 100644 --- a/public/app/plugins/datasource/opentsdb/datasource.js +++ b/public/app/plugins/datasource/opentsdb/datasource.js @@ -50,6 +50,9 @@ function (angular, _, kbn) { var metricToTargetMapping = mapMetricsToTargets(response.data, options); var result = _.map(response.data, function(metricData, index) { index = metricToTargetMapping[index]; + if (index === -1) { + index = 0; + } return transformMetricData(metricData, groupByTags, options.targets[index], options); }); return { data: result }; From 50795adcf0acf21b14b1eabf890f1945cd919972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 9 Jul 2015 16:36:47 +0200 Subject: [PATCH 067/287] feat(influxdb 0.9): field lookup and other enhancements, #2311 --- .../plugins/datasource/influxdb/funcEditor.js | 26 ++++++++++++++++--- .../influxdb/partials/query.editor.html | 2 +- .../datasource/influxdb/queryBuilder.js | 18 ++++++++++--- .../plugins/datasource/influxdb/queryCtrl.js | 14 ++-------- .../test/specs/influx09-querybuilder-specs.js | 14 ++++++++++ 5 files changed, 54 insertions(+), 20 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/funcEditor.js b/public/app/plugins/datasource/influxdb/funcEditor.js index 05bcae6835b..7631363fa12 100644 --- a/public/app/plugins/datasource/influxdb/funcEditor.js +++ b/public/app/plugins/datasource/influxdb/funcEditor.js @@ -16,14 +16,32 @@ function (angular, _, $) { var paramTemplate = ''; + var functionList = [ + 'count', 'mean', 'sum', 'min', 'max', 'mode', 'distinct', 'median', + 'derivative', 'stddev', 'first', 'last', 'difference' + ]; + + var functionMenu = _.map(functionList, function(func) { + return { text: func, click: "changeFunction('" + func + "');" }; + }); + return { restrict: 'A', scope: { field: "=", + getFields: "&", + onChange: "&", }, link: function postLink($scope, elem) { var $funcLink = $(funcSpanTemplate); + $scope.functionMenu = functionMenu; + + $scope.changeFunction = function(func) { + $scope.field.func = func; + $scope.onChange(); + }; + function clickFuncParam() { /*jshint validthis:true */ @@ -55,7 +73,7 @@ function (angular, _, $) { $link.text($input.val()); $scope.field.name = $input.val(); - $scope.$apply($scope.get_data); + $scope.$apply($scope.onChange()); } $input.hide(); @@ -79,8 +97,10 @@ function (angular, _, $) { $input.attr('data-provide', 'typeahead'); $input.typeahead({ - source: function () { - return $scope.getFields.apply(null, arguments); + source: function (query, callback) { + return $scope.getFields().then(function(results) { + callback(results); + }); }, minLength: 0, items: 20, diff --git a/public/app/plugins/datasource/influxdb/partials/query.editor.html b/public/app/plugins/datasource/influxdb/partials/query.editor.html index 81e0b07cbd9..216f099d125 100644 --- a/public/app/plugins/datasource/influxdb/partials/query.editor.html +++ b/public/app/plugins/datasource/influxdb/partials/query.editor.html @@ -66,7 +66,7 @@ SELECT
  • - +
    • diff --git a/public/app/plugins/datasource/influxdb/queryBuilder.js b/public/app/plugins/datasource/influxdb/queryBuilder.js index ca7bb02496d..717ae22847f 100644 --- a/public/app/plugins/datasource/influxdb/queryBuilder.js +++ b/public/app/plugins/datasource/influxdb/queryBuilder.js @@ -76,15 +76,25 @@ function (_) { throw "Metric measurement is missing"; } - var query = 'SELECT '; - var measurement = target.measurement; - var aggregationFunc = target.function || 'mean'; + if (!target.fields) { + target.fields = [{name: 'value', func: target.function || 'mean'}]; + } + var query = 'SELECT '; + var i; + for (i = 0; i < target.fields.length; i++) { + var field = target.fields[i]; + if (i > 0) { + query += ', '; + } + query += field.func + '(' + field.name + ')'; + } + + var measurement = target.measurement; if (!measurement.match('^/.*/') && !measurement.match(/^merge\(.*\)/)) { measurement = '"' + measurement+ '"'; } - query += aggregationFunc + '(value)'; query += ' FROM ' + measurement + ' WHERE '; var conditions = _.map(target.tags, function(tag, index) { return renderTagCondition(tag, index); diff --git a/public/app/plugins/datasource/influxdb/queryCtrl.js b/public/app/plugins/datasource/influxdb/queryCtrl.js index 16e74c8e6bf..e28c6bc034d 100644 --- a/public/app/plugins/datasource/influxdb/queryCtrl.js +++ b/public/app/plugins/datasource/influxdb/queryCtrl.js @@ -10,15 +10,6 @@ function (angular, _, InfluxQueryBuilder) { module.controller('InfluxQueryCtrl', function($scope, $timeout, $sce, templateSrv, $q) { - $scope.functionList = [ - 'count', 'mean', 'sum', 'min', 'max', 'mode', 'distinct', 'median', - 'derivative', 'stddev', 'first', 'last', 'difference' - ]; - - $scope.functionMenu = _.map($scope.functionList, function(func) { - return { text: func, click: "changeFunction('" + func + "');" }; - }); - $scope.init = function() { var target = $scope.target; target.tags = target.tags || []; @@ -97,12 +88,11 @@ function (angular, _, InfluxQueryBuilder) { $scope.$parent.get_data(); }; - $scope.getFields = function(query, callback) { + $scope.getFields = function() { var fieldsQuery = $scope.queryBuilder.buildExploreQuery('FIELDS'); return $scope.datasource.metricFindQuery(fieldsQuery) .then(function(results) { - var fields = _.pluck(results, 'text'); - callback(fields); + return _.pluck(results, 'text'); }); }; diff --git a/public/test/specs/influx09-querybuilder-specs.js b/public/test/specs/influx09-querybuilder-specs.js index 6a18f0013bd..b515cfab6fb 100644 --- a/public/test/specs/influx09-querybuilder-specs.js +++ b/public/test/specs/influx09-querybuilder-specs.js @@ -38,6 +38,20 @@ define([ }); }); + describe('series with multiple fields', function() { + var builder = new InfluxQueryBuilder({ + measurement: 'cpu', + tags: [], + fields: [{ name: 'tx_in', func: 'sum' }, { name: 'tx_out', func: 'mean' }] + }); + + var query = builder.build(); + + it('should generate correct query', function() { + expect(query).to.be('SELECT sum(tx_in), mean(tx_out) FROM "cpu" WHERE $timeFilter GROUP BY time($interval) ORDER BY asc'); + }); + }); + describe('series with multiple tags only', function() { var builder = new InfluxQueryBuilder({ measurement: 'cpu', From 150fd869a01c70a26432fe57dab6f814c4994a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 9 Jul 2015 17:17:17 +0200 Subject: [PATCH 068/287] Finalized work on influxdb field and func selection, #2311 --- .../plugins/datasource/influxdb/funcEditor.js | 2 +- .../influxdb/partials/query.editor.html | 5 +++- .../plugins/datasource/influxdb/queryCtrl.js | 27 ++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/funcEditor.js b/public/app/plugins/datasource/influxdb/funcEditor.js index 7631363fa12..ae4248cf2a6 100644 --- a/public/app/plugins/datasource/influxdb/funcEditor.js +++ b/public/app/plugins/datasource/influxdb/funcEditor.js @@ -124,7 +124,7 @@ function (angular, _, $) { function addElementsAndCompile() { $funcLink.appendTo(elem); - var $paramLink = $('value'); + var $paramLink = $('' + $scope.field.name + ''); var $input = $(paramTemplate); $paramLink.appendTo(elem); diff --git a/public/app/plugins/datasource/influxdb/partials/query.editor.html b/public/app/plugins/datasource/influxdb/partials/query.editor.html index 216f099d125..d63e6853831 100644 --- a/public/app/plugins/datasource/influxdb/partials/query.editor.html +++ b/public/app/plugins/datasource/influxdb/partials/query.editor.html @@ -66,9 +66,12 @@ SELECT
  • - +
    • +
  • + +
    • diff --git a/public/app/plugins/datasource/influxdb/queryCtrl.js b/public/app/plugins/datasource/influxdb/queryCtrl.js index e28c6bc034d..a57737cba93 100644 --- a/public/app/plugins/datasource/influxdb/queryCtrl.js +++ b/public/app/plugins/datasource/influxdb/queryCtrl.js @@ -27,6 +27,8 @@ function (angular, _, InfluxQueryBuilder) { $scope.measurementSegment = new MetricSegment(target.measurement); } + $scope.addFieldSegment = MetricSegment.newPlusButton(); + $scope.tagSegments = []; _.each(target.tags, function(tag) { if (tag.condition) { @@ -92,7 +94,11 @@ function (angular, _, InfluxQueryBuilder) { var fieldsQuery = $scope.queryBuilder.buildExploreQuery('FIELDS'); return $scope.datasource.metricFindQuery(fieldsQuery) .then(function(results) { - return _.pluck(results, 'text'); + var values = _.pluck(results, 'text'); + if ($scope.target.fields.length > 1) { + values.splice(0, 0, "-- remove from select --"); + } + return values; }); }; @@ -161,6 +167,25 @@ function (angular, _, InfluxQueryBuilder) { .then(null, $scope.handleQueryError); }; + $scope.getFieldSegments = function() { + var fieldsQuery = $scope.queryBuilder.buildExploreQuery('FIELDS'); + return $scope.datasource.metricFindQuery(fieldsQuery) + .then($scope.transformToSegments) + .then(null, $scope.handleQueryError); + }; + + $scope.addField = function() { + $scope.target.fields.push({name: $scope.addFieldSegment.value, func: 'mean'}); + _.extend($scope.addFieldSegment, MetricSegment.newPlusButton()); + }; + + $scope.fieldChanged = function(field) { + if (field.name === '-- remove from select --') { + $scope.target.fields = _.without($scope.target.fields, field); + } + $scope.get_data(); + }; + $scope.getGroupByTagSegments = function(segment) { var query = $scope.queryBuilder.buildExploreQuery('TAG_KEYS'); From 14e25100d4066caaa366ffed9aa0f9d87fa30c78 Mon Sep 17 00:00:00 2001 From: gmembre Date: Thu, 9 Jul 2015 22:17:12 +0200 Subject: [PATCH 069/287] Adding GOPATH --- docs/sources/project/building_from_source.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/sources/project/building_from_source.md b/docs/sources/project/building_from_source.md index dc08f5a261a..b59d067eec4 100644 --- a/docs/sources/project/building_from_source.md +++ b/docs/sources/project/building_from_source.md @@ -17,6 +17,7 @@ dev environment. ## Get Code ``` +export GOPATH=``pwd` go get github.com/grafana/grafana ``` @@ -24,11 +25,11 @@ go get github.com/grafana/grafana ``` cd $GOPATH/src/github.com/grafana/grafana go run build.go setup (only needed once to install godep) -godep restore (will pull down all golang lib dependecies in your current GOPATH) +$GOPATH/bin/godep restore (will pull down all golang lib dependecies in your current GOPATH) go build . ``` -# Building on Windows +## Building on Windows The Grafana backend includes Sqlite3 which requires GCC to compile. So in order to compile Grafana on windows you need to install GCC. We recommend [TDM-GCC](http://tdm-gcc.tdragon.net/download). From 3fa228dcbba4d0440ea2d4a4c8644bdec4ddc3d0 Mon Sep 17 00:00:00 2001 From: gmembre Date: Thu, 9 Jul 2015 22:49:27 +0200 Subject: [PATCH 070/287] Correcting typo --- docs/sources/project/building_from_source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/project/building_from_source.md b/docs/sources/project/building_from_source.md index b59d067eec4..b0ae5cec724 100644 --- a/docs/sources/project/building_from_source.md +++ b/docs/sources/project/building_from_source.md @@ -17,7 +17,7 @@ dev environment. ## Get Code ``` -export GOPATH=``pwd` +export GOPATH=`pwd` go get github.com/grafana/grafana ``` From 244e682c2ba1cbdccc6cb1ee15c5785adbed5e8e Mon Sep 17 00:00:00 2001 From: Johannes Wienke Date: Fri, 10 Jul 2015 12:16:41 +0200 Subject: [PATCH 071/287] Series override option to hide legend entries Adds a series override option to hide certain series from the legend. * public/app/components/timeSeries.js: boolean flag to decide display in legend and application of override option for this flag * public/app/panels/graph/legend.js: exclude hidden series * public/app/panels/graph/seriesOverridesCtrl.js: add legend override option This fixes #2324 --- public/app/components/timeSeries.js | 2 ++ public/app/panels/graph/legend.js | 4 ++++ public/app/panels/graph/seriesOverridesCtrl.js | 1 + 3 files changed, 7 insertions(+) diff --git a/public/app/components/timeSeries.js b/public/app/components/timeSeries.js index 3c41378947e..74194e68ff9 100644 --- a/public/app/components/timeSeries.js +++ b/public/app/components/timeSeries.js @@ -13,6 +13,7 @@ function (_, kbn) { this.color = opts.color; this.valueFormater = kbn.valueFormats.none; this.stats = {}; + this.legend = true; } function matchSeriesOverride(aliasOrRegex, seriesAlias) { @@ -55,6 +56,7 @@ function (_, kbn) { if (override.fillBelowTo !== void 0) { this.fillBelowTo = override.fillBelowTo; } if (override.color !== void 0) { this.color = override.color; } if (override.transform !== void 0) { this.transform = override.transform; } + if (override.legend !== void 0) { this.legend = override.legend; } if (override.yaxis !== void 0) { this.yaxis = override.yaxis; diff --git a/public/app/panels/graph/legend.js b/public/app/panels/graph/legend.js index 1f016f2c3a3..beea27ec8b7 100644 --- a/public/app/panels/graph/legend.js +++ b/public/app/panels/graph/legend.js @@ -130,6 +130,10 @@ function (angular, app, _, kbn, $) { if (panel.legend.hideEmpty && series.allIsNull) { continue; } + // ignore series excluded via override + if (!series.legend) { + continue; + } var html = '
    ) will become Red, Orange, Green (). + + +### Spark Lines + +Spark lines are a great way of seeing the historical data associated with a single stat value, providing valuable context at a glance. Spark lines act differently than traditional graph panels and do not include x or y axis, coordinates, a legend, or ability to interact with the graph. + +![](/img/v1/Singlestat-Sparklines.png) + +1. `Show`: The show checkbox will toggle whether the spark line is shown in the panel. When unselected, only the value will appear. +2. `Background`: **ASK TORKEL.** +3. `Line Color`: This color selection applies to the color of the sparkline itself. +4. `Fill Color`: This color selection applies to the area below the sparkline.

    ***Pro-tip:*** Reduce the opacity on fill colors for nice looking panels. + + +### Value to text mapping + +Value to text mapping allows you to translate values into explcit text. The text will respect all styling, thresholds and customization defined for the value. + +![](/img/v1/Singlestat-ValueMapping.png) + +***Does this support regexes or anything other than a single value?*** + From db1847bc1d5ae3aa91903adb0e0cbeec3118ea67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 10 Jul 2015 15:29:34 +0200 Subject: [PATCH 074/287] More work on ldap, gotten ldap search (read attributes) to work --- conf/defaults.ini | 3 +-- pkg/auth/ldap.go | 35 ++++++++++++++++++++++++++++------- pkg/setting/setting.go | 6 ++++-- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index a44d47e2daf..e9b158d11bb 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -184,8 +184,7 @@ auto_sign_up = true enabled = true hosts = ldap://127.0.0.1:389 use_ssl = false -base_dn = dc=grafana,dc=org -bind_path = cn=%username%,dc=grafana,dc=org +bind_path = cn=%s,dc=grafana,dc=org attr_username = cn attr_name = cn attr_surname = sn diff --git a/pkg/auth/ldap.go b/pkg/auth/ldap.go index 4a9b524cdbb..97e89f1453d 100644 --- a/pkg/auth/ldap.go +++ b/pkg/auth/ldap.go @@ -17,7 +17,6 @@ func loginUsingLdap(query *AuthenticateUserQuery) error { return err } - log.Info("Host: %v", url.Host) conn, err := ldap.Dial("tcp", url.Host) if err != nil { return err @@ -25,10 +24,8 @@ func loginUsingLdap(query *AuthenticateUserQuery) error { defer conn.Close() - bindFormat := "cn=%s,dc=grafana,dc=org" - - nx := fmt.Sprintf(bindFormat, query.Username) - err = conn.Bind(nx, query.Password) + bindPath := fmt.Sprintf(setting.LdapBindPath, query.Username) + err = conn.Bind(bindPath, query.Password) if err != nil { if ldapErr, ok := err.(*ldap.Error); ok { @@ -39,12 +36,31 @@ func loginUsingLdap(query *AuthenticateUserQuery) error { return err } - userQuery := m.GetUserByLoginQuery{LoginOrEmail: "admin"} + searchReq := ldap.SearchRequest{ + BaseDN: "dc=grafana,dc=org", + Scope: ldap.ScopeWholeSubtree, + DerefAliases: ldap.NeverDerefAliases, + Attributes: []string{"cn", "sn", "email"}, + Filter: fmt.Sprintf("(cn=%s)", query.Username), + } + + result, err := conn.Search(&searchReq) + if err != nil { + return err + } + + log.Info("Search result: %v, error: %v", result, err) + + for _, entry := range result.Entries { + log.Info("cn: %s", entry.Attributes[0].Values[0]) + log.Info("email: %s", entry.Attributes[2].Values[0]) + } + + userQuery := m.GetUserByLoginQuery{LoginOrEmail: query.Username} err = bus.Dispatch(&userQuery) if err != nil { if err == m.ErrUserNotFound { - return ErrInvalidCredentials } return err } @@ -53,3 +69,8 @@ func loginUsingLdap(query *AuthenticateUserQuery) error { return nil } + +func createUserFromLdapInfo() error { + return nil + +} diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 6dd583a586b..c6c5630d2f8 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -118,8 +118,9 @@ var ( GoogleAnalyticsId string // LDAP - LdapEnabled bool - LdapHosts []string + LdapEnabled bool + LdapHosts []string + LdapBindPath string // SMTP email settings Smtp SmtpSettings @@ -419,6 +420,7 @@ func NewConfigContext(args *CommandLineArgs) { ldapSec := Cfg.Section("auth.ldap") LdapEnabled = ldapSec.Key("enabled").MustBool(false) LdapHosts = ldapSec.Key("hosts").Strings(" ") + LdapBindPath = ldapSec.Key("bind_path").String() readSessionConfig() readSmtpSettings() From 9f8eae908d86e6432ae284791245d4aa647ed382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 10 Jul 2015 15:34:54 +0200 Subject: [PATCH 075/287] Updated singlestat docs --- docs/sources/reference/singlestat.md | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/sources/reference/singlestat.md b/docs/sources/reference/singlestat.md index 67c5fb4d4a2..13e680c6e5c 100644 --- a/docs/sources/reference/singlestat.md +++ b/docs/sources/reference/singlestat.md @@ -13,47 +13,47 @@ provides thresholds to color that singlestat metric or the panel background. ### Big Value Configuration -The big value configuration allows you to both customize the look of your singlestat metric, as well as add additional labels to contexualize the metric. +The big value configuration allows you to both customize the look of your singlestat metric, as well as add additional labels to contexualize the metric. -![](/img/v1/Singlestat-BaseSettings.png) + -1. `Big Value`: Big Value refers to the collection of values displayed in the singlestat panel. -2. `Prefixes`: The Prefix fields let you define a custom label and font-size (as a %) to appear *before* the singlestat metric. -3. `Values`: The Value fields let you set the (min, max, average, current, total) and font-size (as a %) of the singlestat metric. -4. `Potsfixes`: The Postfix fields let you define a custom label and font-size (as a %) to appear *after* the singlestat metric. -5. `Units`: Units are appended to the the singlestat metric within the panel, and will respect the color and threshold settings for the Value. -6. `Decimals`: The Decimal field allows you to override automatic decimal precision, inceasing the digits displayed for your singlestat metric. +1. `Big Value`: Big Value refers to the collection of values displayed in the singlestat panel. +2. `Prefixes`: The Prefix fields let you define a custom label and font-size (as a %) to appear *before* the singlestat metric. +3. `Values`: The Value fields let you set the (min, max, average, current, total) and font-size (as a %) of the singlestat metric. +4. `Potsfixes`: The Postfix fields let you define a custom label and font-size (as a %) to appear *after* the singlestat metric. +5. `Units`: Units are appended to the the singlestat metric within the panel, and will respect the color and threshold settings for the Value. +6. `Decimals`: The Decimal field allows you to override automatic decimal precision, inceasing the digits displayed for your singlestat metric. ### Coloring -The coloring options of the singlestat config allow you to dynamically change the colors based on the displayed data. +The coloring options of the singlestat config allow you to dynamically change the colors based on the displayed data. -![](/img/v1/Singlestat-Coloring.png) + -1. `Background`: The Background checkbox applies the configured thresholds and colors to the entirity of the singlestat panel background. +1. `Background`: The Background checkbox applies the configured thresholds and colors to the entirity of the singlestat panel background. 2. `Value`: The Value checkbox applies the configured thresholds and colors to the value within the singlestat panel. 3. `Thresholds`: Thresholds allow you to change the background and value colors dyanmically within the panel. The threshold field accepts **3 comma-separated** values, corresponding to the three colors directly to the right. -4. `Colors`: The color picker allows you to select a color and opacity +4. `Colors`: The color picker allows you to select a color and opacity 5. `Invert order`: This link toggles the threshold color order.
    For example: Green, Orange, Red () will become Red, Orange, Green (). ### Spark Lines -Spark lines are a great way of seeing the historical data associated with a single stat value, providing valuable context at a glance. Spark lines act differently than traditional graph panels and do not include x or y axis, coordinates, a legend, or ability to interact with the graph. +Spark lines are a great way of seeing the historical data associated with a single stat value, providing valuable context at a glance. Spark lines act differently than traditional graph panels and do not include x or y axis, coordinates, a legend, or ability to interact with the graph. -![](/img/v1/Singlestat-Sparklines.png) + 1. `Show`: The show checkbox will toggle whether the spark line is shown in the panel. When unselected, only the value will appear. 2. `Background`: **ASK TORKEL.** 3. `Line Color`: This color selection applies to the color of the sparkline itself. -4. `Fill Color`: This color selection applies to the area below the sparkline.

    ***Pro-tip:*** Reduce the opacity on fill colors for nice looking panels. +4. `Fill Color`: This color selection applies to the area below the sparkline. +> ***Pro-tip:*** Reduce the opacity on fill colors for nice looking panels. ### Value to text mapping -Value to text mapping allows you to translate values into explcit text. The text will respect all styling, thresholds and customization defined for the value. +Value to text mapping allows you to translate values into explcit text. The text will respect all styling, thresholds and customization defined for the value. -![](/img/v1/Singlestat-ValueMapping.png) + -***Does this support regexes or anything other than a single value?*** From ca1cd8919073794ebeb665276cf20ada6716c0a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 10 Jul 2015 15:53:39 +0200 Subject: [PATCH 076/287] Updated singlestat docs again --- docs/sources/reference/singlestat.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/sources/reference/singlestat.md b/docs/sources/reference/singlestat.md index 13e680c6e5c..4e5a38544c5 100644 --- a/docs/sources/reference/singlestat.md +++ b/docs/sources/reference/singlestat.md @@ -34,8 +34,7 @@ The coloring options of the singlestat config allow you to dynamically change th 2. `Value`: The Value checkbox applies the configured thresholds and colors to the value within the singlestat panel. 3. `Thresholds`: Thresholds allow you to change the background and value colors dyanmically within the panel. The threshold field accepts **3 comma-separated** values, corresponding to the three colors directly to the right. 4. `Colors`: The color picker allows you to select a color and opacity -5. `Invert order`: This link toggles the threshold color order.
    For example: Green, Orange, Red () will become Red, Orange, Green (). - +5. `Invert order`: This link toggles the threshold color order.
    For example: Green, Orange, Red () will become Red, Orange, Green (). ### Spark Lines @@ -44,7 +43,7 @@ Spark lines are a great way of seeing the historical data associated with a sing 1. `Show`: The show checkbox will toggle whether the spark line is shown in the panel. When unselected, only the value will appear. -2. `Background`: **ASK TORKEL.** +2. `Background`: Check if you want the sparklines to take up the full panel width or uncheck if they should only be at the bottom. 3. `Line Color`: This color selection applies to the color of the sparkline itself. 4. `Fill Color`: This color selection applies to the area below the sparkline. From 9f6c9cd6ffbc03475f4908313f91fa63063beca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Sun, 12 Jul 2015 18:18:04 +0200 Subject: [PATCH 077/287] fix(graphite): raw query mode (disable graphite query editor mode) is now persisted property on the query, Fixes #2328, Fixes #2307 --- .../graphite/partials/query.editor.html | 13 ++++++------- .../app/plugins/datasource/graphite/queryCtrl.js | 15 +++++++++++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/graphite/partials/query.editor.html b/public/app/plugins/datasource/graphite/partials/query.editor.html index dcbbdc69a8a..add85c26773 100755 --- a/public/app/plugins/datasource/graphite/partials/query.editor.html +++ b/public/app/plugins/datasource/graphite/partials/query.editor.html @@ -12,7 +12,7 @@
  • - +
    • @@ -65,15 +65,14 @@ - + ng-model-onblur ng-change="get_data()" + ng-show="target.textEditor" /> -
    - +