Add support for deployment ID (#6144)

deployment ID helps in identifying a minio deployment in the case of remote
logging targets.

cmd/format-fs.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -55,11 +56,18 @@ type formatFSVersionDetect struct {
 	} `json:"fs"`
 }
 
+// Generic structure to manage both v1 and v2 structures
+type formatFS struct {
+	formatMetaV1
+	FS interface{} `json:"fs"`
+}
+
 // Returns the latest "fs" format V1
 func newFormatFSV1() (format *formatFSV1) {
 	f := &formatFSV1{}
 	f.Version = formatMetaVersionV1
 	f.Format = formatBackendFS
+	f.ID = mustGetUUID()
 	f.FS.Version = formatFSVersionV1
 	return f
 }
@@ -69,6 +77,7 @@ func newFormatFSV2() (format *formatFSV2) {
 	f := &formatFSV2{}
 	f.Version = formatMetaVersionV1
 	f.Format = formatBackendFS
+	f.ID = mustGetUUID()
 	f.FS.Version = formatFSVersionV2
 	return f
 }
@@ -115,7 +124,16 @@ func formatFSMigrateV1ToV2(ctx context.Context, wlk *lock.LockedFile, fsPath str
 		return err
 	}
 
-	return jsonSave(wlk.File, newFormatFSV2())
+	formatV1 := formatFSV1{}
+	if err = jsonLoad(wlk, &formatV1); err != nil {
+		return err
+	}
+
+	formatV2 := formatFSV2{}
+	formatV2.formatMetaV1 = formatV1.formatMetaV1
+	formatV2.FS.Version = formatFSVersionV2
+
+	return jsonSave(wlk.File, formatV2)
 }
 
 // Migrate the "fs" backend.
@@ -179,6 +197,12 @@ func createFormatFS(ctx context.Context, fsFormatPath string) error {
 // migrate the backend when we are actively working on the backend.
 func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, err error) {
 	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
+
+	// Add a deployment ID, if it does not exist.
+	if err := formatFSFixDeploymentID(fsFormatPath); err != nil {
+		return nil, err
+	}
+
 	// Any read on format.json should be done with read-lock.
 	// Any write on format.json should be done with write-lock.
 	for {
@@ -235,7 +259,8 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			rlk.Close()
 			// Hold write lock during migration so that we do not disturb any
 			// minio processes running in parallel.
-			wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+			var wlk *lock.LockedFile
+			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
 			if err == lock.ErrAlreadyLocked {
 				// Lock already present, sleep and attempt again.
 				time.Sleep(100 * time.Millisecond)
@@ -253,7 +278,94 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			// Successfully migrated, now try to hold a read-lock on format.json
 			continue
 		}
-
+		var id string
+		if id, err = formatFSGetDeploymentID(rlk); err != nil {
+			rlk.Close()
+			return nil, err
+		}
+		logger.SetDeploymentID(id)
 		return rlk, nil
 	}
 }
+
+func formatFSGetDeploymentID(rlk *lock.RLockedFile) (id string, err error) {
+	format := &formatFS{}
+	if err := jsonLoad(rlk, format); err != nil {
+		return "", err
+	}
+	return format.ID, nil
+}
+
+// Generate a deployment ID if one does not exist already.
+func formatFSFixDeploymentID(fsFormatPath string) error {
+	rlk, err := lock.RLockedOpenFile(fsFormatPath)
+	if err == nil {
+		// format.json can be empty in a rare condition when another
+		// minio process just created the file but could not hold lock
+		// and write to it.
+		var fi os.FileInfo
+		fi, err = rlk.Stat()
+		if err != nil {
+			rlk.Close()
+			return err
+		}
+		if fi.Size() == 0 {
+			rlk.Close()
+			return nil
+		}
+	}
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	formatBackend, err := formatMetaGetFormatBackendFS(rlk)
+	if err != nil {
+		rlk.Close()
+		return err
+	}
+	if formatBackend != formatBackendFS {
+		rlk.Close()
+		return fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
+	}
+
+	format := &formatFS{}
+	err = jsonLoad(rlk, format)
+	rlk.Close()
+	if err != nil {
+		return err
+	}
+
+	// Check if it needs to be updated
+	if format.ID != "" {
+		return nil
+	}
+
+	for {
+		wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+		if err == lock.ErrAlreadyLocked {
+			// Lock already present, sleep and attempt again.
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		defer wlk.Close()
+
+		err = jsonLoad(wlk, format)
+		if err != nil {
+			return err
+		}
+
+		// Check if it needs to be updated
+		if format.ID != "" {
+			return nil
+		}
+		format.ID = mustGetUUID()
+		return jsonSave(wlk, format)
+	}
+
+}

cmd/format-meta.go

@@ -49,4 +49,6 @@ type formatMetaV1 struct {
 	Version string `json:"version"`
 	// Format indicates the backend format type, supports two values 'xl' and 'fs'.
 	Format string `json:"format"`
+	// ID is the identifier for the minio deployment
+	ID string `json:"id"`
 }

cmd/format-xl.go

@@ -22,11 +22,13 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"sync"
 
 	"encoding/hex"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/logger"
 	sha256 "github.com/minio/sha256-simd"
 )
 
@@ -127,6 +129,7 @@ func newFormatXLV3(numSets int, setLen int) *formatXLV3 {
 	format := &formatXLV3{}
 	format.Version = formatMetaVersionV1
 	format.Format = formatBackendXL
+	format.ID = mustGetUUID()
 	format.XL.Version = formatXLVersionV3
 	format.XL.DistributionAlgo = formatXLVersionV2DistributionAlgo
 	format.XL.Sets = make([][]string, numSets)
@@ -443,6 +446,112 @@ func checkFormatXLValues(formats []*formatXLV3) error {
 	return nil
 }
 
+// Get Deployment ID for the XL sets from format.json.
+// This need not be in quorum. Even if one of the format.json
+// file has this value, we assume it is valid.
+// If more than one format.json's have different id, it is considered a corrupt
+// backend format.
+func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (string, error) {
+	var deploymentID string
+	for _, format := range formats {
+		if format == nil || format.ID == "" {
+			continue
+		}
+		if reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) {
+			// Found an ID in one of the format.json file
+			// Set deploymentID for the first time.
+			if deploymentID == "" {
+				deploymentID = format.ID
+			} else if deploymentID != format.ID {
+				// DeploymentID found earlier doesn't match with the
+				// current format.json's ID.
+				return "", errCorruptedFormat
+			}
+		}
+	}
+	return deploymentID, nil
+}
+
+// formatXLFixDeploymentID - Add deployment id if it is not present.
+func formatXLFixDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
+	// Acquire lock on format.json
+	mutex := newNSLock(globalIsDistXL)
+	formatLock := mutex.NewNSLock(minioMetaBucket, formatConfigFile)
+	if err = formatLock.GetLock(globalHealingTimeout); err != nil {
+		return err
+	}
+	defer formatLock.Unlock()
+
+	// Attempt to load all `format.json` from all disks.
+	var sErrs []error
+	formats, sErrs := loadFormatXLAll(storageDisks)
+	for i, sErr := range sErrs {
+		if _, ok := formatCriticalErrors[sErr]; ok {
+			return fmt.Errorf("Disk %s: %s", globalEndpoints[i], sErr)
+		}
+	}
+
+	for index := range formats {
+		// If the XL sets do not match, set those formats to nil,
+		// We do not have to update the ID on those format.json file.
+		if formats[index] != nil && !reflect.DeepEqual(formats[index].XL.Sets, refFormat.XL.Sets) {
+			formats[index] = nil
+		}
+	}
+	refFormat.ID, err = formatXLGetDeploymentID(refFormat, formats)
+	if err != nil {
+		return err
+	}
+
+	// If ID is set, then some other node got the lock
+	// before this node could and generated an ID
+	// for the deployment. No need to generate one.
+	if refFormat.ID != "" {
+		return nil
+	}
+
+	// ID is generated for the first time,
+	// We set the ID in all the formats and update.
+	refFormat.ID = mustGetUUID()
+	for _, format := range formats {
+		if format != nil {
+			format.ID = refFormat.ID
+		}
+	}
+	// Deployment ID needs to be set on all the disks.
+	// Save `format.json` across all disks.
+	return saveFormatXLAll(ctx, storageDisks, formats)
+
+}
+
+// Update only the valid local disks which have not been updated before.
+func formatXLFixLocalDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) error {
+	// If this server was down when the deploymentID was updated
+	// then we make sure that we update the local disks with the deploymentID.
+	for index, storageDisk := range storageDisks {
+		if globalEndpoints[index].IsLocal && storageDisk != nil && storageDisk.IsOnline() {
+			format, err := loadFormatXL(storageDisk)
+			if err != nil {
+				// Disk can be offline etc.
+				// ignore the errors seen here.
+				continue
+			}
+			if format.ID != "" {
+				continue
+			}
+			if !reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) {
+				continue
+			}
+			format.ID = refFormat.ID
+			if err := saveFormatXL(storageDisk, format); err != nil {
+				logger.LogIf(ctx, err)
+				return fmt.Errorf("Unable to save format.json, %s", err)
+			}
+		}
+	}
+	return nil
+}
+
 // Get backend XL format in quorum `format.json`.
 func getFormatXLInQuorum(formats []*formatXLV3) (*formatXLV3, error) {
 	formatHashes := make([]string, len(formats))

cmd/format-xl_test.go

@@ -472,6 +472,61 @@ func TestGetFormatXLInQuorumCheck(t *testing.T) {
 	}
 }
 
+// Tests formatXLGetDeploymentID()
+func TestGetXLID(t *testing.T) {
+	setCount := 2
+	disksPerSet := 8
+
+	format := newFormatXLV3(setCount, disksPerSet)
+	formats := make([]*formatXLV3, 16)
+
+	for i := 0; i < setCount; i++ {
+		for j := 0; j < disksPerSet; j++ {
+			newFormat := *format
+			newFormat.XL.This = format.XL.Sets[i][j]
+			formats[i*disksPerSet+j] = &newFormat
+		}
+	}
+
+	// Return a format from list of formats in quorum.
+	quorumFormat, err := getFormatXLInQuorum(formats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check if the reference format and input formats are same.
+	var id string
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[0] = nil
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[1].XL.Sets[0][0] = "bad-uuid"
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[2].ID = "bad-id"
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
+		t.Fatal("Unexpected Success")
+	}
+}
+
 // Initialize new format sets.
 func TestNewFormatSets(t *testing.T) {
 	setCount := 2

cmd/logger/logger.go

@@ -122,14 +122,15 @@ type api struct {
 }
 
 type logEntry struct {
-	Level      string      `json:"level"`
+	DeploymentID string      `json:"deploymentid,omitempty"`
-	Time       string      `json:"time"`
+	Level        string      `json:"level"`
-	API        *api        `json:"api,omitempty"`
+	Time         string      `json:"time"`
-	RemoteHost string      `json:"remotehost,omitempty"`
+	API          *api        `json:"api,omitempty"`
-	RequestID  string      `json:"requestID,omitempty"`
+	RemoteHost   string      `json:"remotehost,omitempty"`
-	UserAgent  string      `json:"userAgent,omitempty"`
+	RequestID    string      `json:"requestID,omitempty"`
-	Message    string      `json:"message,omitempty"`
+	UserAgent    string      `json:"userAgent,omitempty"`
-	Trace      *traceEntry `json:"error,omitempty"`
+	Message      string      `json:"message,omitempty"`
+	Trace        *traceEntry `json:"error,omitempty"`
 }
 
 // quiet: Hide startup messages if enabled
@@ -138,8 +139,15 @@ var (
 	quiet, jsonFlag bool
 	// Custom function to format error
 	errorFmtFunc func(string, error, bool) string
+
+	deploymentID string
 )
 
+// SetDeploymentID - Used to set the deployment ID, in XL and FS mode
+func SetDeploymentID(id string) {
+	deploymentID = id
+}
+
 // EnableQuiet - turns quiet option on.
 func EnableQuiet() {
 	quiet = true

cmd/prepare-storage.go

@@ -186,6 +186,23 @@ func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, dr
 		return nil, err
 	}
 
+	// Get the deploymentID if set.
+	format.ID, err = formatXLGetDeploymentID(format, formatConfigs)
+	if err != nil {
+		return nil, err
+	}
+
+	if format.ID == "" {
+		if err = formatXLFixDeploymentID(context.Background(), storageDisks, format); err != nil {
+			return nil, err
+		}
+	}
+
+	logger.SetDeploymentID(format.ID)
+
+	if err = formatXLFixLocalDeploymentID(context.Background(), storageDisks, format); err != nil {
+		return nil, err
+	}
 	return format, nil
 }