Implement mgmt REST APIs to heal storage format. (#3604)

* Implement heal format REST API handler
* Implement admin peer rpc handler to re-initialize storage
* Implement HealFormat API in pkg/madmin
* Update pkg/madmin API.md to incl. HealFormat
* Added unit tests for ReInitDisks rpc handler and HealFormatHandler

cmd/admin-handlers.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"encoding/json"
 	"encoding/xml"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -374,6 +375,7 @@ func (adminAPI adminAPIHandlers) ListBucketsHealHandler(w http.ResponseWriter, r
 }
 
 // HealBucketHandler - POST /?heal&bucket=mybucket
+// - x-minio-operation = bucket
 // - bucket is mandatory query parameter
 // Heal a given bucket, if present.
 func (adminAPI adminAPIHandlers) HealBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -425,6 +427,7 @@ func isDryRun(qval url.Values) bool {
 }
 
 // HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject
+// - x-minio-operation = object
 // - bucket and object are both mandatory query parameters
 // Heal a given object, if present.
 func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *http.Request) {
@@ -473,3 +476,69 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
 	// Return 200 on success.
 	writeSuccessResponseHeadersOnly(w)
 }
+
+// HealFormatHandler - POST /?heal
+// - x-minio-operation = format
+// - bucket and object are both mandatory query parameters
+// Heal a given object, if present.
+func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *http.Request) {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Check if this setup is an erasure code backend, since
+	// heal-format is only applicable to single node XL and
+	// distributed XL setup.
+	if !globalIsXL {
+		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		return
+	}
+
+	// Create a new set of storage instances to heal format.json.
+	bootstrapDisks, err := initStorageDisks(globalEndpoints)
+	if err != nil {
+		fmt.Println(traceError(err))
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Heal format.json on available storage.
+	err = healFormatXL(bootstrapDisks)
+	if err != nil {
+		fmt.Println(traceError(err))
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Instantiate new object layer with newly formatted storage.
+	newObjectAPI, err := newXLObjects(bootstrapDisks)
+	if err != nil {
+		fmt.Println(traceError(err))
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Set object layer with newly formatted storage to globalObjectAPI.
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = newObjectAPI
+	globalObjLayerMutex.Unlock()
+
+	// Shutdown storage belonging to old object layer instance.
+	objectAPI.Shutdown()
+
+	// Inform peers to reinitialize storage with newly formatted storage.
+	reInitPeerDisks(globalAdminPeers)
+
+	// Return 200 on success.
+	writeSuccessResponseHeadersOnly(w)
+}

cmd/admin-handlers_test.go

@@ -29,6 +29,79 @@ import (
 	router "github.com/gorilla/mux"
 )
 
+// adminXLTestBed - encapsulates subsystems that need to be setup for
+// admin-handler unit tests.
+type adminXLTestBed struct {
+	configPath string
+	xlDirs     []string
+	objLayer   ObjectLayer
+	mux        *router.Router
+}
+
+// prepareAdminXLTestBed - helper function that setups a single-node
+// XL backend for admin-handler tests.
+func prepareAdminXLTestBed() (*adminXLTestBed, error) {
+	// reset global variables to start afresh.
+	resetTestGlobals()
+	// Initialize minio server config.
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		return nil, err
+	}
+	// Initializing objectLayer for HealFormatHandler.
+	objLayer, xlDirs, xlErr := initTestXLObjLayer()
+	if xlErr != nil {
+		return nil, xlErr
+	}
+
+	// Set globalEndpoints for a single node XL setup.
+	for _, xlDir := range xlDirs {
+		globalEndpoints = append(globalEndpoints, &url.URL{
+			Path: xlDir,
+		})
+	}
+
+	// Set globalIsXL to indicate that the setup uses an erasure code backend.
+	globalIsXL = true
+
+	// initialize NSLock.
+	isDistXL := false
+	initNSLock(isDistXL)
+
+	// Setup admin mgmt REST API handlers.
+	adminRouter := router.NewRouter()
+	registerAdminRouter(adminRouter)
+
+	return &adminXLTestBed{
+		configPath: rootPath,
+		xlDirs:     xlDirs,
+		objLayer:   objLayer,
+		mux:        adminRouter,
+	}, nil
+}
+
+// TearDown - method that resets the test bed for subsequent unit
+// tests to start afresh.
+func (atb *adminXLTestBed) TearDown() {
+	removeAll(atb.configPath)
+	removeRoots(atb.xlDirs)
+	resetTestGlobals()
+}
+
+// initTestObjLayer - Helper function to initialize an XL-based object
+// layer and set globalObjectAPI.
+func initTestXLObjLayer() (ObjectLayer, []string, error) {
+	objLayer, xlDirs, xlErr := prepareXL()
+	if xlErr != nil {
+		return nil, nil, xlErr
+	}
+	// Make objLayer available to all internal services via globalObjectAPI.
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = objLayer
+	globalObjLayerMutex.Unlock()
+	return objLayer, xlDirs, nil
+}
+
 // cmdType - Represents different service subcomands like status, stop
 // and restart.
 type cmdType int
@@ -115,17 +188,11 @@ func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Requ
 // testServicesCmdHandler - parametrizes service subcommand tests on
 // cmdType value.
 func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing.T) {
-	// reset globals.
+	adminTestBed, err := prepareAdminXLTestBed()
-	// this is to make sure that the tests are not affected by modified value.
-	resetTestGlobals()
-	// initialize NSLock.
-	initNSLock(false)
-	// Initialize configuration for access/secret credentials.
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
-	defer removeAll(rootPath)
+	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls. Note: In a
 	// single node setup, this degenerates to a simple function
@@ -139,29 +206,12 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing
 	globalMinioAddr = eps[0].Host
 	initGlobalAdminPeers(eps)
 
-	if cmd == statusCmd {
-		// Initializing objectLayer and corresponding
-		// []StorageAPI since DiskInfo() method requires it.
-		objLayer, xlDirs, xlErr := prepareXL()
-		if xlErr != nil {
-			t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
-		}
-		defer removeRoots(xlDirs)
-		// Make objLayer available to all internal services via globalObjectAPI.
-		globalObjLayerMutex.Lock()
-		globalObjectAPI = objLayer
-		globalObjLayerMutex.Unlock()
-	}
-
 	// Setting up a go routine to simulate ServerMux's
 	// handleServiceSignals for stop and restart commands.
 	if cmd == restartCmd {
 		go testServiceSignalReceiver(cmd, t)
 	}
 	credentials := serverConfig.GetCredential()
-	adminRouter := router.NewRouter()
-	registerAdminRouter(adminRouter)
-
 	var body []byte
 
 	if cmd == setCreds {
@@ -174,7 +224,7 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing
 	}
 
 	rec := httptest.NewRecorder()
-	adminRouter.ServeHTTP(rec, req)
+	adminTestBed.mux.ServeHTTP(rec, req)
 
 	if cmd == statusCmd {
 		expectedInfo := newObjectLayerFn().StorageInfo()
@@ -232,17 +282,11 @@ func mkLockQueryVal(bucket, prefix, relTimeStr string) url.Values {
 
 // Test for locks list management REST API.
 func TestListLocksHandler(t *testing.T) {
-	// reset globals.
+	adminTestBed, err := prepareAdminXLTestBed()
-	// this is to make sure that the tests are not affected by modified globals.
-	resetTestGlobals()
-	// initialize NSLock.
-	initNSLock(false)
-
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
-	defer removeAll(rootPath)
+	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
 	eps, err := parseStorageEndpoints([]string{"http://localhost"})
@@ -254,10 +298,6 @@ func TestListLocksHandler(t *testing.T) {
 	globalMinioAddr = eps[0].Host
 	initGlobalAdminPeers(eps)
 
-	// Setup admin mgmt REST API handlers.
-	adminRouter := router.NewRouter()
-	registerAdminRouter(adminRouter)
-
 	testCases := []struct {
 		bucket         string
 		prefix         string
@@ -308,7 +348,7 @@ func TestListLocksHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminRouter.ServeHTTP(rec, req)
+		adminTestBed.mux.ServeHTTP(rec, req)
 		if test.expectedStatus != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
 		}
@@ -317,17 +357,11 @@ func TestListLocksHandler(t *testing.T) {
 
 // Test for locks clear management REST API.
 func TestClearLocksHandler(t *testing.T) {
-	// reset globals.
+	adminTestBed, err := prepareAdminXLTestBed()
-	// this is to make sure that the tests are not affected by modified globals.
-	resetTestGlobals()
-	// initialize NSLock.
-	initNSLock(false)
-
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
-	defer removeAll(rootPath)
+	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
 	eps, err := parseStorageEndpoints([]string{"http://localhost"})
@@ -336,10 +370,6 @@ func TestClearLocksHandler(t *testing.T) {
 	}
 	initGlobalAdminPeers(eps)
 
-	// Setup admin mgmt REST API handlers.
-	adminRouter := router.NewRouter()
-	registerAdminRouter(adminRouter)
-
 	testCases := []struct {
 		bucket         string
 		prefix         string
@@ -390,7 +420,7 @@ func TestClearLocksHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminRouter.ServeHTTP(rec, req)
+		adminTestBed.mux.ServeHTTP(rec, req)
 		if test.expectedStatus != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
 		}
@@ -556,36 +586,19 @@ func TestValidateHealQueryParams(t *testing.T) {
 
 // TestListObjectsHeal - Test for ListObjectsHealHandler.
 func TestListObjectsHealHandler(t *testing.T) {
-	rootPath, err := newTestConfig("us-east-1")
+	adminTestBed, err := prepareAdminXLTestBed()
 	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
-	defer removeAll(rootPath)
+	defer adminTestBed.TearDown()
 
-	// Initializing objectLayer and corresponding []StorageAPI
+	err = adminTestBed.objLayer.MakeBucket("mybucket")
-	// since ListObjectsHeal() method requires it.
-	objLayer, xlDirs, xlErr := prepareXL()
-	if xlErr != nil {
-		t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
-	}
-	defer removeRoots(xlDirs)
-
-	err = objLayer.MakeBucket("mybucket")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
 
 	// Delete bucket after running all test cases.
-	defer objLayer.DeleteBucket("mybucket")
+	defer adminTestBed.objLayer.DeleteBucket("mybucket")
-
-	// Make objLayer available to all internal services via globalObjectAPI.
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
-	// Setup admin mgmt REST API handlers.
-	adminRouter := router.NewRouter()
-	registerAdminRouter(adminRouter)
 
 	testCases := []struct {
 		bucket     string
@@ -695,7 +708,7 @@ func TestListObjectsHealHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign list objects needing heal request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminRouter.ServeHTTP(rec, req)
+		adminTestBed.mux.ServeHTTP(rec, req)
 		if test.statusCode != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
 		}
@@ -704,36 +717,19 @@ func TestListObjectsHealHandler(t *testing.T) {
 
 // TestHealBucketHandler - Test for HealBucketHandler.
 func TestHealBucketHandler(t *testing.T) {
-	rootPath, err := newTestConfig("us-east-1")
+	adminTestBed, err := prepareAdminXLTestBed()
 	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
-	defer removeAll(rootPath)
+	defer adminTestBed.TearDown()
 
-	// Initializing objectLayer and corresponding []StorageAPI
+	err = adminTestBed.objLayer.MakeBucket("mybucket")
-	// since MakeBucket() and DeleteBucket() methods requires it.
-	objLayer, xlDirs, xlErr := prepareXL()
-	if xlErr != nil {
-		t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
-	}
-	defer removeRoots(xlDirs)
-
-	err = objLayer.MakeBucket("mybucket")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
 
 	// Delete bucket after running all test cases.
-	defer objLayer.DeleteBucket("mybucket")
+	defer adminTestBed.objLayer.DeleteBucket("mybucket")
-
-	// Make objLayer available to all internal services via globalObjectAPI.
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
-	// Setup admin mgmt REST API handlers.
-	adminRouter := router.NewRouter()
-	registerAdminRouter(adminRouter)
 
 	testCases := []struct {
 		bucket     string
@@ -771,7 +767,8 @@ func TestHealBucketHandler(t *testing.T) {
 
 		req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
 		if err != nil {
-			t.Fatalf("Test %d - Failed to construct heal bucket request - %v", i+1, err)
+			t.Fatalf("Test %d - Failed to construct heal bucket request - %v",
+				i+1, err)
 		}
 
 		req.Header.Set(minioAdminOpHeader, "bucket")
@@ -779,12 +776,14 @@ func TestHealBucketHandler(t *testing.T) {
 		cred := serverConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
-			t.Fatalf("Test %d - Failed to sign heal bucket request - %v", i+1, err)
+			t.Fatalf("Test %d - Failed to sign heal bucket request - %v",
+				i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminRouter.ServeHTTP(rec, req)
+		adminTestBed.mux.ServeHTTP(rec, req)
 		if test.statusCode != rec.Code {
-			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
+			t.Errorf("Test %d - Expected HTTP status code %d but received %d",
+				i+1, test.statusCode, rec.Code)
 		}
 
 	}
@@ -792,29 +791,22 @@ func TestHealBucketHandler(t *testing.T) {
 
 // TestHealObjectHandler - Test for HealObjectHandler.
 func TestHealObjectHandler(t *testing.T) {
-	rootPath, err := newTestConfig("us-east-1")
+	adminTestBed, err := prepareAdminXLTestBed()
 	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
-	defer removeAll(rootPath)
+	defer adminTestBed.TearDown()
-
-	// Initializing objectLayer and corresponding []StorageAPI
-	// since MakeBucket(), PutObject() and DeleteBucket() method requires it.
-	objLayer, xlDirs, xlErr := prepareXL()
-	if xlErr != nil {
-		t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
-	}
-	defer removeRoots(xlDirs)
 
 	// Create an object myobject under bucket mybucket.
 	bucketName := "mybucket"
 	objName := "myobject"
-	err = objLayer.MakeBucket(bucketName)
+	err = adminTestBed.objLayer.MakeBucket(bucketName)
 	if err != nil {
 		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
 	}
 
-	_, err = objLayer.PutObject(bucketName, objName, int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
+	_, err = adminTestBed.objLayer.PutObject(bucketName, objName,
+		int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
 	if err != nil {
 		t.Fatalf("Failed to create %s - %v", objName, err)
 	}
@@ -823,16 +815,7 @@ func TestHealObjectHandler(t *testing.T) {
 	defer func(objLayer ObjectLayer, bucketName, objName string) {
 		objLayer.DeleteObject(bucketName, objName)
 		objLayer.DeleteBucket(bucketName)
-	}(objLayer, bucketName, objName)
+	}(adminTestBed.objLayer, bucketName, objName)
-
-	// Make objLayer available to all internal services via globalObjectAPI.
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
-	// Setup admin mgmt REST API handlers.
-	adminRouter := router.NewRouter()
-	registerAdminRouter(adminRouter)
 
 	testCases := []struct {
 		bucket     string
@@ -899,9 +882,42 @@ func TestHealObjectHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign heal object request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminRouter.ServeHTTP(rec, req)
+		adminTestBed.mux.ServeHTTP(rec, req)
 		if test.statusCode != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
 		}
 	}
 }
+
+// TestHealFormatHandler - test for HealFormatHandler.
+func TestHealFormatHandler(t *testing.T) {
+	adminTestBed, err := prepareAdminXLTestBed()
+	if err != nil {
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+	}
+	defer adminTestBed.TearDown()
+
+	// Prepare query params for heal-format mgmt REST API.
+	queryVal := url.Values{}
+	queryVal.Set("heal", "")
+	req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct heal object request - %v", err)
+	}
+
+	// Set x-minio-operation header to format.
+	req.Header.Set(minioAdminOpHeader, "format")
+
+	// Sign the request using signature v4.
+	cred := serverConfig.GetCredential()
+	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
+	if err != nil {
+		t.Fatalf("Failed to sign heal object request - %v", err)
+	}
+
+	rec := httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Errorf("Expected to succeed but failed with %d", rec.Code)
+	}
+}

cmd/admin-router.go

@@ -57,4 +57,6 @@ func registerAdminRouter(mux *router.Router) {
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "bucket").HandlerFunc(adminAPI.HealBucketHandler)
 	// Heal Objects.
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
+	// Heal Format.
+	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
 }

cmd/admin-rpc-client.go

@@ -38,6 +38,7 @@ type remoteAdminClient struct {
 type adminCmdRunner interface {
 	Restart() error
 	ListLocks(bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error)
+	ReInitDisks() error
 }
 
 // Restart - Sends a message over channel to the go-routine
@@ -73,6 +74,20 @@ func (rc remoteAdminClient) ListLocks(bucket, prefix string, relTime time.Durati
 	return reply.volLocks, nil
 }
 
+// ReInitDisks - There is nothing to do here, heal format REST API
+// handler has already formatted and reinitialized the local disks.
+func (lc localAdminClient) ReInitDisks() error {
+	return nil
+}
+
+// ReInitDisks - Signals peers via RPC to reinitialize their disks and
+// object layer.
+func (rc remoteAdminClient) ReInitDisks() error {
+	args := AuthRPCArgs{}
+	reply := AuthRPCReply{}
+	return rc.Call("Admin.ReInitDisks", &args, &reply)
+}
+
 // adminPeer - represents an entity that implements Restart methods.
 type adminPeer struct {
 	addr      string
@@ -159,6 +174,8 @@ func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
 	errs[0] = invokeServiceCmd(cps[0], cmd)
 }
 
+// listPeerLocksInfo - fetch list of locks held on the given bucket,
+// matching prefix older than relTime from all peer servers.
 func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) {
 	// Used to aggregate volume lock information from all nodes.
 	allLocks := make([][]VolumeLockInfo, len(peers))
@@ -206,3 +223,21 @@ func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Dur
 	}
 	return groupedLockInfos, nil
 }
+
+// reInitPeerDisks - reinitialize disks and object layer on peer servers to use the new format.
+func reInitPeerDisks(peers adminPeers) error {
+	errs := make([]error, len(peers))
+
+	// Send ReInitDisks RPC call to all nodes.
+	// for local adminPeer this is a no-op.
+	wg := sync.WaitGroup{}
+	for i, peer := range peers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			errs[idx] = peer.cmdRunner.ReInitDisks()
+		}(i, peer)
+	}
+	wg.Wait()
+	return nil
+}

cmd/admin-rpc-server.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"errors"
 	"net/rpc"
 	"time"
 
@@ -25,6 +26,8 @@ import (
 
 const adminPath = "/admin"
 
+var errUnsupportedBackend = errors.New("not supported for non erasure-code backend")
+
 // adminCmd - exports RPC methods for service status, stop and
 // restart commands.
 type adminCmd struct {
@@ -57,11 +60,51 @@ func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
 
 // ListLocks - lists locks held by requests handled by this server instance.
 func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error {
+	if err := query.IsAuthenticated(); err != nil {
+		return err
+	}
 	volLocks := listLocksInfo(query.bucket, query.prefix, query.relTime)
 	*reply = ListLocksReply{volLocks: volLocks}
 	return nil
 }
 
+// ReInitDisk - reinitialize storage disks and object layer to use the
+// new format.
+func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
+	if err := args.IsAuthenticated(); err != nil {
+		return err
+	}
+
+	if !globalIsXL {
+		return errUnsupportedBackend
+	}
+
+	// Get the current object layer instance.
+	objLayer := newObjectLayerFn()
+
+	// Initialize new disks to include the newly formatted disks.
+	bootstrapDisks, err := initStorageDisks(globalEndpoints)
+	if err != nil {
+		return err
+	}
+
+	// Initialize new object layer with newly formatted disks.
+	newObjectAPI, err := newXLObjects(bootstrapDisks)
+	if err != nil {
+		return err
+	}
+
+	// Replace object layer with newly formatted storage.
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = newObjectAPI
+	globalObjLayerMutex.Unlock()
+
+	// Shutdown storage belonging to old object layer instance.
+	objLayer.Shutdown()
+
+	return nil
+}
+
 // registerAdminRPCRouter - registers RPC methods for service status,
 // stop and restart commands.
 func registerAdminRPCRouter(mux *router.Router) error {

cmd/admin-rpc-server_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"net/url"
 	"testing"
 	"time"
 )
@@ -61,6 +62,85 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 	}
 }
 
+// TestAdminRestart - test for Admin.Restart RPC service.
 func TestAdminRestart(t *testing.T) {
 	testAdminCmd(restartCmd, t)
 }
+
+// TestReInitDisks - test for Admin.ReInitDisks RPC service.
+func TestReInitDisks(t *testing.T) {
+	// Reset global variables to start afresh.
+	resetTestGlobals()
+
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Unable to initialize server config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// Initializing objectLayer for HealFormatHandler.
+	_, xlDirs, xlErr := initTestXLObjLayer()
+	if xlErr != nil {
+		t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
+	}
+	defer removeRoots(xlDirs)
+
+	// Set globalEndpoints for a single node XL setup.
+	for _, xlDir := range xlDirs {
+		globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
+	}
+
+	// Setup admin rpc server for an XL backend.
+	globalIsXL = true
+	adminServer := adminCmd{}
+	creds := serverConfig.GetCredential()
+	args := LoginRPCArgs{
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
+		Version:     Version,
+		RequestTime: time.Now().UTC(),
+	}
+	reply := LoginRPCReply{}
+	err = adminServer.Login(&args, &reply)
+	if err != nil {
+		t.Fatalf("Failed to login to admin server - %v", err)
+	}
+
+	authArgs := AuthRPCArgs{
+		AuthToken:   reply.AuthToken,
+		RequestTime: time.Now().UTC(),
+	}
+	authReply := AuthRPCReply{}
+
+	err = adminServer.ReInitDisks(&authArgs, &authReply)
+	if err != nil {
+		t.Errorf("Expected to pass, but failed with %v", err)
+	}
+
+	// Negative test case with admin rpc server setup for FS.
+	globalIsXL = false
+	fsAdminServer := adminCmd{}
+	fsArgs := LoginRPCArgs{
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
+		Version:     Version,
+		RequestTime: time.Now().UTC(),
+	}
+	fsReply := LoginRPCReply{}
+	err = fsAdminServer.Login(&fsArgs, &fsReply)
+	if err != nil {
+		t.Fatalf("Failed to login to fs admin server - %v", err)
+	}
+
+	authArgs = AuthRPCArgs{
+		AuthToken:   fsReply.AuthToken,
+		RequestTime: time.Now().UTC(),
+	}
+	authReply = AuthRPCReply{}
+	// Attempt ReInitDisks service on a FS backend.
+	err = fsAdminServer.ReInitDisks(&authArgs, &authReply)
+	if err != errUnsupportedBackend {
+		t.Errorf("Expected to fail with %v, but received %v",
+			errUnsupportedBackend, err)
+	}
+}

cmd/globals.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"crypto/x509"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
@@ -70,6 +71,9 @@ var (
 	// Indicates if the running minio server is distributed setup.
 	globalIsDistXL = false
 
+	// Indicates if the running minio server is an erasure-code backend.
+	globalIsXL = false
+
 	// This flag is set to 'true' by default, it is set to `false`
 	// when MINIO_BROWSER env is set to 'off'.
 	globalIsBrowserEnabled = !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off")
@@ -112,6 +116,9 @@ var (
 	// Secret key passed from the environment
 	globalEnvSecretKey = os.Getenv("MINIO_SECRET_KEY")
 
+	// url.URL endpoints of disks that belong to the object storage.
+	globalEndpoints = []*url.URL{}
+
 	// Add new variable global values here.
 )
 

cmd/server-main.go

@@ -418,6 +418,12 @@ func serverMain(c *cli.Context) {
 		fatalIf(initDsyncNodes(endpoints), "Unable to initialize distributed locking clients")
 	}
 
+	// Set globalIsXL if erasure code backend is about to be
+	// initialized for the given endpoints.
+	if len(endpoints) > 1 {
+		globalIsXL = true
+	}
+
 	// Initialize name space lock.
 	initNSLock(globalIsDistXL)
 
@@ -453,6 +459,9 @@ func serverMain(c *cli.Context) {
 		fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server.")
 	}()
 
+	// Set endpoints of []*url.URL type to globalEndpoints.
+	globalEndpoints = endpoints
+
 	newObject, err := newObjectLayer(srvConfig)
 	fatalIf(err, "Initializing object layer failed")
 

cmd/test-utils_test.go

@@ -488,6 +488,14 @@ func resetGlobalEventnotify() {
 	globalEventNotifier = nil
 }
 
+func resetGlobalEndpoints() {
+	globalEndpoints = []*url.URL{}
+}
+
+func resetGlobalIsXL() {
+	globalIsXL = false
+}
+
 // Resets all the globals used modified in tests.
 // Resetting ensures that the changes made to globals by one test doesn't affect others.
 func resetTestGlobals() {
@@ -501,6 +509,10 @@ func resetTestGlobals() {
 	resetGlobalNSLock()
 	// Reset global event notifier.
 	resetGlobalEventnotify()
+	// Reset global endpoints.
+	resetGlobalEndpoints()
+	// Reset global isXL flag.
+	resetGlobalIsXL()
 }
 
 // Configure the server for the test run.

cmd/xl-v1.go

@@ -163,6 +163,14 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
 // Shutdown function for object storage interface.
 func (xl xlObjects) Shutdown() error {
 	// Add any object layer shutdown activities here.
+	for _, disk := range xl.storageDisks {
+		// This closes storage rpc client connections if any.
+		// Otherwise this is a no-op.
+		if disk == nil {
+			continue
+		}
+		disk.Close()
+	}
 	return nil
 }
 

pkg/madmin/API.md

@@ -38,8 +38,11 @@ func main() {
 
 | Service operations|LockInfo operations|Healing operations|
 |:---|:---|:---|
-|[`ServiceStatus`](#ServiceStatus)| | |
+|[`ServiceStatus`](#ServiceStatus)| [`ListLocks`](#ListLocks)| [`ListObjectsHeal`](#ListObjectsHeal)|
-|[`ServiceRestart`](#ServiceRestart)| | |
+|[`ServiceRestart`](#ServiceRestart)| [`ClearLocks`](#ClearLocks)| [`ListBucketsHeal`](#ListBucketsHeal)|
+| | |[`HealBucket`](#HealBucket) |
+| | |[`HealObject`](#HealObject)|
+| | |[`HealFormat`](#HealFormat)|
 
 ## 1. Constructor
 <a name="Minio"></a>
@@ -185,8 +188,8 @@ __Example__
     }
 ```
 
-<a name="ListBucketsList"></a>
+<a name="ListBucketsHeal"></a>
-### ListBucketsList() error
+### ListBucketsHeal() error
 If successful returns information on the list of buckets that need healing.
 
 __Example__
@@ -244,3 +247,19 @@ __Example__
     log.Println("successfully healed mybucket/myobject")
 
 ```
+
+<a name="HealFormat"></a>
+### HealFormat() error
+Heal storage format on available disks. This is used when disks were replaced or were found with missing format. This is supported only for erasure-coded backend.
+
+__Example__
+
+``` go
+    err := madmClnt.HealFormat()
+    if err != nil {
+        log.Fatalln(err)
+    }
+
+    log.Println("successfully healed storage format on available disks.")
+
+```

pkg/madmin/examples/heal-format.go

@@ -0,0 +1,49 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an Minio Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	// Heal storage format on available disks.
+	err = madmClnt.HealFormat()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	log.Println("successfully healed storage format on available disks.")
+}

pkg/madmin/heal-commands.go

@@ -403,3 +403,32 @@ func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) error {
 
 	return nil
 }
+
+// HealFormat - heal storage format on available disks.
+func (adm *AdminClient) HealFormat() error {
+	queryVal := url.Values{}
+	queryVal.Set("heal", "")
+
+	// Set x-minio-operation to format.
+	hdrs := make(http.Header)
+	hdrs.Set(minioAdminOpHeader, "format")
+
+	reqData := requestData{
+		queryValues:   queryVal,
+		customHeaders: hdrs,
+	}
+
+	// Execute POST on /?heal to heal storage format.
+	resp, err := adm.executeMethod("POST", reqData)
+
+	defer closeResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.New("Got HTTP Status: " + resp.Status)
+	}
+
+	return nil
+}