root
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Implement mgmt REST APIs to heal storage format. (#3604) 

* Implement heal format REST API handler
* Implement admin peer rpc handler to re-initialize storage
* Implement HealFormat API in pkg/madmin
* Update pkg/madmin API.md to incl. HealFormat
* Added unit tests for ReInitDisks rpc handler and HealFormatHandler
This commit is contained in:
Krishnan Parthasarathi 2017-01-23 14:02:55 +05:30 committed by Harshavardhana
parent 4e926b292f
commit 586058f079
13 changed files with 511 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
cmd/admin-handlers.go
View File

@ -19,6 +19,7 @@ package cmd
import (
"encoding/json"
"encoding/xml"
"fmt"
"io/ioutil"
"net/http"
"net/url"
@ -374,6 +375,7 @@ func (adminAPI adminAPIHandlers) ListBucketsHealHandler(w http.ResponseWriter, r
}
// HealBucketHandler - POST /?heal&bucket=mybucket
// - x-minio-operation = bucket
// - bucket is mandatory query parameter
// Heal a given bucket, if present.
func (adminAPI adminAPIHandlers) HealBucketHandler(w http.ResponseWriter, r *http.Request) {
@ -425,6 +427,7 @@ func isDryRun(qval url.Values) bool {
}
// HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject
// - x-minio-operation = object
// - bucket and object are both mandatory query parameters
// Heal a given object, if present.
func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *http.Request) {
@ -473,3 +476,69 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
// Return 200 on success.
writeSuccessResponseHeadersOnly(w)
}
// HealFormatHandler - POST /?heal
// - x-minio-operation = format
// - bucket and object are both mandatory query parameters
// Heal a given object, if present.
func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *http.Request) {
// Get current object layer instance.
objectAPI := newObjectLayerFn()
if objectAPI == nil {
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
return
}
// Validate request signature.
adminAPIErr := checkRequestAuthType(r, "", "", "")
if adminAPIErr != ErrNone {
writeErrorResponse(w, adminAPIErr, r.URL)
return
}
// Check if this setup is an erasure code backend, since
// heal-format is only applicable to single node XL and
// distributed XL setup.
if !globalIsXL {
writeErrorResponse(w, ErrNotImplemented, r.URL)
return
}
// Create a new set of storage instances to heal format.json.
bootstrapDisks, err := initStorageDisks(globalEndpoints)
if err != nil {
fmt.Println(traceError(err))
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
return
}
// Heal format.json on available storage.
err = healFormatXL(bootstrapDisks)
if err != nil {
fmt.Println(traceError(err))
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
return
}
// Instantiate new object layer with newly formatted storage.
newObjectAPI, err := newXLObjects(bootstrapDisks)
if err != nil {
fmt.Println(traceError(err))
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
return
}
// Set object layer with newly formatted storage to globalObjectAPI.
globalObjLayerMutex.Lock()
globalObjectAPI = newObjectAPI
globalObjLayerMutex.Unlock()
// Shutdown storage belonging to old object layer instance.
objectAPI.Shutdown()
// Inform peers to reinitialize storage with newly formatted storage.
reInitPeerDisks(globalAdminPeers)
// Return 200 on success.
writeSuccessResponseHeadersOnly(w)
}

272
cmd/admin-handlers_test.go
View File

@ -29,6 +29,79 @@ import (
router "github.com/gorilla/mux"
)
// adminXLTestBed - encapsulates subsystems that need to be setup for
// admin-handler unit tests.
type adminXLTestBed struct {
configPath string
xlDirs []string
objLayer ObjectLayer
mux *router.Router
}
// prepareAdminXLTestBed - helper function that setups a single-node
// XL backend for admin-handler tests.
func prepareAdminXLTestBed() (*adminXLTestBed, error) {
// reset global variables to start afresh.
resetTestGlobals()
// Initialize minio server config.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
return nil, err
}
// Initializing objectLayer for HealFormatHandler.
objLayer, xlDirs, xlErr := initTestXLObjLayer()
if xlErr != nil {
return nil, xlErr
}
// Set globalEndpoints for a single node XL setup.
for _, xlDir := range xlDirs {
globalEndpoints = append(globalEndpoints, &url.URL{
Path: xlDir,
})
}
// Set globalIsXL to indicate that the setup uses an erasure code backend.
globalIsXL = true
// initialize NSLock.
isDistXL := false
initNSLock(isDistXL)
// Setup admin mgmt REST API handlers.
adminRouter := router.NewRouter()
registerAdminRouter(adminRouter)
return &adminXLTestBed{
configPath: rootPath,
xlDirs: xlDirs,
objLayer: objLayer,
mux: adminRouter,
}, nil
}
// TearDown - method that resets the test bed for subsequent unit
// tests to start afresh.
func (atb *adminXLTestBed) TearDown() {
removeAll(atb.configPath)
removeRoots(atb.xlDirs)
resetTestGlobals()
}
// initTestObjLayer - Helper function to initialize an XL-based object
// layer and set globalObjectAPI.
func initTestXLObjLayer() (ObjectLayer, []string, error) {
objLayer, xlDirs, xlErr := prepareXL()
if xlErr != nil {
return nil, nil, xlErr
}
// Make objLayer available to all internal services via globalObjectAPI.
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
return objLayer, xlDirs, nil
}
// cmdType - Represents different service subcomands like status, stop
// and restart.
type cmdType int
@ -115,17 +188,11 @@ func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Requ
// testServicesCmdHandler - parametrizes service subcommand tests on
// cmdType value.
func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing.T) {
// reset globals.
// this is to make sure that the tests are not affected by modified value.
resetTestGlobals()
// initialize NSLock.
initNSLock(false)
// Initialize configuration for access/secret credentials.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
defer removeAll(rootPath)
defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls. Note: In a
// single node setup, this degenerates to a simple function
@ -139,29 +206,12 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
if cmd == statusCmd {
// Initializing objectLayer and corresponding
// []StorageAPI since DiskInfo() method requires it.
objLayer, xlDirs, xlErr := prepareXL()
if xlErr != nil {
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
}
defer removeRoots(xlDirs)
// Make objLayer available to all internal services via globalObjectAPI.
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
}
// Setting up a go routine to simulate ServerMux's
// handleServiceSignals for stop and restart commands.
if cmd == restartCmd {
go testServiceSignalReceiver(cmd, t)
}
credentials := serverConfig.GetCredential()
adminRouter := router.NewRouter()
registerAdminRouter(adminRouter)
var body []byte
if cmd == setCreds {
@ -174,7 +224,7 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing
}
rec := httptest.NewRecorder()
adminRouter.ServeHTTP(rec, req)
adminTestBed.mux.ServeHTTP(rec, req)
if cmd == statusCmd {
expectedInfo := newObjectLayerFn().StorageInfo()
@ -232,17 +282,11 @@ func mkLockQueryVal(bucket, prefix, relTimeStr string) url.Values {
// Test for locks list management REST API.
func TestListLocksHandler(t *testing.T) {
// reset globals.
// this is to make sure that the tests are not affected by modified globals.
resetTestGlobals()
// initialize NSLock.
initNSLock(false)
rootPath, err := newTestConfig(globalMinioDefaultRegion)
adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
defer removeAll(rootPath)
defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://localhost"})
@ -254,10 +298,6 @@ func TestListLocksHandler(t *testing.T) {
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
// Setup admin mgmt REST API handlers.
adminRouter := router.NewRouter()
registerAdminRouter(adminRouter)
testCases := []struct {
bucket string
prefix string
@ -308,7 +348,7 @@ func TestListLocksHandler(t *testing.T) {
t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
}
rec := httptest.NewRecorder()
adminRouter.ServeHTTP(rec, req)
adminTestBed.mux.ServeHTTP(rec, req)
if test.expectedStatus != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
}
@ -317,17 +357,11 @@ func TestListLocksHandler(t *testing.T) {
// Test for locks clear management REST API.
func TestClearLocksHandler(t *testing.T) {
// reset globals.
// this is to make sure that the tests are not affected by modified globals.
resetTestGlobals()
// initialize NSLock.
initNSLock(false)
rootPath, err := newTestConfig(globalMinioDefaultRegion)
adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
defer removeAll(rootPath)
defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://localhost"})
@ -336,10 +370,6 @@ func TestClearLocksHandler(t *testing.T) {
}
initGlobalAdminPeers(eps)
// Setup admin mgmt REST API handlers.
adminRouter := router.NewRouter()
registerAdminRouter(adminRouter)
testCases := []struct {
bucket string
prefix string
@ -390,7 +420,7 @@ func TestClearLocksHandler(t *testing.T) {
t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
}
rec := httptest.NewRecorder()
adminRouter.ServeHTTP(rec, req)
adminTestBed.mux.ServeHTTP(rec, req)
if test.expectedStatus != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
}
@ -556,36 +586,19 @@ func TestValidateHealQueryParams(t *testing.T) {
// TestListObjectsHeal - Test for ListObjectsHealHandler.
func TestListObjectsHealHandler(t *testing.T) {
rootPath, err := newTestConfig("us-east-1")
adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
defer removeAll(rootPath)
defer adminTestBed.TearDown()
// Initializing objectLayer and corresponding []StorageAPI
// since ListObjectsHeal() method requires it.
objLayer, xlDirs, xlErr := prepareXL()
if xlErr != nil {
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
}
defer removeRoots(xlDirs)
err = objLayer.MakeBucket("mybucket")
err = adminTestBed.objLayer.MakeBucket("mybucket")
if err != nil {
t.Fatalf("Failed to make bucket - %v", err)
}
// Delete bucket after running all test cases.
defer objLayer.DeleteBucket("mybucket")
// Make objLayer available to all internal services via globalObjectAPI.
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
// Setup admin mgmt REST API handlers.
adminRouter := router.NewRouter()
registerAdminRouter(adminRouter)
defer adminTestBed.objLayer.DeleteBucket("mybucket")
testCases := []struct {
bucket string
@ -695,7 +708,7 @@ func TestListObjectsHealHandler(t *testing.T) {
t.Fatalf("Test %d - Failed to sign list objects needing heal request - %v", i+1, err)
}
rec := httptest.NewRecorder()
adminRouter.ServeHTTP(rec, req)
adminTestBed.mux.ServeHTTP(rec, req)
if test.statusCode != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
}
@ -704,36 +717,19 @@ func TestListObjectsHealHandler(t *testing.T) {
// TestHealBucketHandler - Test for HealBucketHandler.
func TestHealBucketHandler(t *testing.T) {
rootPath, err := newTestConfig("us-east-1")
adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
defer removeAll(rootPath)
defer adminTestBed.TearDown()
// Initializing objectLayer and corresponding []StorageAPI
// since MakeBucket() and DeleteBucket() methods requires it.
objLayer, xlDirs, xlErr := prepareXL()
if xlErr != nil {
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
}
defer removeRoots(xlDirs)
err = objLayer.MakeBucket("mybucket")
err = adminTestBed.objLayer.MakeBucket("mybucket")
if err != nil {
t.Fatalf("Failed to make bucket - %v", err)
}
// Delete bucket after running all test cases.
defer objLayer.DeleteBucket("mybucket")
// Make objLayer available to all internal services via globalObjectAPI.
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
// Setup admin mgmt REST API handlers.
adminRouter := router.NewRouter()
registerAdminRouter(adminRouter)
defer adminTestBed.objLayer.DeleteBucket("mybucket")
testCases := []struct {
bucket string
@ -771,7 +767,8 @@ func TestHealBucketHandler(t *testing.T) {
req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
if err != nil {
t.Fatalf("Test %d - Failed to construct heal bucket request - %v", i+1, err)
t.Fatalf("Test %d - Failed to construct heal bucket request - %v",
i+1, err)
}
req.Header.Set(minioAdminOpHeader, "bucket")
@ -779,12 +776,14 @@ func TestHealBucketHandler(t *testing.T) {
cred := serverConfig.GetCredential()
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
if err != nil {
t.Fatalf("Test %d - Failed to sign heal bucket request - %v", i+1, err)
t.Fatalf("Test %d - Failed to sign heal bucket request - %v",
i+1, err)
}
rec := httptest.NewRecorder()
adminRouter.ServeHTTP(rec, req)
adminTestBed.mux.ServeHTTP(rec, req)
if test.statusCode != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
t.Errorf("Test %d - Expected HTTP status code %d but received %d",
i+1, test.statusCode, rec.Code)
}
}
@ -792,29 +791,22 @@ func TestHealBucketHandler(t *testing.T) {
// TestHealObjectHandler - Test for HealObjectHandler.
func TestHealObjectHandler(t *testing.T) {
rootPath, err := newTestConfig("us-east-1")
adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
defer removeAll(rootPath)
// Initializing objectLayer and corresponding []StorageAPI
// since MakeBucket(), PutObject() and DeleteBucket() method requires it.
objLayer, xlDirs, xlErr := prepareXL()
if xlErr != nil {
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
}
defer removeRoots(xlDirs)
defer adminTestBed.TearDown()
// Create an object myobject under bucket mybucket.
bucketName := "mybucket"
objName := "myobject"
err = objLayer.MakeBucket(bucketName)
err = adminTestBed.objLayer.MakeBucket(bucketName)
if err != nil {
t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
}
_, err = objLayer.PutObject(bucketName, objName, int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
_, err = adminTestBed.objLayer.PutObject(bucketName, objName,
int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
if err != nil {
t.Fatalf("Failed to create %s - %v", objName, err)
}
@ -823,16 +815,7 @@ func TestHealObjectHandler(t *testing.T) {
defer func(objLayer ObjectLayer, bucketName, objName string) {
objLayer.DeleteObject(bucketName, objName)
objLayer.DeleteBucket(bucketName)
}(objLayer, bucketName, objName)
// Make objLayer available to all internal services via globalObjectAPI.
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
// Setup admin mgmt REST API handlers.
adminRouter := router.NewRouter()
registerAdminRouter(adminRouter)
}(adminTestBed.objLayer, bucketName, objName)
testCases := []struct {
bucket string
@ -899,9 +882,42 @@ func TestHealObjectHandler(t *testing.T) {
t.Fatalf("Test %d - Failed to sign heal object request - %v", i+1, err)
}
rec := httptest.NewRecorder()
adminRouter.ServeHTTP(rec, req)
adminTestBed.mux.ServeHTTP(rec, req)
if test.statusCode != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
}
}
}
// TestHealFormatHandler - test for HealFormatHandler.
func TestHealFormatHandler(t *testing.T) {
adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
defer adminTestBed.TearDown()
// Prepare query params for heal-format mgmt REST API.
queryVal := url.Values{}
queryVal.Set("heal", "")
req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
if err != nil {
t.Fatalf("Failed to construct heal object request - %v", err)
}
// Set x-minio-operation header to format.
req.Header.Set(minioAdminOpHeader, "format")
// Sign the request using signature v4.
cred := serverConfig.GetCredential()
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
if err != nil {
t.Fatalf("Failed to sign heal object request - %v", err)
}
rec := httptest.NewRecorder()
adminTestBed.mux.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Errorf("Expected to succeed but failed with %d", rec.Code)
}
}

2
cmd/admin-router.go
View File

@ -57,4 +57,6 @@ func registerAdminRouter(mux *router.Router) {
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "bucket").HandlerFunc(adminAPI.HealBucketHandler)
// Heal Objects.
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
// Heal Format.
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
}

35
cmd/admin-rpc-client.go
View File

@ -38,6 +38,7 @@ type remoteAdminClient struct {
type adminCmdRunner interface {
Restart() error
ListLocks(bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error)
ReInitDisks() error
}
// Restart - Sends a message over channel to the go-routine
@ -73,6 +74,20 @@ func (rc remoteAdminClient) ListLocks(bucket, prefix string, relTime time.Durati
return reply.volLocks, nil
}
// ReInitDisks - There is nothing to do here, heal format REST API
// handler has already formatted and reinitialized the local disks.
func (lc localAdminClient) ReInitDisks() error {
return nil
}
// ReInitDisks - Signals peers via RPC to reinitialize their disks and
// object layer.
func (rc remoteAdminClient) ReInitDisks() error {
args := AuthRPCArgs{}
reply := AuthRPCReply{}
return rc.Call("Admin.ReInitDisks", &args, &reply)
}
// adminPeer - represents an entity that implements Restart methods.
type adminPeer struct {
addr string
@ -159,6 +174,8 @@ func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
errs[0] = invokeServiceCmd(cps[0], cmd)
}
// listPeerLocksInfo - fetch list of locks held on the given bucket,
// matching prefix older than relTime from all peer servers.
func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) {
// Used to aggregate volume lock information from all nodes.
allLocks := make([][]VolumeLockInfo, len(peers))
@ -206,3 +223,21 @@ func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Dur
}
return groupedLockInfos, nil
}
// reInitPeerDisks - reinitialize disks and object layer on peer servers to use the new format.
func reInitPeerDisks(peers adminPeers) error {
errs := make([]error, len(peers))
// Send ReInitDisks RPC call to all nodes.
// for local adminPeer this is a no-op.
wg := sync.WaitGroup{}
for i, peer := range peers {
wg.Add(1)
go func(idx int, peer adminPeer) {
defer wg.Done()
errs[idx] = peer.cmdRunner.ReInitDisks()
}(i, peer)
}
wg.Wait()
return nil
}

43
cmd/admin-rpc-server.go
View File

@ -17,6 +17,7 @@
package cmd
import (
"errors"
"net/rpc"
"time"
@ -25,6 +26,8 @@ import (
const adminPath = "/admin"
var errUnsupportedBackend = errors.New("not supported for non erasure-code backend")
// adminCmd - exports RPC methods for service status, stop and
// restart commands.
type adminCmd struct {
@ -57,11 +60,51 @@ func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
// ListLocks - lists locks held by requests handled by this server instance.
func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error {
if err := query.IsAuthenticated(); err != nil {
return err
}
volLocks := listLocksInfo(query.bucket, query.prefix, query.relTime)
*reply = ListLocksReply{volLocks: volLocks}
return nil
}
// ReInitDisk - reinitialize storage disks and object layer to use the
// new format.
func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
if err := args.IsAuthenticated(); err != nil {
return err
}
if !globalIsXL {
return errUnsupportedBackend
}
// Get the current object layer instance.
objLayer := newObjectLayerFn()
// Initialize new disks to include the newly formatted disks.
bootstrapDisks, err := initStorageDisks(globalEndpoints)
if err != nil {
return err
}
// Initialize new object layer with newly formatted disks.
newObjectAPI, err := newXLObjects(bootstrapDisks)
if err != nil {
return err
}
// Replace object layer with newly formatted storage.
globalObjLayerMutex.Lock()
globalObjectAPI = newObjectAPI
globalObjLayerMutex.Unlock()
// Shutdown storage belonging to old object layer instance.
objLayer.Shutdown()
return nil
}
// registerAdminRPCRouter - registers RPC methods for service status,
// stop and restart commands.
func registerAdminRPCRouter(mux *router.Router) error {

80
cmd/admin-rpc-server_test.go
View File

@ -17,6 +17,7 @@
package cmd
import (
"net/url"
"testing"
"time"
)
@ -61,6 +62,85 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
}
}
// TestAdminRestart - test for Admin.Restart RPC service.
func TestAdminRestart(t *testing.T) {
testAdminCmd(restartCmd, t)
}
// TestReInitDisks - test for Admin.ReInitDisks RPC service.
func TestReInitDisks(t *testing.T) {
// Reset global variables to start afresh.
resetTestGlobals()
rootPath, err := newTestConfig("us-east-1")
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
}
defer removeAll(rootPath)
// Initializing objectLayer for HealFormatHandler.
_, xlDirs, xlErr := initTestXLObjLayer()
if xlErr != nil {
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
}
defer removeRoots(xlDirs)
// Set globalEndpoints for a single node XL setup.
for _, xlDir := range xlDirs {
globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
}
// Setup admin rpc server for an XL backend.
globalIsXL = true
adminServer := adminCmd{}
creds := serverConfig.GetCredential()
args := LoginRPCArgs{
Username: creds.AccessKey,
Password: creds.SecretKey,
Version: Version,
RequestTime: time.Now().UTC(),
}
reply := LoginRPCReply{}
err = adminServer.Login(&args, &reply)
if err != nil {
t.Fatalf("Failed to login to admin server - %v", err)
}
authArgs := AuthRPCArgs{
AuthToken: reply.AuthToken,
RequestTime: time.Now().UTC(),
}
authReply := AuthRPCReply{}
err = adminServer.ReInitDisks(&authArgs, &authReply)
if err != nil {
t.Errorf("Expected to pass, but failed with %v", err)
}
// Negative test case with admin rpc server setup for FS.
globalIsXL = false
fsAdminServer := adminCmd{}
fsArgs := LoginRPCArgs{
Username: creds.AccessKey,
Password: creds.SecretKey,
Version: Version,
RequestTime: time.Now().UTC(),
}
fsReply := LoginRPCReply{}
err = fsAdminServer.Login(&fsArgs, &fsReply)
if err != nil {
t.Fatalf("Failed to login to fs admin server - %v", err)
}
authArgs = AuthRPCArgs{
AuthToken: fsReply.AuthToken,
RequestTime: time.Now().UTC(),
}
authReply = AuthRPCReply{}
// Attempt ReInitDisks service on a FS backend.
err = fsAdminServer.ReInitDisks(&authArgs, &authReply)
if err != errUnsupportedBackend {
t.Errorf("Expected to fail with %v, but received %v",
errUnsupportedBackend, err)
}
}

7
cmd/globals.go
View File

@ -18,6 +18,7 @@ package cmd
import (
"crypto/x509"
"net/url"
"os"
"runtime"
"strings"
@ -70,6 +71,9 @@ var (
// Indicates if the running minio server is distributed setup.
globalIsDistXL = false
// Indicates if the running minio server is an erasure-code backend.
globalIsXL = false
// This flag is set to 'true' by default, it is set to `false`
// when MINIO_BROWSER env is set to 'off'.
globalIsBrowserEnabled = !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off")
@ -112,6 +116,9 @@ var (
// Secret key passed from the environment
globalEnvSecretKey = os.Getenv("MINIO_SECRET_KEY")
// url.URL endpoints of disks that belong to the object storage.
globalEndpoints = []*url.URL{}
// Add new variable global values here.
)

9
cmd/server-main.go
View File

@ -418,6 +418,12 @@ func serverMain(c *cli.Context) {
fatalIf(initDsyncNodes(endpoints), "Unable to initialize distributed locking clients")
}
// Set globalIsXL if erasure code backend is about to be
// initialized for the given endpoints.
if len(endpoints) > 1 {
globalIsXL = true
}
// Initialize name space lock.
initNSLock(globalIsDistXL)
@ -453,6 +459,9 @@ func serverMain(c *cli.Context) {
fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server.")
}()
// Set endpoints of []*url.URL type to globalEndpoints.
globalEndpoints = endpoints
newObject, err := newObjectLayer(srvConfig)
fatalIf(err, "Initializing object layer failed")

12
cmd/test-utils_test.go
View File

@ -488,6 +488,14 @@ func resetGlobalEventnotify() {
globalEventNotifier = nil
}
func resetGlobalEndpoints() {
globalEndpoints = []*url.URL{}
}
func resetGlobalIsXL() {
globalIsXL = false
}
// Resets all the globals used modified in tests.
// Resetting ensures that the changes made to globals by one test doesn't affect others.
func resetTestGlobals() {
@ -501,6 +509,10 @@ func resetTestGlobals() {
resetGlobalNSLock()
// Reset global event notifier.
resetGlobalEventnotify()
// Reset global endpoints.
resetGlobalEndpoints()
// Reset global isXL flag.
resetGlobalIsXL()
}
// Configure the server for the test run.

8
cmd/xl-v1.go
View File

@ -163,6 +163,14 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
// Shutdown function for object storage interface.
func (xl xlObjects) Shutdown() error {
// Add any object layer shutdown activities here.
for _, disk := range xl.storageDisks {
// This closes storage rpc client connections if any.
// Otherwise this is a no-op.
if disk == nil {
continue
}
disk.Close()
}
return nil
}

27
pkg/madmin/API.md
View File

@ -38,8 +38,11 @@ func main() {
| Service operations|LockInfo operations|Healing operations|
|:---|:---|:---|
|[`ServiceStatus`](#ServiceStatus)| | |
|[`ServiceRestart`](#ServiceRestart)| | |
|[`ServiceStatus`](#ServiceStatus)| [`ListLocks`](#ListLocks)| [`ListObjectsHeal`](#ListObjectsHeal)|
|[`ServiceRestart`](#ServiceRestart)| [`ClearLocks`](#ClearLocks)| [`ListBucketsHeal`](#ListBucketsHeal)|
| | |[`HealBucket`](#HealBucket) |
| | |[`HealObject`](#HealObject)|
| | |[`HealFormat`](#HealFormat)|
## 1. Constructor
<a name="Minio"></a>
@ -185,8 +188,8 @@ __Example__
}
```
<a name="ListBucketsList"></a>
### ListBucketsList() error
<a name="ListBucketsHeal"></a>
### ListBucketsHeal() error
If successful returns information on the list of buckets that need healing.
__Example__
@ -244,3 +247,19 @@ __Example__
log.Println("successfully healed mybucket/myobject")
```
<a name="HealFormat"></a>
### HealFormat() error
Heal storage format on available disks. This is used when disks were replaced or were found with missing format. This is supported only for erasure-coded backend.
__Example__
``` go
err := madmClnt.HealFormat()
if err != nil {
log.Fatalln(err)
}
log.Println("successfully healed storage format on available disks.")
```

49
pkg/madmin/examples/heal-format.go Normal file
View File

@ -0,0 +1,49 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"log"
"github.com/minio/minio/pkg/madmin"
)
func main() {
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
// dummy values, please replace them with original values.
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
// dummy values, please replace them with original values.
// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
// New returns an Minio Admin client object.
madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
if err != nil {
log.Fatalln(err)
}
// Heal storage format on available disks.
err = madmClnt.HealFormat()
if err != nil {
log.Fatalln(err)
}
log.Println("successfully healed storage format on available disks.")
}

29
pkg/madmin/heal-commands.go
View File

@ -403,3 +403,32 @@ func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) error {
return nil
}
// HealFormat - heal storage format on available disks.
func (adm *AdminClient) HealFormat() error {
queryVal := url.Values{}
queryVal.Set("heal", "")
// Set x-minio-operation to format.
hdrs := make(http.Header)
hdrs.Set(minioAdminOpHeader, "format")
reqData := requestData{
queryValues: queryVal,
customHeaders: hdrs,
}
// Execute POST on /?heal to heal storage format.
resp, err := adm.executeMethod("POST", reqData)
defer closeResponse(resp)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.New("Got HTTP Status: " + resp.Status)
}
return nil
}