| 
									
										
										
										
											2021-04-19 03:41:13 +08:00
										 |  |  | // Copyright (c) 2015-2021 MinIO, Inc.
 | 
					
						
							|  |  |  | //
 | 
					
						
							|  |  |  | // This file is part of MinIO Object Storage stack
 | 
					
						
							|  |  |  | //
 | 
					
						
							|  |  |  | // This program is free software: you can redistribute it and/or modify
 | 
					
						
							|  |  |  | // it under the terms of the GNU Affero General Public License as published by
 | 
					
						
							|  |  |  | // the Free Software Foundation, either version 3 of the License, or
 | 
					
						
							|  |  |  | // (at your option) any later version.
 | 
					
						
							|  |  |  | //
 | 
					
						
							|  |  |  | // This program is distributed in the hope that it will be useful
 | 
					
						
							|  |  |  | // but WITHOUT ANY WARRANTY; without even the implied warranty of
 | 
					
						
							|  |  |  | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 | 
					
						
							|  |  |  | // GNU Affero General Public License for more details.
 | 
					
						
							|  |  |  | //
 | 
					
						
							|  |  |  | // You should have received a copy of the GNU Affero General Public License
 | 
					
						
							|  |  |  | // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | 
 | 
					
						
							|  |  |  | package cmd | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | import ( | 
					
						
							| 
									
										
										
										
											2020-02-21 13:59:57 +08:00
										 |  |  | 	"bytes" | 
					
						
							| 
									
										
										
										
											2020-06-18 05:49:26 +08:00
										 |  |  | 	"context" | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | 	"io" | 
					
						
							|  |  |  | 	"net/url" | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-06-02 05:59:40 +08:00
										 |  |  | 	"github.com/minio/minio/internal/dsync" | 
					
						
							|  |  |  | 	"github.com/minio/minio/internal/http" | 
					
						
							|  |  |  | 	xhttp "github.com/minio/minio/internal/http" | 
					
						
							|  |  |  | 	"github.com/minio/minio/internal/rest" | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // lockRESTClient is authenticable lock REST client
 | 
					
						
							|  |  |  | type lockRESTClient struct { | 
					
						
							|  |  |  | 	restClient *rest.Client | 
					
						
							| 
									
										
										
										
											2020-12-10 23:28:37 +08:00
										 |  |  | 	u          *url.URL | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-08-29 07:12:57 +08:00
										 |  |  | func toLockError(err error) error { | 
					
						
							|  |  |  | 	if err == nil { | 
					
						
							|  |  |  | 		return nil | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	switch err.Error() { | 
					
						
							|  |  |  | 	case errLockConflict.Error(): | 
					
						
							|  |  |  | 		return errLockConflict | 
					
						
							| 
									
										
										
										
											2021-03-04 10:36:43 +08:00
										 |  |  | 	case errLockNotFound.Error(): | 
					
						
							|  |  |  | 		return errLockNotFound | 
					
						
							| 
									
										
										
										
											2019-08-29 07:12:57 +08:00
										 |  |  | 	} | 
					
						
							|  |  |  | 	return err | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-11-14 04:17:45 +08:00
										 |  |  | // String stringer *dsync.NetLocker* interface compatible method.
 | 
					
						
							|  |  |  | func (client *lockRESTClient) String() string { | 
					
						
							| 
									
										
										
										
											2020-12-10 23:28:37 +08:00
										 |  |  | 	return client.u.String() | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
 | 
					
						
							|  |  |  | // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 | 
					
						
							|  |  |  | // after verifying format.json
 | 
					
						
							| 
									
										
										
										
											2020-07-30 14:15:34 +08:00
										 |  |  | func (client *lockRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) { | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | 	if values == nil { | 
					
						
							|  |  |  | 		values = make(url.Values) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-09-05 00:45:06 +08:00
										 |  |  | 	respBody, err = client.restClient.Call(ctx, method, values, body, length) | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | 	if err == nil { | 
					
						
							|  |  |  | 		return respBody, nil | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-08-29 07:12:57 +08:00
										 |  |  | 	return nil, toLockError(err) | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // IsOnline - returns whether REST client failed to connect or not.
 | 
					
						
							|  |  |  | func (client *lockRESTClient) IsOnline() bool { | 
					
						
							| 
									
										
										
										
											2020-06-17 09:59:32 +08:00
										 |  |  | 	return client.restClient.IsOnline() | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-10-09 03:32:32 +08:00
										 |  |  | // Not a local locker
 | 
					
						
							|  |  |  | func (client *lockRESTClient) IsLocal() bool { | 
					
						
							|  |  |  | 	return false | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | // Close - marks the client as closed.
 | 
					
						
							|  |  |  | func (client *lockRESTClient) Close() error { | 
					
						
							|  |  |  | 	client.restClient.Close() | 
					
						
							|  |  |  | 	return nil | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // restCall makes a call to the lock REST server.
 | 
					
						
							| 
									
										
										
										
											2020-07-30 14:15:34 +08:00
										 |  |  | func (client *lockRESTClient) restCall(ctx context.Context, call string, args dsync.LockArgs) (reply bool, err error) { | 
					
						
							| 
									
										
										
										
											2021-10-01 02:53:01 +08:00
										 |  |  | 	argsBytes, err := args.MarshalMsg(metaDataPoolGet()[:0]) | 
					
						
							|  |  |  | 	if err != nil { | 
					
						
							|  |  |  | 		return false, err | 
					
						
							| 
									
										
										
										
											2020-02-21 13:59:57 +08:00
										 |  |  | 	} | 
					
						
							| 
									
										
										
										
											2021-10-01 02:53:01 +08:00
										 |  |  | 	defer metaDataPoolPut(argsBytes) | 
					
						
							|  |  |  | 	body := bytes.NewReader(argsBytes) | 
					
						
							|  |  |  | 	respBody, err := client.callWithContext(ctx, call, nil, body, body.Size()) | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | 	defer http.DrainBody(respBody) | 
					
						
							| 
									
										
										
										
											2019-08-29 07:12:57 +08:00
										 |  |  | 	switch err { | 
					
						
							|  |  |  | 	case nil: | 
					
						
							|  |  |  | 		return true, nil | 
					
						
							| 
									
										
										
										
											2021-03-04 10:36:43 +08:00
										 |  |  | 	case errLockConflict, errLockNotFound: | 
					
						
							| 
									
										
										
										
											2019-08-29 07:12:57 +08:00
										 |  |  | 		return false, nil | 
					
						
							|  |  |  | 	default: | 
					
						
							|  |  |  | 		return false, err | 
					
						
							|  |  |  | 	} | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // RLock calls read lock REST API.
 | 
					
						
							| 
									
										
										
										
											2020-07-30 14:15:34 +08:00
										 |  |  | func (client *lockRESTClient) RLock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { | 
					
						
							|  |  |  | 	return client.restCall(ctx, lockRESTMethodRLock, args) | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // Lock calls lock REST API.
 | 
					
						
							| 
									
										
										
										
											2020-07-30 14:15:34 +08:00
										 |  |  | func (client *lockRESTClient) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { | 
					
						
							|  |  |  | 	return client.restCall(ctx, lockRESTMethodLock, args) | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // RUnlock calls read unlock REST API.
 | 
					
						
							| 
									
										
										
										
											2021-05-11 17:11:29 +08:00
										 |  |  | func (client *lockRESTClient) RUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { | 
					
						
							|  |  |  | 	return client.restCall(ctx, lockRESTMethodRUnlock, args) | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-03-04 10:36:43 +08:00
										 |  |  | // RUnlock calls read unlock REST API.
 | 
					
						
							|  |  |  | func (client *lockRESTClient) Refresh(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { | 
					
						
							|  |  |  | 	return client.restCall(ctx, lockRESTMethodRefresh, args) | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | // Unlock calls write unlock RPC.
 | 
					
						
							| 
									
										
										
										
											2021-05-11 17:11:29 +08:00
										 |  |  | func (client *lockRESTClient) Unlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { | 
					
						
							|  |  |  | 	return client.restCall(ctx, lockRESTMethodUnlock, args) | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-01-26 02:01:27 +08:00
										 |  |  | // ForceUnlock calls force unlock handler to forcibly unlock an active lock.
 | 
					
						
							|  |  |  | func (client *lockRESTClient) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { | 
					
						
							|  |  |  | 	return client.restCall(ctx, lockRESTMethodForceUnlock, args) | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-11-14 04:17:45 +08:00
										 |  |  | func newLockAPI(endpoint Endpoint) dsync.NetLocker { | 
					
						
							|  |  |  | 	if endpoint.IsLocal { | 
					
						
							| 
									
										
										
										
											2020-12-10 23:28:37 +08:00
										 |  |  | 		return globalLockServer | 
					
						
							| 
									
										
										
										
											2019-11-14 04:17:45 +08:00
										 |  |  | 	} | 
					
						
							|  |  |  | 	return newlockRESTClient(endpoint) | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // Returns a lock rest client.
 | 
					
						
							| 
									
										
										
										
											2019-11-14 04:17:45 +08:00
										 |  |  | func newlockRESTClient(endpoint Endpoint) *lockRESTClient { | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | 	serverURL := &url.URL{ | 
					
						
							| 
									
										
										
										
											2019-11-14 04:17:45 +08:00
										 |  |  | 		Scheme: endpoint.Scheme, | 
					
						
							|  |  |  | 		Host:   endpoint.Host, | 
					
						
							| 
									
										
										
										
											2020-12-10 23:28:37 +08:00
										 |  |  | 		Path:   pathJoin(lockRESTPrefix, lockRESTVersion), | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-11-02 23:43:11 +08:00
										 |  |  | 	restClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken) | 
					
						
							| 
									
										
										
										
											2020-10-30 00:52:11 +08:00
										 |  |  | 	restClient.ExpectTimeouts = true | 
					
						
							| 
									
										
										
										
											2020-11-11 01:28:23 +08:00
										 |  |  | 	// Use a separate client to avoid recursive calls.
 | 
					
						
							|  |  |  | 	healthClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken) | 
					
						
							|  |  |  | 	healthClient.ExpectTimeouts = true | 
					
						
							| 
									
										
										
										
											2021-06-09 05:09:26 +08:00
										 |  |  | 	healthClient.NoMetrics = true | 
					
						
							| 
									
										
										
										
											2020-06-18 05:49:26 +08:00
										 |  |  | 	restClient.HealthCheckFn = func() bool { | 
					
						
							| 
									
										
										
										
											2021-01-29 05:38:12 +08:00
										 |  |  | 		ctx, cancel := context.WithTimeout(context.Background(), restClient.HealthCheckTimeout) | 
					
						
							| 
									
										
										
										
											2020-11-11 01:28:23 +08:00
										 |  |  | 		defer cancel() | 
					
						
							|  |  |  | 		respBody, err := healthClient.Call(ctx, lockRESTMethodHealth, nil, nil, -1) | 
					
						
							| 
									
										
										
										
											2020-06-18 05:49:26 +08:00
										 |  |  | 		xhttp.DrainBody(respBody) | 
					
						
							| 
									
										
										
										
											2020-11-20 05:53:49 +08:00
										 |  |  | 		return !isNetworkError(err) | 
					
						
							| 
									
										
										
										
											2020-06-18 05:49:26 +08:00
										 |  |  | 	} | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-12-10 23:28:37 +08:00
										 |  |  | 	return &lockRESTClient{u: &url.URL{ | 
					
						
							|  |  |  | 		Scheme: endpoint.Scheme, | 
					
						
							|  |  |  | 		Host:   endpoint.Host, | 
					
						
							|  |  |  | 	}, restClient: restClient} | 
					
						
							| 
									
										
										
										
											2019-04-18 14:16:27 +08:00
										 |  |  | } |