Support live updates for clients during speedtest (#13566)

cmd/admin-handlers.go

@@ -951,47 +951,31 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 		duration = time.Second * 10
 	}
 
-	throughputSize := size
-
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 
-	endBlankRepliesCh := make(chan error)
+	ch := speedTest(ctx, size, concurrent, duration, autotune)
-
+	for {
-	go func() {
+		select {
-		for {
+		case <-ctx.Done():
-			select {
+			return
-			case <-ctx.Done():
+		case <-keepAliveTicker.C:
-				endBlankRepliesCh <- nil
+			// Write a blank entry to prevent client from disconnecting
+			if err := json.NewEncoder(w).Encode(madmin.SpeedTestResult{}); err != nil {
 				return
-			case <-keepAliveTicker.C:
+			}
-				// Write a blank entry to prevent client from disconnecting
+			w.(http.Flusher).Flush()
-				if err := json.NewEncoder(w).Encode(madmin.SpeedTestResult{}); err != nil {
+		case result, ok := <-ch:
-					endBlankRepliesCh <- err
+			if !ok {
-					return
+				defer objectAPI.DeleteBucket(context.Background(), pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), DeleteBucketOptions{Force: true, NoRecreate: true})
-				}
+				return
-				w.(http.Flusher).Flush()
+			}
-			case endBlankRepliesCh <- nil:
+			if err := json.NewEncoder(w).Encode(result); err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
 		}
-	}()
-
-	result, err := speedTest(ctx, throughputSize, concurrent, duration, autotune)
-	if <-endBlankRepliesCh != nil {
-		return
 	}
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	if err := json.NewEncoder(w).Encode(result); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), DeleteBucketOptions{Force: true, NoRecreate: true})
 }
 
 // Admin API errors

cmd/utils.go

@@ -973,99 +973,111 @@ func auditLogInternal(ctx context.Context, bucket, object string, opts AuditLogO
 }
 
 // Get the max throughput and iops numbers.
-func speedTest(ctx context.Context, throughputSize, concurrencyStart int, duration time.Duration, autotune bool) (madmin.SpeedTestResult, error) {
+func speedTest(ctx context.Context, throughputSize, concurrencyStart int, duration time.Duration, autotune bool) chan madmin.SpeedTestResult {
-	var result madmin.SpeedTestResult
+	ch := make(chan madmin.SpeedTestResult, 1)
+	go func() {
+		defer close(ch)
 
-	objAPI := newObjectLayerFn()
+		objAPI := newObjectLayerFn()
-	if objAPI == nil {
+		if objAPI == nil {
-		return result, errServerNotInitialized
+			return
-	}
-
-	concurrency := concurrencyStart
-
-	throughputHighestGet := uint64(0)
-	throughputHighestPut := uint64(0)
-	var throughputHighestGetResults []SpeedtestResult
-
-	for {
-		select {
-		case <-ctx.Done():
-			// If the client got disconnected stop the speedtest.
-			return result, errUnexpected
-		default:
 		}
 
-		results := globalNotificationSys.Speedtest(ctx, throughputSize, concurrency, duration)
+		concurrency := concurrencyStart
-		sort.Slice(results, func(i, j int) bool {
+
-			return results[i].Endpoint < results[j].Endpoint
+		throughputHighestGet := uint64(0)
-		})
+		throughputHighestPut := uint64(0)
-		totalPut := uint64(0)
+		var throughputHighestGetResults []SpeedtestResult
-		totalGet := uint64(0)
+
-		for _, result := range results {
+		sendResult := func() {
-			totalPut += result.Uploads
+			var result madmin.SpeedTestResult
-			totalGet += result.Downloads
+
+			durationSecs := duration.Seconds()
+
+			result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs)
+			result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(throughputSize) / uint64(durationSecs)
+			result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs)
+			result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(throughputSize) / uint64(durationSecs)
+			for i := 0; i < len(throughputHighestGetResults); i++ {
+				errStr := ""
+				if throughputHighestGetResults[i].Error != "" {
+					errStr = throughputHighestGetResults[i].Error
+				}
+				result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{
+					Endpoint:         throughputHighestGetResults[i].Endpoint,
+					ThroughputPerSec: throughputHighestGetResults[i].Uploads / uint64(durationSecs),
+					ObjectsPerSec:    throughputHighestGetResults[i].Uploads / uint64(throughputSize) / uint64(durationSecs),
+					Err:              errStr,
+				})
+				result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{
+					Endpoint:         throughputHighestGetResults[i].Endpoint,
+					ThroughputPerSec: throughputHighestGetResults[i].Downloads / uint64(durationSecs),
+					ObjectsPerSec:    throughputHighestGetResults[i].Downloads / uint64(throughputSize) / uint64(durationSecs),
+					Err:              errStr,
+				})
+			}
+
+			numDisks := 0
+			if pools, ok := objAPI.(*erasureServerPools); ok {
+				for _, set := range pools.serverPools {
+					numDisks = set.setCount * set.setDriveCount
+				}
+			}
+			result.Disks = numDisks
+			result.Servers = len(globalNotificationSys.peerClients) + 1
+			result.Version = Version
+			result.Size = throughputSize
+			result.Concurrent = concurrency
+
+			ch <- result
 		}
 
-		if totalGet < throughputHighestGet {
+		for {
-			break
+			select {
-		}
+			case <-ctx.Done():
+				// If the client got disconnected stop the speedtest.
+				return
+			default:
+			}
 
-		doBreak := false
+			results := globalNotificationSys.Speedtest(ctx, throughputSize, concurrency, duration)
-		if float64(totalGet-throughputHighestGet)/float64(totalGet) < 0.025 {
+			sort.Slice(results, func(i, j int) bool {
-			doBreak = true
+				return results[i].Endpoint < results[j].Endpoint
-		}
+			})
+			totalPut := uint64(0)
+			totalGet := uint64(0)
+			for _, result := range results {
+				totalPut += result.Uploads
+				totalGet += result.Downloads
+			}
+
+			if totalGet < throughputHighestGet {
+				sendResult()
+				break
+			}
+
+			doBreak := false
+			if float64(totalGet-throughputHighestGet)/float64(totalGet) < 0.025 {
+				doBreak = true
+			}
 
-		if totalGet > throughputHighestGet {
 			throughputHighestGet = totalGet
 			throughputHighestGetResults = results
 			throughputHighestPut = totalPut
+
+			if doBreak {
+				sendResult()
+				break
+			}
+
+			if !autotune {
+				sendResult()
+				break
+			}
+			sendResult()
+			// Try with a higher concurrency to see if we get better throughput
+			concurrency += (concurrency + 1) / 2
 		}
-
+	}()
-		if doBreak {
+	return ch
-			break
-		}
-
-		if !autotune {
-			break
-		}
-		// Try with a higher concurrency to see if we get better throughput
-		concurrency += (concurrency + 1) / 2
-	}
-
-	durationSecs := duration.Seconds()
-
-	result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs)
-	result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(throughputSize) / uint64(durationSecs)
-	result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs)
-	result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(throughputSize) / uint64(durationSecs)
-	for i := 0; i < len(throughputHighestGetResults); i++ {
-		errStr := ""
-		if throughputHighestGetResults[i].Error != "" {
-			errStr = throughputHighestGetResults[i].Error
-		}
-		result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{
-			Endpoint:         throughputHighestGetResults[i].Endpoint,
-			ThroughputPerSec: throughputHighestGetResults[i].Uploads / uint64(durationSecs),
-			ObjectsPerSec:    throughputHighestGetResults[i].Uploads / uint64(throughputSize) / uint64(durationSecs),
-			Err:              errStr,
-		})
-		result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{
-			Endpoint:         throughputHighestGetResults[i].Endpoint,
-			ThroughputPerSec: throughputHighestGetResults[i].Downloads / uint64(durationSecs),
-			ObjectsPerSec:    throughputHighestGetResults[i].Downloads / uint64(throughputSize) / uint64(durationSecs),
-			Err:              errStr,
-		})
-	}
-
-	numDisks := 0
-	if pools, ok := objAPI.(*erasureServerPools); ok {
-		for _, set := range pools.serverPools {
-			numDisks = set.setCount * set.setDriveCount
-		}
-	}
-	result.Disks = numDisks
-	result.Servers = len(globalNotificationSys.peerClients) + 1
-	result.Version = Version
-
-	return result, nil
 }

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.1.10
+	github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104
 	github.com/minio/minio-go/v7 v7.0.15
 	github.com/minio/parquet-go v1.0.0
 	github.com/minio/pkg v1.1.5

go.sum

@@ -633,6 +633,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-containerregistry v0.1.2/go.mod h1:GPivBPgdAyd2SU+vf6EpsgOtWDuPqjW0hJZt4rNdTZ4=
 github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -967,6 +968,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -1053,6 +1056,8 @@ github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQc
 github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
 github.com/minio/madmin-go v1.1.10 h1:pfMgXkzdwADnNfVdNMJbwok2fjb2sJ7Q76kDt89RGzE=
 github.com/minio/madmin-go v1.1.10/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
+github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104 h1:/N0JW/1+vbxdDgCI8+tac7GXKkis6rh8kVsLNejN6Ug=
+github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
 github.com/minio/mc v0.0.0-20211027024940-7866f97ef502 h1:7ip9qTspUniv+WDENgOcfUr95IccxG5aDkBM4Z96kQg=
 github.com/minio/mc v0.0.0-20211027024940-7866f97ef502/go.mod h1:vxztwXLB9Gyl/h3Yh08Mpz1CB/0FO5Es0iQRpzxvS5I=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@@ -1343,6 +1348,8 @@ github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+
 github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
 github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg=
 github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
+github.com/shirou/gopsutil/v3 v3.21.10 h1:flTg1DrnV/UVrBqjLgVgDJzx6lf+91rC64/dBHmO2IA=
+github.com/shirou/gopsutil/v3 v3.21.10/go.mod h1:t75NhzCZ/dYyPQjyQmrAYP6c8+LCdFANeBMdLPCNnew=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -1699,6 +1706,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
 golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 h1:VrJZAjbekhoRn7n5FBujY31gboH+iB3pdLxn3gE9FjU=
+golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1813,8 +1822,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo=
 golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c h1:QOfDMdrf/UwlVR0UBq2Mpr58UzNtvgJRXA4BgPfFACs=
+golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=