| 
									
										
										
										
											2016-04-22 11:07:47 +08:00
										 |  |  | /* | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  |  * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc. | 
					
						
							| 
									
										
										
										
											2016-04-22 11:07:47 +08:00
										 |  |  |  * | 
					
						
							|  |  |  |  * Licensed under the Apache License, Version 2.0 (the "License"); | 
					
						
							|  |  |  |  * you may not use this file except in compliance with the License. | 
					
						
							|  |  |  |  * You may obtain a copy of the License at | 
					
						
							|  |  |  |  * | 
					
						
							|  |  |  |  *     http://www.apache.org/licenses/LICENSE-2.0
 | 
					
						
							|  |  |  |  * | 
					
						
							|  |  |  |  * Unless required by applicable law or agreed to in writing, software | 
					
						
							|  |  |  |  * distributed under the License is distributed on an "AS IS" BASIS, | 
					
						
							|  |  |  |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 
					
						
							|  |  |  |  * See the License for the specific language governing permissions and | 
					
						
							|  |  |  |  * limitations under the License. | 
					
						
							|  |  |  |  */ | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-08-19 07:23:42 +08:00
										 |  |  | package cmd | 
					
						
							| 
									
										
										
										
											2016-04-08 10:01:15 +08:00
										 |  |  | 
 | 
					
						
							|  |  |  | import ( | 
					
						
							|  |  |  | 	"time" | 
					
						
							|  |  |  | ) | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-11-03 09:07:52 +08:00
										 |  |  | //go:generate msgp -file=$GOFILE
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-11-05 02:10:54 +08:00
										 |  |  | // DiskInfo is an extended type which returns current
 | 
					
						
							|  |  |  | // disk usage per path.
 | 
					
						
							| 
									
										
										
										
											2021-01-04 03:27:57 +08:00
										 |  |  | //msgp:tuple DiskInfo
 | 
					
						
							|  |  |  | // The above means that any added/deleted fields are incompatible.
 | 
					
						
							| 
									
										
										
										
											2020-11-05 02:10:54 +08:00
										 |  |  | type DiskInfo struct { | 
					
						
							| 
									
										
										
										
											2020-12-27 14:58:06 +08:00
										 |  |  | 	Total      uint64 | 
					
						
							|  |  |  | 	Free       uint64 | 
					
						
							|  |  |  | 	Used       uint64 | 
					
						
							|  |  |  | 	UsedInodes uint64 | 
					
						
							|  |  |  | 	FSType     string | 
					
						
							|  |  |  | 	RootDisk   bool | 
					
						
							|  |  |  | 	Healing    bool | 
					
						
							|  |  |  | 	Endpoint   string | 
					
						
							|  |  |  | 	MountPath  string | 
					
						
							|  |  |  | 	ID         string | 
					
						
							| 
									
										
										
										
											2021-03-17 11:06:57 +08:00
										 |  |  | 	Metrics    DiskMetrics | 
					
						
							| 
									
										
										
										
											2020-12-27 14:58:06 +08:00
										 |  |  | 	Error      string // carries the error over the network
 | 
					
						
							| 
									
										
										
										
											2020-11-05 02:10:54 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-03-17 11:06:57 +08:00
										 |  |  | // DiskMetrics has the information about XL Storage APIs
 | 
					
						
							|  |  |  | // the number of calls of each API and the moving average of
 | 
					
						
							|  |  |  | // the duration of each API.
 | 
					
						
							|  |  |  | type DiskMetrics struct { | 
					
						
							|  |  |  | 	APILatencies map[string]string `json:"apiLatencies,omitempty"` | 
					
						
							|  |  |  | 	APICalls     map[string]uint64 `json:"apiCalls,omitempty"` | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-11-05 02:10:54 +08:00
										 |  |  | // VolsInfo is a collection of volume(bucket) information
 | 
					
						
							|  |  |  | type VolsInfo []VolInfo | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-06-09 21:24:11 +08:00
										 |  |  | // VolInfo - represents volume stat information.
 | 
					
						
							| 
									
										
										
										
											2021-01-04 03:27:57 +08:00
										 |  |  | //msgp:tuple VolInfo
 | 
					
						
							|  |  |  | // The above means that any added/deleted fields are incompatible.
 | 
					
						
							| 
									
										
										
										
											2016-04-09 01:37:38 +08:00
										 |  |  | type VolInfo struct { | 
					
						
							| 
									
										
										
										
											2016-06-09 21:24:11 +08:00
										 |  |  | 	// Name of the volume.
 | 
					
						
							|  |  |  | 	Name string | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// Date and time when the volume was created.
 | 
					
						
							| 
									
										
										
										
											2016-04-09 01:37:38 +08:00
										 |  |  | 	Created time.Time | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-05-15 04:49:10 +08:00
										 |  |  | // FilesInfo represent a list of files, additionally
 | 
					
						
							|  |  |  | // indicates if the list is last.
 | 
					
						
							|  |  |  | type FilesInfo struct { | 
					
						
							|  |  |  | 	Files       []FileInfo | 
					
						
							|  |  |  | 	IsTruncated bool | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | // FilesInfoVersions represents a list of file versions,
 | 
					
						
							|  |  |  | // additionally indicates if the list is last.
 | 
					
						
							|  |  |  | type FilesInfoVersions struct { | 
					
						
							|  |  |  | 	FilesVersions []FileInfoVersions | 
					
						
							|  |  |  | 	IsTruncated   bool | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // FileInfoVersions represent a list of versions for a given file.
 | 
					
						
							|  |  |  | type FileInfoVersions struct { | 
					
						
							|  |  |  | 	// Name of the volume.
 | 
					
						
							|  |  |  | 	Volume string | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// Name of the file.
 | 
					
						
							|  |  |  | 	Name string | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-01-27 18:19:28 +08:00
										 |  |  | 	IsEmptyDir bool | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | 	// Represents the latest mod time of the
 | 
					
						
							|  |  |  | 	// latest version.
 | 
					
						
							|  |  |  | 	LatestModTime time.Time | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	Versions []FileInfo | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-02-12 02:22:03 +08:00
										 |  |  | // findVersionIndex will return the version index where the version
 | 
					
						
							|  |  |  | // was found. Returns -1 if not found.
 | 
					
						
							|  |  |  | func (f *FileInfoVersions) findVersionIndex(v string) int { | 
					
						
							|  |  |  | 	if f == nil || v == "" { | 
					
						
							|  |  |  | 		return -1 | 
					
						
							| 
									
										
										
										
											2020-10-29 00:18:35 +08:00
										 |  |  | 	} | 
					
						
							|  |  |  | 	for i, ver := range f.Versions { | 
					
						
							|  |  |  | 		if ver.VersionID == v { | 
					
						
							| 
									
										
										
										
											2021-02-12 02:22:03 +08:00
										 |  |  | 			return i | 
					
						
							| 
									
										
										
										
											2020-10-29 00:18:35 +08:00
										 |  |  | 		} | 
					
						
							|  |  |  | 	} | 
					
						
							| 
									
										
										
										
											2021-02-12 02:22:03 +08:00
										 |  |  | 	return -1 | 
					
						
							| 
									
										
										
										
											2020-10-29 00:18:35 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-06-09 21:24:11 +08:00
										 |  |  | // FileInfo - represents file stat information.
 | 
					
						
							| 
									
										
										
										
											2020-11-03 09:07:52 +08:00
										 |  |  | //msgp:tuple FileInfo
 | 
					
						
							|  |  |  | // The above means that any added/deleted fields are incompatible.
 | 
					
						
							| 
									
										
										
										
											2016-04-08 10:01:15 +08:00
										 |  |  | type FileInfo struct { | 
					
						
							| 
									
										
										
										
											2016-06-09 21:24:11 +08:00
										 |  |  | 	// Name of the volume.
 | 
					
						
							|  |  |  | 	Volume string | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// Name of the file.
 | 
					
						
							|  |  |  | 	Name string | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | 	// Version of the file.
 | 
					
						
							|  |  |  | 	VersionID string | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// Indicates if the version is the latest
 | 
					
						
							|  |  |  | 	IsLatest bool | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// Deleted is set when this FileInfo represents
 | 
					
						
							|  |  |  | 	// a deleted marker for a versioned bucket.
 | 
					
						
							|  |  |  | 	Deleted bool | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-11-13 04:12:09 +08:00
										 |  |  | 	// TransitionStatus is set to Pending/Complete for transitioned
 | 
					
						
							|  |  |  | 	// entries based on state of transition
 | 
					
						
							|  |  |  | 	TransitionStatus string | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | 	// DataDir of the file
 | 
					
						
							|  |  |  | 	DataDir string | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-07-18 08:41:29 +08:00
										 |  |  | 	// Indicates if this object is still in V1 format.
 | 
					
						
							|  |  |  | 	XLV1 bool | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | 	// Date and time when the file was last modified, if Deleted
 | 
					
						
							|  |  |  | 	// is 'true' this value represents when while was deleted.
 | 
					
						
							| 
									
										
										
										
											2016-04-08 10:01:15 +08:00
										 |  |  | 	ModTime time.Time | 
					
						
							| 
									
										
										
										
											2016-06-09 21:24:11 +08:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	// Total file size.
 | 
					
						
							|  |  |  | 	Size int64 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// File mode bits.
 | 
					
						
							| 
									
										
										
										
											2020-11-03 09:07:52 +08:00
										 |  |  | 	Mode uint32 | 
					
						
							| 
									
										
										
										
											2019-05-15 04:49:10 +08:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	// File metadata
 | 
					
						
							|  |  |  | 	Metadata map[string]string | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// All the parts per object.
 | 
					
						
							|  |  |  | 	Parts []ObjectPartInfo | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | 	// Erasure info for all objects.
 | 
					
						
							|  |  |  | 	Erasure ErasureInfo | 
					
						
							| 
									
										
										
										
											2020-11-20 10:43:58 +08:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	// DeleteMarkerReplicationStatus is set when this FileInfo represents
 | 
					
						
							|  |  |  | 	// replication on a DeleteMarker
 | 
					
						
							|  |  |  | 	MarkDeleted                   bool // mark this version as deleted
 | 
					
						
							|  |  |  | 	DeleteMarkerReplicationStatus string | 
					
						
							|  |  |  | 	VersionPurgeStatus            VersionPurgeStatusType | 
					
						
							| 
									
										
										
										
											2021-01-08 11:27:31 +08:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	Data []byte // optionally carries object data
 | 
					
						
							| 
									
										
										
										
											2021-02-02 01:52:11 +08:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	NumVersions      int | 
					
						
							|  |  |  | 	SuccessorModTime time.Time | 
					
						
							| 
									
										
										
										
											2020-11-20 10:43:58 +08:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // VersionPurgeStatusKey denotes purge status in metadata
 | 
					
						
							|  |  |  | const VersionPurgeStatusKey = "purgestatus" | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication
 | 
					
						
							|  |  |  | type VersionPurgeStatusType string | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | const ( | 
					
						
							|  |  |  | 	// Pending - versioned delete replication is pending.
 | 
					
						
							|  |  |  | 	Pending VersionPurgeStatusType = "PENDING" | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// Complete - versioned delete replication is now complete, erase version on disk.
 | 
					
						
							|  |  |  | 	Complete VersionPurgeStatusType = "COMPLETE" | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// Failed - versioned delete replication failed.
 | 
					
						
							|  |  |  | 	Failed VersionPurgeStatusType = "FAILED" | 
					
						
							|  |  |  | ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // Empty returns true if purge status was not set.
 | 
					
						
							|  |  |  | func (v VersionPurgeStatusType) Empty() bool { | 
					
						
							|  |  |  | 	return string(v) == "" | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // Pending returns true if the version is pending purge.
 | 
					
						
							|  |  |  | func (v VersionPurgeStatusType) Pending() bool { | 
					
						
							|  |  |  | 	return v == Pending || v == Failed | 
					
						
							| 
									
										
										
										
											2016-04-08 10:01:15 +08:00
										 |  |  | } | 
					
						
							| 
									
										
										
										
											2020-02-25 23:52:28 +08:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | // newFileInfo - initializes new FileInfo, allocates a fresh erasure info.
 | 
					
						
							|  |  |  | func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) { | 
					
						
							|  |  |  | 	fi.Erasure = ErasureInfo{ | 
					
						
							|  |  |  | 		Algorithm:    erasureAlgorithm, | 
					
						
							|  |  |  | 		DataBlocks:   dataBlocks, | 
					
						
							|  |  |  | 		ParityBlocks: parityBlocks, | 
					
						
							| 
									
										
											  
											
												[feat]: change erasure coding default block size from 10MiB to 1MiB (#11721)
major performance improvements in range GETs to avoid large
read amplification when ranges are tiny and random
```
-------------------
Operation: GET
Operations: 142014 -> 339421
Duration: 4m50s -> 4m56s
* Average: +139.41% (+1177.3 MiB/s) throughput, +139.11% (+658.4) obj/s
* Fastest: +125.24% (+1207.4 MiB/s) throughput, +132.32% (+612.9) obj/s
* 50% Median: +139.06% (+1175.7 MiB/s) throughput, +133.46% (+660.9) obj/s
* Slowest: +203.40% (+1267.9 MiB/s) throughput, +198.59% (+753.5) obj/s
```
TTFB from 10MiB BlockSize
```
* First Access TTFB: Avg: 81ms, Median: 61ms, Best: 20ms, Worst: 2.056s
```
TTFB from 1MiB BlockSize
```
* First Access TTFB: Avg: 22ms, Median: 21ms, Best: 8ms, Worst: 91ms
```
Full object reads however do see a slight change which won't be
noticeable in real world, so not doing any comparisons
TTFB still had improvements with full object reads with 1MiB
```
* First Access TTFB: Avg: 68ms, Median: 35ms, Best: 11ms, Worst: 1.16s
```
v/s
TTFB with 10MiB
```
* First Access TTFB: Avg: 388ms, Median: 98ms, Best: 20ms, Worst: 4.156s
```
This change should affect all new uploads, previous uploads should
continue to work with business as usual. But dramatic improvements can
be seen with these changes.
											
										 
											2021-03-07 06:09:34 +08:00
										 |  |  | 		BlockSize:    blockSizeV2, | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | 		Distribution: hashOrder(object, dataBlocks+parityBlocks), | 
					
						
							| 
									
										
										
										
											2020-02-25 23:52:28 +08:00
										 |  |  | 	} | 
					
						
							| 
									
										
										
										
											2020-06-13 11:04:01 +08:00
										 |  |  | 	return fi | 
					
						
							| 
									
										
										
										
											2020-02-25 23:52:28 +08:00
										 |  |  | } |