root
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Actions 1 Packages Projects Releases Wiki Activity
minio/cmd/xl-storage-free-version_tes...

231 lines
6.3 KiB
Go
Raw Normal View History

Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
Fix DeleteObject when only free versions remain (#16289)
2022-12-22 08:24:07 +08:00
"errors"
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
"testing"
"time"
"github.com/minio/minio/internal/bucket/lifecycle"
)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
func (x xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
avoid repeated large allocations for large parts (#17968) objects with 10,000 parts and many of them can cause a large memory spike which can potentially lead to OOM due to lack of GC. with previous PR reducing the memory usage significantly in #17963, this PR reduces this further by 80% under repeated calls. Scanner sub-system has no use for the slice of Parts(), it is better left empty. ``` benchmark old ns/op new ns/op delta BenchmarkToFileInfo/ToFileInfo-8 295658 188143 -36.36% benchmark old allocs new allocs delta BenchmarkToFileInfo/ToFileInfo-8 61 60 -1.64% benchmark old bytes new bytes delta BenchmarkToFileInfo/ToFileInfo-8 1097210 227255 -79.29% ```
2023-09-02 22:49:24 +08:00
fivs, err := x.ListVersions(volume, path, true)
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
if err != nil {
return nil, err
}
n := 0
for _, fiv := range fivs {
if fiv.TierFreeVersion() {
fivs[n] = fiv
n++
}
}
fivs = fivs[:n]
return fivs, nil
}
func TestFreeVersion(t *testing.T) {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
fatalErr := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Add a version with tiered content, one with local content
xl := xlMetaV2{}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
counter := 1
report := func() {
t.Helper()
// t.Logf("versions (%d): len = %d", counter, len(xl.versions))
counter++
}
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
fi := FileInfo{
Volume: "volume",
Name: "object-name",
VersionID: "00000000-0000-0000-0000-000000000001",
IsLatest: true,
Deleted: false,
TransitionStatus: "",
DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef",
XLV1: false,
ModTime: time.Now(),
Size: 0,
Mode: 0,
Metadata: nil,
Parts: nil,
Erasure: ErasureInfo{
Algorithm: ReedSolomon.String(),
DataBlocks: 4,
ParityBlocks: 2,
BlockSize: 10000,
Index: 1,
Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
Checksums: []ChecksumInfo{{
PartNumber: 1,
Algorithm: HighwayHash256S,
Hash: nil,
}},
},
Add support for multi site replication (#12880)
2021-09-19 04:31:35 +08:00
MarkDeleted: false,
run gofumpt cleanup across code-base (#14015)
2022-01-03 01:15:06 +08:00
// DeleteMarkerReplicationStatus: "",
// VersionPurgeStatus: "",
Add support for multi site replication (#12880)
2021-09-19 04:31:35 +08:00
NumVersions: 1,
SuccessorModTime: time.Time{},
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
}
// Add a version with local content
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
fatalErr(xl.AddVersion(fi))
report()
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Add null version with tiered content
tierfi := fi
tierfi.VersionID = ""
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
fatalErr(xl.AddVersion(tierfi))
report()
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
tierfi.TransitionStatus = lifecycle.TransitionComplete
tierfi.TransitionedObjName = mustGetUUID()
tierfi.TransitionTier = "MINIOTIER-1"
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
var err error
fix: deleteVersions causing xl.meta to have empty Versions[] slice (#14483) This is a side-affect of the optimization done in PR #13544 which causes a certain type of delete operations on given object versions can cause lastVersion indication to be skipped, which leads to an `xl.meta` where Versions[] slice is empty while the entire file is intact by itself. This PR tries to ensure that such files are visible and deletable by regular means of listing as null 'delete-marker' and also avoid the situation where this potential issue might arise.
2022-03-05 12:01:26 +08:00
_, err = xl.DeleteVersion(tierfi)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
fatalErr(err)
report()
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
fvIDs := []string{
"00000000-0000-0000-0000-0000000000f1",
"00000000-0000-0000-0000-0000000000f2",
}
// Simulate overwrite of null version
newtierfi := tierfi
newtierfi.SetTierFreeVersionID(fvIDs[0])
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
fatalErr(xl.AddFreeVersion(newtierfi))
report()
fatalErr(xl.AddVersion(newtierfi))
report()
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Simulate removal of null version
newtierfi.TransitionTier = ""
newtierfi.TransitionedObjName = ""
newtierfi.TransitionStatus = ""
newtierfi.SetTierFreeVersionID(fvIDs[1])
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
report()
fix: deleteVersions causing xl.meta to have empty Versions[] slice (#14483) This is a side-affect of the optimization done in PR #13544 which causes a certain type of delete operations on given object versions can cause lastVersion indication to be skipped, which leads to an `xl.meta` where Versions[] slice is empty while the entire file is intact by itself. This PR tries to ensure that such files are visible and deletable by regular means of listing as null 'delete-marker' and also avoid the situation where this potential issue might arise.
2022-03-05 12:01:26 +08:00
_, err = xl.DeleteVersion(newtierfi)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
report()
fatalErr(err)
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
Fix DeleteObject when only free versions remain (#16289)
2022-12-22 08:24:07 +08:00
// At this point the version stack must look as below,
// v3 --> free version 00000000-0000-0000-0000-0000000000f2 (from removal of null version)
// v2 --> free version 00000000-0000-0000-0000-0000000000f1 (from overwriting of null version )
// v1 --> non-free version 00000000-0000-0000-0000-000000000001
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Check number of free-versions
freeVersions, err := xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
if err != nil {
t.Fatalf("failed to list free versions %v", err)
}
if len(freeVersions) != 2 {
t.Fatalf("Expected two free versions but got %d", len(freeVersions))
}
Fix DeleteObject when only free versions remain (#16289)
2022-12-22 08:24:07 +08:00
freeVersionsTests := []struct {
vol string
name string
inclFreeVers bool
afterFn func(fi FileInfo) (string, error)
expectedFree bool
expectedErr error
}{
// ToFileInfo with 'inclFreeVers = true' should return the latest
// non-free version if one is present
{
vol: newtierfi.Volume,
name: newtierfi.Name,
inclFreeVers: true,
afterFn: xl.DeleteVersion,
expectedFree: false,
},
// ToFileInfo with 'inclFreeVers = true' must return the latest free
// version when no non-free versions are present.
{
vol: newtierfi.Volume,
name: newtierfi.Name,
inclFreeVers: true,
expectedFree: true,
},
// ToFileInfo with 'inclFreeVers = false' must return errFileNotFound
// when no non-free version exist.
{
vol: newtierfi.Volume,
name: newtierfi.Name,
inclFreeVers: false,
expectedErr: errFileNotFound,
},
}
for _, ft := range freeVersionsTests {
avoid repeated large allocations for large parts (#17968) objects with 10,000 parts and many of them can cause a large memory spike which can potentially lead to OOM due to lack of GC. with previous PR reducing the memory usage significantly in #17963, this PR reduces this further by 80% under repeated calls. Scanner sub-system has no use for the slice of Parts(), it is better left empty. ``` benchmark old ns/op new ns/op delta BenchmarkToFileInfo/ToFileInfo-8 295658 188143 -36.36% benchmark old allocs new allocs delta BenchmarkToFileInfo/ToFileInfo-8 61 60 -1.64% benchmark old bytes new bytes delta BenchmarkToFileInfo/ToFileInfo-8 1097210 227255 -79.29% ```
2023-09-02 22:49:24 +08:00
fi, err := xl.ToFileInfo(ft.vol, ft.name, "", ft.inclFreeVers, true)
Fix DeleteObject when only free versions remain (#16289)
2022-12-22 08:24:07 +08:00
if err != nil && !errors.Is(err, ft.expectedErr) {
t.Fatalf("ToFileInfo failed due to %v", err)
}
if got := fi.TierFreeVersion(); got != ft.expectedFree {
t.Fatalf("Expected free-version=%v but got free-version=%v", ft.expectedFree, got)
}
if ft.afterFn != nil {
_, err = ft.afterFn(fi)
if err != nil {
t.Fatalf("ft.afterFn failed with err %v", err)
}
}
}
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Simulate scanner removing free-version
freefi := newtierfi
for _, fvID := range fvIDs {
freefi.VersionID = fvID
fix: deleteVersions causing xl.meta to have empty Versions[] slice (#14483) This is a side-affect of the optimization done in PR #13544 which causes a certain type of delete operations on given object versions can cause lastVersion indication to be skipped, which leads to an `xl.meta` where Versions[] slice is empty while the entire file is intact by itself. This PR tries to ensure that such files are visible and deletable by regular means of listing as null 'delete-marker' and also avoid the situation where this potential issue might arise.
2022-03-05 12:01:26 +08:00
_, err = xl.DeleteVersion(freefi)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
fatalErr(err)
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
report()
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Check number of free-versions
freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
if err != nil {
t.Fatalf("failed to list free versions %v", err)
}
if len(freeVersions) != 0 {
t.Fatalf("Expected zero free version but got %d", len(freeVersions))
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
report()
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Adding a free version to a version with no tiered content.
newfi := fi
newfi.SetTierFreeVersionID("00000000-0000-0000-0000-0000000000f3")
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-19 04:15:22 +08:00
fatalErr(xl.AddFreeVersion(newfi)) // this shouldn't add a free-version
report()
Add a 'free' version to track deletion of tiered object content (#12470)
2021-07-01 10:32:07 +08:00
// Check number of free-versions
freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
if err != nil {
t.Fatalf("failed to list free versions %v", err)
}
if len(freeVersions) != 0 {
t.Fatalf("Expected zero free version but got %d", len(freeVersions))
}
}