mirror of https://github.com/minio/minio.git
414 lines
9.1 KiB
Go
414 lines
9.1 KiB
Go
// Copyright (c) 2015-2022 MinIO, Inc.
|
|
//
|
|
// This file is part of MinIO Object Storage stack
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
json "github.com/minio/colorjson"
|
|
|
|
"github.com/klauspost/compress/zip"
|
|
"github.com/tinylib/msgp/msgp"
|
|
)
|
|
|
|
func inspectToExportType(downloadPath string, datajson bool) error {
|
|
decode := func(r io.Reader, file string) ([]byte, error) {
|
|
b, e := io.ReadAll(r)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
b, _, minor, e := checkXL2V1(b)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
|
|
buf := bytes.NewBuffer(nil)
|
|
var data xlMetaInlineData
|
|
switch minor {
|
|
case 0:
|
|
_, e = msgp.CopyToJSON(buf, bytes.NewReader(b))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
case 1, 2:
|
|
v, b, e := msgp.ReadBytesZC(b)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if _, nbuf, e := msgp.ReadUint32Bytes(b); e == nil {
|
|
// Read metadata CRC (added in v2, ignore if not found)
|
|
b = nbuf
|
|
}
|
|
|
|
_, e = msgp.CopyToJSON(buf, bytes.NewReader(v))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
data = b
|
|
case 3:
|
|
v, b, e := msgp.ReadBytesZC(b)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if _, nbuf, e := msgp.ReadUint32Bytes(b); e == nil {
|
|
// Read metadata CRC (added in v2, ignore if not found)
|
|
b = nbuf
|
|
}
|
|
|
|
nVers, v, e := decodeXLHeaders(v)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
type version struct {
|
|
Idx int
|
|
Header json.RawMessage
|
|
Metadata json.RawMessage
|
|
}
|
|
versions := make([]version, nVers)
|
|
e = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
|
|
var header xlMetaV2VersionHeaderV2
|
|
if _, e := header.UnmarshalMsg(hdr); e != nil {
|
|
return e
|
|
}
|
|
b, e := header.MarshalJSON()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var buf bytes.Buffer
|
|
if _, e := msgp.UnmarshalAsJSON(&buf, meta); e != nil {
|
|
return e
|
|
}
|
|
versions[idx] = version{
|
|
Idx: idx,
|
|
Header: b,
|
|
Metadata: buf.Bytes(),
|
|
}
|
|
return nil
|
|
})
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
enc := json.NewEncoder(buf)
|
|
if e := enc.Encode(struct {
|
|
Versions []version
|
|
}{Versions: versions}); e != nil {
|
|
return nil, e
|
|
}
|
|
data = b
|
|
default:
|
|
return nil, fmt.Errorf("unknown metadata version %d", minor)
|
|
}
|
|
|
|
if datajson {
|
|
b, e := data.json()
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
buf = bytes.NewBuffer(b)
|
|
}
|
|
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
fmt.Println("{")
|
|
|
|
hasWritten := false
|
|
var r io.Reader
|
|
var sz int64
|
|
f, e := os.Open(downloadPath)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if st, e := f.Stat(); e == nil {
|
|
sz = st.Size()
|
|
}
|
|
defer f.Close()
|
|
r = f
|
|
|
|
zr, e := zip.NewReader(r.(io.ReaderAt), sz)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
for _, file := range zr.File {
|
|
if !file.FileInfo().IsDir() && strings.HasSuffix(file.Name, "xl.meta") {
|
|
r, e := file.Open()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
// Quote string...
|
|
b, _ := json.Marshal(file.Name)
|
|
if hasWritten {
|
|
fmt.Print(",\n")
|
|
}
|
|
fmt.Printf("\t%s: ", string(b))
|
|
|
|
b, e = decode(r, file.Name)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
fmt.Print(string(b))
|
|
hasWritten = true
|
|
}
|
|
}
|
|
fmt.Println("")
|
|
fmt.Println("}")
|
|
|
|
return nil
|
|
}
|
|
|
|
var (
|
|
// XL header specifies the format
|
|
xlHeader = [4]byte{'X', 'L', '2', ' '}
|
|
|
|
// Current version being written.
|
|
xlVersionCurrent [4]byte
|
|
)
|
|
|
|
const (
|
|
// Breaking changes.
|
|
// Newer versions cannot be read by older software.
|
|
// This will prevent downgrades to incompatible versions.
|
|
xlVersionMajor = 1
|
|
|
|
// Non breaking changes.
|
|
// Bumping this is informational, but should be done
|
|
// if any change is made to the data stored, bumping this
|
|
// will allow to detect the exact version later.
|
|
xlVersionMinor = 1
|
|
)
|
|
|
|
func init() {
|
|
binary.LittleEndian.PutUint16(xlVersionCurrent[0:2], xlVersionMajor)
|
|
binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor)
|
|
}
|
|
|
|
// checkXL2V1 will check if the metadata has correct header and is a known major version.
|
|
// The remaining payload and versions are returned.
|
|
func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, e error) {
|
|
if len(buf) <= 8 {
|
|
return payload, 0, 0, fmt.Errorf("xlMeta: no data")
|
|
}
|
|
|
|
if !bytes.Equal(buf[:4], xlHeader[:]) {
|
|
return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4])
|
|
}
|
|
|
|
if bytes.Equal(buf[4:8], []byte("1 ")) {
|
|
// Set as 1,0.
|
|
major, minor = 1, 0
|
|
} else {
|
|
major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8])
|
|
}
|
|
if major > xlVersionMajor {
|
|
return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major)
|
|
}
|
|
|
|
return buf[8:], major, minor, nil
|
|
}
|
|
|
|
const xlMetaInlineDataVer = 1
|
|
|
|
type xlMetaInlineData []byte
|
|
|
|
// afterVersion returns the payload after the version, if any.
|
|
func (x xlMetaInlineData) afterVersion() []byte {
|
|
if len(x) == 0 {
|
|
return x
|
|
}
|
|
return x[1:]
|
|
}
|
|
|
|
// versionOK returns whether the version is ok.
|
|
func (x xlMetaInlineData) versionOK() bool {
|
|
if len(x) == 0 {
|
|
return true
|
|
}
|
|
return x[0] > 0 && x[0] <= xlMetaInlineDataVer
|
|
}
|
|
|
|
func (x xlMetaInlineData) json() ([]byte, error) {
|
|
if len(x) == 0 {
|
|
return []byte("{}"), nil
|
|
}
|
|
if !x.versionOK() {
|
|
return nil, errors.New("xlMetaInlineData: unknown version")
|
|
}
|
|
|
|
sz, buf, e := msgp.ReadMapHeaderBytes(x.afterVersion())
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
res := []byte("{")
|
|
|
|
for i := uint32(0); i < sz; i++ {
|
|
var key, val []byte
|
|
key, buf, e = msgp.ReadMapKeyZC(buf)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if len(key) == 0 {
|
|
return nil, fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
|
|
}
|
|
// Skip data...
|
|
val, buf, e = msgp.ReadBytesZC(buf)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if i > 0 {
|
|
res = append(res, ',')
|
|
}
|
|
s := fmt.Sprintf(`"%s":%d`, string(key), len(val))
|
|
res = append(res, []byte(s)...)
|
|
}
|
|
res = append(res, '}')
|
|
return res, nil
|
|
}
|
|
|
|
const (
|
|
xlHeaderVersion = 2
|
|
xlMetaVersion = 1
|
|
)
|
|
|
|
func decodeXLHeaders(buf []byte) (versions int, b []byte, e error) {
|
|
hdrVer, buf, e := msgp.ReadUintBytes(buf)
|
|
if e != nil {
|
|
return 0, buf, e
|
|
}
|
|
metaVer, buf, e := msgp.ReadUintBytes(buf)
|
|
if e != nil {
|
|
return 0, buf, e
|
|
}
|
|
if hdrVer > xlHeaderVersion {
|
|
return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", metaVer)
|
|
}
|
|
if metaVer > xlMetaVersion {
|
|
return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer)
|
|
}
|
|
versions, buf, e = msgp.ReadIntBytes(buf)
|
|
if e != nil {
|
|
return 0, buf, e
|
|
}
|
|
if versions < 0 {
|
|
return 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions)
|
|
}
|
|
return versions, buf, nil
|
|
}
|
|
|
|
// decodeVersions will decode a number of versions from a buffer
|
|
// and perform a callback for each version in order, newest first.
|
|
// Any non-nil error is returned.
|
|
func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (e error) {
|
|
var tHdr, tMeta []byte // Zero copy bytes
|
|
for i := 0; i < versions; i++ {
|
|
tHdr, buf, e = msgp.ReadBytesZC(buf)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
tMeta, buf, e = msgp.ReadBytesZC(buf)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if e = fn(i, tHdr, tMeta); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type xlMetaV2VersionHeaderV2 struct {
|
|
VersionID [16]byte
|
|
ModTime int64
|
|
Signature [4]byte
|
|
Type uint8
|
|
Flags uint8
|
|
}
|
|
|
|
// UnmarshalMsg implements msgp.Unmarshaler
|
|
func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) {
|
|
var zb0001 uint32
|
|
zb0001, bts, e = msgp.ReadArrayHeaderBytes(bts)
|
|
if e != nil {
|
|
e = msgp.WrapError(e)
|
|
return
|
|
}
|
|
if zb0001 != 5 {
|
|
e = msgp.ArrayError{Wanted: 5, Got: zb0001}
|
|
return
|
|
}
|
|
bts, e = msgp.ReadExactBytes(bts, (z.VersionID)[:])
|
|
if e != nil {
|
|
e = msgp.WrapError(e, "VersionID")
|
|
return
|
|
}
|
|
z.ModTime, bts, e = msgp.ReadInt64Bytes(bts)
|
|
if e != nil {
|
|
e = msgp.WrapError(e, "ModTime")
|
|
return
|
|
}
|
|
bts, e = msgp.ReadExactBytes(bts, (z.Signature)[:])
|
|
if e != nil {
|
|
e = msgp.WrapError(e, "Signature")
|
|
return
|
|
}
|
|
{
|
|
var zb0002 uint8
|
|
zb0002, bts, e = msgp.ReadUint8Bytes(bts)
|
|
if e != nil {
|
|
e = msgp.WrapError(e, "Type")
|
|
return
|
|
}
|
|
z.Type = zb0002
|
|
}
|
|
{
|
|
var zb0003 uint8
|
|
zb0003, bts, e = msgp.ReadUint8Bytes(bts)
|
|
if e != nil {
|
|
e = msgp.WrapError(e, "Flags")
|
|
return
|
|
}
|
|
z.Flags = zb0003
|
|
}
|
|
o = bts
|
|
return
|
|
}
|
|
|
|
func (z xlMetaV2VersionHeaderV2) MarshalJSON() (o []byte, e error) {
|
|
tmp := struct {
|
|
VersionID string
|
|
ModTime time.Time
|
|
Signature string
|
|
Type uint8
|
|
Flags uint8
|
|
}{
|
|
VersionID: hex.EncodeToString(z.VersionID[:]),
|
|
ModTime: time.Unix(0, z.ModTime),
|
|
Signature: hex.EncodeToString(z.Signature[:]),
|
|
Type: z.Type,
|
|
Flags: z.Flags,
|
|
}
|
|
return json.Marshal(tmp)
|
|
}
|