root
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Actions 1 Packages Projects Releases Wiki Activity

s3-check-md5: Add --modified-since flag to skip some objects (#17410)

This commit is contained in:
Anis Eleuch 2023-06-15 20:44:09 +01:00 committed by GitHub
parent ad4e511026
commit 8c33fdf5f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 35 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
docs/debugging/s3-check-md5/main.go
View File

@ -26,8 +26,10 @@ import (
"log"
"net/url"
"os"
"path"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@ -35,6 +37,7 @@ import (
var (
endpoint, accessKey, secretKey string
minModTimeStr string
bucket, prefix string
debug bool
versions bool
@ -57,6 +60,7 @@ func main() {
flag.BoolVar(&debug, "debug", false, "Prints HTTP network calls to S3 endpoint")
flag.BoolVar(&versions, "versions", false, "Verify all versions")
flag.BoolVar(&insecure, "insecure", false, "Disable TLS verification")
flag.StringVar(&minModTimeStr, "modified-since", "", "Specify a minimum object last modified time, e.g.: 2023-01-02T15:04:05Z")
flag.Parse()
if endpoint == "" {
@ -75,6 +79,15 @@ func main() {
log.Fatalln("--prefix is specified without --bucket.")
}
var minModTime time.Time
if minModTimeStr != "" {
var e error
minModTime, e = time.Parse(time.RFC3339, minModTimeStr)
if e != nil {
log.Fatalln("Unable to parse --modified-since:", e)
}
}
u, err := url.Parse(endpoint)
if err != nil {
log.Fatalln(err)
@ -124,22 +137,33 @@ func main() {
WithMetadata: true,
}
objFullPath := func(obj minio.ObjectInfo) (fpath string) {
fpath = path.Join(bucket, obj.Key)
if versions {
fpath += ":" + obj.VersionID
}
return
}
// List all objects from a bucket-name with a matching prefix.
for object := range s3Client.ListObjects(context.Background(), bucket, opts) {
if object.Err != nil {
log.Println("LIST error:", object.Err)
log.Println("FAILED: LIST with error:", object.Err)
continue
}
if !minModTime.IsZero() && object.LastModified.Before(minModTime) {
continue
}
if object.IsDeleteMarker {
log.Println("DELETE marker skipping object:", object.Key)
log.Println("SKIPPED: DELETE marker object:", objFullPath(object))
continue
}
if _, ok := object.UserMetadata["X-Amz-Server-Side-Encryption-Customer-Algorithm"]; ok {
log.Println("Objects encrypted with SSE-C do not have md5sum as ETag:", object.Key)
log.Println("SKIPPED: Objects encrypted with SSE-C do not have md5sum as ETag:", objFullPath(object))
continue
}
if v, ok := object.UserMetadata["X-Amz-Server-Side-Encryption"]; ok && v == "aws:kms" {
log.Println("Objects encrypted with SSE-KMS do not have md5sum as ETag:", object.Key)
log.Println("FAILED: encrypted with SSE-KMS do not have md5sum as ETag:", objFullPath(object))
continue
}
parts := 1
@ -152,12 +176,12 @@ func main() {
if p, err := strconv.Atoi(s[1]); err == nil {
parts = p
} else {
log.Println("ETAG: wrong format:", err)
log.Println("FAILED: ETAG of", objFullPath(object), "has a wrong format:", err)
continue
}
multipart = true
default:
log.Println("Unexpected ETAG format", object.ETag)
log.Println("FAILED: Unexpected ETAG", object.ETag, "for object:", objFullPath(object))
continue
}
@ -170,13 +194,13 @@ func main() {
}
obj, err := s3Client.GetObject(context.Background(), bucket, object.Key, opts)
if err != nil {
log.Println("GET", bucket, object.Key, object.VersionID, "=>", err)
log.Println("FAILED: GET", objFullPath(object), "=>", err)
failedMD5 = true
break
}
h := md5.New()
if _, err := io.Copy(h, obj); err != nil {
log.Println("MD5 calculation error:", bucket, object.Key, object.VersionID, "=>", err)
log.Println("FAILED: MD5 calculation error:", objFullPath(object), "=>", err)
failedMD5 = true
break
}
@ -184,7 +208,7 @@ func main() {
}
if failedMD5 {
log.Println("CORRUPTED object:", bucket, object.Key, object.VersionID)
log.Println("CORRUPTED object:", objFullPath(object))
continue
}
@ -206,9 +230,9 @@ func main() {
}
if corrupted {
log.Println("CORRUPTED object:", bucket, object.Key, object.VersionID)
log.Println("CORRUPTED object:", objFullPath(object))
} else {
log.Println("INTACT object:", bucket, object.Key, object.VersionID)
log.Println("INTACT object:", objFullPath(object))
}
}
}