Merge 0110061b79 into 71f293d9ab

- Add MINIO_MIN_PART_SIZE environment variable support
- Default value remains 5MiB if not set
- Initialize min part size in initAllSubsystems
- Add comprehensive tests for the new functionality

cmd/metacache-set.go

@@ -225,7 +225,10 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 				continue
 			}
 			if yes := o.shouldSkip(ctx, entry); yes {
-				results.lastSkippedEntry = entry.name
+				// when we have not enough results, record the skipped entry
+				if o.Limit > 0 && results.len() < o.Limit {
+					results.lastSkippedEntry = entry.name
+				}
 				continue
 			}
 			if o.Limit > 0 && results.len() >= o.Limit {

cmd/object-handlers_test.go

@@ -1739,8 +1739,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	}
 	uploadID := multipartResponse.UploadID
 
-	a := 0
-	b := globalMinPartSize
+	a, b := 0, int(globalMinPartSize)
 	var parts []CompletePart
 	for partNumber := 1; partNumber <= 2; partNumber++ {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
@@ -1760,7 +1759,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 
 		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 		// Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request.
-		a = globalMinPartSize + 1
+		a = int(globalMinPartSize + 1)
 		b = len(bytesData[0].byteData) - 1
 		apiRouter.ServeHTTP(rec, req)
 		if rec.Code != http.StatusOK {

cmd/object_api_suite_test.go

@@ -20,10 +20,13 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"math/rand"
 	"strconv"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/kms"
@@ -432,6 +435,46 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix2", result.Objects[0].Name)
 		}
 	}
+
+	// check paging works.
+	ag := []string{"a", "b", "c", "d", "e", "f", "g"}
+	checkObjCount := make(map[string]int)
+	for i := 0; i < 7; i++ {
+		dirName := strings.Repeat(ag[i], 3)
+		key := fmt.Sprintf("testPrefix/%s/obj%s", dirName, dirName)
+		checkObjCount[key]++
+		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts)
+		if err != nil {
+			t.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+	}
+	{
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+		defer cancel()
+		token := ""
+		for ctx.Err() == nil {
+			result, err := obj.ListObjectsV2(ctx, "bucket", "testPrefix", token, "", 2, false, "")
+			if err != nil {
+				t.Fatalf("%s: <ERROR> %s", instanceType, err)
+			}
+			token = result.NextContinuationToken
+			if len(result.Objects) == 0 {
+				break
+			}
+			for _, obj := range result.Objects {
+				checkObjCount[obj.Name]--
+			}
+			if token == "" {
+				break
+			}
+		}
+		for key, value := range checkObjCount {
+			if value != 0 {
+				t.Errorf("%s: Expected value of objects to be %d, instead found to be %d", instanceType, 0, value)
+			}
+			delete(checkObjCount, key)
+		}
+	}
 }
 
 // Wrapper for calling testObjectOverwriteWorks for both Erasure and FS.

cmd/server-main.go

@@ -448,6 +448,9 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 }
 
 func initAllSubsystems(ctx context.Context) {
+	// Initialize minimum part size configuration
+	initMinPartSize()
+
 	// Initialize notification peer targets
 	globalNotificationSys = NewNotificationSys(globalEndpoints)
 

cmd/utils.go

@@ -289,14 +289,31 @@ const (
 	// using 'curl' and presigned URL.
 	globalMaxObjectSize = 5 * humanize.TiByte
 
-	// Minimum Part size for multipart upload is 5MiB
-	globalMinPartSize = 5 * humanize.MiByte
-
 	// Maximum Part ID for multipart upload is 10000
 	// (Acceptable values range from 1 to 10000 inclusive)
 	globalMaxPartID = 10000
 )
 
+// Minimum Part size for multipart upload, configurable via environment variable
+var globalMinPartSize int64
+
+// initMinPartSize initializes the minimum part size from environment variable
+// or uses default value of 5MiB
+//
+//nolint:unused
+func initMinPartSize() {
+	defaultMinPartSize := int64(5 * humanize.MiByte)
+	if envMinPartSize := env.Get("MINIO_MIN_PART_SIZE", ""); envMinPartSize != "" {
+		if size, err := humanize.ParseBytes(envMinPartSize); err == nil {
+			globalMinPartSize = int64(size)
+		} else {
+			globalMinPartSize = defaultMinPartSize
+		}
+	} else {
+		globalMinPartSize = defaultMinPartSize
+	}
+}
+
 // isMaxObjectSize - verify if max object size
 func isMaxObjectSize(size int64) bool {
 	return size > globalMaxObjectSize

cmd/utils_test.go

@@ -23,9 +23,12 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/dustin/go-humanize"
 )
 
 // Tests maximum object size.
@@ -227,7 +230,7 @@ func TestDumpRequest(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	req.RequestURI = "/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=USWUXHGYZQYFYFFIT3RE%2F20170529%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20170529T190139Z&X-Amz-Expires=600&X-Amz-Signature=19b58080999df54b446fc97304eb8dda60d3df1812ae97f3e8783351bfd9781d&X-Amz-SignedHeaders=host&prefix=Hello%2AWorld%2A"
+	req.RequestURI = "/?" + req.URL.RawQuery
 	req.Header.Set("content-md5", "====test")
 	jsonReq := dumpRequest(req)
 	type jsonResult struct {
@@ -246,15 +249,9 @@ func TestDumpRequest(t *testing.T) {
 	}
 
 	// Look for expected query values
-	expectedQuery := url.Values{}
-	expectedQuery.Set("prefix", "Hello*World*")
-	expectedQuery.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
-	expectedQuery.Set("X-Amz-Credential", "USWUXHGYZQYFYFFIT3RE/20170529/us-east-1/s3/aws4_request")
-	expectedQuery.Set("X-Amz-Date", "20170529T190139Z")
-	expectedQuery.Set("X-Amz-Expires", "600")
-	expectedQuery.Set("X-Amz-SignedHeaders", "host")
+	expectedQuery := req.URL.Query()
 	expectedQuery.Set("X-Amz-Signature", "19b58080999df54b446fc97304eb8dda60d3df1812ae97f3e8783351bfd9781d")
-	expectedRequestURI := "/?" + expectedQuery.Encode()
+	expectedRequestURI := req.URL.Path + "?" + expectedQuery.Encode()
 	if !reflect.DeepEqual(res.RequestURI, expectedRequestURI) {
 		t.Fatalf("Expected %#v, got %#v", expectedRequestURI, res.RequestURI)
 	}
@@ -399,3 +396,43 @@ func TestGetMinioMode(t *testing.T) {
 	globalIsDistErasure, globalIsErasure = false, false
 	testMinioMode(globalMinioModeFS)
 }
+
+// Test initMinPartSize function
+func TestInitMinPartSize(t *testing.T) {
+	// Save original value
+	originalValue := globalMinPartSize
+
+	// Test case 1: No environment variable set, should use default
+	os.Unsetenv("MINIO_MIN_PART_SIZE")
+	initMinPartSize()
+	expectedDefault := int64(5 * humanize.MiByte)
+	if globalMinPartSize != expectedDefault {
+		t.Errorf("Expected default value %d, got %d", expectedDefault, globalMinPartSize)
+	}
+
+	// Test case 2: Valid environment variable set
+	os.Setenv("MINIO_MIN_PART_SIZE", "10MiB")
+	initMinPartSize()
+	expectedFromEnv := int64(10 * humanize.MiByte)
+	if globalMinPartSize != expectedFromEnv {
+		t.Errorf("Expected value from env %d, got %d", expectedFromEnv, globalMinPartSize)
+	}
+
+	// Test case 3: Invalid environment variable set, should fallback to default
+	os.Setenv("MINIO_MIN_PART_SIZE", "invalid-size")
+	initMinPartSize()
+	if globalMinPartSize != expectedDefault {
+		t.Errorf("Expected fallback to default %d, got %d", expectedDefault, globalMinPartSize)
+	}
+
+	// Test case 4: Large value
+	os.Setenv("MINIO_MIN_PART_SIZE", "100MiB")
+	initMinPartSize()
+	expectedLarge := int64(100 * humanize.MiByte)
+	if globalMinPartSize != expectedLarge {
+		t.Errorf("Expected large value %d, got %d", expectedLarge, globalMinPartSize)
+	}
+
+	// Restore original value
+	globalMinPartSize = originalValue
+}