539 lines
16 KiB
Go
539 lines
16 KiB
Go
package upload
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/textproto"
|
|
"os"
|
|
"path"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"gitlab.com/gitlab-org/gitlab/workhorse/internal/api"
|
|
"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper"
|
|
"gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy"
|
|
"gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper"
|
|
"gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test"
|
|
"gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream/roundtripper"
|
|
)
|
|
|
|
var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
|
|
|
|
type testFormProcessor struct{ SavedFileTracker }
|
|
|
|
func (a *testFormProcessor) ProcessField(ctx context.Context, formName string, writer *multipart.Writer) error {
|
|
if formName != "token" && !strings.HasPrefix(formName, "file.") && !strings.HasPrefix(formName, "other.") {
|
|
return fmt.Errorf("illegal field: %v", formName)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *testFormProcessor) Finalize(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
func TestUploadTempPathRequirement(t *testing.T) {
|
|
apiResponse := &api.Response{}
|
|
preparer := &DefaultPreparer{}
|
|
_, err := preparer.Prepare(apiResponse)
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestUploadHandlerForwardingRawData(t *testing.T) {
|
|
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
|
|
require.Equal(t, "PATCH", r.Method, "method")
|
|
|
|
body, err := ioutil.ReadAll(r.Body)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "REQUEST", string(body), "request body")
|
|
|
|
w.WriteHeader(202)
|
|
fmt.Fprint(w, "RESPONSE")
|
|
})
|
|
defer ts.Close()
|
|
|
|
httpRequest, err := http.NewRequest("PATCH", ts.URL+"/url/path", bytes.NewBufferString("REQUEST"))
|
|
require.NoError(t, err)
|
|
|
|
tempPath := t.TempDir()
|
|
response := httptest.NewRecorder()
|
|
|
|
handler := newProxy(ts.URL)
|
|
fa := &eagerAuthorizer{&api.Response{TempPath: tempPath}}
|
|
preparer := &DefaultPreparer{}
|
|
|
|
interceptMultipartFiles(response, httpRequest, handler, nil, fa, preparer)
|
|
|
|
require.Equal(t, 202, response.Code)
|
|
require.Equal(t, "RESPONSE", response.Body.String(), "response body")
|
|
}
|
|
|
|
func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
|
|
var filePath string
|
|
tempPath := t.TempDir()
|
|
|
|
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
|
|
require.Equal(t, "PUT", r.Method, "method")
|
|
require.NoError(t, r.ParseMultipartForm(100000))
|
|
|
|
require.Empty(t, r.MultipartForm.File, "Expected to not receive any files")
|
|
require.Equal(t, "test", r.FormValue("token"), "Expected to receive token")
|
|
require.Equal(t, "my.file", r.FormValue("file.name"), "Expected to receive a filename")
|
|
|
|
filePath = r.FormValue("file.path")
|
|
require.True(t, strings.HasPrefix(filePath, tempPath), "Expected to the file to be in tempPath")
|
|
|
|
require.Empty(t, r.FormValue("file.remote_url"), "Expected to receive empty remote_url")
|
|
require.Empty(t, r.FormValue("file.remote_id"), "Expected to receive empty remote_id")
|
|
require.Equal(t, "4", r.FormValue("file.size"), "Expected to receive the file size")
|
|
|
|
hashes := map[string]string{
|
|
"md5": "098f6bcd4621d373cade4e832627b4f6",
|
|
"sha1": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
|
|
"sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
|
|
"sha512": "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff",
|
|
}
|
|
|
|
for algo, hash := range hashes {
|
|
require.Equal(t, hash, r.FormValue("file."+algo), "file hash %s", algo)
|
|
}
|
|
|
|
require.Len(t, r.MultipartForm.Value, 12, "multipart form values")
|
|
|
|
w.WriteHeader(202)
|
|
fmt.Fprint(w, "RESPONSE")
|
|
})
|
|
|
|
var buffer bytes.Buffer
|
|
|
|
writer := multipart.NewWriter(&buffer)
|
|
writer.WriteField("token", "test")
|
|
file, err := writer.CreateFormFile("file", "my.file")
|
|
require.NoError(t, err)
|
|
fmt.Fprint(file, "test")
|
|
writer.Close()
|
|
|
|
httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", nil)
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
httpRequest = httpRequest.WithContext(ctx)
|
|
httpRequest.Body = ioutil.NopCloser(&buffer)
|
|
httpRequest.ContentLength = int64(buffer.Len())
|
|
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
|
|
response := httptest.NewRecorder()
|
|
|
|
handler := newProxy(ts.URL)
|
|
|
|
fa := &eagerAuthorizer{&api.Response{TempPath: tempPath}}
|
|
preparer := &DefaultPreparer{}
|
|
|
|
interceptMultipartFiles(response, httpRequest, handler, &testFormProcessor{}, fa, preparer)
|
|
require.Equal(t, 202, response.Code)
|
|
|
|
cancel() // this will trigger an async cleanup
|
|
waitUntilDeleted(t, filePath)
|
|
}
|
|
|
|
func TestUploadHandlerDetectingInjectedMultiPartData(t *testing.T) {
|
|
var filePath string
|
|
|
|
tests := []struct {
|
|
name string
|
|
field string
|
|
response int
|
|
}{
|
|
{
|
|
name: "injected file.path",
|
|
field: "file.path",
|
|
response: 400,
|
|
},
|
|
{
|
|
name: "injected file.remote_id",
|
|
field: "file.remote_id",
|
|
response: 400,
|
|
},
|
|
{
|
|
name: "field with other prefix",
|
|
field: "other.path",
|
|
response: 202,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
|
|
require.Equal(t, "PUT", r.Method, "method")
|
|
|
|
w.WriteHeader(202)
|
|
fmt.Fprint(w, "RESPONSE")
|
|
})
|
|
|
|
var buffer bytes.Buffer
|
|
|
|
writer := multipart.NewWriter(&buffer)
|
|
file, err := writer.CreateFormFile("file", "my.file")
|
|
require.NoError(t, err)
|
|
fmt.Fprint(file, "test")
|
|
|
|
writer.WriteField(test.field, "value")
|
|
writer.Close()
|
|
|
|
httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", &buffer)
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
httpRequest = httpRequest.WithContext(ctx)
|
|
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
|
|
response := httptest.NewRecorder()
|
|
|
|
handler := newProxy(ts.URL)
|
|
|
|
testInterceptMultipartFiles(t, response, httpRequest, handler, &testFormProcessor{})
|
|
require.Equal(t, test.response, response.Code)
|
|
|
|
cancel() // this will trigger an async cleanup
|
|
waitUntilDeleted(t, filePath)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUploadProcessingField(t *testing.T) {
|
|
var buffer bytes.Buffer
|
|
|
|
writer := multipart.NewWriter(&buffer)
|
|
writer.WriteField("token2", "test")
|
|
writer.Close()
|
|
|
|
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
|
|
require.NoError(t, err)
|
|
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
response := httptest.NewRecorder()
|
|
|
|
testInterceptMultipartFiles(t, response, httpRequest, nilHandler, &testFormProcessor{})
|
|
|
|
require.Equal(t, 500, response.Code)
|
|
}
|
|
|
|
func TestUploadingMultipleFiles(t *testing.T) {
|
|
testhelper.ConfigureSecret()
|
|
|
|
var buffer bytes.Buffer
|
|
|
|
writer := multipart.NewWriter(&buffer)
|
|
for i := 0; i < 11; i++ {
|
|
_, err := writer.CreateFormFile(fmt.Sprintf("file %v", i), "my.file")
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, writer.Close())
|
|
|
|
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
|
|
require.NoError(t, err)
|
|
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
response := httptest.NewRecorder()
|
|
|
|
testInterceptMultipartFiles(t, response, httpRequest, nilHandler, &testFormProcessor{})
|
|
|
|
require.Equal(t, 400, response.Code)
|
|
require.Equal(t, "upload request contains more than 10 files\n", response.Body.String())
|
|
}
|
|
|
|
func TestUploadProcessingFile(t *testing.T) {
|
|
testhelper.ConfigureSecret()
|
|
tempPath := t.TempDir()
|
|
|
|
objectStore, testServer := test.StartObjectStore()
|
|
defer testServer.Close()
|
|
|
|
storeUrl := testServer.URL + test.ObjectPath
|
|
|
|
tests := []struct {
|
|
name string
|
|
preauth *api.Response
|
|
content func(t *testing.T) []byte
|
|
}{
|
|
{
|
|
name: "FileStore Upload",
|
|
preauth: &api.Response{TempPath: tempPath},
|
|
content: func(t *testing.T) []byte {
|
|
entries, err := os.ReadDir(tempPath)
|
|
require.NoError(t, err)
|
|
require.Len(t, entries, 1)
|
|
content, err := os.ReadFile(path.Join(tempPath, entries[0].Name()))
|
|
require.NoError(t, err)
|
|
return content
|
|
},
|
|
},
|
|
{
|
|
name: "ObjectStore Upload",
|
|
preauth: &api.Response{RemoteObject: api.RemoteObject{StoreURL: storeUrl, ID: "123"}},
|
|
content: func(*testing.T) []byte { return objectStore.GetObject(test.ObjectPath) },
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var buffer bytes.Buffer
|
|
writer := multipart.NewWriter(&buffer)
|
|
file, err := writer.CreateFormFile("file", "my.file")
|
|
require.NoError(t, err)
|
|
fmt.Fprint(file, "test")
|
|
writer.Close()
|
|
|
|
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
|
|
require.NoError(t, err)
|
|
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
response := httptest.NewRecorder()
|
|
fa := &eagerAuthorizer{test.preauth}
|
|
preparer := &DefaultPreparer{}
|
|
|
|
interceptMultipartFiles(response, httpRequest, nilHandler, &testFormProcessor{}, fa, preparer)
|
|
|
|
require.Equal(t, 200, response.Code)
|
|
require.Equal(t, "test", string(test.content(t)))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestInvalidFileNames(t *testing.T) {
|
|
testhelper.ConfigureSecret()
|
|
|
|
for _, testCase := range []struct {
|
|
filename string
|
|
code int
|
|
expectedPrefix string
|
|
}{
|
|
{"foobar", 200, "foobar"}, // sanity check for test setup below
|
|
{"foo/bar", 200, "bar"},
|
|
{"foo/bar/baz", 200, "baz"},
|
|
{"/../../foobar", 200, "foobar"},
|
|
{".", 500, ""},
|
|
{"..", 500, ""},
|
|
{"./", 500, ""},
|
|
} {
|
|
buffer := &bytes.Buffer{}
|
|
|
|
writer := multipart.NewWriter(buffer)
|
|
file, err := writer.CreateFormFile("file", testCase.filename)
|
|
require.NoError(t, err)
|
|
fmt.Fprint(file, "test")
|
|
writer.Close()
|
|
|
|
httpRequest, err := http.NewRequest("POST", "/example", buffer)
|
|
require.NoError(t, err)
|
|
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
response := httptest.NewRecorder()
|
|
testInterceptMultipartFiles(t, response, httpRequest, nilHandler, &SavedFileTracker{Request: httpRequest})
|
|
require.Equal(t, testCase.code, response.Code)
|
|
}
|
|
}
|
|
|
|
func TestContentDispositionRewrite(t *testing.T) {
|
|
testhelper.ConfigureSecret()
|
|
|
|
tests := []struct {
|
|
desc string
|
|
header string
|
|
code int
|
|
sanitizedHeader string
|
|
}{
|
|
{
|
|
desc: "with name",
|
|
header: `form-data; name="foo"`,
|
|
code: 200,
|
|
sanitizedHeader: `form-data; name=foo`,
|
|
},
|
|
{
|
|
desc: "with name and name*",
|
|
header: `form-data; name="foo"; name*=UTF-8''bar`,
|
|
code: 200,
|
|
sanitizedHeader: `form-data; name=bar`,
|
|
},
|
|
{
|
|
desc: "with name and invalid name*",
|
|
header: `form-data; name="foo"; name*=UTF-16''bar`,
|
|
code: 200,
|
|
sanitizedHeader: `form-data; name=foo`,
|
|
},
|
|
}
|
|
|
|
for _, testCase := range tests {
|
|
t.Run(testCase.desc, func(t *testing.T) {
|
|
h := make(textproto.MIMEHeader)
|
|
h.Set("Content-Disposition", testCase.header)
|
|
h.Set("Content-Type", "application/octet-stream")
|
|
|
|
buffer := &bytes.Buffer{}
|
|
writer := multipart.NewWriter(buffer)
|
|
file, err := writer.CreatePart(h)
|
|
require.NoError(t, err)
|
|
fmt.Fprint(file, "test")
|
|
writer.Close()
|
|
|
|
httpRequest := httptest.NewRequest("POST", "/example", buffer)
|
|
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
var upstreamRequestBuffer bytes.Buffer
|
|
customHandler := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
|
r.Write(&upstreamRequestBuffer)
|
|
})
|
|
|
|
response := httptest.NewRecorder()
|
|
testInterceptMultipartFiles(t, response, httpRequest, customHandler, &SavedFileTracker{Request: httpRequest})
|
|
|
|
upstreamRequest, err := http.ReadRequest(bufio.NewReader(&upstreamRequestBuffer))
|
|
require.NoError(t, err)
|
|
|
|
reader, err := upstreamRequest.MultipartReader()
|
|
require.NoError(t, err)
|
|
|
|
for i := 0; ; i++ {
|
|
p, err := reader.NextPart()
|
|
if err == io.EOF {
|
|
require.Equal(t, i, 1)
|
|
break
|
|
}
|
|
require.NoError(t, err)
|
|
require.Equal(t, testCase.sanitizedHeader, p.Header.Get("Content-Disposition"))
|
|
}
|
|
|
|
require.Equal(t, testCase.code, response.Code)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUploadHandlerRemovingExif(t *testing.T) {
|
|
content, err := ioutil.ReadFile("exif/testdata/sample_exif.jpg")
|
|
require.NoError(t, err)
|
|
|
|
runUploadTest(t, content, "sample_exif.jpg", 200, func(w http.ResponseWriter, r *http.Request) {
|
|
err := r.ParseMultipartForm(100000)
|
|
require.NoError(t, err)
|
|
|
|
size, err := strconv.Atoi(r.FormValue("file.size"))
|
|
require.NoError(t, err)
|
|
require.True(t, size < len(content), "Expected the file to be smaller after removal of exif")
|
|
require.True(t, size > 0, "Expected to receive not empty file")
|
|
|
|
w.WriteHeader(200)
|
|
fmt.Fprint(w, "RESPONSE")
|
|
})
|
|
}
|
|
|
|
func TestUploadHandlerRemovingExifTiff(t *testing.T) {
|
|
content, err := ioutil.ReadFile("exif/testdata/sample_exif.tiff")
|
|
require.NoError(t, err)
|
|
|
|
runUploadTest(t, content, "sample_exif.tiff", 200, func(w http.ResponseWriter, r *http.Request) {
|
|
err := r.ParseMultipartForm(100000)
|
|
require.NoError(t, err)
|
|
|
|
size, err := strconv.Atoi(r.FormValue("file.size"))
|
|
require.NoError(t, err)
|
|
require.True(t, size < len(content), "Expected the file to be smaller after removal of exif")
|
|
require.True(t, size > 0, "Expected to receive not empty file")
|
|
|
|
w.WriteHeader(200)
|
|
fmt.Fprint(w, "RESPONSE")
|
|
})
|
|
}
|
|
|
|
func TestUploadHandlerRemovingExifInvalidContentType(t *testing.T) {
|
|
content, err := ioutil.ReadFile("exif/testdata/sample_exif_invalid.jpg")
|
|
require.NoError(t, err)
|
|
|
|
runUploadTest(t, content, "sample_exif_invalid.jpg", 200, func(w http.ResponseWriter, r *http.Request) {
|
|
err := r.ParseMultipartForm(100000)
|
|
require.NoError(t, err)
|
|
|
|
output, err := ioutil.ReadFile(r.FormValue("file.path"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, content, output, "Expected the file to be same as before")
|
|
|
|
w.WriteHeader(200)
|
|
fmt.Fprint(w, "RESPONSE")
|
|
})
|
|
}
|
|
|
|
func TestUploadHandlerRemovingExifCorruptedFile(t *testing.T) {
|
|
content, err := ioutil.ReadFile("exif/testdata/sample_exif_corrupted.jpg")
|
|
require.NoError(t, err)
|
|
|
|
runUploadTest(t, content, "sample_exif_corrupted.jpg", 422, func(w http.ResponseWriter, r *http.Request) {
|
|
err := r.ParseMultipartForm(100000)
|
|
require.Error(t, err)
|
|
})
|
|
}
|
|
|
|
func runUploadTest(t *testing.T, image []byte, filename string, httpCode int, tsHandler func(http.ResponseWriter, *http.Request)) {
|
|
var buffer bytes.Buffer
|
|
|
|
writer := multipart.NewWriter(&buffer)
|
|
file, err := writer.CreateFormFile("file", filename)
|
|
require.NoError(t, err)
|
|
|
|
_, err = file.Write(image)
|
|
require.NoError(t, err)
|
|
|
|
err = writer.Close()
|
|
require.NoError(t, err)
|
|
|
|
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), tsHandler)
|
|
defer ts.Close()
|
|
|
|
httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", &buffer)
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
httpRequest = httpRequest.WithContext(ctx)
|
|
httpRequest.ContentLength = int64(buffer.Len())
|
|
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
|
|
response := httptest.NewRecorder()
|
|
|
|
handler := newProxy(ts.URL)
|
|
|
|
testInterceptMultipartFiles(t, response, httpRequest, handler, &testFormProcessor{})
|
|
require.Equal(t, httpCode, response.Code)
|
|
}
|
|
|
|
func newProxy(url string) *proxy.Proxy {
|
|
parsedURL := helper.URLMustParse(url)
|
|
return proxy.NewProxy(parsedURL, "123", roundtripper.NewTestBackendRoundTripper(parsedURL))
|
|
}
|
|
|
|
func waitUntilDeleted(t *testing.T, path string) {
|
|
var err error
|
|
require.Eventually(t, func() bool {
|
|
_, err = os.Stat(path)
|
|
return err != nil
|
|
}, 10*time.Second, 10*time.Millisecond)
|
|
require.True(t, os.IsNotExist(err), "expected the file to be deleted")
|
|
}
|
|
|
|
func testInterceptMultipartFiles(t *testing.T, w http.ResponseWriter, r *http.Request, h http.Handler, filter MultipartFormProcessor) {
|
|
t.Helper()
|
|
|
|
fa := &eagerAuthorizer{&api.Response{TempPath: t.TempDir()}}
|
|
preparer := &DefaultPreparer{}
|
|
|
|
interceptMultipartFiles(w, r, h, filter, fa, preparer)
|
|
}
|