Commit b7a1db9d authored by manas kashyap's avatar manas kashyap 💬

Update upstream source from tag 'upstream/5.2.0+debian'

Update to upstream version '5.2.0+debian'
with Debian dir 4eafbee93cb7d06be6af3b15ba093c61c28ab7d5
parents db8d957f c3c6684b
...@@ -2,6 +2,18 @@ ...@@ -2,6 +2,18 @@
Formerly known as 'gitlab-git-http-server'. Formerly known as 'gitlab-git-http-server'.
v 5.2.0
- Populate Git Protocol !276
- Add support for GitConfigOptions required for git-receive-pack command !281
v 5.1.0
- Log using correlation-id bound to the incoming request !258
- Prevent uploading two files as artifacts in single request !273
- Prometheus instrumentation !279
v 5.0.0 v 5.0.0
- Update httprs for broken range implementations !266 - Update httprs for broken range implementations !266
......
...@@ -6,32 +6,34 @@ export PATH:=${GOPATH}/bin:${PATH} ...@@ -6,32 +6,34 @@ export PATH:=${GOPATH}/bin:${PATH}
GOBUILD=go build -ldflags "-X main.Version=${VERSION}" GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
PKG=gitlab.com/gitlab-org/gitlab-workhorse PKG=gitlab.com/gitlab-org/gitlab-workhorse
PKG_ALL = $(shell GOPATH=${GOPATH} go list ${PKG}/... | grep -v /vendor/) PKG_ALL = $(shell GOPATH=${GOPATH} go list ${PKG}/... | grep -v /vendor/)
EXE_ALL = gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
all: clean-build gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse all: clean-build $(EXE_ALL)
gitlab-zip-cat: ${BUILD_DIR}/_build $(shell find cmd/gitlab-zip-cat/ -name '*.go') gitlab-zip-cat: ${BUILD_DIR}/_build/.sync $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG}/cmd/$@ ${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG}/cmd/$@
gitlab-zip-metadata: ${BUILD_DIR}/_build $(shell find cmd/gitlab-zip-metadata/ -name '*.go') gitlab-zip-metadata: ${BUILD_DIR}/_build/.sync $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG}/cmd/$@ ${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG}/cmd/$@
gitlab-workhorse: ${BUILD_DIR}/_build $(shell find . -name '*.go' | grep -v '^\./_') gitlab-workhorse: ${BUILD_DIR}/_build/.sync $(shell find . -name '*.go' | grep -v '^\./_')
${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG} ${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG}
install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
mkdir -p $(DESTDIR)${PREFIX}/bin/ mkdir -p $(DESTDIR)${PREFIX}/bin/
cd ${BUILD_DIR} && install gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata ${DESTDIR}${PREFIX}/bin/ cd ${BUILD_DIR} && install gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata ${DESTDIR}${PREFIX}/bin/
${BUILD_DIR}/_build: ${BUILD_DIR}/_build/.sync:
mkdir -p $@/src/${PKG} mkdir -p ${BUILD_DIR}/_build/src/${PKG}
tar -cf - --exclude _build --exclude .git . | (cd $@/src/${PKG} && tar -xf -) tar -cf - --exclude _build --exclude .git . | (cd ${BUILD_DIR}/_build/src/${PKG} && tar -xf -)
touch $@ touch $@
.PHONY: test .PHONY: test
test: clean-build clean-workhorse all govendor test: clean-build clean-workhorse govendor prepare-tests
go fmt ${PKG_ALL} | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }' go fmt ${PKG_ALL} | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
_support/detect-context.sh _support/detect-context.sh
cd ${GOPATH}/src/${PKG} && govendor sync cd ${GOPATH}/src/${PKG} && govendor sync
cp $(EXE_ALL) ${GOPATH}/src/${PKG}
go test ${PKG_ALL} go test ${PKG_ALL}
@echo SUCCESS @echo SUCCESS
...@@ -61,3 +63,9 @@ release: ...@@ -61,3 +63,9 @@ release:
.PHONY: clean-build .PHONY: clean-build
clean-build: clean-build:
rm -rf ${BUILD_DIR}/_build rm -rf ${BUILD_DIR}/_build
.PHONY: prepare-tests
prepare-tests: testdata/data/group/test.git $(EXE_ALL)
testdata/data/group/test.git:
git clone --quiet --bare https://gitlab.com/gitlab-org/gitlab-test.git $@
# GitLab-Workhorse development process
## Maintainers
GitLab-Workhorse has the following maintainers:
- Nick Thomas `@nick.thomas`
- Jacob Vosmaer `@jacobvosmaer-gitlab`
This list is defined at https://about.gitlab.com/team/.
## Merging and reviewing contributions
Contributions must be reviewed by at least one Workhorse maintainer.
The final merge must be performed by a maintainer.
## Releases
New versions of Workhorse can be released by one of the Workhorse
maintainers. The release process is:
- create a merge request to update CHANGELOG and VERSION on the
respective release branch (usually `master`)
- merge the merge request
- run `make release` on the release branch
...@@ -65,9 +65,19 @@ func TestGetInfoRefsProxiedToGitalySuccessfully(t *testing.T) { ...@@ -65,9 +65,19 @@ func TestGetInfoRefsProxiedToGitalySuccessfully(t *testing.T) {
apiResponse := gitOkBody(t) apiResponse := gitOkBody(t)
apiResponse.GitalyServer.Address = gitalyAddress apiResponse.GitalyServer.Address = gitalyAddress
for _, showAllRefs := range []bool{true, false} { testCases := []struct {
t.Run(fmt.Sprintf("ShowAllRefs=%v", showAllRefs), func(t *testing.T) { showAllRefs bool
apiResponse.ShowAllRefs = showAllRefs gitRpc string
}{
{showAllRefs: false, gitRpc: "git-upload-pack"},
{showAllRefs: true, gitRpc: "git-upload-pack"},
{showAllRefs: false, gitRpc: "git-receive-pack"},
{showAllRefs: true, gitRpc: "git-receive-pack"},
}
for _, tc := range testCases {
t.Run(fmt.Sprintf("ShowAllRefs=%v,gitRpc=%v", tc.showAllRefs, tc.gitRpc), func(t *testing.T) {
apiResponse.ShowAllRefs = tc.showAllRefs
ts := testAuthServer(nil, 200, apiResponse) ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close() defer ts.Close()
...@@ -75,19 +85,30 @@ func TestGetInfoRefsProxiedToGitalySuccessfully(t *testing.T) { ...@@ -75,19 +85,30 @@ func TestGetInfoRefsProxiedToGitalySuccessfully(t *testing.T) {
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
resource := "/gitlab-org/gitlab-test.git/info/refs?service=git-upload-pack" gitProtocol := "fake git protocol"
_, body := httpGet(t, ws.URL+resource) resource := "/gitlab-org/gitlab-test.git/info/refs?service=" + tc.gitRpc
resp, body := httpGet(t, ws.URL+resource, map[string]string{"Git-Protocol": gitProtocol})
require.Equal(t, 200, resp.StatusCode)
bodySplit := strings.SplitN(body, "\000", 3)
require.Len(t, bodySplit, 3)
expectedContent := "\n\000" + string(testhelper.GitalyInfoRefsResponseMock) + "\000" gitalyRequest := &pb.InfoRefsRequest{}
if showAllRefs { require.NoError(t, jsonpb.UnmarshalString(bodySplit[0], gitalyRequest))
expectedContent = git.GitConfigShowAllRefs + expectedContent
require.Equal(t, gitProtocol, gitalyRequest.GitProtocol)
if tc.showAllRefs {
require.Equal(t, []string{git.GitConfigShowAllRefs}, gitalyRequest.GitConfigOptions)
} else {
require.Empty(t, gitalyRequest.GitConfigOptions)
} }
assert.Equal(t, expectedContent, body, "GET %q: response body", resource) require.Equal(t, tc.gitRpc, bodySplit[1])
require.Equal(t, string(testhelper.GitalyInfoRefsResponseMock), bodySplit[2], "GET %q: response body", resource)
}) })
} }
} }
func TestGetInfoRefsProxiedToGitalyInterruptedStream(t *testing.T) { func TestGetInfoRefsProxiedToGitalyInterruptedStream(t *testing.T) {
...@@ -132,30 +153,40 @@ func TestPostReceivePackProxiedToGitalySuccessfully(t *testing.T) { ...@@ -132,30 +153,40 @@ func TestPostReceivePackProxiedToGitalySuccessfully(t *testing.T) {
defer gitalyServer.Stop() defer gitalyServer.Stop()
apiResponse.GitalyServer.Address = "unix://" + socketPath apiResponse.GitalyServer.Address = "unix://" + socketPath
apiResponse.GitConfigOptions = []string{"git-config-hello=world"}
ts := testAuthServer(nil, 200, apiResponse) ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
gitProtocol := "fake Git protocol"
resource := "/gitlab-org/gitlab-test.git/git-receive-pack" resource := "/gitlab-org/gitlab-test.git/git-receive-pack"
resp, body := httpPost( resp, body := httpPost(
t, t,
ws.URL+resource, ws.URL+resource,
"application/x-git-receive-pack-request", map[string]string{
"Content-Type": "application/x-git-receive-pack-request",
"Git-Protocol": gitProtocol,
},
testhelper.GitalyReceivePackResponseMock, testhelper.GitalyReceivePackResponseMock,
) )
expectedBody := strings.Join([]string{ split := strings.SplitN(body, "\000", 2)
apiResponse.Repository.StorageName, require.Len(t, split, 2)
apiResponse.Repository.RelativePath,
apiResponse.GL_ID, gitalyRequest := &pb.PostReceivePackRequest{}
apiResponse.GL_USERNAME, require.NoError(t, jsonpb.UnmarshalString(split[0], gitalyRequest))
string(testhelper.GitalyReceivePackResponseMock),
}, "\000") assert.Equal(t, apiResponse.Repository.StorageName, gitalyRequest.Repository.StorageName)
assert.Equal(t, apiResponse.Repository.RelativePath, gitalyRequest.Repository.RelativePath)
assert.Equal(t, apiResponse.GL_ID, gitalyRequest.GlId)
assert.Equal(t, apiResponse.GL_USERNAME, gitalyRequest.GlUsername)
assert.Equal(t, apiResponse.GitConfigOptions, gitalyRequest.GitConfigOptions)
assert.Equal(t, gitProtocol, gitalyRequest.GitProtocol)
assert.Equal(t, 200, resp.StatusCode, "POST %q", resource) assert.Equal(t, 200, resp.StatusCode, "POST %q", resource)
assert.Equal(t, expectedBody, body, "POST %q: response body", resource) require.Equal(t, string(testhelper.GitalyReceivePackResponseMock), split[1])
testhelper.AssertResponseHeader(t, resp, "Content-Type", "application/x-git-receive-pack-result") testhelper.AssertResponseHeader(t, resp, "Content-Type", "application/x-git-receive-pack-result")
} }
...@@ -222,30 +253,38 @@ func TestPostUploadPackProxiedToGitalySuccessfully(t *testing.T) { ...@@ -222,30 +253,38 @@ func TestPostUploadPackProxiedToGitalySuccessfully(t *testing.T) {
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
gitProtocol := "fake git protocol"
resource := "/gitlab-org/gitlab-test.git/git-upload-pack" resource := "/gitlab-org/gitlab-test.git/git-upload-pack"
resp, body := httpPost( resp, body := httpPost(
t, t,
ws.URL+resource, ws.URL+resource,
"application/x-git-upload-pack-request", map[string]string{
"Content-Type": "application/x-git-upload-pack-request",
"Git-Protocol": gitProtocol,
},
testhelper.GitalyUploadPackResponseMock, testhelper.GitalyUploadPackResponseMock,
) )
expectedBodyParts := []string{ require.Equal(t, 200, resp.StatusCode, "POST %q", resource)
apiResponse.Repository.StorageName, testhelper.AssertResponseHeader(t, resp, "Content-Type", "application/x-git-upload-pack-result")
apiResponse.Repository.RelativePath,
} bodySplit := strings.SplitN(body, "\000", 2)
require.Len(t, bodySplit, 2)
gitalyRequest := &pb.PostUploadPackRequest{}
require.NoError(t, jsonpb.UnmarshalString(bodySplit[0], gitalyRequest))
require.Equal(t, apiResponse.Repository.StorageName, gitalyRequest.Repository.StorageName)
require.Equal(t, apiResponse.Repository.RelativePath, gitalyRequest.Repository.RelativePath)
require.Equal(t, gitProtocol, gitalyRequest.GitProtocol)
if tc.showAllRefs { if tc.showAllRefs {
expectedBodyParts = append(expectedBodyParts, git.GitConfigShowAllRefs+"\n") require.Equal(t, []string{git.GitConfigShowAllRefs}, gitalyRequest.GitConfigOptions)
} else { } else {
expectedBodyParts = append(expectedBodyParts, "\n") require.Empty(t, gitalyRequest.GitConfigOptions)
} }
expectedBodyParts = append(expectedBodyParts, string(testhelper.GitalyUploadPackResponseMock)) require.Equal(t, string(testhelper.GitalyUploadPackResponseMock), bodySplit[1], "POST %q: response body", resource)
expectedBody := strings.Join(expectedBodyParts, "\000")
assert.Equal(t, 200, resp.StatusCode, "POST %q", resource)
assert.Equal(t, expectedBody, body, "POST %q: response body", resource)
testhelper.AssertResponseHeader(t, resp, "Content-Type", "application/x-git-upload-pack-result")
}) })
} }
} }
...@@ -302,7 +341,7 @@ func TestGetInfoRefsHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) { ...@@ -302,7 +341,7 @@ func TestGetInfoRefsHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) {
defer ws.Close() defer ws.Close()
resource := "/gitlab-org/gitlab-test.git/info/refs?service=git-upload-pack" resource := "/gitlab-org/gitlab-test.git/info/refs?service=git-upload-pack"
resp, body := httpGet(t, ws.URL+resource) resp, body := httpGet(t, ws.URL+resource, nil)
assert.Equal(t, 200, resp.StatusCode, "GET %q", resource) assert.Equal(t, 200, resp.StatusCode, "GET %q", resource)
assert.NotContains(t, string(testhelper.GitalyInfoRefsResponseMock), body, "GET %q: should not have been proxied to Gitaly", resource) assert.NotContains(t, string(testhelper.GitalyInfoRefsResponseMock), body, "GET %q: should not have been proxied to Gitaly", resource)
...@@ -323,7 +362,12 @@ func TestPostReceivePackHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) { ...@@ -323,7 +362,12 @@ func TestPostReceivePackHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) {
resource := "/gitlab-org/gitlab-test.git/git-receive-pack" resource := "/gitlab-org/gitlab-test.git/git-receive-pack"
payload := []byte("This payload should not reach Gitaly") payload := []byte("This payload should not reach Gitaly")
resp, body := httpPost(t, ws.URL+resource, "application/x-git-receive-pack-request", payload) resp, body := httpPost(
t,
ws.URL+resource,
map[string]string{"Content-type": "application/x-git-receive-pack-request"},
payload,
)
assert.Equal(t, 200, resp.StatusCode, "POST %q: status code", resource) assert.Equal(t, 200, resp.StatusCode, "POST %q: status code", resource)
assert.NotContains(t, payload, body, "POST %q: request should not have been proxied to Gitaly", resource) assert.NotContains(t, payload, body, "POST %q: request should not have been proxied to Gitaly", resource)
...@@ -344,7 +388,12 @@ func TestPostUploadPackHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) { ...@@ -344,7 +388,12 @@ func TestPostUploadPackHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) {
resource := "/gitlab-org/gitlab-test.git/git-upload-pack" resource := "/gitlab-org/gitlab-test.git/git-upload-pack"
payload := []byte("This payload should not reach Gitaly") payload := []byte("This payload should not reach Gitaly")
resp, body := httpPost(t, ws.URL+resource, "application/x-git-upload-pack-request", payload) resp, body := httpPost(
t,
ws.URL+resource,
map[string]string{"Content-type": "application/x-git-upload-pack-request"},
payload,
)
assert.Equal(t, 200, resp.StatusCode, "POST %q: status code", resource) assert.Equal(t, 200, resp.StatusCode, "POST %q: status code", resource)
assert.NotContains(t, payload, body, "POST %q: request should not have been proxied to Gitaly", resource) assert.NotContains(t, payload, body, "POST %q: request should not have been proxied to Gitaly", resource)
......
...@@ -107,6 +107,8 @@ type Response struct { ...@@ -107,6 +107,8 @@ type Response struct {
// RepoPath is the full path on disk to the Git repository the request is // RepoPath is the full path on disk to the Git repository the request is
// about // about
RepoPath string RepoPath string
// GitConfigOptions holds the custom options that we want to pass to the git command
GitConfigOptions []string
// StoreLFSPath is provided by the GitLab Rails application to mark where the tmp file should be placed. // StoreLFSPath is provided by the GitLab Rails application to mark where the tmp file should be placed.
// This field is deprecated. GitLab will use TempPath instead // This field is deprecated. GitLab will use TempPath instead
StoreLFSPath string StoreLFSPath string
......
...@@ -136,7 +136,8 @@ func TestUploadHandlerSendingToExternalStorage(t *testing.T) { ...@@ -136,7 +136,8 @@ func TestUploadHandlerSendingToExternalStorage(t *testing.T) {
contentBuffer, contentType := createTestMultipartForm(t, archiveData) contentBuffer, contentType := createTestMultipartForm(t, archiveData)
response := testUploadArtifacts(contentType, &contentBuffer, t, ts) response := testUploadArtifacts(contentType, &contentBuffer, t, ts)
testhelper.AssertResponseCode(t, response, 200) testhelper.AssertResponseCode(t, response, http.StatusOK)
testhelper.AssertResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
assert.Equal(t, 1, storeServerCalled, "store should be called only once") assert.Equal(t, 1, storeServerCalled, "store should be called only once")
assert.Equal(t, 1, responseProcessorCalled, "response processor should be called only once") assert.Equal(t, 1, responseProcessorCalled, "response processor should be called only once")
}) })
...@@ -166,7 +167,7 @@ func TestUploadHandlerSendingToExternalStorageAndStorageServerUnreachable(t *tes ...@@ -166,7 +167,7 @@ func TestUploadHandlerSendingToExternalStorageAndStorageServerUnreachable(t *tes
defer ts.Close() defer ts.Close()
response := testUploadArtifactsFromTestZip(t, ts) response := testUploadArtifactsFromTestZip(t, ts)
testhelper.AssertResponseCode(t, response, 500) testhelper.AssertResponseCode(t, response, http.StatusInternalServerError)
} }
func TestUploadHandlerSendingToExternalStorageAndInvalidURLIsUsed(t *testing.T) { func TestUploadHandlerSendingToExternalStorageAndInvalidURLIsUsed(t *testing.T) {
...@@ -192,7 +193,7 @@ func TestUploadHandlerSendingToExternalStorageAndInvalidURLIsUsed(t *testing.T) ...@@ -192,7 +193,7 @@ func TestUploadHandlerSendingToExternalStorageAndInvalidURLIsUsed(t *testing.T)
defer ts.Close() defer ts.Close()
response := testUploadArtifactsFromTestZip(t, ts) response := testUploadArtifactsFromTestZip(t, ts)
testhelper.AssertResponseCode(t, response, 500) testhelper.AssertResponseCode(t, response, http.StatusInternalServerError)
} }
func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T) { func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T) {
...@@ -230,7 +231,7 @@ func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T) ...@@ -230,7 +231,7 @@ func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T)
defer ts.Close() defer ts.Close()
response := testUploadArtifactsFromTestZip(t, ts) response := testUploadArtifactsFromTestZip(t, ts)
testhelper.AssertResponseCode(t, response, 500) testhelper.AssertResponseCode(t, response, http.StatusInternalServerError)
assert.Equal(t, 1, putCalledTimes, "upload should be called only once") assert.Equal(t, 1, putCalledTimes, "upload should be called only once")
} }
...@@ -271,7 +272,7 @@ func TestUploadHandlerSendingToExternalStorageAndSupportRequestTimeout(t *testin ...@@ -271,7 +272,7 @@ func TestUploadHandlerSendingToExternalStorageAndSupportRequestTimeout(t *testin
defer ts.Close() defer ts.Close()
response := testUploadArtifactsFromTestZip(t, ts) response := testUploadArtifactsFromTestZip(t, ts)
testhelper.AssertResponseCode(t, response, 500) testhelper.AssertResponseCode(t, response, http.StatusInternalServerError)
assert.Equal(t, 1, putCalledTimes, "upload should be called only once") assert.Equal(t, 1, putCalledTimes, "upload should be called only once")
} }
......
...@@ -10,15 +10,10 @@ import ( ...@@ -10,15 +10,10 @@ import (
) )
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
cleanup, err := testhelper.BuildExecutables() if err := testhelper.BuildExecutables(); err != nil {
if err != nil { log.WithError(err).Fatal()
log.WithError(err).Print("Test setup: failed to build executables")
os.Exit(1)
} }
os.Exit(func() int { os.Exit(m.Run())
defer cleanup()
return m.Run()
}())
} }
...@@ -18,9 +18,8 @@ import ( ...@@ -18,9 +18,8 @@ import (
) )
type artifactsUploadProcessor struct { type artifactsUploadProcessor struct {
opts *filestore.SaveFileOpts opts *filestore.SaveFileOpts
metadataFile string stored bool
stored bool
} }
func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, file *filestore.FileHandler) (*filestore.FileHandler, error) { func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, file *filestore.FileHandler) (*filestore.FileHandler, error) {
...@@ -80,9 +79,10 @@ func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName str ...@@ -80,9 +79,10 @@ func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName str
if formName != "file" { if formName != "file" {
return fmt.Errorf("Invalid form field: %q", formName) return fmt.Errorf("Invalid form field: %q", formName)
} }
if a.metadataFile != "" { if a.stored {
return fmt.Errorf("Artifacts request contains more than one file!") return fmt.Errorf("Artifacts request contains more than one file")
} }
a.stored = true
select { select {
case <-ctx.Done(): case <-ctx.Done():
......
...@@ -23,6 +23,12 @@ import ( ...@@ -23,6 +23,12 @@ import (
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts" "gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
) )
const (
MetadataHeaderKey = "Metadata-Status"
MetadataHeaderPresent = "present"
MetadataHeaderMissing = "missing"
)
func testArtifactsUploadServer(t *testing.T, authResponse api.Response, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *httptest.Server { func testArtifactsUploadServer(t *testing.T, authResponse api.Response, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *httptest.Server {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/url/path/authorize", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/url/path/authorize", func(w http.ResponseWriter, r *http.Request) {
...@@ -46,52 +52,54 @@ func testArtifactsUploadServer(t *testing.T, authResponse api.Response, bodyProc ...@@ -46,52 +52,54 @@ func testArtifactsUploadServer(t *testing.T, authResponse api.Response, bodyProc
} }
if opts.IsLocal() { if opts.IsLocal() {
if r.FormValue("file.path") == "" { if r.FormValue("file.path") == "" {
w.WriteHeader(501) t.Fatal("Expected file to be present")
return return
} }
_, err := ioutil.ReadFile(r.FormValue("file.path")) _, err := ioutil.ReadFile(r.FormValue("file.path"))
if err != nil { if err != nil {
w.WriteHeader(404) t.Fatal("Expected file to be readable")
return
}
} else if opts.IsRemote() {
if r.FormValue("file.remote_url") == "" {
t.Fatal("Expected file to be remote accessible")
return return
} }
} }
if opts.IsRemote() && r.FormValue("file.remote_url") == "" { if r.FormValue("metadata.path") != "" {
w.WriteHeader(501) metadata, err := ioutil.ReadFile(r.FormValue("metadata.path"))
return if err != nil {
} t.Fatal("Expected metadata to be readable")
return
}
gz, err := gzip.NewReader(bytes.NewReader(metadata))
if err != nil {
t.Fatal("Expected metadata to be valid gzip")
return
}
defer gz.Close()
metadata, err = ioutil.ReadAll(gz)
if err != nil {
t.Fatal("Expected metadata to be valid")
return
}
if !bytes.HasPrefix(metadata, []byte(zipartifacts.MetadataHeaderPrefix+zipartifacts.MetadataHeader)) {
t.Fatal("Expected metadata to be of valid format")
return
}
if r.FormValue("metadata.path") == "" { w.Header().Set(MetadataHeaderKey, MetadataHeaderPresent)
w.WriteHeader(502)
return
}
metadata, err := ioutil.ReadFile(r.FormValue("metadata.path")) } else {
if err != nil { w.Header().Set(MetadataHeaderKey, MetadataHeaderMissing)
w.WriteHeader(404)
return
}
gz, err := gzip.NewReader(bytes.NewReader(metadata))
if err != nil {
w.WriteHeader(405)
return
}
defer gz.Close()
metadata, err = ioutil.ReadAll(gz)
if err != nil {
w.WriteHeader(404)
return
}
if !bytes.HasPrefix(metadata, []byte(zipartifacts.MetadataHeaderPrefix+zipartifacts.MetadataHeader)) {
w.WriteHeader(400)
return
} }
w.WriteHeader(http.StatusOK)
if bodyProcessor != nil { if bodyProcessor != nil {
bodyProcessor(w, r) bodyProcessor(w, r)
} else {
w.WriteHeader(200)
} }
}) })
return testhelper.TestServerWithHandler(nil, mux.ServeHTTP) return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
...@@ -141,7 +149,8 @@ func TestUploadHandlerAddingMetadata(t *testing.T) { ...@@ -141,7 +149,8 @@ func TestUploadHandlerAddingMetadata(t *testing.T) {
writer.Close() writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts) response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
testhelper.AssertResponseCode(t, response, 200) testhelper.AssertResponseCode(t, response, http.StatusOK)
testhelper.AssertResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
} }
func TestUploadHandlerForUnsupportedArchive(t *testing.T) { func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
...@@ -164,8 +173,37 @@ func TestUploadHandlerForUnsupportedArchive(t *testing.T) { ...@@ -164,8 +173,37 @@ func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
writer.Close() writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts) response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
// 502 is a custom response code from the mock server in testUploadArtifacts testhelper.AssertResponseCode(t, response, http.StatusOK)
testhelper.AssertResponseCode(t, response, 502) testhelper.AssertResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderMissing)
}
func TestUploadHandlerForMultipleFiles(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, api.Response{TempPath: tempPath}, nil)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
file, err = writer.CreateFormFile("file", "my.file")