New upstream version 7.4.0+debian

parent 4fc68dfe
image: golang:1.8
image: golang:1.10
verify:
image: golang:1.10
script:
- make verify
.test_template: &test_definition
services:
- name: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest
# Disable the hooks so we don't have to stub the GitLab API
command: ["bash", "-c", "mkdir -p /home/git/repositories && rm -rf /srv/gitlab-shell/hooks/* && exec /scripts/process-wrapper"]
alias: gitaly
variables:
GITALY_ADDRESS: "tcp://gitaly:8075"
script:
- apt update -qq && apt install -y unzip bzip2
- go version
- make test
test using go 1.8:
<<: *test_definition
test using go 1.9:
image: golang:1.9
test using go 1.10:
<<: *test_definition
test using go 1.10:
image: golang:1.10
test using go 1.11:
image: golang:1.11
<<: *test_definition
test:release:
......
......@@ -2,6 +2,60 @@
Formerly known as 'gitlab-git-http-server'.
UNRELEASED
v 7.4.0
- Strip port and include remote IP in access logs !337
v 7.3.0
- Redact sensitive url params as in Rails
v 7.2.1
- Extract correlation code out to the LabKit project !323
- Log X-Forwarded-For IPs when UNIX domain sockets are in use !324
v 7.2.0
- Update CI matrix to go1.10 + go1.11 and fix ResponseWriter bugs !309
- Add support for Redis URLs (redis:// and rediss://) in Workhorse !321
v 7.1.3
- Redact sensitive url params as in Rails
v 7.1.1
Bad release, use 7.2.0 instead.
v 7.1.0
- Add structured logFormat for text based logging !275
- Run make fmt on master !306
- Allow to configure `BUILD_DIR` and `TARGET_DIR` !308
- Resolve "Rework test suite to allow dead code to be removed" !307
- Update Prometheus vendoring !305
- General vendoring cleanup !310
- Remove Go 1.8 support !314
- Remove unused 'body' argument !315
- Refactor badgateway to use standardlib interfaces !316
- Pass Correlation-Ids down to backend systems !311
- Don't fail if /home/git/repositories already exists in Gitaly container !317
v 7.0.1
- Redact sensitive url params as in Rails
v 7.0.0
- Use the new Gitaly auth scheme (v2) !298
v 6.1.2
- Redact sensitive url params as in Rails
v 6.1.1
- Allow custom error messages to pass through to Rails !300
......
PREFIX=/usr/local
PKG := gitlab.com/gitlab-org/gitlab-workhorse
BUILD_DIR := $(CURDIR)
TARGET_DIR := $(BUILD_DIR)/_build
BUILD_DIR ?= $(CURDIR)
TARGET_DIR ?= $(BUILD_DIR)/_build
TARGET_SETUP := $(TARGET_DIR)/.ok
BIN_BUILD_DIR := $(TARGET_DIR)/bin
PKG_BUILD_DIR := $(TARGET_DIR)/src/$(PKG)
COVERAGE_DIR := $(TARGET_DIR)/cover
VERSION := $(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
VERSION_STRING := $(shell git describe)
ifeq ($(strip $(VERSION_STRING)),)
VERSION_STRING := v$(shell cat VERSION)
endif
VERSION := ${VERSION_STRING}-$(shell date -u +%Y%m%d.%H%M%S)
GOBUILD := go build -ldflags "-X main.Version=$(VERSION)"
EXE_ALL := gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
INSTALL := install
......@@ -38,7 +42,7 @@ $(TARGET_SETUP):
$(call message,"Setting up target directory")
rm -rf $(TARGET_DIR)
mkdir -p "$(dir $(PKG_BUILD_DIR))"
ln -sf ../../../.. "$(PKG_BUILD_DIR)"
ln -sf "$(CURDIR)" "$(PKG_BUILD_DIR)"
mkdir -p "$(BIN_BUILD_DIR)"
touch "$(TARGET_SETUP)"
......@@ -146,7 +150,7 @@ govendor-sync: $(TARGET_SETUP)
.PHONY: fmt
fmt: $(TARGET_SETUP) install-goimports
$(call message,$@)
@goimports -w -l $(LOCAL_GO_FILES)
@goimports -w -local $(PKG) -l $(LOCAL_GO_FILES)
.PHONY: goimports
install-goimports: $(TARGET_SETUP)
......
......@@ -21,5 +21,28 @@ maintainers. The release process is:
- create a merge request to update CHANGELOG and VERSION on the
respective release branch (usually `master`)
- make sure the new version number adheres to our [versioning standard](#versioning)
- merge the merge request
- run `make release` on the release branch
## Versioning
Workhorse uses a variation of SemVer. We don't use "normal" SemVer
because we have to be able to integrate into GitLab stable branches.
A version has the format MAJOR.MINOR.PATCH.
- Major and minor releases are tagged on the `master` branch
- If the change is backwards compatible, increment the MINOR counter
- If the change breaks compatibility, increment MAJOR and set MINOR to `0`
- Patch release tags must be made on stable branches
- Only make a patch release when targeting a GitLab stable branch
This means that tags that end in `.0` (e.g. `8.5.0`) must always be on
the master branch, and tags that end in anthing other than `.0` (e.g.
`8.5.2`) must always be on a stable branch.
> The reason we do this is that SemVer suggests something like a
> refactoring constitutes a "patch release", while the GitLab stable
> branch quality standards do not allow for back-porting refactorings
> into a stable branch.
......@@ -49,7 +49,7 @@ Options:
-authSocket string
Optional: Unix domain socket to dial authBackend at
-developmentMode
Allow to serve assets from Rails app
Allow the assets to be served from Rails app
-documentRoot string
Path to static files content (default "public")
-listenAddr string
......
......@@ -2,7 +2,7 @@
set -eu
IMPORT_RESULT=$(goimports -e -l "$@")
IMPORT_RESULT=$(goimports -e -local "gitlab.com/gitlab-org/gitlab-workhorse" -l "$@")
if [ -n "${IMPORT_RESULT}" ]; then
echo >&2 "Formatting or imports need fixing: 'make fmt'"
......
......@@ -7,13 +7,13 @@ import (
"regexp"
"testing"
jwt "github.com/dgrijalva/jwt-go"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/secret"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"github.com/dgrijalva/jwt-go"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/roundtripper"
)
func okHandler(w http.ResponseWriter, _ *http.Request, _ *api.Response) {
......@@ -34,7 +34,7 @@ func runPreAuthorizeHandler(t *testing.T, ts *httptest.Server, suffix string, ur
}
parsedURL := helper.URLMustParse(ts.URL)
testhelper.ConfigureSecret()
a := api.NewAPI(parsedURL, "123", badgateway.TestRoundTripper(parsedURL))
a := api.NewAPI(parsedURL, "123", roundtripper.NewTestBackendRoundTripper(parsedURL))
response := httptest.NewRecorder()
a.PreAuthorizeHandler(okHandler, suffix).ServeHTTP(response, httpRequest)
......
// Tests in this file need access to a real Gitaly server to run. The address
// is supplied via the GITALY_ADDRESS environment variable
package main
import (
"context"
"fmt"
"os"
"os/exec"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
pb "gitlab.com/gitlab-org/gitaly-proto/go"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
)
var (
gitalyAddress string
)
func init() {
gitalyAddress = os.Getenv("GITALY_ADDRESS")
}
func skipUnlessRealGitaly(t *testing.T) {
t.Log(gitalyAddress)
if gitalyAddress != "" {
return
}
t.Skip(`Please set GITALY_ADDRESS="..." to run Gitaly integration tests`)
}
func realGitalyAuthResponse(apiResponse *api.Response) *api.Response {
apiResponse.GitalyServer.Address = gitalyAddress
return apiResponse
}
func realGitalyOkBody(t *testing.T) *api.Response {
return realGitalyAuthResponse(gitOkBody(t))
}
func ensureGitalyRepository(t *testing.T, apiResponse *api.Response) error {
namespace, err := gitaly.NewNamespaceClient(apiResponse.GitalyServer)
if err != nil {
return err
}
repository, err := gitaly.NewRepositoryClient(apiResponse.GitalyServer)
if err != nil {
return err
}
// Remove the repository if it already exists, for consistency
rmNsReq := &pb.RemoveNamespaceRequest{
StorageName: apiResponse.Repository.StorageName,
Name: apiResponse.Repository.RelativePath,
}
_, err = namespace.RemoveNamespace(context.Background(), rmNsReq)
if err != nil {
return err
}
createReq := &pb.CreateRepositoryFromURLRequest{
Repository: &apiResponse.Repository,
Url: "https://gitlab.com/gitlab-org/gitlab-test.git",
}
_, err = repository.CreateRepositoryFromURL(context.Background(), createReq)
return err
}
func TestAllowedClone(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
// Prepare test server and backend
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
// Do the git clone
require.NoError(t, os.RemoveAll(scratchDir))
cloneCmd := exec.Command("git", "clone", fmt.Sprintf("%s/%s", ws.URL, testRepo), checkoutDir)
runOrFail(t, cloneCmd)
// We may have cloned an 'empty' repository, 'git log' will fail in it
logCmd := exec.Command("git", "log", "-1", "--oneline")
logCmd.Dir = checkoutDir
runOrFail(t, logCmd)
}
func TestAllowedShallowClone(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
// Prepare test server and backend
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
// Shallow git clone (depth 1)
require.NoError(t, os.RemoveAll(scratchDir))
cloneCmd := exec.Command("git", "clone", "--depth", "1", fmt.Sprintf("%s/%s", ws.URL, testRepo), checkoutDir)
runOrFail(t, cloneCmd)
// We may have cloned an 'empty' repository, 'git log' will fail in it
logCmd := exec.Command("git", "log", "-1", "--oneline")
logCmd.Dir = checkoutDir
runOrFail(t, logCmd)
}
func TestAllowedPush(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
// Prepare the test server and backend
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
// Perform the git push
pushCmd := exec.Command("git", "push", fmt.Sprintf("%s/%s", ws.URL, testRepo), fmt.Sprintf("master:%s", newBranch()))
pushCmd.Dir = checkoutDir
runOrFail(t, pushCmd)
}
func TestAllowedGetGitBlob(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
// the LICENSE file in the test repository
oid := "50b27c6518be44c42c4d87966ae2481ce895624c"
expectedBody := "The MIT License (MIT)"
bodyLen := 1075
jsonParams := fmt.Sprintf(
`{
"GitalyServer":{"Address":"%s", "Token":""},
"GetBlobRequest":{
"repository":{"storage_name":"%s", "relative_path":"%s"},
"oid":"%s",
"limit":-1
}
}`,
gitalyAddress, apiResponse.Repository.StorageName, apiResponse.Repository.RelativePath, oid,
)
resp, body, err := doSendDataRequest("/something", "git-blob", jsonParams)
require.NoError(t, err)
shortBody := string(body[:len(expectedBody)])
assert.Equal(t, 200, resp.StatusCode, "GET %q: status code", resp.Request.URL)
assert.Equal(t, expectedBody, shortBody, "GET %q: response body", resp.Request.URL)
testhelper.AssertResponseHeader(t, resp, "Content-Length", strconv.Itoa(bodyLen))
assertNginxResponseBuffering(t, "no", resp, "GET %q: nginx response buffering", resp.Request.URL)
}
......@@ -11,24 +11,22 @@ import (
"os"
"os/exec"
"path"
"strconv"
"strings"
"testing"
"time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/git"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
pb "gitlab.com/gitlab-org/gitaly-proto/go"
"github.com/golang/protobuf/jsonpb"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
pb "gitlab.com/gitlab-org/gitaly-proto/go"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/git"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
)
func TestFailedCloneNoGitaly(t *testing.T) {
......@@ -328,100 +326,6 @@ func TestPostUploadPackProxiedToGitalyInterrupted(t *testing.T) {
}
}
func TestGetInfoRefsHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) {
gitalyServer, _ := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
apiResponse := gitOkBody(t)
apiResponse.GitalyServer.Address = ""
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
resource := "/gitlab-org/gitlab-test.git/info/refs?service=git-upload-pack"
resp, body := httpGet(t, ws.URL+resource, nil)
assert.Equal(t, 200, resp.StatusCode, "GET %q", resource)
assert.NotContains(t, string(testhelper.GitalyInfoRefsResponseMock), body, "GET %q: should not have been proxied to Gitaly", resource)
testhelper.AssertResponseHeader(t, resp, "Content-Type", "application/x-git-upload-pack-advertisement")
}
func TestPostReceivePackHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) {
gitalyServer, _ := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
apiResponse := gitOkBody(t)
apiResponse.GitalyServer.Address = ""
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
resource := "/gitlab-org/gitlab-test.git/git-receive-pack"
payload := []byte("This payload should not reach Gitaly")
resp, body := httpPost(
t,
ws.URL+resource,
map[string]string{"Content-type": "application/x-git-receive-pack-request"},
payload,
)
assert.Equal(t, 200, resp.StatusCode, "POST %q: status code", resource)
assert.NotContains(t, payload, body, "POST %q: request should not have been proxied to Gitaly", resource)
testhelper.AssertResponseHeader(t, resp, "Content-Type", "application/x-git-receive-pack-result")
}
func TestPostUploadPackHandledLocallyDueToEmptyGitalySocketPath(t *testing.T) {
gitalyServer, _ := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
apiResponse := gitOkBody(t)
apiResponse.GitalyServer.Address = ""
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
resource := "/gitlab-org/gitlab-test.git/git-upload-pack"
payload := []byte("This payload should not reach Gitaly")
resp, body := httpPost(
t,
ws.URL+resource,
map[string]string{"Content-type": "application/x-git-upload-pack-request"},
payload,
)
assert.Equal(t, 200, resp.StatusCode, "POST %q: status code", resource)
assert.NotContains(t, payload, body, "POST %q: request should not have been proxied to Gitaly", resource)
testhelper.AssertResponseHeader(t, resp, "Content-Type", "application/x-git-upload-pack-result")
}
func TestGetBlobProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
repoStorage := "default"
oid := "54fcc214b94e78d7a41a9a8fe6d87a5e59500e51"
repoRelativePath := "foo/bar.git"
jsonParams := fmt.Sprintf(`{"GitalyServer":{"Address":"%s","Token":""},"GetBlobRequest":{"repository":{"storage_name":"%s","relative_path":"%s"},"oid":"%s","limit":-1}}`,
gitalyAddress, repoStorage, repoRelativePath, oid)
expectedBody := testhelper.GitalyGetBlobResponseMock
blobLength := len(expectedBody)
resp, body, err := doSendDataRequest("/something", "git-blob", jsonParams)
require.NoError(t, err)
assert.Equal(t, 200, resp.StatusCode, "GET %q: status code", resp.Request.URL)
assert.Equal(t, expectedBody, string(body), "GET %q: response body", resp.Request.URL)
assert.Equal(t, blobLength, len(body), "GET %q: body size", resp.Request.URL)
testhelper.AssertResponseHeader(t, resp, "Content-Length", strconv.Itoa(blobLength))
}
func TestGetDiffProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
......
......@@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
......@@ -14,7 +13,6 @@ import (
"github.com/prometheus/client_golang/prometheus"
pb "gitlab.com/gitlab-org/gitaly-proto/go"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/secret"
......@@ -57,7 +55,7 @@ func init() {
prometheus.MustRegister(bytesTotal)
}
func NewAPI(myURL *url.URL, version string, roundTripper *badgateway.RoundTripper) *API {
func NewAPI(myURL *url.URL, version string, roundTripper http.RoundTripper) *API {
return &API{
Client: &http.Client{Transport: roundTripper},
URL: myURL,
......@@ -170,34 +168,30 @@ func rebaseUrl(url *url.URL, onto *url.URL, suffix string) *url.URL {
return &newUrl
}
func (api *API) newRequest(r *http.Request, body io.Reader, suffix string) (*http.Request, error) {
func (api *API) newRequest(r *http.Request, suffix string) (*http.Request, error) {
authReq := &http.Request{
Method: r.Method,
URL: rebaseUrl(r.URL, api.URL, suffix),
Header: helper.HeaderClone(r.Header),
}
if body != nil {
authReq.Body = ioutil.NopCloser(body)
}
// Clean some headers when issuing a new request without body
if body == nil {
authReq.Header.Del("Content-Type")
authReq.Header.Del("Content-Encoding")
authReq.Header.Del("Content-Length")
authReq.Header.Del("Content-Disposition")
authReq.Header.Del("Accept-Encoding")
// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
authReq.Header.Del("Transfer-Encoding")
authReq.Header.Del("Connection")
authReq.Header.Del("Keep-Alive")
authReq.Header.Del("Proxy-Authenticate")
authReq.Header.Del("Proxy-Authorization")
authReq.Header.Del("Te")
authReq.Header.Del("Trailers")
authReq.Header.Del("Upgrade")
}
authReq.Header.Del("Content-Type")
authReq.Header.Del("Content-Encoding")
authReq.Header.Del("Content-Length")
authReq.Header.Del("Content-Disposition")
authReq.Header.Del("Accept-Encoding")
// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
authReq.Header.Del("Transfer-Encoding")
authReq.Header.Del("Connection")
authReq.Header.Del("Keep-Alive")
authReq.Header.Del("Proxy-Authenticate")
authReq.Header.Del("Proxy-Authorization")
authReq.Header.Del("Te")
authReq.Header.Del("Trailers")
authReq.Header.Del("Upgrade")
// Also forward the Host header, which is excluded from the Header map by the http libary.
// This allows the Host header received by the backend to be consistent with other
......@@ -218,7 +212,7 @@ func (api *API) newRequest(r *http.Request, body io.Reader, suffix string) (*htt
return authReq, nil
}
// Perform a pre-authorization check against the API for the given HTTP request
// PreAuthorize performs a pre-authorization check against the API for the given HTTP request
//
// If `outErr` is set, the other fields will be nil and it should be treated as
// a 500 error.
......@@ -227,7 +221,7 @@ func (api *API) newRequest(r *http.Request, body io.Reader, suffix string) (*htt
//
// authResponse will only be present if the authorization check was successful
func (api *API) PreAuthorize(suffix string, r *http.Request) (httpResponse *http.Response, authResponse *Response, outErr error) {
authReq, err := api.newRequest(r, nil, suffix)
authReq, err := api.newRequest(r, suffix)
if err != nil {
return nil, nil, fmt.Errorf("preAuthorizeHandler newUpstreamRequest: %v", err)
}
......
......@@ -12,7 +12,7 @@ import (
func Block(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rw := &blocker{rw: w, r: r}
defer rw.Flush()
defer rw.flush()
h.ServeHTTP(rw, r)
})
}
......@@ -33,7 +33,7 @@ func (b *blocker) Write(data []byte) (int, error) {
b.WriteHeader(http.StatusOK)
}
if b.hijacked {
return 0, nil
return len(data), nil
}
return b.rw.Write(data)
......@@ -56,6 +56,6 @@ func (b *blocker) WriteHeader(status int) {
b.rw.WriteHeader(b.status)
}
func (b *blocker) Flush() {
func (b *blocker) flush() {
b.WriteHeader(http.StatusOK)
}
package api
import (
"io/ioutil"