New upstream version 0.8.2

parent 94bbbdac
before_script:
- apt-get update -qq && apt-get install -y curl unzip bzip2
- curl -O https://storage.googleapis.com/golang/go1.5.3.linux-amd64.tar.gz
- echo '43afe0c5017e502630b1aea4d44b8a7f059bf60d7f29dfd58db454d4e4e0ae53 go1.5.3.linux-amd64.tar.gz' | shasum -c -a256 -
- rm -rf /usr/local/go && tar -C /usr/local -xzf go1.5.3.linux-amd64.tar.gz
- export PATH=/usr/local/go/bin:$PATH
image: golang:1.6.2
test:
script: make clean test
script:
- apt-get update -qq && apt-get install -y unzip bzip2
- make test
......@@ -2,12 +2,69 @@
Formerly known as 'gitlab-git-http-server'.
v0.7.2
v0.8.2
Recognize more archive formats in git.SendArchive. Make 502 errors
(failed proxy requests to Unicorn) easier to recognize in Sentry.
v0.8.1
Add Sentry (raven-go) for remote error tracking.
v0.8.0
Add JWT signed communication between gitlab-workhorse and gitlab-rails.
v0.7.11
Fix 'nil dereference' crash on Go 1.7 when parsing authBackend
parameter. Fix 'hard-wire backend host' crashes.
v0.7.10
Fix typo in metrics header name.
v0.7.9
Hard-wire backend host when using TCP.
v0.7.8
Send artifact zip file entries via the 'senddata' mechanism.
v0.7.7
Add the protocol used (HTTP) to each gitCommand call in order to check
for restricted protocol access on GitLab's side.
v0.7.6
Add the capability to inject `git format-patch` output.
v0.7.5
Add the capability to inject `git diff` output as HTTP response bodies
(@zj).
v0.7.4
Pass a timestamp when forwarding requests to Rails. Hopefully this
will give us insight into Unicorn queueing behavior.
v0.7.3
Revert 'buffer Git HTTP responses'. Set default listen socket
permissions to world read/writeable.
v0.7.2 DO NOT USE
Integrate with GOPATH during development (remove relative imports
etc.). Buffer Git HTTP responses so that we may return an error if the
local command fails early.
Update: the 'buffer Git HTTP responses' change in 0.7.2 is BAD, it
breaks shallow Git clone. Don't use 0.7.2!
v0.7.1
Set Content-Length (retrieved from Git) on raw blob data responses.
......
{
"ImportPath": "gitlab.com/gitlab-org/gitlab-workhorse",
"GoVersion": "go1.7",
"GodepVersion": "v74",
"Deps": [
{
"ImportPath": "github.com/certifi/gocertifi",
"Comment": "2016.08.31",
"Rev": "ec89d50f00d39494f5b3ec5cf2fe75c53467a937"
},
{
"ImportPath": "github.com/dgrijalva/jwt-go",
"Comment": "v3.0.0",
"Rev": "d2709f9f1f31ebcda9651b03077758c1f3a0018c"
},
{
"ImportPath": "github.com/getsentry/raven-go",
"Rev": "379f8d0a68ca237cf8893a1cdfd4f574125e2c51"
}
]
}
This directory tree is generated automatically by godep.
Please do not edit.
See https://github.com/tools/godep for more information.
PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
export GOPATH=$(shell pwd)/_build
BUILD_DIR = $(shell pwd)
export GOPATH=${BUILD_DIR}/_build
export GO15VENDOREXPERIMENT=1
GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
PKG=gitlab.com/gitlab-org/gitlab-workhorse
all: clean-build gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
gitlab-zip-cat: _build $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ${PKG}/cmd/$@
gitlab-zip-cat: ${BUILD_DIR}/_build $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG}/cmd/$@
gitlab-zip-metadata: _build $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o $@ ${PKG}/cmd/$@
gitlab-zip-metadata: ${BUILD_DIR}/_build $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG}/cmd/$@
gitlab-workhorse: _build $(shell find . -name '*.go' | grep -v '^\./_')
${GOBUILD} -o $@ ${PKG}
gitlab-workhorse: ${BUILD_DIR}/_build $(shell find . -name '*.go' | grep -v '^\./_')
${GOBUILD} -o ${BUILD_DIR}/$@ ${PKG}
install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
mkdir -p $(DESTDIR)${PREFIX}/bin/
install gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata ${DESTDIR}${PREFIX}/bin/
cd ${BUILD_DIR} && install gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata ${DESTDIR}${PREFIX}/bin/
_build:
${BUILD_DIR}/_build:
mkdir -p $@/src/${PKG}
tar -cf - --exclude $@ --exclude .git . | (cd $@/src/${PKG} && tar -xf -)
tar -cf - --exclude _build --exclude .git . | (cd $@/src/${PKG} && tar -xf -)
touch $@
.PHONY: test
test: testdata/data/group/test.git clean-build clean-workhorse all
test: clean-build clean-workhorse all
go fmt ${PKG}/... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
support/path go test ${PKG}/...
go test ${PKG}/...
@echo SUCCESS
coverage: testdata/data/group/test.git
coverage:
go test -cover -coverprofile=test.coverage
go tool cover -html=test.coverage -o coverage.html
rm -f test.coverage
......@@ -38,20 +40,14 @@ coverage: testdata/data/group/test.git
fmt:
go fmt ./...
testdata/data/group/test.git: testdata/data
git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@
testdata/data:
mkdir -p $@
.PHONY: clean
clean: clean-workhorse clean-build
rm -rf testdata/data testdata/scratch
.PHONY: clean-workhorse
clean-workhorse:
rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
cd ${BUILD_DIR} && rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
.PHONY: clean-build
clean-build:
rm -rf _build
rm -rf ${BUILD_DIR}/_build
......@@ -4,6 +4,8 @@ Gitlab-workhorse is a smart reverse proxy for GitLab. It handles
"large" HTTP requests such as file downloads, file uploads, Git
push/pull and Git archive downloads.
For more information see ['A brief history of
gitlab-workhorse'][brief-history-blog].
## Usage
......@@ -24,11 +26,13 @@ Options:
-listenNetwork string
Listen 'network' (tcp, tcp4, tcp6, unix) (default "tcp")
-listenUmask int
Umask for Unix socket, default: 022 (default 18)
Umask for Unix socket
-pprofListenAddr string
pprof listening address, e.g. 'localhost:6060'
-proxyHeadersTimeout duration
How long to wait for response headers when proxying the request (default 1m0s)
How long to wait for response headers when proxying the request (default 5m0s)
-secretPath string
File with secret key to authenticate with authBackend (default "./.gitlab_workhorse_secret")
-version
Print version and exit
```
......@@ -54,7 +58,8 @@ gitlab-workhorse -authBackend http://localhost:8080/gitlab
## Installation
To install gitlab-workhorse you need [Go 1.5 or
newer](https://golang.org/dl).
newer](https://golang.org/dl) and [GNU
Make](https://www.gnu.org/software/make/).
To install into `/usr/local/bin` run `make install`.
......@@ -68,6 +73,27 @@ To install into `/foo/bin` set the PREFIX variable.
make install PREFIX=/foo
```
On some operating systems, such as FreeBSD, you may have to use
`gmake` instead of `make`.
## Error tracking
GitLab-Workhorse supports remote error tracking with
[Sentry](https://sentry.io). To enable this feature set the
GITLAB_WORKHORSE_SENTRY_DSN environment variable.
Omnibus (`/etc/gitlab/gitlab.rb`):
```
gitlab_workhorse['env'] = {'GITLAB_WORKHORSE_SENTRY_DSN' => 'https://foobar'}
```
Source installations (`/etc/default/gitlab`):
```
export GITLAB_WORKHORSE_SENTRY_DSN='https://foobar'
```
## Tests
Run the tests with:
......@@ -89,3 +115,5 @@ It is OK if a feature is only covered by integration tests.
## License
This code is distributed under the MIT license, see the LICENSE file.
[brief-history-blog]: https://about.gitlab.com/2016/04/12/a-brief-history-of-gitlab-workhorse/
......@@ -8,8 +8,11 @@ import (
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"github.com/dgrijalva/jwt-go"
)
func okHandler(w http.ResponseWriter, _ *http.Request, _ *api.Response) {
......@@ -17,17 +20,19 @@ func okHandler(w http.ResponseWriter, _ *http.Request, _ *api.Response) {
fmt.Fprint(w, "{\"status\":\"ok\"}")
}
func runPreAuthorizeHandler(t *testing.T, suffix string, url *regexp.Regexp, apiResponse interface{}, returnCode, expectedCode int) *httptest.ResponseRecorder {
// Prepare test server and backend
ts := testAuthServer(url, returnCode, apiResponse)
defer ts.Close()
func runPreAuthorizeHandler(t *testing.T, ts *httptest.Server, suffix string, url *regexp.Regexp, apiResponse interface{}, returnCode, expectedCode int) *httptest.ResponseRecorder {
if ts == nil {
ts = testAuthServer(url, returnCode, apiResponse)
defer ts.Close()
}
// Create http request
httpRequest, err := http.NewRequest("GET", "/address", nil)
if err != nil {
t.Fatal(err)
}
a := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
parsedURL := helper.URLMustParse(ts.URL)
a := api.NewAPI(parsedURL, "123", testhelper.SecretPath(), badgateway.TestRoundTripper(parsedURL))
response := httptest.NewRecorder()
a.PreAuthorizeHandler(okHandler, suffix).ServeHTTP(response, httpRequest)
......@@ -37,7 +42,7 @@ func runPreAuthorizeHandler(t *testing.T, suffix string, url *regexp.Regexp, api
func TestPreAuthorizeHappyPath(t *testing.T) {
runPreAuthorizeHandler(
t, "/authorize",
t, nil, "/authorize",
regexp.MustCompile(`/authorize\z`),
&api.Response{},
200, 201)
......@@ -45,7 +50,7 @@ func TestPreAuthorizeHappyPath(t *testing.T) {
func TestPreAuthorizeSuffix(t *testing.T) {
runPreAuthorizeHandler(
t, "/different-authorize",
t, nil, "/different-authorize",
regexp.MustCompile(`/authorize\z`),
&api.Response{},
200, 404)
......@@ -53,8 +58,68 @@ func TestPreAuthorizeSuffix(t *testing.T) {
func TestPreAuthorizeJsonFailure(t *testing.T) {
runPreAuthorizeHandler(
t, "/authorize",
t, nil, "/authorize",
regexp.MustCompile(`/authorize\z`),
"not-json",
200, 500)
}
func TestPreAuthorizeContentTypeFailure(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if _, err := w.Write([]byte(`{"hello":"world"}`)); err != nil {
t.Fatalf("write auth response: %v", err)
}
}))
defer ts.Close()
runPreAuthorizeHandler(
t, ts, "/authorize",
regexp.MustCompile(`/authorize\z`),
"",
200, 500)
}
func TestPreAuthorizeJWT(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
token, err := jwt.Parse(r.Header.Get(api.RequestHeader), func(token *jwt.Token) (interface{}, error) {
// Don't forget to validate the alg is what you expect:
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
}
secretBytes, err := (&api.Secret{Path: testhelper.SecretPath()}).Bytes()
if err != nil {
return nil, fmt.Errorf("read secret from file: %v", err)
}
return secretBytes, nil
})
if err != nil {
t.Fatalf("decode token: %v", err)
}
claims, ok := token.Claims.(jwt.MapClaims)
if !ok {
t.Fatal("claims cast failed")
}
if !token.Valid {
t.Fatal("JWT token invalid")
}
if claims["iss"] != "gitlab-workhorse" {
t.Fatalf("execpted issuer gitlab-workhorse, got %q", claims["iss"])
}
w.Header().Set("Content-Type", api.ResponseContentType)
if _, err := w.Write([]byte(`{"hello":"world"}`)); err != nil {
t.Fatalf("write auth response: %v", err)
}
}))
defer ts.Close()
runPreAuthorizeHandler(
t, ts, "/authorize",
regexp.MustCompile(`/authorize\z`),
"",
200, 201)
}
package main
import (
"fmt"
"net/url"
)
func parseAuthBackend(authBackend string) (*url.URL, error) {
backendURL, err := url.Parse(authBackend)
if err != nil {
return nil, err
}
if backendURL.Host == "" {
backendURL, err = url.Parse("http://" + authBackend)
if err != nil {
return nil, err
}
}
if backendURL.Scheme != "http" {
return nil, fmt.Errorf("invalid scheme, only 'http' is allowed: %q", authBackend)
}
if backendURL.Host == "" {
return nil, fmt.Errorf("missing host in %q", authBackend)
}
return backendURL, nil
}
package main
import (
"testing"
)
func TestParseAuthBackend(t *testing.T) {
failures := []string{
"",
"ftp://localhost",
"https://example.com",
}
for _, example := range failures {
if _, err := parseAuthBackend(example); err == nil {
t.Errorf("error expected for %q", example)
}
}
successes := []struct{ input, host, scheme string }{
{"http://localhost:8080", "localhost:8080", "http"},
{"localhost:3000", "localhost:3000", "http"},
{"http://localhost", "localhost", "http"},
{"localhost", "localhost", "http"},
}
for _, example := range successes {
result, err := parseAuthBackend(example.input)
if err != nil {
t.Errorf("parse %q: %v", example.input, err)
break
}
if result.Host != example.host {
t.Errorf("example %q: expected %q, got %q", example.input, example.host, result.Host)
}
if result.Scheme != example.scheme {
t.Errorf("example %q: expected %q, got %q", example.input, example.scheme, result.Scheme)
}
}
}
......@@ -11,22 +11,28 @@ import (
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"github.com/dgrijalva/jwt-go"
)
// Custom content type for API responses, to catch routing / programming mistakes
const ResponseContentType = "application/vnd.gitlab-workhorse+json"
const RequestHeader = "Gitlab-Workhorse-Api-Request"
type API struct {
Client *http.Client
URL *url.URL
Version string
Secret *Secret
}
func NewAPI(myURL *url.URL, version string, roundTripper *badgateway.RoundTripper) *API {
if roundTripper == nil {
roundTripper = badgateway.NewRoundTripper("", 0)
}
func NewAPI(myURL *url.URL, version, secretPath string, roundTripper *badgateway.RoundTripper) *API {
return &API{
Client: &http.Client{Transport: roundTripper},
URL: myURL,
Version: version,
Secret: &Secret{Path: secretPath},
}
}
......@@ -122,6 +128,18 @@ func (api *API) newRequest(r *http.Request, body io.Reader, suffix string) (*htt
// configurations (Passenger) to solve auth request routing problems.
authReq.Header.Set("Gitlab-Workhorse", api.Version)
secretBytes, err := api.Secret.Bytes()
if err != nil {
return nil, fmt.Errorf("newRequest: %v", err)
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.StandardClaims{Issuer: "gitlab-workhorse"})
tokenString, err := token.SignedString(secretBytes)
if err != nil {
return nil, fmt.Errorf("newRequest: sign JWT: %v", err)
}
authReq.Header.Set(RequestHeader, tokenString)
return authReq, nil
}
......@@ -141,11 +159,6 @@ func (api *API) PreAuthorizeHandler(h HandleFunc, suffix string) http.Handler {
defer authResponse.Body.Close()
if authResponse.StatusCode != 200 {
// The Git request is not allowed by the backend. Maybe the
// client needs to send HTTP Basic credentials. Forward the
// response from the auth backend to our client. This includes
// the 'WWW-Authenticate' header that acts as a hint that
// Basic auth credentials are needed.
for k, v := range authResponse.Header {
// Accomodate broken clients that do case-sensitive header lookup
if k == "Www-Authenticate" {
......@@ -159,6 +172,11 @@ func (api *API) PreAuthorizeHandler(h HandleFunc, suffix string) http.Handler {
return
}
if contentType := authResponse.Header.Get("Content-Type"); contentType != ResponseContentType {
helper.Fail500(w, fmt.Errorf("preAuthorizeHandler: API responded with wrong content type: %v", contentType))
return
}
a := &Response{}
// The auth backend validated the client request and told us additional
// request metadata. We must extract this information from the auth
......
package api
import (
"fmt"
"net/http"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
// Prevent internal API responses intended for gitlab-workhorse from
// leaking to the end user
func Block(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rw := &blocker{rw: w}
defer rw.Flush()
h.ServeHTTP(rw, r)
})
}
type blocker struct {
rw http.ResponseWriter
hijacked bool
status int
}
func (b *blocker) Header() http.Header {
return b.rw.Header()
}
func (b *blocker) Write(data []byte) (int, error) {
if b.status == 0 {
b.WriteHeader(http.StatusOK)
}
if b.hijacked {
return 0, nil
}
return b.rw.Write(data)
}
func (b *blocker) WriteHeader(status int) {
if b.status != 0 {
return
}
if b.Header().Get("Content-Type") == ResponseContentType {
b.status = 500
b.Header().Del("Content-Length")
b.hijacked = true
helper.Fail500(b.rw, fmt.Errorf("api.blocker: forbidden content-type: %q", ResponseContentType))
return
}
b.status = status
b.rw.WriteHeader(b.status)
}
func (b *blocker) Flush() {
b.WriteHeader(http.StatusOK)
}
package api
import (
"encoding/base64"
"fmt"
"io/ioutil"
"sync"
)
const numSecretBytes = 32
type Secret struct {
Path string
bytes []byte
sync.RWMutex
}
// Lazy access to the HMAC secret key. We must be lazy because if the key
// is not already there, it will be generated by gitlab-rails, and
// gitlab-rails is slow.
func (s *Secret) Bytes() ([]byte, error) {
if bytes := s.getBytes(); bytes != nil {
return bytes, nil
}
return s.setBytes()
}
func (s *Secret) getBytes() []byte {
s.RLock()
defer s.RUnlock()
return s.bytes
}
func (s *Secret) setBytes() ([]byte, error) {
s.Lock()
defer s.Unlock()
if s.bytes != nil {
return s.bytes, nil
}
base64Bytes, err := ioutil.ReadFile(s.Path)
if err != nil {
return nil, fmt.Errorf("read Secret.Path: %v", err)
}
secretBytes := make([]byte, base64.StdEncoding.DecodedLen(len(base64Bytes)))
n, err := base64.StdEncoding.Decode(secretBytes, base64Bytes)
if err != nil {
return nil, fmt.Errorf("decode secret: %v", err)
}
if n != numSecretBytes {
return nil, fmt.Errorf("expected %d secretBytes in %s, found %d", numSecretBytes, s.Path, n)
}
s.bytes = secretBytes
return s.bytes, nil
}
package artifacts
import (
"log"
"os"
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
)
func TestMain(m *testing.M) {
cleanup, err := testhelper.BuildExecutables()
if err != nil {
log.Printf("Test setup: failed to build executables: %v", err)
os.Exit(1)
}
os.Exit(func() int {
defer cleanup()
return m.Run()
}())
}
......@@ -15,6 +15,7 @@ import (
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
......@@ -28,7 +29,7 @@ func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
t.Fatal("Expected POST request")
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Type", api.ResponseContentType)
data, err := json.Marshal(&api.Response{
TempPath: tempPath,
......@@ -90,8 +91,10 @@ func testUploadArtifacts(contentType string, body io.Reader, t *testing.T, ts *h
}
httpRequest.Header.Set("Content-Type", contentType)
response := httptest.NewRecorder()
apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
proxyClient := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
parsedURL := helper.URLMustParse(ts.URL)
roundTripper := badgateway.TestRoundTripper(parsedURL)
apiClient := api.NewAPI(parsedURL, "123", testhelper.SecretPath(), roundTripper)
proxyClient := proxy.NewProxy(parsedURL, "123", roundTripper)
UploadArtifacts(apiClient, proxyClient).ServeHTTP(response, httpRequest)
return response
}
......
......@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
"log"
"mime"
"net/http"
"os"
......@@ -13,11 +14,40 @@ import (
"strings"
"syscall"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"