Imported Upstream version 0.6.3

parent 92ce05b9
test/data
test/scratch
gitlab-workhorse
testdata/data
testdata/scratch
testdata/public
gitlab-zip-cat
gitlab-zip-metadata
before_script:
- rm -rf /usr/local/go
- apt-get update -qq
- apt-get install -y curl unzip bzip2
- curl -O https://storage.googleapis.com/golang/go1.5.2.linux-amd64.tar.gz
- echo 'cae87ed095e8d94a81871281d35da7829bd1234e go1.5.2.linux-amd64.tar.gz' | shasum -c -
- tar -C /usr/local -xzf go1.5.2.linux-amd64.tar.gz
- apt-get update -qq && apt-get install -y curl unzip bzip2
- curl -O https://storage.googleapis.com/golang/go1.5.3.linux-amd64.tar.gz
- echo '43afe0c5017e502630b1aea4d44b8a7f059bf60d7f29dfd58db454d4e4e0ae53 go1.5.3.linux-amd64.tar.gz' | shasum -c -a256 -
- rm -rf /usr/local/go && tar -C /usr/local -xzf go1.5.3.linux-amd64.tar.gz
- export PATH=/usr/local/go/bin:$PATH
test:
......
......@@ -2,6 +2,49 @@
Formerly known as 'gitlab-git-http-server'.
0.6.3
Add support for sending Git raw git blobs via gitlab-workhorse.
0.6.2
We now fill in missing directory entries in archize zip metadata
files; also some other minor changes.
0.6.1
Add support for generating zip artifacts metadata and serving single
files from zip archives.
Gitlab-workhorse now consists of multiple executables. We also fixed a
routing bug introduced by the 0.6.0 refactor that broke relative URL
support.
0.6.0
Overhauled the source code organization; no user-facing changes
(intended). The application code is now split into Go 'packages'
(modules). As of 0.6.0 gitlab-workhorse requires Go 1.5 or newer.
0.5.4
Fix /api/v3/projects routing bug introduced in 0.5.2-0.5.3.
0.5.3
Fixes merge error in 0.5.2.
0.5.2 (broken!)
- Always check with upstream if files in /uploads/ may be served
- Fix project%2Fnamespace API project ID's
- Prevent archive zombies when using gzip or bzip2
- Don't show pretty error pages in development mode
0.5.1
Deprecate -relativeURLRoot option, use -authBackend instead.
0.5.0
Send ALL GitLab requests through gitlab-workhorse.
......@@ -46,4 +89,4 @@ This makes the REPO_ROOT command line argument obsolete.
0.2.14
This is the last version that works with GitLab 8.0.
\ No newline at end of file
This is the last version that works with GitLab 8.0.
PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
gitlab-workhorse: $(wildcard *.go)
go build -ldflags "-X main.Version ${VERSION}" -o gitlab-workhorse
all: gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
install: gitlab-workhorse
install gitlab-workhorse ${PREFIX}/bin/
gitlab-zip-cat: $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-zip-metadata: $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-workhorse: $(shell find . -name '*.go')
${GOBUILD} -o $@
install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
mkdir -p $(DESTDIR)${PREFIX}/bin/
install gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata ${DESTDIR}${PREFIX}/bin/
.PHONY: test
test: test/data/group/test.git clean-workhorse gitlab-workhorse
go fmt | awk '{ print "Please run go fmt"; exit 1 }'
go test
test: testdata/data/group/test.git clean-workhorse all
go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
support/path go test ./...
@echo SUCCESS
coverage: test/data/group/test.git
coverage: testdata/data/group/test.git
go test -cover -coverprofile=test.coverage
go tool cover -html=test.coverage -o coverage.html
rm -f test.coverage
test/data/group/test.git: test/data
git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git test/data/group/test.git
testdata/data/group/test.git: testdata/data
git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@
test/data:
mkdir -p test/data
testdata/data:
mkdir -p $@
.PHONY: clean
clean: clean-workhorse
rm -rf test/data test/scratch
rm -rf testdata/data testdata/scratch
.PHONY: clean-workhorse
clean-workhorse:
rm -f gitlab-workhorse
rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
# gitlab-workhorse
gitlab-workhorse was designed to unload Git HTTP traffic from
the GitLab Rails app (Unicorn) to a separate daemon. It also serves
'git archive' downloads for GitLab. All authentication and
authorization logic is still handled by the GitLab Rails app.
Gitlab-workhorse is a smart reverse proxy for GitLab. It handles
"large" HTTP requests such as file downloads, file uploads, Git
push/pull and Git archive downloads.
Architecture: Git client -> NGINX -> gitlab-workhorse (makes
auth request to GitLab Rails app) -> git-upload-pack
## Usage
......@@ -18,6 +15,10 @@ Options:
Authentication/authorization backend (default "http://localhost:8080")
-authSocket string
Optional: Unix domain socket to dial authBackend at
-developmentMode
Allow to serve assets from Rails app
-documentRoot string
Path to static files content (default "public")
-listenAddr string
Listen address for HTTP server (default "localhost:8181")
-listenNetwork string
......@@ -26,24 +27,35 @@ Options:
Umask for Unix socket, default: 022 (default 18)
-pprofListenAddr string
pprof listening address, e.g. 'localhost:6060'
-proxyHeadersTimeout duration
How long to wait for response headers when proxying the request (default 1m0s)
-version
Print version and exit
```
gitlab-workhorse allows Git HTTP clients to push and pull to
and from Git repositories. Each incoming request is first replayed
(with an empty request body) to an external authentication/authorization
HTTP server: the 'auth backend'. The auth backend is expected to
be a GitLab Unicorn process. The 'auth response' is a JSON message
which tells gitlab-workhorse the path of the Git repository
to read from/write to.
The 'auth backend' refers to the GitLab Rails application. The name is
a holdover from when gitlab-workhorse only handled Git push/pull over
HTTP.
gitlab-workhorse can listen on either a TCP or a Unix domain socket. It
Gitlab-workhorse can listen on either a TCP or a Unix domain socket. It
can also open a second listening TCP listening socket with the Go
[net/http/pprof profiler server](http://golang.org/pkg/net/http/pprof/).
### Relative URL support
If you are mounting GitLab at a relative URL, e.g.
`example.com/gitlab`, then you should also use this relative URL in
the `authBackend` setting:
```
gitlab-workhorse -authBackend http://localhost:8080/gitlab
```
## Installation
To install gitlab-workhorse you need [Go 1.5 or
newer](https://golang.org/dl).
To install into `/usr/local/bin` run `make install`.
```
......
package main
func artifactsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
return preAuthorizeHandler(handleFunc, "/authorize")
}
package main
import (
"./internal/api"
"./internal/helper"
"./internal/testhelper"
"fmt"
"net/http"
"net/http/httptest"
......@@ -8,14 +11,14 @@ import (
"testing"
)
func okHandler(w http.ResponseWriter, r *gitRequest) {
func okHandler(w http.ResponseWriter, _ *http.Request, _ *api.Response) {
w.WriteHeader(201)
fmt.Fprint(w, "{\"status\":\"ok\"}")
}
func runPreAuthorizeHandler(t *testing.T, suffix string, url *regexp.Regexp, authorizationResponse interface{}, returnCode, expectedCode int) *httptest.ResponseRecorder {
func runPreAuthorizeHandler(t *testing.T, suffix string, url *regexp.Regexp, apiResponse interface{}, returnCode, expectedCode int) *httptest.ResponseRecorder {
// Prepare test server and backend
ts := testAuthServer(url, returnCode, authorizationResponse)
ts := testAuthServer(url, returnCode, apiResponse)
defer ts.Close()
// Create http request
......@@ -23,15 +26,11 @@ func runPreAuthorizeHandler(t *testing.T, suffix string, url *regexp.Regexp, aut
if err != nil {
t.Fatal(err)
}
request := gitRequest{
Request: httpRequest,
u: newUpstream(ts.URL, nil),
}
a := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
response := httptest.NewRecorder()
preAuthorizeHandler(okHandler, suffix)(response, &request)
assertResponseCode(t, response, expectedCode)
a.PreAuthorizeHandler(okHandler, suffix).ServeHTTP(response, httpRequest)
testhelper.AssertResponseCode(t, response, expectedCode)
return response
}
......@@ -39,7 +38,7 @@ func TestPreAuthorizeHappyPath(t *testing.T) {
runPreAuthorizeHandler(
t, "/authorize",
regexp.MustCompile(`/authorize\z`),
&authorizationResponse{},
&api.Response{},
200, 201)
}
......@@ -47,7 +46,7 @@ func TestPreAuthorizeSuffix(t *testing.T) {
runPreAuthorizeHandler(
t, "/different-authorize",
regexp.MustCompile(`/authorize\z`),
&authorizationResponse{},
&api.Response{},
200, 404)
}
......
package main
import (
"../../internal/zipartifacts"
"archive/zip"
"flag"
"fmt"
"io"
"os"
)
const progName = "gitlab-zip-cat"
var Version = "unknown"
var printVersion = flag.Bool("version", false, "Print version and exit")
func main() {
flag.Parse()
version := fmt.Sprintf("%s %s", progName, Version)
if *printVersion {
fmt.Println(version)
os.Exit(0)
}
if len(os.Args) != 3 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP ENTRY", progName)
os.Exit(1)
}
archiveFileName := os.Args[1]
fileName, err := zipartifacts.DecodeFileEntry(os.Args[2])
if err != nil {
fatalError(fmt.Errorf("decode entry %q: %v", os.Args[2], err))
}
archive, err := zip.OpenReader(archiveFileName)
if err != nil {
notFoundError(fmt.Errorf("open %q: %v", archiveFileName, err))
}
defer archive.Close()
file := findFileInZip(fileName, &archive.Reader)
if file == nil {
notFoundError(fmt.Errorf("find %q in %q: not found", fileName, archiveFileName))
}
// Start decompressing the file
reader, err := file.Open()
if err != nil {
fatalError(fmt.Errorf("open %q in %q: %v", fileName, archiveFileName, err))
}
defer reader.Close()
if _, err := fmt.Printf("%d\n", file.UncompressedSize64); err != nil {
fatalError(fmt.Errorf("write file size: %v", err))
}
if _, err := io.Copy(os.Stdout, reader); err != nil {
fatalError(fmt.Errorf("write %q from %q to stdout: %v", fileName, archiveFileName, err))
}
}
func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
for _, file := range archive.File {
if file.Name == fileName {
return file
}
}
return nil
}
func printError(err error) {
fmt.Fprintf(os.Stderr, "%s: %v", progName, err)
}
func fatalError(err error) {
printError(err)
os.Exit(1)
}
func notFoundError(err error) {
printError(err)
os.Exit(zipartifacts.StatusEntryNotFound)
}
package main
import (
"../../internal/zipartifacts"
"flag"
"fmt"
"os"
)
const progName = "gitlab-zip-metadata"
var Version = "unknown"
var printVersion = flag.Bool("version", false, "Print version and exit")
func main() {
flag.Parse()
version := fmt.Sprintf("%s %s", progName, Version)
if *printVersion {
fmt.Println(version)
os.Exit(0)
}
if len(os.Args) != 2 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP\n", progName)
os.Exit(1)
}
if err := zipartifacts.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", progName, err)
if err == os.ErrInvalid {
os.Exit(zipartifacts.StatusNotZip)
}
os.Exit(1)
}
}
package main
import "net/http"
func handleDevelopmentMode(developmentMode *bool, handler serviceHandleFunc) serviceHandleFunc {
return func(w http.ResponseWriter, r *gitRequest) {
if !*developmentMode {
http.NotFound(w, r.Request)
return
}
handler(w, r)
}
}
package main
package api
import (
"../badgateway"
"../helper"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
func (u *upstream) newUpstreamRequest(r *http.Request, body io.Reader, suffix string) (*http.Request, error) {
url := u.authBackend + r.URL.RequestURI() + suffix
authReq, err := http.NewRequest(r.Method, url, body)
if err != nil {
return nil, err
type API struct {
Client *http.Client
URL *url.URL
Version string
}
func NewAPI(myURL *url.URL, version string, roundTripper *badgateway.RoundTripper) *API {
if roundTripper == nil {
roundTripper = badgateway.NewRoundTripper("", 0)
}
// Forward all headers from our client to the auth backend. This includes
// HTTP Basic authentication credentials (the 'Authorization' header).
for k, v := range r.Header {
authReq.Header[k] = v
return &API{
Client: &http.Client{Transport: roundTripper},
URL: myURL,
Version: version,
}
}
type HandleFunc func(http.ResponseWriter, *http.Request, *Response)
type Response struct {
// GL_ID is an environment variable used by gitlab-shell hooks during 'git
// push' and 'git pull'
GL_ID string
// RepoPath is the full path on disk to the Git repository the request is
// about
RepoPath string
// ArchivePath is the full path where we should find/create a cached copy
// of a requested archive
ArchivePath string
// ArchivePrefix is used to put extracted archive contents in a
// subdirectory
ArchivePrefix string
// CommitId is used do prevent race conditions between the 'time of check'
// in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
CommitId string
// StoreLFSPath is provided by the GitLab Rails application
// to mark where the tmp file should be placed
StoreLFSPath string
// LFS object id
LfsOid string
// LFS object size
LfsSize int64
// TmpPath is the path where we should store temporary files
// This is set by authorization middleware
TempPath string
// Archive is the path where the artifacts archive is stored
Archive string `json:"archive"`
// Entry is a filename inside the archive point to file that needs to be extracted
Entry string `json:"entry"`
}
// singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
// rebaseUrl is taken from reverseproxy.go:NewSingleHostReverseProxy
func rebaseUrl(url *url.URL, onto *url.URL, suffix string) *url.URL {
newUrl := *url
newUrl.Scheme = onto.Scheme
newUrl.Host = onto.Host
if suffix != "" {
newUrl.Path = singleJoiningSlash(url.Path, suffix)
}
if onto.RawQuery == "" || newUrl.RawQuery == "" {
newUrl.RawQuery = onto.RawQuery + newUrl.RawQuery
} else {
newUrl.RawQuery = onto.RawQuery + "&" + newUrl.RawQuery
}
return &newUrl
}
func (api *API) newRequest(r *http.Request, body io.Reader, suffix string) (*http.Request, error) {
authReq := &http.Request{
Method: r.Method,
URL: rebaseUrl(r.URL, api.URL, suffix),
Header: helper.HeaderClone(r.Header),
}
if body != nil {
authReq.Body = ioutil.NopCloser(body)
}
// Clean some headers when issuing a new request without body
......@@ -46,22 +128,22 @@ func (u *upstream) newUpstreamRequest(r *http.Request, body io.Reader, suffix st
authReq.Host = r.Host
// Set a custom header for the request. This can be used in some
// configurations (Passenger) to solve auth request routing problems.
authReq.Header.Set("Gitlab-Workhorse", Version)
authReq.Header.Set("Gitlab-Workhorse", api.Version)
return authReq, nil
}
func preAuthorizeHandler(handleFunc serviceHandleFunc, suffix string) serviceHandleFunc {
return func(w http.ResponseWriter, r *gitRequest) {
authReq, err := r.u.newUpstreamRequest(r.Request, nil, suffix)
func (api *API) PreAuthorizeHandler(h HandleFunc, suffix string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
authReq, err := api.newRequest(r, nil, suffix)
if err != nil {
fail500(w, fmt.Errorf("preAuthorizeHandler: newUpstreamRequest: %v", err))
helper.Fail500(w, fmt.Errorf("preAuthorizeHandler: newUpstreamRequest: %v", err))
return
}
authResponse, err := r.u.httpClient.Do(authReq)
authResponse, err := api.Client.Do(authReq)
if err != nil {
fail500(w, fmt.Errorf("preAuthorizeHandler: do %v: %v", authReq.URL.Path, err))
helper.Fail500(w, fmt.Errorf("preAuthorizeHandler: do %v: %v", authReq.URL.Path, err))
return
}
defer authResponse.Body.Close()
......@@ -85,11 +167,12 @@ func preAuthorizeHandler(handleFunc serviceHandleFunc, suffix string) serviceHan
return
}
a := &Response{}
// The auth backend validated the client request and told us additional
// request metadata. We must extract this information from the auth
// response body.
if err := json.NewDecoder(authResponse.Body).Decode(&r.authorizationResponse); err != nil {
fail500(w, fmt.Errorf("preAuthorizeHandler: decode authorization response: %v", err))
if err := json.NewDecoder(authResponse.Body).Decode(a); err != nil {
helper.Fail500(w, fmt.Errorf("preAuthorizeHandler: decode authorization response: %v", err))
return
}
// Don't hog a TCP connection in CLOSE_WAIT, we can already close it now
......@@ -104,6 +187,6 @@ func preAuthorizeHandler(handleFunc serviceHandleFunc, suffix string) serviceHan
}
}
handleFunc(w, r)
}
h(w, r, a)
})
}
package artifacts
import (
"../api"
"../helper"
"../zipartifacts"
"bufio"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
)
func detectFileContentType(fileName string) string {
contentType := mime.TypeByExtension(filepath.Ext(fileName))
if contentType == "" {
contentType = "application/octet-stream"
}
return contentType
}
func unpackFileFromZip(archiveFileName, encodedFilename string, headers http.Header, output io.Writer) error {
fileName, err := zipartifacts.DecodeFileEntry(encodedFilename)
if err != nil {
return err
}
catFile := exec.Command("gitlab-zip-cat", archiveFileName, encodedFilename)
catFile.Stderr = os.Stderr
catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := catFile.StdoutPipe()
if err != nil {
return fmt.Errorf("create gitlab-zip-cat stdout pipe: %v", err)
}
if err := catFile.Start(); err != nil {
return fmt.Errorf("start %v: %v", catFile.Args, err)
}
defer helper.CleanUpProcessGroup(catFile)
basename := filepath.Base(fileName)
reader := bufio.NewReader(stdout)
contentLength, err := reader.ReadString('\n')
if err != nil {
if catFileErr := waitCatFile(catFile); catFileErr != nil {
return catFileErr
}
return fmt.Errorf("read content-length: %v", err)
}
contentLength = strings.TrimSuffix(contentLength, "\n")
// Write http headers about the file
headers.Set("Content-Length", contentLength)
headers.Set("Content-Type", detectFileContentType(fileName))
headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
// Copy file body to client
if _, err := io.Copy(output, reader); err != nil {
return fmt.Errorf("copy stdout of %v: %v", catFile.Args, err)
}
return waitCatFile(catFile)
}
func waitCatFile(cmd *exec.Cmd) error {
err := cmd.Wait()
if err == nil {
return nil
}
if st, ok := helper.ExitStatus(err); ok && st == zipartifacts.StatusEntryNotFound {
return os.ErrNotExist
}
return fmt.Errorf("wait for %v to finish: %v", cmd.Args, err)
}
// Artifacts downloader doesn't support ranges when downloading a single file
func DownloadArtifact(myAPI *api.API) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.Archive == "" || a.Entry == "" {
helper.Fail500(w, errors.New("DownloadArtifact: Archive or Path is empty"))
return
}
err := unpackFileFromZip(a.Archive, a.Entry, w.Header(), w)
if os.IsNotExist(err) {
http.NotFound(w, r)
return
} else if err != nil {
helper.Fail500(w, fmt.Errorf("DownloadArtifact: %v", err))
}
}, "")
}
package artifacts
import (
"../api"
"../helper"
"../testhelper"
"archive/zip"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func testArtifactDownloadServer(t *testing.T, archive string, entry string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Fatal("Expected GET request")
}
w.Header().Set("Content-Type", "application/json")
data, err := json.Marshal(&api.Response{
Archive: archive,
Entry: base64.StdEncoding.EncodeToString([]byte(entry)),
})
if err != nil {
t.Fatal(err)