Commit 94bbbdac authored by Dmitry Smirnov's avatar Dmitry Smirnov

Imported Upstream version 0.7.2

parent fa95f113
......@@ -4,3 +4,4 @@ testdata/scratch
testdata/public
gitlab-zip-cat
gitlab-zip-metadata
_build
......@@ -2,6 +2,29 @@
Formerly known as 'gitlab-git-http-server'.
v0.7.2
Integrate with GOPATH during development (remove relative imports
etc.). Buffer Git HTTP responses so that we may return an error if the
local command fails early.
v0.7.1
Set Content-Length (retrieved from Git) on raw blob data responses.
v0.7.0
Start using a 'v' prefix on the version string.
0.6.5
Inject 'git archive' data the same way as Git blob data.
0.6.4
Increase default ProxyHeadersTimeout to 5 minutes. Fix injecting raw
blobs for /api/v3 requetsts.
0.6.3
Add support for sending Git raw git blobs via gitlab-workhorse.
......
PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
export GOPATH=$(shell pwd)/_build
GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
PKG=gitlab.com/gitlab-org/gitlab-workhorse
all: gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
all: clean-build gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
gitlab-zip-cat: $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-zip-cat: _build $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ${PKG}/cmd/$@
gitlab-zip-metadata: $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-zip-metadata: _build $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o $@ ${PKG}/cmd/$@
gitlab-workhorse: $(shell find . -name '*.go')
${GOBUILD} -o $@
gitlab-workhorse: _build $(shell find . -name '*.go' | grep -v '^\./_')
${GOBUILD} -o $@ ${PKG}
install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
mkdir -p $(DESTDIR)${PREFIX}/bin/
install gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata ${DESTDIR}${PREFIX}/bin/
_build:
mkdir -p $@/src/${PKG}
tar -cf - --exclude $@ --exclude .git . | (cd $@/src/${PKG} && tar -xf -)
touch $@
.PHONY: test
test: testdata/data/group/test.git clean-workhorse all
go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
support/path go test ./...
test: testdata/data/group/test.git clean-build clean-workhorse all
go fmt ${PKG}/... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
support/path go test ${PKG}/...
@echo SUCCESS
coverage: testdata/data/group/test.git
......@@ -28,6 +35,9 @@ coverage: testdata/data/group/test.git
go tool cover -html=test.coverage -o coverage.html
rm -f test.coverage
fmt:
go fmt ./...
testdata/data/group/test.git: testdata/data
git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@
......@@ -35,9 +45,13 @@ testdata/data:
mkdir -p $@
.PHONY: clean
clean: clean-workhorse
clean: clean-workhorse clean-build
rm -rf testdata/data testdata/scratch
.PHONY: clean-workhorse
clean-workhorse:
rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
.PHONY: clean-build
clean-build:
rm -rf _build
......@@ -70,42 +70,21 @@ make install PREFIX=/foo
## Tests
```
make clean test
```
## Try it out
You can try out the Git server without authentication as follows:
Run the tests with:
```
# Start a fake auth backend that allows everything/everybody
make test/data/test.git
go run support/fake-auth-backend.go ~+/test/data/test.git &
# Start gitlab-workhorse
make
./gitlab-workhorse
make clean test
```
Now you can try things like:
### Coverage / what to test
```
git clone http://localhost:8181/test.git
curl -JO http://localhost:8181/test/repository/archive.zip
```
Each feature in gitlab-workhorse should have an integration test that
verifies that the feature 'kicks in' on the right requests and leaves
other requests unaffected. It is better to also have package-level tests
for specific behavior but the high-level integration tests should have
the first priority during development.
## Example request flow
- start POST repo.git/git-receive-pack to NGINX
- ..start POST repo.git/git-receive-pack to gitlab-workhorse
- ....start POST repo.git/git-receive-pack to Unicorn for auth
- ....end POST to Unicorn for auth
- ....start git-receive-pack process from gitlab-workhorse
- ......start POST /api/v3/internal/allowed to Unicorn from Git hook (check protected branches)
- ......end POST to Unicorn from Git hook
- ....end git-receive-pack process
- ..end POST to gitlab-workhorse
- end POST to NGINX
It is OK if a feature is only covered by integration tests.
## License
......
package main
import (
"./internal/api"
"./internal/helper"
"./internal/testhelper"
"fmt"
"net/http"
"net/http/httptest"
"regexp"
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
)
func okHandler(w http.ResponseWriter, _ *http.Request, _ *api.Response) {
......
package main
import (
"../../internal/zipartifacts"
"archive/zip"
"flag"
"fmt"
"io"
"os"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
)
const progName = "gitlab-zip-cat"
......@@ -25,7 +26,7 @@ func main() {
}
if len(os.Args) != 3 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP ENTRY", progName)
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP ENTRY\n", progName)
os.Exit(1)
}
......
package main
import (
"../../internal/zipartifacts"
"flag"
"fmt"
"os"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
)
const progName = "gitlab-zip-metadata"
......
package api
import (
"../badgateway"
"../helper"
"encoding/json"
"fmt"
"io"
......@@ -10,6 +8,9 @@ import (
"net/http"
"net/url"
"strings"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
type API struct {
......@@ -38,15 +39,6 @@ type Response struct {
// RepoPath is the full path on disk to the Git repository the request is
// about
RepoPath string
// ArchivePath is the full path where we should find/create a cached copy
// of a requested archive
ArchivePath string
// ArchivePrefix is used to put extracted archive contents in a
// subdirectory
ArchivePrefix string
// CommitId is used do prevent race conditions between the 'time of check'
// in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
CommitId string
// StoreLFSPath is provided by the GitLab Rails application
// to mark where the tmp file should be placed
StoreLFSPath string
......
package artifacts
import (
"../api"
"../helper"
"../zipartifacts"
"bufio"
"errors"
"fmt"
......@@ -15,6 +12,10 @@ import (
"path/filepath"
"strings"
"syscall"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
)
func detectFileContentType(fileName string) string {
......
package artifacts
import (
"../api"
"../helper"
"../testhelper"
"archive/zip"
"encoding/base64"
"encoding/json"
......@@ -13,6 +10,10 @@ import (
"net/http/httptest"
"os"
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
)
func testArtifactDownloadServer(t *testing.T, archive string, entry string) *httptest.Server {
......
package artifacts
import (
"../api"
"../helper"
"../upload"
"../zipartifacts"
"errors"
"fmt"
"io/ioutil"
......@@ -13,6 +9,11 @@ import (
"os"
"os/exec"
"syscall"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
)
type artifactsUploadProcessor struct {
......
package artifacts
import (
"../api"
"../helper"
"../proxy"
"../testhelper"
"../zipartifacts"
"archive/zip"
"bytes"
"compress/gzip"
......@@ -18,6 +13,12 @@ import (
"net/http/httptest"
"os"
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
)
func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
......
package badgateway
import (
"../helper"
"bytes"
"fmt"
"io/ioutil"
"net"
"net/http"
"time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
// Values from http.DefaultTransport
......
// Package delay exports delay.ResponseWriter. This type implements
// http.ResponseWriter with the ability to delay setting the HTTP
// response code (with WriteHeader()) until writing the first bufferSize
// bytes. This makes it possible, up to a point, to 'change your mind'
// about the HTTP status code. The caller must call
// ResponseWriter.Flush() before returning from the handler (e.g. using
// 'defer').
package delay
import (
"bytes"
"io"
"net/http"
)
const bufferSize = 8192
type ResponseWriter struct {
writer http.ResponseWriter
status int
bufWritten int
cap int
flushed bool
buffer *bytes.Buffer
}
func NewResponseWriter(w http.ResponseWriter) *ResponseWriter {
return &ResponseWriter{
writer: w,
buffer: bytes.NewBuffer(make([]byte, 0, bufferSize)),
cap: bufferSize,
}
}
func (rw *ResponseWriter) Write(buf []byte) (int, error) {
if !rw.flushed && len(buf)+rw.bufWritten <= rw.cap {
n, err := rw.buffer.Write(buf)
rw.bufWritten += n
return n, err
}
if err := rw.Flush(); err != nil {
return 0, err
}
return rw.writer.Write(buf)
}
func (rw *ResponseWriter) Header() http.Header {
return rw.writer.Header()
}
func (rw *ResponseWriter) WriteHeader(code int) {
if rw.status != 0 {
return
}
rw.status = code
}
func (rw *ResponseWriter) Flush() error {
if rw.flushed {
return nil
}
rw.flushed = true
if rw.status == 0 {
rw.writer.WriteHeader(http.StatusOK)
} else {
rw.writer.WriteHeader(rw.status)
}
_, err := io.Copy(rw.writer, rw.buffer)
rw.buffer = nil // "Release" the buffer for GC
return err
}
package delay
import (
"fmt"
"net/http/httptest"
"strings"
"testing"
)
func TestSanity(t *testing.T) {
first, second := 200, 500
w := httptest.NewRecorder()
w.WriteHeader(first)
w.WriteHeader(second)
if code := w.Code; code != first {
t.Fatalf("Expected HTTP code %d, got %d", first, code)
}
}
func TestSmallResponse(t *testing.T) {
code := 500
body := "hello"
w := httptest.NewRecorder()
rw := NewResponseWriter(w)
fmt.Fprint(rw, body)
rw.WriteHeader(code)
rw.Flush()
if actualCode := w.Code; actualCode != code {
t.Fatalf("Expected code %d, got %d", code, actualCode)
}
if actualBody := w.Body.String(); actualBody != body {
t.Fatalf("Expected body %q, got %q", body, actualBody)
}
}
func TestLargeResponse(t *testing.T) {
code := 200
body := strings.Repeat("0123456789", bufferSize/5) // must exceed the buffer size
w := httptest.NewRecorder()
rw := NewResponseWriter(w)
fmt.Fprint(rw, body)
// Because the 'body' was too long this 500 should be ignored
rw.WriteHeader(500)
rw.Flush()
if actualCode := w.Code; actualCode != code {
t.Fatalf("Expected code %d, got %d", code, actualCode)
}
if actualBody := w.Body.String(); actualBody != body {
t.Fatalf("Expected body %q, got %q", body, actualBody)
}
}
......@@ -5,8 +5,6 @@ In this file we handle 'git archive' downloads
package git
import (
"../api"
"../helper"
"fmt"
"io"
"io/ioutil"
......@@ -18,13 +16,28 @@ import (
"path/filepath"
"syscall"
"time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
)
func GetArchive(a *api.API) http.Handler {
return repoPreAuthorizeHandler(a, handleGetArchive)
type archive struct{ senddata.Prefix }
type archiveParams struct {
RepoPath string
ArchivePath string
ArchivePrefix string
CommitId string
}
func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
var SendArchive = &archive{"git-archive:"}
func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
var params archiveParams
if err := a.Unpack(&params, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendArchive: unpack sendData: %v", err))
return
}
var format string
urlPath := r.URL.Path
switch filepath.Base(urlPath) {
......@@ -41,11 +54,11 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
return
}
archiveFilename := path.Base(a.ArchivePath)
archiveFilename := path.Base(params.ArchivePath)
if cachedArchive, err := os.Open(a.ArchivePath); err == nil {
if cachedArchive, err := os.Open(params.ArchivePath); err == nil {
defer cachedArchive.Close()
log.Printf("Serving cached file %q", a.ArchivePath)
log.Printf("Serving cached file %q", params.ArchivePath)
setArchiveHeaders(w, format, archiveFilename)
// Even if somebody deleted the cachedArchive from disk since we opened
// the file, Unix file semantics guarantee we can still read from the
......@@ -58,7 +71,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
// safe. We create the tempfile in the same directory as the final cached
// archive we want to create so that we can use an atomic link(2) operation
// to finalize the cached archive.
tempFile, err := prepareArchiveTempfile(path.Dir(a.ArchivePath), archiveFilename)
tempFile, err := prepareArchiveTempfile(path.Dir(params.ArchivePath), archiveFilename)
if err != nil {
helper.Fail500(w, fmt.Errorf("handleGetArchive: create tempfile: %v", err))
return
......@@ -68,7 +81,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
compressCmd, archiveFormat := parseArchiveFormat(format)
archiveCmd := gitCommand("", "git", "--git-dir="+a.RepoPath, "archive", "--format="+archiveFormat, "--prefix="+a.ArchivePrefix+"/", a.CommitId)
archiveCmd := gitCommand("", "git", "--git-dir="+params.RepoPath, "archive", "--format="+archiveFormat, "--prefix="+params.ArchivePrefix+"/", params.CommitId)
archiveStdout, err := archiveCmd.StdoutPipe()
if err != nil {
helper.Fail500(w, fmt.Errorf("handleGetArchive: archive stdout: %v", err))
......@@ -125,13 +138,14 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
}
}
if err := finalizeCachedArchive(tempFile, a.ArchivePath); err != nil {
if err := finalizeCachedArchive(tempFile, params.ArchivePath); err != nil {
helper.LogError(fmt.Errorf("handleGetArchive: finalize cached archive: %v", err))
return
}
}
func setArchiveHeaders(w http.ResponseWriter, format string, archiveFilename string) {
w.Header().Del("Content-Length")
w.Header().Add("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, archiveFilename))
if format == "zip" {
w.Header().Add("Content-Type", "application/zip")
......
package git
import (
"../helper"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"strings"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
)
type blobParams struct {
RepoPath string
BlobId string
}
type blob struct{ senddata.Prefix }
type blobParams struct{ RepoPath, BlobId string }
const SendBlobPrefix = "git-blob:"
var SendBlob = &blob{"git-blob:"}
func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
params, err := unpackSendData(sendData)
if err != nil {
func (b *blob) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
var params blobParams
if err := b.Unpack(&params, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: unpack sendData: %v", err))
return
}
log.Printf("SendBlob: sending %q for %q", params.BlobId, r.URL.Path)
sizeOutput, err := gitCommand("", "git", "--git-dir="+params.RepoPath, "cat-file", "-s", params.BlobId).Output()
if err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: get blob size: %v", err))
return
}
gitShowCmd := gitCommand("", "git", "--git-dir="+params.RepoPath, "cat-file", "blob", params.BlobId)
stdout, err := gitShowCmd.StdoutPipe()
if err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: git stdout: %v", err))
helper.Fail500(w, fmt.Errorf("SendBlob: git cat-file stdout: %v", err))
return
}
if err := gitShowCmd.Start(); err != nil {
......@@ -38,6 +43,7 @@ func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
}
defer helper.CleanUpProcessGroup(gitShowCmd)
w.Header().Set("Content-Length", strings.TrimSpace(string(sizeOutput)))
if _, err := io.Copy(w, stdout); err != nil {
helper.LogError(fmt.Errorf("SendBlob: copy git cat-file stdout: %v", err))
return
......@@ -47,15 +53,3 @@ func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
return
}
}
func unpackSendData(sendData string) (*blobParams, error) {
jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, SendBlobPrefix))
if err != nil {
return nil, err
}
result := &blobParams{}
if err := json.Unmarshal([]byte(jsonBytes), result); err != nil {
return nil, err
}
return result, nil
}
......@@ -5,8 +5,6 @@ In this file we handle the Git 'smart HTTP' protocol
package git
import (
"../api"
"../helper"
"errors"
"fmt"
"io"
......@@ -16,6 +14,10 @@ import (
"path"
"path/filepath"
"strings"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/delay"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
func GetInfoRefs(a *api.API) http.Handler {
......@@ -52,7 +54,10 @@ func repoPreAuthorizeHandler(myAPI *api.API, handleFunc api.HandleFunc) http.Han
}, "")
}
func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response) {
func handleGetInfoRefs(_w http.ResponseWriter, r *http.Request, a *api.Response) {
w := delay.NewResponseWriter(_w)
defer w.Flush()
rpc := r.URL.Query().Get("service")
if !(rpc == "git-upload-pack" || rpc == "git-receive-pack") {
// The 'dumb' Git HTTP protocol is not supported
......@@ -77,26 +82,28 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
// Start writing the response
w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
w.Header().Add("Cache-Control", "no-cache")
w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
if err := pktLine(w, fmt.Sprintf("# service=%s\n", rpc)); err != nil {
helper.LogError(fmt.Errorf("handleGetInfoRefs: pktLine: %v", err))
helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: pktLine: %v", err))
return
}
if err := pktFlush(w); err != nil {
helper.LogError(fmt.Errorf("handleGetInfoRefs: pktFlush: %v", err))
helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: pktFlush: %v", err))
return
}
if _, err := io.Copy(w, stdout); err != nil {
helper.LogError(fmt.Errorf("handleGetInfoRefs: copy output of %v: %v", cmd.Args, err))
helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: copy output of %v: %v", cmd.Args, err))
return
}
if err := cmd.Wait(); err != nil {
helper.LogError(fmt.Errorf("handleGetInfoRefs: wait for %v: %v", cmd.Args, err))
helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: wait for %v: %v", cmd.Args, err))
return
}
}
func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
func handlePostRPC(_w http.ResponseWriter, r *http.Request, a *api.Response) {
w := delay.NewResponseWriter(_w)
defer w.Flush()
var err error
// Get Git action from URL
......@@ -142,15 +149,14 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
// Start writing the response
w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-result", action))
w.Header().Add("Cache-Control", "no-cache")
w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
// This io.Copy may take a long time, both for Git push and pull.
if _, err := io.Copy(w, stdout); err != nil {
helper.LogError(fmt.Errorf("handlePostRPC copy output of %v: %v", cmd.Args, err))
helper.Fail500(w, fmt.Errorf("handlePostRPC copy output of %v: %v", cmd.Args, err))
return
}
if err := cmd.Wait(); err != nil {
helper.LogError(fmt.Errorf("handlePostRPC wait for %v: %v", cmd.Args, err))
helper.Fail500(w, fmt.Errorf("handlePostRPC wait for %v: %v", cmd.Args, err))
return
}
}
......
......@@ -5,9 +5,6 @@ In this file we handle git lfs objects downloads and uploads