Commit 8e2188be authored by Praveen Arimbrathodiyil's avatar Praveen Arimbrathodiyil

Update upstream source from tag 'upstream/7.6.0+debian'

Update to upstream version '7.6.0+debian'
with Debian dir 8547cc97034923ee7df2987ec851d39fb970eec0
parents 02f6cf01 e4cc1382
......@@ -2,7 +2,16 @@
Formerly known as 'gitlab-git-http-server'.
UNRELEASED
v 7.6.0
- Rename correlation-id structured logging field to correlation_id !343
- Remove local git receive-pack implementation !326
- Remove curl from sendfile_test.go !344
- Update README.md usage example !342
v 7.5.0
- Add proxy layer to calculate content type and disposition headers !335
v 7.4.0
......
......@@ -37,37 +37,43 @@ gitlab-workhorse'][brief-history-blog].
Options:
-apiCiLongPollingDuration duration
Long polling duration for job requesting for runners (default 0s - disabled)
Long polling duration for job requesting for runners (default 50s - enabled) (default 50ns)
-apiLimit uint
Number of API requests allowed at single time
Number of API requests allowed at single time
-apiQueueDuration duration
Maximum queueing duration of requests (default 30s)
Maximum queueing duration of requests (default 30s)
-apiQueueLimit uint
Number of API requests allowed to be queued
Number of API requests allowed to be queued
-authBackend string
Authentication/authorization backend (default "http://localhost:8080")
Authentication/authorization backend (default "http://localhost:8080")
-authSocket string
Optional: Unix domain socket to dial authBackend at
Optional: Unix domain socket to dial authBackend at
-config string
TOML file to load config from
-developmentMode
Allow the assets to be served from Rails app
Allow the assets to be served from Rails app
-documentRoot string
Path to static files content (default "public")
Path to static files content (default "public")
-listenAddr string
Listen address for HTTP server (default "localhost:8181")
Listen address for HTTP server (default "localhost:8181")
-listenNetwork string
Listen 'network' (tcp, tcp4, tcp6, unix) (default "tcp")
Listen 'network' (tcp, tcp4, tcp6, unix) (default "tcp")
-listenUmask int
Umask for Unix socket
Umask for Unix socket
-logFile string
Log file location
-logFormat string
Log format to use defaults to text (text, json, structured, none) (default "text")
-pprofListenAddr string
pprof listening address, e.g. 'localhost:6060'
pprof listening address, e.g. 'localhost:6060'
-prometheusListenAddr string
Prometheus listening address, e.g. 'localhost:9229'
-proxyHeadersTimeout duration
How long to wait for response headers when proxying the request (default 5m0s)
How long to wait for response headers when proxying the request (default 5m0s)
-secretPath string
File with secret key to authenticate with authBackend (default "./.gitlab_workhorse_secret")
-config string
File that hold configuration. Currently only for redis. File is in TOML-format (default "")
File with secret key to authenticate with authBackend (default "./.gitlab_workhorse_secret")
-version
Print version and exit
Print version and exit
```
The 'auth backend' refers to the GitLab Rails application. The name is
......@@ -80,8 +86,8 @@ can also open a second listening TCP listening socket with the Go
Gitlab-workhorse can listen on redis events (currently only builds/register
for runners). This requires you to pass a valid TOML config file via
`-config` flag.
For regular setups it only requires the following (replacing the string
`-config` flag.
For regular setups it only requires the following (replacing the string
with the actual socket)
### Redis
......
......@@ -8,9 +8,7 @@ import (
"fmt"
"io"
"net/http"
"os/exec"
"path/filepath"
"strings"
"sync"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
......@@ -67,22 +65,6 @@ func repoPreAuthorizeHandler(myAPI *api.API, handleFunc api.HandleFunc) http.Han
}, "")
}
func startGitCommand(a *api.Response, stdin io.Reader, stdout io.Writer, action string, options ...string) (cmd *exec.Cmd, err error) {
// Prepare our Git subprocess
args := []string{subCommand(action), "--stateless-rpc"}
args = append(args, options...)
args = append(args, a.RepoPath)
cmd = gitCommandApi(a, "git", args...)
cmd.Stdin = stdin
cmd.Stdout = stdout
if err = cmd.Start(); err != nil {
return nil, fmt.Errorf("start %v: %v", cmd.Args, err)
}
return cmd, nil
}
func writePostRPCHeader(w http.ResponseWriter, action string) {
w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-result", action))
w.Header().Set("Cache-Control", "no-cache")
......@@ -95,10 +77,6 @@ func getService(r *http.Request) string {
return filepath.Base(r.URL.Path)
}
func subCommand(rpc string) string {
return strings.TrimPrefix(rpc, "git-")
}
type countReadCloser struct {
n int64
io.ReadCloser
......
package git
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
)
const (
expectedBytes = 102400
GL_ID = "test-user"
)
// From https://npf.io/2015/06/testing-exec-command/
func fakeExecCommand(command string, args ...string) *exec.Cmd {
cs := []string{"-test.run=TestGitCommandProcess", "--", command}
cs = append(cs, args...)
cmd := exec.Command(os.Args[0], cs...)
return cmd
}
func createTestPayload() []byte {
return bytes.Repeat([]byte{'0'}, expectedBytes)
}
func TestHandleReceivePack(t *testing.T) {
testHandlePostRpc(t, "git-receive-pack", handleReceivePack)
}
func testHandlePostRpc(t *testing.T, action string, handler func(*HttpResponseWriter, *http.Request, *api.Response) error) {
execCommand = fakeExecCommand
defer func() { execCommand = exec.Command }()
testInput := createTestPayload()
body := bytes.NewReader([]byte(testInput))
url := fmt.Sprintf("/gitlab/gitlab-ce.git/?service=%s", action)
req, err := http.NewRequest("GET", url, body)
if err != nil {
t.Fatal(err)
}
resp := &api.Response{GL_ID: GL_ID}
rr := httptest.NewRecorder()
handler(NewHttpResponseWriter(rr), req, resp)
// Check HTTP status code
if status := rr.Code; status != http.StatusOK {
t.Errorf("handler returned wrong status code: expected: %v, got %v",
http.StatusOK, status)
}
ct := fmt.Sprintf("application/x-%s-result", action)
headers := []struct {
key string
value string
}{
{"Content-Type", ct},
{"Cache-Control", "no-cache"},
}
// Check HTTP headers
for _, h := range headers {
if value := rr.Header().Get(h.key); value != h.value {
t.Errorf("HTTP header %v does not match: expected: %v, got %v",
h.key, h.value, value)
}
}
if rr.Body.String() != string(testInput) {
t.Errorf("handler did not receive expected data: got %d, expected %d bytes",
len(rr.Body.String()), len(testInput))
}
}
func stringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
func TestGitCommandProcess(t *testing.T) {
if os.Getenv("GL_ID") != GL_ID {
return
}
defer os.Exit(0)
uploadPack := stringInSlice("upload-pack", os.Args)
if uploadPack {
// First, send a large payload to stdout so that this executable will be blocked
// until the reader consumes the data
testInput := createTestPayload()
body := bytes.NewReader([]byte(testInput))
io.Copy(os.Stdout, body)
// Now consume all the data to unblock the sender
ioutil.ReadAll(os.Stdin)
} else {
io.Copy(os.Stdout, os.Stdin)
}
}
package git
import (
"context"
"fmt"
"io"
"net/http"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
......@@ -20,41 +18,14 @@ func handleReceivePack(w *HttpResponseWriter, r *http.Request, a *api.Response)
cr, cw := helper.NewWriteAfterReader(r.Body, w)
defer cw.Flush()
var err error
if a.GitalyServer.Address == "" {
err = handleReceivePackLocally(a, r, cr, cw, action)
} else {
gitProtocol := r.Header.Get("Git-Protocol")
gitProtocol := r.Header.Get("Git-Protocol")
err = handleReceivePackWithGitaly(r.Context(), a, cr, cw, gitProtocol)
}
return err
}
func handleReceivePackLocally(a *api.Response, r *http.Request, stdin io.Reader, stdout io.Writer, action string) error {
cmd, err := startGitCommand(a, stdin, stdout, action)
if err != nil {
return fmt.Errorf("startGitCommand: %v", err)
}
defer helper.CleanUpProcessGroup(cmd)
if err := cmd.Wait(); err != nil {
helper.LogError(r, fmt.Errorf("wait for %v: %v", cmd.Args, err))
// Return nil because the response body has been written to already.
return nil
}
return nil
}
func handleReceivePackWithGitaly(ctx context.Context, a *api.Response, clientRequest io.Reader, clientResponse io.Writer, gitProtocol string) error {
smarthttp, err := gitaly.NewSmartHTTPClient(a.GitalyServer)
if err != nil {
return fmt.Errorf("smarthttp.ReceivePack: %v", err)
}
if err := smarthttp.ReceivePack(ctx, &a.Repository, a.GL_ID, a.GL_USERNAME, a.GL_REPOSITORY, a.GitConfigOptions, clientRequest, clientResponse, gitProtocol); err != nil {
if err := smarthttp.ReceivePack(r.Context(), &a.Repository, a.GL_ID, a.GL_USERNAME, a.GL_REPOSITORY, a.GitConfigOptions, cr, cw, gitProtocol); err != nil {
return fmt.Errorf("smarthttp.ReceivePack: %v", err)
}
......
package headers
import (
"mime"
"net/http"
"regexp"
svg "github.com/h2non/go-is-svg"
)
var (
ImageTypeRegex = regexp.MustCompile(`^image/*`)
SvgMimeTypeRegex = regexp.MustCompile(`^image/svg\+xml$`)
TextTypeRegex = regexp.MustCompile(`^text/*`)
VideoTypeRegex = regexp.MustCompile(`^video/*`)
AttachmentRegex = regexp.MustCompile(`^attachment`)
)
// Mime types that can't be inlined. Usually subtypes of main types
var forbiddenInlineTypes = []*regexp.Regexp{SvgMimeTypeRegex}
// Mime types that can be inlined. We can add global types like "image/" or
// specific types like "text/plain". If there is a specific type inside a global
// allowed type that can't be inlined we must add it to the forbiddenInlineTypes var.
// One example of this is the mime type "image". We allow all images to be
// inlined except for SVGs.
var allowedInlineTypes = []*regexp.Regexp{ImageTypeRegex, TextTypeRegex, VideoTypeRegex}
func SafeContentHeaders(data []byte, contentDisposition string) (string, string) {
contentType := safeContentType(data)
contentDisposition = safeContentDisposition(contentType, contentDisposition)
return contentType, contentDisposition
}
func safeContentType(data []byte) string {
// Special case for svg because DetectContentType detects it as text
if svg.Is(data) {
return "image/svg+xml"
}
// Override any existing Content-Type header from other ResponseWriters
contentType := http.DetectContentType(data)
// If the content is text type, we set to plain, because we don't
// want to render it inline if they're html or javascript
if isType(contentType, TextTypeRegex) {
return "text/plain; charset=utf-8"
}
return contentType
}
func safeContentDisposition(contentType string, contentDisposition string) string {
existingDisposition, file := extractContentDispositionFile(contentDisposition)
// If the existing disposition is attachment we return that. This allow us
// to force a download from GitLab (ie: RawController)
if AttachmentRegex.MatchString(existingDisposition) {
return attachmentDisposition(file)
}
// Checks for mime types that are forbidden to be inline
for _, element := range forbiddenInlineTypes {
if isType(contentType, element) {
return attachmentDisposition(file)
}
}
// Checks for mime types allowed to be inline
for _, element := range allowedInlineTypes {
if isType(contentType, element) {
return inlineDisposition(file)
}
}
// Anything else is set to attachment
return attachmentDisposition(file)
}
func extractContentDispositionFile(disposition string) (string, string) {
if disposition == "" {
return "", ""
}
existingDisposition, params, err := mime.ParseMediaType(disposition)
if err != nil {
return "", ""
}
return existingDisposition, params["filename"]
}
func attachmentDisposition(file string) string {
return disposition("attachment", file)
}
func inlineDisposition(file string) string {
return disposition("inline", file)
}
func disposition(disposition string, file string) string {
params := map[string]string{}
if file != "" {
params["filename"] = file
}
return mime.FormatMediaType(disposition, params)
}
func isType(contentType string, mimeType *regexp.Regexp) bool {
return mimeType.MatchString(contentType)
}
package headers
import (
"net/http"
"strconv"
)
// Max number of bytes that http.DetectContentType needs to get the content type
const MaxDetectSize = 512
// HTTP Headers
const (
ContentDispositionHeader = "Content-Disposition"
ContentTypeHeader = "Content-Type"
// Workhorse related headers
GitlabWorkhorseSendDataHeader = "Gitlab-Workhorse-Send-Data"
XSendFileHeader = "X-Sendfile"
XSendFileTypeHeader = "X-Sendfile-Type"
// Signal header that indicates Workhorse should detect and set the content headers
GitlabWorkhorseDetectContentTypeHeader = "Gitlab-Workhorse-Detect-Content-Type"
)
var ResponseHeaders = []string{
XSendFileHeader,
GitlabWorkhorseSendDataHeader,
GitlabWorkhorseDetectContentTypeHeader,
}
func IsDetectContentTypeHeaderPresent(rw http.ResponseWriter) bool {
header, err := strconv.ParseBool(rw.Header().Get(GitlabWorkhorseDetectContentTypeHeader))
if err != nil || !header {
return false
}
return true
}
// AnyResponseHeaderPresent checks in the ResponseWriter if there is any Response Header
func AnyResponseHeaderPresent(rw http.ResponseWriter) bool {
// If this header is not present means that we want the old behavior
if !IsDetectContentTypeHeaderPresent(rw) {
return false
}
for _, header := range ResponseHeaders {
if rw.Header().Get(header) != "" {
return true
}
}
return false
}
// RemoveResponseHeaders removes any ResponseHeader from the ResponseWriter
func RemoveResponseHeaders(rw http.ResponseWriter) {
for _, header := range ResponseHeaders {
rw.Header().Del(header)
}
}
package headers
import (
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
)
func TestIsDetectContentTypeHeaderPresent(t *testing.T) {
rw := httptest.NewRecorder()
rw.Header().Del(GitlabWorkhorseDetectContentTypeHeader)
require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw))
rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "true")
require.Equal(t, true, IsDetectContentTypeHeaderPresent(rw))
rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "false")
require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw))
rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "foobar")
require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw))
}
......@@ -28,11 +28,11 @@ func getCorrelationID(ctx context.Context) string {
return correlationID
}
// WithContext provides a *logrus.Entry with the proper "correlation-id" field.
// WithContext provides a *logrus.Entry with the proper "correlation_id" field.
//
// "[MISSING]" will be used when ctx has no value for KeyCorrelationID
func WithContext(ctx context.Context) *logrus.Entry {
return logrus.WithField("correlation-id", getCorrelationID(ctx))
return logrus.WithField("correlation_id", getCorrelationID(ctx))
}
// NoContext provides logrus.StandardLogger()
......@@ -40,22 +40,22 @@ func NoContext() *logrus.Logger {
return logrus.StandardLogger()
}
// WrapEntry adds the proper "correlation-id" field to the provided *logrus.Entry
// WrapEntry adds the proper "correlation_id" field to the provided *logrus.Entry
func WrapEntry(ctx context.Context, e *logrus.Entry) *logrus.Entry {
return e.WithField("correlation-id", getCorrelationID(ctx))
return e.WithField("correlation_id", getCorrelationID(ctx))
}
// WithFields decorates logrus.WithFields with the proper "correlation-id"
// WithFields decorates logrus.WithFields with the proper "correlation_id"
func WithFields(ctx context.Context, f Fields) *logrus.Entry {
return WithContext(ctx).WithFields(toLogrusFields(f))
}
// WithField decorates logrus.WithField with the proper "correlation-id"
// WithField decorates logrus.WithField with the proper "correlation_id"
func WithField(ctx context.Context, key string, value interface{}) *logrus.Entry {
return WithContext(ctx).WithField(key, value)
}
// WithError decorates logrus.WithError with the proper "correlation-id"
// WithError decorates logrus.WithError with the proper "correlation_id"
func WithError(ctx context.Context, err error) *logrus.Entry {
return WithContext(ctx).WithError(err)
}
......@@ -19,8 +19,8 @@ func requireCorrelationID(t *testing.T, getEntry func(ctx context.Context) *logr
e := getEntry(ctx)
require.NotNil(t, e)
require.Contains(t, e.Data, "correlation-id")
require.Equal(t, id, e.Data["correlation-id"])
require.Contains(t, e.Data, "correlation_id")
require.Equal(t, id, e.Data["correlation_id"])
return e
}
......@@ -84,7 +84,7 @@ func TestNoContext(t *testing.T) {
require.Equal(t, logrus.StandardLogger(), logger)
e := logger.WithField(key, value)
require.NotContains(t, e.Data, "correlation-id")
require.NotContains(t, e.Data, "correlation_id")
require.Contains(t, e.Data, key)
require.Equal(t, value, e.Data[key])
......
package contentprocessor
import (
"bytes"
"io"
"net/http"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/headers"
)
type contentDisposition struct {
rw http.ResponseWriter
buf *bytes.Buffer
wroteHeader bool
flushed bool
active bool
removedResponseHeaders bool
status int
sentStatus bool
}
// SetContentHeaders buffers the response if Gitlab-Workhorse-Detect-Content-Type
// header is found and set the proper content headers based on the current
// value of content type and disposition
func SetContentHeaders(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
cd := &contentDisposition{
rw: w,
buf: &bytes.Buffer{},
status: http.StatusOK,
}
defer cd.flush()
h.ServeHTTP(cd, r)
})
}
func (cd *contentDisposition) Header() http.Header {
return cd.rw.Header()
}
func (cd *contentDisposition) Write(data []byte) (int, error) {
// Normal write if we don't need to buffer
if cd.isUnbuffered() {
cd.WriteHeader(cd.status)
return cd.rw.Write(data)
}
// Write the new data into the buffer
n, _ := cd.buf.Write(data)
// If we have enough data to calculate the content headers then flush the Buffer
var err error
if cd.buf.Len() >= headers.MaxDetectSize {
err = cd.flushBuffer()
}
return n, err
}
func (cd *contentDisposition) flushBuffer() error {
if cd.isUnbuffered() {
return nil
}
cd.flushed = true
// If the buffer has any content then we calculate the content headers and
// write in the response
if cd.buf.Len() > 0 {
cd.writeContentHeaders()
cd.WriteHeader(cd.status)
_, err := io.Copy(cd.rw, cd.buf)
return err
}
// If no content is present in the buffer we still need to send the headers
cd.WriteHeader(cd.status)
return nil
}
func (cd *contentDisposition) writeContentHeaders() {
if cd.wroteHeader {
return
}
cd.wroteHeader = true
contentType, contentDisposition := headers.SafeContentHeaders(cd.buf.Bytes(), cd.Header().Get(headers.ContentDispositionHeader))
cd.Header().Set(headers.ContentTypeHeader, contentType)
cd.Header().Set(headers.ContentDispositionHeader, contentDisposition)
}
func (cd *contentDisposition) WriteHeader(status int) {
if cd.sentStatus {
return
}
cd.status = status
if cd.isUnbuffered() {
cd.rw.WriteHeader(cd.status)
cd.sentStatus = true
}
}
// If we find any response header, then we must calculate the content headers
// If we don't find any, the data is not buffered and it works as
// a usual ResponseWriter
func (cd *contentDisposition) isUnbuffered() bool {
if !cd.removedResponseHeaders {
if headers.IsDetectContentTypeHeaderPresent(cd.rw) {
cd.active = true
}
cd.removedResponseHeaders = true
// We ensure to clear any response header from the response
headers.RemoveResponseHeaders(cd.rw)
}
return cd.flushed || !cd.active
}
func (cd *contentDisposition) flush() {
cd.flushBuffer()
}
package contentprocessor
import (
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/headers"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"github.com/stretchr/testify/require"
)
func TestFailSetContentTypeAndDisposition(t *testing.T) {
testCaseBody := "Hello world!"
h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, err := io.WriteString(w, testCaseBody)