Commit 9d694c7b authored by Utkarsh Gupta's avatar Utkarsh Gupta

New upstream version 8.5.2+debian

parent e4cc1382
image: golang:1.10
image: golang:1.11
verify:
script:
......@@ -8,20 +8,20 @@ verify:
services:
- name: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest
# Disable the hooks so we don't have to stub the GitLab API
command: ["bash", "-c", "mkdir -p /home/git/repositories && rm -rf /srv/gitlab-shell/hooks/* && exec /scripts/process-wrapper"]
command: ["bash", "-c", "mkdir -p /home/git/repositories && rm -rf /srv/gitlab-shell/hooks/* && exec /usr/bin/env GITALY_TESTING_NO_GIT_HOOKS=1 /scripts/process-wrapper"]
alias: gitaly
variables:
GITALY_ADDRESS: "tcp://gitaly:8075"
script:
- apt update -qq && apt install -y unzip bzip2
- go version
- apt-get update && apt-get -y install libimage-exiftool-perl
- make test
test using go 1.10:
image: golang:1.10
<<: *test_definition
test using go 1.11:
image: golang:1.11
<<: *test_definition
test:release:
......
......@@ -2,6 +2,86 @@
Formerly known as 'gitlab-git-http-server'.
v8.5.2
- Don't log http.ErrAbortHandler panics in sentry !392
v8.5.1
- Remove duplicate X-Request-Id response header !384
v8.5.0
- Replace terminal terminology to channel !382
v8.4.0
- Adds X-Request-Id response header for Workhorse !363
- Change content type detection header size to 4k !366
- Allow unknown fields in jsonpb gitaly-proto messages !367
- Filter `sharedSecret` param from Jira !369
- Get git-archive with GetArchiveRequest !375
v8.3.3
- Preserve orientation when removing EXIF
v8.3.2
- Remove EXIF from JPEG/TIFF images
v 8.3.1
- Update gitaly-proto to 1.10.0 !363
v 8.3.0
- Count ^/-/ requests separately in prometheus !355
- Statically link jaeger into Workhorse by default !359
- Support encoded Content-Disposition fields !360
v 8.2.0
- Sign LFS upload requests that have been handled by workhorse
- Fixed svg recognition to get the proper content type !353
v 8.1.1
- Sign LFS upload requests that have been handled by workhorse
v 8.1.0
- Upgrade the gitaly client to v1.13.0 (includes TLS support) !351
- Update gitaly-proto to 0.124.0 !331
- Add distributed tracing with LabKit !325
v 8.0.4
- Preserve orientation when removing EXIF
v 8.0.3
- Remove EXIF from JPEG/TIFF images
v 8.0.2
- Fixed svg recognition to get the proper content type !353
v 8.0.1
- Sign LFS upload requests that have been handled by workhorse
v 8.0.0
- Remove local git archive support !304
- Remove local git diff handling !345
- Remove local git format-patch handling !346
- Remove RepoPath from the API response
v 7.6.1
- Sign LFS upload requests that have been handled by workhorse
v 7.6.0
- Rename correlation-id structured logging field to correlation_id !343
......@@ -9,6 +89,10 @@ v 7.6.0
- Remove curl from sendfile_test.go !344
- Update README.md usage example !342
v 7.5.1
- Rename correlation-id structured logging field to correlation_id !343
v 7.5.0
- Add proxy layer to calculate content type and disposition headers !335
......@@ -31,6 +115,10 @@ v 7.2.0
- Update CI matrix to go1.10 + go1.11 and fix ResponseWriter bugs !309
- Add support for Redis URLs (redis:// and rediss://) in Workhorse !321
v 7.1.4
- Sign LFS upload requests that have been handled by workhorse
v 7.1.3
- Redact sensitive url params as in Rails
......
......@@ -14,6 +14,7 @@ VERSION := ${VERSION_STRING}-$(shell date -u +%Y%m%d.%H%M%S)
GOBUILD := go build -ldflags "-X main.Version=$(VERSION)"
EXE_ALL := gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
INSTALL := install
BUILD_TAGS := tracer_static tracer_static_jaeger
MINIMUM_SUPPORTED_GO_VERSION := 1.8
......@@ -48,15 +49,15 @@ $(TARGET_SETUP):
gitlab-zip-cat: $(TARGET_SETUP) $(shell find cmd/gitlab-zip-cat/ -name '*.go')
$(call message,Building $@)
$(GOBUILD) -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
gitlab-zip-metadata: $(TARGET_SETUP) $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
$(call message,Building $@)
$(GOBUILD) -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
gitlab-workhorse: $(TARGET_SETUP) $(shell find . -name '*.go' | grep -v '^\./_')
$(call message,Building $@)
$(GOBUILD) -o $(BUILD_DIR)/$@ $(PKG)
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)
.PHONY: install
install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
......@@ -67,13 +68,13 @@ install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
.PHONY: test
test: $(TARGET_SETUP) prepare-tests
$(call message,$@)
@go test $(LOCAL_PACKAGES)
@go test -tags "$(BUILD_TAGS)" $(LOCAL_PACKAGES)
@echo SUCCESS
.PHONY: coverage
coverage: $(TARGET_SETUP) prepare-tests
$(call message,$@)
@go test -cover -coverprofile=test.coverage $(LOCAL_PACKAGES)
@go test -tags "$(BUILD_TAGS)" -cover -coverprofile=test.coverage $(LOCAL_PACKAGES)
go tool cover -html=test.coverage -o coverage.html
rm -f test.coverage
......@@ -87,10 +88,15 @@ clean-workhorse:
$(call message,$@)
rm -f $(EXE_ALL)
.PHONY: release
release:
.PHONY: tag
tag:
$(call message,$@)
sh _support/release.sh
sh _support/tag.sh
.PHONY: signed_tag
signed_tag:
$(call message,$@)
TAG_OPTS=-s sh _support/tag.sh
.PHONY: clean-build
clean-build:
......@@ -105,7 +111,7 @@ testdata/data/group/test.git:
git clone --quiet --bare https://gitlab.com/gitlab-org/gitlab-test.git $@
.PHONY: verify
verify: lint vet detect-context check-formatting megacheck
verify: lint vet detect-context check-formatting staticcheck
.PHONY: lint
lint: $(TARGET_SETUP) govendor-sync
......@@ -131,11 +137,11 @@ check-formatting: $(TARGET_SETUP) install-goimports
# Megacheck will tailor some responses given a minimum Go version, so pass that through the CLI
# Additionally, megacheck will not return failure exit codes unless explicitely told to via the
# `-simple.exit-non-zero` `-unused.exit-non-zero` and `-staticcheck.exit-non-zero` flags
.PHONY: megacheck
megacheck: $(TARGET_SETUP) govendor-sync
.PHONY: staticcheck
staticcheck: $(TARGET_SETUP) govendor-sync
$(call message,Verify: $@)
@command -v megacheck || go get -v honnef.co/go/tools/cmd/megacheck
@megacheck -go $(MINIMUM_SUPPORTED_GO_VERSION) -simple.exit-non-zero -unused.exit-non-zero -staticcheck.exit-non-zero $(LOCAL_PACKAGES)
@command -v staticcheck || go get -v honnef.co/go/tools/cmd/staticcheck
@staticcheck -go $(MINIMUM_SUPPORTED_GO_VERSION) $(LOCAL_PACKAGES)
# Some vendor components, used for testing are GPL, so we don't distribute them
# and need to go a sync before using them
......
......@@ -19,11 +19,14 @@ The final merge must be performed by a maintainer.
New versions of Workhorse can be released by one of the Workhorse
maintainers. The release process is:
- create a merge request to update CHANGELOG and VERSION on the
respective release branch (usually `master`)
- make sure the new version number adheres to our [versioning standard](#versioning)
- merge the merge request
- run `make release` on the release branch
- pick a release branch. For x.y.0, use `master`. For all other
versions (x.y.1, x.y.2 etc.) , use `x-y-stable`. Also see [below](#versioning)
- create a merge request to update CHANGELOG and VERSION on the
release branch
- merge the merge request
- run `make tag` or `make signed_tag` on the release branch. This will
make a tag matching the VERSION file.
- push the tag to gitlab.com
## Versioning
......
......@@ -168,6 +168,25 @@ make install PREFIX=/foo
On some operating systems, such as FreeBSD, you may have to use
`gmake` instead of `make`.
## Dependencies
### Exiftool
Workhorse uses [exiftool](https://www.sno.phy.queensu.ca/~phil/exiftool/) for
removing EXIF data (which may contain sensitive information) from uploaded
images. If you installed GitLab:
- Using the Omnibus package, you're all set.
- From source, make sure `exiftool` is installed:
```sh
# Debian/Ubuntu
sudo apt-get install libimage-exiftool-perl
# RHEL/CentOS
sudo yum install perl-Image-ExifTool
```
## Error tracking
GitLab-Workhorse supports remote error tracking with
......@@ -204,6 +223,26 @@ the first priority during development.
It is OK if a feature is only covered by integration tests.
## Distributed Tracing
Workhorse supports distributed tracing through [LabKit](https://gitlab.com/gitlab-org/labkit/) using [OpenTracing APIs](https://opentracing.io).
By default, no tracing implementation is linked into the binary, but different OpenTracing providers can be linked in using [build tags](https://golang.org/pkg/go/build/#hdr-Build_Constraints)/[build constraints](https://golang.org/pkg/go/build/#hdr-Build_Constraints). This can be done by setting the `BUILD_TAGS` make variable.
For more details of the supported providers, see LabKit, but as an example, for Jaeger tracing support, include the tags: `BUILD_TAGS="tracer_static tracer_static_jaeger"`.
```shell
make BUILD_TAGS="tracer_static tracer_static_jaeger"
```
Once Workhorse is compiled with an opentracing provider, the tracing configuration is configured via the `GITLAB_TRACING` environment variable.
For example:
```shell
GITLAB_TRACING=opentracing://jaeger ./gitlab-workhorse
```
## License
This code is distributed under the MIT license, see the LICENSE file.
......
set -e
remotes='https://gitlab.com/gitlab-org/gitlab-workhorse.git'
main() {
get_version
tag_name="v${version}"
git tag -m "Version ${version}" -a ${tag_name}
git tag $TAG_OPTS -m "Version ${version}" -a ${tag_name}
git show ${tag_name}
echo
echo "Does this look OK? Enter 'yes' to push to ${remotes}"
read confirmation
if [ "x${confirmation}" != xyes ] ; then
echo "Aborting"
exit 1
fi
for r in ${remotes}; do
git push "${r}" HEAD ${tag_name}
done
cat <<'EOF'
Remember to now push your tag, either to gitlab.com (for a
normal release) or dev.gitlab.org (for a security release).
EOF
}
get_version() {
......
package main
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"strings"
"testing"
)
func TestAllowedDownloadZip(t *testing.T) {
prepareDownloadDir(t)
// Prepare test server and backend
archiveName := "foobar.zip"
ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/%s/repository/archive.zip", ws.URL, testProject))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
extractCmd := exec.Command("unzip", archiveName)
extractCmd.Dir = scratchDir
runOrFail(t, extractCmd)
}
func TestAllowedDownloadTar(t *testing.T) {
prepareDownloadDir(t)
// Prepare test server and backend
archiveName := "foobar.tar"
ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/%s/repository/archive.tar", ws.URL, testProject))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
extractCmd := exec.Command("tar", "xf", archiveName)
extractCmd.Dir = scratchDir
runOrFail(t, extractCmd)
}
func TestAllowedDownloadTarGz(t *testing.T) {
prepareDownloadDir(t)
// Prepare test server and backend
archiveName := "foobar.tar.gz"
ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/%s/repository/archive.tar.gz", ws.URL, testProject))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
extractCmd := exec.Command("tar", "zxf", archiveName)
extractCmd.Dir = scratchDir
runOrFail(t, extractCmd)
}
func TestAllowedDownloadTarBz2(t *testing.T) {
prepareDownloadDir(t)
// Prepare test server and backend
archiveName := "foobar.tar.bz2"
ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/%s/repository/archive.tar.bz2", ws.URL, testProject))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
extractCmd := exec.Command("tar", "jxf", archiveName)
extractCmd.Dir = scratchDir
runOrFail(t, extractCmd)
}
func TestAllowedApiDownloadZip(t *testing.T) {
prepareDownloadDir(t)
// Prepare test server and backend
archiveName := "foobar.zip"
ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/api/v3/projects/123/repository/archive.zip", ws.URL))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
extractCmd := exec.Command("unzip", archiveName)
extractCmd.Dir = scratchDir
runOrFail(t, extractCmd)
}
func TestAllowedApiDownloadZipWithSlash(t *testing.T) {
prepareDownloadDir(t)
// Prepare test server and backend
archiveName := "foobar.zip"
ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
// Use foo%2Fbar instead of a numeric ID
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/api/v3/projects/foo%%2Fbar/repository/archive.zip", ws.URL))
if !strings.Contains(downloadCmd.Args[3], `projects/foo%2Fbar/repository`) {
t.Fatalf("Cannot find percent-2F: %v", downloadCmd.Args)
}
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
extractCmd := exec.Command("unzip", archiveName)
extractCmd.Dir = scratchDir
runOrFail(t, extractCmd)
}
func TestDownloadCacheHit(t *testing.T) {
prepareDownloadDir(t)
// Prepare test server and backend
archiveName := "foobar.zip"
ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
if err := os.MkdirAll(cacheDir, 0755); err != nil {
t.Fatal(err)
}
cachedContent := []byte("cached")
if err := ioutil.WriteFile(path.Join(cacheDir, archiveName), cachedContent, 0644); err != nil {
t.Fatal(err)
}
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/api/v3/projects/123/repository/archive.zip", ws.URL))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
actual, err := ioutil.ReadFile(path.Join(scratchDir, archiveName))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(actual, cachedContent) {
t.Fatal("Unexpected file contents in download")
}
}
func TestDownloadCacheCreate(t *testing.T) {
prepareDownloadDir(t)
// Prepare test server and backend
archiveName := "foobar.zip"
ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/api/v3/projects/123/repository/archive.zip", ws.URL))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
compareCmd := exec.Command("cmp", path.Join(cacheDir, archiveName), path.Join(scratchDir, archiveName))
if err := compareCmd.Run(); err != nil {
t.Fatalf("Comparison between downloaded file and cache item failed: %s", err)
}
}
......@@ -83,7 +83,7 @@ func TestPreAuthorizeContentTypeFailure(t *testing.T) {
func TestPreAuthorizeRedirect(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/", 301)
http.Redirect(w, r, "/", http.StatusMovedPermanently)
}))
defer ts.Close()
......@@ -95,7 +95,7 @@ func TestPreAuthorizeRedirect(t *testing.T) {
func TestPreAuthorizeJWT(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
token, err := jwt.Parse(r.Header.Get(api.RequestHeader), func(token *jwt.Token) (interface{}, error) {
token, err := jwt.Parse(r.Header.Get(secret.RequestHeader), func(token *jwt.Token) (interface{}, error) {
// Don't forget to validate the alg is what you expect:
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
......
......@@ -29,7 +29,50 @@ type connWithReq struct {
req *http.Request
}
func TestTerminalHappyPath(t *testing.T) {
func TestChannelHappyPath(t *testing.T) {
tests := []struct {
name string
channelPath string
}{
{"environments", envTerminalPath},
{"jobs", jobTerminalPath},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
serverConns, clientURL, close := wireupChannel(test.channelPath, nil, "channel.k8s.io")
defer close()
client, _, err := dialWebsocket(clientURL, nil, "terminal.gitlab.com")
if err != nil {
t.Fatal(err)
}
server := (<-serverConns).conn
defer server.Close()
message := "test message"
// channel.k8s.io: server writes to channel 1, STDOUT
if err := say(server, "\x01"+message); err != nil {
t.Fatal(err)