Commit bd5ffab3 authored by Dmitry Smirnov's avatar Dmitry Smirnov

Imported Upstream version 1.1.14+dfsg

parent 83f789c1
......@@ -7,3 +7,4 @@ awstesting/integration/smoke/_test/
/vendor/bin/
/vendor/pkg/
/vendor/src/
/private/model/cli/gen-api/gen-api
{
"PkgHandler": {
"Pattern": "/sdk-for-go/api/",
"StripPrefix": "/sdk-for-go/api",
"Include": ["/src/github.com/aws/aws-sdk-go/aws", "/src/github.com/aws/aws-sdk-go/service"],
"Exclude": ["/src/cmd", "/src/github.com/aws/aws-sdk-go/awstesting", "/src/github.com/aws/aws-sdk-go/awsmigrate"],
"IgnoredSuffixes": ["iface"]
},
"Github": {
"Tag": "master",
"Repo": "/aws/aws-sdk-go",
"UseGithub": true
}
}
......@@ -5,9 +5,10 @@ sudo: false
go:
- 1.4
- 1.5
- 1.6
- tip
# Use Go 1.5's vendoring experiment for 1.5 tests. 1.4 tests will use the tip of the dependancies repo.
# Use Go 1.5's vendoring experiment for 1.5 tests. 1.4 tests will use the tip of the dependencies repo.
env:
- GO15VENDOREXPERIMENT=1
......@@ -15,7 +16,7 @@ install:
- make get-deps
script:
- make unit
- make unit-with-race-cover
matrix:
allow_failures:
......
......@@ -3,12 +3,14 @@ LINTIGNOREDOC='service/[^/]+/(api|service|waiters)\.go:.+(comment on exported|sh
LINTIGNORECONST='service/[^/]+/(api|service|waiters)\.go:.+(type|struct field|const|func) ([^ ]+) should be ([^ ]+)'
LINTIGNORESTUTTER='service/[^/]+/(api|service)\.go:.+(and that stutters)'
LINTIGNOREINFLECT='service/[^/]+/(api|service)\.go:.+method .+ should be '
LINTIGNOREINFLECTS3UPLOAD='service/s3/s3manager/upload\.go:.+struct field SSEKMSKeyId should be '
LINTIGNOREDEPS='vendor/.+\.go'
SDK_WITH_VENDOR_PKGS=$(shell go list ./... | grep -v "/vendor/src")
SDK_ONLY_PKGS=$(shell go list ./... | grep -v "/vendor/")
SDK_GO_1_4=$(shell go version | grep "go1.4")
all: generate unit
all: get-deps generate unit
help:
@echo "Please use \`make <target>' where <target> is one of"
......@@ -17,6 +19,7 @@ help:
@echo " build to go build the SDK"
@echo " unit to run unit tests"
@echo " integration to run integration tests"
@echo " performance to run performance tests"
@echo " verify to verify tests"
@echo " lint to lint the SDK"
@echo " vet to vet the SDK"
......@@ -24,8 +27,7 @@ help:
@echo " gen-test to generate protocol tests"
@echo " gen-services to generate services"
@echo " get-deps to go get the SDK dependencies"
@echo " get-deps-unit to get the SDK's unit test dependencies"
@echo " get-deps-integ to get the SDK's integration test dependencies"
@echo " get-deps-tests to get the SDK's test dependencies"
@echo " get-deps-verify to get the SDK's verification dependencies"
generate: gen-test gen-endpoints gen-services
......@@ -45,42 +47,59 @@ build:
@echo "go build SDK and vendor packages"
@go build $(SDK_WITH_VENDOR_PKGS)
unit: get-deps-unit build verify
unit: get-deps-tests build verify
@echo "go test SDK and vendor packages"
@go test $(SDK_WITH_VENDOR_PKGS)
integration: get-deps-integ
unit-with-race-cover: get-deps-tests build verify
@echo "go test SDK and vendor packages"
@go test -v -race -cpu=1,2,4 -covermode=atomic $(SDK_WITH_VENDOR_PKGS)
integration: get-deps-tests integ-custom smoke-tests performance
integ-custom:
go test -tags=integration ./awstesting/integration/customizations/...
smoke-tests: get-deps-tests
gucumber ./awstesting/integration/smoke
performance: get-deps-tests
AWS_TESTING_LOG_RESULTS=${log-detailed} AWS_TESTING_REGION=$(region) AWS_TESTING_DB_TABLE=$(table) gucumber ./awstesting/performance
sandbox-tests:
docker build -f ./awstesting/sandbox/Dockerfile.test.go1.4 -t "aws-sdk-go-1.4" .
docker build -f ./awstesting/sandbox/Dockerfile.test.go1.5 -t "aws-sdk-go-1.5" .
docker build -f ./awstesting/sandbox/Dockerfile.test.go1.6 -t "aws-sdk-go-1.6" .
docker run -t aws-sdk-go-1.4
docker run -t aws-sdk-go-1.5
docker run -t aws-sdk-go-1.6
verify: get-deps-verify lint vet
lint:
@echo "go lint SDK and vendor packages"
@lint=`golint ./...`; \
lint=`echo "$$lint" | grep -E -v -e ${LINTIGNOREDOT} -e ${LINTIGNOREDOC} -e ${LINTIGNORECONST} -e ${LINTIGNORESTUTTER} -e ${LINTIGNOREINFLECT} -e ${LINTIGNOREDEPS}`; \
@lint=`if [ -z "${SDK_GO_1_4}" ]; then golint ./...; else echo "skipped"; fi`; \
lint=`echo "$$lint" | grep -E -v -e ${LINTIGNOREDOT} -e ${LINTIGNOREDOC} -e ${LINTIGNORECONST} -e ${LINTIGNORESTUTTER} -e ${LINTIGNOREINFLECT} -e ${LINTIGNOREDEPS} -e ${LINTIGNOREINFLECTS3UPLOAD}`; \
echo "$$lint"; \
if [ "$$lint" != "" ]; then exit 1; fi
if [ "$$lint" != "" ] && [ "$$lint" != "skipped" ]; then exit 1; fi
vet:
go tool vet -all -shadow $(shell ls -d */ | grep -v vendor)
go tool vet -all -shadow $(shell if go version | grep -v 1.5 | grep -v 1.4 >> /dev/null; then echo "-example=false" | tr -d '\n'; fi) $(shell ls -d */ | grep -v vendor | grep -v awsmigrate)
get-deps: get-deps-unit get-deps-integ get-deps-verify
get-deps: get-deps-tests get-deps-verify
@echo "go get SDK dependencies"
@go get -v $(SDK_ONLY_PKGS)
get-deps-unit:
@echo "go get SDK unit testing dependancies"
get-deps-tests:
@echo "go get SDK testing dependencies"
go get github.com/lsegal/gucumber/cmd/gucumber
go get github.com/stretchr/testify
go get github.com/smartystreets/goconvey
get-deps-integ: get-deps-unit
@echo "go get SDK integration testing dependencies"
go get github.com/lsegal/gucumber/cmd/gucumber
get-deps-verify:
@echo "go get SDK verification utilities"
go get github.com/golang/lint/golint
@if [ -z "${SDK_GO_1_4}" ]; then go get github.com/golang/lint/golint; else echo "skipped getting golint"; fi
go get golang.org/x/tools/cmd/vet
bench:
@echo "go bench SDK packages"
......@@ -93,6 +112,8 @@ bench-protocol:
docs:
@echo "generate SDK docs"
rm -rf doc && bundle install && bundle exec yard
# This env variable, DOCS, is for internal use
if [ -n $(AWS_DOC_GEN_TOOL) ]; then echo "For internal use. Subject to change."; $(AWS_DOC_GEN_TOOL) `pwd`; fi
api_info:
@go run private/model/cli/api-info/api-info.go
......@@ -11,11 +11,11 @@ Checkout our [release notes](https://github.com/aws/aws-sdk-go/releases) for inf
## Installing
If you are using Go 1.5 with the `GO15VENDOREXPERIMENT=1` vendoring flag you can use the following to get the SDK as the SDK's runtime dependancies are vendored in the `vendor` folder.
If you are using Go 1.5 with the `GO15VENDOREXPERIMENT=1` vendoring flag you can use the following to get the SDK as the SDK's runtime dependencies are vendored in the `vendor` folder.
$ go get -u github.com/aws/aws-sdk-go
Otherwise you'll need to tell Go to get the SDK and all of its dependancies.
Otherwise you'll need to tell Go to get the SDK and all of its dependencies.
$ go get -u github.com/aws/aws-sdk-go/...
......@@ -41,6 +41,9 @@ AWS_ACCESS_KEY_ID=AKID1234567890
AWS_SECRET_ACCESS_KEY=MY-SECRET-KEY
```
### AWS CLI config file (`~/aws/config`)
The AWS SDK for Go does not support the AWS CLI's config file. The SDK will not use any contents from this file. The SDK only supports the shared credentials file (`~/.aws/credentials`). #384 tracks this feature request discussion.
## Using the Go SDK
To use a service in the SDK, create a service variable by calling the `New()`
......
......@@ -14,13 +14,13 @@ package awserr
// if err != nil {
// if awsErr, ok := err.(awserr.Error); ok {
// // Get error details
// log.Println("Error:", err.Code(), err.Message())
// log.Println("Error:", awsErr.Code(), awsErr.Message())
//
// // Prints out full error message, including original error if there was one.
// log.Println("Error:", err.Error())
// log.Println("Error:", awsErr.Error())
//
// // Get original error
// if origErr := err.Err(); origErr != nil {
// if origErr := awsErr.OrigErr(); origErr != nil {
// // operate on original error.
// }
// } else {
......@@ -42,15 +42,55 @@ type Error interface {
OrigErr() error
}
// BatchError is a batch of errors which also wraps lower level errors with
// code, message, and original errors. Calling Error() will include all errors
// that occured in the batch.
//
// Deprecated: Replaced with BatchedErrors. Only defined for backwards
// compatibility.
type BatchError interface {
// Satisfy the generic error interface.
error
// Returns the short phrase depicting the classification of the error.
Code() string
// Returns the error details message.
Message() string
// Returns the original error if one was set. Nil is returned if not set.
OrigErrs() []error
}
// BatchedErrors is a batch of errors which also wraps lower level errors with
// code, message, and original errors. Calling Error() will include all errors
// that occured in the batch.
//
// Replaces BatchError
type BatchedErrors interface {
// Satisfy the base Error interface.
Error
// Returns the original error if one was set. Nil is returned if not set.
OrigErrs() []error
}
// New returns an Error object described by the code, message, and origErr.
//
// If origErr satisfies the Error interface it will not be wrapped within a new
// Error object and will instead be returned.
func New(code, message string, origErr error) Error {
if e, ok := origErr.(Error); ok && e != nil {
return e
var errs []error
if origErr != nil {
errs = append(errs, origErr)
}
return newBaseError(code, message, origErr)
return newBaseError(code, message, errs)
}
// NewBatchError returns an BatchedErrors with a collection of errors as an
// array of errors.
func NewBatchError(code, message string, errs []error) BatchedErrors {
return newBaseError(code, message, errs)
}
// A RequestFailure is an interface to extract request failure information from
......@@ -63,9 +103,9 @@ func New(code, message string, origErr error) Error {
// output, err := s3manage.Upload(svc, input, opts)
// if err != nil {
// if reqerr, ok := err.(RequestFailure); ok {
// log.Printf("Request failed", reqerr.Code(), reqerr.Message(), reqerr.RequestID())
// log.Println("Request failed", reqerr.Code(), reqerr.Message(), reqerr.RequestID())
// } else {
// log.Printf("Error:", err.Error()
// log.Println("Error:", err.Error())
// }
// }
//
......
......@@ -31,23 +31,27 @@ type baseError struct {
// Optional original error this error is based off of. Allows building
// chained errors.
origErr error
errs []error
}
// newBaseError returns an error object for the code, message, and err.
// newBaseError returns an error object for the code, message, and errors.
//
// code is a short no whitespace phrase depicting the classification of
// the error that is being created.
//
// message is the free flow string containing detailed information about the error.
// message is the free flow string containing detailed information about the
// error.
//
// origErr is the error object which will be nested under the new error to be returned.
func newBaseError(code, message string, origErr error) *baseError {
return &baseError{
// origErrs is the error objects which will be nested under the new errors to
// be returned.
func newBaseError(code, message string, origErrs []error) *baseError {
b := &baseError{
code: code,
message: message,
origErr: origErr,
errs: origErrs,
}
return b
}
// Error returns the string representation of the error.
......@@ -56,7 +60,12 @@ func newBaseError(code, message string, origErr error) *baseError {
//
// Satisfies the error interface.
func (b baseError) Error() string {
return SprintError(b.code, b.message, "", b.origErr)
size := len(b.errs)
if size > 0 {
return SprintError(b.code, b.message, "", errorList(b.errs))
}
return SprintError(b.code, b.message, "", nil)
}
// String returns the string representation of the error.
......@@ -75,10 +84,28 @@ func (b baseError) Message() string {
return b.message
}
// OrigErr returns the original error if one was set. Nil is returned if no error
// was set.
// OrigErr returns the original error if one was set. Nil is returned if no
// error was set. This only returns the first element in the list. If the full
// list is needed, use BatchedErrors.
func (b baseError) OrigErr() error {
return b.origErr
switch len(b.errs) {
case 0:
return nil
case 1:
return b.errs[0]
default:
if err, ok := b.errs[0].(Error); ok {
return NewBatchError(err.Code(), err.Message(), b.errs[1:])
}
return NewBatchError("BatchedErrors",
"multiple errors occured", b.errs)
}
}
// OrigErrs returns the original errors if one was set. An empty slice is
// returned if no error was set.
func (b baseError) OrigErrs() []error {
return b.errs
}
// So that the Error interface type can be included as an anonymous field
......@@ -94,8 +121,8 @@ type requestError struct {
requestID string
}
// newRequestError returns a wrapped error with additional information for request
// status code, and service requestID.
// newRequestError returns a wrapped error with additional information for
// request status code, and service requestID.
//
// Should be used to wrap all request which involve service requests. Even if
// the request failed without a service response, but had an HTTP status code
......@@ -133,3 +160,35 @@ func (r requestError) StatusCode() int {
func (r requestError) RequestID() string {
return r.requestID
}
// OrigErrs returns the original errors if one was set. An empty slice is
// returned if no error was set.
func (r requestError) OrigErrs() []error {
if b, ok := r.awsError.(BatchedErrors); ok {
return b.OrigErrs()
}
return []error{r.OrigErr()}
}
// An error list that satisfies the golang interface
type errorList []error
// Error returns the string representation of the error.
//
// Satisfies the error interface.
func (e errorList) Error() string {
msg := ""
// How do we want to handle the array size being zero
if size := len(e); size > 0 {
for i := 0; i < size; i++ {
msg += fmt.Sprintf("%s", e[i].Error())
// We check the next index to see if it is within the slice.
// If it is, then we append a newline. We do this, because unit tests
// could be broken with the additional '\n'
if i+1 < size {
msg += "\n"
}
}
}
return msg
}
......@@ -91,6 +91,10 @@ func prettify(v reflect.Value, indent int, buf *bytes.Buffer) {
buf.WriteString("\n" + strings.Repeat(" ", indent) + "}")
default:
if !v.IsValid() {
fmt.Fprint(buf, "<invalid value>")
return
}
format := "%v"
switch v.Interface().(type) {
case string:
......
......@@ -108,7 +108,7 @@ const logRespMsg = `DEBUG: Response %s/%s Details:
-----------------------------------------------------`
func logResponse(r *request.Request) {
var msg = "no reponse data"
var msg = "no response data"
if r.HTTPResponse != nil {
logBody := r.Config.LogLevel.Matches(aws.LogDebugWithHTTPBody)
dumpedBody, _ := httputil.DumpResponse(r.HTTPResponse, logBody)
......
package client
import (
"math"
"math/rand"
"time"
......@@ -32,7 +31,13 @@ func (d DefaultRetryer) MaxRetries() int {
// RetryRules returns the delay duration before retrying this request again
func (d DefaultRetryer) RetryRules(r *request.Request) time.Duration {
delay := int(math.Pow(2, float64(r.RetryCount))) * (rand.Intn(30) + 30)
// Set the upper limit of delay in retrying at ~five minutes
retryCount := r.RetryCount
if retryCount > 13 {
retryCount = 13
}
delay := (1 << uint(retryCount)) * (rand.Intn(30) + 30)
return time.Duration(delay) * time.Millisecond
}
......
......@@ -18,6 +18,11 @@ type RequestRetryer interface{}
// A Config provides service configuration for service clients. By default,
// all clients will use the {defaults.DefaultConfig} structure.
type Config struct {
// Enables verbose error printing of all credential chain errors.
// Should be used when wanting to see all errors while attempting to retreive
// credentials.
CredentialsChainVerboseErrors *bool
// The credentials object to use when signing requests. Defaults to
// a chain of credential providers to search for credentials in environment
// variables, shared credential file, and EC2 Instance Roles.
......@@ -95,6 +100,20 @@ type Config struct {
// Amazon S3: Virtual Hosting of Buckets
S3ForcePathStyle *bool
// Set this to `true` to disable the EC2Metadata client from overriding the
// default http.Client's Timeout. This is helpful if you do not want the EC2Metadata
// client to create a new http.Client. This options is only meaningful if you're not
// already using a custom HTTP client with the SDK. Enabled by default.
//
// Must be set and provided to the session.New() in order to disable the EC2Metadata
// overriding the timeout for default credentials chain.
//
// Example:
// sess := session.New(aws.NewConfig().WithEC2MetadataDiableTimeoutOverride(true))
// svc := s3.New(sess)
//
EC2MetadataDisableTimeoutOverride *bool
SleepDelay func(time.Duration)
}
......@@ -107,6 +126,13 @@ func NewConfig() *Config {
return &Config{}
}
// WithCredentialsChainVerboseErrors sets a config verbose errors boolean and returning
// a Config pointer.
func (c *Config) WithCredentialsChainVerboseErrors(verboseErrs bool) *Config {
c.CredentialsChainVerboseErrors = &verboseErrs
return c
}
// WithCredentials sets a config Credentials value returning a Config pointer
// for chaining.
func (c *Config) WithCredentials(creds *credentials.Credentials) *Config {
......@@ -184,6 +210,13 @@ func (c *Config) WithS3ForcePathStyle(force bool) *Config {
return c
}
// WithEC2MetadataDisableTimeoutOverride sets a config EC2MetadataDisableTimeoutOverride value
// returning a Config pointer for chaining.
func (c *Config) WithEC2MetadataDisableTimeoutOverride(enable bool) *Config {
c.EC2MetadataDisableTimeoutOverride = &enable
return c
}
// WithSleepDelay overrides the function used to sleep while waiting for the
// next retry. Defaults to time.Sleep.
func (c *Config) WithSleepDelay(fn func(time.Duration)) *Config {
......@@ -203,6 +236,10 @@ func mergeInConfig(dst *Config, other *Config) {
return
}
if other.CredentialsChainVerboseErrors != nil {
dst.CredentialsChainVerboseErrors = other.CredentialsChainVerboseErrors
}
if other.Credentials != nil {
dst.Credentials = other.Credentials
}
......@@ -251,6 +288,10 @@ func mergeInConfig(dst *Config, other *Config) {
dst.S3ForcePathStyle = other.S3ForcePathStyle
}
if other.EC2MetadataDisableTimeoutOverride != nil {
dst.EC2MetadataDisableTimeoutOverride = other.EC2MetadataDisableTimeoutOverride
}
if other.SleepDelay != nil {
dst.SleepDelay = other.SleepDelay
}
......
......@@ -64,6 +64,11 @@ var SendHandler = request.NamedHandler{Name: "core.SendHandler", Fn: func(r *req
var err error
r.HTTPResponse, err = r.Config.HTTPClient.Do(r.HTTPRequest)
if err != nil {
// Prevent leaking if an HTTPResponse was returned. Clean up
// the body.
if r.HTTPResponse != nil {
r.HTTPResponse.Body.Close()
}
// Capture the case where url.Error is returned for error processing
// response. e.g. 301 without location header comes back as string
// error and r.HTTPResponse is nil. Other url redirect errors will
......