Port some replication bits to OSS (hashicorp#2386)

.travis.yml

@@ -7,7 +7,7 @@ services:
   - docker
 
 go:
-  - 1.8rc2
+  - 1.8
 
 matrix:
   allow_failures:

Makefile

@@ -24,6 +24,11 @@ dev-dynamic: generate
 test: generate
 	CGO_ENABLED=0 VAULT_TOKEN= VAULT_ACC= go test -tags='$(BUILD_TAGS)' $(TEST) $(TESTARGS) -timeout=10m -parallel=4
 
+testcompile: generate
+	@for pkg in $(TEST) ; do \
+		go test -v -c -tags='$(BUILD_TAGS)' $$pkg -parallel=4 ; \
+	done
+
 # testacc runs acceptance tests
 testacc: generate
 	@if [ "$(TEST)" = "./..." ]; then \

README.md

@@ -56,9 +56,9 @@ All documentation is available on the [Vault website](https://www.vaultproject.i
 Developing Vault
 --------------------
 
-If you wish to work on Vault itself or any of its built-in systems,
-you'll first need [Go](https://www.golang.org) installed on your
-machine (version 1.8+ is *required*).
+If you wish to work on Vault itself or any of its built-in systems, you'll
+first need [Go](https://www.golang.org) installed on your machine (version 1.8+
+is *required*).
 
 For local dev first make sure Go is properly installed, including setting up a
 [GOPATH](https://golang.org/doc/code.html#GOPATH). Next, clone this repository

builtin/logical/cassandra/test-fixtures/cassandra.yaml

@@ -421,7 +421,7 @@ seed_provider:
       parameters:
           # seeds is actually a comma-delimited list of addresses.
           # Ex: "<ip1>,<ip2>,<ip3>"
-          - seeds: "172.17.0.2"
+          - seeds: "172.17.0.3"
 
 # For workloads with more data than can fit in memory, Cassandra's
 # bottleneck will be reads that need to fetch data from
@@ -572,7 +572,7 @@ ssl_storage_port: 7001
 #
 # Setting listen_address to 0.0.0.0 is always wrong.
 #
-listen_address: 172.17.0.2
+listen_address: 172.17.0.3
 
 # Set listen_address OR listen_interface, not both. Interfaces must correspond
 # to a single address, IP aliasing is not supported.
@@ -586,7 +586,7 @@ listen_address: 172.17.0.2
 
 # Address to broadcast to other Cassandra nodes
 # Leaving this blank will set it to the same value as listen_address
-broadcast_address: 172.17.0.2
+broadcast_address: 172.17.0.3
 
 # When using multiple physical network interfaces, set this
 # to true to listen on broadcast_address in addition to
@@ -668,7 +668,7 @@ rpc_port: 9160
 # be set to 0.0.0.0. If left blank, this will be set to the value of
 # rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must
 # be set.
-broadcast_rpc_address: 172.17.0.2
+broadcast_rpc_address: 172.17.0.3
 
 # enable or disable keepalive on rpc/native connections
 rpc_keepalive: true

helper/consts/error.go

@@ -0,0 +1,13 @@
+package consts
+
+import "errors"
+
+var (
+	// ErrSealed is returned if an operation is performed on a sealed barrier.
+	// No operation is expected to succeed before unsealing
+	ErrSealed = errors.New("Vault is sealed")
+
+	// ErrStandby is returned if an operation is performed on a standby Vault.
+	// No operation is expected to succeed until active.
+	ErrStandby = errors.New("Vault is in standby mode")
+)

http/handler.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/helper/duration"
 	"github.com/hashicorp/vault/helper/jsonutil"
 	"github.com/hashicorp/vault/logical"
@@ -206,11 +207,11 @@ func handleRequestForwarding(core *vault.Core, handler http.Handler) http.Handle
 // case of an error.
 func request(core *vault.Core, w http.ResponseWriter, rawReq *http.Request, r *logical.Request) (*logical.Response, bool) {
 	resp, err := core.HandleRequest(r)
-	if errwrap.Contains(err, vault.ErrStandby.Error()) {
+	if errwrap.Contains(err, consts.ErrStandby.Error()) {
 		respondStandby(core, w, rawReq.URL)
 		return resp, false
 	}
-	if respondErrorCommon(w, resp, err) {
+	if respondErrorCommon(w, r, resp, err) {
 		return resp, false
 	}
 
@@ -310,20 +311,7 @@ func requestWrapInfo(r *http.Request, req *logical.Request) (*logical.Request, e
 }
 
 func respondError(w http.ResponseWriter, status int, err error) {
-	// Adjust status code when sealed
-	if errwrap.Contains(err, vault.ErrSealed.Error()) {
-		status = http.StatusServiceUnavailable
-	}
-
-	// Adjust status code on
-	if errwrap.Contains(err, "http: request body too large") {
-		status = http.StatusRequestEntityTooLarge
-	}
-
-	// Allow HTTPCoded error passthrough to specify a code
-	if t, ok := err.(logical.HTTPCodedError); ok {
-		status = t.Code()
-	}
+	logical.AdjustErrorStatusCode(&status, err)
 
 	w.Header().Add("Content-Type", "application/json")
 	w.WriteHeader(status)
@@ -337,42 +325,13 @@ func respondError(w http.ResponseWriter, status int, err error) {
 	enc.Encode(resp)
 }
 
-func respondErrorCommon(w http.ResponseWriter, resp *logical.Response, err error) bool {
-	// If there are no errors return
-	if err == nil && (resp == nil || !resp.IsError()) {
+func respondErrorCommon(w http.ResponseWriter, req *logical.Request, resp *logical.Response, err error) bool {
+	statusCode, newErr := logical.RespondErrorCommon(req, resp, err)
+	if newErr == nil && statusCode == 0 {
 		return false
 	}
 
-	// Start out with internal server error since in most of these cases there
-	// won't be a response so this won't be overridden
-	statusCode := http.StatusInternalServerError
-	// If we actually have a response, start out with bad request
-	if resp != nil {
-		statusCode = http.StatusBadRequest
-	}
-
-	// Now, check the error itself; if it has a specific logical error, set the
-	// appropriate code
-	if err != nil {
-		switch {
-		case errwrap.ContainsType(err, new(vault.StatusBadRequest)):
-			statusCode = http.StatusBadRequest
-		case errwrap.Contains(err, logical.ErrPermissionDenied.Error()):
-			statusCode = http.StatusForbidden
-		case errwrap.Contains(err, logical.ErrUnsupportedOperation.Error()):
-			statusCode = http.StatusMethodNotAllowed
-		case errwrap.Contains(err, logical.ErrUnsupportedPath.Error()):
-			statusCode = http.StatusNotFound
-		case errwrap.Contains(err, logical.ErrInvalidRequest.Error()):
-			statusCode = http.StatusBadRequest
-		}
-	}
-
-	if resp != nil && resp.IsError() {
-		err = fmt.Errorf("%s", resp.Data["error"].(string))
-	}
-
-	respondError(w, statusCode, err)
+	respondError(w, statusCode, newErr)
 	return true
 }
 

http/handler_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/vault"
 )
@@ -80,6 +81,7 @@ func TestSysMounts_headerAuth(t *testing.T) {
 					"default_lease_ttl": json.Number("0"),
 					"max_lease_ttl":     json.Number("0"),
 				},
+				"local": false,
 			},
 			"sys/": map[string]interface{}{
 				"description": "system endpoints used for control, policy and debugging",
@@ -88,6 +90,7 @@ func TestSysMounts_headerAuth(t *testing.T) {
 					"default_lease_ttl": json.Number("0"),
 					"max_lease_ttl":     json.Number("0"),
 				},
+				"local": false,
 			},
 			"cubbyhole/": map[string]interface{}{
 				"description": "per-token private secret storage",
@@ -96,6 +99,7 @@ func TestSysMounts_headerAuth(t *testing.T) {
 					"default_lease_ttl": json.Number("0"),
 					"max_lease_ttl":     json.Number("0"),
 				},
+				"local": false,
 			},
 		},
 		"secret/": map[string]interface{}{
@@ -105,6 +109,7 @@ func TestSysMounts_headerAuth(t *testing.T) {
 				"default_lease_ttl": json.Number("0"),
 				"max_lease_ttl":     json.Number("0"),
 			},
+			"local": false,
 		},
 		"sys/": map[string]interface{}{
 			"description": "system endpoints used for control, policy and debugging",
@@ -113,6 +118,7 @@ func TestSysMounts_headerAuth(t *testing.T) {
 				"default_lease_ttl": json.Number("0"),
 				"max_lease_ttl":     json.Number("0"),
 			},
+			"local": false,
 		},
 		"cubbyhole/": map[string]interface{}{
 			"description": "per-token private secret storage",
@@ -121,6 +127,7 @@ func TestSysMounts_headerAuth(t *testing.T) {
 				"default_lease_ttl": json.Number("0"),
 				"max_lease_ttl":     json.Number("0"),
 			},
+			"local": false,
 		},
 	}
 	testResponseStatus(t, resp, 200)
@@ -223,7 +230,7 @@ func TestHandler_error(t *testing.T) {
 	// vault.ErrSealed is a special case
 	w3 := httptest.NewRecorder()
 
-	respondError(w3, 400, vault.ErrSealed)
+	respondError(w3, 400, consts.ErrSealed)
 
 	if w3.Code != 503 {
 		t.Fatalf("expected 503, got %d", w3.Code)

http/help.go

@@ -35,7 +35,7 @@ func handleHelp(core *vault.Core, w http.ResponseWriter, req *http.Request) {
 
 	resp, err := core.HandleRequest(lreq)
 	if err != nil {
-		respondErrorCommon(w, resp, err)
+		respondErrorCommon(w, lreq, resp, err)
 		return
 	}
 

http/logical.go

@@ -109,40 +109,13 @@ func handleLogical(core *vault.Core, dataOnly bool, prepareRequestCallback Prepa
 
 		// Make the internal request. We attach the connection info
 		// as well in case this is an authentication request that requires
-		// it. Vault core handles stripping this if we need to.
+		// it. Vault core handles stripping this if we need to. This also
+		// handles all error cases; if we hit respondLogical, the request is a
+		// success.
 		resp, ok := request(core, w, r, req)
 		if !ok {
 			return
 		}
-		switch {
-		case req.Operation == logical.ReadOperation:
-			if resp == nil {
-				respondError(w, http.StatusNotFound, nil)
-				return
-			}
-
-		// Basically: if we have empty "keys" or no keys at all, 404. This
-		// provides consistency with GET.
-		case req.Operation == logical.ListOperation && resp.WrapInfo == nil:
-			if resp == nil || len(resp.Data) == 0 {
-				respondError(w, http.StatusNotFound, nil)
-				return
-			}
-			keysRaw, ok := resp.Data["keys"]
-			if !ok || keysRaw == nil {
-				respondError(w, http.StatusNotFound, nil)
-				return
-			}
-			keys, ok := keysRaw.([]string)
-			if !ok {
-				respondError(w, http.StatusInternalServerError, nil)
-				return
-			}
-			if len(keys) == 0 {
-				respondError(w, http.StatusNotFound, nil)
-				return
-			}
-		}
 
 		// Build the proper response
 		respondLogical(w, r, req, dataOnly, resp)

http/logical_test.go

@@ -101,7 +101,7 @@ func TestLogical_StandbyRedirect(t *testing.T) {
 
 	// Attempt to fix raciness in this test by giving the first core a chance
 	// to grab the lock
-	time.Sleep(time.Second)
+	time.Sleep(2 * time.Second)
 
 	// Create a second HA Vault
 	conf2 := &vault.CoreConfig{

http/sys_audit_test.go

@@ -35,13 +35,15 @@ func TestSysAudit(t *testing.T) {
 				"type":        "noop",
 				"description": "",
 				"options":     map[string]interface{}{},
+				"local":       false,
 			},
 		},
 		"noop/": map[string]interface{}{
 			"path":        "noop/",
 			"type":        "noop",
 			"description": "",
 			"options":     map[string]interface{}{},
+			"local":       false,
 		},
 	}
 	testResponseStatus(t, resp, 200)

http/sys_auth_test.go

@@ -32,6 +32,7 @@ func TestSysAuth(t *testing.T) {
 					"default_lease_ttl": json.Number("0"),
 					"max_lease_ttl":     json.Number("0"),
 				},
+				"local": false,
 			},
 		},
 		"token/": map[string]interface{}{
@@ -41,6 +42,7 @@ func TestSysAuth(t *testing.T) {
 				"default_lease_ttl": json.Number("0"),
 				"max_lease_ttl":     json.Number("0"),
 			},
+			"local": false,
 		},
 	}
 	testResponseStatus(t, resp, 200)
@@ -83,6 +85,7 @@ func TestSysEnableAuth(t *testing.T) {
 					"default_lease_ttl": json.Number("0"),
 					"max_lease_ttl":     json.Number("0"),
 				},
+				"local": false,
 			},
 			"token/": map[string]interface{}{
 				"description": "token based credentials",
@@ -91,6 +94,7 @@ func TestSysEnableAuth(t *testing.T) {
 					"default_lease_ttl": json.Number("0"),
 					"max_lease_ttl":     json.Number("0"),
 				},
+				"local": false,
 			},
 		},
 		"foo/": map[string]interface{}{
@@ -100,6 +104,7 @@ func TestSysEnableAuth(t *testing.T) {
 				"default_lease_ttl": json.Number("0"),
 				"max_lease_ttl":     json.Number("0"),
 			},
+			"local": false,
 		},
 		"token/": map[string]interface{}{
 			"description": "token based credentials",
@@ -108,6 +113,7 @@ func TestSysEnableAuth(t *testing.T) {
 				"default_lease_ttl": json.Number("0"),
 				"max_lease_ttl":     json.Number("0"),
 			},
+			"local": false,
 		},
 	}
 	testResponseStatus(t, resp, 200)
@@ -153,6 +159,7 @@ func TestSysDisableAuth(t *testing.T) {
 				},
 				"description": "token based credentials",
 				"type":        "token",
+				"local":       false,
 			},
 		},
 		"token/": map[string]interface{}{
@@ -162,6 +169,7 @@ func TestSysDisableAuth(t *testing.T) {
 			},
 			"description": "token based credentials",
 			"type":        "token",
+			"local":       false,
 		},
 	}
 	testResponseStatus(t, resp, 200)