SV: CAS: Implement Check and Set for Delete and Upsert (#13429)

* SV: CAS
    * Implement Check and Set for Delete and Upsert
    * Reading the conflict from the state store
    * Update endpoint for new error text
    * Updated HTTP api tests
    * Conflicts to the HTTP api

* SV: structs: Update SV time to UnixNanos
    * update mock to UnixNano; refactor

* SV: encrypter: quote KeyID in error
* SV: mock: add mock for namespace w/ SV

command/agent/secure_variable_endpoint.go

@@ -1,7 +1,9 @@
 package agent
 
 import (
+	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -33,7 +35,7 @@ func (s *HTTPServer) SecureVariablesListRequest(resp http.ResponseWriter, req *h
 func (s *HTTPServer) SecureVariableSpecificRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	path := strings.TrimPrefix(req.URL.Path, "/v1/var/")
 	if len(path) == 0 {
-		return nil, CodedError(http.StatusBadRequest, "Missing secure variable path")
+		return nil, CodedError(http.StatusBadRequest, "missing secure variable path")
 	}
 	switch req.Method {
 	case http.MethodGet:
@@ -63,7 +65,7 @@ func (s *HTTPServer) secureVariableQuery(resp http.ResponseWriter, req *http.Req
 	setMeta(resp, &out.QueryMeta)
 
 	if out.Data == nil {
-		return nil, CodedError(http.StatusNotFound, "Secure variable not found")
+		return nil, CodedError(http.StatusNotFound, "secure variable not found")
 	}
 	return out.Data, nil
 }
@@ -75,23 +77,49 @@ func (s *HTTPServer) secureVariableUpsert(resp http.ResponseWriter, req *http.Re
 	if err := decodeBody(req, &SecureVariable); err != nil {
 		return nil, CodedError(http.StatusBadRequest, err.Error())
 	}
-	if SecureVariable.Items == nil {
-		return nil, CodedError(http.StatusBadRequest, "Secure variable missing required Items object.")
+	if len(SecureVariable.Items) == 0 {
+		return nil, CodedError(http.StatusBadRequest, "secure variable missing required Items object")
 	}
+
 	SecureVariable.Path = path
 
-	// Format the request
+	// This function always makes an upsert request with length of 1, which is
+	// an important proviso for when we check for conflicts and return them
 	args := structs.SecureVariablesUpsertRequest{
 		Data: []*structs.SecureVariableDecrypted{&SecureVariable},
 	}
 	s.parseWriteRequest(req, &args.WriteRequest)
+	if err := parseCAS(req, &args); err != nil {
+		return nil, err
+	}
 
 	var out structs.SecureVariablesUpsertResponse
 	if err := s.agent.RPC(structs.SecureVariablesUpsertRPCMethod, &args, &out); err != nil {
+
+		// This handles the cases where there is an error in the CAS checking
+		// function that renders it unable to return the conflicting variable
+		// so it returns a text error. We can at least consider these unknown
+		// moments to be CAS violations
+		if strings.Contains(err.Error(), "cas error:") {
+			resp.WriteHeader(http.StatusConflict)
+		}
+
+		// Otherwise it's a non-CAS error
+		setIndex(resp, out.WriteMeta.Index)
 		return nil, err
 	}
-	setIndex(resp, out.WriteMeta.Index)
 
+	// As noted earlier, the upsert request generated by this endpoint always
+	// has length of 1, so we expect a non-Nil Conflicts slice to have len(1).
+	// We then extract the conflict value at index 0
+	if len(out.Conflicts) == 1 {
+		setIndex(resp, out.Conflicts[0].ModifyIndex)
+		resp.WriteHeader(http.StatusConflict)
+		return out.Conflicts[0], nil
+	}
+
+	// Finally, we know that this is a success response, send it to the caller
+	setIndex(resp, out.WriteMeta.Index)
 	return nil, nil
 }
 
@@ -102,11 +130,49 @@ func (s *HTTPServer) secureVariableDelete(resp http.ResponseWriter, req *http.Re
 		Path: path,
 	}
 	s.parseWriteRequest(req, &args.WriteRequest)
+	if err := parseCAS(req, &args); err != nil {
+		return nil, err
+	}
 
 	var out structs.SecureVariablesDeleteResponse
 	if err := s.agent.RPC(structs.SecureVariablesDeleteRPCMethod, &args, &out); err != nil {
+
+		// This handles the cases where there is an error in the CAS checking
+		// function that renders it unable to return the conflicting variable
+		// so it returns a text error. We can at least consider these unknown
+		// moments to be CAS violations
+		if strings.HasPrefix(err.Error(), "cas error:") {
+			resp.WriteHeader(http.StatusConflict)
+		}
+		setIndex(resp, out.WriteMeta.Index)
 		return nil, err
 	}
+
+	// If the CAS validation can decode the conflicting value, Conflict is
+	// non-Nil. Write out a 409 Conflict response.
+	if out.Conflict != nil {
+		setIndex(resp, out.Conflict.ModifyIndex)
+		resp.WriteHeader(http.StatusConflict)
+		return out.Conflict, nil
+	}
+
+	// Finally, we know that this is a success response, send it to the caller
 	setIndex(resp, out.WriteMeta.Index)
+	resp.WriteHeader(http.StatusNoContent)
 	return nil, nil
 }
+
+func parseCAS(req *http.Request, rpc CheckIndexSetter) error {
+	if cq := req.URL.Query().Get("cas"); cq != "" {
+		ci, err := strconv.ParseUint(cq, 10, 64)
+		if err != nil {
+			return CodedError(http.StatusBadRequest, fmt.Sprintf("can not parse cas: %v", err))
+		}
+		rpc.SetCheckIndex(ci)
+	}
+	return nil
+}
+
+type CheckIndexSetter interface {
+	SetCheckIndex(uint64)
+}