kemko/nomad
1
0
Fork 0
mirror of https://github.com/kemko/nomad.git synced 2026-01-06 10:25:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

csi: server-side plugin state tracking and api (#6966)

Browse Source 
* structs: CSIPlugin indexes jobs acting as plugins and node updates

* schema: csi_plugins table for CSIPlugin

* nomad: csi_endpoint use vol.Denormalize, plugin requests

* nomad: csi_volume_endpoint: rename to csi_endpoint

* agent: add CSI plugin endpoints

* state_store_test: use generated ids to avoid t.Parallel conflicts

* contributing: add note about registering new RPC structs

* command: agent http register plugin lists

* api: CSI plugin queries, ControllerHealthy -> ControllersHealthy

* state_store: copy on write for volumes and plugins

* structs: copy on write for volumes and plugins

* state_store: CSIVolumeByID returns an unhealthy volume, denormalize

* nomad: csi_endpoint use CSIVolumeDenormalizePlugins

* structs: remove struct errors for missing objects

* nomad: csi_endpoint return nil for missing objects, not errors

* api: return meta from Register to avoid EOF error

* state_store: CSIVolumeDenormalize keep allocs in their own maps

* state_store: CSIVolumeDeregister error on missing volume

* state_store: CSIVolumeRegister set indexes

* nomad: csi_endpoint use CSIVolumeDenormalizePlugins tests
This commit is contained in:
Lang Martin
2020-01-28 10:28:34 -05:00
committed by Tim Gross
parent 1250d56333
commit 15ffae2798
14 changed files with 1252 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
api/csi.go
View File

@@ -10,12 +10,12 @@ type CSIVolumes struct {
client *Client
}
// CSIVolumes returns a handle on the allocs endpoints.
// CSIVolumes returns a handle on the CSIVolumes endpoint
func (c *Client) CSIVolumes() *CSIVolumes {
return &CSIVolumes{client: c}
}
// List returns all CSI volumes, ignoring driver
// List returns all CSI volumes
func (v *CSIVolumes) List(q *QueryOptions) ([]*CSIVolumeListStub, *QueryMeta, error) {
var resp []*CSIVolumeListStub
qm, err := v.client.query("/v1/csi/volumes", &resp, q)
@@ -26,12 +26,12 @@ func (v *CSIVolumes) List(q *QueryOptions) ([]*CSIVolumeListStub, *QueryMeta, er
return resp, qm, nil
}
// DriverList returns all CSI volumes for the specified driver
func (v *CSIVolumes) DriverList(driver string) ([]*CSIVolumeListStub, *QueryMeta, error) {
return v.List(&QueryOptions{Prefix: driver})
// PluginList returns all CSI volumes for the specified plugin id
func (v *CSIVolumes) PluginList(pluginID string) ([]*CSIVolumeListStub, *QueryMeta, error) {
return v.List(&QueryOptions{Prefix: pluginID})
}
// Info is used to retrieve a single allocation.
// Info is used to retrieve a single CSIVolume
func (v *CSIVolumes) Info(id string, q *QueryOptions) (*CSIVolume, *QueryMeta, error) {
var resp CSIVolume
qm, err := v.client.query("/v1/csi/volume/"+id, &resp, q)
@@ -41,13 +41,12 @@ func (v *CSIVolumes) Info(id string, q *QueryOptions) (*CSIVolume, *QueryMeta, e
return &resp, qm, nil
}
func (v *CSIVolumes) Register(vol *CSIVolume, w *WriteOptions) error {
func (v *CSIVolumes) Register(vol *CSIVolume, w *WriteOptions) (*WriteMeta, error) {
req := CSIVolumeRegisterRequest{
Volumes: []*CSIVolume{vol},
}
var resp struct{}
_, err := v.client.write("/v1/csi/volume/"+vol.ID, req, &resp, w)
return err
meta, err := v.client.write("/v1/csi/volume/"+vol.ID, req, nil, w)
return meta, err
}
func (v *CSIVolumes) Deregister(id string, w *WriteOptions) error {
@@ -81,7 +80,6 @@ const (
// CSIVolume is used for serialization, see also nomad/structs/csi.go
type CSIVolume struct {
ID string
Driver string
Namespace string
Topologies []*CSITopology
AccessMode CSIVolumeAccessMode
@@ -92,16 +90,17 @@ type CSIVolume struct {
// Healthy is true iff all the denormalized plugin health fields are true, and the
// volume has not been marked for garbage collection
Healthy bool
VolumeGC time.Time
ControllerName string
ControllerHealthy bool
NodeHealthy int
NodeExpected int
ResourceExhausted time.Time
Healthy bool
VolumeGC time.Time
PluginID string
ControllersHealthy int
ControllersExpected int
NodesHealthy int
NodesExpected int
ResourceExhausted time.Time
CreatedIndex uint64
ModifiedIndex uint64
CreateIndex uint64
ModifyIndex uint64
}
type CSIVolumeIndexSort []*CSIVolumeListStub
@@ -111,7 +110,7 @@ func (v CSIVolumeIndexSort) Len() int {
}
func (v CSIVolumeIndexSort) Less(i, j int) bool {
return v[i].CreatedIndex > v[j].CreatedIndex
return v[i].CreateIndex > v[j].CreateIndex
}
func (v CSIVolumeIndexSort) Swap(i, j int) {
@@ -121,7 +120,6 @@ func (v CSIVolumeIndexSort) Swap(i, j int) {
// CSIVolumeListStub omits allocations. See also nomad/structs/csi.go
type CSIVolumeListStub struct {
ID string
Driver string
Namespace string
Topologies []*CSITopology
AccessMode CSIVolumeAccessMode
@@ -129,16 +127,17 @@ type CSIVolumeListStub struct {
// Healthy is true iff all the denormalized plugin health fields are true, and the
// volume has not been marked for garbage collection
Healthy bool
VolumeGC time.Time
ControllerName string
ControllerHealthy bool
NodeHealthy int
NodeExpected int
ResourceExhausted time.Time
Healthy bool
VolumeGC time.Time
PluginID string
ControllersHealthy int
ControllersExpected int
NodesHealthy int
NodesExpected int
ResourceExhausted time.Time
CreatedIndex uint64
ModifiedIndex uint64
CreateIndex uint64
ModifyIndex uint64
}
type CSIVolumeRegisterRequest struct {
@@ -150,3 +149,75 @@ type CSIVolumeDeregisterRequest struct {
VolumeIDs []string
WriteRequest
}
// CSI Plugins are jobs with plugin specific data
type CSIPlugins struct {
client *Client
}
type CSIPlugin struct {
ID string
Type CSIPluginType
Namespace string
Jobs map[string]map[string]*Job
ControllersHealthy int
Controllers map[string]*CSIInfo
NodesHealthy int
Nodes map[string]*CSIInfo
CreateIndex uint64
ModifyIndex uint64
}
type CSIPluginListStub struct {
ID string
Type CSIPluginType
JobIDs map[string]map[string]struct{}
ControllersHealthy int
ControllersExpected int
NodesHealthy int
NodesExpected int
CreateIndex uint64
ModifyIndex uint64
}
type CSIPluginIndexSort []*CSIPluginListStub
func (v CSIPluginIndexSort) Len() int {
return len(v)
}
func (v CSIPluginIndexSort) Less(i, j int) bool {
return v[i].CreateIndex > v[j].CreateIndex
}
func (v CSIPluginIndexSort) Swap(i, j int) {
v[i], v[j] = v[j], v[i]
}
// CSIPlugins returns a handle on the CSIPlugins endpoint
func (c *Client) CSIPlugins() *CSIPlugins {
return &CSIPlugins{client: c}
}
// List returns all CSI plugins
func (v *CSIPlugins) List(q *QueryOptions) ([]*CSIPluginListStub, *QueryMeta, error) {
var resp []*CSIPluginListStub
qm, err := v.client.query("/v1/csi/plugins", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(CSIPluginIndexSort(resp))
return resp, qm, nil
}
// Info is used to retrieve a single CSI Plugin Job
func (v *CSIPlugins) Info(id string, q *QueryOptions) (*CSIPlugin, *QueryMeta, error) {
var resp *CSIPlugin
qm, err := v.client.query("/v1/csi/plugin/"+id, &resp, q)
if err != nil {
return nil, nil, err
}
return resp, qm, nil
}

93
api/csi_test.go
View File

@@ -3,7 +3,7 @@ package api
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCSIVolumes_CRUD(t *testing.T) {
@@ -14,9 +14,9 @@ func TestCSIVolumes_CRUD(t *testing.T) {
// Successful empty result
vols, qm, err := v.List(nil)
assert.NoError(t, err)
assert.NotEqual(t, 0, qm.LastIndex)
assert.Equal(t, 0, len(vols))
require.NoError(t, err)
require.NotEqual(t, 0, qm.LastIndex)
require.Equal(t, 0, len(vols))
// Authorized QueryOpts. Use the root token to just bypass ACL details
opts := &QueryOptions{
@@ -32,38 +32,89 @@ func TestCSIVolumes_CRUD(t *testing.T) {
}
// Register a volume
v.Register(&CSIVolume{
ID: "DEADBEEF-63C7-407F-AE82-C99FBEF78FEB",
Driver: "minnie",
id := "DEADBEEF-31B5-8F78-7986-DD404FDA0CD1"
_, err = v.Register(&CSIVolume{
ID: id,
Namespace: "default",
PluginID: "adam",
AccessMode: CSIVolumeAccessModeMultiNodeSingleWriter,
AttachmentMode: CSIVolumeAttachmentModeFilesystem,
Topologies: []*CSITopology{{Segments: map[string]string{"foo": "bar"}}},
}, wpts)
require.NoError(t, err)
// Successful result with volumes
vols, qm, err = v.List(opts)
assert.NoError(t, err)
assert.NotEqual(t, 0, qm.LastIndex)
assert.Equal(t, 1, len(vols))
require.NoError(t, err)
require.NotEqual(t, 0, qm.LastIndex)
require.Equal(t, 1, len(vols))
// Successful info query
vol, qm, err := v.Info("DEADBEEF-63C7-407F-AE82-C99FBEF78FEB", opts)
assert.NoError(t, err)
assert.Equal(t, "minnie", vol.Driver)
assert.Equal(t, "bar", vol.Topologies[0].Segments["foo"])
vol, qm, err := v.Info(id, opts)
require.NoError(t, err)
require.Equal(t, "bar", vol.Topologies[0].Segments["foo"])
// Deregister the volume
err = v.Deregister("DEADBEEF-63C7-407F-AE82-C99FBEF78FEB", wpts)
assert.NoError(t, err)
err = v.Deregister(id, wpts)
require.NoError(t, err)
// Successful empty result
vols, qm, err = v.List(nil)
assert.NoError(t, err)
assert.NotEqual(t, 0, qm.LastIndex)
assert.Equal(t, 0, len(vols))
require.NoError(t, err)
require.NotEqual(t, 0, qm.LastIndex)
require.Equal(t, 0, len(vols))
// Failed info query
vol, qm, err = v.Info("DEADBEEF-63C7-407F-AE82-C99FBEF78FEB", opts)
assert.Error(t, err, "missing")
vol, qm, err = v.Info(id, opts)
require.Error(t, err, "missing")
}
func TestCSIPlugins_viaJob(t *testing.T) {
t.Parallel()
c, s, root := makeACLClient(t, nil, nil)
defer s.Stop()
p := c.CSIPlugins()
// Successful empty result
plugs, qm, err := p.List(nil)
require.NoError(t, err)
require.NotEqual(t, 0, qm.LastIndex)
require.Equal(t, 0, len(plugs))
// Authorized QueryOpts. Use the root token to just bypass ACL details
opts := &QueryOptions{
Region: "global",
Namespace: "default",
AuthToken: root.SecretID,
}
wpts := &WriteOptions{
Region: "global",
Namespace: "default",
AuthToken: root.SecretID,
}
// Register a plugin job
j := c.Jobs()
job := testJob()
job.Namespace = stringToPtr("default")
job.TaskGroups[0].Tasks[0].CSIPluginConfig = &TaskCSIPluginConfig{
ID: "foo",
Type: "monolith",
MountDir: "/not-empty",
}
_, _, err = j.Register(job, wpts)
require.NoError(t, err)
// Successful result with the plugin
plugs, qm, err = p.List(opts)
require.NoError(t, err)
require.NotEqual(t, 0, qm.LastIndex)
require.Equal(t, 1, len(plugs))
// Successful info query
plug, qm, err := p.Info("foo", opts)
require.NoError(t, err)
require.NotNil(t, plug.Jobs[*job.Namespace][*job.ID])
require.Equal(t, *job.ID, *plug.Jobs[*job.Namespace][*job.ID].ID)
}