diff --git a/e2e/.gitignore b/e2e/.gitignore index adad33a7b..cfc151d21 100644 --- a/e2e/.gitignore +++ b/e2e/.gitignore @@ -1,2 +1 @@ provisioning.json -csi/input/volumes.json diff --git a/e2e/csi/csi.go b/e2e/csi/csi.go index 4029e103d..81c35fb16 100644 --- a/e2e/csi/csi.go +++ b/e2e/csi/csi.go @@ -3,22 +3,22 @@ package csi import ( "bytes" "context" - "encoding/json" "io/ioutil" "os" "time" + "github.com/hashicorp/hcl" "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/e2e/e2eutil" "github.com/hashicorp/nomad/e2e/framework" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/stretchr/testify/require" ) type CSIVolumesTest struct { framework.TC - jobIds []string - volumeIDs *volumeConfig + jobIds []string } func init() { @@ -32,27 +32,8 @@ func init() { }) } -type volumeConfig struct { - EBSVolumeID string `json:"ebs_volume"` - EFSVolumeID string `json:"efs_volume"` -} - func (tc *CSIVolumesTest) BeforeAll(f *framework.F) { t := f.T() - // The volume IDs come from the external provider, so we need - // to read the configuration out of our Terraform output. - rawjson, err := ioutil.ReadFile("csi/input/volumes.json") - if err != nil { - t.Skip("volume ID configuration not found, try running 'terraform output volumes > ../csi/input/volumes.json'") - } - volumeIDs := &volumeConfig{} - err = json.Unmarshal(rawjson, volumeIDs) - if err != nil { - t.Fatal("volume ID configuration could not be read") - } - - tc.volumeIDs = volumeIDs - // Ensure cluster has leader and at least two client // nodes in a ready state before running tests e2eutil.WaitForLeader(t, tc.Nomad()) @@ -97,15 +78,9 @@ func (tc *CSIVolumesTest) TestEBSVolumeClaim(f *framework.F) { // register a volume volID := "ebs-vol0" - vol := &api.CSIVolume{ - ID: volID, - Name: volID, - ExternalID: tc.volumeIDs.EBSVolumeID, - AccessMode: "single-node-writer", - AttachmentMode: "file-system", - PluginID: "aws-ebs0", - } - _, err := nomadClient.CSIVolumes().Register(vol, nil) + vol, err := parseVolumeFile("csi/input/volume-ebs.hcl") + require.NoError(err) + _, err = nomadClient.CSIVolumes().Register(vol, nil) require.NoError(err) defer nomadClient.CSIVolumes().Deregister(volID, nil) @@ -136,7 +111,6 @@ func (tc *CSIVolumesTest) TestEBSVolumeClaim(f *framework.F) { "csi/input/use-ebs-volume.nomad", readJobID, "") readAllocID := readAllocs[0].ID e2eutil.WaitForAllocRunning(t, nomadClient, readAllocID) - // ensure we clean up claim before we deregister volumes defer nomadClient.Jobs().Deregister(readJobID, true, nil) @@ -179,15 +153,9 @@ func (tc *CSIVolumesTest) TestEFSVolumeClaim(f *framework.F) { // register a volume volID := "efs-vol0" - vol := &api.CSIVolume{ - ID: volID, - Name: volID, - ExternalID: tc.volumeIDs.EFSVolumeID, - AccessMode: "single-node-writer", - AttachmentMode: "file-system", - PluginID: "aws-efs0", - } - _, err := nomadClient.CSIVolumes().Register(vol, nil) + vol, err := parseVolumeFile("csi/input/volume-efs.hcl") + require.NoError(err) + _, err = nomadClient.CSIVolumes().Register(vol, nil) require.NoError(err) defer nomadClient.CSIVolumes().Deregister(volID, nil) @@ -249,3 +217,34 @@ func readFile(client *api.Client, alloc *api.Allocation, path string) (bytes.Buf make(chan api.TerminalSize), nil) return stdout, err } + +// TODO(tgross): this is taken from `nomad volume register` but +// it would be nice if we could expose this with a ParseFile as +// we do for api.Job. +func parseVolumeFile(filepath string) (*api.CSIVolume, error) { + + rawInput, err := ioutil.ReadFile(filepath) + if err != nil { + return nil, err + } + ast, err := hcl.Parse(string(rawInput)) + if err != nil { + return nil, err + } + + output := &api.CSIVolume{} + err = hcl.DecodeObject(output, ast) + if err != nil { + return nil, err + } + + // api.CSIVolume doesn't have the type field, it's used only for + // dispatch in parseVolumeType + helper.RemoveEqualFold(&output.ExtraKeysHCL, "type") + err = helper.UnusedKeys(output) + if err != nil { + return nil, err + } + + return output, nil +} diff --git a/e2e/csi/input/.gitignore b/e2e/csi/input/.gitignore new file mode 100644 index 000000000..ce9b9f225 --- /dev/null +++ b/e2e/csi/input/.gitignore @@ -0,0 +1,2 @@ +volume-ebs.hcl +volume-efs.hcl diff --git a/e2e/terraform/provisioning.tf b/e2e/terraform/provisioning.tf index 5d69b13b4..5e68d22ca 100644 --- a/e2e/terraform/provisioning.tf +++ b/e2e/terraform/provisioning.tf @@ -9,15 +9,6 @@ export NOMAD_E2E=1 EOM } -output "volumes" { - description = "get volume IDs needed to register volumes for CSI testing." - value = jsonencode( - { - "ebs_volume" : aws_ebs_volume.csi.id, - "efs_volume" : aws_efs_file_system.csi.id, - }) -} - output "provisioning" { description = "output to a file to be use w/ E2E framework -provision.terraform" value = jsonencode( diff --git a/e2e/terraform/volumes.tf b/e2e/terraform/volumes.tf index 324a430e6..966d041e5 100644 --- a/e2e/terraform/volumes.tf +++ b/e2e/terraform/volumes.tf @@ -22,3 +22,39 @@ resource "aws_ebs_volume" "csi" { User = data.aws_caller_identity.current.arn } } + +data "template_file" "ebs_volume_hcl" { + template = <