e2e: have TF write-out HCL for CSI volume registration (#7599)

e2e/.gitignore

@@ -1,2 +1 @@
 provisioning.json
-csi/input/volumes.json

e2e/csi/csi.go

@@ -3,22 +3,22 @@ package csi
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"io/ioutil"
 	"os"
 	"time"
 
+	"github.com/hashicorp/hcl"
 	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/e2e/e2eutil"
 	"github.com/hashicorp/nomad/e2e/framework"
+	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/stretchr/testify/require"
 )
 
 type CSIVolumesTest struct {
 	framework.TC
-	jobIds    []string
-	volumeIDs *volumeConfig
+	jobIds []string
 }
 
 func init() {
@@ -32,27 +32,8 @@ func init() {
 	})
 }
 
-type volumeConfig struct {
-	EBSVolumeID string `json:"ebs_volume"`
-	EFSVolumeID string `json:"efs_volume"`
-}
-
 func (tc *CSIVolumesTest) BeforeAll(f *framework.F) {
 	t := f.T()
-	// The volume IDs come from the external provider, so we need
-	// to read the configuration out of our Terraform output.
-	rawjson, err := ioutil.ReadFile("csi/input/volumes.json")
-	if err != nil {
-		t.Skip("volume ID configuration not found, try running 'terraform output volumes > ../csi/input/volumes.json'")
-	}
-	volumeIDs := &volumeConfig{}
-	err = json.Unmarshal(rawjson, volumeIDs)
-	if err != nil {
-		t.Fatal("volume ID configuration could not be read")
-	}
-
-	tc.volumeIDs = volumeIDs
-
 	// Ensure cluster has leader and at least two client
 	// nodes in a ready state before running tests
 	e2eutil.WaitForLeader(t, tc.Nomad())
@@ -97,15 +78,9 @@ func (tc *CSIVolumesTest) TestEBSVolumeClaim(f *framework.F) {
 
 	// register a volume
 	volID := "ebs-vol0"
-	vol := &api.CSIVolume{
-		ID:             volID,
-		Name:           volID,
-		ExternalID:     tc.volumeIDs.EBSVolumeID,
-		AccessMode:     "single-node-writer",
-		AttachmentMode: "file-system",
-		PluginID:       "aws-ebs0",
-	}
-	_, err := nomadClient.CSIVolumes().Register(vol, nil)
+	vol, err := parseVolumeFile("csi/input/volume-ebs.hcl")
+	require.NoError(err)
+	_, err = nomadClient.CSIVolumes().Register(vol, nil)
 	require.NoError(err)
 	defer nomadClient.CSIVolumes().Deregister(volID, nil)
 
@@ -136,7 +111,6 @@ func (tc *CSIVolumesTest) TestEBSVolumeClaim(f *framework.F) {
 		"csi/input/use-ebs-volume.nomad", readJobID, "")
 	readAllocID := readAllocs[0].ID
 	e2eutil.WaitForAllocRunning(t, nomadClient, readAllocID)
-
 	// ensure we clean up claim before we deregister volumes
 	defer nomadClient.Jobs().Deregister(readJobID, true, nil)
 
@@ -179,15 +153,9 @@ func (tc *CSIVolumesTest) TestEFSVolumeClaim(f *framework.F) {
 
 	// register a volume
 	volID := "efs-vol0"
-	vol := &api.CSIVolume{
-		ID:             volID,
-		Name:           volID,
-		ExternalID:     tc.volumeIDs.EFSVolumeID,
-		AccessMode:     "single-node-writer",
-		AttachmentMode: "file-system",
-		PluginID:       "aws-efs0",
-	}
-	_, err := nomadClient.CSIVolumes().Register(vol, nil)
+	vol, err := parseVolumeFile("csi/input/volume-efs.hcl")
+	require.NoError(err)
+	_, err = nomadClient.CSIVolumes().Register(vol, nil)
 	require.NoError(err)
 	defer nomadClient.CSIVolumes().Deregister(volID, nil)
 
@@ -249,3 +217,34 @@ func readFile(client *api.Client, alloc *api.Allocation, path string) (bytes.Buf
 		make(chan api.TerminalSize), nil)
 	return stdout, err
 }
+
+// TODO(tgross): this is taken from `nomad volume register` but
+// it would be nice if we could expose this with a ParseFile as
+// we do for api.Job.
+func parseVolumeFile(filepath string) (*api.CSIVolume, error) {
+
+	rawInput, err := ioutil.ReadFile(filepath)
+	if err != nil {
+		return nil, err
+	}
+	ast, err := hcl.Parse(string(rawInput))
+	if err != nil {
+		return nil, err
+	}
+
+	output := &api.CSIVolume{}
+	err = hcl.DecodeObject(output, ast)
+	if err != nil {
+		return nil, err
+	}
+
+	// api.CSIVolume doesn't have the type field, it's used only for
+	// dispatch in parseVolumeType
+	helper.RemoveEqualFold(&output.ExtraKeysHCL, "type")
+	err = helper.UnusedKeys(output)
+	if err != nil {
+		return nil, err
+	}
+
+	return output, nil
+}

e2e/csi/input/.gitignore

@@ -0,0 +1,2 @@
+volume-ebs.hcl
+volume-efs.hcl

e2e/terraform/provisioning.tf

@@ -9,15 +9,6 @@ export NOMAD_E2E=1
 EOM
 }
 
-output "volumes" {
-  description = "get volume IDs needed to register volumes for CSI testing."
-  value = jsonencode(
-    {
-      "ebs_volume" : aws_ebs_volume.csi.id,
-      "efs_volume" : aws_efs_file_system.csi.id,
-  })
-}
-
 output "provisioning" {
   description = "output to a file to be use w/ E2E framework -provision.terraform"
   value = jsonencode(

e2e/terraform/volumes.tf

@@ -22,3 +22,39 @@ resource "aws_ebs_volume" "csi" {
     User = data.aws_caller_identity.current.arn
   }
 }
+
+data "template_file" "ebs_volume_hcl" {
+  template = <<EOT
+type = "csi"
+id = "ebs-vol0"
+name = "ebs-vol0"
+external_id = "${aws_ebs_volume.csi.id}"
+access_mode = "single-node-writer"
+attachment_mode = "file-system"
+plugin_id = "aws-ebs0"
+EOT
+}
+
+data "template_file" "efs_volume_hcl" {
+  template = <<EOT
+type = "csi"
+id = "efs-vol0"
+name = "efs-vol0"
+external_id = "${aws_efs_file_system.csi.id}"
+access_mode = "single-node-writer"
+attachment_mode = "file-system"
+plugin_id = "aws-efs0"
+EOT
+}
+
+resource "local_file" "ebs_volume_hcl" {
+  content         = data.template_file.ebs_volume_hcl.rendered
+  filename        = "${path.module}/../csi/input/volume-ebs.hcl"
+  file_permission = "0664"
+}
+
+resource "local_file" "efs_volume_hcl" {
+  content         = data.template_file.efs_volume_hcl.rendered
+  filename        = "${path.module}/../csi/input/volume-efs.hcl"
+  file_permission = "0664"
+}