diff --git a/plugins/shared/util.go b/plugins/shared/util.go new file mode 100644 index 000000000..252180ddd --- /dev/null +++ b/plugins/shared/util.go @@ -0,0 +1,69 @@ +package shared + +import ( + "bytes" + "fmt" + + hjson "github.com/hashicorp/hcl2/hcl/json" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/ugorji/go/codec" + + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcldec" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" + "github.com/zclconf/go-cty/cty/function/stdlib" +) + +// ParseHclInterface is used to convert an interface value representing a hcl2 +// body and return the interpolated value. +func ParseHclInterface(val interface{}, spec hcldec.Spec, ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { + // Encode to json + var buf bytes.Buffer + enc := codec.NewEncoder(&buf, structs.JsonHandle) + err := enc.Encode(val) + if err != nil { + // Convert to a hcl diagnostics message + return cty.NilVal, hcl.Diagnostics([]*hcl.Diagnostic{ + { + Severity: hcl.DiagError, + Summary: "Failed to JSON encode value", + Detail: fmt.Sprintf("JSON encoding failed: %v", err), + }}) + } + + // Parse the json as hcl2 + hclFile, diag := hjson.Parse(buf.Bytes(), "") + if diag.HasErrors() { + return cty.NilVal, diag + } + + value, decDiag := hcldec.Decode(hclFile.Body, spec, ctx) + diag = diag.Extend(decDiag) + if diag.HasErrors() { + return cty.NilVal, diag + } + + return value, diag +} + +// GetStdlibFuncs returns the set of stdlib functions. +func GetStdlibFuncs() map[string]function.Function { + return map[string]function.Function{ + "abs": stdlib.AbsoluteFunc, + "coalesce": stdlib.CoalesceFunc, + "concat": stdlib.ConcatFunc, + "hasindex": stdlib.HasIndexFunc, + "int": stdlib.IntFunc, + "jsondecode": stdlib.JSONDecodeFunc, + "jsonencode": stdlib.JSONEncodeFunc, + "length": stdlib.LengthFunc, + "lower": stdlib.LowerFunc, + "max": stdlib.MaxFunc, + "min": stdlib.MinFunc, + "reverse": stdlib.ReverseFunc, + "strlen": stdlib.StrlenFunc, + "substr": stdlib.SubstrFunc, + "upper": stdlib.UpperFunc, + } +} diff --git a/plugins/shared/util_test.go b/plugins/shared/util_test.go new file mode 100644 index 000000000..75b8ab38d --- /dev/null +++ b/plugins/shared/util_test.go @@ -0,0 +1,692 @@ +package shared + +import ( + "testing" + + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl2/gohcl" + hcl2 "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcldec" + "github.com/hashicorp/hcl2/hclparse" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/kr/pretty" + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/require" + "github.com/ugorji/go/codec" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/gocty" +) + +/* + +Martin suggests writing a function that takes a spec and map[string]interface{} +and essentially fixes the []map[string]interface{} -> map[string]interface{} +*/ + +var ( + dockerSpec hcldec.Spec = hcldec.ObjectSpec(map[string]hcldec.Spec{ + "image": &hcldec.AttrSpec{ + Name: "image", + Type: cty.String, + Required: true, + }, + "args": &hcldec.AttrSpec{ + Name: "args", + Type: cty.List(cty.String), + }, + "pids_limit": &hcldec.AttrSpec{ + Name: "pids_limit", + Type: cty.Number, + }, + "port_map": &hcldec.AttrSpec{ + Name: "port_map", + + // This should be a block. cty.Map(cty.String) + Type: cty.List(cty.Map(cty.String)), + }, + + "devices": &hcldec.BlockListSpec{ + TypeName: "devices", + Nested: hcldec.ObjectSpec(map[string]hcldec.Spec{ + "host_path": &hcldec.AttrSpec{ + Name: "host_path", + Type: cty.String, + }, + "container_path": &hcldec.AttrSpec{ + Name: "container_path", + Type: cty.String, + }, + "cgroup_permissions": &hcldec.DefaultSpec{ + Primary: &hcldec.AttrSpec{ + Name: "cgroup_permissions", + Type: cty.String, + }, + Default: &hcldec.LiteralSpec{ + Value: cty.StringVal(""), + }, + }, + }), + }, + }, + ) +) + +type dockerConfig struct { + Image string `cty:"image"` + Args []string `cty:"args"` + PidsLimit *int64 `cty:"pids_limit"` + PortMap []map[string]string `cty:"port_map"` + Devices []DockerDevice `cty:"devices"` +} + +type DockerDevice struct { + HostPath string `cty:"host_path"` + ContainerPath string `cty:"container_path"` + CgroupPermissions string `cty:"cgroup_permissions"` +} + +func hclConfigToInterface(t *testing.T, config string) interface{} { + t.Helper() + + // Parse as we do in the jobspec parser + root, err := hcl.Parse(config) + if err != nil { + t.Fatalf("failed to hcl parse the config: %v", err) + } + + // Top-level item should be a list + list, ok := root.Node.(*ast.ObjectList) + if !ok { + t.Fatalf("root should be an object") + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, list.Items[0]); err != nil { + t.Fatalf("failed to decode object: %v", err) + } + + var m2 map[string]interface{} + if err := mapstructure.WeakDecode(m, &m2); err != nil { + t.Fatalf("failed to weak decode object: %v", err) + } + + return m2["config"] +} + +func jsonConfigToInterface(t *testing.T, config string) interface{} { + t.Helper() + + // Decode from json + dec := codec.NewDecoderBytes([]byte(config), structs.JsonHandle) + + var m map[string]interface{} + err := dec.Decode(&m) + if err != nil { + t.Fatalf("failed to decode: %v", err) + } + + return m["Config"] +} + +func TestParseHclInterface_Hcl(t *testing.T) { + defaultCtx := &hcl2.EvalContext{ + Functions: GetStdlibFuncs(), + } + variableCtx := &hcl2.EvalContext{ + Functions: GetStdlibFuncs(), + Variables: map[string]cty.Value{ + "NOMAD_ALLOC_INDEX": cty.NumberIntVal(2), + "NOMAD_META_hello": cty.StringVal("world"), + }, + } + + // XXX Useful for determining what cty thinks the type is + //implied, err := gocty.ImpliedType(&dockerConfig{}) + //if err != nil { + //t.Fatalf("implied type failed: %v", err) + //} + + //t.Logf("Implied type: %v", implied.GoString()) + + cases := []struct { + name string + config interface{} + spec hcldec.Spec + ctx *hcl2.EvalContext + expected interface{} + expectedType interface{} + }{ + { + name: "single string attr", + config: hclConfigToInterface(t, ` + config { + image = "redis:3.2" + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + { + name: "single string attr json", + config: jsonConfigToInterface(t, ` + { + "Config": { + "image": "redis:3.2" + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + // ------------------------------------------------ + { + name: "number attr", + config: hclConfigToInterface(t, ` + config { + image = "redis:3.2" + pids_limit = 2 + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + PidsLimit: helper.Int64ToPtr(2), + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + { + name: "number attr json", + config: jsonConfigToInterface(t, ` + { + "Config": { + "image": "redis:3.2", + "pids_limit": "2" + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + PidsLimit: helper.Int64ToPtr(2), + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + // ------------------------------------------------ + { + name: "number attr interpolated", + config: hclConfigToInterface(t, ` + config { + image = "redis:3.2" + pids_limit = "${2 + 2}" + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + PidsLimit: helper.Int64ToPtr(4), + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + { + name: "number attr interploated json", + config: jsonConfigToInterface(t, ` + { + "Config": { + "image": "redis:3.2", + "pids_limit": "${2 + 2}" + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + PidsLimit: helper.Int64ToPtr(4), + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + // ------------------------------------------------ + { + name: "multi attr", + config: hclConfigToInterface(t, ` + config { + image = "redis:3.2" + args = ["foo", "bar"] + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + Args: []string{"foo", "bar"}, + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + { + name: "multi attr json", + config: jsonConfigToInterface(t, ` + { + "Config": { + "image": "redis:3.2", + "args": ["foo", "bar"] + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + Args: []string{"foo", "bar"}, + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + // ------------------------------------------------ + { + name: "multi attr variables", + config: hclConfigToInterface(t, ` + config { + image = "redis:3.2" + args = ["${NOMAD_META_hello}", "${NOMAD_ALLOC_INDEX}"] + pids_limit = "${NOMAD_ALLOC_INDEX + 2}" + }`), + spec: dockerSpec, + ctx: variableCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + Args: []string{"world", "2"}, + PidsLimit: helper.Int64ToPtr(4), + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + { + name: "multi attr variables json", + config: jsonConfigToInterface(t, ` + { + "Config": { + "image": "redis:3.2", + "args": ["foo", "bar"] + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + Args: []string{"foo", "bar"}, + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + // ------------------------------------------------ + { + name: "port_map", + config: hclConfigToInterface(t, ` + config { + image = "redis:3.2" + port_map { + foo = "db" + bar = "db2" + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + PortMap: []map[string]string{ + { + "foo": "db", + "bar": "db2", + }}, + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + { + name: "port_map json", + config: jsonConfigToInterface(t, ` + { + "Config": { + "image": "redis:3.2", + "port_map": [{ + "foo": "db", + "bar": "db2" + }] + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + PortMap: []map[string]string{ + { + "foo": "db", + "bar": "db2", + }}, + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + // ------------------------------------------------ + /* + { + name: "port_map non-list json", + config: jsonConfigToInterface(t, ` + { + "Config": { + "image": "redis:3.2", + "port_map": { + "foo": "db", + "bar": "db2" + } + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + PortMap: []map[string]string{ + { + "foo": "db", + "bar": "db2", + }}, + Devices: []DockerDevice{}, + }, + expectedType: &dockerConfig{}, + }, + */ + // ------------------------------------------------ + { + name: "devices", + config: hclConfigToInterface(t, ` + config { + image = "redis:3.2" + devices = [ + { + host_path = "/dev/sda1" + container_path = "/dev/xvdc" + cgroup_permissions = "r" + }, + { + host_path = "/dev/sda2" + container_path = "/dev/xvdd" + } + ] + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + Devices: []DockerDevice{ + { + HostPath: "/dev/sda1", + ContainerPath: "/dev/xvdc", + CgroupPermissions: "r", + }, + { + HostPath: "/dev/sda2", + ContainerPath: "/dev/xvdd", + }, + }, + }, + expectedType: &dockerConfig{}, + }, + { + name: "devices json", + config: jsonConfigToInterface(t, ` + { + "Config": { + "image": "redis:3.2", + "devices": [ + { + "host_path": "/dev/sda1", + "container_path": "/dev/xvdc", + "cgroup_permissions": "r" + }, + { + "host_path": "/dev/sda2", + "container_path": "/dev/xvdd" + } + ] + } + }`), + spec: dockerSpec, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + Devices: []DockerDevice{ + { + HostPath: "/dev/sda1", + ContainerPath: "/dev/xvdc", + CgroupPermissions: "r", + }, + { + HostPath: "/dev/sda2", + ContainerPath: "/dev/xvdd", + }, + }, + }, + expectedType: &dockerConfig{}, + }, + // ------------------------------------------------ + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Logf("Val: % #v", pretty.Formatter(c.config)) + // Parse the interface + ctyValue, diag := ParseHclInterface(c.config, c.spec, c.ctx) + if diag.HasErrors() { + for _, err := range diag.Errs() { + t.Error(err) + } + t.FailNow() + } + + // Convert cty-value to go structs + require.NoError(t, gocty.FromCtyValue(ctyValue, c.expectedType)) + + require.EqualValues(t, c.expected, c.expectedType) + + }) + } +} + +// ------------------------------------------------------------------------- + +var ( + dockerSpec2 hcldec.Spec = hcldec.ObjectSpec(map[string]hcldec.Spec{ + "image": &hcldec.AttrSpec{ + Name: "image", + Type: cty.String, + Required: true, + }, + "args": &hcldec.AttrSpec{ + Name: "args", + Type: cty.List(cty.String), + }, + //"port_map": &hcldec.AttrSpec{ + //Name: "port_map", + //Type: cty.List(cty.Map(cty.String)), + //}, + + //"devices": &hcldec.AttrSpec{ + //Name: "devices", + //Type: cty.List(cty.Object(map[string]cty.Type{ + //"host_path": cty.String, + //"container_path": cty.String, + //"cgroup_permissions": cty.String, + //})), + //Type: cty.Tuple([]cty.Type{cty.Object(map[string]cty.Type{ + //"host_path": cty.String, + //"container_path": cty.String, + //"cgroup_permissions": cty.String, + //})}), + //}, + }, + ) +) + +func configToHcl2Interface(t *testing.T, config string) interface{} { + t.Helper() + + // Parse as we do in the jobspec parser + file, diag := hclparse.NewParser().ParseHCL([]byte(config), "config") + if diag.HasErrors() { + t.Fatalf("failed to hcl parse the config: %v", diag.Error()) + } + + //t.Logf("Body: % #v", pretty.Formatter(file.Body)) + + var c struct { + m map[string]interface{} + } + implied, partial := gohcl.ImpliedBodySchema(&c) + t.Logf("partial=%v implied=% #v", partial, pretty.Formatter(implied)) + + contents, diag := file.Body.Content(implied) + if diag.HasErrors() { + t.Fatalf("failed to get contents: %v", diag.Error()) + } + + t.Fatalf("content=% #v", pretty.Formatter(contents)) + + //defaultCtx := &hcl2.EvalContext{ + //Functions: GetStdlibFuncs(), + //} + return nil +} + +func TestParseHclInterface_Hcl2(t *testing.T) { + t.SkipNow() + + defaultCtx := &hcl2.EvalContext{ + Functions: GetStdlibFuncs(), + } + + cases := []struct { + name string + config string + spec hcldec.Spec + ctx *hcl2.EvalContext + expected interface{} + expectedType interface{} + }{ + { + name: "single attr", + config: ` + image = "redis:3.2" + `, + spec: dockerSpec2, + ctx: defaultCtx, + expected: &dockerConfig{ + Image: "redis:3.2", + }, + expectedType: &dockerConfig{}, + }, + //{ + //name: "multi attr", + //config: ` + //config { + //image = "redis:3.2" + //args = ["foo", "bar"] + //}`, + //spec: dockerSpec2, + //ctx: defaultCtx, + //expected: &dockerConfig{ + //Image: "redis:3.2", + //Args: []string{"foo", "bar"}, + //}, + //expectedType: &dockerConfig{}, + //}, + //{ + //name: "port_map", + //config: ` + //config { + //image = "redis:3.2" + //port_map { + //foo = "db" + //} + //}`, + //spec: dockerSpec, + //ctx: defaultCtx, + //expected: &dockerConfig{ + //Image: "redis:3.2", + //PortMap: []map[string]string{{"foo": "db"}}, + //}, + //expectedType: &dockerConfig{}, + //}, + + //{ + //name: "devices", + //config: ` + //config { + //image = "redis:3.2" + //devices = [ + //{ + //host_path = "/dev/sda1" + //container_path = "/dev/xvdc" + //cgroup_permissions = "r" + //}, + //{ + //host_path = "/dev/sda2" + //container_path = "/dev/xvdd" + //} + //] + //}`, + //spec: dockerSpec, + //ctx: defaultCtx, + //expected: &dockerConfig{ + //Image: "redis:3.2", + //Args: []string{"foo", "bar"}, + //Devices: []DockerDevice{ + //{ + //HostPath: "/dev/sda1", + //ContainerPath: "/dev/xvdc", + //CgroupPermissions: "r", + //}, + //{ + //HostPath: "/dev/sda2", + //ContainerPath: "/dev/xvdd", + //}, + //}, + //}, + //expectedType: &dockerConfig{}, + //}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + // Convert the config to a value + v := configToHcl2Interface(t, c.config) + t.Logf("value: % #v", pretty.Formatter(v)) + + // Parse the interface + ctyValue, diag := ParseHclInterface(v, c.spec, c.ctx) + if diag.HasErrors() { + for _, err := range diag.Errs() { + t.Error(err) + } + t.FailNow() + } + + // Convert cty-value to go structs + require.NoError(t, gocty.FromCtyValue(ctyValue, c.expectedType)) + + require.EqualValues(t, c.expected, c.expectedType) + + }) + } +}