ci: use groups of tests in gha (#15018)

* [no ci] use json for grouping packages for testing

* [no ci] able to get packages in group

* [no ci] able to run groups of tests

* [no ci] more

* [no ci] try disable circle unit tests

* ci: use actions/checkout@v3

* ci: rename to quick

* ci: need make dev in mods cache step

* ci: make compile step depend on checks step

* ci: bump consul and vault versions

* ci: need make dev for group tests

* ci: update ci unit testing docs

* docs: spell plumbing correctly

Co-authored-by: Tim Gross <tgross@hashicorp.com>

Co-authored-by: Tim Gross <tgross@hashicorp.com>

tools/go.mod

@@ -4,13 +4,11 @@ go 1.18
 
 require (
 	github.com/aws/aws-sdk-go v1.37.26
-	github.com/stretchr/testify v1.7.1
-	gopkg.in/yaml.v2 v2.2.8
+	github.com/hashicorp/go-set v0.1.6
+	github.com/shoenig/test v0.4.0
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )

tools/go.sum

@@ -2,6 +2,10 @@ github.com/aws/aws-sdk-go v1.37.26 h1:D9Qvyjlr6xFR0CspZ0imdASc5Y1WE/Sgyte4l+cUp4
 github.com/aws/aws-sdk-go v1.37.26/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/hashicorp/go-set v0.1.6 h1:fj/JG5B97sAOd9OQN4GL880yCE384fz3asNDpGbxkPo=
+github.com/hashicorp/go-set v0.1.6/go.mod h1:ELvMcE+1mRHYPVgTFSQiecObIwZRxY5Q11EhbtmM5KQ=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -9,9 +13,9 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/shoenig/test v0.4.0 h1:3X4xG/Chx7mzi0h71Sm6Vo38q0EYaQIBZpYFRcA1HVM=
+github.com/shoenig/test v0.4.0/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -22,9 +26,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tools/missing/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -10,7 +11,7 @@ import (
 	"sort"
 	"strings"
 
-	"gopkg.in/yaml.v2"
+	"github.com/hashicorp/go-set"
 )
 
 func main() {
@@ -20,21 +21,28 @@ func main() {
 	}
 }
 
-type YamlFile struct {
-	Jobs struct {
-		TestPackages struct {
-			Strategy struct {
-				Matrix struct {
-					Packages []string `yaml:"pkg"`
-				} `yaml:"matrix"`
-			} `yaml:"strategy"`
-		} `yaml:"tests-pkgs"`
-	} `yaml:"jobs"`
+// Manifest represents groupings of packages for testing
+// see: ci/test-core.json
+type Manifest map[string][]string
+
+func (m Manifest) covers(pkg string) bool {
+	for _, list := range m {
+		if isCovered(list, pkg) {
+			return true
+		}
+	}
+	return false
 }
 
+const (
+	verify = 1
+	group  = 2
+)
+
 func run(args []string) error {
-	if len(args) != 1 {
-		return errors.New("requires filename")
+	mode := len(args)
+	if !(mode == verify || mode == group) {
+		return errors.New("usage: [filename] <group>")
 	}
 
 	f, err := os.Open(args[0])
@@ -45,11 +53,22 @@ func run(args []string) error {
 		_ = f.Close()
 	}()
 
-	coverage, err := inMatrix(f)
+	manifest, err := getManifest(f)
 	if err != nil {
 		return err
 	}
 
+	switch mode {
+	case verify:
+		return runVerify(manifest)
+	case group:
+		return runGroups(manifest, args[1])
+	default:
+		panic("oops")
+	}
+}
+
+func runVerify(manifest Manifest) error {
 	packages, err := inCode(".")
 	if err != nil {
 		return err
@@ -57,7 +76,7 @@ func run(args []string) error {
 
 	var isMissing []string
 	for _, pkg := range packages {
-		if !isCovered(coverage, pkg) {
+		if !manifest.covers(pkg) {
 			isMissing = append(isMissing, pkg)
 		}
 	}
@@ -74,7 +93,17 @@ func run(args []string) error {
 	return nil
 }
 
-// isCovered returns true if pkg is covered by a package in coverage.
+func runGroups(manifest Manifest, group string) error {
+	list := manifest[group]
+	for i := 0; i < len(list); i++ {
+		list[i] = "./" + list[i]
+	}
+	s := strings.Join(list, " ")
+	fmt.Print(s)
+	return nil
+}
+
+// isCovered returns true if pkg is tested by directory in manifest.
 func isCovered(coverage []string, pkg string) bool {
 	for _, p := range coverage {
 		if isCoveredOne(p, pkg) {
@@ -101,19 +130,14 @@ func isCoveredOne(p string, pkg string) bool {
 	return false
 }
 
-func inMatrix(r io.Reader) ([]string, error) {
-	var yFile YamlFile
-	if err := yaml.NewDecoder(r).Decode(&yFile); err != nil {
+func getManifest(r io.Reader) (Manifest, error) {
+	m := make(Manifest)
+	if err := json.NewDecoder(r).Decode(&m); err != nil {
 		return nil, err
 	}
-	p := yFile.Jobs.TestPackages.Strategy.Matrix.Packages
-	return p, nil
+	return m, nil
 }
 
-type nothing struct{}
-
-var null = nothing{}
-
 // uninteresting lists remaining packages that contain Go code but still
 // do not need to be covered by test cases.
 var uninteresting = []string{
@@ -147,7 +171,7 @@ func skip(p string) bool {
 }
 
 func inCode(root string) ([]string, error) {
-	m := map[string]nothing{}
+	pkgs := set.New[string](100)
 
 	err := filepath.Walk(root, func(path string, info fs.FileInfo, err error) error {
 		if info.IsDir() {
@@ -159,7 +183,7 @@ func inCode(root string) ([]string, error) {
 		}
 
 		if ext := filepath.Ext(path); ext == ".go" {
-			m[filepath.Dir(path)] = null
+			pkgs.Insert(filepath.Dir(path))
 		}
 
 		return nil
@@ -168,12 +192,9 @@ func inCode(root string) ([]string, error) {
 		return nil, err
 	}
 
-	delete(m, ".") // package main
+	pkgs.Remove(".") // main
 
-	var packages []string
-	for p := range m {
-		packages = append(packages, p)
-	}
+	packages := pkgs.List()
 	sort.Strings(packages)
 	return packages, nil
 }

tools/missing/main_test.go

@@ -3,13 +3,13 @@ package main
 import (
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	"github.com/shoenig/test/must"
 )
 
 func Test_isCoveredOne(t *testing.T) {
 	try := func(p string, exp bool) {
 		result := isCoveredOne(p, "foo/bar")
-		require.Equal(t, exp, result)
+		must.Eq(t, exp, result)
 	}
 	try("baz", false)
 	try("foo", false)