diff --git a/GNUmakefile b/GNUmakefile index f60aac9d6..3e77c5a34 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -425,3 +425,8 @@ endif missing: ## Check for packages not being tested @echo "==> Checking for packages not being tested ..." @go run -modfile tools/go.mod tools/missing/main.go .github/workflows/test-core.yaml + +.PHONY: ec2info +ec2info: ## Generate AWS EC2 CPU specification table + @echo "==> Generating AWS EC2 specifications ..." + @go run -modfile tools/go.mod tools/ec2info/main.go diff --git a/tools/ec2info/aws.go b/tools/ec2info/aws.go deleted file mode 100644 index 6382885c5..000000000 --- a/tools/ec2info/aws.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "fmt" - "log" - "sort" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" -) - -func clientForRegion(region string) (*ec2.EC2, error) { - sess, err := session.NewSession(&aws.Config{ - Region: ®ion, - }) - if err != nil { - return nil, err - } - return ec2.New(sess), nil -} - -func getRegions(client *ec2.EC2) ([]*ec2.Region, error) { - all := false // beyond account access - regions, err := client.DescribeRegions(&ec2.DescribeRegionsInput{ - AllRegions: &all, - }) - if err != nil { - return nil, err - } - return regions.Regions, nil -} - -type specs struct { - Cores int - Speed float64 -} - -func (s specs) String() string { - return fmt.Sprintf("(%d %.2f)", s.Cores, s.Speed) -} - -func getData(regions []*ec2.Region, verbose bool) (map[string]map[string]specs, error) { - data := make(map[string]map[string]specs) - - for _, region := range regions { - rData, rProblems, err := getDataForRegion(*region.RegionName) - if err != nil { - return nil, err - } - data[*region.RegionName] = rData - - if verbose { - log.Println("region", *region.RegionName, "got data for", len(rData), "instance types", len(rProblems), "incomplete") - instanceProblems(rProblems) - } - } - - return data, nil -} - -func instanceProblems(problems map[string]string) { - types := make([]string, 0, len(problems)) - for k := range problems { - types = append(types, k) - } - sort.Strings(types) - for _, iType := range types { - log.Println(" ->", iType, problems[iType]) - } -} - -func getDataForRegion(region string) (map[string]specs, map[string]string, error) { - client, err := clientForRegion(region) - if err != nil { - return nil, nil, err - } - - data := make(map[string]specs) - problems := make(map[string]string) - regionInfoPage(client, true, region, nil, data, problems) - return data, problems, nil -} - -func regionInfoPage(client *ec2.EC2, first bool, region string, token *string, data map[string]specs, problems map[string]string) { - if first || token != nil { - output, err := client.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ - NextToken: token, - }) - if err != nil { - log.Fatal(err) - } - - // recursively accumulate each page of data - regionInfoAccumulate(output, data, problems) - regionInfoPage(client, false, region, output.NextToken, data, problems) - } -} - -func regionInfoAccumulate(output *ec2.DescribeInstanceTypesOutput, data map[string]specs, problems map[string]string) { - for _, iType := range output.InstanceTypes { - switch { - - case iType.ProcessorInfo == nil: - fallthrough - case iType.ProcessorInfo.SustainedClockSpeedInGhz == nil: - problems[*iType.InstanceType] = "missing clock Speed" - continue - - case iType.VCpuInfo == nil: - fallthrough - case iType.VCpuInfo.DefaultVCpus == nil: - problems[*iType.InstanceType] = "missing virtual cpu Cores" - continue - - default: - data[*iType.InstanceType] = specs{ - Speed: *iType.ProcessorInfo.SustainedClockSpeedInGhz, - Cores: int(*iType.VCpuInfo.DefaultVCpus), - } - continue - } - } -} diff --git a/tools/ec2info/main.go b/tools/ec2info/main.go index 8e2f17294..aa707735d 100644 --- a/tools/ec2info/main.go +++ b/tools/ec2info/main.go @@ -6,29 +6,24 @@ // // Requires AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN. // -// Options -// --package : configure package name of generated output file -// --region : configure initial region from which to lookup all regions -// --outfile : configure filepath of generated output file -// --verbose : print log messages while running +// Usage (invoke from Nomad's makefile) // -// Usage -// $ go run . +// make ec2info package main import ( - "flag" + "fmt" + "io" "log" -) + "os" + "os/exec" + "sort" + "text/template" -func args() (string, string, string, bool) { - pkg := flag.String("package", "fingerprint", "generate package name") - region := flag.String("region", "us-west-1", "initial region for listing regions") - outfile := flag.String("output", "../../client/fingerprint/env_aws_cpu.go", "output filepath") - verbose := flag.Bool("verbose", true, "print extra information while running") - flag.Parse() - return *pkg, *region, *outfile, *verbose -} + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" +) func check(err error) { if err != nil { @@ -37,7 +32,7 @@ func check(err error) { } func main() { - pkg, region, output, verbose := args() + pkg, region, output := "fingerprint", "us-west-1", "client/fingerprint/env_aws_cpu.go" client, err := clientForRegion(region) check(err) @@ -45,7 +40,7 @@ func main() { regions, err := getRegions(client) check(err) - data, err := getData(regions, verbose) + data, err := getData(regions) check(err) flat := flatten(data) @@ -59,3 +54,157 @@ func main() { check(write(f, flat, pkg)) check(format(output)) } + +func clientForRegion(region string) (*ec2.EC2, error) { + sess, err := session.NewSession(&aws.Config{ + Region: ®ion, + }) + if err != nil { + return nil, err + } + return ec2.New(sess), nil +} + +func getRegions(client *ec2.EC2) ([]*ec2.Region, error) { + all := false // beyond account access + regions, err := client.DescribeRegions(&ec2.DescribeRegionsInput{ + AllRegions: &all, + }) + if err != nil { + log.Println("failed to create AWS session; make sure environment is setup") + log.Println("must have environment variables AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN") + log.Println("or ~/.aws/credentials configured properly") + return nil, err + } + return regions.Regions, nil +} + +type specs struct { + Cores int + Speed float64 +} + +func (s specs) String() string { + return fmt.Sprintf("(%d %.2f)", s.Cores, s.Speed) +} + +func getData(regions []*ec2.Region) (map[string]map[string]specs, error) { + data := make(map[string]map[string]specs) + + for _, region := range regions { + rData, rProblems, err := getDataForRegion(*region.RegionName) + if err != nil { + return nil, err + } + data[*region.RegionName] = rData + + log.Println("region", *region.RegionName, "got data for", len(rData), "instance types", len(rProblems), "incomplete") + instanceProblems(rProblems) + } + + return data, nil +} + +func instanceProblems(problems map[string]string) { + types := make([]string, 0, len(problems)) + for k := range problems { + types = append(types, k) + } + sort.Strings(types) + for _, iType := range types { + log.Println(" ->", iType, problems[iType]) + } +} + +func getDataForRegion(region string) (map[string]specs, map[string]string, error) { + client, err := clientForRegion(region) + if err != nil { + return nil, nil, err + } + + data := make(map[string]specs) + problems := make(map[string]string) + regionInfoPage(client, true, region, nil, data, problems) + return data, problems, nil +} + +func regionInfoPage(client *ec2.EC2, first bool, region string, token *string, data map[string]specs, problems map[string]string) { + if first || token != nil { + output, err := client.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ + NextToken: token, + }) + if err != nil { + log.Fatal(err) + } + + // recursively accumulate each page of data + regionInfoAccumulate(output, data, problems) + regionInfoPage(client, false, region, output.NextToken, data, problems) + } +} + +func regionInfoAccumulate(output *ec2.DescribeInstanceTypesOutput, data map[string]specs, problems map[string]string) { + for _, iType := range output.InstanceTypes { + switch { + + case iType.ProcessorInfo == nil: + fallthrough + case iType.ProcessorInfo.SustainedClockSpeedInGhz == nil: + problems[*iType.InstanceType] = "missing clock Speed" + continue + + case iType.VCpuInfo == nil: + fallthrough + case iType.VCpuInfo.DefaultVCpus == nil: + problems[*iType.InstanceType] = "missing virtual cpu Cores" + continue + + default: + data[*iType.InstanceType] = specs{ + Speed: *iType.ProcessorInfo.SustainedClockSpeedInGhz, + Cores: int(*iType.VCpuInfo.DefaultVCpus), + } + continue + } + } +} + +// open the output file for writing. +func open(output string) (io.ReadWriteCloser, error) { + return os.Create(output) +} + +// flatten region data, assuming instance type is the same across regions. +func flatten(data map[string]map[string]specs) map[string]specs { + result := make(map[string]specs) + for _, m := range data { + for iType, specifications := range m { + result[iType] = specifications + } + } + return result +} + +type Template struct { + Package string + Data map[string]specs +} + +// write the data using the cpu_table.go.template to w. +func write(w io.Writer, data map[string]specs, pkg string) error { + tmpl, err := template.ParseFiles("tools/ec2info/cpu_table.go.template") + if err != nil { + return err + } + return tmpl.Execute(w, Template{ + Package: pkg, + Data: data, + }) +} + +// format the file using gofmt. +func format(file string) error { + cmd := exec.Command("gofmt", "-w", file) + _, err := cmd.CombinedOutput() + return err +} diff --git a/tools/ec2info/output.go b/tools/ec2info/output.go deleted file mode 100644 index 75d27be27..000000000 --- a/tools/ec2info/output.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "io" - "os" - "os/exec" - "text/template" -) - -// open the output file for writing. -func open(output string) (io.ReadWriteCloser, error) { - return os.Create(output) -} - -// flatten region data, assuming instance type is the same across regions. -func flatten(data map[string]map[string]specs) map[string]specs { - result := make(map[string]specs) - for _, m := range data { - for iType, specs := range m { - result[iType] = specs - } - } - return result -} - -type Template struct { - Package string - Data map[string]specs -} - -// write the data using the cpu_table.go.template to w. -func write(w io.Writer, data map[string]specs, pkg string) error { - tmpl, err := template.ParseFiles("cpu_table.go.template") - if err != nil { - return err - } - return tmpl.Execute(w, Template{ - Package: pkg, - Data: data, - }) -} - -// format the file using gofmt. -func format(file string) error { - cmd := exec.Command("gofmt", "-w", file) - _, err := cmd.CombinedOutput() - return err -}