From d627d4ea95e80d3074bb4fd858024c2ac238446d Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Wed, 30 Jan 2019 08:00:17 -0800 Subject: [PATCH] Install versioned release of codecgen Gets us one step closer to a reproducible build. Also removes the unused vendored version. --- GNUmakefile | 7 +- scripts/install-codecgen.sh | 12 + .../ugorji/go/codec/codecgen/README.md | 37 -- .../ugorji/go/codec/codecgen/gen.go | 324 ------------------ .../github.com/ugorji/go/codec/codecgen/z.go | 3 - vendor/vendor.json | 1 - 6 files changed, 17 insertions(+), 367 deletions(-) create mode 100644 scripts/install-codecgen.sh delete mode 100644 vendor/github.com/ugorji/go/codec/codecgen/README.md delete mode 100644 vendor/github.com/ugorji/go/codec/codecgen/gen.go delete mode 100644 vendor/github.com/ugorji/go/codec/codecgen/z.go diff --git a/GNUmakefile b/GNUmakefile index 7b190fe30..7d0ae185c 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -139,13 +139,13 @@ bootstrap: deps lint-deps # Install all dependencies deps: ## Install build and development dependencies @echo "==> Updating build dependencies..." go get -u github.com/kardianos/govendor - go get -u github.com/ugorji/go/codec/codecgen go get -u github.com/hashicorp/go-bindata/go-bindata go get -u github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs go get -u github.com/a8m/tree/cmd/tree go get -u github.com/magiconair/vendorfmt/cmd/vendorfmt - @bash -C "$(PROJECT_ROOT)/scripts/install-protoc-gen-go.sh" go get -u gotest.tools/gotestsum + @bash -C "$(PROJECT_ROOT)/scripts/install-codecgen.sh" + @bash -C "$(PROJECT_ROOT)/scripts/install-protoc-gen-go.sh" .PHONY: lint-deps lint-deps: ## Install linter dependencies @@ -190,10 +190,12 @@ generate-all: generate-structs proto .PHONY: generate-structs generate-structs: LOCAL_PACKAGES = $(shell go list ./... | grep -v '/vendor/') generate-structs: ## Update generated code + @echo "--> Running go generate..." @go generate $(LOCAL_PACKAGES) .PHONY: proto proto: + @echo "--> Generating proto bindings..." @for file in $$(git ls-files "*.proto" | grep -v "vendor\/.*.proto"); do \ protoc -I . -I ../../.. --go_out=plugins=grpc:. $$file; \ done @@ -280,6 +282,7 @@ clean: ## Remove build artifacts @rm -rf "$(PROJECT_ROOT)/bin/" @rm -rf "$(PROJECT_ROOT)/pkg/" @rm -f "$(GOPATH)/bin/nomad" + @find client nomad -name '*.generated.*' -delete .PHONY: travis travis: ## Run Nomad test suites with output to prevent timeouts under Travis CI diff --git a/scripts/install-codecgen.sh b/scripts/install-codecgen.sh new file mode 100644 index 000000000..4fb7212f6 --- /dev/null +++ b/scripts/install-codecgen.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +# Match entry in vendor.json +GIT_TAG="v1.1.2" +echo "Installing codec/codecgen@${GIT_TAG} ..." + +# Either fetch in existing git repo or use go get to clone +git -C "$(go env GOPATH)"/src/github.com/ugorji/go/codec fetch -q || go get -d -u github.com/ugorji/go/codec/codecgen +git -C "$(go env GOPATH)"/src/github.com/ugorji/go/codec checkout --quiet $GIT_TAG +go install github.com/ugorji/go/codec/codecgen diff --git a/vendor/github.com/ugorji/go/codec/codecgen/README.md b/vendor/github.com/ugorji/go/codec/codecgen/README.md deleted file mode 100644 index 854b64bfc..000000000 --- a/vendor/github.com/ugorji/go/codec/codecgen/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# codecgen tool - -Generate is given a list of *.go files to parse, and an output file (fout), -codecgen will create an output file __file.go__ which -contains `codec.Selfer` implementations for the named types found -in the files parsed. - -Using codecgen is very straightforward. - -**Download and install the tool** - -`go get -u github.com/ugorji/go/codec/codecgen` - -**Run the tool on your files** - -The command line format is: - -`codecgen [options] (-o outfile) (infile ...)` - -```sh -% codecgen -? -Usage of codecgen: - -c="github.com/ugorji/go/codec": codec path - -o="": out file - -r=".*": regex for type name to match - -nr="": regex for type name to exclude - -rt="": tags for go run - -t="": build tag to put in file - -u=false: Use unsafe, e.g. to avoid unnecessary allocation on []byte->string - -x=false: keep temp file - -% codecgen -o values_codecgen.go values.go values2.go moretypedefs.go -``` - -Please see the [blog article](http://ugorji.net/blog/go-codecgen) -for more information on how to use the tool. - diff --git a/vendor/github.com/ugorji/go/codec/codecgen/gen.go b/vendor/github.com/ugorji/go/codec/codecgen/gen.go deleted file mode 100644 index 346d780ee..000000000 --- a/vendor/github.com/ugorji/go/codec/codecgen/gen.go +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright (c) 2012-2015 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -// codecgen generates codec.Selfer implementations for a set of types. -package main - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "go/ast" - "go/build" - "go/parser" - "go/token" - "math/rand" - "os" - "os/exec" - "path/filepath" - "regexp" - "strconv" - "strings" - "text/template" - "time" -) - -const genCodecPkg = "codec1978" // keep this in sync with codec.genCodecPkg - -const genFrunMainTmpl = `//+build ignore - -package main -{{ if .Types }}import "{{ .ImportPath }}"{{ end }} -func main() { - {{ $.PackageName }}.CodecGenTempWrite{{ .RandString }}() -} -` - -// const genFrunPkgTmpl = `//+build codecgen -const genFrunPkgTmpl = ` -package {{ $.PackageName }} - -import ( - {{ if not .CodecPkgFiles }}{{ .CodecPkgName }} "{{ .CodecImportPath }}"{{ end }} - "os" - "reflect" - "bytes" - "strings" - "go/format" -) - -func CodecGenTempWrite{{ .RandString }}() { - fout, err := os.Create("{{ .OutFile }}") - if err != nil { - panic(err) - } - defer fout.Close() - var out bytes.Buffer - - var typs []reflect.Type -{{ range $index, $element := .Types }} - var t{{ $index }} {{ . }} - typs = append(typs, reflect.TypeOf(t{{ $index }})) -{{ end }} - {{ if not .CodecPkgFiles }}{{ .CodecPkgName }}.{{ end }}Gen(&out, "{{ .BuildTag }}", "{{ .PackageName }}", "{{ .RandString }}", {{ .UseUnsafe }}, {{ if not .CodecPkgFiles }}{{ .CodecPkgName }}.{{ end }}NewTypeInfos(strings.Split("{{ .StructTags }}", ",")), typs...) - bout, err := format.Source(out.Bytes()) - if err != nil { - fout.Write(out.Bytes()) - panic(err) - } - fout.Write(bout) -} - -` - -// Generate is given a list of *.go files to parse, and an output file (fout). -// -// It finds all types T in the files, and it creates 2 tmp files (frun). -// - main package file passed to 'go run' -// - package level file which calls *genRunner.Selfer to write Selfer impls for each T. -// We use a package level file so that it can reference unexported types in the package being worked on. -// Tool then executes: "go run __frun__" which creates fout. -// fout contains Codec(En|De)codeSelf implementations for every type T. -// -func Generate(outfile, buildTag, codecPkgPath string, uid int64, useUnsafe bool, goRunTag string, - st string, regexName *regexp.Regexp, notRegexName *regexp.Regexp, deleteTempFile bool, infiles ...string) (err error) { - // For each file, grab AST, find each type, and write a call to it. - if len(infiles) == 0 { - return - } - if outfile == "" || codecPkgPath == "" { - err = errors.New("outfile and codec package path cannot be blank") - return - } - if uid < 0 { - uid = -uid - } - if uid == 0 { - rr := rand.New(rand.NewSource(time.Now().UnixNano())) - uid = 101 + rr.Int63n(9777) - } - // We have to parse dir for package, before opening the temp file for writing (else ImportDir fails). - // Also, ImportDir(...) must take an absolute path. - lastdir := filepath.Dir(outfile) - absdir, err := filepath.Abs(lastdir) - if err != nil { - return - } - pkg, err := build.Default.ImportDir(absdir, build.AllowBinary) - if err != nil { - return - } - type tmplT struct { - CodecPkgName string - CodecImportPath string - ImportPath string - OutFile string - PackageName string - RandString string - BuildTag string - StructTags string - Types []string - CodecPkgFiles bool - UseUnsafe bool - } - tv := tmplT{ - CodecPkgName: genCodecPkg, - OutFile: outfile, - CodecImportPath: codecPkgPath, - BuildTag: buildTag, - UseUnsafe: useUnsafe, - RandString: strconv.FormatInt(uid, 10), - StructTags: st, - } - tv.ImportPath = pkg.ImportPath - if tv.ImportPath == tv.CodecImportPath { - tv.CodecPkgFiles = true - tv.CodecPkgName = "codec" - } else { - // HACK: always handle vendoring. It should be typically on in go 1.6, 1.7 - tv.ImportPath = stripVendor(tv.ImportPath) - } - astfiles := make([]*ast.File, len(infiles)) - for i, infile := range infiles { - if filepath.Dir(infile) != lastdir { - err = errors.New("in files must all be in same directory as outfile") - return - } - fset := token.NewFileSet() - astfiles[i], err = parser.ParseFile(fset, infile, nil, 0) - if err != nil { - return - } - if i == 0 { - tv.PackageName = astfiles[i].Name.Name - if tv.PackageName == "main" { - // codecgen cannot be run on types in the 'main' package. - // A temporary 'main' package must be created, and should reference the fully built - // package containing the types. - // Also, the temporary main package will conflict with the main package which already has a main method. - err = errors.New("codecgen cannot be run on types in the 'main' package") - return - } - } - } - - // keep track of types with selfer methods - // selferMethods := []string{"CodecEncodeSelf", "CodecDecodeSelf"} - selferEncTyps := make(map[string]bool) - selferDecTyps := make(map[string]bool) - for _, f := range astfiles { - for _, d := range f.Decls { - // if fd, ok := d.(*ast.FuncDecl); ok && fd.Recv != nil && fd.Recv.NumFields() == 1 { - if fd, ok := d.(*ast.FuncDecl); ok && fd.Recv != nil && len(fd.Recv.List) == 1 { - recvType := fd.Recv.List[0].Type - if ptr, ok := recvType.(*ast.StarExpr); ok { - recvType = ptr.X - } - if id, ok := recvType.(*ast.Ident); ok { - switch fd.Name.Name { - case "CodecEncodeSelf": - selferEncTyps[id.Name] = true - case "CodecDecodeSelf": - selferDecTyps[id.Name] = true - } - } - } - } - } - - // now find types - for _, f := range astfiles { - for _, d := range f.Decls { - if gd, ok := d.(*ast.GenDecl); ok { - for _, dd := range gd.Specs { - if td, ok := dd.(*ast.TypeSpec); ok { - // if len(td.Name.Name) == 0 || td.Name.Name[0] > 'Z' || td.Name.Name[0] < 'A' { - if len(td.Name.Name) == 0 { - continue - } - - // only generate for: - // struct: StructType - // primitives (numbers, bool, string): Ident - // map: MapType - // slice, array: ArrayType - // chan: ChanType - // do not generate: - // FuncType, InterfaceType, StarExpr (ptr), etc - switch td.Type.(type) { - case *ast.StructType, *ast.Ident, *ast.MapType, *ast.ArrayType, *ast.ChanType: - // only add to tv.Types iff - // - it matches per the -r parameter - // - it doesn't match per the -nr parameter - // - it doesn't have any of the Selfer methods in the file - if regexName.FindStringIndex(td.Name.Name) != nil && - notRegexName.FindStringIndex(td.Name.Name) == nil && - !selferEncTyps[td.Name.Name] && - !selferDecTyps[td.Name.Name] { - tv.Types = append(tv.Types, td.Name.Name) - } - } - } - } - } - } - } - - if len(tv.Types) == 0 { - return - } - - // we cannot use ioutil.TempFile, because we cannot guarantee the file suffix (.go). - // Also, we cannot create file in temp directory, - // because go run will not work (as it needs to see the types here). - // Consequently, create the temp file in the current directory, and remove when done. - - // frun, err = ioutil.TempFile("", "codecgen-") - // frunName := filepath.Join(os.TempDir(), "codecgen-"+strconv.FormatInt(time.Now().UnixNano(), 10)+".go") - - frunMainName := "codecgen-main-" + tv.RandString + ".generated.go" - frunPkgName := "codecgen-pkg-" + tv.RandString + ".generated.go" - if deleteTempFile { - defer os.Remove(frunMainName) - defer os.Remove(frunPkgName) - } - // var frunMain, frunPkg *os.File - if _, err = gen1(frunMainName, genFrunMainTmpl, &tv); err != nil { - return - } - if _, err = gen1(frunPkgName, genFrunPkgTmpl, &tv); err != nil { - return - } - - // remove outfile, so "go run ..." will not think that types in outfile already exist. - os.Remove(outfile) - - // execute go run frun - cmd := exec.Command("go", "run", "-tags="+goRunTag, frunMainName) //, frunPkg.Name()) - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - if err = cmd.Run(); err != nil { - err = fmt.Errorf("error running 'go run %s': %v, console: %s", - frunMainName, err, buf.Bytes()) - return - } - os.Stdout.Write(buf.Bytes()) - return -} - -func gen1(frunName, tmplStr string, tv interface{}) (frun *os.File, err error) { - os.Remove(frunName) - if frun, err = os.Create(frunName); err != nil { - return - } - defer frun.Close() - - t := template.New("") - if t, err = t.Parse(tmplStr); err != nil { - return - } - bw := bufio.NewWriter(frun) - if err = t.Execute(bw, tv); err != nil { - return - } - if err = bw.Flush(); err != nil { - return - } - return -} - -// copied from ../gen.go (keep in sync). -func stripVendor(s string) string { - // HACK: Misbehaviour occurs in go 1.5. May have to re-visit this later. - // if s contains /vendor/ OR startsWith vendor/, then return everything after it. - const vendorStart = "vendor/" - const vendorInline = "/vendor/" - if i := strings.LastIndex(s, vendorInline); i >= 0 { - s = s[i+len(vendorInline):] - } else if strings.HasPrefix(s, vendorStart) { - s = s[len(vendorStart):] - } - return s -} - -func main() { - o := flag.String("o", "", "out file") - c := flag.String("c", genCodecPath, "codec path") - t := flag.String("t", "", "build tag to put in file") - r := flag.String("r", ".*", "regex for type name to match") - nr := flag.String("nr", "^$", "regex for type name to exclude") - rt := flag.String("rt", "", "tags for go run") - st := flag.String("st", "codec,json", "struct tag keys to introspect") - x := flag.Bool("x", false, "keep temp file") - u := flag.Bool("u", false, "Use unsafe, e.g. to avoid unnecessary allocation on []byte->string") - d := flag.Int64("d", 0, "random identifier for use in generated code") - flag.Parse() - if err := Generate(*o, *t, *c, *d, *u, *rt, *st, - regexp.MustCompile(*r), regexp.MustCompile(*nr), !*x, flag.Args()...); err != nil { - fmt.Fprintf(os.Stderr, "codecgen error: %v\n", err) - os.Exit(1) - } -} diff --git a/vendor/github.com/ugorji/go/codec/codecgen/z.go b/vendor/github.com/ugorji/go/codec/codecgen/z.go deleted file mode 100644 index e120a4eb9..000000000 --- a/vendor/github.com/ugorji/go/codec/codecgen/z.go +++ /dev/null @@ -1,3 +0,0 @@ -package main - -const genCodecPath = "github.com/ugorji/go/codec" diff --git a/vendor/vendor.json b/vendor/vendor.json index 4ff3be734..d98e74cac 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -359,7 +359,6 @@ {"path":"github.com/tonnerre/golang-text","checksumSHA1":"t24KnvC9jRxiANVhpw2pqFpmEu8=","revision":"048ed3d792f7104850acbc8cfc01e5a6070f4c04","revisionTime":"2013-09-25T19:58:46Z"}, {"path":"github.com/tv42/httpunix","checksumSHA1":"2xcr/mhxBFlDjpxe/Mc2Wb4RGR8=","revision":"b75d8614f926c077e48d85f1f8f7885b758c6225","revisionTime":"2015-04-27T01:28:21Z"}, {"path":"github.com/ugorji/go/codec","checksumSHA1":"8G1zvpE4gTtWQRuP/x2HPVDmflo=","revision":"0053ebfd9d0ee06ccefbfe17072021e1d4acebee","revisionTime":"2017-06-20T06:01:02Z"}, - {"path":"github.com/ugorji/go/codec/codecgen","checksumSHA1":"OgParimNuU2CJqr3pcTympeQZUc=","revision":"5efa3251c7f7d05e5d9704a69a984ec9f1386a40","revisionTime":"2017-06-20T10:48:52Z"}, {"path":"github.com/ulikunitz/xz","checksumSHA1":"qgMa75aMGbkFY0jIqqqgVnCUoNA=","revision":"0c6b41e72360850ca4f98dc341fd999726ea007f","revisionTime":"2017-06-05T21:53:11Z"}, {"path":"github.com/ulikunitz/xz/internal/hash","checksumSHA1":"vjnTkzNrMs5Xj6so/fq0mQ6dT1c=","revision":"0c6b41e72360850ca4f98dc341fd999726ea007f","revisionTime":"2017-06-05T21:53:11Z"}, {"path":"github.com/ulikunitz/xz/internal/xlog","checksumSHA1":"m0pm57ASBK/CTdmC0ppRHO17mBs=","revision":"0c6b41e72360850ca4f98dc341fd999726ea007f","revisionTime":"2017-06-05T21:53:11Z"},