diff --git a/api/agent.go b/api/agent.go index 884d44688..5ae71ca72 100644 --- a/api/agent.go +++ b/api/agent.go @@ -319,8 +319,7 @@ func (a *Agent) CPUProfile(serverID, nodeID string, seconds int, q *QueryOptions return nil, err } - var resp []byte - resp, err = ioutil.ReadAll(body) + resp, err := ioutil.ReadAll(body) if err != nil { return nil, err } @@ -357,8 +356,7 @@ func (a *Agent) Trace(serverID, nodeID string, seconds int, q *QueryOptions) ([] return nil, err } - var resp []byte - resp, err = ioutil.ReadAll(body) + resp, err := ioutil.ReadAll(body) if err != nil { return nil, err } @@ -395,8 +393,7 @@ func (a *Agent) Profile(serverID, nodeID, profile string, debug int, q *QueryOpt return nil, err } - var resp []byte - resp, err = ioutil.ReadAll(body) + resp, err := ioutil.ReadAll(body) if err != nil { return nil, err } diff --git a/client/agent_endpoint.go b/client/agent_endpoint.go index d57fc9498..0618cc7e7 100644 --- a/client/agent_endpoint.go +++ b/client/agent_endpoint.go @@ -8,7 +8,7 @@ import ( "time" "github.com/hashicorp/nomad/command/agent/monitor" - "github.com/hashicorp/nomad/command/agent/profile" + "github.com/hashicorp/nomad/command/agent/pprof" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" "github.com/ugorji/go/codec" @@ -39,10 +39,8 @@ func (a *Agent) Profile(args *structs.AgentPprofRequest, reply *structs.AgentPpr } // If ACLs are disabled, EnableDebug must be enabled - if aclObj == nil { - if !a.c.config.EnableDebug { - return structs.ErrPermissionDenied - } + if aclObj == nil && !a.c.config.EnableDebug { + return structs.ErrPermissionDenied } var resp []byte @@ -53,18 +51,18 @@ func (a *Agent) Profile(args *structs.AgentPprofRequest, reply *structs.AgentPpr // Our RPC endpoints currently don't support context // or request cancellation so stubbing with TODO switch args.ReqType { - case profile.CPUReq: - resp, headers, err = profile.CPUProfile(context.TODO(), args.Seconds) - case profile.CmdReq: - resp, headers, err = profile.Cmdline() - case profile.LookupReq: - resp, headers, err = profile.Profile(args.Profile, args.Debug) - case profile.TraceReq: - resp, headers, err = profile.Trace(context.TODO(), args.Seconds) + case pprof.CPUReq: + resp, headers, err = pprof.CPUProfile(context.TODO(), args.Seconds) + case pprof.CmdReq: + resp, headers, err = pprof.Cmdline() + case pprof.LookupReq: + resp, headers, err = pprof.Profile(args.Profile, args.Debug, args.GC) + case pprof.TraceReq: + resp, headers, err = pprof.Trace(context.TODO(), args.Seconds) } if err != nil { - if profile.IsErrProfileNotFound(err) { + if pprof.IsErrProfileNotFound(err) { return structs.NewErrRPCCoded(404, err.Error()) } return structs.NewErrRPCCoded(500, err.Error()) diff --git a/client/agent_endpoint_test.go b/client/agent_endpoint_test.go index 20010ff99..6d391824a 100644 --- a/client/agent_endpoint_test.go +++ b/client/agent_endpoint_test.go @@ -13,7 +13,7 @@ import ( "github.com/hashicorp/nomad/client/config" sframer "github.com/hashicorp/nomad/client/lib/streamframer" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/command/agent/profile" + "github.com/hashicorp/nomad/command/agent/pprof" "github.com/hashicorp/nomad/nomad" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -232,7 +232,7 @@ func TestAgentProfile_DefaultDisabled(t *testing.T) { defer cleanupC() req := structs.AgentPprofRequest{ - ReqType: profile.CPUReq, + ReqType: pprof.CPUReq, NodeID: c.NodeID(), } @@ -261,7 +261,7 @@ func TestAgentProfile(t *testing.T) { // Successful request { req := structs.AgentPprofRequest{ - ReqType: profile.CPUReq, + ReqType: pprof.CPUReq, NodeID: c.NodeID(), } @@ -277,7 +277,7 @@ func TestAgentProfile(t *testing.T) { // Unknown profile request { req := structs.AgentPprofRequest{ - ReqType: profile.LookupReq, + ReqType: pprof.LookupReq, Profile: "unknown", NodeID: c.NodeID(), } @@ -333,7 +333,7 @@ func TestAgentProfile_ACL(t *testing.T) { for _, tc := range cases { t.Run(tc.Name, func(t *testing.T) { req := &structs.AgentPprofRequest{ - ReqType: profile.CmdReq, + ReqType: pprof.CmdReq, QueryOptions: structs.QueryOptions{ Namespace: structs.DefaultNamespace, Region: "global", diff --git a/command/agent/agent_endpoint.go b/command/agent/agent_endpoint.go index b1e266dd6..e0b4df19f 100644 --- a/command/agent/agent_endpoint.go +++ b/command/agent/agent_endpoint.go @@ -16,7 +16,7 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/acl" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/command/agent/profile" + "github.com/hashicorp/nomad/command/agent/pprof" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/serf/serf" "github.com/mitchellh/copystructure" @@ -341,11 +341,11 @@ func (s *HTTPServer) AgentPprofRequest(resp http.ResponseWriter, req *http.Reque // no root index route return nil, CodedError(404, ErrInvalidMethod) case "cmdline": - return s.agentPprof(profile.CmdReq, resp, req) + return s.agentPprof(pprof.CmdReq, resp, req) case "profile": - return s.agentPprof(profile.CPUReq, resp, req) + return s.agentPprof(pprof.CPUReq, resp, req) case "trace": - return s.agentPprof(profile.TraceReq, resp, req) + return s.agentPprof(pprof.TraceReq, resp, req) default: // Add profile to request values := req.URL.Query() @@ -353,26 +353,20 @@ func (s *HTTPServer) AgentPprofRequest(resp http.ResponseWriter, req *http.Reque req.URL.RawQuery = values.Encode() // generic pprof profile request - return s.agentPprof(profile.LookupReq, resp, req) + return s.agentPprof(pprof.LookupReq, resp, req) } } -func (s *HTTPServer) agentPprof(reqType profile.ReqType, resp http.ResponseWriter, req *http.Request) ([]byte, error) { - var secret string - s.parseToken(req, &secret) +func (s *HTTPServer) agentPprof(reqType pprof.ReqType, resp http.ResponseWriter, req *http.Request) ([]byte, error) { - // Parse profile duration, default to 1 second - var err error - secondsParam := req.URL.Query().Get("seconds") - var seconds int - if secondsParam == "" { + // Parse query param int values + seconds, _ := strconv.Atoi(req.URL.Query().Get("seconds")) + debug, _ := strconv.Atoi(req.URL.Query().Get("debug")) + gc, _ := strconv.Atoi(req.URL.Query().Get("gc")) + + // default to 1 second + if seconds == 0 { seconds = 1 - } else { - seconds, err = strconv.Atoi(secondsParam) - if err != nil { - errStr := fmt.Sprintf("Error parsing seconds parameter %s", secondsParam) - return nil, CodedError(400, errStr) - } } // Create the request @@ -380,6 +374,8 @@ func (s *HTTPServer) agentPprof(reqType profile.ReqType, resp http.ResponseWrite NodeID: req.URL.Query().Get("node_id"), Profile: req.URL.Query().Get("profile"), ServerID: req.URL.Query().Get("server_id"), + Debug: debug, + GC: gc, ReqType: reqType, Seconds: seconds, } diff --git a/command/agent/profile/pprof.go b/command/agent/pprof/pprof.go similarity index 87% rename from command/agent/profile/pprof.go rename to command/agent/pprof/pprof.go index 51bbc81ff..868e85e8b 100644 --- a/command/agent/profile/pprof.go +++ b/command/agent/pprof/pprof.go @@ -1,10 +1,16 @@ -package profile +// Package profile is meant to be a near identical implemenation of +// https://golang.org/src/net/http/pprof/pprof.go +// It's purpose is to provide a way to accommodate the RPC endpoint style +// we use instead of traditional http handlers. + +package pprof import ( "bytes" "context" "fmt" "os" + "runtime" "runtime/pprof" "runtime/trace" "strings" @@ -17,7 +23,7 @@ const ( CmdReq ReqType = "cmdline" CPUReq ReqType = "cpu" TraceReq ReqType = "trace" - LookupReq ReqType = "profile" + LookupReq ReqType = "lookup" ErrProfileNotFoundPrefix = "Pprof profile not found profile:" ) @@ -49,12 +55,16 @@ func Cmdline() ([]byte, map[string]string, error) { // Profile generates a pprof.Profile report for the given profile name // see runtime/pprof/pprof.go for available profiles. -func Profile(profile string, debug int) ([]byte, map[string]string, error) { +func Profile(profile string, debug, gc int) ([]byte, map[string]string, error) { p := pprof.Lookup(profile) if p == nil { return nil, nil, NewErrProfileNotFound(profile) } + if profile == "heap" && gc > 0 { + runtime.GC() + } + var buf bytes.Buffer if err := p.WriteTo(&buf, debug); err != nil { return nil, nil, err diff --git a/command/agent/profile/pprof_test.go b/command/agent/pprof/pprof_test.go similarity index 96% rename from command/agent/profile/pprof_test.go rename to command/agent/pprof/pprof_test.go index 103ee373b..cf68fc7a7 100644 --- a/command/agent/profile/pprof_test.go +++ b/command/agent/pprof/pprof_test.go @@ -1,4 +1,4 @@ -package profile +package pprof import ( "context" @@ -12,6 +12,7 @@ func TestProfile(t *testing.T) { desc string profile string debug int + gc int expectedHeaders map[string]string expectedErr error }{ @@ -43,7 +44,7 @@ func TestProfile(t *testing.T) { for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { - resp, headers, err := Profile(tc.profile, tc.debug) + resp, headers, err := Profile(tc.profile, tc.debug, tc.gc) require.Equal(t, tc.expectedHeaders, headers) if tc.expectedErr != nil { diff --git a/nomad/client_agent_endpoint.go b/nomad/client_agent_endpoint.go index d27b57226..ca2d00229 100644 --- a/nomad/client_agent_endpoint.go +++ b/nomad/client_agent_endpoint.go @@ -13,7 +13,7 @@ import ( sframer "github.com/hashicorp/nomad/client/lib/streamframer" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/command/agent/monitor" - "github.com/hashicorp/nomad/command/agent/profile" + "github.com/hashicorp/nomad/command/agent/pprof" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" @@ -85,20 +85,20 @@ func (a *Agent) Profile(args *structs.AgentPprofRequest, reply *structs.AgentPpr // or request cancellation so using server shutdownCtx as a // best effort. switch args.ReqType { - case profile.CPUReq: - resp, headers, err = profile.CPUProfile(a.srv.shutdownCtx, args.Seconds) - case profile.CmdReq: - resp, headers, err = profile.Cmdline() - case profile.LookupReq: - resp, headers, err = profile.Profile(args.Profile, args.Debug) - case profile.TraceReq: - resp, headers, err = profile.Trace(a.srv.shutdownCtx, args.Seconds) + case pprof.CPUReq: + resp, headers, err = pprof.CPUProfile(a.srv.shutdownCtx, args.Seconds) + case pprof.CmdReq: + resp, headers, err = pprof.Cmdline() + case pprof.LookupReq: + resp, headers, err = pprof.Profile(args.Profile, args.Debug, args.GC) + case pprof.TraceReq: + resp, headers, err = pprof.Trace(a.srv.shutdownCtx, args.Seconds) default: err = structs.NewErrRPCCoded(404, "Unknown profile request type") } if err != nil { - if profile.IsErrProfileNotFound(err) { + if pprof.IsErrProfileNotFound(err) { return structs.NewErrRPCCoded(404, err.Error()) } return structs.NewErrRPCCoded(500, err.Error()) diff --git a/nomad/client_agent_endpoint_test.go b/nomad/client_agent_endpoint_test.go index bc9e92c20..931d03a60 100644 --- a/nomad/client_agent_endpoint_test.go +++ b/nomad/client_agent_endpoint_test.go @@ -15,7 +15,7 @@ import ( "github.com/hashicorp/nomad/client/config" sframer "github.com/hashicorp/nomad/client/lib/streamframer" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/command/agent/profile" + "github.com/hashicorp/nomad/command/agent/pprof" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -492,7 +492,7 @@ func TestAgentProfile_RemoteClient(t *testing.T) { }) req := structs.AgentPprofRequest{ - ReqType: profile.CPUReq, + ReqType: pprof.CPUReq, NodeID: c.NodeID(), QueryOptions: structs.QueryOptions{Region: "global"}, } @@ -531,7 +531,7 @@ func TestAgentProfile_RemoteRegionMisMatch(t *testing.T) { testutil.WaitForLeader(t, s1.RPC) req := structs.AgentPprofRequest{ - ReqType: profile.CPUReq, + ReqType: pprof.CPUReq, ServerID: s1.serf.LocalMember().Name, QueryOptions: structs.QueryOptions{ Region: "bar", @@ -568,7 +568,7 @@ func TestAgentProfile_RemoteRegion(t *testing.T) { testutil.WaitForLeader(t, s1.RPC) req := structs.AgentPprofRequest{ - ReqType: profile.CPUReq, + ReqType: pprof.CPUReq, ServerID: s2.serf.LocalMember().Name, QueryOptions: structs.QueryOptions{ Region: "bar", @@ -621,41 +621,41 @@ func TestAgentProfile_Server(t *testing.T) { origin *Server expectedErr string expectedAgentID string - reqType profile.ReqType + reqType pprof.ReqType }{ { desc: "remote leader", serverID: "leader", origin: nonLeader, - reqType: profile.CmdReq, + reqType: pprof.CmdReq, expectedAgentID: leader.serf.LocalMember().Name, }, { desc: "remote server", serverID: nonLeader.serf.LocalMember().Name, origin: leader, - reqType: profile.CmdReq, + reqType: pprof.CmdReq, expectedAgentID: nonLeader.serf.LocalMember().Name, }, { desc: "serverID is current leader", serverID: "leader", origin: leader, - reqType: profile.CmdReq, + reqType: pprof.CmdReq, expectedAgentID: leader.serf.LocalMember().Name, }, { desc: "serverID is current server", serverID: nonLeader.serf.LocalMember().Name, origin: nonLeader, - reqType: profile.CPUReq, + reqType: pprof.CPUReq, expectedAgentID: nonLeader.serf.LocalMember().Name, }, { desc: "serverID is unknown", serverID: uuid.Generate(), origin: nonLeader, - reqType: profile.CmdReq, + reqType: pprof.CmdReq, expectedErr: "unknown nomad server", expectedAgentID: "", }, @@ -724,7 +724,7 @@ func TestAgentProfile_ACL(t *testing.T) { for _, tc := range cases { t.Run(tc.Name, func(t *testing.T) { req := &structs.AgentPprofRequest{ - ReqType: profile.CmdReq, + ReqType: pprof.CmdReq, QueryOptions: structs.QueryOptions{ Namespace: structs.DefaultNamespace, Region: "global", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 98e934d4a..eb8ff2627 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -28,7 +28,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-version" "github.com/hashicorp/nomad/acl" - "github.com/hashicorp/nomad/command/agent/profile" + "github.com/hashicorp/nomad/command/agent/pprof" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/args" "github.com/hashicorp/nomad/helper/constraints/semver" @@ -266,7 +266,7 @@ func (q QueryOptions) AllowStaleRead() bool { // AgentPprofRequest is used to request a pprof report for a given node. type AgentPprofRequest struct { // ReqType specifies the profile to use - ReqType profile.ReqType + ReqType pprof.ReqType // Profile specifies the runtime/pprof profile to lookup and generate. Profile string @@ -277,6 +277,10 @@ type AgentPprofRequest struct { // Debug specifies if pprof profile should inclue debug output Debug int + // GC specifies if the profile should call runtime.GC() before + // running its profile. This is only used for "heap" profiles + GC int + // NodeID is the node we want to track the logs of NodeID string