From 91075e130a8666493daad7a50f7cd523153d1f55 Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Sat, 19 Dec 2015 21:05:17 +0100 Subject: [PATCH] Short identifiers functionality * Use go-memdb prefix indexer for lookups * Add Job lookups * Update state store with new ByIDPrefix get methods * Call new methods when exact lookup fails or is not applicable --- command/monitor.go | 2 +- command/status.go | 4 +- nomad/alloc_endpoint.go | 38 ++++- nomad/eval_endpoint.go | 37 ++++- nomad/job_endpoint.go | 28 ++++ nomad/node_endpoint.go | 23 +++ nomad/state/state_store.go | 116 +++++++------ nomad/state/state_store_test.go | 277 ++++++++++++++++++++++++++++++++ 8 files changed, 456 insertions(+), 69 deletions(-) diff --git a/command/monitor.go b/command/monitor.go index be9b816dc..a6615c5a1 100644 --- a/command/monitor.go +++ b/command/monitor.go @@ -196,7 +196,7 @@ func (m *monitor) monitor(evalID string) int { state.index = eval.CreateIndex // Query the allocations associated with the evaluation - allocs, _, err := m.client.Evaluations().Allocations(evalID, nil) + allocs, _, err := m.client.Evaluations().Allocations(eval.ID, nil) if err != nil { m.ui.Error(fmt.Sprintf("Error reading allocations: %s", err)) return 1 diff --git a/command/status.go b/command/status.go index 4a736dc7a..ed7301639 100644 --- a/command/status.go +++ b/command/status.go @@ -106,14 +106,14 @@ func (c *StatusCommand) Run(args []string) int { var evals, allocs []string if !short { // Query the evaluations - jobEvals, _, err := client.Jobs().Evaluations(jobID, nil) + jobEvals, _, err := client.Jobs().Evaluations(job.ID, nil) if err != nil { c.Ui.Error(fmt.Sprintf("Error querying job evaluations: %s", err)) return 1 } // Query the allocations - jobAllocs, _, err := client.Jobs().Allocations(jobID, nil) + jobAllocs, _, err := client.Jobs().Allocations(job.ID, nil) if err != nil { c.Ui.Error(fmt.Sprintf("Error querying job allocations: %s", err)) return 1 diff --git a/nomad/alloc_endpoint.go b/nomad/alloc_endpoint.go index c07d5549d..da059f451 100644 --- a/nomad/alloc_endpoint.go +++ b/nomad/alloc_endpoint.go @@ -1,6 +1,7 @@ package nomad import ( + "fmt" "time" "github.com/armon/go-metrics" @@ -80,9 +81,40 @@ func (a *Alloc) GetAlloc(args *structs.AllocSpecificRequest, if err != nil { return err } - out, err := snap.AllocByID(args.AllocID) - if err != nil { - return err + + var out *structs.Allocation + + // Exact lookup if the identifier length is 36 (full UUID) + if len(args.AllocID) == 36 { + out, err = snap.AllocByID(args.AllocID) + if err != nil { + return err + } + } else { + iter, err := snap.AllocByIDPrefix(args.AllocID) + if err != nil { + return err + } + + // Gather all matching nodes + var allocs []*structs.Allocation + var allocIds []string + for { + raw := iter.Next() + if raw == nil { + break + } + alloc := raw.(*structs.Allocation) + allocIds = append(allocIds, alloc.ID) + allocs = append(allocs, alloc) + } + + if len(allocs) == 1 { + // Return unique allocation + out = allocs[0] + } else if len(allocs) > 1 { + return fmt.Errorf("Ambiguous identifier: %+v", allocIds) + } } // Setup the output diff --git a/nomad/eval_endpoint.go b/nomad/eval_endpoint.go index bc74e85f3..c9f68c0c6 100644 --- a/nomad/eval_endpoint.go +++ b/nomad/eval_endpoint.go @@ -38,9 +38,40 @@ func (e *Eval) GetEval(args *structs.EvalSpecificRequest, if err != nil { return err } - out, err := snap.EvalByID(args.EvalID) - if err != nil { - return err + + var out *structs.Evaluation + + // Exact lookup if the identifier length is 36 (full UUID) + if len(args.EvalID) == 36 { + out, err = snap.EvalByID(args.EvalID) + if err != nil { + return err + } + } else { + iter, err := snap.EvalByIDPrefix(args.EvalID) + if err != nil { + return err + } + + // Gather all matching evaluations + var evals []*structs.Evaluation + var evalIds []string + for { + raw := iter.Next() + if raw == nil { + break + } + eval := raw.(*structs.Evaluation) + evalIds = append(evalIds, eval.ID) + evals = append(evals, eval) + } + + if len(evals) == 1 { + // Return unique evaluation + out = evals[0] + } else if len(evals) > 1 { + return fmt.Errorf("Ambiguous identifier: %+v", evalIds) + } } // Setup the output diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 18da75268..f7a17fb22 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -218,6 +218,34 @@ func (j *Job) GetJob(args *structs.JobSpecificRequest, return err } + // Exact lookup failed so try a prefix based lookup + if out == nil { + iter, err := snap.JobByIDPrefix(args.JobID) + if err != nil { + return err + } + + // Gather all matching jobs + var jobs []*structs.Job + var jobIds []string + for { + raw := iter.Next() + if raw == nil { + break + } + job := raw.(*structs.Job) + jobIds = append(jobIds, job.ID) + jobs = append(jobs, job) + } + + if len(jobs) == 1 { + // Return unique match + out = jobs[0] + } else if len(jobs) > 1 { + return fmt.Errorf("Ambiguous identifier: %+v", jobIds) + } + } + // Setup the output reply.Job = out if out != nil { diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index 5bd600380..f65a9a096 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -304,6 +304,29 @@ func (n *Node) GetNode(args *structs.NodeSpecificRequest, return err } + if out == nil { + iter, err := snap.NodeByIDPrefix(args.NodeID) + if err != nil { + return err + } + + var nodes []*structs.Node + for { + raw := iter.Next() + if raw == nil { + break + } + node := raw.(*structs.Node) + nodes = append(nodes, node) + } + + if len(nodes) == 1 { + out = nodes[0] + } else { + return fmt.Errorf("Ambiguous identifier: %v", nodes) + } + } + // Setup the output reply.Node = out if out != nil { diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index c7e962c39..d15500346 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -241,30 +241,29 @@ func (s *StateStore) UpdateNodeDrain(index uint64, nodeID string, drain bool) er func (s *StateStore) NodeByID(nodeID string) (*structs.Node, error) { txn := s.db.Txn(false) - existing, err := txn.Find("nodes", "id", nodeID) + existing, err := txn.First("nodes", "id", nodeID) if err != nil { return nil, fmt.Errorf("node lookup failed: %v", err) } if existing != nil { - // Return exact match directly - if len(existing) == 1 { - return existing[0].(*structs.Node), nil - } - - // The results were ambiguous for the given node identifier. Return - // an error with possible options so that the user can try again with - // a more specific identifier. - var nodes []string - for _, result := range existing { - node := result.(*structs.Node) - nodes = append(nodes, node.ID) - } - return nil, fmt.Errorf("Ambiguous identifier: %v", nodes) + return existing.(*structs.Node), nil } return nil, nil } +// NodeByIDPrefix is used to lookup a node by partial ID +func (s *StateStore) NodeByIDPrefix(nodeID string) (memdb.ResultIterator, error) { + txn := s.db.Txn(false) + + iter, err := txn.Get("nodes", "id_prefix", nodeID) + if err != nil { + return nil, fmt.Errorf("node lookup failed: %v", err) + } + + return iter, nil +} + // Nodes returns an iterator over all the nodes func (s *StateStore) Nodes() (memdb.ResultIterator, error) { txn := s.db.Txn(false) @@ -349,30 +348,29 @@ func (s *StateStore) DeleteJob(index uint64, jobID string) error { func (s *StateStore) JobByID(id string) (*structs.Job, error) { txn := s.db.Txn(false) - existing, err := txn.Find("jobs", "id", id) + existing, err := txn.First("jobs", "id", id) if err != nil { return nil, fmt.Errorf("job lookup failed: %v", err) } if existing != nil { - // Return exact match directly - if len(existing) == 1 { - return existing[0].(*structs.Job), nil - } - - // The results were ambiguous for the given job identifier. Return - // an error with possible options so that the user can try again with - // a more specific identifier. - var jobs []string - for _, result := range existing { - job := result.(*structs.Job) - jobs = append(jobs, job.ID) - } - return nil, fmt.Errorf("Ambiguous identifier: %v", jobs) + return existing.(*structs.Job), nil } return nil, nil } +// JobByIDPrefix is used to lookup a job by partial ID +func (s *StateStore) JobByIDPrefix(id string) (memdb.ResultIterator, error) { + txn := s.db.Txn(false) + + iter, err := txn.Get("jobs", "id_prefix", id) + if err != nil { + return nil, fmt.Errorf("job lookup failed: %v", err) + } + + return iter, nil +} + // Jobs returns an iterator over all the jobs func (s *StateStore) Jobs() (memdb.ResultIterator, error) { txn := s.db.Txn(false) @@ -515,30 +513,29 @@ func (s *StateStore) DeleteEval(index uint64, evals []string, allocs []string) e func (s *StateStore) EvalByID(id string) (*structs.Evaluation, error) { txn := s.db.Txn(false) - existing, err := txn.Find("evals", "id", id) + existing, err := txn.First("evals", "id", id) if err != nil { return nil, fmt.Errorf("eval lookup failed: %v", err) } if existing != nil { - // Return exact match directly - if len(existing) == 1 { - return existing[0].(*structs.Evaluation), nil - } - - // The results were ambiguous for the given eval identifier. Return - // an error with possible options so that the user can try again with - // a more specific identifier. - var evals []string - for _, result := range existing { - eval := result.(*structs.Evaluation) - evals = append(evals, eval.ID) - } - return nil, fmt.Errorf("Ambiguous identifier: %v", evals) + return existing.(*structs.Evaluation), nil } return nil, nil } +// EvalByIDPrefix is used to lookup an eval by partial ID +func (s *StateStore) EvalByIDPrefix(id string) (memdb.ResultIterator, error) { + txn := s.db.Txn(false) + + iter, err := txn.Get("evals", "id_prefix", id) + if err != nil { + return nil, fmt.Errorf("eval lookup failed: %v", err) + } + + return iter, nil +} + // EvalsByJob returns all the evaluations by job id func (s *StateStore) EvalsByJob(jobID string) ([]*structs.Evaluation, error) { txn := s.db.Txn(false) @@ -677,30 +674,29 @@ func (s *StateStore) UpsertAllocs(index uint64, allocs []*structs.Allocation) er func (s *StateStore) AllocByID(id string) (*structs.Allocation, error) { txn := s.db.Txn(false) - existing, err := txn.Find("allocs", "id", id) + existing, err := txn.First("allocs", "id", id) if err != nil { return nil, fmt.Errorf("alloc lookup failed: %v", err) } if existing != nil { - // Return exact match directly - if len(existing) == 1 { - return existing[0].(*structs.Allocation), nil - } - - // The results were ambiguous for the given job identifier. Return - // an error with possible options so that the user can try again with - // a more specific identifier. - var allocs []string - for _, result := range existing { - alloc := result.(*structs.Allocation) - allocs = append(allocs, alloc.ID) - } - return nil, fmt.Errorf("Ambiguous identifier: %v", allocs) + return existing.(*structs.Allocation), nil } return nil, nil } +// AllocByIDPrefix is used to lookup an alloc by partial ID +func (s *StateStore) AllocByIDPrefix(id string) (memdb.ResultIterator, error) { + txn := s.db.Txn(false) + + iter, err := txn.Get("allocs", "id_prefix", id) + if err != nil { + return nil, fmt.Errorf("alloc lookup failed: %v", err) + } + + return iter, nil +} + // AllocsByNode returns all the allocations by node func (s *StateStore) AllocsByNode(node string) ([]*structs.Allocation, error) { txn := s.db.Txn(false) diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 0609f3048..2cd17cce4 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -6,6 +6,7 @@ import ( "sort" "testing" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/watch" @@ -215,6 +216,77 @@ func TestStateStore_Nodes(t *testing.T) { } } +func TestStateStore_NodeByIDPrefix(t *testing.T) { + state := testStateStore(t) + node := mock.Node() + + node.ID = "11111111-662e-d0ab-d1c9-3e434af7bdb4" + err := state.UpsertNode(1000, node) + if err != nil { + t.Fatalf("err: %v", err) + } + + iter, err := state.NodeByIDPrefix(node.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + + gatherNodes := func(iter memdb.ResultIterator) []*structs.Node { + var nodes []*structs.Node + for { + raw := iter.Next() + if raw == nil { + break + } + node := raw.(*structs.Node) + nodes = append(nodes, node) + } + return nodes + } + + nodes := gatherNodes(iter) + if len(nodes) != 1 { + t.Fatalf("err: %v", err) + } + + iter, err = state.NodeByIDPrefix("11") + if err != nil { + t.Fatalf("err: %v", err) + } + + nodes = gatherNodes(iter) + if len(nodes) != 1 { + t.Fatalf("err: %v", err) + } + + node = mock.Node() + node.ID = "11222222-662e-d0ab-d1c9-3e434af7bdb4" + err = state.UpsertNode(1001, node) + if err != nil { + t.Fatalf("err: %v", err) + } + + iter, err = state.NodeByIDPrefix("11") + if err != nil { + t.Fatalf("err: %v", err) + } + + nodes = gatherNodes(iter) + if len(nodes) != 2 { + t.Fatalf("err: %v", err) + } + + iter, err = state.NodeByIDPrefix("111") + if err != nil { + t.Fatalf("err: %v", err) + } + + nodes = gatherNodes(iter) + if len(nodes) != 1 { + t.Fatalf("err: %v", err) + } +} + func TestStateStore_RestoreNode(t *testing.T) { state := testStateStore(t) node := mock.Node() @@ -404,6 +476,76 @@ func TestStateStore_Jobs(t *testing.T) { } } +func TestStateStore_JobByIDPrefix(t *testing.T) { + state := testStateStore(t) + job := mock.Job() + + job.ID = "redis" + err := state.UpsertJob(1000, job) + if err != nil { + t.Fatalf("err: %v", err) + } + + iter, err := state.JobByIDPrefix(job.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + + gatherJobs := func(iter memdb.ResultIterator) []*structs.Job { + var jobs []*structs.Job + for { + raw := iter.Next() + if raw == nil { + break + } + jobs = append(jobs, raw.(*structs.Job)) + } + return jobs + } + + jobs := gatherJobs(iter) + if len(jobs) != 1 { + t.Fatalf("err: %v", err) + } + + iter, err = state.JobByIDPrefix("re") + if err != nil { + t.Fatalf("err: %v", err) + } + + jobs = gatherJobs(iter) + if len(jobs) != 1 { + t.Fatalf("err: %v", err) + } + + job = mock.Job() + job.ID = "riak" + err = state.UpsertJob(1001, job) + if err != nil { + t.Fatalf("err: %v", err) + } + + iter, err = state.JobByIDPrefix("r") + if err != nil { + t.Fatalf("err: %v", err) + } + + jobs = gatherJobs(iter) + if len(jobs) != 2 { + t.Fatalf("err: %v", err) + } + + iter, err = state.JobByIDPrefix("ri") + if err != nil { + t.Fatalf("err: %v", err) + } + + jobs = gatherJobs(iter) + if len(jobs) != 1 { + t.Fatalf("err: %v", err) + } +} + func TestStateStore_JobsByScheduler(t *testing.T) { state := testStateStore(t) var serviceJobs []*structs.Job @@ -859,6 +1001,74 @@ func TestStateStore_Evals(t *testing.T) { } } +func TestStateStore_EvalByIDPrefix(t *testing.T) { + state := testStateStore(t) + var evals []*structs.Evaluation + + ids := []string{ + "aaaaaaaa-7bfb-395d-eb95-0685af2176b2", + "aaaaaaab-7bfb-395d-eb95-0685af2176b2", + "aaaaaabb-7bfb-395d-eb95-0685af2176b2", + "aaaaabbb-7bfb-395d-eb95-0685af2176b2", + "aaaabbbb-7bfb-395d-eb95-0685af2176b2", + "aaabbbbb-7bfb-395d-eb95-0685af2176b2", + "aabbbbbb-7bfb-395d-eb95-0685af2176b2", + "abbbbbbb-7bfb-395d-eb95-0685af2176b2", + "bbbbbbbb-7bfb-395d-eb95-0685af2176b2", + } + for i := 0; i < 9; i++ { + eval := mock.Eval() + eval.ID = ids[i] + evals = append(evals, eval) + } + + err := state.UpsertEvals(1000, evals) + if err != nil { + t.Fatalf("err: %v", err) + } + + iter, err := state.EvalByIDPrefix("aaaa") + if err != nil { + t.Fatalf("err: %v", err) + } + + gatherEvals := func(iter memdb.ResultIterator) []*structs.Evaluation { + var evals []*structs.Evaluation + for { + raw := iter.Next() + if raw == nil { + break + } + evals = append(evals, raw.(*structs.Evaluation)) + } + return evals + } + + out := gatherEvals(iter) + if len(out) != 5 { + t.Fatalf("bad: expected five evaluations, got: %#v", out) + } + + sort.Sort(EvalIDSort(evals)) + + for index, eval := range out { + if ids[index] != eval.ID { + t.Fatalf("bad: got unexpected id: %s", eval.ID) + } + } + + iter, err = state.EvalByIDPrefix("b-a7bfb") + if err != nil { + t.Fatalf("err: %v", err) + } + + out = gatherEvals(iter) + if len(out) != 0 { + t.Fatalf("bad: unexpected zero evaluations, got: %#v", out) + } + +} + func TestStateStore_RestoreEval(t *testing.T) { state := testStateStore(t) eval := mock.Eval() @@ -1119,6 +1329,73 @@ func TestStateStore_AllocsByJob(t *testing.T) { } } +func TestStateStore_AllocByIDPrefix(t *testing.T) { + state := testStateStore(t) + var allocs []*structs.Allocation + + ids := []string{ + "aaaaaaaa-7bfb-395d-eb95-0685af2176b2", + "aaaaaaab-7bfb-395d-eb95-0685af2176b2", + "aaaaaabb-7bfb-395d-eb95-0685af2176b2", + "aaaaabbb-7bfb-395d-eb95-0685af2176b2", + "aaaabbbb-7bfb-395d-eb95-0685af2176b2", + "aaabbbbb-7bfb-395d-eb95-0685af2176b2", + "aabbbbbb-7bfb-395d-eb95-0685af2176b2", + "abbbbbbb-7bfb-395d-eb95-0685af2176b2", + "bbbbbbbb-7bfb-395d-eb95-0685af2176b2", + } + for i := 0; i < 9; i++ { + alloc := mock.Alloc() + alloc.ID = ids[i] + allocs = append(allocs, alloc) + } + + err := state.UpsertAllocs(1000, allocs) + if err != nil { + t.Fatalf("err: %v", err) + } + + iter, err := state.AllocByIDPrefix("aaaa") + if err != nil { + t.Fatalf("err: %v", err) + } + + gatherAllocs := func(iter memdb.ResultIterator) []*structs.Allocation { + var allocs []*structs.Allocation + for { + raw := iter.Next() + if raw == nil { + break + } + allocs = append(allocs, raw.(*structs.Allocation)) + } + return allocs + } + + out := gatherAllocs(iter) + if len(out) != 5 { + t.Fatalf("bad: expected five allocations, got: %#v", out) + } + + sort.Sort(AllocIDSort(allocs)) + + for index, alloc := range out { + if ids[index] != alloc.ID { + t.Fatalf("bad: got unexpected id: %s", alloc.ID) + } + } + + iter, err = state.AllocByIDPrefix("b-a7bfb") + if err != nil { + t.Fatalf("err: %v", err) + } + + out = gatherAllocs(iter) + if len(out) != 0 { + t.Fatalf("bad: unexpected zero allocations, got: %#v", out) + } +} + func TestStateStore_Allocs(t *testing.T) { state := testStateStore(t) var allocs []*structs.Allocation