From 8cf34bde62be8beba07bb7f2ba988bd51878a951 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 13 Mar 2025 10:32:02 -0400 Subject: [PATCH 01/16] upgrade testing: allow configurable artifactory repo (#25350) Prerelease builds are in a different Artifactory repository than release builds. Make this a variable option so we can test prerelease builds in the nightly/weekly runs. --- enos/enos-scenario-upgrade.hcl | 2 ++ enos/enos-vars.hcl | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index 580406600..7e149c3aa 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -48,6 +48,7 @@ scenario "upgrade" { variables { artifactory_username = var.artifactory_username artifactory_token = var.artifactory_token + artifactory_repo = var.artifactory_repo_start arch = local.arch edition = matrix.edition product_version = var.product_version @@ -242,6 +243,7 @@ scenario "upgrade" { variables { artifactory_username = var.artifactory_username artifactory_token = var.artifactory_token + artifactory_repo = var.artifactory_repo_upgrade arch = local.arch edition = matrix.edition product_version = var.upgrade_version diff --git a/enos/enos-vars.hcl b/enos/enos-vars.hcl index 4914515f4..4e5118552 100644 --- a/enos/enos-vars.hcl +++ b/enos/enos-vars.hcl @@ -27,12 +27,28 @@ variable "product_version" { default = null } +variable "artifactory_repo_start" { + description = "The Artifactory repository we'll download the starting binary from" + type = string + + # note: this default only works for released binaries + default = "hashicorp-crt-staging-local*" +} + variable "upgrade_version" { description = "The version of Nomad we want to upgrade the cluster to" type = string default = null } +variable "artifactory_repo_upgrade" { + description = "The Artifactory repository we'll download the upgraded binary from" + type = string + + # note: this default only works for released binaries + default = "hashicorp-crt-staging-local*" +} + variable "download_binary_path" { description = "The path to a local directory where binaries will be downloaded to provision" } From 433f8c9a8b25af7ef83546aafc8f08be0c7fde52 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 13 Mar 2025 15:27:01 -0400 Subject: [PATCH 02/16] dynamic host volumes: don't wait for fingerprint to reserve node (#25386) If multiple dynamic host volumes are created in quick succession, it's possible for the server to attempt placement on a host where another volume has been placed but not yet fingerprinted as ready. Once a `VolumeCreate` RPC returns a response, we've already invoked the plugin successfully and written to state, so we're just waiting on the fingerprint for scheduling purposes. Change the placement selection so that we skip a node if it has a volume, regardless of whether that volume is ready yet. --- nomad/host_volume_endpoint.go | 9 +++++---- nomad/host_volume_endpoint_test.go | 4 ++++ nomad/state/state_store_host_volumes.go | 18 ++++++++++++++++++ nomad/state/state_store_host_volumes_test.go | 10 ++++++++++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/nomad/host_volume_endpoint.go b/nomad/host_volume_endpoint.go index 8e3d1cbf7..6eae2e88c 100644 --- a/nomad/host_volume_endpoint.go +++ b/nomad/host_volume_endpoint.go @@ -549,10 +549,11 @@ func (v *HostVolume) placeHostVolume(snap *state.StateSnapshot, vol *structs.Hos candidate := raw.(*structs.Node) // note: this is a race if multiple users create volumes of the same - // name concurrently, but we can't solve it on the server because we - // haven't yet written to state. The client will reject requests to - // create/register a volume with the same name with a different ID. - if _, hasVol := candidate.HostVolumes[vol.Name]; hasVol { + // name concurrently, but we can't completely solve it on the server + // because we haven't yet written to state. The client single-threads + // requests by volume name, so will reject requests to create/register a + // volume with the same name but a different ID. + if snap.NodeHasHostVolume(candidate.ID, vol.Name) { filteredByExisting++ continue } diff --git a/nomad/host_volume_endpoint_test.go b/nomad/host_volume_endpoint_test.go index 8f420e05d..bb57307f9 100644 --- a/nomad/host_volume_endpoint_test.go +++ b/nomad/host_volume_endpoint_test.go @@ -697,6 +697,10 @@ func TestHostVolumeEndpoint_placeVolume(t *testing.T) { must.NoError(t, store.UpsertNode(structs.MsgTypeTestSetup, 1000, node2)) must.NoError(t, store.UpsertNode(structs.MsgTypeTestSetup, 1000, node3)) + vol := mock.HostVolume() + vol.NodeID = node3.ID + must.NoError(t, store.UpsertHostVolume(1000, vol)) + testCases := []struct { name string vol *structs.HostVolume diff --git a/nomad/state/state_store_host_volumes.go b/nomad/state/state_store_host_volumes.go index 06a5e64c5..33117540b 100644 --- a/nomad/state/state_store_host_volumes.go +++ b/nomad/state/state_store_host_volumes.go @@ -290,3 +290,21 @@ func upsertHostVolumeForNode(txn *txn, node *structs.Node, index uint64) error { return nil } + +func (s *StateStore) NodeHasHostVolume(nodeID, volName string) bool { + iter, err := s.HostVolumesByNodeID(nil, nodeID, SortDefault) + if err != nil { + return true + } + for { + raw := iter.Next() + if raw == nil { + break + } + match := raw.(*structs.HostVolume) + if match.Name == volName { + return true + } + } + return false +} diff --git a/nomad/state/state_store_host_volumes_test.go b/nomad/state/state_store_host_volumes_test.go index b390eaa60..0d0b4746c 100644 --- a/nomad/state/state_store_host_volumes_test.go +++ b/nomad/state/state_store_host_volumes_test.go @@ -234,6 +234,11 @@ func TestStateStore_UpdateHostVolumesFromFingerprint(t *testing.T) { vols[3].Name = "dhv-two" vols[3].NodeID = node.ID + must.False(t, store.NodeHasHostVolume(node.ID, "dhv-zero")) + must.False(t, store.NodeHasHostVolume(node.ID, "dhv-one")) + must.False(t, store.NodeHasHostVolume(node.ID, "dhv-two")) + must.False(t, store.NodeHasHostVolume(otherNode.ID, "dhv-one")) + index++ oldIndex := index must.NoError(t, store.UpsertHostVolume(index, vols[0])) @@ -241,6 +246,11 @@ func TestStateStore_UpdateHostVolumesFromFingerprint(t *testing.T) { must.NoError(t, store.UpsertHostVolume(index, vols[2])) must.NoError(t, store.UpsertHostVolume(index, vols[3])) + must.True(t, store.NodeHasHostVolume(node.ID, "dhv-zero")) + must.True(t, store.NodeHasHostVolume(node.ID, "dhv-one")) + must.True(t, store.NodeHasHostVolume(node.ID, "dhv-two")) + must.True(t, store.NodeHasHostVolume(otherNode.ID, "dhv-one")) + vol0, err := store.HostVolumeByID(nil, ns, vols[0].ID, false) must.NoError(t, err) must.Eq(t, structs.HostVolumeStateReady, vol0.State, From 239ac3e4bdec33cd2181333605d8b9fee1c06af4 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 13 Mar 2025 16:39:19 -0400 Subject: [PATCH 03/16] [ui] Case-insensitive jobs list filtering (#25378) --- .changelog/25378.txt | 3 +++ ui/app/controllers/jobs/index.js | 2 +- ui/mirage/config.js | 6 ++++++ ui/tests/acceptance/jobs-list-test.js | 26 ++++++++++++++++++++++++-- 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 .changelog/25378.txt diff --git a/.changelog/25378.txt b/.changelog/25378.txt new file mode 100644 index 000000000..6de1b693e --- /dev/null +++ b/.changelog/25378.txt @@ -0,0 +1,3 @@ +```release-note:improvement +ui: Makes jobs list filtering case-insensitive +``` diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index b13be7e7f..c643ff260 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -653,7 +653,7 @@ export default class JobsIndexController extends Controller { this.searchText = newFilter; } else { // If it's a string without a filter operator, assume the user is trying to look up a job name - this.searchText = `Name contains "${newFilter}"`; + this.searchText = `Name matches "(?i)${newFilter}"`; } } diff --git a/ui/mirage/config.js b/ui/mirage/config.js index b304ae237..f2aed8dec 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -231,6 +231,12 @@ export default function () { job[condition.field] && job[condition.field].includes(condition.value) ); + } else if (condition.operator === 'matches') { + // strip the (?i) bit out of the value; used for case-insensitive matching + // but JS doesn't support PCRE-style regex modifiers the way our backend does, + // so strip 'em out here. + const value = condition.value.replace('(?i)', ''); + return new RegExp(value, 'i').test(job[condition.field]); } else if (condition.operator === '==') { return job[condition.field] === condition.value; } else if (condition.operator === '!=') { diff --git a/ui/tests/acceptance/jobs-list-test.js b/ui/tests/acceptance/jobs-list-test.js index 56f488d15..3a9ceee49 100644 --- a/ui/tests/acceptance/jobs-list-test.js +++ b/ui/tests/acceptance/jobs-list-test.js @@ -188,7 +188,7 @@ module('Acceptance | jobs list', function (hooks) { assert.equal( currentURL(), - '/jobs?filter=Name%20contains%20%22foobar%22', + '/jobs?filter=Name%20matches%20%22(%3Fi)foobar%22', 'No page query param' ); }); @@ -1271,7 +1271,7 @@ module('Acceptance | jobs list', function (hooks) { assert.ok( server.pretender.handledRequests.find((req) => decodeURIComponent(req.url).includes( - '?filter=Name contains "something-that-surely-doesnt-exist"' + '?filter=Name matches "(?i)something-that-surely-doesnt-exist"' ) ), 'A request was made with a filter query param that assumed job name' @@ -1386,6 +1386,28 @@ module('Acceptance | jobs list', function (hooks) { localStorage.removeItem('nomadPageSize'); }); + test('Searching by name filters the list case-insensitively', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + createJobs(server, 10); + server.create('job', { + name: 'hashi-one', + id: 'hashi-one', + modifyIndex: 0, + }); + server.create('job', { + name: 'Hashi-two', + id: 'hashi-two', + modifyIndex: 0, + }); + + await JobsList.visit(); + + await JobsList.search.fillIn('Hashi'); + assert.dom('.job-row').exists({ count: 2 }); + assert.dom('[data-test-job-row="hashi-one"]').exists(); + assert.dom('[data-test-job-row="hashi-two"]').exists(); + }); + test('Searching by type filters the list', async function (assert) { localStorage.setItem('nomadPageSize', '10'); server.createList('job', 10, { From 3e1f56c1c0ec2c253f66bcdb7d042f6d3bb0564a Mon Sep 17 00:00:00 2001 From: James Rasell Date: Fri, 14 Mar 2025 15:59:41 +0100 Subject: [PATCH 04/16] cli: Add volume type to delete error messages when API call fails. (#25392) --- command/volume_delete.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/command/volume_delete.go b/command/volume_delete.go index 427fd9c59..6098d4329 100644 --- a/command/volume_delete.go +++ b/command/volume_delete.go @@ -146,7 +146,7 @@ func (c *VolumeDeleteCommand) deleteCSIVolume(client *api.Client, volID string, Namespace: c.namespace, }) if err != nil { - c.Ui.Error(fmt.Sprintf("Could not find existing volume to delete: %s", err)) + c.Ui.Error(fmt.Sprintf(`Could not find existing CSI volume to delete: %s`, err)) return 1 } if len(possible) > 0 { @@ -179,7 +179,7 @@ func (c *VolumeDeleteCommand) deleteHostVolume(client *api.Client, volID string) if !helper.IsUUID(volID) { stub, possible, err := getHostVolumeByPrefix(client, volID, c.namespace) if err != nil { - c.Ui.Error(fmt.Sprintf("Could not find existing volume to delete: %s", err)) + c.Ui.Error(fmt.Sprintf(`Could not find existing host volume to delete: %s`, err)) return 1 } if len(possible) > 0 { From 3af2da73623d36b88837ace5de5e7e9285cb33fc Mon Sep 17 00:00:00 2001 From: Juanadelacuesta <8647634+Juanadelacuesta@users.noreply.github.com> Date: Fri, 14 Mar 2025 16:38:23 +0100 Subject: [PATCH 05/16] fix: add default policy to consul acl configurations for the e2e cluster --- .../provision-infra/provision-nomad/etc/consul.d/servers.hcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/terraform/provision-infra/provision-nomad/etc/consul.d/servers.hcl b/e2e/terraform/provision-infra/provision-nomad/etc/consul.d/servers.hcl index 54f35892c..c41ab7150 100644 --- a/e2e/terraform/provision-infra/provision-nomad/etc/consul.d/servers.hcl +++ b/e2e/terraform/provision-infra/provision-nomad/etc/consul.d/servers.hcl @@ -15,7 +15,8 @@ ui_config { } acl { - enabled = true + enabled = true + default_policy = "deny" tokens { initial_management = "${management_token}" agent = "${token}" From 88ff5a7cae67152e4e6745100bff41af2efc05f6 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 14 Mar 2025 12:37:39 -0400 Subject: [PATCH 06/16] [ui] Scope selection for Sentinel Policies (#25390) * An option to select, and column etc. to view, sentinel policy scope * Flake potential: Seed(1) had a couple jobs with the same ModifyIndex * More de-flaking --- .changelog/25390.txt | 3 +++ ui/app/components/sentinel-policy-editor.hbs | 12 ++++++++++ ui/app/components/sentinel-policy-editor.js | 4 ++++ .../administration/sentinel-policies/index.js | 5 ++++ .../sentinel-policies/index.hbs | 1 + ui/mirage/factories/sentinel-policy.js | 2 +- ui/mirage/scenarios/default.js | 15 ++++++++++++ ui/tests/acceptance/job-detail-test.js | 6 ++++- ui/tests/acceptance/jobs-list-test.js | 3 ++- ui/tests/acceptance/sentinel-policies-test.js | 24 +++++++++++++++++++ ui/tests/acceptance/task-detail-test.js | 2 ++ 11 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 .changelog/25390.txt diff --git a/.changelog/25390.txt b/.changelog/25390.txt new file mode 100644 index 000000000..e755c2434 --- /dev/null +++ b/.changelog/25390.txt @@ -0,0 +1,3 @@ +```release-note:improvement +ui: Added a scope selector for sentinel policy page +``` diff --git a/ui/app/components/sentinel-policy-editor.hbs b/ui/app/components/sentinel-policy-editor.hbs index 723b5a52a..99931e7fd 100644 --- a/ui/app/components/sentinel-policy-editor.hbs +++ b/ui/app/components/sentinel-policy-editor.hbs @@ -77,6 +77,18 @@ +
+ + Scope + + Submit Job + + + Submit Host Volume + + +
+