From b46955f9e13e8784d2c2c379ff2d59d34ca0fcf1 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Mon, 24 Aug 2020 09:24:32 -0700 Subject: [PATCH] Allow for custom resource requirements for jobs that are recognized by allocations The job factory will now accept an array of resourceSpecs that is a shorthand notation for memory, cpu, disk, and iops requirements. These specs get passed down to task groups. The task group factory will split the resource requirements near evenly (there is variance threshold) across all expected tasks. Allocations then construct task-resource objects based on the resources from the matching task. --- ui/mirage/factories/allocation.js | 36 +++++++++------- ui/mirage/factories/job.js | 33 ++++++++++++-- ui/mirage/factories/task-group.js | 71 ++++++++++++++++++++++++++++++- 3 files changed, 120 insertions(+), 20 deletions(-) diff --git a/ui/mirage/factories/allocation.js b/ui/mirage/factories/allocation.js index c91ba9c95..c571271d0 100644 --- a/ui/mirage/factories/allocation.js +++ b/ui/mirage/factories/allocation.js @@ -40,16 +40,18 @@ export default Factory.extend({ withTaskWithPorts: trait({ afterCreate(allocation, server) { const taskGroup = server.db.taskGroups.findBy({ name: allocation.taskGroup }); - const resources = taskGroup.taskIds.map(id => - server.create( + const resources = taskGroup.taskIds.map(id => { + const task = server.db.tasks.find(id); + return server.create( 'task-resource', { allocation, - name: server.db.tasks.find(id).name, + name: task.name, + resources: task.Resources, }, 'withReservedPorts' - ) - ); + ); + }); allocation.update({ taskResourceIds: resources.mapBy('id') }); }, @@ -58,16 +60,18 @@ export default Factory.extend({ withoutTaskWithPorts: trait({ afterCreate(allocation, server) { const taskGroup = server.db.taskGroups.findBy({ name: allocation.taskGroup }); - const resources = taskGroup.taskIds.map(id => - server.create( + const resources = taskGroup.taskIds.map(id => { + const task = server.db.tasks.find(id); + return server.create( 'task-resource', { allocation, - name: server.db.tasks.find(id).name, + name: task.name, + resources: task.Resources, }, 'withoutReservedPorts' - ) - ); + ); + }); allocation.update({ taskResourceIds: resources.mapBy('id') }); }, @@ -191,12 +195,14 @@ export default Factory.extend({ }) ); - const resources = taskGroup.taskIds.map(id => - server.create('task-resource', { + const resources = taskGroup.taskIds.map(id => { + const task = server.db.tasks.find(id); + return server.create('task-resource', { allocation, - name: server.db.tasks.find(id).name, - }) - ); + name: task.name, + resources: task.Resources, + }); + }); allocation.update({ taskStateIds: allocation.clientStatus === 'pending' ? [] : states.mapBy('id'), diff --git a/ui/mirage/factories/job.js b/ui/mirage/factories/job.js index 04fb128df..c4958bb5b 100644 --- a/ui/mirage/factories/job.js +++ b/ui/mirage/factories/job.js @@ -20,7 +20,19 @@ export default Factory.extend({ version: 1, - groupsCount: () => faker.random.number({ min: 1, max: 2 }), + // When provided, the resourceSpec will inform how many task groups to create + // and how much of each resource that task group reserves. + // + // One task group, 256 MiB memory and 500 Mhz cpu + // resourceSpec: ['M: 256, C: 500'] + // + // Two task groups + // resourceSpec: ['M: 256, C: 500', 'M: 1024, C: 1200'] + resourceSpec: null, + + groupsCount() { + return this.resourceSpec ? this.resourceSpec.length : faker.random.number({ min: 1, max: 2 }); + }, region: () => 'global', type: () => faker.helpers.randomize(JOB_TYPES), @@ -135,9 +147,22 @@ export default Factory.extend({ groupProps.count = job.groupTaskCount; } - const groups = job.noHostVolumes - ? server.createList('task-group', job.groupsCount, 'noHostVolumes', groupProps) - : server.createList('task-group', job.groupsCount, groupProps); + let groups; + if (job.noHostVolumes) { + groups = provide(job.groupsCount, (_, idx) => + server.create('task-group', 'noHostVolumes', { + ...groupProps, + resourceSpec: job.resourceSpec && job.resourceSpec.length && job.resourceSpec[idx], + }) + ); + } else { + groups = provide(job.groupsCount, (_, idx) => + server.create('task-group', { + ...groupProps, + resourceSpec: job.resourceSpec && job.resourceSpec.length && job.resourceSpec[idx], + }) + ); + } job.update({ taskGroupIds: groups.mapBy('id'), diff --git a/ui/mirage/factories/task-group.js b/ui/mirage/factories/task-group.js index 7d6822c22..8e3cbc9ac 100644 --- a/ui/mirage/factories/task-group.js +++ b/ui/mirage/factories/task-group.js @@ -1,6 +1,7 @@ import { Factory, trait } from 'ember-cli-mirage'; import faker from 'nomad-ui/mirage/faker'; import { provide } from '../utils'; +import { generateResources } from '../common'; const DISK_RESERVATIONS = [200, 500, 1000, 2000, 5000, 10000, 100000]; @@ -36,6 +37,9 @@ export default Factory.extend({ // When true, only creates allocations shallow: false, + // When set, passed into tasks to set resource values + resourceSpec: null, + afterCreate(group, server) { let taskIds = []; let volumes = Object.keys(group.volumes); @@ -66,12 +70,20 @@ export default Factory.extend({ } if (!group.shallow) { - const tasks = provide(group.count, () => { + const resources = + group.resourceSpec && divide(group.count, parseResourceSpec(group.resourceSpec)); + const tasks = provide(group.count, (_, idx) => { const mounts = faker.helpers .shuffle(volumes) .slice(0, faker.random.number({ min: 1, max: 3 })); + + const maybeResources = {}; + if (resources) { + maybeResources.Resources = generateResources(resources[idx]); + } return server.create('task', { taskGroup: group, + ...maybeResources, volumeMounts: mounts.map(mount => ({ Volume: mount, Destination: `/${faker.internet.userName()}/${faker.internet.domainWord()}/${faker.internet.color()}`, @@ -136,3 +148,60 @@ function makeHostVolumes() { return hash; }, {}); } + +function parseResourceSpec(spec) { + const mapping = { + M: 'MemoryMB', + C: 'CPU', + D: 'DiskMB', + I: 'IOPS', + }; + + const terms = spec.split(',').map(t => { + const [k, v] = t + .trim() + .split(':') + .map(kv => kv.trim()); + return [k, +v]; + }); + + return terms.reduce((hash, term) => { + hash[mapping[term[0]]] = term[1]; + return hash; + }, {}); +} + +// Split a single resources object into N resource objects where +// the sum of each property of the new resources objects equals +// the original resources properties +// ex: divide(2, { Mem: 400, Cpu: 250 }) -> [{ Mem: 80, Cpu: 50 }, { Mem: 320, Cpu: 200 }] +function divide(count, resources) { + const wheel = roulette(1, count); + + const ret = provide(count, (_, idx) => { + return Object.keys(resources).reduce((hash, key) => { + hash[key] = Math.round(resources[key] * wheel[idx]); + return hash; + }, {}); + }); + + return ret; +} + +// Roulette splits a number into N divisions +// Variance is a value between 0 and 1 that determines how much each division in +// size. At 0 each division is even, at 1, it's entirely random but the sum of all +// divisions is guaranteed to equal the total value. +function roulette(total, divisions, variance = 0.8) { + let roulette = new Array(divisions).fill(total / divisions); + roulette.forEach((v, i) => { + if (i === roulette.length - 1) return; + roulette.splice(i, 2, ...rngDistribute(roulette[i], roulette[i + 1], variance)); + }); + return roulette; +} + +function rngDistribute(a, b, variance = 0.8) { + const move = a * faker.random.number({ min: 0, max: variance, precision: 0.01 }); + return [a - move, b + move]; +}