From 9a24cda68ce5f8ccedd2b1f8823ca7ee3eeed102 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Thu, 26 Jul 2018 22:23:45 +0300 Subject: [PATCH] inventory and ci setup --- practice/ceph/_deploy_cluster.sh | 14 + practice/ceph/inventory/group_vars/all.yml | 19 ++ practice/ceph/inventory/hosts | 20 ++ practice/ci-cd/setup.sh | 86 +++++++ .../kubespray}/_deploy_cluster.sh | 0 .../inventory/slurm/group_vars/all.yml | 151 +++++++++++ .../slurm/group_vars/k8s-cluster.yml | 240 ++++++++++++++++++ .../slurm/group_vars/kube-ingress.yml | 2 + practice/kubespray/inventory/slurm/hosts.ini | 31 +++ 9 files changed, 563 insertions(+) create mode 100644 practice/ceph/_deploy_cluster.sh create mode 100644 practice/ceph/inventory/group_vars/all.yml create mode 100644 practice/ceph/inventory/hosts create mode 100755 practice/ci-cd/setup.sh rename {scripts => practice/kubespray}/_deploy_cluster.sh (100%) create mode 100644 practice/kubespray/inventory/slurm/group_vars/all.yml create mode 100644 practice/kubespray/inventory/slurm/group_vars/k8s-cluster.yml create mode 100644 practice/kubespray/inventory/slurm/group_vars/kube-ingress.yml create mode 100644 practice/kubespray/inventory/slurm/hosts.ini diff --git a/practice/ceph/_deploy_cluster.sh b/practice/ceph/_deploy_cluster.sh new file mode 100644 index 0000000..0b6b794 --- /dev/null +++ b/practice/ceph/_deploy_cluster.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +###ssh-agent bash +#ssh-add ~/.ssh/id_rsa + +if [ -z "$1" ]; then + echo "Usage: $0 adminname" + exit 1 +fi + +d=$(date '+%Y.%m.%d_%H:%M') +ANSIBLE_FORCE_COLOR=true ansible-playbook -u $1 -k -i inventory/hosts site.yml -b --diff 2>&1 | tee "./deploy-$d.log" + +sed -i -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" "./deploy-$d.log" diff --git a/practice/ceph/inventory/group_vars/all.yml b/practice/ceph/inventory/group_vars/all.yml new file mode 100644 index 0000000..95b8c1c --- /dev/null +++ b/practice/ceph/inventory/group_vars/all.yml @@ -0,0 +1,19 @@ +ceph_origin: repository +ceph_repository: community +ceph_repository_type: cdn +ceph_stable_release: luminous +public_network: "172.20.100.0/24" +cluster_network: "172.20.100.0/24" + +osd_objectstore: bluestore +osd_scenario: collocated +devices: + - /dev/sdb + +osd_pool_default_pg_num: 30 + +ceph_conf_overrides: + global: + "osd journal size": 1024 + "osd pool default size": 3 + "osd pool default min size": 2 diff --git a/practice/ceph/inventory/hosts b/practice/ceph/inventory/hosts new file mode 100644 index 0000000..0073638 --- /dev/null +++ b/practice/ceph/inventory/hosts @@ -0,0 +1,20 @@ +[mons] +ingress-1.slurm.io monitor_address=172.20.100.5 +node-1.slurm.io monitor_address=172.20.100.6 +node-2.slurm.io monitor_address=172.20.100.7 + +[osds] +ingress-1.slurm.io +node-1.slurm.io +node-2.slurm.io + +[mgrs] +ingress-1.slurm.io +node-1.slurm.io +node-2.slurm.io + + +[mdss] +ingress-1.slurm.io +node-1.slurm.io +node-2.slurm.io diff --git a/practice/ci-cd/setup.sh b/practice/ci-cd/setup.sh new file mode 100755 index 0000000..0a264ed --- /dev/null +++ b/practice/ci-cd/setup.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +CI_PROJECT_PATH_SLUG=$1 +CI_ENVIRONMENT_NAME=$2 + + +GREEN='\033[0;32m' +NC='\033[0m' + + +usage() { + echo "Usage: $0 CI_PROJECT_PATH_SLUG CI_ENVIRONMENT_NAME" +} + +base64_decode_key() { +if [[ "$OSTYPE" == "linux"* ]] +then + echo "-d" + +elif [[ "$OSTYPE" == "darwin"* ]] +then + echo "-D" + +else + echo "--help" +fi +} + + +if [ -n "$CI_PROJECT_PATH_SLUG" ] && [ -n "$CI_ENVIRONMENT_NAME" ] +then + echo -e "${GREEN}creating namespace for project${NC}" + kubectl create namespace \ + $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME + + echo + echo -e "${GREEN}creating CI serviceaccount for project${NC}" + kubectl create serviceaccount \ + --namespace $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME + + echo + echo -e "${GREEN}creating CI role for project${NC}" + cat << EOF | kubectl create --namespace $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME -f - + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME + rules: + - apiGroups: ["", "extensions", "apps", "certmanager.k8s.io"] + resources: ["*"] + verbs: ["*"] +EOF + + echo + echo -e "${GREEN}creating CI rolebinding for project${NC}" + kubectl create rolebinding \ + --namespace $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + --serviceaccount $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME:$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + --role $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME + + echo + echo -e "${GREEN}installing Tiller for project's CI${NC}" + helm init \ + --service-account $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + --tiller-namespace $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + --history-max 10 \ + --wait + + echo + echo -e "${GREEN}access token for new CI user:${NC}" + kubectl get secret \ + --namespace $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + $( \ + kubectl get serviceaccount \ + --namespace $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME \ + -o jsonpath='{.secrets[].name}'\ + ) \ + -o jsonpath='{.data.token}' | base64 $(base64_decode_key) + echo + +else + usage +fi diff --git a/scripts/_deploy_cluster.sh b/practice/kubespray/_deploy_cluster.sh similarity index 100% rename from scripts/_deploy_cluster.sh rename to practice/kubespray/_deploy_cluster.sh diff --git a/practice/kubespray/inventory/slurm/group_vars/all.yml b/practice/kubespray/inventory/slurm/group_vars/all.yml new file mode 100644 index 0000000..389f89d --- /dev/null +++ b/practice/kubespray/inventory/slurm/group_vars/all.yml @@ -0,0 +1,151 @@ +# Valid bootstrap options (required): ubuntu, coreos, centos, none +bootstrap_os: none + +## Flannel +flannel_backend_type: "host-gw" +flannel_interface_regexp: '172\\.20\\.100\\.\\d{1,3}' + +###flannel_interface: "eth0" + +#Directory where etcd data stored +etcd_data_dir: /var/lib/etcd + +# Directory where the binaries will be installed +bin_dir: /usr/local/bin + +## The access_ip variable is used to define how other nodes should access +## the node. This is used in flannel to allow other flannel nodes to see +## this node for example. The access_ip is really useful AWS and Google +## environments where the nodes are accessed remotely by the "public" ip, +## but don't know about that address themselves. +#access_ip: 1.1.1.1 + +### LOADBALANCING AND ACCESS MODES +## Enable multiaccess to configure etcd clients to access all of the etcd members directly +## as the "http://hostX:port, http://hostY:port, ..." and ignore the proxy loadbalancers. +## This may be the case if clients support and loadbalance multiple etcd servers natively. +etcd_multiaccess: true + +### ETCD: disable peer client cert authentication. +# This affects ETCD_PEER_CLIENT_CERT_AUTH variable +#etcd_peer_client_auth: true + +## External LB example config +## apiserver_loadbalancer_domain_name: "elb.some.domain" +#loadbalancer_apiserver: +# address: 1.2.3.4 +# port: 1234 + +## Internal loadbalancers for apiservers +#loadbalancer_apiserver_localhost: true + +## Local loadbalancer should use this port instead, if defined. +## Defaults to kube_apiserver_port (6443) +#nginx_kube_apiserver_port: 8443 + +### OTHER OPTIONAL VARIABLES +## For some things, kubelet needs to load kernel modules. For example, dynamic kernel services are needed +## for mounting persistent volumes into containers. These may not be loaded by preinstall kubernetes +## processes. For example, ceph and rbd backed volumes. Set to true to allow kubelet to load kernel +## modules. +kubelet_load_modules: true + +## Internal network total size. This is the prefix of the +## entire network. Must be unused in your environment. +#kube_network_prefix: 18 + +## With calico it is possible to distributed routes with border routers of the datacenter. +## Warning : enabling router peering will disable calico's default behavior ('node mesh'). +## The subnets of each nodes will be distributed by the datacenter router +#peer_with_router: false + +## Upstream dns servers used by dnsmasq +#upstream_dns_servers: +# - 8.8.8.8 +# - 8.8.4.4 + +## There are some changes specific to the cloud providers +## for instance we need to encapsulate packets with some network plugins +## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', or 'external' +## When openstack is used make sure to source in the openstack credentials +## like you would do when using nova-client before starting the playbook. +#cloud_provider: + +## When azure is used, you need to also set the following variables. +## see docs/azure.md for details on how to get these values +#azure_tenant_id: +#azure_subscription_id: +#azure_aad_client_id: +#azure_aad_client_secret: +#azure_resource_group: +#azure_location: +#azure_subnet_name: +#azure_security_group_name: +#azure_vnet_name: +#azure_vnet_resource_group: +#azure_route_table_name: + +## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) +#openstack_blockstorage_version: "v1/v2/auto (default)" +## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. +#openstack_lbaas_enabled: True +#openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP" +## To enable automatic floating ip provisioning, specify a subnet. +#openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default" +## Override default LBaaS behavior +#openstack_lbaas_use_octavia: False +#openstack_lbaas_method: "ROUND_ROBIN" +#openstack_lbaas_provider: "haproxy" +#openstack_lbaas_create_monitor: "yes" +#openstack_lbaas_monitor_delay: "1m" +#openstack_lbaas_monitor_timeout: "30s" +#openstack_lbaas_monitor_max_retries: "3" + +## Uncomment to enable experimental kubeadm deployment mode +#kubeadm_enabled: false +## Set these proxy values in order to update package manager and docker daemon to use proxies +#http_proxy: "" +#https_proxy: "" +## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy +#no_proxy: "" + +## Uncomment this if you want to force overlay/overlay2 as docker storage driver +## Please note that overlay2 is only supported on newer kernels +docker_storage_options: -s overlay2 + +# Uncomment this if you have more than 3 nameservers, then we'll only use the first 3. +docker_dns_servers_strict: false + +## Default packages to install within the cluster, f.e: +#kpm_packages: +# - name: kube-system/grafana + +## Certificate Management +## This setting determines whether certs are generated via scripts or whether a +## cluster of Hashicorp's Vault is started to issue certificates (using etcd +## as a backend). Options are "script" or "vault" +#cert_management: script + +# Set to true to allow pre-checks to fail and continue deployment +#ignore_assert_errors: false + +#etcd_heartbeat_interval: "100" +#etcd_election_timeout: "5000" +etcd_snapshot_count: "5000" + +# -c Takes an integer between 0 and 3 or one of the strings none, realtime, best-effort or idle. +# -n Takes an integer between 0 (highest priority) and 7 (lowest priority) +etcd_ionice: "-c2 -n0" + +## Etcd auto compaction retention for mvcc key value store in hour +#etcd_compaction_retention: 0 + +## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics. +#etcd_metrics: basic + +## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing. +## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM. +etcd_memory_limit: 0 + +# The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable. +kube_read_only_port: 10255 diff --git a/practice/kubespray/inventory/slurm/group_vars/k8s-cluster.yml b/practice/kubespray/inventory/slurm/group_vars/k8s-cluster.yml new file mode 100644 index 0000000..bba0421 --- /dev/null +++ b/practice/kubespray/inventory/slurm/group_vars/k8s-cluster.yml @@ -0,0 +1,240 @@ +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernets. +# This puts them in a sane location and namespace. +# Editting those values will almost surely break something. +kube_config_dir: /etc/kubernetes +kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" +kube_manifest_dir: "{{ kube_config_dir }}/manifests" +system_namespace: kube-system + +# This is where all the cert scripts and certs will be located +kube_cert_dir: "{{ kube_config_dir }}/ssl" + +# This is where all of the bearer tokens will be stored +kube_token_dir: "{{ kube_config_dir }}/tokens" + +# This is where to save basic auth file +kube_users_dir: "{{ kube_config_dir }}/users" + +kube_api_anonymous_auth: false + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" +# Random shifts for retrying failed ops like pushing/downloading +retry_stagger: 5 + +# This is the group that the cert creation scripts chgrp the +# cert files to. Not really changable... +kube_cert_group: kube-cert + +# Cluster Loglevel configuration +kube_log_level: 1 + +# Users to create for basic auth in Kubernetes API via HTTP +# Optionally add groups for user +kube_api_pwd: "{{ lookup('password', inventory_dir + '/credentials/kube_user length=15 chars=ascii_letters,digits') }}" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + groups: + - system:masters + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +kube_oidc_auth: false +kube_basic_auth: false +#kube_token_auth: false + + +## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ +## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) + +# kube_oidc_url: https:// ... +# kube_oidc_client_id: kubernetes +## Optional settings for OIDC +# kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem +# kube_oidc_username_claim: sub +# kube_oidc_groups_claim: groups + + +# Choose network plugin (cilium, calico, contiv, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: flannel + +# weave's network password for encryption +# if null then no network encryption +# you can use --extra-vars to pass the password in command line +weave_password: EnterPasswordHere + +# Weave uses consensus mode by default +# Enabling seed mode allow to dynamically add or remove hosts +# https://www.weave.works/docs/net/latest/ipam/ +weave_mode_seed: false + +# This two variable are automatically changed by the weave's role, do not manually change these values +# To reset values : +# weave_seed: uninitialized +# weave_peers: uninitialized +weave_seed: uninitialized +weave_peers: uninitialized + +# Set the MTU of Weave (default 1376, Jumbo Frames: 8916) +weave_mtu: 1376 + +# Enable kubernetes network policies +enable_network_policy: false + +# Kubernetes internal network for services, unused block of space. +kube_service_addresses: 10.0.0.0/16 + +# internal network. When used, it will assign IP +# addresses from this range to individual pods. +# This network must be unused in your network infrastructure! +kube_pods_subnet: 10.100.0.0/16 + +# internal network node size allocation (optional). This is the size allocated +# to each node on your network. With these defaults you should have +# room for 4096 nodes with 254 pods per node. +kube_network_node_prefix: 24 + +# The port the API Server will be listening on. +kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" +kube_apiserver_port: 6443 # (https) +kube_apiserver_insecure_port: 8080 # (http) +# Set to 0 to disable insecure port - Requires RBAC in authorization_modes and kube_api_anonymous_auth: true +#kube_apiserver_insecure_port: 0 # (disabled) + +# Kube-proxy proxyMode configuration. +# Can be ipvs, iptables +kube_proxy_mode: iptables + +# DNS configuration. +# Kubernetes cluster name, also will be used as DNS domain +cluster_name: slurm.local +# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods +ndots: 2 +# Can be dnsmasq_kubedns, kubedns, manual or none +dns_mode: kubedns +# Set manual server if using a custom cluster DNS server +#manual_dns_server: 10.x.x.x + +# Can be docker_dns, host_resolvconf or none +resolvconf_mode: docker_dns +# Deploy netchecker app to verify DNS resolve as an HTTP service +deploy_netchecker: false +# Ip address of the kubernetes skydns service +skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" +dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" +dns_domain: "{{ cluster_name }}" + +# Path used to store Docker data +docker_daemon_graph: "/var/lib/docker" + +## A string of extra options to pass to the docker daemon. +## This string should be exactly as you wish it to appear. +## An obvious use case is allowing insecure-registry access +## to self hosted registries like so: + +docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }} {{ docker_log_opts }}" +docker_bin_dir: "/usr/bin" + +# Settings for containerized control plane (etcd/kubelet/secrets) +etcd_deployment_type: docker +kubelet_deployment_type: host +vault_deployment_type: docker +helm_deployment_type: host + +# K8s image pull policy (imagePullPolicy) +k8s_image_pull_policy: IfNotPresent + +# Kubernetes dashboard +# RBAC required. see docs/getting-started.md for access details. +dashboard_enabled: true + +# Monitoring apps for k8s +efk_enabled: false + +# Helm deployment +helm_enabled: false + +# Istio deployment +istio_enabled: false + +# Registry deployment +registry_enabled: false + +# Local volume provisioner deployment +local_volume_provisioner_enabled: false +# local_volume_provisioner_namespace: "{{ system_namespace }}" +# local_volume_provisioner_base_dir: /mnt/disks +# local_volume_provisioner_mount_dir: /mnt/disks +# local_volume_provisioner_storage_class: local-storage + +# CephFS provisioner deployment +cephfs_provisioner_enabled: false +# cephfs_provisioner_namespace: "{{ system_namespace }}" +# cephfs_provisioner_cluster: ceph +# cephfs_provisioner_monitors: +# - 172.24.0.1:6789 +# - 172.24.0.2:6789 +# - 172.24.0.3:6789 +# cephfs_provisioner_admin_id: admin +# cephfs_provisioner_secret: secret +# cephfs_provisioner_storage_class: cephfs + +# Nginx ingress controller deployment +ingress_nginx_enabled: true +ingress_nginx_host_network: true +ingress_nginx_nodeselector: + node-role.kubernetes.io/ingress: "true" +# ingress_nginx_namespace: "ingress-nginx" +# ingress_nginx_insecure_port: 80 +# ingress_nginx_secure_port: 443 +ingress_nginx_configmap: + server-tokens: "False" + proxy-body-size: "2G" + proxy-buffer-size: "16k" + worker-shutdown-timeout: "180" +# map-hash-bucket-size: "128" +# ssl-protocols: "SSLv2" +# ingress_nginx_configmap_tcp_services: +# 9000: "default/example-go:8080" +# ingress_nginx_configmap_udp_services: +# 53: "kube-system/kube-dns:53" + +# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now ) +persistent_volumes_enabled: false + +# Make a copy of kubeconfig on the host that runs Ansible in GITDIR/artifacts +kubeconfig_localhost: true +# Download kubectl onto the host that runs Ansible in GITDIR/artifacts +kubectl_localhost: true + +# dnsmasq +# dnsmasq_upstream_dns_servers: +# - /resolvethiszone.with/10.0.4.250 +# - 8.8.8.8 + +# Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created. (default true) +# kubelet_cgroups_per_qos: true + +# A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. +# Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "". +# kubelet_enforce_node_allocatable: pods + +## Supplementary addresses that can be added in kubernetes ssl keys. +## That can be useful for example to setup a keepalived virtual IP +# supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3] + +## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler. +## See https://github.com/kubernetes-incubator/kubespray/issues/2141 +## Set this variable to true to get rid of this issue +volume_cross_zone_attachment: false + +# Added support for kubelet webhook authentication/authorization #2184 +# https://github.com/kubernetes-incubator/kubespray/pull/2184 +# https://factory.southbridge.io/issues/262494 +kubelet_authentication_token_webhook: true +kubelet_authorization_mode_webhook: true diff --git a/practice/kubespray/inventory/slurm/group_vars/kube-ingress.yml b/practice/kubespray/inventory/slurm/group_vars/kube-ingress.yml new file mode 100644 index 0000000..0b71ac7 --- /dev/null +++ b/practice/kubespray/inventory/slurm/group_vars/kube-ingress.yml @@ -0,0 +1,2 @@ +node_labels: + node-role.kubernetes.io/ingress: "true" diff --git a/practice/kubespray/inventory/slurm/hosts.ini b/practice/kubespray/inventory/slurm/hosts.ini new file mode 100644 index 0000000..a8cea1b --- /dev/null +++ b/practice/kubespray/inventory/slurm/hosts.ini @@ -0,0 +1,31 @@ +# ## Configure 'ip' variable to bind kubernetes services on a +# ## different ip than the default iface +master-1.slurm.io ansible_ssh_host=172.20.100.2 ip=172.20.100.2 +master-2.slurm.io ansible_ssh_host=172.20.100.3 ip=172.20.100.3 +master-3.slurm.io ansible_ssh_host=172.20.100.4 ip=172.20.100.4 +ingress-1.slurm.io ansible_ssh_host=172.20.100.5 ip=172.20.100.5 +node-1.slurm.io ansible_ssh_host=172.20.100.6 ip=172.20.100.6 +node-2.slurm.io ansible_ssh_host=172.20.100.7 ip=172.20.100.7 + +[kube-master] +master-1.slurm.io +master-2.slurm.io +master-3.slurm.io + +[etcd] +master-1.slurm.io +master-2.slurm.io +master-3.slurm.io + +[kube-node] +node-1.slurm.io +node-2.slurm.io +ingress-1.slurm.io + +[kube-ingress] +ingress-1.slurm.io + +[k8s-cluster:children] +kube-node +kube-master +