mirror of
https://github.com/kemko/xc.git
synced 2026-01-01 15:55:43 +03:00
added xcAwsInventory.py (#15)
* add .github/workflows/build.yml add Makefile go.mod: add govvv go.sum: add govvv * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * remote/serial.go "panic: bytes: negative Repeat count" * fix .github/workflows/build.yml * fix .github/workflows/build.yml * remove cache * add .github/workflows/release.yml * add .github/workflows/release.yml * .github/workflows/build.yml enable ARM * remote/serial.go fix syscall.Dup2 for ARM * .github/workflows/release.yml enable ARM * .github/workflows/release.yml delete zip * add aws/xcAwsInventory.py * aws/xcAwsInventory.py: rename iniFileName -> iniFilePath * aws/xcAwsInventory.py fix python run * README.md added section "remote environment settings" * xcAwsInventory.py: fixed groupName and added "workgroups"
This commit is contained in:
Grigory Efimov
20
.github/workflows/release.yml
vendored
20
.github/workflows/release.yml
vendored
@@ -30,7 +30,7 @@ jobs:
|
||||
matrix:
|
||||
arch:
|
||||
- amd64
|
||||
#- arm64
|
||||
- arm64
|
||||
os:
|
||||
- linux
|
||||
go-version:
|
||||
@@ -38,8 +38,8 @@ jobs:
|
||||
include:
|
||||
- arch: amd64
|
||||
rpm_arch: x86_64
|
||||
# - arch: arm64
|
||||
# rpm_arch: aarch64
|
||||
- arch: arm64
|
||||
rpm_arch: aarch64
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v4
|
||||
@@ -65,9 +65,7 @@ jobs:
|
||||
|
||||
# create asset {{
|
||||
- name: create archives
|
||||
run: |
|
||||
zip --junk-paths ${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip bin/*
|
||||
tar --create --gzip --verbose --exclude='.gitignore' --file=${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.tgz --directory=bin/ .
|
||||
run: tar --create --gzip --verbose --exclude='.gitignore' --file=${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.tgz --directory=bin/ .
|
||||
- name: create package deb
|
||||
uses: bpicode/github-action-fpm@master
|
||||
with:
|
||||
@@ -85,16 +83,6 @@ jobs:
|
||||
ls -al ./
|
||||
|
||||
# upload-release-asset {{
|
||||
- name: upload-release-asset zip
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ needs.create_release.outputs.upload_url }}
|
||||
asset_path: ./${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip
|
||||
asset_name: ${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip
|
||||
asset_content_type: application/zip
|
||||
|
||||
- name: upload-release-asset tgz
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
|
||||
1
Makefile
1
Makefile
@@ -17,3 +17,4 @@ build:
|
||||
-o bin/xc \
|
||||
-ldflags="$(FLAGS)" \
|
||||
cmd/xc/main.go
|
||||
cp aws/xcAwsInventory.py bin/
|
||||
|
||||
12
README.md
12
README.md
@@ -143,3 +143,15 @@ The only mandatory option is `path` which is used to load your library by xc its
|
||||
Any time xc needs a password for a host, the `GetPass(hostname string)string` function is called. This behaviour may be temporarily turned off by typing `use_password_manager off`.
|
||||
|
||||
There may be an optional `PrintDebug()` function in your plugin which is accessible by typing `_passmgr_debug`. This is useful to debug your code.
|
||||
|
||||
## integration for aws ec2
|
||||
[generate xcdata.ini from aws ec2 cli](aws/README.md)
|
||||
|
||||
## remote environment settings
|
||||
example
|
||||
```
|
||||
[remote_environ]
|
||||
PATH = /opt/puppetlabs/bin:$PATH:/usr/sbin:/sbin
|
||||
LC_ALL = en_US.UTF8
|
||||
FACTERLIB = /opt/puppetlabs/puppet/cache/lib/facter
|
||||
```
|
||||
|
||||
21
aws/README.md
Normal file
21
aws/README.md
Normal file
@@ -0,0 +1,21 @@
|
||||
# xcAwsInventory.py
|
||||
generate xcdata.ini from aws ec2
|
||||
|
||||
## dependencies
|
||||
* python3
|
||||
* aws cli
|
||||
|
||||
## usage
|
||||
just run xcAwsInventory.py
|
||||
|
||||
## settings
|
||||
config files paths: /etc/xcAwsInventory/config.yaml, ~/.xcAwsInventory.yaml
|
||||
default settings:
|
||||
```
|
||||
logFile: 'stdout'
|
||||
logLevel: 'info'
|
||||
regions: [] # all regions
|
||||
iniFilePath: '~/xcdata.ini' # path to result ini file
|
||||
tagForMainGroup: 'Name' # tag 'Key' for 'mainGroup'
|
||||
tagForParentGroup: 'role' # tag 'Key' for 'parentGroup'
|
||||
```
|
||||
263
aws/xcAwsInventory.py
Executable file
263
aws/xcAwsInventory.py
Executable file
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# base import {{
|
||||
import os
|
||||
from os.path import expanduser
|
||||
import sys
|
||||
import argparse
|
||||
import yaml
|
||||
import re
|
||||
import logging
|
||||
import inspect
|
||||
# }}
|
||||
|
||||
# local import {{
|
||||
import traceback
|
||||
import subprocess
|
||||
import json
|
||||
# }}
|
||||
|
||||
# base {{
|
||||
# default vars
|
||||
scriptName = os.path.basename(sys.argv[0]).split('.')[0]
|
||||
homeDir = expanduser("~")
|
||||
defaultConfigFiles = [
|
||||
'/etc/' + scriptName + '/config.yaml',
|
||||
homeDir + '/.' + scriptName + '.yaml',
|
||||
]
|
||||
cfg = {
|
||||
'logFile': '/var/log/' + scriptName + '/' + scriptName + '.log',
|
||||
'logFile': 'stdout',
|
||||
'logLevel': 'info',
|
||||
'regions': [],
|
||||
'iniFilePath': '~/xcdata.ini',
|
||||
'tagForMainGroup': 'Name', # имя тега для "основной группы" хоста, все остальные запихнуться в "parent" или в "tags"
|
||||
'tagForParentGroup': 'role', # имя тега для "родительской группы"
|
||||
'workgroup': 'devops', # имя для дефолтной workgroup
|
||||
}
|
||||
|
||||
# parse args
|
||||
parser = argparse.ArgumentParser( description = '''
|
||||
default config files: %s
|
||||
|
||||
''' % ', '.join(defaultConfigFiles),
|
||||
formatter_class=argparse.RawTextHelpFormatter
|
||||
)
|
||||
parser.add_argument(
|
||||
'-c',
|
||||
'--config',
|
||||
help = 'path to config file',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
argConfigFile = args.config
|
||||
|
||||
# get settings
|
||||
if argConfigFile:
|
||||
if os.path.isfile(argConfigFile):
|
||||
try:
|
||||
with open(argConfigFile, 'r') as ymlfile:
|
||||
cfg.update(yaml.load(ymlfile,Loader=yaml.Loader))
|
||||
except Exception as e:
|
||||
logging.error('failed load config file: "%s", error: "%s"', argConfigFile, e)
|
||||
exit(1)
|
||||
else:
|
||||
for configFile in defaultConfigFiles:
|
||||
if os.path.isfile(configFile):
|
||||
try:
|
||||
with open(configFile, 'r') as ymlfile:
|
||||
try:
|
||||
cfg.update(yaml.load(ymlfile,Loader=yaml.Loader))
|
||||
except Exception as e:
|
||||
logging.warning('skipping load load config file: "%s", error "%s"', configFile, e)
|
||||
continue
|
||||
except:
|
||||
continue
|
||||
|
||||
# fix logDir
|
||||
cfg['logDir'] = os.path.dirname(cfg['logFile'])
|
||||
if cfg['logDir'] == '':
|
||||
cfg['logDir'] = '.'
|
||||
# }}
|
||||
|
||||
# defs
|
||||
def runCmd(commands,communicate=True,stdoutJson=True):
|
||||
""" запуск shell команд, вернёт хеш:
|
||||
{
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
"exitCode": exitCode,
|
||||
}
|
||||
"""
|
||||
|
||||
defName = inspect.stack()[0][3]
|
||||
logging.debug("%s: '%s'" % (defName,commands))
|
||||
if communicate:
|
||||
process = subprocess.Popen('/bin/bash', stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
||||
out, err = process.communicate(commands.encode())
|
||||
returnCode = process.returncode
|
||||
try:
|
||||
outFormatted = out.rstrip().decode("utf-8")
|
||||
except:
|
||||
outFormatted = out.decode("utf-8")
|
||||
if stdoutJson:
|
||||
try:
|
||||
stdout = json.loads(outFormatted)
|
||||
except Exception:
|
||||
logging.error("%s: failed runCmd, cmd='%s', error='%s'" % (defName,commands,traceback.format_exc()))
|
||||
return None
|
||||
else:
|
||||
stdout = outFormatted
|
||||
return {
|
||||
"stdout": stdout,
|
||||
"stderr": err,
|
||||
"exitCode": returnCode,
|
||||
}
|
||||
else:
|
||||
subprocess.call(commands, shell=True)
|
||||
return None
|
||||
|
||||
def getInstancesInfo(region):
|
||||
defName = inspect.stack()[0][3]
|
||||
|
||||
instancesInfo = list()
|
||||
describeInstances = runCmd("aws ec2 --region %s describe-instances" % region)['stdout']
|
||||
reservations = describeInstances['Reservations']
|
||||
for reservation in reservations:
|
||||
instances = reservation['Instances']
|
||||
for instance in instances:
|
||||
if instance['State']['Name'] != 'running':
|
||||
continue
|
||||
info = {
|
||||
'dc': instance['Placement']['AvailabilityZone'],
|
||||
'tags': instance['Tags'],
|
||||
'host': instance['PublicDnsName'],
|
||||
}
|
||||
if info not in instancesInfo:
|
||||
instancesInfo.append(info)
|
||||
return instancesInfo
|
||||
|
||||
if __name__ == "__main__":
|
||||
# basic config {{
|
||||
for dirPath in [
|
||||
cfg['logDir'],
|
||||
]:
|
||||
try:
|
||||
os.makedirs(dirPath)
|
||||
except OSError:
|
||||
if not os.path.isdir(dirPath):
|
||||
raise
|
||||
|
||||
# выбор логлевела
|
||||
if re.match(r"^(warn|warning)$", cfg['logLevel'], re.IGNORECASE):
|
||||
logLevel = logging.WARNING
|
||||
elif re.match(r"^debug$", cfg['logLevel'], re.IGNORECASE):
|
||||
logLevel = logging.DEBUG
|
||||
else:
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
logging.getLogger("requests").setLevel(logging.WARNING)
|
||||
logLevel = logging.INFO
|
||||
|
||||
if cfg['logFile'] == 'stdout':
|
||||
logging.basicConfig(
|
||||
level = logLevel,
|
||||
format = '%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s',
|
||||
datefmt = '%Y-%m-%dT%H:%M:%S',
|
||||
)
|
||||
else:
|
||||
logging.basicConfig(
|
||||
filename = cfg['logFile'],
|
||||
level = logLevel,
|
||||
format = '%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s',
|
||||
datefmt = '%Y-%m-%dT%H:%M:%S',
|
||||
)
|
||||
# }}
|
||||
|
||||
defName = "main"
|
||||
if cfg['regions']:
|
||||
regions = cfg['regions']
|
||||
else:
|
||||
regions = runCmd("aws ec2 describe-regions --query 'Regions[].RegionName'")['stdout']
|
||||
instancesInfoArray = list()
|
||||
for region in regions:
|
||||
instancesInfoArray = instancesInfoArray + getInstancesInfo(region)
|
||||
|
||||
logging.debug("%s: instancesInfoArray='%s'" % (defName,json.dumps(instancesInfoArray,indent=4)))
|
||||
|
||||
datacenters = list()
|
||||
groups = list()
|
||||
hosts = list()
|
||||
for instanceInfo in instancesInfoArray:
|
||||
# добавляем датацентр в общий список дц
|
||||
if instanceInfo['dc'] not in datacenters:
|
||||
datacenters.append(instanceInfo['dc'])
|
||||
# формируем workgroups
|
||||
# workgroups это обязательный параметр, без него не будут работать tags=
|
||||
# берём дефолт из конфига
|
||||
workgroups = [ cfg['workgroup'] ]
|
||||
# проходиться по тегам и сортируем их по типам {{
|
||||
mainGroupName = None
|
||||
parentGroupName = None
|
||||
tagsGroupNames = None
|
||||
for tag in instanceInfo['tags']:
|
||||
groupName = 'tag' + '_' + tag['Key'].replace('-','_') + '_' + tag['Value'].replace('-','_')
|
||||
groupName = re.sub(r'\s+', '_', groupName)
|
||||
if tag['Key'] == cfg['tagForMainGroup']:
|
||||
# из этого тега делаем "основную группу"
|
||||
mainGroupName = groupName
|
||||
elif tag['Key'] == cfg['tagForParentGroup']:
|
||||
# из этого тега делаем "родительскую группу"
|
||||
parentGroupName = groupName
|
||||
# сразу добавляем её в список "всех групп" как есть
|
||||
if parentGroupName not in groups:
|
||||
groups.append(parentGroupName)
|
||||
else:
|
||||
# если ни с чём не совпало, то закидываем их в сущность 'tags'
|
||||
if tagsGroupNames:
|
||||
tagsGroupNames = tagsGroupNames + ',' + groupName
|
||||
else:
|
||||
tagsGroupNames = groupName
|
||||
# }}
|
||||
# формируем строку для группу в секции [groups] {{
|
||||
if mainGroupName:
|
||||
groupLine = mainGroupName + ' wg=' + cfg['workgroup']
|
||||
else:
|
||||
# mainGroupName обязательный
|
||||
logging.warning("%s: mainGroupName not found for host, instanceInfo='%s', skipping" % (defName,json.dumps(instanceInfo)))
|
||||
continue
|
||||
if parentGroupName:
|
||||
groupLine = groupLine + ' parent=' + parentGroupName
|
||||
if tagsGroupNames:
|
||||
groupLine = groupLine + ' tags=' + tagsGroupNames
|
||||
if groupLine not in groups:
|
||||
groups.append(groupLine)
|
||||
# }}
|
||||
|
||||
# формируем строку host в секции [hosts]
|
||||
host = instanceInfo['host'] + ' group=' + mainGroupName + ' dc=' + instanceInfo['dc']
|
||||
if host not in hosts:
|
||||
hosts.append(host)
|
||||
|
||||
logging.debug("%s: datacenters='%s'" % (defName,datacenters))
|
||||
logging.debug("%s: workgroups='%s'" % (defName,workgroups))
|
||||
logging.debug("%s: groups='%s'" % (defName,groups))
|
||||
logging.debug("%s: hosts='%s'" % (defName,hosts))
|
||||
|
||||
# генерим data file для инвентори {{
|
||||
config = '[datacenters]'
|
||||
for datacenter in datacenters:
|
||||
config = config + '\n' + str(datacenter)
|
||||
config = config + '\n\n' + '[workgroups]'
|
||||
for workgroup in workgroups:
|
||||
config = config + '\n' + str(workgroup)
|
||||
config = config + '\n\n' + '[groups]'
|
||||
for group in groups:
|
||||
config = config + '\n' + str(group)
|
||||
config = config + '\n\n' + '[hosts]'
|
||||
for host in hosts:
|
||||
config = config + '\n' + str(host)
|
||||
config = config + '\n'
|
||||
logging.debug("%s: config='%s'" % (defName,config))
|
||||
with open(os.path.expanduser(cfg['iniFilePath']), 'w') as f:
|
||||
f.write(config)
|
||||
# }}
|
||||
1
go.mod
1
go.mod
@@ -11,6 +11,7 @@ require (
|
||||
golang.org/x/crypto v0.1.0
|
||||
golang.org/x/sys v0.1.0
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.28
|
||||
github.com/ahmetb/govvv v0.3.0
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
2
go.sum
2
go.sum
@@ -41,3 +41,5 @@ golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
|
||||
github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0=
|
||||
github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s=
|
||||
|
||||
Reference in New Issue
Block a user