From bb7013f5effd7c2ba9f76ea6f319048eb35cdf9d Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Tue, 23 May 2023 13:36:37 +0300 Subject: [PATCH] added xcAwsInventory.py (#15) * add .github/workflows/build.yml add Makefile go.mod: add govvv go.sum: add govvv * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * fix .github/workflows/build.yml * remote/serial.go "panic: bytes: negative Repeat count" * fix .github/workflows/build.yml * fix .github/workflows/build.yml * remove cache * add .github/workflows/release.yml * add .github/workflows/release.yml * .github/workflows/build.yml enable ARM * remote/serial.go fix syscall.Dup2 for ARM * .github/workflows/release.yml enable ARM * .github/workflows/release.yml delete zip * add aws/xcAwsInventory.py * aws/xcAwsInventory.py: rename iniFileName -> iniFilePath * aws/xcAwsInventory.py fix python run * README.md added section "remote environment settings" * xcAwsInventory.py: fixed groupName and added "workgroups" --- .github/workflows/release.yml | 20 +-- Makefile | 1 + README.md | 12 ++ aws/README.md | 21 +++ aws/xcAwsInventory.py | 263 ++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + 7 files changed, 304 insertions(+), 16 deletions(-) create mode 100644 aws/README.md create mode 100755 aws/xcAwsInventory.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5fde5e..1d7b61f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: matrix: arch: - amd64 - #- arm64 + - arm64 os: - linux go-version: @@ -38,8 +38,8 @@ jobs: include: - arch: amd64 rpm_arch: x86_64 - # - arch: arm64 - # rpm_arch: aarch64 + - arch: arm64 + rpm_arch: aarch64 steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 @@ -65,9 +65,7 @@ jobs: # create asset {{ - name: create archives - run: | - zip --junk-paths ${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip bin/* - tar --create --gzip --verbose --exclude='.gitignore' --file=${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.tgz --directory=bin/ . + run: tar --create --gzip --verbose --exclude='.gitignore' --file=${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.tgz --directory=bin/ . - name: create package deb uses: bpicode/github-action-fpm@master with: @@ -85,16 +83,6 @@ jobs: ls -al ./ # upload-release-asset {{ - - name: upload-release-asset zip - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip - asset_name: ${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip - asset_content_type: application/zip - - name: upload-release-asset tgz uses: actions/upload-release-asset@v1 env: diff --git a/Makefile b/Makefile index e6d48c4..e435782 100644 --- a/Makefile +++ b/Makefile @@ -17,3 +17,4 @@ build: -o bin/xc \ -ldflags="$(FLAGS)" \ cmd/xc/main.go + cp aws/xcAwsInventory.py bin/ diff --git a/README.md b/README.md index b3d4ce9..d2555e0 100644 --- a/README.md +++ b/README.md @@ -143,3 +143,15 @@ The only mandatory option is `path` which is used to load your library by xc its Any time xc needs a password for a host, the `GetPass(hostname string)string` function is called. This behaviour may be temporarily turned off by typing `use_password_manager off`. There may be an optional `PrintDebug()` function in your plugin which is accessible by typing `_passmgr_debug`. This is useful to debug your code. + +## integration for aws ec2 +[generate xcdata.ini from aws ec2 cli](aws/README.md) + +## remote environment settings +example +``` +[remote_environ] +PATH = /opt/puppetlabs/bin:$PATH:/usr/sbin:/sbin +LC_ALL = en_US.UTF8 +FACTERLIB = /opt/puppetlabs/puppet/cache/lib/facter +``` diff --git a/aws/README.md b/aws/README.md new file mode 100644 index 0000000..8e0e21d --- /dev/null +++ b/aws/README.md @@ -0,0 +1,21 @@ +# xcAwsInventory.py +generate xcdata.ini from aws ec2 + +## dependencies +* python3 +* aws cli + +## usage +just run xcAwsInventory.py + +## settings +config files paths: /etc/xcAwsInventory/config.yaml, ~/.xcAwsInventory.yaml +default settings: +``` +logFile: 'stdout' +logLevel: 'info' +regions: [] # all regions +iniFilePath: '~/xcdata.ini' # path to result ini file +tagForMainGroup: 'Name' # tag 'Key' for 'mainGroup' +tagForParentGroup: 'role' # tag 'Key' for 'parentGroup' +``` diff --git a/aws/xcAwsInventory.py b/aws/xcAwsInventory.py new file mode 100755 index 0000000..361ee7f --- /dev/null +++ b/aws/xcAwsInventory.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# base import {{ +import os +from os.path import expanduser +import sys +import argparse +import yaml +import re +import logging +import inspect +# }} + +# local import {{ +import traceback +import subprocess +import json +# }} + +# base {{ +# default vars +scriptName = os.path.basename(sys.argv[0]).split('.')[0] +homeDir = expanduser("~") +defaultConfigFiles = [ + '/etc/' + scriptName + '/config.yaml', + homeDir + '/.' + scriptName + '.yaml', +] +cfg = { + 'logFile': '/var/log/' + scriptName + '/' + scriptName + '.log', + 'logFile': 'stdout', + 'logLevel': 'info', + 'regions': [], + 'iniFilePath': '~/xcdata.ini', + 'tagForMainGroup': 'Name', # имя тега для "основной группы" хоста, все остальные запихнуться в "parent" или в "tags" + 'tagForParentGroup': 'role', # имя тега для "родительской группы" + 'workgroup': 'devops', # имя для дефолтной workgroup +} + +# parse args +parser = argparse.ArgumentParser( description = ''' +default config files: %s + +''' % ', '.join(defaultConfigFiles), +formatter_class=argparse.RawTextHelpFormatter +) +parser.add_argument( + '-c', + '--config', + help = 'path to config file', +) +args = parser.parse_args() +argConfigFile = args.config + +# get settings +if argConfigFile: + if os.path.isfile(argConfigFile): + try: + with open(argConfigFile, 'r') as ymlfile: + cfg.update(yaml.load(ymlfile,Loader=yaml.Loader)) + except Exception as e: + logging.error('failed load config file: "%s", error: "%s"', argConfigFile, e) + exit(1) +else: + for configFile in defaultConfigFiles: + if os.path.isfile(configFile): + try: + with open(configFile, 'r') as ymlfile: + try: + cfg.update(yaml.load(ymlfile,Loader=yaml.Loader)) + except Exception as e: + logging.warning('skipping load load config file: "%s", error "%s"', configFile, e) + continue + except: + continue + +# fix logDir +cfg['logDir'] = os.path.dirname(cfg['logFile']) +if cfg['logDir'] == '': + cfg['logDir'] = '.' +# }} + +# defs +def runCmd(commands,communicate=True,stdoutJson=True): + """ запуск shell команд, вернёт хеш: + { + "stdout": stdout, + "stderr": stderr, + "exitCode": exitCode, + } + """ + + defName = inspect.stack()[0][3] + logging.debug("%s: '%s'" % (defName,commands)) + if communicate: + process = subprocess.Popen('/bin/bash', stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + out, err = process.communicate(commands.encode()) + returnCode = process.returncode + try: + outFormatted = out.rstrip().decode("utf-8") + except: + outFormatted = out.decode("utf-8") + if stdoutJson: + try: + stdout = json.loads(outFormatted) + except Exception: + logging.error("%s: failed runCmd, cmd='%s', error='%s'" % (defName,commands,traceback.format_exc())) + return None + else: + stdout = outFormatted + return { + "stdout": stdout, + "stderr": err, + "exitCode": returnCode, + } + else: + subprocess.call(commands, shell=True) + return None + +def getInstancesInfo(region): + defName = inspect.stack()[0][3] + + instancesInfo = list() + describeInstances = runCmd("aws ec2 --region %s describe-instances" % region)['stdout'] + reservations = describeInstances['Reservations'] + for reservation in reservations: + instances = reservation['Instances'] + for instance in instances: + if instance['State']['Name'] != 'running': + continue + info = { + 'dc': instance['Placement']['AvailabilityZone'], + 'tags': instance['Tags'], + 'host': instance['PublicDnsName'], + } + if info not in instancesInfo: + instancesInfo.append(info) + return instancesInfo + +if __name__ == "__main__": + # basic config {{ + for dirPath in [ + cfg['logDir'], + ]: + try: + os.makedirs(dirPath) + except OSError: + if not os.path.isdir(dirPath): + raise + + # выбор логлевела + if re.match(r"^(warn|warning)$", cfg['logLevel'], re.IGNORECASE): + logLevel = logging.WARNING + elif re.match(r"^debug$", cfg['logLevel'], re.IGNORECASE): + logLevel = logging.DEBUG + else: + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("requests").setLevel(logging.WARNING) + logLevel = logging.INFO + + if cfg['logFile'] == 'stdout': + logging.basicConfig( + level = logLevel, + format = '%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s', + datefmt = '%Y-%m-%dT%H:%M:%S', + ) + else: + logging.basicConfig( + filename = cfg['logFile'], + level = logLevel, + format = '%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s', + datefmt = '%Y-%m-%dT%H:%M:%S', + ) + # }} + + defName = "main" + if cfg['regions']: + regions = cfg['regions'] + else: + regions = runCmd("aws ec2 describe-regions --query 'Regions[].RegionName'")['stdout'] + instancesInfoArray = list() + for region in regions: + instancesInfoArray = instancesInfoArray + getInstancesInfo(region) + + logging.debug("%s: instancesInfoArray='%s'" % (defName,json.dumps(instancesInfoArray,indent=4))) + + datacenters = list() + groups = list() + hosts = list() + for instanceInfo in instancesInfoArray: + # добавляем датацентр в общий список дц + if instanceInfo['dc'] not in datacenters: + datacenters.append(instanceInfo['dc']) + # формируем workgroups + # workgroups это обязательный параметр, без него не будут работать tags= + # берём дефолт из конфига + workgroups = [ cfg['workgroup'] ] + # проходиться по тегам и сортируем их по типам {{ + mainGroupName = None + parentGroupName = None + tagsGroupNames = None + for tag in instanceInfo['tags']: + groupName = 'tag' + '_' + tag['Key'].replace('-','_') + '_' + tag['Value'].replace('-','_') + groupName = re.sub(r'\s+', '_', groupName) + if tag['Key'] == cfg['tagForMainGroup']: + # из этого тега делаем "основную группу" + mainGroupName = groupName + elif tag['Key'] == cfg['tagForParentGroup']: + # из этого тега делаем "родительскую группу" + parentGroupName = groupName + # сразу добавляем её в список "всех групп" как есть + if parentGroupName not in groups: + groups.append(parentGroupName) + else: + # если ни с чём не совпало, то закидываем их в сущность 'tags' + if tagsGroupNames: + tagsGroupNames = tagsGroupNames + ',' + groupName + else: + tagsGroupNames = groupName + # }} + # формируем строку для группу в секции [groups] {{ + if mainGroupName: + groupLine = mainGroupName + ' wg=' + cfg['workgroup'] + else: + # mainGroupName обязательный + logging.warning("%s: mainGroupName not found for host, instanceInfo='%s', skipping" % (defName,json.dumps(instanceInfo))) + continue + if parentGroupName: + groupLine = groupLine + ' parent=' + parentGroupName + if tagsGroupNames: + groupLine = groupLine + ' tags=' + tagsGroupNames + if groupLine not in groups: + groups.append(groupLine) + # }} + + # формируем строку host в секции [hosts] + host = instanceInfo['host'] + ' group=' + mainGroupName + ' dc=' + instanceInfo['dc'] + if host not in hosts: + hosts.append(host) + + logging.debug("%s: datacenters='%s'" % (defName,datacenters)) + logging.debug("%s: workgroups='%s'" % (defName,workgroups)) + logging.debug("%s: groups='%s'" % (defName,groups)) + logging.debug("%s: hosts='%s'" % (defName,hosts)) + + # генерим data file для инвентори {{ + config = '[datacenters]' + for datacenter in datacenters: + config = config + '\n' + str(datacenter) + config = config + '\n\n' + '[workgroups]' + for workgroup in workgroups: + config = config + '\n' + str(workgroup) + config = config + '\n\n' + '[groups]' + for group in groups: + config = config + '\n' + str(group) + config = config + '\n\n' + '[hosts]' + for host in hosts: + config = config + '\n' + str(host) + config = config + '\n' + logging.debug("%s: config='%s'" % (defName,config)) + with open(os.path.expanduser(cfg['iniFilePath']), 'w') as f: + f.write(config) + # }} diff --git a/go.mod b/go.mod index 84e984c..e36ea39 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( golang.org/x/crypto v0.1.0 golang.org/x/sys v0.1.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 + github.com/ahmetb/govvv v0.3.0 ) require ( diff --git a/go.sum b/go.sum index 36b8c0a..c5a88b7 100644 --- a/go.sum +++ b/go.sum @@ -41,3 +41,5 @@ golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0= +github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s=