added xcAwsInventory.py (#15)

* add .github/workflows/build.yml
add Makefile
go.mod: add govvv
go.sum: add govvv

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* remote/serial.go "panic: bytes: negative Repeat count"

* fix .github/workflows/build.yml

* fix .github/workflows/build.yml

* remove cache

* add .github/workflows/release.yml

* add .github/workflows/release.yml

* .github/workflows/build.yml enable ARM

* remote/serial.go fix syscall.Dup2 for ARM

* .github/workflows/release.yml enable ARM

* .github/workflows/release.yml delete zip

* add aws/xcAwsInventory.py

* aws/xcAwsInventory.py: rename iniFileName -> iniFilePath

* aws/xcAwsInventory.py fix python run

* README.md added section "remote environment settings"

* xcAwsInventory.py: fixed groupName and added "workgroups"

.github/workflows/release.yml

@@ -30,7 +30,7 @@ jobs:
       matrix:
         arch:
           - amd64
-          #- arm64
+          - arm64
         os:
           - linux
         go-version:
@@ -38,8 +38,8 @@ jobs:
         include:
           - arch: amd64
             rpm_arch: x86_64
-        #  - arch: arm64
+          - arch: arm64
-        #    rpm_arch: aarch64
+            rpm_arch: aarch64
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
@@ -65,9 +65,7 @@ jobs:
 
       # create asset {{
       - name: create archives
-        run: |
+        run: tar --create --gzip --verbose --exclude='.gitignore' --file=${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.tgz --directory=bin/ .
-          zip --junk-paths ${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip bin/*
-          tar --create --gzip --verbose --exclude='.gitignore' --file=${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.tgz --directory=bin/ .
       - name: create package deb
         uses: bpicode/github-action-fpm@master
         with:
@@ -85,16 +83,6 @@ jobs:
           ls -al ./
 
       # upload-release-asset {{
-      - name: upload-release-asset zip
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ needs.create_release.outputs.upload_url }}
-          asset_path: ./${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip
-          asset_name: ${{ github.event.repository.name }}-${{ steps.release-version.outputs.RELEASE_VERSION }}.${{ matrix.os }}-${{ matrix.arch }}.zip
-          asset_content_type: application/zip
-
       - name: upload-release-asset tgz
         uses: actions/upload-release-asset@v1
         env:

Makefile

@@ -17,3 +17,4 @@ build:
 		-o bin/xc \
 		-ldflags="$(FLAGS)" \
 		cmd/xc/main.go
+	cp aws/xcAwsInventory.py bin/

README.md

@@ -143,3 +143,15 @@ The only mandatory option is `path` which is used to load your library by xc its
 Any time xc needs a password for a host, the `GetPass(hostname string)string` function is called. This behaviour may be temporarily turned off by typing `use_password_manager off`.
 
 There may be an optional `PrintDebug()` function in your plugin which is accessible by typing `_passmgr_debug`. This is useful to debug your code.
+
+## integration for aws ec2
+[generate xcdata.ini from aws ec2 cli](aws/README.md)
+
+## remote environment settings
+example
+```
+[remote_environ]
+PATH = /opt/puppetlabs/bin:$PATH:/usr/sbin:/sbin
+LC_ALL = en_US.UTF8
+FACTERLIB = /opt/puppetlabs/puppet/cache/lib/facter
+```

aws/README.md

@@ -0,0 +1,21 @@
+# xcAwsInventory.py
+generate xcdata.ini from aws ec2
+
+## dependencies
+* python3
+* aws cli
+
+## usage
+just run xcAwsInventory.py
+
+## settings
+config files paths: /etc/xcAwsInventory/config.yaml, ~/.xcAwsInventory.yaml
+default settings:
+```
+logFile: 'stdout'
+logLevel: 'info'
+regions: [] # all regions
+iniFilePath: '~/xcdata.ini' # path to result ini file
+tagForMainGroup: 'Name' # tag 'Key' for 'mainGroup'
+tagForParentGroup: 'role' # tag 'Key' for 'parentGroup'
+```

aws/xcAwsInventory.py

@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# base import {{
+import os
+from os.path import expanduser
+import sys
+import argparse
+import yaml
+import re
+import logging
+import inspect
+# }}
+
+# local import {{
+import traceback
+import subprocess
+import json
+# }}
+
+# base {{
+# default vars
+scriptName          = os.path.basename(sys.argv[0]).split('.')[0]
+homeDir             = expanduser("~")
+defaultConfigFiles  = [
+    '/etc/' + scriptName + '/config.yaml',
+    homeDir + '/.' + scriptName + '.yaml',
+]
+cfg = {
+    'logFile': '/var/log/' + scriptName + '/' + scriptName + '.log',
+    'logFile': 'stdout',
+    'logLevel': 'info',
+    'regions': [],
+    'iniFilePath': '~/xcdata.ini',
+    'tagForMainGroup': 'Name', # имя тега для "основной группы" хоста, все остальные запихнуться в "parent" или в "tags"
+    'tagForParentGroup': 'role', # имя тега для "родительской группы"
+    'workgroup': 'devops', # имя для дефолтной workgroup
+}
+
+# parse args
+parser = argparse.ArgumentParser( description = '''
+default config files: %s
+
+''' % ', '.join(defaultConfigFiles),
+formatter_class=argparse.RawTextHelpFormatter
+)
+parser.add_argument(
+    '-c',
+    '--config',
+    help = 'path to config file',
+)
+args = parser.parse_args()
+argConfigFile = args.config
+
+# get settings
+if argConfigFile:
+    if os.path.isfile(argConfigFile):
+        try:
+            with open(argConfigFile, 'r') as ymlfile:
+                cfg.update(yaml.load(ymlfile,Loader=yaml.Loader))
+        except Exception as e:
+            logging.error('failed load config file: "%s", error: "%s"', argConfigFile, e)
+            exit(1)
+else:
+    for configFile in defaultConfigFiles:
+        if os.path.isfile(configFile):
+            try:
+                with open(configFile, 'r') as ymlfile:
+                    try:
+                        cfg.update(yaml.load(ymlfile,Loader=yaml.Loader))
+                    except Exception as e:
+                        logging.warning('skipping load load config file: "%s", error "%s"', configFile, e)
+                        continue
+            except:
+                continue
+
+# fix logDir
+cfg['logDir'] = os.path.dirname(cfg['logFile'])
+if cfg['logDir'] == '':
+    cfg['logDir'] = '.'
+# }}
+
+# defs
+def runCmd(commands,communicate=True,stdoutJson=True):
+    """ запуск shell команд, вернёт хеш:
+    {
+        "stdout": stdout,
+        "stderr": stderr,
+        "exitCode": exitCode,
+    }
+    """
+
+    defName = inspect.stack()[0][3]
+    logging.debug("%s: '%s'" % (defName,commands))
+    if communicate:
+        process = subprocess.Popen('/bin/bash', stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+        out, err = process.communicate(commands.encode())
+        returnCode = process.returncode
+        try:
+            outFormatted = out.rstrip().decode("utf-8")
+        except:
+            outFormatted = out.decode("utf-8")
+        if stdoutJson:
+            try:
+                stdout = json.loads(outFormatted)
+            except Exception:
+                logging.error("%s: failed runCmd, cmd='%s', error='%s'" % (defName,commands,traceback.format_exc()))
+                return None
+        else:
+            stdout = outFormatted
+        return {
+            "stdout": stdout,
+            "stderr": err,
+            "exitCode": returnCode,
+        }
+    else:
+        subprocess.call(commands, shell=True)
+        return None
+
+def getInstancesInfo(region):
+    defName = inspect.stack()[0][3]
+
+    instancesInfo = list()
+    describeInstances = runCmd("aws ec2 --region %s describe-instances" % region)['stdout']
+    reservations = describeInstances['Reservations']
+    for reservation in reservations:
+        instances = reservation['Instances']
+        for instance in instances:
+            if instance['State']['Name'] != 'running':
+                continue
+            info = {
+                'dc': instance['Placement']['AvailabilityZone'],
+                'tags': instance['Tags'],
+                'host': instance['PublicDnsName'],
+            }
+            if info not in instancesInfo:
+                instancesInfo.append(info)
+    return instancesInfo
+
+if __name__ == "__main__":
+    # basic config {{
+    for dirPath in [
+        cfg['logDir'],
+    ]:
+        try:
+            os.makedirs(dirPath)
+        except OSError:
+            if not os.path.isdir(dirPath):
+                raise
+
+    # выбор логлевела
+    if re.match(r"^(warn|warning)$", cfg['logLevel'], re.IGNORECASE):
+        logLevel = logging.WARNING
+    elif re.match(r"^debug$", cfg['logLevel'], re.IGNORECASE):
+        logLevel = logging.DEBUG
+    else:
+        logging.getLogger("urllib3").setLevel(logging.WARNING)
+        logging.getLogger("requests").setLevel(logging.WARNING)
+        logLevel = logging.INFO
+
+    if cfg['logFile'] == 'stdout':
+        logging.basicConfig(
+            level       = logLevel,
+            format      = '%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s',
+            datefmt     = '%Y-%m-%dT%H:%M:%S',
+        )
+    else:
+        logging.basicConfig(
+            filename    = cfg['logFile'],
+            level       = logLevel,
+            format      = '%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s',
+            datefmt     = '%Y-%m-%dT%H:%M:%S',
+        )
+    # }}
+
+    defName = "main"
+    if cfg['regions']:
+        regions = cfg['regions']
+    else:
+        regions = runCmd("aws ec2 describe-regions --query 'Regions[].RegionName'")['stdout']
+    instancesInfoArray = list()
+    for region in regions:
+        instancesInfoArray = instancesInfoArray + getInstancesInfo(region)
+
+    logging.debug("%s: instancesInfoArray='%s'" % (defName,json.dumps(instancesInfoArray,indent=4)))
+
+    datacenters = list()
+    groups = list()
+    hosts = list()
+    for instanceInfo in instancesInfoArray:
+        # добавляем датацентр в общий список дц
+        if instanceInfo['dc'] not in datacenters:
+            datacenters.append(instanceInfo['dc'])
+        # формируем workgroups
+        # workgroups это обязательный параметр, без него не будут работать tags=
+        # берём дефолт из конфига
+        workgroups = [ cfg['workgroup'] ]
+        # проходиться по тегам и сортируем их по типам {{
+        mainGroupName = None
+        parentGroupName = None
+        tagsGroupNames = None
+        for tag in instanceInfo['tags']:
+            groupName = 'tag' + '_' + tag['Key'].replace('-','_') + '_' + tag['Value'].replace('-','_')
+            groupName = re.sub(r'\s+', '_', groupName)
+            if tag['Key'] == cfg['tagForMainGroup']:
+                # из этого тега делаем "основную группу"
+                mainGroupName = groupName
+            elif tag['Key'] == cfg['tagForParentGroup']:
+                # из этого тега делаем "родительскую группу"
+                parentGroupName = groupName
+                # сразу добавляем её в список "всех групп" как есть
+                if parentGroupName not in groups:
+                    groups.append(parentGroupName)
+            else:
+                # если ни с чём не совпало, то закидываем их в сущность 'tags'
+                if tagsGroupNames:
+                    tagsGroupNames = tagsGroupNames + ',' + groupName
+                else:
+                    tagsGroupNames = groupName
+        # }}
+        # формируем строку для группу в секции [groups] {{
+        if mainGroupName:
+            groupLine = mainGroupName + ' wg=' + cfg['workgroup']
+        else:
+            # mainGroupName обязательный
+            logging.warning("%s: mainGroupName not found for host, instanceInfo='%s', skipping" % (defName,json.dumps(instanceInfo)))
+            continue
+        if parentGroupName:
+            groupLine = groupLine + ' parent=' + parentGroupName
+        if tagsGroupNames:
+            groupLine = groupLine + ' tags=' + tagsGroupNames
+        if groupLine not in groups:
+            groups.append(groupLine)
+        # }}
+
+        # формируем строку host в секции [hosts]
+        host = instanceInfo['host'] + ' group=' + mainGroupName + ' dc=' + instanceInfo['dc']
+        if host not in hosts:
+            hosts.append(host)
+
+    logging.debug("%s: datacenters='%s'" % (defName,datacenters))
+    logging.debug("%s: workgroups='%s'" % (defName,workgroups))
+    logging.debug("%s: groups='%s'" % (defName,groups))
+    logging.debug("%s: hosts='%s'" % (defName,hosts))
+
+    # генерим data file для инвентори {{
+    config = '[datacenters]'
+    for datacenter in datacenters:
+        config = config + '\n' + str(datacenter)
+    config = config + '\n\n' + '[workgroups]'
+    for workgroup in workgroups:
+        config = config + '\n' + str(workgroup)
+    config = config + '\n\n' + '[groups]'
+    for group in groups:
+        config = config + '\n' + str(group)
+    config = config + '\n\n' + '[hosts]'
+    for host in hosts:
+        config = config + '\n' + str(host)
+    config = config + '\n'
+    logging.debug("%s: config='%s'" % (defName,config))
+    with open(os.path.expanduser(cfg['iniFilePath']), 'w') as f:
+        f.write(config)
+    # }}

go.mod

@@ -11,6 +11,7 @@ require (
 	golang.org/x/crypto v0.1.0
 	golang.org/x/sys v0.1.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
+	github.com/ahmetb/govvv v0.3.0
 )
 
 require (

go.sum

@@ -41,3 +41,5 @@ golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=
 gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0=
+github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s=