risotto/bootstrap.py

#!/usr/bin/env python3

from asyncio import run
from os import listdir, link, makedirs, environ
from os.path import isdir, isfile, join
from shutil import rmtree, copy2, copytree
from json import load as json_load
from yaml import load, SafeLoader
from toml import load as toml_load
from pprint import pprint
from typing import Any
from warnings import warn_explicit
from copy import copy

from tiramisu import Config
from tiramisu.error import ValueWarning
from rougail import RougailConfig, RougailConvert, RougailSystemdTemplate
from rougail.utils import normalize_family

from risotto.utils import MULTI_FUNCTIONS, CONFIGS


with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh:
    config = toml_load(fh)


DATASET_DIRECTORY = config['directories']['dataset']
INSTALL_DIR = config['directories']['dest']
FUNCTIONS = 'funcs.py'
CONFIG_DEST_DIR = 'configurations'
SRV_DEST_DIR = 'srv'


with open('servers.json', 'r') as server_fh:
    jsonfile = json_load(server_fh)
    SERVERS = jsonfile['servers']
    MODULES = jsonfile['modules']


async def set_linked_multi_variables(value: str,
                                     linked_server: str=None,
                                     **kwargs: dict,
                                     ) -> None:
    if value is not None and linked_server is not None and 'linked_value_0' not in kwargs:
        kwargs['linked_value_0'] = value
    elif linked_server is None:
        linked_server = value
    if linked_server is None:
        return
    if linked_server not in CONFIGS:
        warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
                      ValueWarning,
                      __file__,
                      3,
                      )
        return
    config = CONFIGS[linked_server][0]
    dico = {}
    variables = {}
    for key, value in kwargs.items():
        if value is None:
            return
        if key.startswith('linked_provider_'):
            index = int(key[16])
            path = await config.information.get('provider:' + value, None)
            if not path:
                return
            if index not in variables:
                variables[index] = {'path': None, 'value': None}
            variables[index]['path'] = path
        elif key.startswith('linked_value_'):
            index = int(key[13])
            if index not in variables:
                variables[index] = {'path': None, 'value': None}
            variables[index]['value'] = value
        else:
            raise AttributeError(f'unknown parameter {key}')
    dynamic = None
    slave_idx = None
    await config.property.read_write()
    first_variable = variables[0]['value']
    if not isinstance(variables[0]['value'], list):
        variables[0]['value'] = [variables[0]['value']]
    try:
        for var_index in range(len(variables[0]['value'])):
            pass
            for index in sorted(list(variables)):
                path = variables[index]['path']
                if index == 0:
                    value = variables[index]['value'][var_index]
                    dynamic = normalize_family(value)
                else:
                    value = variables[index]['value']
                    path = path.replace('{suffix}', dynamic)
                option = config.forcepermissive.option(path, slave_idx)
                multi = await option.option.ismulti()
                if multi and await option.option.isfollower():
                    multi = await option.option.issubmulti()
                if multi:
                    values = await option.value.get()
                    if value not in values:
                        values.append(value)
                        await option.value.set(values)
                    if await option.option.isleader():
                        slave_idx = values.index(value)
                else:
                    if isinstance(value, list):
                        value = value[var_index]
                    await option.value.set(value)
    except Exception as err:
        await config.property.read_only()
        raise err from err
    await config.property.read_only()


async def set_linked(linked_server: str,
                     linked_provider: str,
                     linked_value: str,
                     linked_returns: str=None,
                     dynamic: str=None,
                     ):
    if None in (linked_server, linked_provider, linked_value):
        return
    if linked_server not in CONFIGS:
        warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
                      ValueWarning,
                      __file__,
                      0,
                      )
        return
    config = CONFIGS[linked_server][0]
    path = await config.information.get('provider:' + linked_provider, None)
    if not path:
        warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'),
                      ValueWarning,
                      __file__,
                      0,
                      )
        return
    await config.property.read_write()
    try:
        option = config.forcepermissive.option(path)
        if await option.option.ismulti():
            values = await option.value.get()
            if linked_value not in values:
                values.append(linked_value)
                await option.value.set(values)
        else:
            await option.value.set(linked_value)
    except Exception as err:
        await config.property.read_only()
        raise err from err
    await config.property.read_only()
    if linked_returns is not None:
        linked_variable = await config.information.get('provider:' + linked_returns, None)
        if not linked_variable:
            warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'),
                          ValueWarning,
                          __file__,
                          0,
                          )
            return
    else:
        linked_variable = None
    if linked_variable is not None:
        if dynamic:
            linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic))
        elif '{suffix}' in linked_variable:
            idx = CONFIGS[linked_server][3]
            linked_variable = linked_variable.replace('{suffix}', str(idx))
        ret = await config.forcepermissive.option(linked_variable).value.get()
    else:
        ret = normalize_family(linked_value)
    return ret


async def get_linked_configuration(linked_server: str,
                                   linked_provider: str,
                                   dynamic: str=None,
                                   ):
    if linked_server not in CONFIGS:
        warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
                      ValueWarning,
                      __file__,
                      1,
                      )
        return
    config = CONFIGS[linked_server][0]
    path = await config.information.get('provider:' + linked_provider, None)
    if not path:
        warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
                      ValueWarning,
                      __file__,
                      1,
                      )
        return
    if dynamic:
        path = path.replace('{suffix}', normalize_family(dynamic))
    try:
        return await config.forcepermissive.option(path).value.get()
    except AttributeError as err:
        warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'),
                      ValueWarning,
                      __file__,
                      1,
                      )


class Empty:
    pass
empty = Empty()


async def set_linked_configuration(_linked_value: Any,
                                   linked_server: str,
                                   linked_provider: str,
                                   linked_value: Any=empty,
                                   dynamic: str=None,
                                   leader_provider: str=None,
                                   leader_value: Any=None,
                                   leader_index: int=None,
                                   ):
    if linked_value is not empty:
        _linked_value = linked_value
    linked_value = _linked_value
    if linked_server is None:
        return
    if linked_value is None or linked_server not in CONFIGS:
        warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
                      ValueWarning,
                      __file__,
                      2,
                      )
        return
    config = CONFIGS[linked_server][0]
    path = await config.information.get('provider:' + linked_provider, None)
    if not path:
        warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
                      ValueWarning,
                      __file__,
                      2,
                      )
        return
    if dynamic:
        path = path.replace('{suffix}', normalize_family(dynamic))
    await config.property.read_write()
    try:
        if leader_provider is not None:
            leader_path = await config.information.get('provider:' + leader_provider, None)
            if not leader_path:
                await config.property.read_only()
                warn_explicit(ValueWarning(f'cannot find leader variable with leader_provider "{leader_provider}" in linked server "{linked_server}"'),
                              ValueWarning,
                              __file__,
                              2,
                              )
                return
            if dynamic:
                leader_path = leader_path.replace('{suffix}', normalize_family(dynamic))
            values = await config.forcepermissive.option(leader_path).value.get()
            if not isinstance(leader_value, list):
                leader_value = [leader_value]
            for lv in leader_value:
                if lv in values:
                    slave_idx = values.index(lv)
                    slave_option = config.forcepermissive.option(path, slave_idx)
                    if await slave_option.option.issubmulti():
                        slave_values = await slave_option.value.get()
                        if linked_value not in slave_values:
                            slave_values.append(linked_value)
                            await slave_option.value.set(slave_values)
                    else:
                        await slave_option.value.set(linked_value)
        else:
            option = config.forcepermissive.option(path, leader_index)
            if leader_index is None and await option.option.ismulti() and not isinstance(linked_value, list):
                values = await option.value.get()
                if linked_value not in values:
                    values.append(linked_value)
                    await option.value.set(values)
            else:
                await option.value.set(linked_value)
    except AttributeError as err:
        #raise ValueError(str(err)) from err
        pass
    except Exception as err:
        await config.property.read_only()
        raise err from err
    await config.property.read_only()


def tiramisu_display_name(kls,
                          dyn_name: 'Base'=None,
                          suffix: str=None,
                          ) -> str:
    if dyn_name is not None:
        name = kls.impl_getpath() + suffix
    else:
        name = kls.impl_getpath()
    return name


def load_applications():
    applications = {}
    distrib_dir = join(DATASET_DIRECTORY, 'applicationservice')
    for release in listdir(distrib_dir):
        release_dir = join(distrib_dir, release)
        if not isdir(release_dir):
            continue
        for applicationservice in listdir(release_dir):
            applicationservice_dir = join(release_dir, applicationservice)
            if not isdir(applicationservice_dir):
                continue
            if applicationservice in applications:
                raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
            applications[applicationservice] = applicationservice_dir
    return applications


class ModuleCfg():
    def __init__(self):
        self.dictionaries_dir = []
        self.modules = []
        self.functions_file = [FUNCTIONS]
        self.templates_dir = []
        self.extra_dictionaries = {}
        self.servers = []


def build_module(module_name, datas, module_infos):
    install_dir = join(INSTALL_DIR, module_name)
    makedirs(install_dir)
    applications = load_applications()
    cfg = ModuleCfg()
    module_infos[module_name] = cfg
    def calc_depends(appname, added):
        if appname in added:
            return
        if appname not in applications:
            raise Exception(f'cannot find application dependency "{appname}" in application "{module_name}"')
        as_dir = applications[appname]
        cfg.modules.append(appname)
        dictionaries_dir = join(as_dir, 'dictionaries')
        if isdir(dictionaries_dir):
            cfg.dictionaries_dir.append(dictionaries_dir)
        funcs_dir = join(as_dir, 'funcs')
        if isdir(funcs_dir):
            for f in listdir(funcs_dir):
                if f.startswith('__'):
                    continue
                cfg.functions_file.append(join(funcs_dir, f))
        templates_dir = join(as_dir, 'templates')
        if isdir(templates_dir):
            cfg.templates_dir.append(templates_dir)
        extras_dir = join(as_dir, 'extras')
        if isdir(extras_dir):
            for extra in listdir(extras_dir):
                extra_dir = join(extras_dir, extra)
                if isdir(extra_dir):
                    cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
        for type in ['image', 'install']:
            manual_dir = join(as_dir, 'manual', type)
            if isdir(manual_dir):
                for filename in listdir(manual_dir):
                    src_file = join(manual_dir, filename)
                    if type == 'image':
                        dst_file = join(install_dir, filename)
                        verify = False
                    else:
                        dst_file= join(INSTALL_DIR, filename)
                        verify = True
                    if isdir(src_file):
                        if not isdir(dst_file):
                            makedirs(dst_file)
                        for subfilename in listdir(src_file):
                            if not verify or not isfile(dst_file):
                                src = join(src_file, subfilename)
                                dst = join(dst_file, subfilename)
                                if isfile(src):
                                    copy2(src, dst)
                                else:
                                    copytree(src, dst)
                    elif not verify or not isfile(dst_file):
                        src = join(manual_dir, filename)
                        dst = dst_file
                        if isfile(src):
                            copy2(src, dst)
                        else:
                            copytree(src, dst)
        added.append(appname)
        with open(join(as_dir, 'applicationservice.yml')) as yaml:
            app = load(yaml, Loader=SafeLoader)

        for xml in app.get('depends', []):
            calc_depends(xml, added)
    added = []
    for applicationservice in datas['applicationservices']:
        calc_depends(applicationservice, added)


async def build(server_name, datas, module_infos):
    if server_name in CONFIGS:
        raise Exception(f'server "{server_name}" is duplicate')
    cfg = RougailConfig.copy()
    module_info = module_infos[datas['module']]
    module_info.servers.append(server_name)
    if datas['module'] == 'host':
        cfg['tmpfile_dest_dir'] = datas['values']['rougail.host_install_dir'] + '/host/configurations/' + server_name
    cfg['templates_dir'] = module_info.templates_dir
    cfg['dictionaries_dir'] = module_info.dictionaries_dir
    cfg['functions_file'] = module_info.functions_file
    cfg['multi_functions'] = MULTI_FUNCTIONS 
    cfg['extra_dictionaries'] = module_info.extra_dictionaries
    cfg['extra_annotators'].append('risotto.rougail')
    optiondescription = {'set_linked': set_linked,
                         'set_linked_multi_variables': set_linked_multi_variables,
                         'get_linked_configuration': get_linked_configuration,
                         'set_linked_configuration': set_linked_configuration,
                         }
    cfg['internal_functions'] = list(optiondescription.keys())
    try:
        eolobj = RougailConvert(cfg)
    except Exception as err:
        print(f'Try to load {module_info.modules}')
        raise err from err
    tiram_obj = eolobj.save(None)
#    if server_name == 'revprox.in.silique.fr':
#        print(tiram_obj)
    #cfg['patches_dir'] = join(test_dir, 'patches')
    cfg['tmp_dir'] = 'tmp'
    cfg['destinations_dir'] = join(INSTALL_DIR, datas['module'], CONFIG_DEST_DIR, server_name)
    if isdir('tmp'):
        rmtree('tmp')
    makedirs('tmp')
    makedirs(cfg['destinations_dir'])
    try:
        exec(tiram_obj, None, optiondescription)
    except Exception as err:
        print(tiram_obj)
        raise Exception(f'unknown error when load tiramisu object {err}') from err
    config = await Config(optiondescription['option_0'], display_name=tiramisu_display_name)
    await config.property.read_write()
    try:
        if await config.option('machine.add_srv').value.get():
            srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name)
        else:
            srv = None
    except AttributeError:
        srv = None
    await config.property.read_write()
    CONFIGS[server_name] = (config, cfg, srv, 0)


async def value_pprint(dico, config):
    pprint_dict = {}
    for path, value in dico.items():
        if await config.option(path).option.type() == 'password' and value:
            value = 'X' * len(value)
        pprint_dict[path] = value
    pprint(pprint_dict)


async def set_values(server_name, config, datas):
    if 'informations' in datas:
        for information, value in datas['informations'].items():
            await config.information.set(information, value)
        if 'extra_domainnames' in datas['informations']:
            for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']):
                if extra_domainname in CONFIGS:
                    raise Exception(f'server "{server_name}" is duplicate')
                value = list(CONFIGS[server_name])
                value[3] = idx + 1
                CONFIGS[extra_domainname] = tuple(value)
    await config.information.set('server_name', server_name)
    await config.property.read_write()
    try:
        if 'values' in datas:
            for path, value in datas['values'].items():
                if isinstance(value, dict):
                    for idx, val in value.items():
                        await config.option(path, int(idx)).value.set(val)
                else:
                    await config.option(path).value.set(value)
    except Exception as err:
        await value_pprint(await config.value.dict(), config)
        error_msg = f'cannot configure server "{server_name}": {err}'
        raise Exception(error_msg) from err
    await config.property.read_only()
    #await config.value.dict()


async def valid_mandatories(server_name, config):
    mandatories = await config.value.mandatory()
    if mandatories:
        print()
        print(f'=== Configuration: {server_name} ===')
        await config.property.pop('mandatory')
        await value_pprint(await config.value.dict(), config)
        raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"')


async def templates(server_name, config, cfg, srv, int_idx):
    values = await config.value.dict()
    engine = RougailSystemdTemplate(config, cfg)
#    if server_name == 'dovecot.in.silique.fr':
#        print()
#        print(f'=== Configuration: {server_name} ===')
#        pprint(values)
    try:
        await engine.instance_files()
    except Exception as err:
        print()
        print(f'=== Configuration: {server_name} ===')
        await value_pprint(values, config)
        raise err from err
    if srv:
        makedirs(srv)


async def main():
    if isdir(INSTALL_DIR):
        rmtree(INSTALL_DIR)
    makedirs(INSTALL_DIR)
    module_infos = {}
    for module_name, datas in MODULES.items():
        build_module(module_name, datas, module_infos)
    for server_name, datas in SERVERS.items():
        await build(server_name, datas, module_infos)
    for module_name, cfg in module_infos.items():
        with open(join(INSTALL_DIR, module_name, 'install_machines'), 'w') as fh:
            for server_name in cfg.servers:
                fh.write(f'./install_machine {module_name} {server_name}\n')
    for server_name, datas in SERVERS.items():
        await set_values(server_name, CONFIGS[server_name][0], datas)
    for server_name in SERVERS:
        config = CONFIGS[server_name][0]
        await config.property.pop('mandatory')
        await config.value.dict()
        await config.property.add('mandatory')
    for server_name in SERVERS:
        await valid_mandatories(server_name, CONFIGS[server_name][0])
#    print(await CONFIGS['revprox.in.gnunux.info'][0].option('nginx.reverse_proxy_for_netbox_in_gnunux_info.reverse_proxy_netbox_in_gnunux_info.revprox_url_netbox_in_gnunux_info', 0).value.get())
    for server_name in SERVERS:
        await templates(server_name, *CONFIGS[server_name])


run(main())