#!/usr/bin/env python3 from asyncio import run from os import listdir, link, makedirs from os.path import isdir, isfile, join from shutil import rmtree, copy2, copytree from json import load as json_load from yaml import load, SafeLoader from pprint import pprint from typing import Any from warnings import warn_explicit from copy import copy from tiramisu import Config from tiramisu.error import ValueWarning from rougail import RougailConfig, RougailConvert, RougailSystemdTemplate from rougail.utils import normalize_family #from rougail.error import TemplateError from utils import MULTI_FUNCTIONS, CONFIGS DATASET_DIRECTORY = '/home/gnunux/git/risotto_cadoles/risotto-dataset/seed' FUNCTIONS = 'funcs.py' CONFIG_DEST_DIR = 'configurations' SRV_DEST_DIR = 'srv' INSTALL_DIR = 'installations' # "netbox.in.gnunux.info": {"applicationservices": ["netbox", "provider-systemd-machined"], # "informations": {"zones_name": ["gnunux"]}, # "values": {"rougail.postgresql.pg_client_server_domainname": "postgresql.in.gnunux.info", # "rougail.redis.redis_client_server_domainname": "redis.in.gnunux.info", # "rougail.nginx.revprox_client_server_domainname": "revprox.in.gnunux.info", # "rougail.nginx.revprox_client_external_domainname": "in.gnunux.info" # } # }, with open('servers.json', 'r') as server_fh: jsonfile = json_load(server_fh) SERVERS = jsonfile['servers'] MODULES = jsonfile['modules'] async def set_linked(linked_server: str, linked_provider: str, linked_value: str, linked_returns: str=None, dynamic: str=None, ): if None in (linked_server, linked_provider, linked_value): return if linked_server not in CONFIGS: warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), ValueWarning, __file__, 0, ) return config = CONFIGS[linked_server][0] path = await config.information.get('provider:' + linked_provider, None) if not path: warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'), ValueWarning, __file__, 0, ) return await config.property.read_write() try: option = config.forcepermissive.option(path) if await option.option.ismulti(): values = await option.value.get() if linked_value not in values: values.append(linked_value) await option.value.set(values) else: await option.value.set(linked_value) except Exception as err: await config.property.read_only() raise err from err await config.property.read_only() if linked_returns is not None: linked_variable = await config.information.get('provider:' + linked_returns, None) if not linked_variable: warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'), ValueWarning, __file__, 0, ) return else: linked_variable = None if linked_variable is not None: if dynamic: linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic)) elif '{suffix}' in linked_variable: idx = CONFIGS[linked_server][3] linked_variable = linked_variable.replace('{suffix}', str(idx)) ret = await config.forcepermissive.option(linked_variable).value.get() else: ret = normalize_family(linked_value) return ret async def get_linked_configuration(linked_server: str, linked_provider: str, dynamic: str=None, ): if linked_server not in CONFIGS: warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), ValueWarning, __file__, 1, ) return config = CONFIGS[linked_server][0] path = await config.information.get('provider:' + linked_provider, None) if not path: warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'), ValueWarning, __file__, 1, ) return if dynamic: path = path.replace('{suffix}', normalize_family(dynamic)) try: return await config.forcepermissive.option(path).value.get() except AttributeError as err: warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'), ValueWarning, __file__, 1, ) class Empty: pass empty = Empty() async def set_linked_configuration(_linked_value: Any, linked_server: str, linked_provider: str, linked_value: Any=empty, dynamic: str=None, leader_provider: str=None, leader_value: Any=None, ): if linked_value is not empty: _linked_value = linked_value linked_value = _linked_value if linked_server is None: return if linked_value is None or linked_server not in CONFIGS: warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), ValueWarning, __file__, 2, ) return config = CONFIGS[linked_server][0] path = await config.information.get('provider:' + linked_provider, None) if not path: warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'), ValueWarning, __file__, 2, ) return if dynamic: path = path.replace('{suffix}', normalize_family(dynamic)) await config.property.read_write() try: if leader_provider is not None: leader_path = await config.information.get('provider:' + leader_provider, None) if not leader_path: await config.property.read_only() warn_explicit(ValueWarning(f'cannot find leader variable "{path}" in linked server "{linked_server}"'), ValueWarning, __file__, 2, ) return if dynamic: leader_path = leader_path.replace('{suffix}', normalize_family(dynamic)) values = await config.forcepermissive.option(leader_path).value.get() if leader_value in values: slave_idx = values.index(leader_value) slave_option = config.forcepermissive.option(path, slave_idx) if await slave_option.option.issubmulti(): slave_values = await slave_option.value.get() if linked_value not in slave_values: slave_values.append(linked_value) await slave_option.value.set(slave_values) else: await slave_option.value.set(linked_value) else: option = config.forcepermissive.option(path) if await option.option.ismulti() and not isinstance(linked_value, list): values = await option.value.get() if linked_value not in values: values.append(linked_value) await option.value.set(values) else: await option.value.set(linked_value) except AttributeError as err: #raise ValueError(str(err)) from err pass except Exception as err: await config.property.read_only() raise err from err await config.property.read_only() def tiramisu_display_name(kls, dyn_name: 'Base'=None, suffix: str=None, ) -> str: if dyn_name is not None: name = kls.impl_getpath() + suffix else: name = kls.impl_getpath() return name def load_applications(): applications = {} for distrib in listdir(DATASET_DIRECTORY): distrib_dir = join(DATASET_DIRECTORY, distrib, 'applicationservice') if not isdir(distrib_dir): continue for release in listdir(distrib_dir): release_dir = join(distrib_dir, release) if not isdir(release_dir): continue for applicationservice in listdir(release_dir): applicationservice_dir = join(release_dir, applicationservice) if not isdir(applicationservice_dir): continue if applicationservice in applications: raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})') applications[applicationservice] = applicationservice_dir return applications class ModuleCfg(): def __init__(self): self.dictionaries_dir = [] self.modules = [] self.functions_file = [FUNCTIONS] self.templates_dir = [] self.extra_dictionaries = {} self.servers = [] def build_module(module_name, datas, module_infos): install_dir = join(INSTALL_DIR, module_name) makedirs(install_dir) applications = load_applications() cfg = ModuleCfg() module_infos[module_name] = cfg def calc_depends(appname, added): if appname in added: return as_dir = applications[appname] cfg.modules.append(appname) dictionaries_dir = join(as_dir, 'dictionaries') if isdir(dictionaries_dir): cfg.dictionaries_dir.append(dictionaries_dir) funcs_dir = join(as_dir, 'funcs') if isdir(funcs_dir): for f in listdir(funcs_dir): if f.startswith('__'): continue cfg.functions_file.append(join(funcs_dir, f)) templates_dir = join(as_dir, 'templates') if isdir(templates_dir): cfg.templates_dir.append(templates_dir) extras_dir = join(as_dir, 'extras') if isdir(extras_dir): for extra in listdir(extras_dir): extra_dir = join(extras_dir, extra) if isdir(extra_dir): cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir) for type in ['image', 'install']: manual_dir = join(as_dir, 'manual', type) if isdir(manual_dir): for filename in listdir(manual_dir): src_file = join(manual_dir, filename) if type == 'image': dst_file = join(install_dir, filename) verify = False else: dst_file= join(INSTALL_DIR, filename) verify = True if isdir(src_file): if not isdir(dst_file): makedirs(dst_file) for subfilename in listdir(src_file): if not verify or not isfile(dst_file): src = join(src_file, subfilename) dst = join(dst_file, subfilename) if isfile(src): copy2(src, dst) else: copytree(src, dst) elif not verify or not isfile(dst_file): src = join(manual_dir, filename) dst = dst_file if isfile(src): copy2(src, dst) else: copytree(src, dst) added.append(appname) with open(join(as_dir, 'applicationservice.yml')) as yaml: app = load(yaml, Loader=SafeLoader) for xml in app.get('depends', []): calc_depends(xml, added) added = [] for applicationservice in datas['applicationservices']: calc_depends(applicationservice, added) async def build(server_name, datas, module_infos): if server_name in CONFIGS: raise Exception(f'server "{server_name}" is duplicate') cfg = RougailConfig.copy() module_info = module_infos[datas['module']] module_info.servers.append(server_name) if datas['module'] == 'host': cfg['tmpfile_dest_dir'] = datas['values']['rougail.host_install_dir'] + '/host/configurations/' + server_name cfg['templates_dir'] = module_info.templates_dir cfg['dictionaries_dir'] = module_info.dictionaries_dir cfg['functions_file'] = module_info.functions_file cfg['multi_functions'] = MULTI_FUNCTIONS cfg['extra_dictionaries'] = module_info.extra_dictionaries cfg['extra_annotators'].append('risotto_setting.rougail') optiondescription = {'set_linked': set_linked, 'get_linked_configuration': get_linked_configuration, 'set_linked_configuration': set_linked_configuration, } cfg['internal_functions'] = list(optiondescription.keys()) try: eolobj = RougailConvert(cfg) except Exception as err: print(f'Try to load {module_info.modules}') raise err from err xml = eolobj.save(None) #print(xml) #cfg['patches_dir'] = join(test_dir, 'patches') cfg['tmp_dir'] = 'tmp' cfg['destinations_dir'] = join(INSTALL_DIR, datas['module'], CONFIG_DEST_DIR, server_name) if isdir('tmp'): rmtree('tmp') makedirs('tmp') makedirs(cfg['destinations_dir']) try: exec(xml, None, optiondescription) except Exception as err: print(xml) raise Exception(f'unknown error when load tiramisu object {err}') from err config = await Config(optiondescription['option_0'], display_name=tiramisu_display_name) await config.property.read_write() try: if await config.option('machine.add_srv').value.get(): srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name) else: srv = None except AttributeError: srv = None await config.property.read_write() CONFIGS[server_name] = (config, cfg, srv, 0) async def value_pprint(dico, config): pprint_dict = {} for path, value in dico.items(): if await config.option(path).option.type() == 'password' and value: value = 'X' * len(value) pprint_dict[path] = value pprint(pprint_dict) async def set_values(server_name, config, datas): if 'informations' in datas: for information, value in datas['informations'].items(): await config.information.set(information, value) if 'extra_domainnames' in datas['informations']: for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']): if extra_domainname in CONFIGS: raise Exception(f'server "{server_name}" is duplicate') value = list(CONFIGS[server_name]) value[3] = idx + 1 CONFIGS[extra_domainname] = tuple(value) await config.information.set('server_name', server_name) await config.property.read_write() try: if 'values' in datas: for path, value in datas['values'].items(): if isinstance(value, dict): for idx, val in value.items(): await config.option(path, int(idx)).value.set(val) else: await config.option(path).value.set(value) except Exception as err: await value_pprint(await config.value.dict(), config) error_msg = f'cannot configure server "{server_name}": {err}' raise Exception(error_msg) from err await config.property.read_only() #await config.value.dict() async def valid_mandatories(server_name, config): mandatories = await config.value.mandatory() if mandatories: print() print(f'=== Configuration: {server_name} ===') await config.property.pop('mandatory') await value_pprint(await config.value.dict(), config) raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"') async def templates(server_name, config, cfg, srv, int_idx): values = await config.value.dict() engine = RougailSystemdTemplate(config, cfg) # if server_name == 'roundcube.in.gnunux.info': # print() # print(f'=== Configuration: {server_name} ===') # pprint(values) try: await engine.instance_files() except Exception as err: print() print(f'=== Configuration: {server_name} ===') await value_pprint(values, config) raise err from err if srv: makedirs(srv) async def main(): if isdir(INSTALL_DIR): rmtree(INSTALL_DIR) makedirs(INSTALL_DIR) module_infos = {} for module_name, datas in MODULES.items(): build_module(module_name, datas, module_infos) for server_name, datas in SERVERS.items(): await build(server_name, datas, module_infos) for module_name, cfg in module_infos.items(): with open(join(INSTALL_DIR, module_name, 'install_machines'), 'w') as fh: for server_name in cfg.servers: fh.write(f'./install_machine {module_name} {server_name}\n') for server_name, datas in SERVERS.items(): await set_values(server_name, CONFIGS[server_name][0], datas) for server_name in SERVERS: config = CONFIGS[server_name][0] await config.property.pop('mandatory') await config.value.dict() await config.property.add('mandatory') for server_name in SERVERS: await valid_mandatories(server_name, CONFIGS[server_name][0]) # print(await CONFIGS['revprox.in.gnunux.info'][0].option('nginx.reverse_proxy_for_netbox_in_gnunux_info.reverse_proxy_netbox_in_gnunux_info.revprox_url_netbox_in_gnunux_info', 0).value.get()) for server_name in SERVERS: await templates(server_name, *CONFIGS[server_name]) run(main())