From 382a5322a677cfb744037a3ef53f95a679fd11e8 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Fri, 24 Jun 2022 19:02:45 +0200 Subject: [PATCH] split code --- bootstrap.py | 585 ++++------------------------------------- funcs.py | 105 ++------ src/risotto/image.py | 272 +++++++++++++++++++ src/risotto/machine.py | 394 +++++++++++++++++++++++++++ src/risotto/utils.py | 92 +++++++ 5 files changed, 834 insertions(+), 614 deletions(-) create mode 100644 src/risotto/image.py create mode 100644 src/risotto/machine.py diff --git a/bootstrap.py b/bootstrap.py index 4a4a9a0..1c4a8d1 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -1,541 +1,43 @@ #!/usr/bin/env python3 from asyncio import run -from os import listdir, link, makedirs, environ -from os.path import isdir, isfile, join -from shutil import rmtree, copy2, copytree -from json import load as json_load -from yaml import load, SafeLoader -from toml import load as toml_load -from pprint import pprint -from typing import Any -from warnings import warn_explicit +from os import listdir, link, makedirs +from os.path import isdir, join +from shutil import rmtree from copy import copy -from tiramisu import Config -from tiramisu.error import ValueWarning -from rougail import RougailConfig, RougailConvert, RougailSystemdTemplate -from rougail.utils import normalize_family +from rougail import RougailSystemdTemplate -from risotto.utils import MULTI_FUNCTIONS, CONFIGS, DOMAINS +from risotto.utils import CONFIGS, RISOTTO_CONFIG, SERVERS, value_pprint +from risotto.image import load -with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh: - config = toml_load(fh) - - -DATASET_DIRECTORY = config['directories']['dataset'] -INSTALL_DIR = config['directories']['dest'] -FUNCTIONS = 'funcs.py' +INSTALL_DIR = RISOTTO_CONFIG['directories']['dest'] CONFIG_DEST_DIR = 'configurations' +CONFIG_DIFF_DIR = 'diff' SRV_DEST_DIR = 'srv' -with open('servers.json', 'r') as server_fh: - jsonfile = json_load(server_fh) - SERVERS = jsonfile['servers'] - MODULES = jsonfile['modules'] - - -async def set_linked_multi_variables(value: str, - linked_server: str=None, - variable_index: int=None, - **kwargs: dict, - ) -> None: - if value is not None and linked_server is not None and 'linked_value_0' not in kwargs: - kwargs['linked_value_0'] = value - elif not linked_server: - linked_server = value - if not linked_server: - return - if linked_server not in CONFIGS: - warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), - ValueWarning, - __file__, - 3, - ) - return - config = CONFIGS[linked_server][0] - dico = {} - variables = {} - for key, value in kwargs.items(): - if value is None: - return - if key.startswith('linked_provider_'): - index = int(key[16]) - path = await config.information.get('provider:' + value, None) - if not path: - return - if index not in variables: - variables[index] = {'path': None, 'value': None, 'variable_index': False} - variables[index]['path'] = path - elif key.startswith('linked_value_'): - index = int(key[13]) - if index not in variables: - variables[index] = {'path': None, 'value': None, 'variable_index': False} - variables[index]['value'] = value - elif key.startswith('variable_index_'): - index = int(key[15]) - if index not in variables: - variables[index] = {'path': None, 'value': None, 'variable_index': False} - variables[index]['variable_index'] = True - else: - raise AttributeError(f'unknown parameter {key}') - await config.property.read_write() -# print('=====================================') -# pprint(variables) - if not isinstance(variables[0]['value'], list): - variables[0]['value'] = [variables[0]['value']] - dynamic = None - try: - if variables[0]['value']: - for first_idx, first_value in enumerate(variables[0]['value']): - slave_idxes = [] - dynamic = normalize_family(first_value) - for index in sorted(list(variables)): - path = variables[index]['path'] - if '{suffix}' in path: - path = path.replace('{suffix}', dynamic) - elif first_idx != 0: - continue - value = variables[index]['value'] - option = config.forcepermissive.option(path) - if not await option.option.isfollower(): - #print('===>', path, value, await option.option.ismulti(), await option.option.issubmulti()) - multi = await option.option.ismulti() - if multi: - isleader = await option.option.isleader() - if not isinstance(value, list): - value = [value] - # elif isleader: - # raise Exception('leader must not be a multi from now ...') - values = await option.value.get() - for val in value: - if val not in values: - if isleader: - slave_idxes.append(len(values)) - values.append(val) - elif isleader: - slave_idxes.append(values.index(val)) - await option.value.set(values) - else: - await option.value.set(value) - else: - #print('===<', path, value, await option.option.ismulti(), await option.option.issubmulti()) - if not slave_idxes: - raise Exception('please declare the leader variable before the follower') - if variables[index]['variable_index']: - value = value[variable_index] - if not isinstance(value, list): - value = [value] * len(slave_idxes) - # if isinstance(value, list) and not await option.option.issubmulti(): - for idx, val in enumerate(value): - option = config.forcepermissive.option(path, slave_idxes[idx]) - await option.value.set(val) - except Exception as err: - await config.property.read_only() - raise err from err - await config.property.read_only() - return get_ip_from_domain(linked_server) - - -async def set_linked(linked_server: str, - linked_provider: str, - linked_value: str, - linked_returns: str=None, - dynamic: str=None, - ): - if None in (linked_server, linked_provider, linked_value): - return - if linked_server not in CONFIGS: - warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), - ValueWarning, - __file__, - 0, - ) - return - config = CONFIGS[linked_server][0] - path = await config.information.get('provider:' + linked_provider, None) - if not path: - warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'), - ValueWarning, - __file__, - 0, - ) - return - await config.property.read_write() - try: - option = config.forcepermissive.option(path) - if await option.option.ismulti(): - values = await option.value.get() - if linked_value not in values: - values.append(linked_value) - await option.value.set(values) - else: - await option.value.set(linked_value) - except Exception as err: - await config.property.read_only() - raise err from err - await config.property.read_only() - if linked_returns is not None: - linked_variable = await config.information.get('provider:' + linked_returns, None) - if not linked_variable: - warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'), - ValueWarning, - __file__, - 0, - ) - return - else: - linked_variable = None - if linked_variable is not None: - if dynamic: - linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic)) - elif '{suffix}' in linked_variable: - idx = CONFIGS[linked_server][3] - linked_variable = linked_variable.replace('{suffix}', str(idx)) - ret = await config.forcepermissive.option(linked_variable).value.get() - else: - ret = normalize_family(linked_value) - return ret - - -async def get_linked_configuration(linked_server: str, - linked_provider: str, - dynamic: str=None, - ): - if linked_server not in CONFIGS: - warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), - ValueWarning, - __file__, - 1, - ) - return - config = CONFIGS[linked_server][0] - path = await config.information.get('provider:' + linked_provider, None) - if not path: - warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'), - ValueWarning, - __file__, - 1, - ) - return - if dynamic: - path = path.replace('{suffix}', normalize_family(dynamic)) - try: - return await config.forcepermissive.option(path).value.get() - except AttributeError as err: - warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'), - ValueWarning, - __file__, - 1, - ) - - -class Empty: - pass -empty = Empty() - - -async def set_linked_configuration(_linked_value: Any, - linked_server: str, - linked_provider: str, - linked_value: Any=empty, - dynamic: str=None, - leader_provider: str=None, - leader_value: Any=None, - leader_index: int=None, - ): - if linked_value is not empty: - _linked_value = linked_value - linked_value = _linked_value - if linked_server is None: - return - if linked_value is None or linked_server not in CONFIGS: - warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), - ValueWarning, - __file__, - 2, - ) - return - config = CONFIGS[linked_server][0] - path = await config.information.get('provider:' + linked_provider, None) - if not path: - warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'), - ValueWarning, - __file__, - 2, - ) - return - if dynamic: - path = path.replace('{suffix}', normalize_family(dynamic)) - await config.property.read_write() - try: - if leader_provider is not None: - leader_path = await config.information.get('provider:' + leader_provider, None) - if not leader_path: - await config.property.read_only() - warn_explicit(ValueWarning(f'cannot find leader variable with leader_provider "{leader_provider}" in linked server "{linked_server}"'), - ValueWarning, - __file__, - 2, - ) - return - if dynamic: - leader_path = leader_path.replace('{suffix}', normalize_family(dynamic)) - values = await config.forcepermissive.option(leader_path).value.get() - if not isinstance(leader_value, list): - leader_value = [leader_value] - for lv in leader_value: - if lv in values: - slave_idx = values.index(lv) - slave_option = config.forcepermissive.option(path, slave_idx) - if await slave_option.option.issubmulti(): - slave_values = await slave_option.value.get() - if linked_value not in slave_values: - slave_values.append(linked_value) - await slave_option.value.set(slave_values) - else: - await slave_option.value.set(linked_value) - else: - option = config.forcepermissive.option(path, leader_index) - if leader_index is None and await option.option.ismulti() and not isinstance(linked_value, list): - values = await option.value.get() - if linked_value not in values: - values.append(linked_value) - await option.value.set(values) - else: - await option.value.set(linked_value) - except AttributeError as err: - #raise ValueError(str(err)) from err - pass - except Exception as err: - await config.property.read_only() - raise err from err - await config.property.read_only() - - def tiramisu_display_name(kls, dyn_name: 'Base'=None, suffix: str=None, ) -> str: + # FIXME if dyn_name is not None: - name = kls.impl_getpath() + suffix + name = kls.impl_getpath() + str(suffix) else: name = kls.impl_getpath() return name -def load_applications(): - applications = {} - distrib_dir = join(DATASET_DIRECTORY, 'applicationservice') - for release in listdir(distrib_dir): - release_dir = join(distrib_dir, release) - if not isdir(release_dir): - continue - for applicationservice in listdir(release_dir): - applicationservice_dir = join(release_dir, applicationservice) - if not isdir(applicationservice_dir): - continue - if applicationservice in applications: - raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})') - applications[applicationservice] = applicationservice_dir - return applications - - -class ModuleCfg(): - def __init__(self): - self.dictionaries_dir = [] - self.modules = [] - self.functions_file = [FUNCTIONS] - self.templates_dir = [] - self.extra_dictionaries = {} - self.servers = [] - - -def build_module(module_name, datas, module_infos): - install_dir = join(INSTALL_DIR, module_name) - makedirs(install_dir) - applications = load_applications() - cfg = ModuleCfg() - module_infos[module_name] = cfg - def calc_depends(appname, added): - if appname in added: - return - if appname not in applications: - raise Exception(f'cannot find application dependency "{appname}" in application "{module_name}"') - as_dir = applications[appname] - cfg.modules.append(appname) - dictionaries_dir = join(as_dir, 'dictionaries') - if isdir(dictionaries_dir): - cfg.dictionaries_dir.append(dictionaries_dir) - funcs_dir = join(as_dir, 'funcs') - if isdir(funcs_dir): - for f in listdir(funcs_dir): - if f.startswith('__'): - continue - cfg.functions_file.append(join(funcs_dir, f)) - templates_dir = join(as_dir, 'templates') - if isdir(templates_dir): - cfg.templates_dir.append(templates_dir) - extras_dir = join(as_dir, 'extras') - if isdir(extras_dir): - for extra in listdir(extras_dir): - extra_dir = join(extras_dir, extra) - if isdir(extra_dir): - cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir) - for type in ['image', 'install']: - manual_dir = join(as_dir, 'manual', type) - if isdir(manual_dir): - for filename in listdir(manual_dir): - src_file = join(manual_dir, filename) - if type == 'image': - dst_file = join(install_dir, 'manual', filename) - verify = False - else: - dst_file= join(INSTALL_DIR, filename) - verify = True - if isdir(src_file): - if not isdir(dst_file): - makedirs(dst_file) - for subfilename in listdir(src_file): - if not verify or not isfile(dst_file): - src = join(src_file, subfilename) - dst = join(dst_file, subfilename) - if isfile(src): - copy2(src, dst) - else: - copytree(src, dst) - elif not verify or not isfile(dst_file): - src = join(manual_dir, filename) - dst = dst_file - if isfile(src): - copy2(src, dst) - else: - copytree(src, dst) - added.append(appname) - with open(join(as_dir, 'applicationservice.yml')) as yaml: - app = load(yaml, Loader=SafeLoader) - - for xml in app.get('depends', []): - calc_depends(xml, added) - added = [] - for applicationservice in datas['applicationservices']: - calc_depends(applicationservice, added) - - -def get_ip_from_domain(domain): - if not domain: - return - hostname, domainname = domain.split('.', 1) - return DOMAINS[domainname][1][DOMAINS[domainname][0].index(hostname)] - - -async def build(server_name, datas, module_infos): - if server_name in CONFIGS: - raise Exception(f'server "{server_name}" is duplicate') - cfg = RougailConfig.copy() - module_info = module_infos[datas['module']] - module_info.servers.append(server_name) - if datas['module'] == 'host': - cfg['tmpfile_dest_dir'] = datas['values']['rougail.host_install_dir'] + '/host/configurations/' + server_name - cfg['templates_dir'] = module_info.templates_dir - cfg['dictionaries_dir'] = module_info.dictionaries_dir - cfg['functions_file'] = module_info.functions_file - cfg['multi_functions'] = MULTI_FUNCTIONS - cfg['extra_dictionaries'] = module_info.extra_dictionaries - cfg['extra_annotators'].append('risotto.rougail') - optiondescription = {'set_linked': set_linked, - 'set_linked_multi_variables': set_linked_multi_variables, - 'get_linked_configuration': get_linked_configuration, - 'set_linked_configuration': set_linked_configuration, - 'get_ip_from_domain': get_ip_from_domain, - } - cfg['internal_functions'] = list(optiondescription.keys()) - try: - eolobj = RougailConvert(cfg) - except Exception as err: - print(f'Try to load {module_info.modules}') - raise err from err - tiram_obj = eolobj.save(None) -# if server_name == 'revprox.in.silique.fr': -# print(tiram_obj) - #cfg['patches_dir'] = join(test_dir, 'patches') - cfg['tmp_dir'] = 'tmp' - cfg['destinations_dir'] = join(INSTALL_DIR, datas['module'], CONFIG_DEST_DIR, server_name) - if isdir('tmp'): - rmtree('tmp') - makedirs('tmp') - makedirs(cfg['destinations_dir']) - try: - exec(tiram_obj, None, optiondescription) - except Exception as err: - print(tiram_obj) - raise Exception(f'unknown error when load tiramisu object {err}') from err - config = await Config(optiondescription['option_0'], display_name=tiramisu_display_name) - await config.property.read_write() - try: - if await config.option('machine.add_srv').value.get(): - srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name) - else: - srv = None - except AttributeError: - srv = None - await config.property.read_write() - CONFIGS[server_name] = (config, cfg, srv, 0) - - -async def value_pprint(dico, config): - pprint_dict = {} - for path, value in dico.items(): - if await config.option(path).option.type() == 'password' and value: - value = 'X' * len(value) - pprint_dict[path] = value - pprint(pprint_dict) - - -async def set_values(server_name, config, datas): - if 'informations' in datas: - for information, value in datas['informations'].items(): - await config.information.set(information, value) - if 'extra_domainnames' in datas['informations']: - for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']): - if extra_domainname in CONFIGS: - raise Exception(f'server "{server_name}" is duplicate') - value = list(CONFIGS[server_name]) - value[3] = idx + 1 - CONFIGS[extra_domainname] = tuple(value) - await config.information.set('server_name', server_name) - await config.property.read_write() - try: - if 'values' in datas: - for path, value in datas['values'].items(): - if isinstance(value, dict): - for idx, val in value.items(): - await config.option(path, int(idx)).value.set(val) - else: - await config.option(path).value.set(value) - except Exception as err: - await value_pprint(await config.value.dict(), config) - error_msg = f'cannot configure server "{server_name}": {err}' - raise Exception(error_msg) from err - await config.property.read_only() - #await config.value.dict() - - -async def valid_mandatories(server_name, config): - mandatories = await config.value.mandatory() - if mandatories: - print() - print(f'=== Configuration: {server_name} ===') - await config.property.pop('mandatory') - await value_pprint(await config.value.dict(), config) - raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"') - - -async def templates(server_name, config, cfg, srv, int_idx): +async def templates(server_name, + config, + templates_informations, + srv=False, + **kwargs, + ): values = await config.value.dict() - engine = RougailSystemdTemplate(config, cfg) + engine = RougailSystemdTemplate(config, templates_informations) # if server_name == 'dovecot.in.silique.fr': # print() # print(f'=== Configuration: {server_name} ===') @@ -555,30 +57,43 @@ async def main(): if isdir(INSTALL_DIR): rmtree(INSTALL_DIR) makedirs(INSTALL_DIR) - module_infos = {} - for module_name, datas in MODULES.items(): - build_module(module_name, datas, module_infos) - for server_name, datas in SERVERS.items(): - await build(server_name, datas, module_infos) + module_infos = await load(display_name=tiramisu_display_name, clean_directories=True, copy_manual_dir=True) +# pprint(await CONFIGS['lemonldap.in.silique.fr'][0].value.dict()) + for server_name in SERVERS: + module_name = CONFIGS[server_name]['module_name'] + add_srv = CONFIGS[server_name]['add_srv'] + cfg = CONFIGS[server_name]['templates_informations'] + cfg['tmp_dir'] = 'tmp' + cfg['destinations_dir'] = join(INSTALL_DIR, module_name, CONFIG_DEST_DIR, server_name) + if isdir('tmp'): + rmtree('tmp') + makedirs(cfg['tmp_dir']) + makedirs(cfg['destinations_dir']) + if add_srv: + srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name) + else: + srv = None + await templates(server_name, **CONFIGS[server_name], srv=srv) + for server_name in SERVERS: + config = CONFIGS[server_name]['config'] + await config.property.read_write() + try: +# pass + await config.option('general.hide_secret').value.set(True) + except AttributeError: + # if rougail.general.hide_secret not exists + pass + await config.property.read_only() + for server_name in SERVERS: + module_name = CONFIGS[server_name]['module_name'] + destinations_dir = join(INSTALL_DIR, module_name, CONFIG_DIFF_DIR, server_name) + makedirs(destinations_dir) + CONFIGS[server_name]['templates_informations']['destinations_dir'] = destinations_dir + await templates(server_name, **CONFIGS[server_name]) for module_name, cfg in module_infos.items(): with open(join(INSTALL_DIR, module_name, 'install_machines'), 'w') as fh: for server_name in cfg.servers: fh.write(f'./install_machine {module_name} {server_name}\n') - for server_name, datas in SERVERS.items(): - await set_values(server_name, CONFIGS[server_name][0], datas) - for server_name in SERVERS: - config = CONFIGS[server_name][0] - await config.property.pop('mandatory') - try: - await config.value.dict() - except Exception as err: - raise Exception(f'cannot display config for "{server_name}": {err}') - await config.property.add('mandatory') - for server_name in SERVERS: - await valid_mandatories(server_name, CONFIGS[server_name][0]) -# print(await CONFIGS['dovecot.in.silique.fr'][0].value.dict()) - for server_name in SERVERS: - await templates(server_name, *CONFIGS[server_name]) run(main()) diff --git a/funcs.py b/funcs.py index b59c0e1..3997858 100644 --- a/funcs.py +++ b/funcs.py @@ -1,88 +1,24 @@ from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value -from ipaddress import ip_address from os.path import dirname, abspath, join as _join, isdir as _isdir, isfile as _isfile from typing import List -from json import load from secrets import token_urlsafe as _token_urlsafe from rougail.utils import normalize_family -from risotto.utils import multi_function, CONFIGS, DOMAINS +from risotto.utils import multi_function, DOMAINS, ZONES, load_zones, load_zones_server, load_domains, ZONES_SERVER from risotto.x509 import gen_cert as _x509_gen_cert, gen_ca as _x509_gen_ca, gen_pub as _x509_gen_pub, has_pub as _x509_has_pub -# ============================================================= -# fork of risotto-setting/src/risotto_setting/config/config.py - -with open('servers.json', 'r') as server_fh: - ZONES_SERVER = load(server_fh) -ZONES = None HERE = dirname(abspath(__file__)) -def load_zones(): - global ZONES - if ZONES is not None: - return - ZONES = ZONES_SERVER['zones'] - for server_name, server in ZONES_SERVER['servers'].items(): - if 'informations' not in server: - continue - server_zones = server['informations']['zones_name'] - server_extra_domainnames = server['informations'].get('extra_domainnames', []) - if len(server_zones) > 1 and len(server_zones) != len(server_extra_domainnames) + 1: - raise Exception(f'the server "{server_name}" has more that one zone, please set correct number of extra_domainnames ({len(server_zones) - 1} instead of {len(server_extra_domainnames)})') - - for idx, zone_name in enumerate(server_zones): - zone_domain_name = ZONES[zone_name]['domain_name'] - if idx == 0: - zone_server_name = server_name - else: - zone_server_name = server_extra_domainnames[idx - 1] - server_domain_name = zone_server_name.split('.', 1)[1] - if zone_domain_name and zone_domain_name != server_domain_name: - raise Exception(f'wrong server_name "{zone_server_name}" in zone "{zone_name}" should ends with "{zone_domain_name}"') - ZONES[zone_name].setdefault('hosts', []).append(server_name) - - -def load_domains(): - load_zones() - global DOMAINS - if DOMAINS: - return - for zone_name, zone in ZONES_SERVER['zones'].items(): - if 'domain_name' in zone: - hosts = [] - ips = [] - for host in ZONES[zone_name].get('hosts', []): - hosts.append(host.split('.', 1)[0]) - ips.append(get_ip(host, [zone_name], 0)) - DOMAINS[zone['domain_name']] = (tuple(hosts), tuple(ips)) - - -def get_ip(server_name: str, - zones_name: List[str], - index: str, - ) -> str: - if server_name is None: - return - load_zones() - index = int(index) - zone_name = zones_name[index] - if zone_name not in ZONES: - raise ValueError(f"cannot set IP in unknown zone '{zone_name}'") - zone = ZONES[zone_name] - if server_name not in zone['hosts']: - raise ValueError(f"cannot set IP in unknown server '{server_name}'") - server_index = zone['hosts'].index(server_name) -# print(server_name, zones_name, index, str(ip_address(zone['start_ip']) + server_index)) - return str(ip_address(zone['start_ip']) + server_index) - - @multi_function -def get_chain(authority_cn, - authority_name, +def get_chain(authority_cn: str, + authority_name: str, + hide: bool, ): + if hide: + return "XXXXX" if not authority_name or authority_name is None: if isinstance(authority_name, list): return [] @@ -107,11 +43,14 @@ def get_chain(authority_cn, @multi_function def get_certificate(cn, - authority_name, - authority_cn=None, - extra_domainnames=[], - type='server', + authority_name: str, + hide: bool, + authority_cn: str=None, + extra_domainnames: list=[], + type: str='server', ): + if hide: + return "XXXXX" if isinstance(cn, list) and extra_domainnames: raise Exception('cn cannot be a list with extra_domainnames set') if not cn or authority_name is None: @@ -129,11 +68,14 @@ def get_certificate(cn, @multi_function -def get_private_key(cn, - authority_name=None, - authority_cn=None, - type='server', +def get_private_key(cn: str, + hide: bool, + authority_name: str=None, + authority_cn: str=None, + type: str='server', ): + if hide: + return "XXXXX" if not cn: if isinstance(cn, list): return [] @@ -157,7 +99,11 @@ def get_private_key(cn, ) -def get_public_key(cn): +def get_public_key(cn: str, + hide: bool, + ): + if hide: + return "XXXXX" if not cn: return return _x509_gen_pub(cn, @@ -194,6 +140,7 @@ def get_internal_zones() -> List[str]: @multi_function def get_zones_info(type: str) -> str: + load_zones_server() ret = [] for data in ZONES_SERVER['zones'].values(): ret.append(data[type]) diff --git a/src/risotto/image.py b/src/risotto/image.py new file mode 100644 index 0000000..c252b21 --- /dev/null +++ b/src/risotto/image.py @@ -0,0 +1,272 @@ +from shutil import copy2, copytree, rmtree +from os import listdir, makedirs +from os.path import join, isdir, isfile +from yaml import load as yaml_load, SafeLoader +from json import load as json_load + +from .utils import CONFIGS, RISOTTO_CONFIG, SERVERS, value_pprint +from .machine import load_machine_config + + +FUNCTIONS = 'funcs.py' + + +class ModuleCfg(): + def __init__(self): + self.dictionaries_dir = [] + self.modules = [] + self.functions_file = [FUNCTIONS] + self.templates_dir = [] + self.extra_dictionaries = {} + self.servers = [] + + +def list_applications() -> dict: + """ + {: applicationservice// + """ + dataset_directory = RISOTTO_CONFIG['directories']['dataset'] + applications = {} + distrib_dir = join(dataset_directory, 'applicationservice') + for release in listdir(distrib_dir): + release_dir = join(distrib_dir, release) + if not isdir(release_dir): + continue + for applicationservice in listdir(release_dir): + applicationservice_dir = join(release_dir, applicationservice) + if not isdir(applicationservice_dir): + continue + if applicationservice in applications: + raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})') + applications[applicationservice] = applicationservice_dir + return applications + + +def applicationservice_copy(src_file: str, + dst_file: str, + copy_if_not_exists: bool, + manual_dir: str, + filename: str, + ) -> None: + if isdir(src_file): + if not isdir(dst_file): + makedirs(dst_file) + for subfilename in listdir(src_file): + if not copy_if_not_exists or not isfile(dst_file): + src = join(src_file, subfilename) + dst = join(dst_file, subfilename) + if isfile(src): + copy2(src, dst) + else: + copytree(src, dst) + elif not copy_if_not_exists or not isfile(dst_file): + src = join(manual_dir, filename) + dst = dst_file + if isfile(src): + copy2(src, dst) + else: + copytree(src, dst) + + +def load_applicationservice_cfg(appname: str, + as_dir: str, + install_dir: str, + cfg: ModuleCfg, + copy_manual_dir: bool, + ) -> None: + cfg.modules.append(appname) + # dictionaries + dictionaries_dir = join(as_dir, 'dictionaries') + if isdir(dictionaries_dir): + cfg.dictionaries_dir.append(dictionaries_dir) + # funcs + funcs_dir = join(as_dir, 'funcs') + if isdir(funcs_dir): + for f in listdir(funcs_dir): + if f.startswith('__'): + continue + cfg.functions_file.append(join(funcs_dir, f)) + # templates + templates_dir = join(as_dir, 'templates') + if isdir(templates_dir): + cfg.templates_dir.append(templates_dir) + # extras + extras_dir = join(as_dir, 'extras') + if isdir(extras_dir): + for extra in listdir(extras_dir): + extra_dir = join(extras_dir, extra) + if isdir(extra_dir): + cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir) + if copy_manual_dir: + # manual + for type in ['image', 'install']: + manual_dir = join(as_dir, 'manual', type) + if not isdir(manual_dir): + continue + for filename in listdir(manual_dir): + src_file = join(manual_dir, filename) + if type == 'image': + dst_file = join(install_dir, 'manual', filename) + copy_if_not_exists = False + else: + dst_file= join(install_dir, '..', filename) + copy_if_not_exists = True + applicationservice_copy(src_file, + dst_file, + copy_if_not_exists, + manual_dir, + filename, + ) + + +def load_applicationservice(appname: str, + added: list, + install_dir: str, + cfg: ModuleCfg, + applications: dict, + copy_manual_dir: bool, + ) -> None: + if appname not in applications: + raise Exception(f'cannot find application dependency "{appname}" in application "{module_name}"') + as_dir = applications[appname] + applicationservice_file = join(as_dir, 'applicationservice.yml') + if not isfile(applicationservice_file): + raise Exception(f'cannot find application service file "{applicationservice_file}"') + load_applicationservice_cfg(appname, + as_dir, + install_dir, + cfg, + copy_manual_dir, + ) + added.append(appname) + with open(applicationservice_file) as yaml: + app = yaml_load(yaml, Loader=SafeLoader) + for xml in app.get('depends', []): + if xml in added: + continue + load_applicationservice(xml, + added, + install_dir, + cfg, + applications, + copy_manual_dir, + ) + + +def load_image_informations(install_dir: str, + datas: dict, + applications: dict, + copy_manual_dir: bool, + ) -> ModuleCfg: + cfg = ModuleCfg() + added = [] + for applicationservice in datas['applicationservices']: + load_applicationservice(applicationservice, + added, + install_dir, + cfg, + applications, + copy_manual_dir, + ) + return cfg + + +async def load_informations(server_name, datas, config): + await config.information.set('server_name', server_name) + if 'informations' not in datas: + return + for information, value in datas['informations'].items(): + await config.information.set(information, value) + if 'extra_domainnames' in datas['informations']: + for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']): + if extra_domainname in CONFIGS: + raise Exception(f'server "{server_name}" is duplicate') + value = list(CONFIGS[server_name]) + value[4] = idx + 1 + CONFIGS[extra_domainname] = tuple(value) + + +async def set_values(server_name, config, datas): + try: + if 'values' in datas: + for path, value in datas['values'].items(): + if isinstance(value, dict): + for idx, val in value.items(): + await config.option(path, int(idx)).value.set(val) + else: + await config.option(path).value.set(value) + except Exception as err: + await value_pprint(await config.value.dict(), config) + error_msg = f'cannot configure server "{server_name}": {err}' + raise Exception(error_msg) from err + #await config.value.dict() + + +async def valid_mandatories(server_name, config): + mandatories = await config.value.mandatory() + if mandatories: + print() + print(f'=== Configuration: {server_name} ===') + await config.property.pop('mandatory') + await value_pprint(await config.value.dict(), config) + raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"') + + +async def load(display_name=None, + clean_directories=False, + copy_manual_dir=False, + hide_secret=False, + ): + with open('servers.json', 'r') as server_fh: + jsonfile = json_load(server_fh) + SERVERS.update(jsonfile['servers']) + modules = jsonfile['modules'] + module_infos = {} + applications = list_applications() + # load images + for module_name, datas in modules.items(): + install_dir = join(RISOTTO_CONFIG['directories']['dest'], module_name) + if clean_directories: + if isdir(install_dir): + rmtree(install_dir) + makedirs(install_dir) + module_infos[module_name] = load_image_informations(install_dir, + datas, + applications, + copy_manual_dir, + ) + # load machines + for server_name, datas in SERVERS.items(): + if server_name in CONFIGS: + raise Exception(f'server "{server_name}" is duplicate') + CONFIGS[server_name] = await load_machine_config(server_name, + datas, + module_infos[datas['module']], + display_name=display_name, + ) + # set servers.json values + for server_name, datas in SERVERS.items(): + config = CONFIGS[server_name]['config'] + await load_informations(server_name, datas, config) + await config.property.read_write() + if hide_secret: + try: + await config.option('general.hide_secret').value.set(True) + except AttributeError: + pass + await set_values(server_name, config, datas) + await config.property.read_only() + # force calculates all values (for linked values) + for server_name in SERVERS: + config = CONFIGS[server_name]['config'] + await config.property.pop('mandatory') + try: + await config.value.dict() + except Exception as err: + raise Exception(f'cannot display config for "{server_name}": {err}') + await config.property.add('mandatory') + # validate mandatories values + for server_name in SERVERS: + await valid_mandatories(server_name, CONFIGS[server_name]['config']) + + return module_infos diff --git a/src/risotto/machine.py b/src/risotto/machine.py new file mode 100644 index 0000000..1af963d --- /dev/null +++ b/src/risotto/machine.py @@ -0,0 +1,394 @@ +from os import makedirs +from os.path import join, isdir +from warnings import warn_explicit +from typing import Any + +from tiramisu import Config +from tiramisu.error import ValueWarning +from rougail import RougailConfig, RougailConvert +from .utils import MULTI_FUNCTIONS, CONFIGS, DOMAINS +from rougail.utils import normalize_family + + +ROUGAIL_NAMESPACE = 'general' +ROUGAIL_NAMESPACE_DESCRIPTION = 'Général' + + +async def set_linked_multi_variables(value: str, + linked_server: str=None, + variable_index: int=None, + linked_returns: str=None, + dynamic: str=None, + **kwargs: dict, + ) -> None: + if value is not None and linked_server is not None and 'linked_value_0' not in kwargs: + kwargs['linked_value_0'] = value + elif not linked_server: + linked_server = value + if not linked_server: + return + if linked_server not in CONFIGS: + warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), + ValueWarning, + __file__, + 3, + ) + return + config = CONFIGS[linked_server]['config'] + dico = {} + variables = {} + for key, kvalue in kwargs.items(): + try: + index = int(key.rsplit('_', 1)[-1]) + except ValueError: + raise Exception(f'unknown variable {key}') + if kvalue is None and not kwargs.get(f'allow_none_{index}', False): + return + if key.startswith('linked_provider_'): + path = await config.information.get('provider:' + kvalue, None) + if not path: + return + if index not in variables: + variables[index] = {'path': None, 'value': None, 'variable_index': False} + variables[index]['path'] = path + elif key.startswith('linked_value_'): + index = int(key[13]) + if index not in variables: + variables[index] = {'path': None, 'value': None, 'variable_index': False} + variables[index]['value'] = kvalue + elif key.startswith('variable_index_'): + index = int(key[15]) + if index not in variables: + variables[index] = {'path': None, 'value': None, 'variable_index': False} + variables[index]['variable_index'] = True + elif key.startswith('allow_none_'): + pass + else: + raise AttributeError(f'unknown parameter {key}') + await config.property.read_write() + if not isinstance(variables[0]['value'], list): + variables[0]['value'] = [variables[0]['value']] + if dynamic: + dynamic = normalize_family(dynamic) + _dynamic = None + try: + if variables[0]['value']: + for first_idx, first_value in enumerate(variables[0]['value']): + slave_idxes = [] + if dynamic: + _dynamic = dynamic + else: + _dynamic = normalize_family(first_value) + for index in sorted(list(variables)): + path = variables[index]['path'] + if '{suffix}' in path: + path = path.replace('{suffix}', _dynamic) + elif first_idx != 0: + continue + vvalue = variables[index]['value'] + option = config.forcepermissive.option(path) + if not await option.option.isfollower(): + #print('===>', path, vvalue, await option.option.ismulti(), await option.option.issubmulti()) + multi = await option.option.ismulti() + if multi: + isleader = await option.option.isleader() + if not isinstance(vvalue, list): + vvalue = [vvalue] + # elif isleader: + # raise Exception('leader must not be a multi from now ...') + values = await option.value.get() + for val in vvalue: + if val not in values: + if isleader: + slave_idxes.append(len(values)) + values.append(val) + elif isleader: + slave_idxes.append(values.index(val)) + await option.value.set(values) + await option.owner.set('link') + else: + if isinstance(vvalue, list) and len(vvalue) == 1: + vvalue = vvalue[0] + await option.value.set(vvalue) + await option.owner.set('link') + else: + # print('===<', path, vvalue, await option.option.ismulti(), await option.option.issubmulti()) + if not slave_idxes: + raise Exception('please declare the leader variable before the follower') + if variables[index]['variable_index']: + vvalue = vvalue[variable_index] + if not isinstance(vvalue, list): + vvalue = [vvalue] * len(slave_idxes) + # if isinstance(vvalue, list) and not await option.option.issubmulti(): + for idx, val in enumerate(vvalue): + option = config.forcepermissive.option(path, slave_idxes[idx]) + await option.value.set(val) + await option.owner.set('link') + except Exception as err: + await config.property.read_only() + raise err from err + await config.property.read_only() + if not dynamic: + dynamic = _dynamic + + + if linked_returns is not None: + linked_variable = await config.information.get('provider:' + linked_returns, None) + if not linked_variable: + warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 0, + ) + return + if dynamic: + linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic)) + elif '{suffix}' in linked_variable: + idx = CONFIGS[linked_server]['interface_index'] + linked_variable = linked_variable.replace('{suffix}', str(idx)) + ret = await config.forcepermissive.option(linked_variable).value.get() + else: + ret = get_ip_from_domain(linked_server) + return ret + + +async def set_linked(linked_server: str, + linked_provider: str, + linked_value: str, + linked_returns: str=None, + dynamic: str=None, + ): + if None in (linked_server, linked_provider, linked_value): + return + if linked_server not in CONFIGS: + warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), + ValueWarning, + __file__, + 0, + ) + return + config = CONFIGS[linked_server]['config'] + path = await config.information.get('provider:' + linked_provider, None) + if not path: + warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 0, + ) + return + await config.property.read_write() + try: + option = config.forcepermissive.option(path) + if await option.option.ismulti(): + values = await option.value.get() + if linked_value not in values: + values.append(linked_value) + await option.value.set(values) + await option.owner.set('link') + else: + await option.value.set(linked_value) + await option.owner.set('link') + except Exception as err: + await config.property.read_only() + raise err from err + await config.property.read_only() + if linked_returns is not None: + linked_variable = await config.information.get('provider:' + linked_returns, None) + if not linked_variable: + warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 0, + ) + return + else: + linked_variable = None + if linked_variable is not None: + if dynamic: + linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic)) + elif '{suffix}' in linked_variable: + idx = CONFIGS[linked_server]['interface_index'] + linked_variable = linked_variable.replace('{suffix}', str(idx)) + ret = await config.forcepermissive.option(linked_variable).value.get() + else: + ret = normalize_family(linked_value) + return ret + + +async def get_linked_configuration(linked_server: str, + linked_provider: str, + dynamic: str=None, + ): + if linked_server not in CONFIGS: + warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), + ValueWarning, + __file__, + 1, + ) + return + config = CONFIGS[linked_server]['config'] + path = await config.information.get('provider:' + linked_provider, None) + if not path: + warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 1, + ) + return + if dynamic: + path = path.replace('{suffix}', normalize_family(dynamic)) + try: + return await config.forcepermissive.option(path).value.get() + except AttributeError as err: + warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'), + ValueWarning, + __file__, + 1, + ) + + +class Empty: + pass +empty = Empty() + + +async def set_linked_configuration(_linked_value: Any, + linked_server: str, + linked_provider: str, + linked_value: Any=empty, + dynamic: str=None, + leader_provider: str=None, + leader_value: Any=None, + leader_index: int=None, + ): + if linked_value is not empty: + _linked_value = linked_value + linked_value = _linked_value + if linked_server is None: + return + if linked_value is None or linked_server not in CONFIGS: + warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), + ValueWarning, + __file__, + 2, + ) + return + config = CONFIGS[linked_server]['config'] + path = await config.information.get('provider:' + linked_provider, None) + if not path: + warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 2, + ) + return + if dynamic: + path = path.replace('{suffix}', normalize_family(dynamic)) + await config.property.read_write() + try: + if leader_provider is not None: + leader_path = await config.information.get('provider:' + leader_provider, None) + if not leader_path: + await config.property.read_only() + warn_explicit(ValueWarning(f'cannot find leader variable with leader_provider "{leader_provider}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 2, + ) + return + if dynamic: + leader_path = leader_path.replace('{suffix}', normalize_family(dynamic)) + values = await config.forcepermissive.option(leader_path).value.get() + if not isinstance(leader_value, list): + leader_value = [leader_value] + for lv in leader_value: + if lv in values: + slave_idx = values.index(lv) + slave_option = config.forcepermissive.option(path, slave_idx) + if await slave_option.option.issubmulti(): + slave_values = await slave_option.value.get() + if linked_value not in slave_values: + slave_values.append(linked_value) + await slave_option.value.set(slave_values) + await slave_option.owner.set('link') + else: + await slave_option.value.set(linked_value) + await slave_option.owner.set('link') + else: + option = config.forcepermissive.option(path, leader_index) + if leader_index is None and await option.option.ismulti() and not isinstance(linked_value, list): + values = await option.value.get() + if linked_value not in values: + values.append(linked_value) + await option.value.set(values) + await option.owner.set('link') + else: + await option.value.set(linked_value) + await option.owner.set('link') + except AttributeError as err: + if linked_provider == 'oauth2_external': + raise ValueError(str(err)) from err + pass + except Exception as err: + await config.property.read_only() + raise err from err + await config.property.read_only() + + +def get_ip_from_domain(domain): + if not domain: + return + hostname, domainname = domain.split('.', 1) + return DOMAINS[domainname][1][DOMAINS[domainname][0].index(hostname)] + + +async def load_machine_config(server_name: str, + datas: dict, + module_info: dict, + display_name, + ): + optiondescription = {'set_linked': set_linked, + 'set_linked_multi_variables': set_linked_multi_variables, + 'get_linked_configuration': get_linked_configuration, + 'set_linked_configuration': set_linked_configuration, + 'get_ip_from_domain': get_ip_from_domain, + } + cfg = RougailConfig.copy() + module_info.servers.append(server_name) + cfg['variable_namespace'] = ROUGAIL_NAMESPACE + cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION + if datas['module'] == 'host': + cfg['tmpfile_dest_dir'] = datas['values'][f'{ROUGAIL_NAMESPACE}.host_install_dir'] + '/host/configurations/' + server_name + cfg['templates_dir'] = module_info.templates_dir + cfg['dictionaries_dir'] = module_info.dictionaries_dir + cfg['functions_file'] = module_info.functions_file + cfg['multi_functions'] = MULTI_FUNCTIONS + cfg['extra_dictionaries'] = module_info.extra_dictionaries + cfg['extra_annotators'].append('risotto.rougail') + cfg['internal_functions'] = list(optiondescription.keys()) + try: + eolobj = RougailConvert(cfg) + except Exception as err: + print(f'Try to load {module_info.modules}') + raise err from err + tiram_obj = eolobj.save(None) +# if server_name == 'revprox.in.silique.fr': +# print(tiram_obj) + #cfg['patches_dir'] = join(test_dir, 'patches') + try: + exec(tiram_obj, None, optiondescription) + except Exception as err: + print(tiram_obj) + raise Exception(f'unknown error when load tiramisu object {err}') from err + config = await Config(optiondescription['option_0'], display_name=display_name) + await config.property.read_write() + try: + add_srv = await config.option('machine.add_srv').value.get() + except AttributeError: + add_srv = False + return {'config': config, + 'templates_informations': cfg, + 'interface_index': 0, + 'module_name': datas['module'], + 'add_srv': add_srv, + } diff --git a/src/risotto/utils.py b/src/risotto/utils.py index 0e76159..0b95148 100644 --- a/src/risotto/utils.py +++ b/src/risotto/utils.py @@ -1,6 +1,21 @@ +from os import environ +from json import load +from typing import List +from ipaddress import ip_address +from toml import load as toml_load +from pprint import pprint + + MULTI_FUNCTIONS = [] CONFIGS = {} DOMAINS = {} +ZONES = {} +ZONES_SERVER = {} +SERVERS = {} + + +with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh: + RISOTTO_CONFIG = toml_load(fh) def _(s): @@ -13,3 +28,80 @@ def multi_function(function): if name not in MULTI_FUNCTIONS: MULTI_FUNCTIONS.append(name) return function + + +async def value_pprint(dico, config): + pprint_dict = {} + for path, value in dico.items(): + if await config.option(path).option.type() == 'password' and value: + value = 'X' * len(value) + pprint_dict[path] = value + pprint(pprint_dict) + + +def load_zones_server(): + if ZONES_SERVER: + return + with open('servers.json', 'r') as server_fh: + ZONES_SERVER.update(load(server_fh)) + + +def load_zones(): + global ZONES + if ZONES: + return + + load_zones_server() + ZONES.update(ZONES_SERVER['zones']) + for server_name, server in ZONES_SERVER['servers'].items(): + if 'informations' not in server: + continue + server_zones = server['informations']['zones_name'] + server_extra_domainnames = server['informations'].get('extra_domainnames', []) + if len(server_zones) > 1 and len(server_zones) != len(server_extra_domainnames) + 1: + raise Exception(f'the server "{server_name}" has more that one zone, please set correct number of extra_domainnames ({len(server_zones) - 1} instead of {len(server_extra_domainnames)})') + + for idx, zone_name in enumerate(server_zones): + zone_domain_name = ZONES[zone_name]['domain_name'] + if idx == 0: + zone_server_name = server_name + else: + zone_server_name = server_extra_domainnames[idx - 1] + server_domain_name = zone_server_name.split('.', 1)[1] + if zone_domain_name and zone_domain_name != server_domain_name: + raise Exception(f'wrong server_name "{zone_server_name}" in zone "{zone_name}" should ends with "{zone_domain_name}"') + ZONES[zone_name].setdefault('hosts', []).append(server_name) + + +def load_domains(): + global DOMAINS + if DOMAINS: + return + load_zones() + for zone_name, zone in ZONES_SERVER['zones'].items(): + if 'domain_name' in zone: + hosts = [] + ips = [] + for host in ZONES[zone_name].get('hosts', []): + hosts.append(host.split('.', 1)[0]) + ips.append(_get_ip(host, [zone_name], 0)) + DOMAINS[zone['domain_name']] = (tuple(hosts), tuple(ips)) + + +def _get_ip(server_name: str, + zones_name: List[str], + index: str, + ) -> str: + if server_name is None or zones_name is None: + return + load_zones() + index = int(index) + zone_name = zones_name[index] + if zone_name not in ZONES: + raise ValueError(f"cannot set IP in unknown zone '{zone_name}'") + zone = ZONES[zone_name] + if server_name not in zone['hosts']: + raise ValueError(f"cannot set IP in unknown server '{server_name}'") + server_index = zone['hosts'].index(server_name) +# print(server_name, zones_name, index, str(ip_address(zone['start_ip']) + server_index)) + return str(ip_address(zone['start_ip']) + server_index)