From 186c886e843e208865bb0c5f57e7cc5afdd481e3 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Thu, 22 Jun 2023 16:19:44 +0200 Subject: [PATCH] tiramisu 3 to 4 --- ansible/inventory.py | 65 +++-- sbin/risotto_display | 103 ++++---- sbin/risotto_templates | 17 +- src/risotto/image.py | 68 +++-- src/risotto/machine.py | 310 ++++++++++++---------- src/risotto/rougail/annotator.py | 433 ++++++++++++++++++------------- src/risotto/utils.py | 64 +++-- 7 files changed, 608 insertions(+), 452 deletions(-) diff --git a/ansible/inventory.py b/ansible/inventory.py index 36384be..3ec4881 100755 --- a/ansible/inventory.py +++ b/ansible/inventory.py @@ -8,8 +8,8 @@ from argparse import ArgumentParser from json import load as json_load, dumps, JSONEncoder from os import remove from os.path import isfile -from asyncio import run from traceback import print_exc +from sys import stderr, argv from risotto.machine import load, TIRAMISU_CACHE, VALUES_CACHE, INFORMATIONS_CACHE, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION from tiramisu import Config @@ -40,12 +40,13 @@ class RisottoInventory(object): parser.add_argument('--host', action='store') parser.add_argument('--nocache', action='store_true') parser.add_argument('--debug', action='store_true') + parser.add_argument('--pretty_print', action='store_true') self.args = parser.parse_args() if self.args.debug: global DEBUG DEBUG = True - async def run(self): + def run(self): if self.args.list and self.args.host: raise Exception('cannot have --list and --host together') if self.args.list or self.args.nocache: @@ -55,20 +56,20 @@ class RisottoInventory(object): remove(VALUES_CACHE) if isfile(INFORMATIONS_CACHE): remove(INFORMATIONS_CACHE) - config = await load(TIRAMISU_CACHE, - VALUES_CACHE, - INFORMATIONS_CACHE, - ) + config = load(TIRAMISU_CACHE, + VALUES_CACHE, + INFORMATIONS_CACHE, + ) if self.args.list: - return await self.do_inventory(config) + return self.do_inventory(config) elif self.args.host: - return await self.get_vars(config, self.args.host) + return self.get_vars(config, self.args.host) raise Exception('pfff') - async def do_inventory(self, - config: Config, - ) -> dict: - servers = [await subconfig.option.doc() for subconfig in await config.option.list('optiondescription') if await subconfig.information.get('module') == 'host'] + def do_inventory(self, + config: Config, + ) -> dict: + servers = [subconfig.doc() for subconfig in config.option.list('optiondescription') if subconfig.information.get('module') == 'host'] return dumps({ 'group': { 'hosts': servers, @@ -81,44 +82,54 @@ class RisottoInventory(object): } }) - async def get_vars(self, - config: Config, - host_name: str, - ) -> dict: + def get_vars(self, + config: Config, + host_name: str, + ) -> dict: ret = {} rougailconfig = RougailConfig.copy() rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION - for subconfig in await config.option.list('optiondescription'): - server_name = await subconfig.option.description() - module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get() + for subconfig in config.option.list('optiondescription'): + server_name = subconfig.description() + module_name = subconfig.option(subconfig.information.get('provider:global:module_name')).value.get() if module_name == 'host' and server_name != host_name: continue engine = RougailSystemdTemplate(subconfig, rougailconfig) - await engine.load_variables() + engine.load_variables(with_flatten=False) if module_name != 'host' and engine.rougail_variables_dict['general']['host'] != host_name: continue ret[server_name] = engine.rougail_variables_dict - ret['modules'] = await config.information.get('modules') + ret['modules'] = config.information.get('modules') ret['delete_old_image'] = False ret['configure_host'] = True ret['only_machine'] = None ret['copy_templates'] = False ret['copy_tests'] = False - ret['host_install_dir'] = ret[host_name].pop('host_install_dir') + ret['host_install_dir'] = ret[host_name]['general']['host_install_dir'] return dumps(ret, cls=RougailEncoder) # Get the inventory. -async def main(): +def main(): try: inv = RisottoInventory() - values = await inv.run() - print(values) + values = inv.run() + if inv.args.pretty_print: + from pprint import pprint + from json import loads + pprint(loads(values)) + else: + print(values) except Exception as err: if DEBUG: print_exc() - print(err) + print('---', file=stderr) + extra='' + else: + extra=f'\nmore informations with commandline "{" ".join(argv)} --debug"' + print(f'{err}{extra}', file=stderr) + exit(1) -run(main()) +main() diff --git a/sbin/risotto_display b/sbin/risotto_display index 4a15ef8..d923405 100755 --- a/sbin/risotto_display +++ b/sbin/risotto_display @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from asyncio import run from tabulate import tabulate from argparse import ArgumentParser @@ -18,19 +17,19 @@ def list_to_string(lst): return lst -async def get_files_subelements(type_name, element, files_subelement, files_cols): +def get_files_subelements(type_name, element, files_subelement, files_cols): data = {} - if not await element.option('activate').value.get(): + if not element.option('activate').value.get(): return data for subelement in files_subelement.values(): if subelement['type'] == 'subelement': try: - value = list_to_string(await element.option(subelement['key']).value.get()) + value = list_to_string(element.option(subelement['key']).value.get()) # FIXME except AttributeError: except Exception: value = '' elif subelement['type'] == 'information': - value = await element.information.get(subelement['key'], '') + value = element.information.get(subelement['key'], '') elif subelement['type'] == 'none': value = subelement['value'] else: @@ -47,7 +46,7 @@ async def get_files_subelements(type_name, element, files_subelement, files_cols return data -async def services(config, values): +def services(config, values): files_subelement = {'Source': {'key': 'source', 'type': 'information'}, 'Nom': {'key': 'name', 'type': 'subelement'}, 'Variable': {'key': 'variable', 'type': 'subelement'}, @@ -57,26 +56,26 @@ async def services(config, values): 'Moteur': {'key': 'engine', 'type': 'information'}, } disabled_services = [] - for service in await config.option.list(type="all"): - doc = await service.option.doc() + for service in config.option.list(type="all"): + doc = service.option.doc() files_lst = [] files_cols = set() - if not await service.option('manage').value.get(): + if not service.option('manage').value.get(): doc += " - unmanaged" - if not await service.option('activate').value.get(): + if not service.option('activate').value.get(): disabled_services.append([doc]) else: - for type in await service.list(type="all"): - type_name = await type.option.doc() + for type in service.list(type="all"): + type_name = type.option.doc() if type_name in ['files', 'overrides']: - for element in await type.list(type="all"): - data = await get_files_subelements(type_name, element, files_subelement, files_cols) + for element in type.list(type="all"): + data = get_files_subelements(type_name, element, files_subelement, files_cols) if data: files_lst.append(data) elif type_name == 'manage': pass elif type_name == 'activate': - if not await type.value.get(): + if not type.value.get(): doc += " - unactivated" else: print("FIXME " + type_name) @@ -89,19 +88,19 @@ async def services(config, values): values["Services désactivés"] = {'keys': ['Nom'], 'lst': disabled_services} -async def table_leader(config, read_only): +def table_leader(config, read_only): keys = ['Description'] if read_only: keys.append('Cachée') - leadership_lst = await config.list(type="all") + leadership_lst = config.list(type="all") leader = leadership_lst.pop(0) - leader_owner = await leader.owner.get() - follower_names = [await follower.option.name() for follower in leadership_lst] - doc = await leader.option.doc() - properties = await leader.property.get() + leader_owner = leader.owner.get() + follower_names = [follower.option.name() for follower in leadership_lst] + doc = leader.option.doc() + properties = leader.property.get() if 'mandatory' in properties: doc += '*' - name = await leader.option.name() + name = leader.option.name() lst = [[f'{doc} ({name})']] if read_only: if 'hidden' in properties: @@ -109,7 +108,7 @@ async def table_leader(config, read_only): else: hidden = '' lst[0].append(hidden) - for idx, leader_value in enumerate(await leader.value.get()): + for idx, leader_value in enumerate(leader.value.get()): keys.append(f'Valeur {idx}') keys.append(f'Utilisateur {idx}') lst[0].append(leader_value) @@ -117,11 +116,11 @@ async def table_leader(config, read_only): for follower_idx, follower_name in enumerate(follower_names): follower_option = config.option(follower_name, idx) if idx == 0: - doc = await follower_option.option.doc() - properties = await follower_option.property.get() + doc = follower_option.option.doc() + properties = follower_option.property.get() if 'mandatory' in properties: doc += '*' - name = await follower_option.option.name() + name = follower_option.option.name() lst.append([f'{doc} ({name})']) if read_only: if 'hidden' in properties: @@ -130,48 +129,48 @@ async def table_leader(config, read_only): hidden = '' lst[-1].append(hidden) try: - lst[follower_idx + 1].append(list_to_string(await follower_option.value.get())) - lst[follower_idx + 1].append(await follower_option.owner.get()) + lst[follower_idx + 1].append(list_to_string(follower_option.value.get())) + lst[follower_idx + 1].append(follower_option.owner.get()) except PropertiesOptionError: pass # leader = next leader_iter # if master_values is None: -# master_values = await subconfig.value.get() +# master_values = subconfig.value.get() return {'keys': keys, 'lst': lst} -async def table(config, prefix_len, values, read_only): +def table(config, prefix_len, values, read_only): lst = [] - for subconfig in await config.option.list(type="all"): + for subconfig in config.option.list(type="all"): # prefix = prefix_len * 2 * ' ' -# if await subconfig.option.isoptiondescription(): +# if subconfig.option.isoptiondescription(): # prefix += '=>' # else: # prefix += '-' # display_str = f'{prefix} {description}' # if name != description: # display_str = f'{display_str} ({name})' - name = await subconfig.option.name() - doc = await subconfig.option.doc() + name = subconfig.option.name() + doc = subconfig.option.doc() if prefix_len == 0 and ROUGAIL_NAMESPACE != name: doc = doc.capitalize() if prefix_len == 0 and name == 'services': values['Services'] = {} - await services(subconfig, values['Services']) - elif await subconfig.option.isoptiondescription(): - od_name = f'{doc} ({(await subconfig.option.path()).split(".", 1)[1]})' + services(subconfig, values['Services']) + elif subconfig.option.isoptiondescription(): + od_name = f'{doc} ({(subconfig.option.path()).split(".", 1)[1]})' values[od_name] = None - if await subconfig.option.isleadership(): - values[od_name] = await table_leader(subconfig, read_only) + if subconfig.option.isleadership(): + values[od_name] = table_leader(subconfig, read_only) else: - values[od_name] = await table(subconfig, prefix_len + 1, values, read_only) + values[od_name] = table(subconfig, prefix_len + 1, values, read_only) else: - value = list_to_string(await subconfig.value.get()) - doc = await subconfig.option.doc() - properties = await subconfig.property.get() + value = list_to_string(subconfig.value.get()) + doc = subconfig.option.doc() + properties = subconfig.property.get() if 'mandatory' in properties: doc += '*' - name = await subconfig.option.name() + name = subconfig.option.name() lst.append([f'{doc} ({name})', value]) if read_only: if 'hidden' in properties: @@ -179,7 +178,7 @@ async def table(config, prefix_len, values, read_only): else: hidden = '' lst[-1].append(hidden) - lst[-1].append(await subconfig.owner.get()) + lst[-1].append(subconfig.owner.get()) keys = ['Description', 'Valeur'] if read_only: keys.append('Cachée') @@ -187,7 +186,7 @@ async def table(config, prefix_len, values, read_only): return {'keys': keys, 'lst': lst} -async def main(): +def main(): parser = ArgumentParser() parser.add_argument('server_name') parser.add_argument('--read_only', action='store_true') @@ -199,18 +198,18 @@ async def main(): values = {} server_name = args.server_name - config = await load(hide_secret=HIDE_SECRET, + config = load(hide_secret=HIDE_SECRET, original_display_name=True, valid_mandatories=args.read_only, ) if not args.read_only: - await config.property.read_write() + config.property.read_write() root_option = config.option(normalize_family(server_name)) try: - await root_option.option.get() + root_option.option.get() except AttributeError: - exit(f'Unable to find {server_name} configuration: {[await o.option.description() for o in await config.option.list(type="optiondescription")]}') - await table(root_option, 0, values, args.read_only) + exit(f'Unable to find {server_name} configuration: {[o.option.description() for o in config.option.list(type="optiondescription")]}') + table(root_option, 0, values, args.read_only) for title, dico in values.items(): if title == 'Services': if not dico: @@ -233,4 +232,4 @@ async def main(): print(tabulate(dico['lst'], headers=dico['keys'], tablefmt="fancy_grid")) -run(main()) +main() diff --git a/sbin/risotto_templates b/sbin/risotto_templates index 8f76391..4151cdc 100755 --- a/sbin/risotto_templates +++ b/sbin/risotto_templates @@ -1,13 +1,12 @@ #!/usr/bin/env python3 -from asyncio import run from argparse import ArgumentParser from traceback import print_exc from risotto.machine import remove_cache, build_files, INSTALL_DIR -async def main(): +def main(): parser = ArgumentParser() parser.add_argument('server_name', nargs='?') parser.add_argument('--nocache', action='store_true') @@ -19,12 +18,12 @@ async def main(): remove_cache() try: - await build_files(None, - args.server_name, - False, - args.copy_tests, - template=args.template, - ) + build_files(None, + args.server_name, + False, + args.copy_tests, + template=args.template, + ) except Exception as err: if args.debug: print_exc() @@ -32,4 +31,4 @@ async def main(): print(f'templates generated in "{INSTALL_DIR}" directory') -run(main()) +main() diff --git a/src/risotto/image.py b/src/risotto/image.py index 02accca..4de24b4 100644 --- a/src/risotto/image.py +++ b/src/risotto/image.py @@ -19,8 +19,8 @@ class ModuleCfg(): self.depends = [] self.manuals = [] self.tests = [] - self.providers = [] - self.suppliers = [] + #self.providers = [] + #self.suppliers = [] def __repr__(self): return str(vars(self)) @@ -131,10 +131,10 @@ class Modules: if appname not in cfg.providers[provider]: cfg.providers[provider].append(appname) supplier = app.get('supplier') - if supplier: - self.suppliers.setdefault(supplier, []) - if appname not in self.suppliers[supplier]: - self.suppliers[supplier].append(appname) + #if supplier: + # self.suppliers.setdefault(supplier, []) + # if appname not in self.suppliers[supplier]: + # self.suppliers[supplier].append(appname) if 'distribution' in app and app['distribution']: distribution = appname else: @@ -216,33 +216,57 @@ def applicationservice_copy(src_file: str, copytree(src_file, dst_file) -async def valid_mandatories(config): - mandatories = await config.value.mandatory() - await config.property.pop('mandatory') +def valid_mandatories(config): + mandatories = config.value.mandatory() + config.property.remove('mandatory') hidden = {} variables = {} title = None if mandatories: server_name = None - for mandatory in mandatories: - path_server_name, path = mandatory.split('.', 1) - var_server_name = await config.option(path_server_name).option.description() + for mandatory_option in mandatories: + path_server_name, path = mandatory_option.path().split('.', 1) + var_server_name = config.option(path_server_name).description() if server_name != var_server_name: server_name = var_server_name title = f'=== Missing variables for {server_name} ===' - suboption = config.option(mandatory) - text = await suboption.option.doc() + text = mandatory_option.doc() msg = f' - {text} ({path})' - supplier = await suboption.information.get('supplier', None) + supplier = mandatory_option.information.get('supplier', None) if supplier: msg += f' you could add a service that provides "{supplier}"' - try: - await config.option(mandatory).value.get() - variables.setdefault(title, []).append(msg) - except PropertiesOptionError as err: - if 'hidden' not in err.proptype: - raise PropertiesOptionError(err) - hidden.setdefault(title, []).append(msg) + if mandatory_option.isfollower(): + leader = mandatory_option.leader() + try: + leader_value = leader.value.get() + except PropertiesOptionError as err: + if 'hidden' not in err.proptype: + raise err from err + hidden.setdefault(title, []).append(msg) + else: + config.property.add('mandatory') + for idx in range(mandatory_option.value.len()): + try: + config.option(mandatory_option.path(), idx).value.get() + except PropertiesOptionError as err: + path = leader.path() + spath = path.split('.', 1)[1] + submsg = f'{msg} at index {idx} (value of leader "{leader.doc()}" ({spath}) is "{leader_value[idx]}")' + if 'hidden' in err.proptype: + hidden.setdefault(title, []).append(submsg) + elif 'mandatory' in err.proptype: + variables.setdefault(title, []).append(submsg) + else: + raise err from err + config.property.remove('mandatory') + else: + try: + mandatory_option.value.get() + variables.setdefault(title, []).append(msg) + except PropertiesOptionError as err: + if 'hidden' not in err.proptype: + raise err from err + hidden.setdefault(title, []).append(msg) if not variables: variables = hidden return variables diff --git a/src/risotto/machine.py b/src/risotto/machine.py index 7c57e34..6b8d28f 100644 --- a/src/risotto/machine.py +++ b/src/risotto/machine.py @@ -5,11 +5,11 @@ from .rougail.annotator import calc_providers, calc_providers_global, calc_provi from rougail import RougailConfig, RougailConvert from os import remove, makedirs, listdir, chmod from os.path import isfile, isdir, abspath, join, dirname -from json import dump as json_dump, load as json_load +from pickle import dump as pickle_dump, load as pickle_load from yaml import load as yaml_load, SafeLoader -from ipaddress import ip_network +from ipaddress import IPv4Interface, ip_network # -from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value +from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value, calc_value_property_help from rougail.utils import normalize_family from rougail import RougailSystemdTemplate from shutil import copy2, copytree, rmtree @@ -29,8 +29,8 @@ def tiramisu_display_name(kls, CONFIG_FILE = 'servers.yml' TIRAMISU_CACHE = 'tiramisu_cache.py' -VALUES_CACHE = 'values_cache.json' -INFORMATIONS_CACHE = 'informations_cache.json' +VALUES_CACHE = 'values_cache.pickle' +INFORMATIONS_CACHE = 'informations_cache.pickle' INSTALL_DIR = RISOTTO_CONFIG['directories']['dest'] INSTALL_CONFIG_DIR = 'configurations' INSTALL_TMPL_DIR= 'templates' @@ -47,6 +47,7 @@ FUNCTIONS = {'calc_providers': calc_providers, 'valid_in_network': valid_in_network, 'valid_not_equal': valid_not_equal, 'calc_value': calc_value, + 'calc_value_property_help': calc_value_property_help, 'normalize_family': normalize_family, } @@ -88,28 +89,28 @@ def remove_cache(): remove(INFORMATIONS_CACHE) -async def templates(server_name, - config, - just_copy=False, - copy_manuals=False, - template=None, - extra_variables=None, - ): +def templates(server_name, + config, + just_copy=False, + copy_manuals=False, + template=None, + extra_variables=None, + ): subconfig = config.option(normalize_family(server_name)) try: - await subconfig.option.get() + subconfig.get() except: - servers = [await server.option.description() for server in await config.option.list('optiondescription')] + servers = [server.description() for server in config.list('optiondescription')] raise Exception(f'cannot find server name "{server_name}": {servers}') rougailconfig = RougailConfig.copy() rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION rougailconfig['tmp_dir'] = 'tmp' - rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir') - rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir') - rougailconfig['functions_file'] = await subconfig.information.get('functions_files') - module = await subconfig.information.get('module') + rougailconfig['templates_dir'] = subconfig.information.get('templates_dir') + rougailconfig['patches_dir'] = subconfig.information.get('patches_dir') + rougailconfig['functions_file'] = subconfig.information.get('functions_files') + module = subconfig.information.get('module') is_host = module == 'host' if is_host: rougailconfig['systemd_tmpfile_delete_before_create'] = True @@ -138,15 +139,15 @@ async def templates(server_name, engine.engines[eng] = engine.engines['none'] try: if not template: - await engine.instance_files(extra_variables=extra_variables) + engine.instance_files(extra_variables=extra_variables) else: - await engine.instance_file(template, extra_variables=extra_variables) + engine.instance_file(template, extra_variables=extra_variables) except Exception as err: print() print(f'=== Configuration: {server_name} ===') try: - values = await subconfig.value.dict() - await value_pprint(values, subconfig) + values = subconfig.value.dict() + value_pprint(values, subconfig) except: pass raise err from err @@ -159,17 +160,17 @@ async def templates(server_name, if copy_manuals and not is_host: dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module) if not isdir(dest_dir): - for manual in await subconfig.information.get('manuals_dirs'): + for manual in subconfig.information.get('manuals_dirs'): for filename in listdir(manual): src_file = join(manual, filename) dst_file = join(dest_dir, filename) copy(src_file, dst_file) - copy_tests = await config.information.get('copy_tests') + copy_tests = config.information.get('copy_tests') if copy_tests and not is_host: dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module) if not isdir(dest_dir): - for tests in await subconfig.information.get('tests_dirs'): + for tests in subconfig.information.get('tests_dirs'): for filename in listdir(tests): src_file = join(tests, filename) dst_file = join(dest_dir, filename) @@ -178,7 +179,6 @@ async def templates(server_name, class Loader: def __init__(self, - clean_directories, hide_secret, original_display_name, valid_mandatories, @@ -188,10 +188,6 @@ class Loader: self.original_display_name = original_display_name self.valid_mandatories = valid_mandatories self.config_file = config_file - if clean_directories: - if isdir(INSTALL_DIR): - rmtree(INSTALL_DIR) - makedirs(INSTALL_DIR) def load_tiramisu_file(self): """Load config file (servers.yml) and build tiramisu file with dataset informations @@ -224,28 +220,54 @@ class Loader: rougail = RougailConvert(cfg) for host_name, datas in self.servers_json['hosts'].items(): for server_name, server_datas in datas['servers'].items(): - for zone in server_datas['informations']['zones_name']: + if 'provider_zone' not in server_datas and 'zones_name' not in server_datas: + raise Exception(f'cannot find "zones_name" attribute for server "{server_name}"') + if 'provider_zone' in server_datas: + zones_name.setdefault(server_datas['provider_zone'], []).append(server_name) + if 'zones_name' not in server_datas: + server_datas['zones_name'] = [] + if server_datas['provider_zone'] in server_datas['zones_name']: + raise Exception(_('provider_zone "{server_datas["provider_zone"]}" must not be in "zones" "{server_datas["zones_name"]}"')) + # external zone is better in first place + if server_datas['zones_name'] and self.servers_json['zones']['external_zone'] == server_datas['zones_name'][0]: + server_datas['zones_name'].append(server_datas['provider_zone']) + else: + server_datas['zones_name'].insert(0, server_datas['provider_zone']) +# if server_datas['zones_name'] and server_datas['provider_zone'] == self.servers_json['zones']['external_zone']: +# server_datas['zones_name'].insert(0, server_datas['provider_zone']) +# else: +# server_datas['zones_name'].append(server_datas['provider_zone']) + for zone in server_datas['zones_name']: zones_name.setdefault(zone, []).append(server_name) self.zones = {} zones_network = ip_network(self.servers_json['zones']['network']) zone_start_ip = zones_network.network_address domain_name = self.servers_json['zones']['prefix_domain_name'] - for idx, zone_name in enumerate(zones_name): - sub_network = ip_network(f'{zone_start_ip}/28') - if not sub_network.subnet_of(zones_network): - raise Exception('not enough IP available') - if sub_network.num_addresses < len(zones_name[zone_name]): - #FIXME should try to increase network! - raise Exception(f'network too small for zone {zone_name}') - if idx == 0: + for zone_name in zones_name: + len_zone = len(zones_name[zone_name]) + for zone_cidr in [29, 28, 27, 26]: + try: + sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}') + except ValueError: + # calc network address for this mask + zone_start_ip = IPv4Interface(f'{zone_start_ip}/{zone_cidr}').network.broadcast_address + 1 + sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}') + if not sub_network.subnet_of(zones_network): + raise Exception('not enough IP available') + length = sub_network.num_addresses - 3 # network + broadcast + host + if length >= len_zone: + break + else: + raise Exception(f'network too small for zone "{zone_name}" ({sub_network.num_addresses - 2} < {len_zone})') + if self.servers_json['zones']['external_zone'] == zone_name: zone_domaine_name = domain_name else: zone_domaine_name = zone_name + '.' + domain_name - network = sub_network.network_address self.zones[zone_name] = {'domain_name': zone_domaine_name, 'network': str(sub_network), 'host_ip': str(network + 1), + 'length': length, 'start_ip': str(network + 2) } zone_start_ip = str(sub_network.broadcast_address + 1) @@ -266,9 +288,13 @@ class Loader: # load host module_info = modules.get('host') + tls_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}' cfg['risotto_globals'][host_name] = {'global:server_name': host_name, + 'global:server_names': [host_name for zone in self.zones], + 'global:zones_name': list(self.zones), 'global:module_name': 'host', 'global:host_install_dir': abspath(INSTALL_DIR), + 'global:tls_server': tls_host_name, } functions_files |= set(module_info.functions_file) self.load_dictionaries(cfg, @@ -280,7 +306,7 @@ class Loader: modules_info = {} for server_name, server_datas in datas['servers'].items(): module_info = modules.get(server_datas['applicationservice']) - zones_name = server_datas['informations']['zones_name'] + zones_name = server_datas['zones_name'] values = [f'{server_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name] if server_datas['applicationservice'] == 'tls': true_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}' @@ -292,7 +318,10 @@ class Loader: 'global:zones_name': zones_name, 'global:zones_list': list(range(len(zones_name))), 'global:module_name': server_datas['applicationservice'], + 'global:prefix_domain_name': self.servers_json['zones']['prefix_domain_name'] } + if 'provider_zone' in server_datas: + cfg['risotto_globals'][true_host_name]['global:provider_zone'] = server_datas['provider_zone'] server_datas['server_name'] = true_host_name functions_files |= set(module_info.functions_file) self.load_dictionaries(cfg, @@ -309,26 +338,24 @@ class Loader: zones = set() dns_module_name = None for host in self.servers_json['hosts'].values(): - zones = [None, None] + zones = [self.servers_json['zones']['external_zone'], None] for server_name, datas in host['servers'].items(): + if not 'applicationservice' in datas: + raise Exception(f'cannot find applicationservice for "{server_name}"') if datas['applicationservice'] == 'tls': - raise Exception(f'forbidden module name "tls" for server {server_name}') + raise Exception(f'forbidden module name "tls" for server "{server_name}"') #FIXME use provider! - if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['informations']['zones_name']) > 0: + if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['zones_name']) > 0: if dns_module_name: break - # always add tls machine in second zone of reverse proxy - zones[1] = datas['informations']['zones_name'][0] - if datas['applicationservice'] == 'unbound': - # always add tls machine in second zone of reverse proxy - zones[0] = datas['informations']['zones_name'][0] + zones[1] = datas['provider_zone'] if None in zones: zones = [] else: if zones[0] == zones[1]: zones = [zones[0]] host['servers']['tls'] = {'applicationservice': 'tls', - 'informations': {'zones_name': list(zones)}, + 'zones_name': list(zones), } def load_dictionaries(self, cfg, module_info, server_name, rougail): @@ -344,7 +371,7 @@ class Loader: self.manuals_dirs[server_name] = module_info.manuals self.tests_dirs[server_name] = module_info.tests - async def tiramisu_file_to_tiramisu(self): + def tiramisu_file_to_tiramisu(self): # l tiramisu_space = FUNCTIONS.copy() try: @@ -356,78 +383,78 @@ class Loader: display_name = None else: display_name = tiramisu_display_name - self.config = await Config(tiramisu_space['option_0'], + self.config = Config(tiramisu_space['option_0'], display_name=display_name, ) - async def load_values_and_informations(self): + def load_values_and_informations(self): config = self.config - await config.property.read_write() - await config.property.pop('validator') - await config.property.pop('cache') + config.property.read_write() + config.property.remove('validator') + config.property.remove('cache') load_zones(self.zones, self.servers_json['hosts']) - await config.information.set('zones', self.zones) + config.information.set('zones', self.zones) for host_name, hosts_datas in self.servers_json['hosts'].items(): information = config.option(normalize_family(host_name)).information - await information.set('module', 'host') - await information.set('templates_dir', self.templates_dir[host_name]) - await information.set('patches_dir', self.patches_dir[host_name]) - await information.set('functions_files', self.functions_files[host_name]) - await self.set_values(host_name, config, hosts_datas) + information.set('module', 'host') + information.set('templates_dir', self.templates_dir[host_name]) + information.set('patches_dir', self.patches_dir[host_name]) + information.set('functions_files', self.functions_files[host_name]) + self.set_values(host_name, config, hosts_datas) for datas in hosts_datas['servers'].values(): server_name = datas['server_name'] information = config.option(normalize_family(server_name)).information - await information.set('module', datas['applicationservice']) - await information.set('templates_dir', self.templates_dir[server_name]) - await information.set('patches_dir', self.patches_dir[server_name]) - await information.set('functions_files', self.functions_files[server_name]) - await information.set('manuals_dirs', self.manuals_dirs[server_name]) - await information.set('tests_dirs', self.tests_dirs[server_name]) - await self.set_values(server_name, config, datas) + information.set('module', datas['applicationservice']) + information.set('templates_dir', self.templates_dir[server_name]) + information.set('patches_dir', self.patches_dir[server_name]) + information.set('functions_files', self.functions_files[server_name]) + information.set('manuals_dirs', self.manuals_dirs[server_name]) + information.set('tests_dirs', self.tests_dirs[server_name]) + self.set_values(server_name, config, datas) - await config.information.set('copy_tests', False) + config.information.set('copy_tests', False) # FIXME only one host_name is supported - await config.information.set('modules', self.modules[host_name]) - # await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules}) - await config.property.add('cache') + config.information.set('modules', self.modules[host_name]) + # config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules}) + config.property.add('cache') if self.valid_mandatories: - messages = await valid_mandatories(config) + messages = valid_mandatories(config) if messages: msg = '' for title, variables in messages.items(): msg += '\n' + title + '\n' msg += '\n'.join(variables) raise Exception(msg) - await config.property.read_only() - with open(VALUES_CACHE, 'w') as fh: - json_dump(await config.value.exportation(), fh) - with open(INFORMATIONS_CACHE, 'w') as fh: - json_dump(await config.information.exportation(), fh) + config.property.read_only() + with open(VALUES_CACHE, 'wb') as fh: + pickle_dump(config.value.exportation(), fh) + with open(INFORMATIONS_CACHE, 'wb') as fh: + pickle_dump(config.information.exportation(), fh) - async def set_values(self, - server_name, - config, - datas, - ): + def set_values(self, + server_name, + config, + datas, + ): if 'values' not in datas: return if not isinstance(datas['values'], dict): raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}') server_path = normalize_family(server_name) - await config.owner.set(self.config_file) + config.owner.set(self.config_file) for vpath, value in datas['values'].items(): path = f'{server_path}.{vpath}' try: if isinstance(value, dict): for idx, val in value.items(): - await config.option(path, int(idx)).value.set(val) + config.option(path, int(idx)).value.set(val) else: - await config.option(path).value.set(value) + config.option(path).value.set(value) except Exception as err: - await value_pprint(await config.value.dict(), config) + value_pprint(config.value.dict(), config) error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}' raise Exception(error_msg) from err - await config.owner.set('user') + config.owner.set('user') class LoaderCache(Loader): @@ -435,63 +462,60 @@ class LoaderCache(Loader): with open(TIRAMISU_CACHE) as fh: self.tiram_obj = fh.read() - async def load_values_and_informations(self): - with open(VALUES_CACHE, 'r') as fh: - await self.config.value.importation(json_load(fh)) - with open(INFORMATIONS_CACHE, 'r') as fh: - informations = json_load(fh) - # null is not a valid key in json => 'null' - informations[None] = informations.pop('null') - await self.config.information.importation(informations) + def load_values_and_informations(self): + with open(VALUES_CACHE, 'rb') as fh: + self.config.value.importation(pickle_load(fh)) + with open(INFORMATIONS_CACHE, 'rb') as fh: + self.config.information.importation(pickle_load(fh)) -async def load(clean_directories=False, - hide_secret=False, - original_display_name: bool=False, - valid_mandatories: bool=True, - copy_tests: bool=False, - ): +def load(hide_secret=False, + original_display_name: bool=False, + valid_mandatories: bool=True, + copy_tests: bool=False, + ): if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE): loader_obj = LoaderCache else: loader_obj = Loader - loader = loader_obj(clean_directories, - hide_secret, + loader = loader_obj(hide_secret, original_display_name, valid_mandatories, ) loader.load_tiramisu_file() - await loader.tiramisu_file_to_tiramisu() - await loader.load_values_and_informations() + loader.tiramisu_file_to_tiramisu() + loader.load_values_and_informations() config = loader.config - await config.property.read_only() - await config.information.set('copy_tests', copy_tests) - await config.cache.reset() + config.property.read_only() + config.information.set('copy_tests', copy_tests) + config.cache.reset() return config -async def build_files(hostname: str, - only_machine: str, - just_copy: bool, - copy_tests: bool, - template: str=None, - ) -> None: +def build_files(hostname: str, + only_machine: str, + just_copy: bool, + copy_tests: bool, + template: str=None, + ) -> None: + if isdir(INSTALL_DIR): + rmtree(INSTALL_DIR) + makedirs(INSTALL_DIR) with open(CONFIG_FILE, 'r') as server_fh: servers_json = yaml_load(server_fh, Loader=SafeLoader) - config = await load(copy_tests=copy_tests) - machines = [await subconfig.option.description() for subconfig in await config.option.list(type='optiondescription')] + config = load(copy_tests=copy_tests) + machines = [subconfig.description() for subconfig in config.option.list(type='optiondescription')] certificates = {'certificates': {}, 'configuration': servers_json['certificates'], } # get certificates informations - tls_machine = None + tls_machine = config.option(f'{normalize_family(hostname)}.general.tls_server').value.get() for machine in machines: - if machine.startswith('tls.'): - tls_machine = machine + if machine == tls_machine: continue if hostname is None: # FIXME multi host! - hostname = await config.option(normalize_family(machine)).option('general.host_name').value.get() + hostname = config.option(normalize_family(machine)).option('general.host_name').value.get() if just_copy: continue is_host = machine == hostname @@ -500,24 +524,24 @@ async def build_files(hostname: str, machine_config = config.option(normalize_family(machine)) certificate_names = [] private_names = [] - for service in await machine_config.option('services').option.list('optiondescription'): - if not await service.option('activate').value.get(): + for service in machine_config.option('services').list('optiondescription'): + if not service.option('activate').value.get(): continue -# if await service.option('manage').value.get(): +# if service.option('manage').value.get(): # certificate_type = 'server' # else: # certificate_type = 'client' - tls_ca_directory = await machine_config.option('general.tls_ca_directory').value.get() - tls_cert_directory = await machine_config.option('general.tls_cert_directory').value.get() - tls_key_directory = await machine_config.option('general.tls_key_directory').value.get() + tls_ca_directory = machine_config.option('general.tls_ca_directory').value.get() + tls_cert_directory = machine_config.option('general.tls_cert_directory').value.get() + tls_key_directory = machine_config.option('general.tls_key_directory').value.get() try: - for certificate in await service.option('certificates').option.list('all'): - if not await certificate.option('activate').value.get(): + for certificate in service.option('certificates').list('all'): + if not certificate.option('activate').value.get(): continue - certificate_data = await certificate.value.dict() - certificate_data['type'] = await certificate.information.get('type') - certificate_data['authority'] = join(tls_ca_directory, await certificate.information.get('authority') + '.crt') - certificate_data['format'] = await certificate.information.get('format') + certificate_data = {key.rsplit('.', 1)[1]: value for key, value in certificate.value.dict().items()} + certificate_data['type'] = certificate.information.get('type') + certificate_data['authority'] = join(tls_ca_directory, certificate.information.get('authority') + '.crt') + certificate_data['format'] = certificate.information.get('format') is_list_name = isinstance(certificate_data['name'], list) is_list_domain = isinstance(certificate_data['domain'], list) if is_list_name != is_list_domain: @@ -526,7 +550,7 @@ async def build_files(hostname: str, certificate_data['provider'] = 'autosigne' if is_list_name: if len(certificate_data['name']) != len(certificate_data['domain']): - raise Exception('certificate name and domain name must have same lenght') + raise Exception('certificate name and domain name must have same length') for idx, certificate_name in enumerate(certificate_data['name']): cert_data = certificate_data.copy() if certificate_data['format'] == 'cert_key': @@ -565,19 +589,19 @@ async def build_files(hostname: str, continue if only_machine and only_machine != machine: continue - await templates(machine, - config, - just_copy=just_copy, - copy_manuals=True, - template=template, - extra_variables=certificates, - ) + templates(machine, + config, + just_copy=just_copy, + copy_manuals=True, + template=template, + extra_variables=certificates, + ) is_host = machine == hostname if is_host: directories[machine] = '/usr/local/lib' elif not just_copy: machine_config = config.option(normalize_family(machine)) - directories[machine] = await machine_config.option('general.config_dir').value.get() + directories[machine] = machine_config.option('general.config_dir').value.get() if only_machine: return directories if only_machine: diff --git a/src/risotto/rougail/annotator.py b/src/risotto/rougail/annotator.py index cffb004..0e3b2c7 100644 --- a/src/risotto/rougail/annotator.py +++ b/src/risotto/rougail/annotator.py @@ -26,19 +26,19 @@ def _parse_kwargs(provider, dns, kwargs, index=None): continue elif data['dns'] not in dns: continue - del data['dns'] +# del data['dns'] yield data @multi_function -def calc_providers_global(provider, multi, value, suffix=None): +def calc_providers_global(provider, multi, unique, value, suffix=None): if suffix is not None: return value[int(suffix)] return value @multi_function -def calc_providers_follower(provider, multi, dns, leader, index, **kwargs): +def calc_providers_follower(provider, multi, unique, dns, leader, index, **kwargs): ret = [] for data in _parse_kwargs(provider, dns, kwargs): if 'value' not in data: @@ -64,7 +64,7 @@ def calc_providers_follower(provider, multi, dns, leader, index, **kwargs): @multi_function -def calc_providers_dynamic_follower(provider, multi, dns, leader, index, suffix, **kwargs): +def calc_providers_dynamic_follower(provider, multi, unique, dns, leader, index, suffix, **kwargs): ret = [] for data in _parse_kwargs(provider, dns, kwargs): if 'value' not in data: @@ -92,7 +92,7 @@ def calc_providers_dynamic_follower(provider, multi, dns, leader, index, suffix, @multi_function -def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs): +def calc_providers_dynamic(provider, multi, unique, dns, suffix, **kwargs): ret = [] for data in _parse_kwargs(provider, dns, kwargs): if 'value' not in data: @@ -101,7 +101,7 @@ def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs): continue if isinstance(data['value'], list): for v in data['value']: - if v not in ret: + if not unique or v not in ret: ret.append(v) elif data['value'] not in ret: ret.append(data['value']) @@ -112,10 +112,14 @@ def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs): @multi_function -def calc_providers(provider, multi, dns, suffix=None, **kwargs): +def calc_providers(provider, multi, unique, dns, **kwargs): ret = [] for data in _parse_kwargs(provider, dns, kwargs): if isinstance(data['value'], list): + commun_dns = list(set(data['dns']) & set(dns)) + if len(commun_dns) == 1: + ret.append(data['value'][data['dns'].index(commun_dns[0])]) + continue for v in data['value']: if v in ret: continue @@ -135,216 +139,289 @@ class Annotator(Walk): objectspace: 'RougailObjSpace', *args): self.objectspace = objectspace - self.set_suppliers() + self.get_suppliers_providers() + self.dispatch_provider_supplier_to_zones() + self.dispatch_provider_to_zones() self.convert_providers() self.convert_suppliers() - def set_suppliers(self) -> dict: + def get_suppliers_providers(self) -> None: """ get supplier informations return something like: {'Host': ['host1.example.net', 'host2.example.net']} """ self.suppliers = {} - for variable in self.get_variables(): - if not hasattr(variable, 'supplier') or ':' in variable.supplier: - continue - nf_dns = variable.path.split('.', 1)[0] - server_name = self.objectspace.space.variables[nf_dns].doc - self.suppliers.setdefault(variable.supplier, []).append({'option': variable, - 'dns': server_name, - 'path_prefix': nf_dns, - 'server_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'], - 'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'], - 'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']) - }) - if not hasattr(variable, 'information'): - variable.information = self.objectspace.information(variable.xmlfiles) - variable.information.supplier = variable.supplier - - def convert_providers(self): self.providers = {} for variable in self.get_variables(): - if not hasattr(variable, 'provider'): + if not hasattr(variable, 'supplier') and not hasattr(variable, 'provider'): continue nf_dns = variable.path.split('.', 1)[0] server_name = self.objectspace.space.variables[nf_dns].doc - provider_name = variable.provider - if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host': - server_names = [server_name] - else: + # supplier + if hasattr(variable, 'supplier') and ':' not in variable.supplier: server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'] - if provider_name != 'Host' and not provider_name.startswith('Host:') and not provider_name.startswith('global:'): - p_data = {'option': variable, + zones = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'] + s_data = {'option': variable, 'dns': server_name, 'path_prefix': nf_dns, 'server_names': server_names, - 'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'], - 'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']), + 'zones': zones, + 'providers_zone': {}, } - else: - p_data = None - if ':' in provider_name: - key_name, key_type = provider_name.rsplit(':', 1) - is_provider = False - else: - key_name = key_type = provider_name - is_provider = True - if provider_name != 'Host': - self.providers.setdefault(provider_name, []).append(p_data) - if key_name != 'global' and key_name not in self.suppliers: - #warn(f'cannot find supplier "{key_name}" for "{server_name}"') - continue - # create a fill for this variable - fill = self.objectspace.fill(variable.xmlfiles) - new_target = self.objectspace.target(variable.xmlfiles) - new_target.name = variable - fill.target = [new_target] - if key_name == 'global': - fill.name = 'calc_providers_global' - elif self.objectspace.paths.is_dynamic(variable): - if self.objectspace.paths.is_follower(variable): - fill.name = 'calc_providers_dynamic_follower' + if variable.supplier != 'Host' and not variable.supplier.startswith('Host:') and not variable.supplier.startswith('global:'): + if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]: + s_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone'] + self.suppliers.setdefault(variable.supplier, []).append(s_data) + if not hasattr(variable, 'information'): + variable.information = self.objectspace.information(variable.xmlfiles) + variable.information.supplier = variable.supplier + # provider + if hasattr(variable, 'provider'): + provider_name = variable.provider + p_data = {'option': variable, + 'dns': server_name, + 'provider_name': provider_name, + 'path_prefix': nf_dns, + 'suppliers_zone': {}, + } + if variable.provider != 'Host' and not variable.provider.startswith('Host:') and not variable.provider.startswith('global:'): + if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]: + p_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone'] + if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host': + server_names = [server_name] else: - fill.name = 'calc_providers_dynamic' - elif self.objectspace.paths.is_follower(variable): - fill.name = 'calc_providers_follower' - else: - fill.name = 'calc_providers' - fill.namespace = variable.namespace - fill.index = 0 - # first parameter: the provider name (something link Host:incoming_ports) - param = self.objectspace.param(variable.xmlfiles) - param.name = 'provider' - param.text = provider_name - fill.param = [param] - # second parameter: current variable is a multi variable? - param = self.objectspace.param(variable.xmlfiles) - param.name = 'multi' - param.text = variable.multi - param.type = 'boolean' - fill.param.append(param) - if self.objectspace.paths.is_follower(variable): + server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'] + p_data['server_names'] = server_names + if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] != 'host': + p_data['zones'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'] + if ':' in provider_name: + p_data['is_main_provider'] = False + provider = provider_name.rsplit(':', 1)[0] + else: + p_data['is_main_provider'] = True + provider = variable.provider + self.providers.setdefault(provider, []).append(p_data) + + def dispatch_provider_supplier_to_zones(self): + """calculate zone where provider and supplier communicate + """ + self.providers_zone = {} + for provider_name, p_datas in self.providers.items(): + for p_data in p_datas: + if provider_name in ['global', 'Host'] or provider_name not in self.suppliers: + continue + if not 'provider_zone' in p_data: + provider_zone = None + else: + provider_zone = p_data['provider_zone'] + for s_data in self.suppliers[provider_name]: + if not provider_zone: + if provider_name.endswith('Client'): + p_server = provider_name[0:-6] + if p_server not in self.providers: + continue + for p_data_t in self.providers[p_server]: + if p_data_t['dns'] == s_data['dns'] and 'provider_zone' in p_data_t: + zone = p_data_t['provider_zone'] + break + else: + continue + else: + continue + else: + zone = provider_zone + if zone not in s_data['zones']: + continue + s_data['providers_zone'][provider_name] = zone + p_data['suppliers_zone'].setdefault(provider_name, {})[s_data['dns']] = zone + self.providers_zone.setdefault(zone, set()).add(provider_name) + + def dispatch_provider_to_zones(self): + """ add information with provider zone domain name + """ + self.providers_zone = {} + for provider_name, p_datas in self.providers.items(): + for p_data in p_datas: + if provider_name in ['global', 'Host'] or provider_name not in self.suppliers: + continue + if not 'provider_zone' in p_data: + continue + provider_zone = p_data['provider_zone'] + family = self.objectspace.paths.get_variable(f"providers", + namespace=self.objectspace.rougailconfig['variable_namespace'], + force_path_prefix=p_data['path_prefix'], + ) + if not hasattr(family, 'information'): + family.information = self.objectspace.information(family.xmlfiles) + name_in_zone = p_data['server_names'][p_data['zones'].index(provider_zone)] + setattr(family.information, provider_name, name_in_zone) + setattr(family.information, f'{provider_name}:zone', provider_zone) + + def convert_providers(self): + for provider_name, providers_data in self.providers.items(): + for provider_data in providers_data: + if provider_name != 'global' and provider_name not in self.suppliers: + continue + # create a fill for this variable + variable = provider_data['option'] + fill = self.objectspace.fill(variable.xmlfiles) + new_target = self.objectspace.target(variable.xmlfiles) + new_target.name = variable + fill.target = [new_target] + if provider_name == 'global': + fill.name = 'calc_providers_global' + elif self.objectspace.paths.is_dynamic(variable): + if self.objectspace.paths.is_follower(variable): + fill.name = 'calc_providers_dynamic_follower' + else: + fill.name = 'calc_providers_dynamic' + elif self.objectspace.paths.is_follower(variable): + fill.name = 'calc_providers_follower' + else: + fill.name = 'calc_providers' + fill.namespace = variable.namespace + fill.index = 0 + # first parameter: the provider name (something link Host:incoming_ports) param = self.objectspace.param(variable.xmlfiles) - param.name = 'leader' - param.text = self.objectspace.paths.get_leader(variable) - param.propertyerror = False - param.type = 'variable' + param.name = 'provider' + param.text = provider_data['provider_name'] + fill.param = [param] + # second parameter: current variable is a multi variable? + param = self.objectspace.param(variable.xmlfiles) + param.name = 'multi' + param.text = variable.multi + param.type = 'boolean' fill.param.append(param) - try: - leader_provider = self.objectspace.paths.get_leader(variable).provider - except: - leader_provider = None # param = self.objectspace.param(variable.xmlfiles) - param.name = 'index' - param.type = 'index' + param.name = 'unique' + param.text = variable.unique != "False" + param.type = 'boolean' fill.param.append(param) - if self.objectspace.paths.is_dynamic(variable): - # if dynamic: current suffix - # and add current DNS name, this is useful to known if supplier is link to this provider - param = self.objectspace.param(variable.xmlfiles) - param.name = 'suffix' - param.type = 'suffix' - fill.param.append(param) - if key_name != 'global': - param = self.objectspace.param(variable.xmlfiles) - param.name = 'dns' - param.text = server_names - fill.param.append(param) - if key_name == 'global': - param = self.objectspace.param(variable.xmlfiles) - param.name = 'value' - if provider_name in self.objectspace.rougailconfig['risotto_globals'][server_name]: - value = self.objectspace.rougailconfig['risotto_globals'][server_name][provider_name] - param.text = value - if isinstance(value, bool): - param.type = 'boolean' - else: - param.text = provider_name - param.type = 'information' - fill.param.append(param) - else: - # parse all supplier link to current provider - for idx, data in enumerate(self.suppliers[key_name]): - if p_data: - common_zones = data['zones'] & p_data['zones'] - if not common_zones: - continue - for zidx, zone in enumerate(data['zone_names']): - if zone in common_zones: - break - dns = data['server_names'][zidx] - else: - dns = data['dns'] - option = data['option'] - # if not provider, get the true option that we want has value - if not is_provider: - path_prefix = data['path_prefix'] - try: - supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_name}', path_prefix) - except KeyError: - #warn(f'cannot find supplier "{provider_name}" for "{dns}"') - continue - # first of all, get the supplier name + # + if self.objectspace.paths.is_follower(variable): param = self.objectspace.param(variable.xmlfiles) - param.name = f'dns_{idx}' - param.text = option + param.name = 'leader' + param.text = self.objectspace.paths.get_leader(variable) param.propertyerror = False param.type = 'variable' fill.param.append(param) - if not is_provider and \ - self.objectspace.paths.is_follower(variable): + try: + leader_provider = self.objectspace.paths.get_leader(variable).provider + except: + leader_provider = None + # + param = self.objectspace.param(variable.xmlfiles) + param.name = 'index' + param.type = 'index' + fill.param.append(param) + if self.objectspace.paths.is_dynamic(variable): + # if dynamic: current suffix + # and add current DNS name, this is useful to known if supplier is link to this provider + param = self.objectspace.param(variable.xmlfiles) + param.name = 'suffix' + param.type = 'suffix' + fill.param.append(param) + if provider_name != 'global': + param = self.objectspace.param(variable.xmlfiles) + param.name = 'dns' + param.text = provider_data['server_names'] + fill.param.append(param) + if provider_name == 'global': + param = self.objectspace.param(variable.xmlfiles) + param.name = 'value' + if provider_data['provider_name'] in self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']]: + value = self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']][provider_data['provider_name']] + param.text = value + if isinstance(value, bool): + param.type = 'boolean' + else: + param.text = provider_data['provider_name'] + param.type = 'information' + fill.param.append(param) + else: + # parse all supplier link to current provider + for idx, data in enumerate(self.suppliers[provider_name]): + if 'zones' in provider_data: + if provider_name not in data['providers_zone']: + continue + zone = data['providers_zone'][provider_name] + zidx = data['zones'].index(zone) + dns = data['server_names'][zidx] + else: + dns = data['dns'] + option = data['option'] + # if not provider, get the true option that we want have value + if not provider_data['is_main_provider']: + path_prefix = data['path_prefix'] + try: + supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_data["provider_name"]}', path_prefix) + except KeyError: + #warn(f'cannot find supplier "{provider_name}" for "{dns}"') + continue + # first of all, get the supplier name param = self.objectspace.param(variable.xmlfiles) - param.name = f'leader_{idx}' - if fill.name == 'calc_providers_follower': + param.name = f'dns_{idx}' + param.text = option + param.propertyerror = False + param.type = 'variable' + fill.param.append(param) + if not provider_data['is_main_provider'] and \ + self.objectspace.paths.is_follower(variable): + param = self.objectspace.param(variable.xmlfiles) + param.name = f'leader_{idx}' + if fill.name == 'calc_providers_follower': + param.text = dns + else: + if self.objectspace.paths.is_follower(supplier_option): + param.text = self.objectspace.paths.get_leader(supplier_option) + else: + param.text = self.objectspace.paths.get_supplier(f'supplier:{leader_provider}', path_prefix) + param.propertyerror = False + param.type = 'variable' + fill.param.append(param) + # get the current DNS name for dynamic variable + if self.objectspace.paths.is_dynamic(variable): + param = self.objectspace.param(variable.xmlfiles) + param.name = f'dynamic_{idx}' + param.text = dns + fill.param.append(param) + # get the current value! + param = self.objectspace.param(variable.xmlfiles) + param.name = f'value_{idx}' + if provider_data['is_main_provider']: param.text = dns else: - if self.objectspace.paths.is_follower(supplier_option): - param.text = self.objectspace.paths.get_leader(supplier_option) - else: - param.text = self.objectspace.paths.get_supplier(f'supplier:{leader_provider}', path_prefix) + param.text = supplier_option param.propertyerror = False param.type = 'variable' fill.param.append(param) - # get the current DNS name for dynamic variable - if self.objectspace.paths.is_dynamic(variable): - param = self.objectspace.param(variable.xmlfiles) - param.name = f'dynamic_{idx}' - param.text = dns - fill.param.append(param) - # get the current value! - param = self.objectspace.param(variable.xmlfiles) - param.name = f'value_{idx}' - if is_provider: - param.text = dns - else: - param.text = supplier_option - param.propertyerror = False - param.type = 'variable' - fill.param.append(param) - if not hasattr(self.objectspace.space.variables[nf_dns], 'constraints'): - self.objectspace.space.variables[nf_dns].constraints = self.objectspace.constraints(None) - if not hasattr(self.objectspace.space.variables[nf_dns].constraints, 'fill'): - self.objectspace.space.variables[nf_dns].constraints.fill = [] - self.objectspace.space.variables[nf_dns].constraints.fill.append(fill) + if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']], 'constraints'): + self.objectspace.space.variables[provider_data['path_prefix']].constraints = self.objectspace.constraints(None) + if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']].constraints, 'fill'): + self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill = [] + self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill.append(fill) def convert_suppliers(self): - for supplier, data in self.suppliers.items(): - if supplier == 'Host': + for supplier_name, s_datas in self.suppliers.items(): + if supplier_name == 'Host': continue - for s_dico in data: - if supplier not in self.providers: + for s_data in s_datas: + if supplier_name not in self.providers: continue - for p_dico in self.providers[supplier]: - common_zones = s_dico['zones'] & p_dico['zones'] - if not common_zones: + for p_data in self.providers[supplier_name]: + if s_data['dns'] == p_data['dns']: + # supplier and provider are in same machine continue - for idx, zone in enumerate(p_dico['zone_names']): - if zone in common_zones: - break - dns = p_dico['server_names'][idx] - s_dico['option'].value = dns + if supplier_name not in p_data['suppliers_zone'] or s_data['dns'] not in p_data['suppliers_zone'][supplier_name]: + continue + # get the DNS name in supplier zone + zone = p_data['suppliers_zone'][supplier_name][s_data['dns']] + if zone not in p_data['zones']: + continue + zidx = p_data['zones'].index(zone) + dns = p_data['server_names'][zidx] new_value = self.objectspace.value(None) new_value.name = dns - s_dico['option'].value = [new_value] + s_data['option'].value = [new_value] break diff --git a/src/risotto/utils.py b/src/risotto/utils.py index 25fd4c2..1372e8a 100644 --- a/src/risotto/utils.py +++ b/src/risotto/utils.py @@ -4,6 +4,7 @@ from typing import List from ipaddress import ip_address from toml import load as toml_load from json import load, dump +from json.decoder import JSONDecodeError from pprint import pprint @@ -15,6 +16,10 @@ HERE = environ['PWD'] IP_DIR = join(HERE, 'ip') +# custom filters from dataset +custom_filters = {} + + config_file = environ.get('CONFIG_FILE', 'risotto.conf') if isfile(config_file): with open(config_file, 'r') as fh: @@ -35,10 +40,10 @@ def multi_function(function): return function -async def value_pprint(dico, config): +def value_pprint(dico, config): pprint_dict = {} for path, value in dico.items(): - if await config.option(path).option.type() == 'password' and value: + if config.option(path).type() == 'password' and value: value = 'X' * len(value) pprint_dict[path] = value pprint(pprint_dict) @@ -49,28 +54,45 @@ def load_zones(zones, hosts): makedirs(IP_DIR) json_file = join(IP_DIR, 'zones.json') if isfile(json_file): - with open(json_file, 'r') as fh: - zones_ip = load(fh) + try: + with open(json_file, 'r') as fh: + ori_zones_ip = load(fh) + except JSONDecodeError: + ori_zones_ip = {} else: - zones_ip = {} - for host_name, hosts in hosts.items(): - for server_name, server in hosts['servers'].items(): - server_zones = server['informations']['zones_name'] + ori_zones_ip = {} + new_zones_ip = {} + # cache, machine should not change IP + for host_name, dhosts in hosts.items(): + for server_name, server in dhosts['servers'].items(): + server_zones = server['zones_name'] for idx, zone_name in enumerate(server_zones): zone = zones[zone_name] zone.setdefault('hosts', {}) - # FIXME make a cache, machine should not change IP - if zone_name not in zones_ip: - zones_ip[zone_name] = {} - if server_name in zones_ip[zone_name]: - server_index = zones_ip[zone_name][server_name] - elif not zones_ip[zone_name]: - server_index = 0 + if zone_name not in new_zones_ip: + new_zones_ip[zone_name] = {} + if zone_name in ori_zones_ip and server_name in ori_zones_ip[zone_name]: + server_index = ori_zones_ip[zone_name][server_name] + if server_index >= zone['length']: + server_index = None + elif server_index in new_zones_ip[zone_name].values(): + server_index = None else: - # it's the last ip + 1 - server_index = zones_ip[zone_name][list(zones_ip[zone_name].keys())[-1]] + 1 - ip = str(ip_address(zone['start_ip']) + server_index) - zone['hosts'][server_name] = ip - zones_ip[zone_name][server_name] = server_index + server_index = None + new_zones_ip[zone_name][server_name] = server_index + for zone_name, servers in new_zones_ip.items(): + for server_name, server_idx in servers.items(): + if server_idx is not None: + continue + for new_idx in range(zones[zone_name]['length']): + if new_idx not in new_zones_ip[zone_name].values(): + new_zones_ip[zone_name][server_name] = new_idx + break + else: + raise Exception(f'cannot find free IP in zone "{zone_name}" for "{server_name}"') + for zone_name, servers in new_zones_ip.items(): + start_ip = ip_address(zones[zone_name]['start_ip']) + for server_name, server_index in servers.items(): + zones[zone_name]['hosts'][server_name] = str(start_ip + server_index) with open(json_file, 'w') as fh: - dump(zones_ip, fh) + dump(new_zones_ip, fh)