stove/risotto
3
0
Fork 2
Code Issues 1 Pull requests Projects Releases Wiki Activity

tiramisu 3 to 4

Browse source
This commit is contained in:
egarette@silique.fr 2023-06-22 16:19:44 +02:00
parent 5e735cf453
commit 186c886e84
7 changed files with 608 additions and 452 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
ansible/inventory.py
View file

@ -8,8 +8,8 @@ from argparse import ArgumentParser
from json import load as json_load, dumps, JSONEncoder from json import load as json_load, dumps, JSONEncoder
from os import remove from os import remove
from os.path import isfile from os.path import isfile
from asyncio import run
from traceback import print_exc from traceback import print_exc
from sys import stderr, argv
from risotto.machine import load, TIRAMISU_CACHE, VALUES_CACHE, INFORMATIONS_CACHE, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION from risotto.machine import load, TIRAMISU_CACHE, VALUES_CACHE, INFORMATIONS_CACHE, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
from tiramisu import Config from tiramisu import Config
@ -40,12 +40,13 @@ class RisottoInventory(object):
parser.add_argument('--host', action='store') parser.add_argument('--host', action='store')
parser.add_argument('--nocache', action='store_true') parser.add_argument('--nocache', action='store_true')
parser.add_argument('--debug', action='store_true') parser.add_argument('--debug', action='store_true')
parser.add_argument('--pretty_print', action='store_true')
self.args = parser.parse_args() self.args = parser.parse_args()
if self.args.debug: if self.args.debug:
global DEBUG global DEBUG
DEBUG = True DEBUG = True
async def run(self): def run(self):
if self.args.list and self.args.host: if self.args.list and self.args.host:
raise Exception('cannot have --list and --host together') raise Exception('cannot have --list and --host together')
if self.args.list or self.args.nocache: if self.args.list or self.args.nocache:
@ -55,20 +56,20 @@ class RisottoInventory(object):
remove(VALUES_CACHE) remove(VALUES_CACHE)
if isfile(INFORMATIONS_CACHE): if isfile(INFORMATIONS_CACHE):
remove(INFORMATIONS_CACHE) remove(INFORMATIONS_CACHE)
config = await load(TIRAMISU_CACHE, config = load(TIRAMISU_CACHE,
VALUES_CACHE, VALUES_CACHE,
INFORMATIONS_CACHE, INFORMATIONS_CACHE,
) )
if self.args.list: if self.args.list:
return await self.do_inventory(config) return self.do_inventory(config)
elif self.args.host: elif self.args.host:
return await self.get_vars(config, self.args.host) return self.get_vars(config, self.args.host)
raise Exception('pfff') raise Exception('pfff')
async def do_inventory(self, def do_inventory(self,
config: Config, config: Config,
) -> dict: ) -> dict:
servers = [await subconfig.option.doc() for subconfig in await config.option.list('optiondescription') if await subconfig.information.get('module') == 'host'] servers = [subconfig.doc() for subconfig in config.option.list('optiondescription') if subconfig.information.get('module') == 'host']
return dumps({ return dumps({
'group': { 'group': {
'hosts': servers, 'hosts': servers,
@ -81,7 +82,7 @@ class RisottoInventory(object):
} }
}) })
async def get_vars(self, def get_vars(self,
config: Config, config: Config,
host_name: str, host_name: str,
) -> dict: ) -> dict:
@ -89,36 +90,46 @@ class RisottoInventory(object):
rougailconfig = RougailConfig.copy() rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
for subconfig in await config.option.list('optiondescription'): for subconfig in config.option.list('optiondescription'):
server_name = await subconfig.option.description() server_name = subconfig.description()
module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get() module_name = subconfig.option(subconfig.information.get('provider:global:module_name')).value.get()
if module_name == 'host' and server_name != host_name: if module_name == 'host' and server_name != host_name:
continue continue
engine = RougailSystemdTemplate(subconfig, rougailconfig) engine = RougailSystemdTemplate(subconfig, rougailconfig)
await engine.load_variables() engine.load_variables(with_flatten=False)
if module_name != 'host' and engine.rougail_variables_dict['general']['host'] != host_name: if module_name != 'host' and engine.rougail_variables_dict['general']['host'] != host_name:
continue continue
ret[server_name] = engine.rougail_variables_dict ret[server_name] = engine.rougail_variables_dict
ret['modules'] = await config.information.get('modules') ret['modules'] = config.information.get('modules')
ret['delete_old_image'] = False ret['delete_old_image'] = False
ret['configure_host'] = True ret['configure_host'] = True
ret['only_machine'] = None ret['only_machine'] = None
ret['copy_templates'] = False ret['copy_templates'] = False
ret['copy_tests'] = False ret['copy_tests'] = False
ret['host_install_dir'] = ret[host_name].pop('host_install_dir') ret['host_install_dir'] = ret[host_name]['general']['host_install_dir']
return dumps(ret, cls=RougailEncoder) return dumps(ret, cls=RougailEncoder)
# Get the inventory. # Get the inventory.
async def main(): def main():
try: try:
inv = RisottoInventory() inv = RisottoInventory()
values = await inv.run() values = inv.run()
if inv.args.pretty_print:
from pprint import pprint
from json import loads
pprint(loads(values))
else:
print(values) print(values)
except Exception as err: except Exception as err:
if DEBUG: if DEBUG:
print_exc() print_exc()
print(err) print('---', file=stderr)
extra=''
else:
extra=f'\nmore informations with commandline "{" ".join(argv)} --debug"'
print(f'{err}{extra}', file=stderr)
exit(1)
run(main()) main()

103
sbin/risotto_display
View file

@ -1,6 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from asyncio import run
from tabulate import tabulate from tabulate import tabulate
from argparse import ArgumentParser from argparse import ArgumentParser
@ -18,19 +17,19 @@ def list_to_string(lst):
return lst return lst
async def get_files_subelements(type_name, element, files_subelement, files_cols): def get_files_subelements(type_name, element, files_subelement, files_cols):
data = {} data = {}
if not await element.option('activate').value.get(): if not element.option('activate').value.get():
return data return data
for subelement in files_subelement.values(): for subelement in files_subelement.values():
if subelement['type'] == 'subelement': if subelement['type'] == 'subelement':
try: try:
value = list_to_string(await element.option(subelement['key']).value.get()) value = list_to_string(element.option(subelement['key']).value.get())
# FIXME except AttributeError: # FIXME except AttributeError:
except Exception: except Exception:
value = '' value = ''
elif subelement['type'] == 'information': elif subelement['type'] == 'information':
value = await element.information.get(subelement['key'], '') value = element.information.get(subelement['key'], '')
elif subelement['type'] == 'none': elif subelement['type'] == 'none':
value = subelement['value'] value = subelement['value']
else: else:
@ -47,7 +46,7 @@ async def get_files_subelements(type_name, element, files_subelement, files_cols
return data return data
async def services(config, values): def services(config, values):
files_subelement = {'Source': {'key': 'source', 'type': 'information'}, files_subelement = {'Source': {'key': 'source', 'type': 'information'},
'Nom': {'key': 'name', 'type': 'subelement'}, 'Nom': {'key': 'name', 'type': 'subelement'},
'Variable': {'key': 'variable', 'type': 'subelement'}, 'Variable': {'key': 'variable', 'type': 'subelement'},
@ -57,26 +56,26 @@ async def services(config, values):
'Moteur': {'key': 'engine', 'type': 'information'}, 'Moteur': {'key': 'engine', 'type': 'information'},
} }
disabled_services = [] disabled_services = []
for service in await config.option.list(type="all"): for service in config.option.list(type="all"):
doc = await service.option.doc() doc = service.option.doc()
files_lst = [] files_lst = []
files_cols = set() files_cols = set()
if not await service.option('manage').value.get(): if not service.option('manage').value.get():
doc += " - unmanaged" doc += " - unmanaged"
if not await service.option('activate').value.get(): if not service.option('activate').value.get():
disabled_services.append([doc]) disabled_services.append([doc])
else: else:
for type in await service.list(type="all"): for type in service.list(type="all"):
type_name = await type.option.doc() type_name = type.option.doc()
if type_name in ['files', 'overrides']: if type_name in ['files', 'overrides']:
for element in await type.list(type="all"): for element in type.list(type="all"):
data = await get_files_subelements(type_name, element, files_subelement, files_cols) data = get_files_subelements(type_name, element, files_subelement, files_cols)
if data: if data:
files_lst.append(data) files_lst.append(data)
elif type_name == 'manage': elif type_name == 'manage':
pass pass
elif type_name == 'activate': elif type_name == 'activate':
if not await type.value.get(): if not type.value.get():
doc += " - unactivated" doc += " - unactivated"
else: else:
print("FIXME " + type_name) print("FIXME " + type_name)
@ -89,19 +88,19 @@ async def services(config, values):
values["Services désactivés"] = {'keys': ['Nom'], 'lst': disabled_services} values["Services désactivés"] = {'keys': ['Nom'], 'lst': disabled_services}
async def table_leader(config, read_only): def table_leader(config, read_only):
keys = ['Description'] keys = ['Description']
if read_only: if read_only:
keys.append('Cachée') keys.append('Cachée')
leadership_lst = await config.list(type="all") leadership_lst = config.list(type="all")
leader = leadership_lst.pop(0) leader = leadership_lst.pop(0)
leader_owner = await leader.owner.get() leader_owner = leader.owner.get()
follower_names = [await follower.option.name() for follower in leadership_lst] follower_names = [follower.option.name() for follower in leadership_lst]
doc = await leader.option.doc() doc = leader.option.doc()
properties = await leader.property.get() properties = leader.property.get()
if 'mandatory' in properties: if 'mandatory' in properties:
doc += '*' doc += '*'
name = await leader.option.name() name = leader.option.name()
lst = [[f'{doc} ({name})']] lst = [[f'{doc} ({name})']]
if read_only: if read_only:
if 'hidden' in properties: if 'hidden' in properties:
@ -109,7 +108,7 @@ async def table_leader(config, read_only):
else: else:
hidden = '' hidden = ''
lst[0].append(hidden) lst[0].append(hidden)
for idx, leader_value in enumerate(await leader.value.get()): for idx, leader_value in enumerate(leader.value.get()):
keys.append(f'Valeur {idx}') keys.append(f'Valeur {idx}')
keys.append(f'Utilisateur {idx}') keys.append(f'Utilisateur {idx}')
lst[0].append(leader_value) lst[0].append(leader_value)
@ -117,11 +116,11 @@ async def table_leader(config, read_only):
for follower_idx, follower_name in enumerate(follower_names): for follower_idx, follower_name in enumerate(follower_names):
follower_option = config.option(follower_name, idx) follower_option = config.option(follower_name, idx)
if idx == 0: if idx == 0:
doc = await follower_option.option.doc() doc = follower_option.option.doc()
properties = await follower_option.property.get() properties = follower_option.property.get()
if 'mandatory' in properties: if 'mandatory' in properties:
doc += '*' doc += '*'
name = await follower_option.option.name() name = follower_option.option.name()
lst.append([f'{doc} ({name})']) lst.append([f'{doc} ({name})'])
if read_only: if read_only:
if 'hidden' in properties: if 'hidden' in properties:
@ -130,48 +129,48 @@ async def table_leader(config, read_only):
hidden = '' hidden = ''
lst[-1].append(hidden) lst[-1].append(hidden)
try: try:
lst[follower_idx + 1].append(list_to_string(await follower_option.value.get())) lst[follower_idx + 1].append(list_to_string(follower_option.value.get()))
lst[follower_idx + 1].append(await follower_option.owner.get()) lst[follower_idx + 1].append(follower_option.owner.get())
except PropertiesOptionError: except PropertiesOptionError:
pass pass
# leader = next leader_iter # leader = next leader_iter
# if master_values is None: # if master_values is None:
# master_values = await subconfig.value.get() # master_values = subconfig.value.get()
return {'keys': keys, 'lst': lst} return {'keys': keys, 'lst': lst}
async def table(config, prefix_len, values, read_only): def table(config, prefix_len, values, read_only):
lst = [] lst = []
for subconfig in await config.option.list(type="all"): for subconfig in config.option.list(type="all"):
# prefix = prefix_len * 2 * ' ' # prefix = prefix_len * 2 * ' '
# if await subconfig.option.isoptiondescription(): # if subconfig.option.isoptiondescription():
# prefix += '=>' # prefix += '=>'
# else: # else:
# prefix += '-' # prefix += '-'
# display_str = f'{prefix} {description}' # display_str = f'{prefix} {description}'
# if name != description: # if name != description:
# display_str = f'{display_str} ({name})' # display_str = f'{display_str} ({name})'
name = await subconfig.option.name() name = subconfig.option.name()
doc = await subconfig.option.doc() doc = subconfig.option.doc()
if prefix_len == 0 and ROUGAIL_NAMESPACE != name: if prefix_len == 0 and ROUGAIL_NAMESPACE != name:
doc = doc.capitalize() doc = doc.capitalize()
if prefix_len == 0 and name == 'services': if prefix_len == 0 and name == 'services':
values['Services'] = {} values['Services'] = {}
await services(subconfig, values['Services']) services(subconfig, values['Services'])
elif await subconfig.option.isoptiondescription(): elif subconfig.option.isoptiondescription():
od_name = f'{doc} ({(await subconfig.option.path()).split(".", 1)[1]})' od_name = f'{doc} ({(subconfig.option.path()).split(".", 1)[1]})'
values[od_name] = None values[od_name] = None
if await subconfig.option.isleadership(): if subconfig.option.isleadership():
values[od_name] = await table_leader(subconfig, read_only) values[od_name] = table_leader(subconfig, read_only)
else: else:
values[od_name] = await table(subconfig, prefix_len + 1, values, read_only) values[od_name] = table(subconfig, prefix_len + 1, values, read_only)
else: else:
value = list_to_string(await subconfig.value.get()) value = list_to_string(subconfig.value.get())
doc = await subconfig.option.doc() doc = subconfig.option.doc()
properties = await subconfig.property.get() properties = subconfig.property.get()
if 'mandatory' in properties: if 'mandatory' in properties:
doc += '*' doc += '*'
name = await subconfig.option.name() name = subconfig.option.name()
lst.append([f'{doc} ({name})', value]) lst.append([f'{doc} ({name})', value])
if read_only: if read_only:
if 'hidden' in properties: if 'hidden' in properties:
@ -179,7 +178,7 @@ async def table(config, prefix_len, values, read_only):
else: else:
hidden = '' hidden = ''
lst[-1].append(hidden) lst[-1].append(hidden)
lst[-1].append(await subconfig.owner.get()) lst[-1].append(subconfig.owner.get())
keys = ['Description', 'Valeur'] keys = ['Description', 'Valeur']
if read_only: if read_only:
keys.append('Cachée') keys.append('Cachée')
@ -187,7 +186,7 @@ async def table(config, prefix_len, values, read_only):
return {'keys': keys, 'lst': lst} return {'keys': keys, 'lst': lst}
async def main(): def main():
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument('server_name') parser.add_argument('server_name')
parser.add_argument('--read_only', action='store_true') parser.add_argument('--read_only', action='store_true')
@ -199,18 +198,18 @@ async def main():
values = {} values = {}
server_name = args.server_name server_name = args.server_name
config = await load(hide_secret=HIDE_SECRET, config = load(hide_secret=HIDE_SECRET,
original_display_name=True, original_display_name=True,
valid_mandatories=args.read_only, valid_mandatories=args.read_only,
) )
if not args.read_only: if not args.read_only:
await config.property.read_write() config.property.read_write()
root_option = config.option(normalize_family(server_name)) root_option = config.option(normalize_family(server_name))
try: try:
await root_option.option.get() root_option.option.get()
except AttributeError: except AttributeError:
exit(f'Unable to find {server_name} configuration: {[await o.option.description() for o in await config.option.list(type="optiondescription")]}') exit(f'Unable to find {server_name} configuration: {[o.option.description() for o in config.option.list(type="optiondescription")]}')
await table(root_option, 0, values, args.read_only) table(root_option, 0, values, args.read_only)
for title, dico in values.items(): for title, dico in values.items():
if title == 'Services': if title == 'Services':
if not dico: if not dico:
@ -233,4 +232,4 @@ async def main():
print(tabulate(dico['lst'], headers=dico['keys'], tablefmt="fancy_grid")) print(tabulate(dico['lst'], headers=dico['keys'], tablefmt="fancy_grid"))
run(main()) main()

7
sbin/risotto_templates
View file

@ -1,13 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from asyncio import run
from argparse import ArgumentParser from argparse import ArgumentParser
from traceback import print_exc from traceback import print_exc
from risotto.machine import remove_cache, build_files, INSTALL_DIR from risotto.machine import remove_cache, build_files, INSTALL_DIR
async def main(): def main():
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument('server_name', nargs='?') parser.add_argument('server_name', nargs='?')
parser.add_argument('--nocache', action='store_true') parser.add_argument('--nocache', action='store_true')
@ -19,7 +18,7 @@ async def main():
remove_cache() remove_cache()
try: try:
await build_files(None, build_files(None,
args.server_name, args.server_name,
False, False,
args.copy_tests, args.copy_tests,
@ -32,4 +31,4 @@ async def main():
print(f'templates generated in "{INSTALL_DIR}" directory') print(f'templates generated in "{INSTALL_DIR}" directory')
run(main()) main()

58
src/risotto/image.py
View file

@ -19,8 +19,8 @@ class ModuleCfg():
self.depends = [] self.depends = []
self.manuals = [] self.manuals = []
self.tests = [] self.tests = []
self.providers = [] #self.providers = []
self.suppliers = [] #self.suppliers = []
def __repr__(self): def __repr__(self):
return str(vars(self)) return str(vars(self))
@ -131,10 +131,10 @@ class Modules:
if appname not in cfg.providers[provider]: if appname not in cfg.providers[provider]:
cfg.providers[provider].append(appname) cfg.providers[provider].append(appname)
supplier = app.get('supplier') supplier = app.get('supplier')
if supplier: #if supplier:
self.suppliers.setdefault(supplier, []) # self.suppliers.setdefault(supplier, [])
if appname not in self.suppliers[supplier]: # if appname not in self.suppliers[supplier]:
self.suppliers[supplier].append(appname) # self.suppliers[supplier].append(appname)
if 'distribution' in app and app['distribution']: if 'distribution' in app and app['distribution']:
distribution = appname distribution = appname
else: else:
@ -216,32 +216,56 @@ def applicationservice_copy(src_file: str,
copytree(src_file, dst_file) copytree(src_file, dst_file)
async def valid_mandatories(config): def valid_mandatories(config):
mandatories = await config.value.mandatory() mandatories = config.value.mandatory()
await config.property.pop('mandatory') config.property.remove('mandatory')
hidden = {} hidden = {}
variables = {} variables = {}
title = None title = None
if mandatories: if mandatories:
server_name = None server_name = None
for mandatory in mandatories: for mandatory_option in mandatories:
path_server_name, path = mandatory.split('.', 1) path_server_name, path = mandatory_option.path().split('.', 1)
var_server_name = await config.option(path_server_name).option.description() var_server_name = config.option(path_server_name).description()
if server_name != var_server_name: if server_name != var_server_name:
server_name = var_server_name server_name = var_server_name
title = f'=== Missing variables for {server_name} ===' title = f'=== Missing variables for {server_name} ==='
suboption = config.option(mandatory) text = mandatory_option.doc()
text = await suboption.option.doc()
msg = f' - {text} ({path})' msg = f' - {text} ({path})'
supplier = await suboption.information.get('supplier', None) supplier = mandatory_option.information.get('supplier', None)
if supplier: if supplier:
msg += f' you could add a service that provides "{supplier}"' msg += f' you could add a service that provides "{supplier}"'
if mandatory_option.isfollower():
leader = mandatory_option.leader()
try: try:
await config.option(mandatory).value.get() leader_value = leader.value.get()
except PropertiesOptionError as err:
if 'hidden' not in err.proptype:
raise err from err
hidden.setdefault(title, []).append(msg)
else:
config.property.add('mandatory')
for idx in range(mandatory_option.value.len()):
try:
config.option(mandatory_option.path(), idx).value.get()
except PropertiesOptionError as err:
path = leader.path()
spath = path.split('.', 1)[1]
submsg = f'{msg} at index {idx} (value of leader "{leader.doc()}" ({spath}) is "{leader_value[idx]}")'
if 'hidden' in err.proptype:
hidden.setdefault(title, []).append(submsg)
elif 'mandatory' in err.proptype:
variables.setdefault(title, []).append(submsg)
else:
raise err from err
config.property.remove('mandatory')
else:
try:
mandatory_option.value.get()
variables.setdefault(title, []).append(msg) variables.setdefault(title, []).append(msg)
except PropertiesOptionError as err: except PropertiesOptionError as err:
if 'hidden' not in err.proptype: if 'hidden' not in err.proptype:
raise PropertiesOptionError(err) raise err from err
hidden.setdefault(title, []).append(msg) hidden.setdefault(title, []).append(msg)
if not variables: if not variables:
variables = hidden variables = hidden

256
src/risotto/machine.py
View file

@ -5,11 +5,11 @@ from .rougail.annotator import calc_providers, calc_providers_global, calc_provi
from rougail import RougailConfig, RougailConvert from rougail import RougailConfig, RougailConvert
from os import remove, makedirs, listdir, chmod from os import remove, makedirs, listdir, chmod
from os.path import isfile, isdir, abspath, join, dirname from os.path import isfile, isdir, abspath, join, dirname
from json import dump as json_dump, load as json_load from pickle import dump as pickle_dump, load as pickle_load
from yaml import load as yaml_load, SafeLoader from yaml import load as yaml_load, SafeLoader
from ipaddress import ip_network from ipaddress import IPv4Interface, ip_network
# #
from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value, calc_value_property_help
from rougail.utils import normalize_family from rougail.utils import normalize_family
from rougail import RougailSystemdTemplate from rougail import RougailSystemdTemplate
from shutil import copy2, copytree, rmtree from shutil import copy2, copytree, rmtree
@ -29,8 +29,8 @@ def tiramisu_display_name(kls,
CONFIG_FILE = 'servers.yml' CONFIG_FILE = 'servers.yml'
TIRAMISU_CACHE = 'tiramisu_cache.py' TIRAMISU_CACHE = 'tiramisu_cache.py'
VALUES_CACHE = 'values_cache.json' VALUES_CACHE = 'values_cache.pickle'
INFORMATIONS_CACHE = 'informations_cache.json' INFORMATIONS_CACHE = 'informations_cache.pickle'
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest'] INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
INSTALL_CONFIG_DIR = 'configurations' INSTALL_CONFIG_DIR = 'configurations'
INSTALL_TMPL_DIR= 'templates' INSTALL_TMPL_DIR= 'templates'
@ -47,6 +47,7 @@ FUNCTIONS = {'calc_providers': calc_providers,
'valid_in_network': valid_in_network, 'valid_in_network': valid_in_network,
'valid_not_equal': valid_not_equal, 'valid_not_equal': valid_not_equal,
'calc_value': calc_value, 'calc_value': calc_value,
'calc_value_property_help': calc_value_property_help,
'normalize_family': normalize_family, 'normalize_family': normalize_family,
} }
@ -88,7 +89,7 @@ def remove_cache():
remove(INFORMATIONS_CACHE) remove(INFORMATIONS_CACHE)
async def templates(server_name, def templates(server_name,
config, config,
just_copy=False, just_copy=False,
copy_manuals=False, copy_manuals=False,
@ -97,19 +98,19 @@ async def templates(server_name,
): ):
subconfig = config.option(normalize_family(server_name)) subconfig = config.option(normalize_family(server_name))
try: try:
await subconfig.option.get() subconfig.get()
except: except:
servers = [await server.option.description() for server in await config.option.list('optiondescription')] servers = [server.description() for server in config.list('optiondescription')]
raise Exception(f'cannot find server name "{server_name}": {servers}') raise Exception(f'cannot find server name "{server_name}": {servers}')
rougailconfig = RougailConfig.copy() rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
rougailconfig['tmp_dir'] = 'tmp' rougailconfig['tmp_dir'] = 'tmp'
rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir') rougailconfig['templates_dir'] = subconfig.information.get('templates_dir')
rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir') rougailconfig['patches_dir'] = subconfig.information.get('patches_dir')
rougailconfig['functions_file'] = await subconfig.information.get('functions_files') rougailconfig['functions_file'] = subconfig.information.get('functions_files')
module = await subconfig.information.get('module') module = subconfig.information.get('module')
is_host = module == 'host' is_host = module == 'host'
if is_host: if is_host:
rougailconfig['systemd_tmpfile_delete_before_create'] = True rougailconfig['systemd_tmpfile_delete_before_create'] = True
@ -138,15 +139,15 @@ async def templates(server_name,
engine.engines[eng] = engine.engines['none'] engine.engines[eng] = engine.engines['none']
try: try:
if not template: if not template:
await engine.instance_files(extra_variables=extra_variables) engine.instance_files(extra_variables=extra_variables)
else: else:
await engine.instance_file(template, extra_variables=extra_variables) engine.instance_file(template, extra_variables=extra_variables)
except Exception as err: except Exception as err:
print() print()
print(f'=== Configuration: {server_name} ===') print(f'=== Configuration: {server_name} ===')
try: try:
values = await subconfig.value.dict() values = subconfig.value.dict()
await value_pprint(values, subconfig) value_pprint(values, subconfig)
except: except:
pass pass
raise err from err raise err from err
@ -159,17 +160,17 @@ async def templates(server_name,
if copy_manuals and not is_host: if copy_manuals and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module) dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module)
if not isdir(dest_dir): if not isdir(dest_dir):
for manual in await subconfig.information.get('manuals_dirs'): for manual in subconfig.information.get('manuals_dirs'):
for filename in listdir(manual): for filename in listdir(manual):
src_file = join(manual, filename) src_file = join(manual, filename)
dst_file = join(dest_dir, filename) dst_file = join(dest_dir, filename)
copy(src_file, dst_file) copy(src_file, dst_file)
copy_tests = await config.information.get('copy_tests') copy_tests = config.information.get('copy_tests')
if copy_tests and not is_host: if copy_tests and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module) dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module)
if not isdir(dest_dir): if not isdir(dest_dir):
for tests in await subconfig.information.get('tests_dirs'): for tests in subconfig.information.get('tests_dirs'):
for filename in listdir(tests): for filename in listdir(tests):
src_file = join(tests, filename) src_file = join(tests, filename)
dst_file = join(dest_dir, filename) dst_file = join(dest_dir, filename)
@ -178,7 +179,6 @@ async def templates(server_name,
class Loader: class Loader:
def __init__(self, def __init__(self,
clean_directories,
hide_secret, hide_secret,
original_display_name, original_display_name,
valid_mandatories, valid_mandatories,
@ -188,10 +188,6 @@ class Loader:
self.original_display_name = original_display_name self.original_display_name = original_display_name
self.valid_mandatories = valid_mandatories self.valid_mandatories = valid_mandatories
self.config_file = config_file self.config_file = config_file
if clean_directories:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
def load_tiramisu_file(self): def load_tiramisu_file(self):
"""Load config file (servers.yml) and build tiramisu file with dataset informations """Load config file (servers.yml) and build tiramisu file with dataset informations
@ -224,28 +220,54 @@ class Loader:
rougail = RougailConvert(cfg) rougail = RougailConvert(cfg)
for host_name, datas in self.servers_json['hosts'].items(): for host_name, datas in self.servers_json['hosts'].items():
for server_name, server_datas in datas['servers'].items(): for server_name, server_datas in datas['servers'].items():
for zone in server_datas['informations']['zones_name']: if 'provider_zone' not in server_datas and 'zones_name' not in server_datas:
raise Exception(f'cannot find "zones_name" attribute for server "{server_name}"')
if 'provider_zone' in server_datas:
zones_name.setdefault(server_datas['provider_zone'], []).append(server_name)
if 'zones_name' not in server_datas:
server_datas['zones_name'] = []
if server_datas['provider_zone'] in server_datas['zones_name']:
raise Exception(_('provider_zone "{server_datas["provider_zone"]}" must not be in "zones" "{server_datas["zones_name"]}"'))
# external zone is better in first place
if server_datas['zones_name'] and self.servers_json['zones']['external_zone'] == server_datas['zones_name'][0]:
server_datas['zones_name'].append(server_datas['provider_zone'])
else:
server_datas['zones_name'].insert(0, server_datas['provider_zone'])
# if server_datas['zones_name'] and server_datas['provider_zone'] == self.servers_json['zones']['external_zone']:
# server_datas['zones_name'].insert(0, server_datas['provider_zone'])
# else:
# server_datas['zones_name'].append(server_datas['provider_zone'])
for zone in server_datas['zones_name']:
zones_name.setdefault(zone, []).append(server_name) zones_name.setdefault(zone, []).append(server_name)
self.zones = {} self.zones = {}
zones_network = ip_network(self.servers_json['zones']['network']) zones_network = ip_network(self.servers_json['zones']['network'])
zone_start_ip = zones_network.network_address zone_start_ip = zones_network.network_address
domain_name = self.servers_json['zones']['prefix_domain_name'] domain_name = self.servers_json['zones']['prefix_domain_name']
for idx, zone_name in enumerate(zones_name): for zone_name in zones_name:
sub_network = ip_network(f'{zone_start_ip}/28') len_zone = len(zones_name[zone_name])
for zone_cidr in [29, 28, 27, 26]:
try:
sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}')
except ValueError:
# calc network address for this mask
zone_start_ip = IPv4Interface(f'{zone_start_ip}/{zone_cidr}').network.broadcast_address + 1
sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}')
if not sub_network.subnet_of(zones_network): if not sub_network.subnet_of(zones_network):
raise Exception('not enough IP available') raise Exception('not enough IP available')
if sub_network.num_addresses < len(zones_name[zone_name]): length = sub_network.num_addresses - 3 # network + broadcast + host
#FIXME should try to increase network! if length >= len_zone:
raise Exception(f'network too small for zone {zone_name}') break
if idx == 0: else:
raise Exception(f'network too small for zone "{zone_name}" ({sub_network.num_addresses - 2} < {len_zone})')
if self.servers_json['zones']['external_zone'] == zone_name:
zone_domaine_name = domain_name zone_domaine_name = domain_name
else: else:
zone_domaine_name = zone_name + '.' + domain_name zone_domaine_name = zone_name + '.' + domain_name
network = sub_network.network_address network = sub_network.network_address
self.zones[zone_name] = {'domain_name': zone_domaine_name, self.zones[zone_name] = {'domain_name': zone_domaine_name,
'network': str(sub_network), 'network': str(sub_network),
'host_ip': str(network + 1), 'host_ip': str(network + 1),
'length': length,
'start_ip': str(network + 2) 'start_ip': str(network + 2)
} }
zone_start_ip = str(sub_network.broadcast_address + 1) zone_start_ip = str(sub_network.broadcast_address + 1)
@ -266,9 +288,13 @@ class Loader:
# load host # load host
module_info = modules.get('host') module_info = modules.get('host')
tls_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
cfg['risotto_globals'][host_name] = {'global:server_name': host_name, cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
'global:server_names': [host_name for zone in self.zones],
'global:zones_name': list(self.zones),
'global:module_name': 'host', 'global:module_name': 'host',
'global:host_install_dir': abspath(INSTALL_DIR), 'global:host_install_dir': abspath(INSTALL_DIR),
'global:tls_server': tls_host_name,
} }
functions_files |= set(module_info.functions_file) functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg, self.load_dictionaries(cfg,
@ -280,7 +306,7 @@ class Loader:
modules_info = {} modules_info = {}
for server_name, server_datas in datas['servers'].items(): for server_name, server_datas in datas['servers'].items():
module_info = modules.get(server_datas['applicationservice']) module_info = modules.get(server_datas['applicationservice'])
zones_name = server_datas['informations']['zones_name'] zones_name = server_datas['zones_name']
values = [f'{server_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name] values = [f'{server_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name]
if server_datas['applicationservice'] == 'tls': if server_datas['applicationservice'] == 'tls':
true_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}' true_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
@ -292,7 +318,10 @@ class Loader:
'global:zones_name': zones_name, 'global:zones_name': zones_name,
'global:zones_list': list(range(len(zones_name))), 'global:zones_list': list(range(len(zones_name))),
'global:module_name': server_datas['applicationservice'], 'global:module_name': server_datas['applicationservice'],
'global:prefix_domain_name': self.servers_json['zones']['prefix_domain_name']
} }
if 'provider_zone' in server_datas:
cfg['risotto_globals'][true_host_name]['global:provider_zone'] = server_datas['provider_zone']
server_datas['server_name'] = true_host_name server_datas['server_name'] = true_host_name
functions_files |= set(module_info.functions_file) functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg, self.load_dictionaries(cfg,
@ -309,26 +338,24 @@ class Loader:
zones = set() zones = set()
dns_module_name = None dns_module_name = None
for host in self.servers_json['hosts'].values(): for host in self.servers_json['hosts'].values():
zones = [None, None] zones = [self.servers_json['zones']['external_zone'], None]
for server_name, datas in host['servers'].items(): for server_name, datas in host['servers'].items():
if not 'applicationservice' in datas:
raise Exception(f'cannot find applicationservice for "{server_name}"')
if datas['applicationservice'] == 'tls': if datas['applicationservice'] == 'tls':
raise Exception(f'forbidden module name "tls" for server {server_name}') raise Exception(f'forbidden module name "tls" for server "{server_name}"')
#FIXME use provider! #FIXME use provider!
if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['informations']['zones_name']) > 0: if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['zones_name']) > 0:
if dns_module_name: if dns_module_name:
break break
# always add tls machine in second zone of reverse proxy zones[1] = datas['provider_zone']
zones[1] = datas['informations']['zones_name'][0]
if datas['applicationservice'] == 'unbound':
# always add tls machine in second zone of reverse proxy
zones[0] = datas['informations']['zones_name'][0]
if None in zones: if None in zones:
zones = [] zones = []
else: else:
if zones[0] == zones[1]: if zones[0] == zones[1]:
zones = [zones[0]] zones = [zones[0]]
host['servers']['tls'] = {'applicationservice': 'tls', host['servers']['tls'] = {'applicationservice': 'tls',
'informations': {'zones_name': list(zones)}, 'zones_name': list(zones),
} }
def load_dictionaries(self, cfg, module_info, server_name, rougail): def load_dictionaries(self, cfg, module_info, server_name, rougail):
@ -344,7 +371,7 @@ class Loader:
self.manuals_dirs[server_name] = module_info.manuals self.manuals_dirs[server_name] = module_info.manuals
self.tests_dirs[server_name] = module_info.tests self.tests_dirs[server_name] = module_info.tests
async def tiramisu_file_to_tiramisu(self): def tiramisu_file_to_tiramisu(self):
# l # l
tiramisu_space = FUNCTIONS.copy() tiramisu_space = FUNCTIONS.copy()
try: try:
@ -356,55 +383,55 @@ class Loader:
display_name = None display_name = None
else: else:
display_name = tiramisu_display_name display_name = tiramisu_display_name
self.config = await Config(tiramisu_space['option_0'], self.config = Config(tiramisu_space['option_0'],
display_name=display_name, display_name=display_name,
) )
async def load_values_and_informations(self): def load_values_and_informations(self):
config = self.config config = self.config
await config.property.read_write() config.property.read_write()
await config.property.pop('validator') config.property.remove('validator')
await config.property.pop('cache') config.property.remove('cache')
load_zones(self.zones, self.servers_json['hosts']) load_zones(self.zones, self.servers_json['hosts'])
await config.information.set('zones', self.zones) config.information.set('zones', self.zones)
for host_name, hosts_datas in self.servers_json['hosts'].items(): for host_name, hosts_datas in self.servers_json['hosts'].items():
information = config.option(normalize_family(host_name)).information information = config.option(normalize_family(host_name)).information
await information.set('module', 'host') information.set('module', 'host')
await information.set('templates_dir', self.templates_dir[host_name]) information.set('templates_dir', self.templates_dir[host_name])
await information.set('patches_dir', self.patches_dir[host_name]) information.set('patches_dir', self.patches_dir[host_name])
await information.set('functions_files', self.functions_files[host_name]) information.set('functions_files', self.functions_files[host_name])
await self.set_values(host_name, config, hosts_datas) self.set_values(host_name, config, hosts_datas)
for datas in hosts_datas['servers'].values(): for datas in hosts_datas['servers'].values():
server_name = datas['server_name'] server_name = datas['server_name']
information = config.option(normalize_family(server_name)).information information = config.option(normalize_family(server_name)).information
await information.set('module', datas['applicationservice']) information.set('module', datas['applicationservice'])
await information.set('templates_dir', self.templates_dir[server_name]) information.set('templates_dir', self.templates_dir[server_name])
await information.set('patches_dir', self.patches_dir[server_name]) information.set('patches_dir', self.patches_dir[server_name])
await information.set('functions_files', self.functions_files[server_name]) information.set('functions_files', self.functions_files[server_name])
await information.set('manuals_dirs', self.manuals_dirs[server_name]) information.set('manuals_dirs', self.manuals_dirs[server_name])
await information.set('tests_dirs', self.tests_dirs[server_name]) information.set('tests_dirs', self.tests_dirs[server_name])
await self.set_values(server_name, config, datas) self.set_values(server_name, config, datas)
await config.information.set('copy_tests', False) config.information.set('copy_tests', False)
# FIXME only one host_name is supported # FIXME only one host_name is supported
await config.information.set('modules', self.modules[host_name]) config.information.set('modules', self.modules[host_name])
# await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules}) # config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
await config.property.add('cache') config.property.add('cache')
if self.valid_mandatories: if self.valid_mandatories:
messages = await valid_mandatories(config) messages = valid_mandatories(config)
if messages: if messages:
msg = '' msg = ''
for title, variables in messages.items(): for title, variables in messages.items():
msg += '\n' + title + '\n' msg += '\n' + title + '\n'
msg += '\n'.join(variables) msg += '\n'.join(variables)
raise Exception(msg) raise Exception(msg)
await config.property.read_only() config.property.read_only()
with open(VALUES_CACHE, 'w') as fh: with open(VALUES_CACHE, 'wb') as fh:
json_dump(await config.value.exportation(), fh) pickle_dump(config.value.exportation(), fh)
with open(INFORMATIONS_CACHE, 'w') as fh: with open(INFORMATIONS_CACHE, 'wb') as fh:
json_dump(await config.information.exportation(), fh) pickle_dump(config.information.exportation(), fh)
async def set_values(self, def set_values(self,
server_name, server_name,
config, config,
datas, datas,
@ -414,20 +441,20 @@ class Loader:
if not isinstance(datas['values'], dict): if not isinstance(datas['values'], dict):
raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}') raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}')
server_path = normalize_family(server_name) server_path = normalize_family(server_name)
await config.owner.set(self.config_file) config.owner.set(self.config_file)
for vpath, value in datas['values'].items(): for vpath, value in datas['values'].items():
path = f'{server_path}.{vpath}' path = f'{server_path}.{vpath}'
try: try:
if isinstance(value, dict): if isinstance(value, dict):
for idx, val in value.items(): for idx, val in value.items():
await config.option(path, int(idx)).value.set(val) config.option(path, int(idx)).value.set(val)
else: else:
await config.option(path).value.set(value) config.option(path).value.set(value)
except Exception as err: except Exception as err:
await value_pprint(await config.value.dict(), config) value_pprint(config.value.dict(), config)
error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}' error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
raise Exception(error_msg) from err raise Exception(error_msg) from err
await config.owner.set('user') config.owner.set('user')
class LoaderCache(Loader): class LoaderCache(Loader):
@ -435,18 +462,14 @@ class LoaderCache(Loader):
with open(TIRAMISU_CACHE) as fh: with open(TIRAMISU_CACHE) as fh:
self.tiram_obj = fh.read() self.tiram_obj = fh.read()
async def load_values_and_informations(self): def load_values_and_informations(self):
with open(VALUES_CACHE, 'r') as fh: with open(VALUES_CACHE, 'rb') as fh:
await self.config.value.importation(json_load(fh)) self.config.value.importation(pickle_load(fh))
with open(INFORMATIONS_CACHE, 'r') as fh: with open(INFORMATIONS_CACHE, 'rb') as fh:
informations = json_load(fh) self.config.information.importation(pickle_load(fh))
# null is not a valid key in json => 'null'
informations[None] = informations.pop('null')
await self.config.information.importation(informations)
async def load(clean_directories=False, def load(hide_secret=False,
hide_secret=False,
original_display_name: bool=False, original_display_name: bool=False,
valid_mandatories: bool=True, valid_mandatories: bool=True,
copy_tests: bool=False, copy_tests: bool=False,
@ -455,43 +478,44 @@ async def load(clean_directories=False,
loader_obj = LoaderCache loader_obj = LoaderCache
else: else:
loader_obj = Loader loader_obj = Loader
loader = loader_obj(clean_directories, loader = loader_obj(hide_secret,
hide_secret,
original_display_name, original_display_name,
valid_mandatories, valid_mandatories,
) )
loader.load_tiramisu_file() loader.load_tiramisu_file()
await loader.tiramisu_file_to_tiramisu() loader.tiramisu_file_to_tiramisu()
await loader.load_values_and_informations() loader.load_values_and_informations()
config = loader.config config = loader.config
await config.property.read_only() config.property.read_only()
await config.information.set('copy_tests', copy_tests) config.information.set('copy_tests', copy_tests)
await config.cache.reset() config.cache.reset()
return config return config
async def build_files(hostname: str, def build_files(hostname: str,
only_machine: str, only_machine: str,
just_copy: bool, just_copy: bool,
copy_tests: bool, copy_tests: bool,
template: str=None, template: str=None,
) -> None: ) -> None:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
with open(CONFIG_FILE, 'r') as server_fh: with open(CONFIG_FILE, 'r') as server_fh:
servers_json = yaml_load(server_fh, Loader=SafeLoader) servers_json = yaml_load(server_fh, Loader=SafeLoader)
config = await load(copy_tests=copy_tests) config = load(copy_tests=copy_tests)
machines = [await subconfig.option.description() for subconfig in await config.option.list(type='optiondescription')] machines = [subconfig.description() for subconfig in config.option.list(type='optiondescription')]
certificates = {'certificates': {}, certificates = {'certificates': {},
'configuration': servers_json['certificates'], 'configuration': servers_json['certificates'],
} }
# get certificates informations # get certificates informations
tls_machine = None tls_machine = config.option(f'{normalize_family(hostname)}.general.tls_server').value.get()
for machine in machines: for machine in machines:
if machine.startswith('tls.'): if machine == tls_machine:
tls_machine = machine
continue continue
if hostname is None: if hostname is None:
# FIXME multi host! # FIXME multi host!
hostname = await config.option(normalize_family(machine)).option('general.host_name').value.get() hostname = config.option(normalize_family(machine)).option('general.host_name').value.get()
if just_copy: if just_copy:
continue continue
is_host = machine == hostname is_host = machine == hostname
@ -500,24 +524,24 @@ async def build_files(hostname: str,
machine_config = config.option(normalize_family(machine)) machine_config = config.option(normalize_family(machine))
certificate_names = [] certificate_names = []
private_names = [] private_names = []
for service in await machine_config.option('services').option.list('optiondescription'): for service in machine_config.option('services').list('optiondescription'):
if not await service.option('activate').value.get(): if not service.option('activate').value.get():
continue continue
# if await service.option('manage').value.get(): # if service.option('manage').value.get():
# certificate_type = 'server' # certificate_type = 'server'
# else: # else:
# certificate_type = 'client' # certificate_type = 'client'
tls_ca_directory = await machine_config.option('general.tls_ca_directory').value.get() tls_ca_directory = machine_config.option('general.tls_ca_directory').value.get()
tls_cert_directory = await machine_config.option('general.tls_cert_directory').value.get() tls_cert_directory = machine_config.option('general.tls_cert_directory').value.get()
tls_key_directory = await machine_config.option('general.tls_key_directory').value.get() tls_key_directory = machine_config.option('general.tls_key_directory').value.get()
try: try:
for certificate in await service.option('certificates').option.list('all'): for certificate in service.option('certificates').list('all'):
if not await certificate.option('activate').value.get(): if not certificate.option('activate').value.get():
continue continue
certificate_data = await certificate.value.dict() certificate_data = {key.rsplit('.', 1)[1]: value for key, value in certificate.value.dict().items()}
certificate_data['type'] = await certificate.information.get('type') certificate_data['type'] = certificate.information.get('type')
certificate_data['authority'] = join(tls_ca_directory, await certificate.information.get('authority') + '.crt') certificate_data['authority'] = join(tls_ca_directory, certificate.information.get('authority') + '.crt')
certificate_data['format'] = await certificate.information.get('format') certificate_data['format'] = certificate.information.get('format')
is_list_name = isinstance(certificate_data['name'], list) is_list_name = isinstance(certificate_data['name'], list)
is_list_domain = isinstance(certificate_data['domain'], list) is_list_domain = isinstance(certificate_data['domain'], list)
if is_list_name != is_list_domain: if is_list_name != is_list_domain:
@ -526,7 +550,7 @@ async def build_files(hostname: str,
certificate_data['provider'] = 'autosigne' certificate_data['provider'] = 'autosigne'
if is_list_name: if is_list_name:
if len(certificate_data['name']) != len(certificate_data['domain']): if len(certificate_data['name']) != len(certificate_data['domain']):
raise Exception('certificate name and domain name must have same lenght') raise Exception('certificate name and domain name must have same length')
for idx, certificate_name in enumerate(certificate_data['name']): for idx, certificate_name in enumerate(certificate_data['name']):
cert_data = certificate_data.copy() cert_data = certificate_data.copy()
if certificate_data['format'] == 'cert_key': if certificate_data['format'] == 'cert_key':
@ -565,7 +589,7 @@ async def build_files(hostname: str,
continue continue
if only_machine and only_machine != machine: if only_machine and only_machine != machine:
continue continue
await templates(machine, templates(machine,
config, config,
just_copy=just_copy, just_copy=just_copy,
copy_manuals=True, copy_manuals=True,
@ -577,7 +601,7 @@ async def build_files(hostname: str,
directories[machine] = '/usr/local/lib' directories[machine] = '/usr/local/lib'
elif not just_copy: elif not just_copy:
machine_config = config.option(normalize_family(machine)) machine_config = config.option(normalize_family(machine))
directories[machine] = await machine_config.option('general.config_dir').value.get() directories[machine] = machine_config.option('general.config_dir').value.get()
if only_machine: if only_machine:
return directories return directories
if only_machine: if only_machine:

235
src/risotto/rougail/annotator.py
View file

@ -26,19 +26,19 @@ def _parse_kwargs(provider, dns, kwargs, index=None):
continue continue
elif data['dns'] not in dns: elif data['dns'] not in dns:
continue continue
del data['dns'] # del data['dns']
yield data yield data
@multi_function @multi_function
def calc_providers_global(provider, multi, value, suffix=None): def calc_providers_global(provider, multi, unique, value, suffix=None):
if suffix is not None: if suffix is not None:
return value[int(suffix)] return value[int(suffix)]
return value return value
@multi_function @multi_function
def calc_providers_follower(provider, multi, dns, leader, index, **kwargs): def calc_providers_follower(provider, multi, unique, dns, leader, index, **kwargs):
ret = [] ret = []
for data in _parse_kwargs(provider, dns, kwargs): for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data: if 'value' not in data:
@ -64,7 +64,7 @@ def calc_providers_follower(provider, multi, dns, leader, index, **kwargs):
@multi_function @multi_function
def calc_providers_dynamic_follower(provider, multi, dns, leader, index, suffix, **kwargs): def calc_providers_dynamic_follower(provider, multi, unique, dns, leader, index, suffix, **kwargs):
ret = [] ret = []
for data in _parse_kwargs(provider, dns, kwargs): for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data: if 'value' not in data:
@ -92,7 +92,7 @@ def calc_providers_dynamic_follower(provider, multi, dns, leader, index, suffix,
@multi_function @multi_function
def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs): def calc_providers_dynamic(provider, multi, unique, dns, suffix, **kwargs):
ret = [] ret = []
for data in _parse_kwargs(provider, dns, kwargs): for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data: if 'value' not in data:
@ -101,7 +101,7 @@ def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs):
continue continue
if isinstance(data['value'], list): if isinstance(data['value'], list):
for v in data['value']: for v in data['value']:
if v not in ret: if not unique or v not in ret:
ret.append(v) ret.append(v)
elif data['value'] not in ret: elif data['value'] not in ret:
ret.append(data['value']) ret.append(data['value'])
@ -112,10 +112,14 @@ def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs):
@multi_function @multi_function
def calc_providers(provider, multi, dns, suffix=None, **kwargs): def calc_providers(provider, multi, unique, dns, **kwargs):
ret = [] ret = []
for data in _parse_kwargs(provider, dns, kwargs): for data in _parse_kwargs(provider, dns, kwargs):
if isinstance(data['value'], list): if isinstance(data['value'], list):
commun_dns = list(set(data['dns']) & set(dns))
if len(commun_dns) == 1:
ret.append(data['value'][data['dns'].index(commun_dns[0])])
continue
for v in data['value']: for v in data['value']:
if v in ret: if v in ret:
continue continue
@ -135,71 +139,136 @@ class Annotator(Walk):
objectspace: 'RougailObjSpace', objectspace: 'RougailObjSpace',
*args): *args):
self.objectspace = objectspace self.objectspace = objectspace
self.set_suppliers() self.get_suppliers_providers()
self.dispatch_provider_supplier_to_zones()
self.dispatch_provider_to_zones()
self.convert_providers() self.convert_providers()
self.convert_suppliers() self.convert_suppliers()
def set_suppliers(self) -> dict: def get_suppliers_providers(self) -> None:
""" get supplier informations """ get supplier informations
return something like: return something like:
{'Host': ['host1.example.net', 'host2.example.net']} {'Host': ['host1.example.net', 'host2.example.net']}
""" """
self.suppliers = {} self.suppliers = {}
self.providers = {}
for variable in self.get_variables(): for variable in self.get_variables():
if not hasattr(variable, 'supplier') or ':' in variable.supplier: if not hasattr(variable, 'supplier') and not hasattr(variable, 'provider'):
continue continue
nf_dns = variable.path.split('.', 1)[0] nf_dns = variable.path.split('.', 1)[0]
server_name = self.objectspace.space.variables[nf_dns].doc server_name = self.objectspace.space.variables[nf_dns].doc
self.suppliers.setdefault(variable.supplier, []).append({'option': variable, # supplier
if hasattr(variable, 'supplier') and ':' not in variable.supplier:
server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
zones = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']
s_data = {'option': variable,
'dns': server_name, 'dns': server_name,
'path_prefix': nf_dns, 'path_prefix': nf_dns,
'server_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'], 'server_names': server_names,
'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'], 'zones': zones,
'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']) 'providers_zone': {},
}) }
if variable.supplier != 'Host' and not variable.supplier.startswith('Host:') and not variable.supplier.startswith('global:'):
if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]:
s_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone']
self.suppliers.setdefault(variable.supplier, []).append(s_data)
if not hasattr(variable, 'information'): if not hasattr(variable, 'information'):
variable.information = self.objectspace.information(variable.xmlfiles) variable.information = self.objectspace.information(variable.xmlfiles)
variable.information.supplier = variable.supplier variable.information.supplier = variable.supplier
# provider
def convert_providers(self): if hasattr(variable, 'provider'):
self.providers = {}
for variable in self.get_variables():
if not hasattr(variable, 'provider'):
continue
nf_dns = variable.path.split('.', 1)[0]
server_name = self.objectspace.space.variables[nf_dns].doc
provider_name = variable.provider provider_name = variable.provider
p_data = {'option': variable,
'dns': server_name,
'provider_name': provider_name,
'path_prefix': nf_dns,
'suppliers_zone': {},
}
if variable.provider != 'Host' and not variable.provider.startswith('Host:') and not variable.provider.startswith('global:'):
if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]:
p_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone']
if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host': if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host':
server_names = [server_name] server_names = [server_name]
else: else:
server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'] server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
if provider_name != 'Host' and not provider_name.startswith('Host:') and not provider_name.startswith('global:'): p_data['server_names'] = server_names
p_data = {'option': variable, if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] != 'host':
'dns': server_name, p_data['zones'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']
'path_prefix': nf_dns,
'server_names': server_names,
'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']),
}
else:
p_data = None
if ':' in provider_name: if ':' in provider_name:
key_name, key_type = provider_name.rsplit(':', 1) p_data['is_main_provider'] = False
is_provider = False provider = provider_name.rsplit(':', 1)[0]
else: else:
key_name = key_type = provider_name p_data['is_main_provider'] = True
is_provider = True provider = variable.provider
if provider_name != 'Host': self.providers.setdefault(provider, []).append(p_data)
self.providers.setdefault(provider_name, []).append(p_data)
if key_name != 'global' and key_name not in self.suppliers: def dispatch_provider_supplier_to_zones(self):
#warn(f'cannot find supplier "{key_name}" for "{server_name}"') """calculate zone where provider and supplier communicate
"""
self.providers_zone = {}
for provider_name, p_datas in self.providers.items():
for p_data in p_datas:
if provider_name in ['global', 'Host'] or provider_name not in self.suppliers:
continue
if not 'provider_zone' in p_data:
provider_zone = None
else:
provider_zone = p_data['provider_zone']
for s_data in self.suppliers[provider_name]:
if not provider_zone:
if provider_name.endswith('Client'):
p_server = provider_name[0:-6]
if p_server not in self.providers:
continue
for p_data_t in self.providers[p_server]:
if p_data_t['dns'] == s_data['dns'] and 'provider_zone' in p_data_t:
zone = p_data_t['provider_zone']
break
else:
continue
else:
continue
else:
zone = provider_zone
if zone not in s_data['zones']:
continue
s_data['providers_zone'][provider_name] = zone
p_data['suppliers_zone'].setdefault(provider_name, {})[s_data['dns']] = zone
self.providers_zone.setdefault(zone, set()).add(provider_name)
def dispatch_provider_to_zones(self):
""" add information with provider zone domain name
"""
self.providers_zone = {}
for provider_name, p_datas in self.providers.items():
for p_data in p_datas:
if provider_name in ['global', 'Host'] or provider_name not in self.suppliers:
continue
if not 'provider_zone' in p_data:
continue
provider_zone = p_data['provider_zone']
family = self.objectspace.paths.get_variable(f"providers",
namespace=self.objectspace.rougailconfig['variable_namespace'],
force_path_prefix=p_data['path_prefix'],
)
if not hasattr(family, 'information'):
family.information = self.objectspace.information(family.xmlfiles)
name_in_zone = p_data['server_names'][p_data['zones'].index(provider_zone)]
setattr(family.information, provider_name, name_in_zone)
setattr(family.information, f'{provider_name}:zone', provider_zone)
def convert_providers(self):
for provider_name, providers_data in self.providers.items():
for provider_data in providers_data:
if provider_name != 'global' and provider_name not in self.suppliers:
continue continue
# create a fill for this variable # create a fill for this variable
variable = provider_data['option']
fill = self.objectspace.fill(variable.xmlfiles) fill = self.objectspace.fill(variable.xmlfiles)
new_target = self.objectspace.target(variable.xmlfiles) new_target = self.objectspace.target(variable.xmlfiles)
new_target.name = variable new_target.name = variable
fill.target = [new_target] fill.target = [new_target]
if key_name == 'global': if provider_name == 'global':
fill.name = 'calc_providers_global' fill.name = 'calc_providers_global'
elif self.objectspace.paths.is_dynamic(variable): elif self.objectspace.paths.is_dynamic(variable):
if self.objectspace.paths.is_follower(variable): if self.objectspace.paths.is_follower(variable):
@ -215,7 +284,7 @@ class Annotator(Walk):
# first parameter: the provider name (something link Host:incoming_ports) # first parameter: the provider name (something link Host:incoming_ports)
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
param.name = 'provider' param.name = 'provider'
param.text = provider_name param.text = provider_data['provider_name']
fill.param = [param] fill.param = [param]
# second parameter: current variable is a multi variable? # second parameter: current variable is a multi variable?
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
@ -223,6 +292,13 @@ class Annotator(Walk):
param.text = variable.multi param.text = variable.multi
param.type = 'boolean' param.type = 'boolean'
fill.param.append(param) fill.param.append(param)
#
param = self.objectspace.param(variable.xmlfiles)
param.name = 'unique'
param.text = variable.unique != "False"
param.type = 'boolean'
fill.param.append(param)
#
if self.objectspace.paths.is_follower(variable): if self.objectspace.paths.is_follower(variable):
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
param.name = 'leader' param.name = 'leader'
@ -246,42 +322,40 @@ class Annotator(Walk):
param.name = 'suffix' param.name = 'suffix'
param.type = 'suffix' param.type = 'suffix'
fill.param.append(param) fill.param.append(param)
if key_name != 'global': if provider_name != 'global':
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
param.name = 'dns' param.name = 'dns'
param.text = server_names param.text = provider_data['server_names']
fill.param.append(param) fill.param.append(param)
if key_name == 'global': if provider_name == 'global':
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
param.name = 'value' param.name = 'value'
if provider_name in self.objectspace.rougailconfig['risotto_globals'][server_name]: if provider_data['provider_name'] in self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']]:
value = self.objectspace.rougailconfig['risotto_globals'][server_name][provider_name] value = self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']][provider_data['provider_name']]
param.text = value param.text = value
if isinstance(value, bool): if isinstance(value, bool):
param.type = 'boolean' param.type = 'boolean'
else: else:
param.text = provider_name param.text = provider_data['provider_name']
param.type = 'information' param.type = 'information'
fill.param.append(param) fill.param.append(param)
else: else:
# parse all supplier link to current provider # parse all supplier link to current provider
for idx, data in enumerate(self.suppliers[key_name]): for idx, data in enumerate(self.suppliers[provider_name]):
if p_data: if 'zones' in provider_data:
common_zones = data['zones'] & p_data['zones'] if provider_name not in data['providers_zone']:
if not common_zones:
continue continue
for zidx, zone in enumerate(data['zone_names']): zone = data['providers_zone'][provider_name]
if zone in common_zones: zidx = data['zones'].index(zone)
break
dns = data['server_names'][zidx] dns = data['server_names'][zidx]
else: else:
dns = data['dns'] dns = data['dns']
option = data['option'] option = data['option']
# if not provider, get the true option that we want has value # if not provider, get the true option that we want have value
if not is_provider: if not provider_data['is_main_provider']:
path_prefix = data['path_prefix'] path_prefix = data['path_prefix']
try: try:
supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_name}', path_prefix) supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_data["provider_name"]}', path_prefix)
except KeyError: except KeyError:
#warn(f'cannot find supplier "{provider_name}" for "{dns}"') #warn(f'cannot find supplier "{provider_name}" for "{dns}"')
continue continue
@ -292,7 +366,7 @@ class Annotator(Walk):
param.propertyerror = False param.propertyerror = False
param.type = 'variable' param.type = 'variable'
fill.param.append(param) fill.param.append(param)
if not is_provider and \ if not provider_data['is_main_provider'] and \
self.objectspace.paths.is_follower(variable): self.objectspace.paths.is_follower(variable):
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
param.name = f'leader_{idx}' param.name = f'leader_{idx}'
@ -315,36 +389,39 @@ class Annotator(Walk):
# get the current value! # get the current value!
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
param.name = f'value_{idx}' param.name = f'value_{idx}'
if is_provider: if provider_data['is_main_provider']:
param.text = dns param.text = dns
else: else:
param.text = supplier_option param.text = supplier_option
param.propertyerror = False param.propertyerror = False
param.type = 'variable' param.type = 'variable'
fill.param.append(param) fill.param.append(param)
if not hasattr(self.objectspace.space.variables[nf_dns], 'constraints'): if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']], 'constraints'):
self.objectspace.space.variables[nf_dns].constraints = self.objectspace.constraints(None) self.objectspace.space.variables[provider_data['path_prefix']].constraints = self.objectspace.constraints(None)
if not hasattr(self.objectspace.space.variables[nf_dns].constraints, 'fill'): if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']].constraints, 'fill'):
self.objectspace.space.variables[nf_dns].constraints.fill = [] self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill = []
self.objectspace.space.variables[nf_dns].constraints.fill.append(fill) self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill.append(fill)
def convert_suppliers(self): def convert_suppliers(self):
for supplier, data in self.suppliers.items(): for supplier_name, s_datas in self.suppliers.items():
if supplier == 'Host': if supplier_name == 'Host':
continue continue
for s_dico in data: for s_data in s_datas:
if supplier not in self.providers: if supplier_name not in self.providers:
continue continue
for p_dico in self.providers[supplier]: for p_data in self.providers[supplier_name]:
common_zones = s_dico['zones'] & p_dico['zones'] if s_data['dns'] == p_data['dns']:
if not common_zones: # supplier and provider are in same machine
continue continue
for idx, zone in enumerate(p_dico['zone_names']): if supplier_name not in p_data['suppliers_zone'] or s_data['dns'] not in p_data['suppliers_zone'][supplier_name]:
if zone in common_zones: continue
break # get the DNS name in supplier zone
dns = p_dico['server_names'][idx] zone = p_data['suppliers_zone'][supplier_name][s_data['dns']]
s_dico['option'].value = dns if zone not in p_data['zones']:
continue
zidx = p_data['zones'].index(zone)
dns = p_data['server_names'][zidx]
new_value = self.objectspace.value(None) new_value = self.objectspace.value(None)
new_value.name = dns new_value.name = dns
s_dico['option'].value = [new_value] s_data['option'].value = [new_value]
break break

62
src/risotto/utils.py
View file

@ -4,6 +4,7 @@ from typing import List
from ipaddress import ip_address from ipaddress import ip_address
from toml import load as toml_load from toml import load as toml_load
from json import load, dump from json import load, dump
from json.decoder import JSONDecodeError
from pprint import pprint from pprint import pprint
@ -15,6 +16,10 @@ HERE = environ['PWD']
IP_DIR = join(HERE, 'ip') IP_DIR = join(HERE, 'ip')
# custom filters from dataset
custom_filters = {}
config_file = environ.get('CONFIG_FILE', 'risotto.conf') config_file = environ.get('CONFIG_FILE', 'risotto.conf')
if isfile(config_file): if isfile(config_file):
with open(config_file, 'r') as fh: with open(config_file, 'r') as fh:
@ -35,10 +40,10 @@ def multi_function(function):
return function return function
async def value_pprint(dico, config): def value_pprint(dico, config):
pprint_dict = {} pprint_dict = {}
for path, value in dico.items(): for path, value in dico.items():
if await config.option(path).option.type() == 'password' and value: if config.option(path).type() == 'password' and value:
value = 'X' * len(value) value = 'X' * len(value)
pprint_dict[path] = value pprint_dict[path] = value
pprint(pprint_dict) pprint(pprint_dict)
@ -49,28 +54,45 @@ def load_zones(zones, hosts):
makedirs(IP_DIR) makedirs(IP_DIR)
json_file = join(IP_DIR, 'zones.json') json_file = join(IP_DIR, 'zones.json')
if isfile(json_file): if isfile(json_file):
try:
with open(json_file, 'r') as fh: with open(json_file, 'r') as fh:
zones_ip = load(fh) ori_zones_ip = load(fh)
except JSONDecodeError:
ori_zones_ip = {}
else: else:
zones_ip = {} ori_zones_ip = {}
for host_name, hosts in hosts.items(): new_zones_ip = {}
for server_name, server in hosts['servers'].items(): # cache, machine should not change IP
server_zones = server['informations']['zones_name'] for host_name, dhosts in hosts.items():
for server_name, server in dhosts['servers'].items():
server_zones = server['zones_name']
for idx, zone_name in enumerate(server_zones): for idx, zone_name in enumerate(server_zones):
zone = zones[zone_name] zone = zones[zone_name]
zone.setdefault('hosts', {}) zone.setdefault('hosts', {})
# FIXME make a cache, machine should not change IP if zone_name not in new_zones_ip:
if zone_name not in zones_ip: new_zones_ip[zone_name] = {}
zones_ip[zone_name] = {} if zone_name in ori_zones_ip and server_name in ori_zones_ip[zone_name]:
if server_name in zones_ip[zone_name]: server_index = ori_zones_ip[zone_name][server_name]
server_index = zones_ip[zone_name][server_name] if server_index >= zone['length']:
elif not zones_ip[zone_name]: server_index = None
server_index = 0 elif server_index in new_zones_ip[zone_name].values():
server_index = None
else: else:
# it's the last ip + 1 server_index = None
server_index = zones_ip[zone_name][list(zones_ip[zone_name].keys())[-1]] + 1 new_zones_ip[zone_name][server_name] = server_index
ip = str(ip_address(zone['start_ip']) + server_index) for zone_name, servers in new_zones_ip.items():
zone['hosts'][server_name] = ip for server_name, server_idx in servers.items():
zones_ip[zone_name][server_name] = server_index if server_idx is not None:
continue
for new_idx in range(zones[zone_name]['length']):
if new_idx not in new_zones_ip[zone_name].values():
new_zones_ip[zone_name][server_name] = new_idx
break
else:
raise Exception(f'cannot find free IP in zone "{zone_name}" for "{server_name}"')
for zone_name, servers in new_zones_ip.items():
start_ip = ip_address(zones[zone_name]['start_ip'])
for server_name, server_index in servers.items():
zones[zone_name]['hosts'][server_name] = str(start_ip + server_index)
with open(json_file, 'w') as fh: with open(json_file, 'w') as fh:
dump(zones_ip, fh) dump(new_zones_ip, fh)